Initial OSV mirror

.gitea/workflows/update.yml

@@ -0,0 +1,71 @@
+name: vuln-osv Synchronisation (lokaler Git-Cache)
+
+on:
+  schedule:
+    - cron: "41 */6 * * *"
+  workflow_dispatch:
+
+permissions:
+  contents: write
+
+concurrency:
+  group: vuln-osv-sync
+  cancel-in-progress: false
+
+jobs:
+  mirror:
+    runs-on: vuln-runner
+    timeout-minutes: 360
+    steps:
+      - name: Rohdaten aktualisieren
+        shell: bash
+        env:
+          GITEA_JOB_TOKEN: ${{ secrets.GITEA_TOKEN }}
+          CACHE_ROOT: /var/lib/act_runner/cache/vuln-osv
+        run: |
+          set -Eeuo pipefail
+          umask 077
+          : "${GITEA_JOB_TOKEN:?GITEA_TOKEN fehlt}"
+          mkdir -p "$CACHE_ROOT"
+          exec 9>"$CACHE_ROOT/.sync.lock"
+          flock -w 300 9 || { echo "Bereits ein Lauf aktiv" >&2; exit 1; }
+
+          ASKPASS="$(mktemp "$CACHE_ROOT/.askpass.XXXXXX")"
+          trap 'rm -f "$ASKPASS"' EXIT
+          cat > "$ASKPASS" <<'ASKPASS_SCRIPT'
+          #!/bin/sh
+          case "$1" in
+            *Username*|*username*) printf '%s\n' 'oauth2' ;;
+            *Password*|*password*) printf '%s\n' "$GITEA_JOB_TOKEN" ;;
+            *) exit 1 ;;
+          esac
+          ASKPASS_SCRIPT
+          chmod 700 "$ASKPASS"
+          export GIT_ASKPASS="$ASKPASS" GIT_TERMINAL_PROMPT=0
+          REMOTE="https://git.apex-project.de/VULN/vuln-osv.git"
+          REPO="$CACHE_ROOT/repo"
+          if [ ! -d "$REPO/.git" ]; then
+            [ ! -e "$REPO" ] || { echo "Unvollstaendiger Cache: $REPO" >&2; exit 1; }
+            git -c credential.helper= clone --branch main --single-branch --depth 1 "$REMOTE" "$REPO"
+          else
+            # Never reset an unfinished local commit or changes after a failed push.
+            if [ -n "$(git -C "$REPO" status --porcelain)" ] || [ "$(git -C "$REPO" rev-list --count origin/main..HEAD)" != 0 ]; then
+              echo "Lokale Aenderungen/Commits vorhanden: manuell pruefen, nicht ueberschreiben" >&2
+              exit 1
+            fi
+            git -C "$REPO" remote set-url origin "$REMOTE"
+            git -C "$REPO" -c credential.helper= fetch --no-tags --depth 1 origin main
+            git -C "$REPO" checkout -B main FETCH_HEAD
+          fi
+          cd "$REPO"
+          python3 scripts/sync.py
+          python3 scripts/validate.py
+          git config user.name "Vulnerability Bot"
+          git config user.email "vuln-bot@users.noreply.local"
+          git add -A -- data/ state/
+          if git diff --cached --quiet; then
+            echo "Keine neuen Daten"
+            exit 0
+          fi
+          git commit -m "vuln-osv update $(date -u +%Y-%m-%dT%H:%M:%SZ)"
+          git -c credential.helper= push origin HEAD:main

.gitignore

@@ -0,0 +1,5 @@
+__pycache__/
+*.pyc
+*.tmp
+.sync.lock
+.DS_Store

README.md

@@ -0,0 +1,21 @@
+# vuln-osv – Gitea OSV raw-data mirror
+
+Speichert Original-JSON aus OSV-Ökosystem-Snapshots direkt in Git unter `data/<ECOSYSTEM>/<ID>.json`.
+
+## Installation
+1. Leeres Gitea-Repository `VULN/vuln-osv` erstellen, dieses Verzeichnis pushen.
+2. Auf der Gitea-VM: `sudo install -d -o act_runner -g act_runner -m 700 /var/lib/act_runner/cache/vuln-osv`.
+3. `vuln-runner:host` muss registriert sein. `python3`, `git` und `flock` müssen verfügbar sein.
+4. Actions-Job-Token benötigt Schreibrechte. Workflow `.gitea/workflows/update.yml` starten.
+
+## Updates
+- Zeitplan: alle sechs Stunden (Minute 41).
+- HTTP `If-None-Match` / `If-Modified-Since`: Bei unverändertem Export Antwort `304`, kein ZIP-Download.
+- Wenn sich ein ZIP ändert, wird **das gesamte Ökosystem-ZIP erneut geladen**, nicht nur die Änderungen. Ein späterer Umbau auf `modified_id.csv` spart weitere Bandbreite.
+- Nur geänderte JSON-Dateien werden geschrieben und committed.
+- Standard-Ökosysteme: PyPI, npm, Go, Maven, NuGet, crates.io, Packagist, RubyGems, GitHub Actions, OSS-Fuzz.
+- Für andere Ökosysteme `python3 scripts/sync.py --ecosystems 'PyPI,npm,Go'` ausführen, beziehungsweise Workflow-Aufruf anpassen.
+- **Löschungen**: Standardmäßig werden keine Dateien entfernt. Erst nach belastbar bestätigtem Full-Snapshot mit `--allow-deletions` aktivieren. Ein fehlender Snapshot darf niemals als Leeren interpretiert werden.
+- Sehr große Git-Repos: Speicherplatz, Inodes und Gitea-Push-Limits kontrollieren. Für npm/Go sind mehrere hunderttausend Dateien möglich.
+
+Offizielle Quelle: https://google.github.io/osv.dev/data/

scripts/sync.py

@@ -0,0 +1,126 @@
+#!/usr/bin/env python3
+"""Mirror full raw OSV JSON files per ecosystem with conditional HTTP requests.
+No third-party Python dependencies. All-or-nothing validation per ecosystem.
+"""
+import argparse
+import io
+import json
+import os
+import re
+import shutil
+import sys
+import tempfile
+import time
+import urllib.error
+import urllib.parse
+import urllib.request
+import zipfile
+from datetime import datetime, timezone
+from pathlib import Path
+
+ROOT = Path(__file__).resolve().parents[1]
+BASE = 'https://storage.googleapis.com/osv-vulnerabilities'
+DEFAULT = 'PyPI,npm,Go,Maven,NuGet,crates.io,Packagist,RubyGems,GitHub Actions,OSS-Fuzz'
+VALID_ECOSYSTEM = re.compile(r'^[A-Za-z0-9._ -]{1,60}$')
+VALID_ID = re.compile(r'^[A-Za-z0-9._-]{2,150}$')
+
+def atomic_write(path, data):
+    path.parent.mkdir(parents=True, exist_ok=True)
+    if path.exists() and path.read_bytes() == data:
+        return False
+    tmp = path.with_name(path.name + '.tmp')
+    try:
+        tmp.write_bytes(data)
+        os.replace(tmp, path)
+    finally:
+        tmp.unlink(missing_ok=True)
+    return True
+
+def fetch(url, etag=None, last_modified=None):
+    headers = {'User-Agent': 'vuln-osv-gitea-mirror/1.0'}
+    if etag:
+        headers['If-None-Match'] = etag
+    elif last_modified:
+        headers['If-Modified-Since'] = last_modified
+    for attempt in range(5):
+        try:
+            with urllib.request.urlopen(urllib.request.Request(url, headers=headers), timeout=180) as response:
+                # Download into a temp file; don't hold huge archives in RAM.
+                with tempfile.TemporaryFile() as temp:
+                    shutil.copyfileobj(response, temp)
+                    temp.seek(0)
+                    body = temp.read()
+                return body, response.headers.get('ETag'), response.headers.get('Last-Modified')
+        except urllib.error.HTTPError as exc:
+            if exc.code == 304:
+                return None, etag, last_modified
+            if exc.code not in (408, 429, 500, 502, 503, 504) or attempt == 4:
+                raise
+        except (urllib.error.URLError, TimeoutError):
+            if attempt == 4:
+                raise
+        time.sleep(min(120, 3 * 2**attempt))
+    raise RuntimeError('Unreachable')
+
+def process(ecosystem, state, allow_deletions):
+    if not VALID_ECOSYSTEM.fullmatch(ecosystem):
+        raise ValueError(f'Invalid ecosystem name: {ecosystem!r}')
+    url = f'{BASE}/{urllib.parse.quote(ecosystem)}/all.zip'
+    previous = state.get('ecosystems', {}).get(ecosystem, {})
+    body, etag, modified = fetch(url, previous.get('etag'), previous.get('last_modified'))
+    if body is None:
+        print(f'{ecosystem}: unchanged (304)', flush=True)
+        return False
+    # Validate all records before changing the working tree.
+    records = {}
+    with zipfile.ZipFile(io.BytesIO(body)) as archive:
+        for info in archive.infolist():
+            if info.is_dir() or not info.filename.endswith('.json'):
+                continue
+            if info.file_size > 32*1024*1024:
+                raise ValueError(f'OSV JSON too large: {info.filename}')
+            identifier = Path(info.filename).stem
+            if not VALID_ID.fullmatch(identifier):
+                raise ValueError(f'Unsafe ID: {identifier!r}')
+            content = archive.read(info)
+            entry = json.loads(content)
+            if not isinstance(entry, dict) or entry.get('id') != identifier:
+                raise ValueError(f'Invalid OSV record: {info.filename}')
+            if identifier in records and records[identifier] != content:
+                raise ValueError(f'Duplicate OSV ID: {identifier}')
+            records[identifier] = content
+    if not records:
+        raise ValueError('Empty OSV archive: refusing to delete data')
+    directory = ROOT / 'data' / ecosystem
+    existing = {p.stem: p for p in directory.glob('*.json')}
+    written = sum(atomic_write(directory / f'{ident}.json', raw) for ident, raw in records.items())
+    deleted = 0
+    if allow_deletions:
+        for ident, path in existing.items():
+            if ident not in records:
+                path.unlink()
+                deleted += 1
+    state.setdefault('ecosystems', {})[ecosystem] = {
+        'etag': etag, 'last_modified': modified,
+        'records': len(records), 'last_success': datetime.now(timezone.utc).isoformat()
+    }
+    print(f'{ecosystem}: {len(records)} records, {written} written, {deleted} deleted', flush=True)
+    return True
+
+def main():
+    parser = argparse.ArgumentParser()
+    parser.add_argument('--ecosystems', default=os.getenv('OSV_ECOSYSTEMS', DEFAULT))
+    parser.add_argument('--allow-deletions', action='store_true', help='Remove records no longer in verified full snapshots')
+    args = parser.parse_args()
+    ecosystems = [s.strip() for s in args.ecosystems.split(',') if s.strip()]
+    if len(set(ecosystems)) != len(ecosystems):
+        raise ValueError('Duplicate ecosystem')
+    state_path = ROOT / 'state' / 'sync.json'
+    state = json.loads(state_path.read_text()) if state_path.exists() else {'version': 1, 'ecosystems': {}}
+    for ecosystem in ecosystems:
+        # Persist each successfully downloaded ecosystem. A retry resumes the rest.
+        if process(ecosystem, state, args.allow_deletions):
+            atomic_write(state_path, (json.dumps(state, ensure_ascii=False, sort_keys=True, indent=2) + '\n').encode())
+
+if __name__ == '__main__':
+    main()

scripts/validate.py

@@ -0,0 +1,20 @@
+#!/usr/bin/env python3
+import json
+import sys
+from pathlib import Path
+root = Path(__file__).resolve().parents[1]
+state_file = root / 'state/sync.json'
+if not state_file.exists():
+    sys.exit('No OSV sync state')
+state = json.loads(state_file.read_text())
+for ecosystem, metadata in state.get('ecosystems', {}).items():
+    files = list((root / 'data' / ecosystem).glob('*.json'))
+    if len(files) < 1:
+        sys.exit(f'Empty ecosystem: {ecosystem}')
+    # Files are checked on import; spot-check retained data here to avoid reparsing millions.
+    for path in files[:10]:
+        item = json.loads(path.read_text())
+        if item.get('id') != path.stem:
+            sys.exit(f'Invalid {path}')
+    print(ecosystem, len(files), 'records')
+print('Validation OK')