scripts/run.sh aktualisiert

2026-11-11 22:26:55 +01:00
1 parent 2417815543
commit 0a393fdcf2
@@ -1,33 +1,145 @@
+
 #!/usr/bin/env bash
 set -Eeuo pipefail
 umask 077
+
+# ==================================================
+# Configuration
+# ==================================================
+
 : "${CACHE_ROOT:=/vuln-cache/vuln-builder}"
-: "${GITEA_BASE_URL:=https://git.apex-project.de}"
-: "${GITEA_JOB_TOKEN:?GITEA_JOB_TOKEN missing}"
-mkdir -p "$CACHE_ROOT" "$CACHE_ROOT/sources" "$CACHE_ROOT/releases"
+
+SOURCES_DIR="${SOURCES_DIR:-$CACHE_ROOT/sources}"
+SOURCES_CONFIG="${SOURCES_CONFIG:-$CACHE_ROOT/sources.json}"
+SOURCE_MANIFEST="$CACHE_ROOT/source-manifest.json"
+RELEASE_DIR="$CACHE_ROOT/releases"
+
+# ==================================================
+# Create directories
+# ==================================================
+
+mkdir -p "$CACHE_ROOT" "$SOURCES_DIR" "$RELEASE_DIR"
+
+# ==================================================
+# Prevent concurrent builds
+# ==================================================
+
 exec 9>"$CACHE_ROOT/.build.lock"
-/usr/bin/flock -w 1800 9 || { echo 'Another builder run in progress' >&2; exit 1; }
-export GIT_CONFIG_COUNT=1 GIT_CONFIG_KEY_0=http.extraHeader
-GIT_CONFIG_VALUE_0="Authorization: Basic $(printf 'oauth2:%s' "$GITEA_JOB_TOKEN" | base64 | tr -d '\n')"
-export GIT_CONFIG_VALUE_0
-# Configurable list; extra feeds tracked in manifest but not yet indexed.
-SOURCES="${VULN_SOURCES:-vuln-cve vuln-nvd vuln-osv vuln-cpe vuln-exploits vuln-microsoft vuln-redhat vuln-oracle vuln-bsi vuln-debian vuln-ubuntu vuln-alpine vuln-suse vuln-suse-vex vuln-almalinux vuln-rocky}"
-for name in $SOURCES; do
-  [[ "$name" =~ ^vuln-[a-z0-9-]+$ ]] || { echo "Invalid source name: $name" >&2; exit 1; }
-  repo="$CACHE_ROOT/sources/$name"
-  url="${GITEA_BASE_URL%/}/VULN/$name.git"
-  if [ ! -d "$repo/.git" ]; then
-    echo "Cloning $name"
-    git clone --quiet --depth 1 --single-branch --branch main "$url" "$repo"
-  else
-    echo "Updating $name"
-    git -C "$repo" fetch --quiet --depth 1 origin main
-    git -C "$repo" reset --hard -q FETCH_HEAD
-    git -C "$repo" clean -fdq
-  fi
-done
-unset GIT_CONFIG_COUNT GIT_CONFIG_KEY_0 GIT_CONFIG_VALUE_0
+
+/usr/bin/flock -w 1800 9 || {
+    echo "ERROR: Another builder run is active" >&2
+    exit 1
+}
+
+# ==================================================
+# Validate configuration
+# ==================================================
+
+echo "========================================"
+echo "Vulnerability SQLite Builder"
+echo "========================================"
+
+if [ ! -s "$SOURCES_CONFIG" ]; then
+    echo "ERROR: sources.json missing: $SOURCES_CONFIG" >&2
+    exit 1
+fi
+
+if [ ! -s "$SOURCE_MANIFEST" ]; then
+    echo "ERROR: Source manifest missing" >&2
+    exit 1
+fi
+
+if [ ! -d "$SOURCES_DIR" ]; then
+    echo "ERROR: Source directory missing" >&2
+    exit 1
+fi
+
+# Verify that all sources listed in the manifest
+# exist as local Git repositories.
+python3 - "$SOURCE_MANIFEST" "$SOURCES_DIR" <<'PY'
+import json
+import subprocess
+import sys
+from pathlib import Path
+
+manifest_path = Path(sys.argv[1])
+sources_dir = Path(sys.argv[2])
+
+entries = json.loads(
+    manifest_path.read_text(encoding="utf-8")
+)
+
+if not isinstance(entries, list) or not entries:
+    raise SystemExit("ERROR: Empty source manifest")
+
+for entry in entries:
+    name = entry["repo"]
+    expected = entry["commit"]
+    repo = sources_dir / name
+
+    if not (repo / ".git").is_dir():
+        raise SystemExit(f"ERROR: Missing repository {name}")
+
+    actual = subprocess.check_output(
+        ["git", "-C", str(repo), "rev-parse", "HEAD"],
+        text=True
+    ).strip()
+
+    if actual != expected:
+        raise SystemExit(
+            f"ERROR: Commit mismatch in {name}: "
+            f"{actual} != {expected}"
+        )
+
+    print(f"OK {name}: {actual[:12]}", flush=True)
+
+print(f"Validated {len(entries)} repositories")
+PY
+
+# ==================================================
+# Build SQLite
+# ==================================================
+
 cd "$(dirname "$0")/.."
-python3 scripts/build.py --sources "$CACHE_ROOT/sources" --output "$CACHE_ROOT/releases"
-echo 'Release files:'
-ls -lh "$CACHE_ROOT/releases"/*.zst "$CACHE_ROOT/releases"/manifest.json
+
+echo "Starting SQLite build..."
+
+python3 scripts/build.py \
+    --sources "$SOURCES_DIR" \
+    --output "$RELEASE_DIR"
+
+# ==================================================
+# Verify release files
+# ==================================================
+
+echo "Checking build results..."
+
+if [ ! -s "$RELEASE_DIR/master.db.zst" ]; then
+    echo "ERROR: master.db.zst missing or empty" >&2
+    exit 1
+fi
+
+if [ ! -s "$RELEASE_DIR/manifest.json" ]; then
+    echo "ERROR: Release manifest missing" >&2
+    exit 1
+fi
+
+# ==================================================
+# Show results
+# ==================================================
+
+echo "========================================"
+echo "Build completed"
+echo "========================================"
+
+ls -lh \
+    "$RELEASE_DIR/master.db.zst" \
+    "$RELEASE_DIR/manifest.json"
+
+find "$RELEASE_DIR" \
+    -maxdepth 1 \
+    -name 'delta-*.zst' \
+    -printf '%f %s bytes\n'
+
+echo "SQLite build finished successfully"
+