diff --git a/.gitea/workflows/build.yml b/.gitea/workflows/build.yml index e68dc98..bfa4648 100644 --- a/.gitea/workflows/build.yml +++ b/.gitea/workflows/build.yml @@ -1,102 +1,302 @@ + name: Build SQLite vulnerability index + on: workflow_dispatch: schedule: - cron: "35 3 * * *" + concurrency: group: vuln-builder cancel-in-progress: false + jobs: build: runs-on: vuln-builder timeout-minutes: 720 + env: CACHE_ROOT: /vuln-cache/vuln-builder GITEA_BASE_URL: https://git.apex-project.de GITEA_JOB_TOKEN: ${{ secrets.GITEA_TOKEN }} + SOURCES_DIR: /vuln-cache/vuln-builder/sources + SOURCES_CONFIG: /vuln-cache/vuln-builder/sources.json + steps: + # ============================================ + # 1. Install dependencies + # ============================================ - name: Install tools shell: sh run: | set -eu + if command -v apk >/dev/null 2>&1; then - apk add --no-cache bash git python3 sqlite zstd util-linux ca-certificates coreutils + apk add --no-cache \ + bash git python3 sqlite zstd \ + util-linux ca-certificates coreutils + elif command -v apt-get >/dev/null 2>&1; then apt-get update - apt-get install -y --no-install-recommends bash git python3 sqlite3 zstd util-linux ca-certificates coreutils + apt-get install -y --no-install-recommends \ + bash git python3 sqlite3 zstd \ + util-linux ca-certificates coreutils else - echo 'Unsupported job image' >&2 + echo "Unsupported job image" >&2 exit 1 fi + + python3 --version + git --version /usr/bin/flock --version - - name: Verify persistent cache + + # ============================================ + # 2. Bootstrap builder + # ============================================ + - name: Bootstrap builder shell: bash run: | set -Eeuo pipefail + umask 077 - echo "=== Cache Mount ===" - findmnt -T /vuln-cache || true + : "${GITEA_JOB_TOKEN:?Missing Gitea token}" - echo "=== Schreibtest ===" - mkdir -p /vuln-cache/vuln-builder - echo "CACHE_TEST_OK" > /vuln-cache/vuln-builder/test.txt - - echo "=== Inhalt ===" - ls -lah /vuln-cache/vuln-builder - - name: Test Gitea cache - shell: sh - run: | - set -eu - mkdir -p /vuln-cache/vuln-builder - echo "GITEA_JOB_OK" > /vuln-cache/vuln-builder/gitea-test.txt - ls -lah /vuln-cache/vuln-builder - - - name: Bootstrap and build - shell: bash - run: | - set -Eeuo pipefail - : "${GITEA_JOB_TOKEN:?missing token}" mkdir -p "$CACHE_ROOT" + export GIT_CONFIG_COUNT=1 export GIT_CONFIG_KEY_0=http.extraHeader export GIT_CONFIG_VALUE_0="Authorization: Basic $(printf 'oauth2:%s' "$GITEA_JOB_TOKEN" | base64 | tr -d '\n')" + if [ ! -d "$CACHE_ROOT/builder/.git" ]; then - git clone --depth 1 --branch main "$GITEA_BASE_URL/VULN/vuln-builder.git" "$CACHE_ROOT/builder" + git clone --depth 1 --branch main \ + "$GITEA_BASE_URL/VULN/vuln-builder.git" \ + "$CACHE_ROOT/builder" else - git -C "$CACHE_ROOT/builder" fetch --depth 1 origin main - git -C "$CACHE_ROOT/builder" reset --hard FETCH_HEAD + git -C "$CACHE_ROOT/builder" \ + fetch --depth 1 origin main + + git -C "$CACHE_ROOT/builder" \ + reset --hard FETCH_HEAD fi + + echo "Builder ready" + + # ============================================ + # 3. Download sources.json + # ============================================ - name: Load sources configuration shell: bash run: | set -Eeuo pipefail - - mkdir -p "$CACHE_ROOT" + umask 077 python3 - <<'PY' + import json import os import urllib.request + from pathlib import Path base = os.environ["GITEA_BASE_URL"].rstrip("/") token = os.environ["GITEA_JOB_TOKEN"] - root = os.environ["CACHE_ROOT"] url = ( f"{base}/VULN/vuln-status/" "raw/branch/main/sources.json" ) - req = urllib.request.Request( + request = urllib.request.Request( url, headers={"Authorization": f"token {token}"} ) - with urllib.request.urlopen(req, timeout=60) as response: + with urllib.request.urlopen( + request, timeout=120 + ) as response: data = response.read() - with open(f"{root}/sources.json", "wb") as f: - f.write(data) + config = json.loads(data) - print("sources.json downloaded") + if not isinstance(config.get("sources"), list): + raise ValueError("Invalid sources.json") + + path = Path(os.environ["SOURCES_CONFIG"]) + path.parent.mkdir(parents=True, exist_ok=True) + path.write_bytes(data) + + print( + f"Loaded {len(config['sources'])} entries", + flush=True + ) PY + # ============================================ + # 4. Synchronize all repositories + # ============================================ + - name: Synchronize vulnerability repositories + shell: bash + run: | + set -Eeuo pipefail + umask 077 + + export GIT_CONFIG_COUNT=1 + export GIT_CONFIG_KEY_0=http.extraHeader + export GIT_CONFIG_VALUE_0="Authorization: Basic $(printf 'oauth2:%s' "$GITEA_JOB_TOKEN" | base64 | tr -d '\n')" + + python3 - <<'PY' + import json + import os + import re + import subprocess + from pathlib import Path + + root = Path(os.environ["CACHE_ROOT"]) + sources_dir = Path(os.environ["SOURCES_DIR"]) + base = os.environ["GITEA_BASE_URL"].rstrip("/") + + config = json.loads( + Path(os.environ["SOURCES_CONFIG"]) + .read_text(encoding="utf-8") + ) + + owner = config.get("owner", "VULN") + branch = config.get("branch", "main") + + valid_repo = re.compile(r"^vuln-[a-z0-9-]+$") + valid_owner = re.compile(r"^[A-Za-z0-9_-]+$") + valid_branch = re.compile(r"^[A-Za-z0-9_./-]+$") + + if not valid_owner.fullmatch(owner): + raise ValueError("Invalid owner") + + if ( + not valid_branch.fullmatch(branch) + or branch.startswith("-") + or ".." in branch + or branch.endswith("/") + ): + raise ValueError("Invalid branch") + + sources_dir.mkdir(parents=True, exist_ok=True) + + manifest = [] + seen = set() + + def git(*args, cwd=None): + subprocess.run( + ["git", *args], + cwd=cwd, + check=True + ) + + for entry in config["sources"]: + if entry.get("enabled", True) is False: + continue + + name = entry["repo"] + + if not valid_repo.fullmatch(name): + raise ValueError(f"Invalid repo: {name}") + + if name in seen: + raise ValueError(f"Duplicate repo: {name}") + + seen.add(name) + + repo_dir = sources_dir / name + repo_url = f"{base}/{owner}/{name}.git" + + print(f"\n=== SYNC {name} ===", flush=True) + + if not (repo_dir / ".git").is_dir(): + git( + "clone", + "--depth", "1", + "--single-branch", + "--branch", branch, + repo_url, + str(repo_dir) + ) + else: + git( + "remote", "set-url", + "origin", repo_url, + cwd=repo_dir + ) + git( + "fetch", + "--depth", "1", + "origin", branch, + cwd=repo_dir + ) + git( + "reset", "--hard", + "FETCH_HEAD", + cwd=repo_dir + ) + + commit = subprocess.check_output( + ["git", "rev-parse", "HEAD"], + cwd=repo_dir, + text=True + ).strip() + + manifest.append({ + "repo": name, + "commit": commit, + "path": str(repo_dir) + }) + + print( + f"OK {name}: {commit[:12]}", + flush=True + ) + + if not manifest: + raise RuntimeError( + "No enabled repositories found" + ) + + manifest_path = root / "source-manifest.json" + temp_path = root / "source-manifest.json.tmp" + + temp_path.write_text( + json.dumps(manifest, indent=2), + encoding="utf-8" + ) + temp_path.replace(manifest_path) + + print( + f"\nSynchronized: {len(manifest)} repositories", + flush=True + ) + PY + + # ============================================ + # 5. Build SQLite master and delta + # ============================================ + - name: Build SQLite database + shell: bash + run: | + set -Eeuo pipefail + + echo "Starting SQLite Builder" + + bash "$CACHE_ROOT/builder/scripts/run.sh" + + # ============================================ + # 6. Show results + # ============================================ + - name: Show build results + if: always() + shell: sh + run: | + echo "=== Cache usage ===" + du -sh "$CACHE_ROOT" 2>/dev/null || true + + echo "=== Sources ===" + ls -lah "$CACHE_ROOT/sources" \ + 2>/dev/null || true + + echo "=== Releases ===" + ls -lah "$CACHE_ROOT/releases" \ + 2>/dev/null || true