From 0aa4cea361aa67f912373d4def9b1c74b7fb9fa5 Mon Sep 17 00:00:00 2001 From: Ronny Grobel Date: Wed, 11 Nov 2026 22:16:00 +0100 Subject: [PATCH 01/14] =?UTF-8?q?Verzeichnis=20'.github/workflows=20'=20l?= =?UTF-8?q?=C3=B6schen?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .github/workflows/build.yml | 50 ------------------------------------- 1 file changed, 50 deletions(-) delete mode 100644 .github/workflows/build.yml diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml deleted file mode 100644 index 0db7446..0000000 --- a/.github/workflows/build.yml +++ /dev/null @@ -1,50 +0,0 @@ -name: Build vulnerability SQLite -on: - schedule: - - cron: '40 */6 * * *' - workflow_dispatch: -permissions: - contents: write -concurrency: - group: sqlite-publish - cancel-in-progress: false -jobs: - publish: - runs-on: [self-hosted, linux, x64, vulnerability] - timeout-minutes: 240 - steps: - - name: Checkout builder - uses: actions/checkout@v4 - - name: Checkout CVE raw-data repo - uses: actions/checkout@v4 - with: - repository: ${{ github.repository_owner }}/vuln-cve - path: sources/vuln-cve - token: ${{ secrets.VULN_READ_TOKEN }} - persist-credentials: false - fetch-depth: 1 - - name: Verify compression tool - run: zstd --version - - name: Download latest released baseline, if available - env: - GH_TOKEN: ${{ github.token }} - run: | - mkdir -p prior - gh release download --repo "$GITHUB_REPOSITORY" --pattern master.db.zst --dir prior || echo 'No prior release; baseline only' - if test -s prior/master.db.zst; then - python3 scripts/unpack.py prior/master.db.zst prior/master.db - fi - - name: Build master and delta - run: | - REV=$(git -C sources/vuln-cve rev-parse HEAD) - if test -f prior/master.db; then - python3 scripts/build.py --source sources/vuln-cve/data --source-revision "$REV" --previous prior/master.db - else - python3 scripts/build.py --source sources/vuln-cve/data --source-revision "$REV" - fi - - name: Publish release - env: - GH_TOKEN: ${{ github.token }} - run: | - TAG="db-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}" - gh release create "$TAG" dist/*.zst dist/manifest.json --title "$TAG" --notes 'Automated vulnerability DB snapshot and incremental delta' From 0c3c9cd5843cdc6b140c26457129e2195c7e725e Mon Sep 17 00:00:00 2001 From: Ronny Grobel Date: Wed, 11 Nov 2026 22:18:47 +0100 Subject: [PATCH 02/14] =?UTF-8?q?scripts/sync=5Fsources.py=20hinzugef?= =?UTF-8?q?=C3=BCgt?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- scripts/sync_sources.py | 120 ++++++++++++++++++++++++++++++++++++++++ 1 file changed, 120 insertions(+) create mode 100644 scripts/sync_sources.py diff --git a/scripts/sync_sources.py b/scripts/sync_sources.py new file mode 100644 index 0000000..5d2213a --- /dev/null +++ b/scripts/sync_sources.py @@ -0,0 +1,120 @@ + +#!/usr/bin/env python3 + +import json +import os +import re +import subprocess +import urllib.request +from pathlib import Path + +GITEA_URL = "https://git.apex-project.de" +SOURCES_URL = ( + f"{GITEA_URL}/VULN/vuln-status/" + "raw/branch/main/sources.json" +) + +CACHE_ROOT = Path( + os.getenv("CACHE_ROOT", "/vuln-cache/vuln-builder") +) +SOURCES_DIR = CACHE_ROOT / "sources" + +TOKEN = os.getenv("SOURCE_READ_TOKEN", "") +REPO_PATTERN = re.compile(r"^[a-zA-Z0-9_.-]+$") + + +def load_sources(): + request = urllib.request.Request(SOURCES_URL) + + if TOKEN: + request.add_header("Authorization", f"token {TOKEN}") + + with urllib.request.urlopen(request, timeout=60) as response: + config = json.load(response) + + if not isinstance(config.get("sources"), list): + raise ValueError("Ungültige sources.json") + + if not REPO_PATTERN.fullmatch(config.get("owner", "")): + raise ValueError("Ungültiger Repository-Owner") + + if config.get("branch") != "main": + raise ValueError("Unerwarteter Git-Branch") + + return config + + +def git(*args, cwd=None): + subprocess.run( + ["git", *args], + cwd=cwd, + check=True, + env={ + **os.environ, + "GIT_TERMINAL_PROMPT": "0" + } + ) + + +def sync_repository(owner, repo, branch): + if not REPO_PATTERN.fullmatch(repo): + raise ValueError(f"Ungültiger Repository-Name: {repo}") + + path = SOURCES_DIR / repo + remote = f"{GITEA_URL}/{owner}/{repo}.git" + + if not (path / ".git").is_dir(): + git( + "clone", + "--depth", "1", + "--branch", branch, + remote, + str(path) + ) + else: + git("fetch", "--depth", "1", "origin", branch, cwd=path) + git("reset", "--hard", "FETCH_HEAD", cwd=path) + + result = subprocess.check_output( + ["git", "rev-parse", "HEAD"], + cwd=path, + text=True + ).strip() + + return { + "repo": repo, + "commit": result, + "path": str(path) + } + + +def main(): + config = load_sources() + SOURCES_DIR.mkdir(parents=True, exist_ok=True) + + result = [] + + for source in config["sources"]: + repo = source["repo"] + print(f"Synchronisiere {repo}", flush=True) + + info = sync_repository( + config["owner"], + repo, + config["branch"] + ) + + result.append(info) + + manifest = CACHE_ROOT / "source-manifest.json" + + manifest.write_text( + json.dumps(result, indent=2), + encoding="utf-8" + ) + + print(f"{len(result)} Repositories synchronisiert") + + +if __name__ == "__main__": + main() From 241781554371b50abf2a0c0638cb4f9b2e8aac05 Mon Sep 17 00:00:00 2001 From: Ronny Grobel Date: Wed, 11 Nov 2026 22:25:10 +0100 Subject: [PATCH 03/14] .gitea/workflows/build.yml aktualisiert --- .gitea/workflows/build.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitea/workflows/build.yml b/.gitea/workflows/build.yml index a1b75a3..4bf3616 100644 --- a/.gitea/workflows/build.yml +++ b/.gitea/workflows/build.yml @@ -45,3 +45,4 @@ jobs: git -C "$CACHE_ROOT/builder" reset --hard FETCH_HEAD fi bash "$CACHE_ROOT/builder/scripts/run.sh" + From 0a393fdcf236111c2a8e99bf0289f6d3e804e790 Mon Sep 17 00:00:00 2001 From: Ronny Grobel Date: Wed, 11 Nov 2026 22:26:55 +0100 Subject: [PATCH 04/14] scripts/run.sh aktualisiert --- scripts/run.sh | 166 +++++++++++++++++++++++++++++++++++++++++-------- 1 file changed, 139 insertions(+), 27 deletions(-) diff --git a/scripts/run.sh b/scripts/run.sh index 44590a2..4692def 100644 --- a/scripts/run.sh +++ b/scripts/run.sh @@ -1,33 +1,145 @@ + #!/usr/bin/env bash set -Eeuo pipefail umask 077 + +# ================================================== +# Configuration +# ================================================== + : "${CACHE_ROOT:=/vuln-cache/vuln-builder}" -: "${GITEA_BASE_URL:=https://git.apex-project.de}" -: "${GITEA_JOB_TOKEN:?GITEA_JOB_TOKEN missing}" -mkdir -p "$CACHE_ROOT" "$CACHE_ROOT/sources" "$CACHE_ROOT/releases" + +SOURCES_DIR="${SOURCES_DIR:-$CACHE_ROOT/sources}" +SOURCES_CONFIG="${SOURCES_CONFIG:-$CACHE_ROOT/sources.json}" +SOURCE_MANIFEST="$CACHE_ROOT/source-manifest.json" +RELEASE_DIR="$CACHE_ROOT/releases" + +# ================================================== +# Create directories +# ================================================== + +mkdir -p "$CACHE_ROOT" "$SOURCES_DIR" "$RELEASE_DIR" + +# ================================================== +# Prevent concurrent builds +# ================================================== + exec 9>"$CACHE_ROOT/.build.lock" -/usr/bin/flock -w 1800 9 || { echo 'Another builder run in progress' >&2; exit 1; } -export GIT_CONFIG_COUNT=1 GIT_CONFIG_KEY_0=http.extraHeader -GIT_CONFIG_VALUE_0="Authorization: Basic $(printf 'oauth2:%s' "$GITEA_JOB_TOKEN" | base64 | tr -d '\n')" -export GIT_CONFIG_VALUE_0 -# Configurable list; extra feeds tracked in manifest but not yet indexed. -SOURCES="${VULN_SOURCES:-vuln-cve vuln-nvd vuln-osv vuln-cpe vuln-exploits vuln-microsoft vuln-redhat vuln-oracle vuln-bsi vuln-debian vuln-ubuntu vuln-alpine vuln-suse vuln-suse-vex vuln-almalinux vuln-rocky}" -for name in $SOURCES; do - [[ "$name" =~ ^vuln-[a-z0-9-]+$ ]] || { echo "Invalid source name: $name" >&2; exit 1; } - repo="$CACHE_ROOT/sources/$name" - url="${GITEA_BASE_URL%/}/VULN/$name.git" - if [ ! -d "$repo/.git" ]; then - echo "Cloning $name" - git clone --quiet --depth 1 --single-branch --branch main "$url" "$repo" - else - echo "Updating $name" - git -C "$repo" fetch --quiet --depth 1 origin main - git -C "$repo" reset --hard -q FETCH_HEAD - git -C "$repo" clean -fdq - fi -done -unset GIT_CONFIG_COUNT GIT_CONFIG_KEY_0 GIT_CONFIG_VALUE_0 + +/usr/bin/flock -w 1800 9 || { + echo "ERROR: Another builder run is active" >&2 + exit 1 +} + +# ================================================== +# Validate configuration +# ================================================== + +echo "========================================" +echo "Vulnerability SQLite Builder" +echo "========================================" + +if [ ! -s "$SOURCES_CONFIG" ]; then + echo "ERROR: sources.json missing: $SOURCES_CONFIG" >&2 + exit 1 +fi + +if [ ! -s "$SOURCE_MANIFEST" ]; then + echo "ERROR: Source manifest missing" >&2 + exit 1 +fi + +if [ ! -d "$SOURCES_DIR" ]; then + echo "ERROR: Source directory missing" >&2 + exit 1 +fi + +# Verify that all sources listed in the manifest +# exist as local Git repositories. +python3 - "$SOURCE_MANIFEST" "$SOURCES_DIR" <<'PY' +import json +import subprocess +import sys +from pathlib import Path + +manifest_path = Path(sys.argv[1]) +sources_dir = Path(sys.argv[2]) + +entries = json.loads( + manifest_path.read_text(encoding="utf-8") +) + +if not isinstance(entries, list) or not entries: + raise SystemExit("ERROR: Empty source manifest") + +for entry in entries: + name = entry["repo"] + expected = entry["commit"] + repo = sources_dir / name + + if not (repo / ".git").is_dir(): + raise SystemExit(f"ERROR: Missing repository {name}") + + actual = subprocess.check_output( + ["git", "-C", str(repo), "rev-parse", "HEAD"], + text=True + ).strip() + + if actual != expected: + raise SystemExit( + f"ERROR: Commit mismatch in {name}: " + f"{actual} != {expected}" + ) + + print(f"OK {name}: {actual[:12]}", flush=True) + +print(f"Validated {len(entries)} repositories") +PY + +# ================================================== +# Build SQLite +# ================================================== + cd "$(dirname "$0")/.." -python3 scripts/build.py --sources "$CACHE_ROOT/sources" --output "$CACHE_ROOT/releases" -echo 'Release files:' -ls -lh "$CACHE_ROOT/releases"/*.zst "$CACHE_ROOT/releases"/manifest.json + +echo "Starting SQLite build..." + +python3 scripts/build.py \ + --sources "$SOURCES_DIR" \ + --output "$RELEASE_DIR" + +# ================================================== +# Verify release files +# ================================================== + +echo "Checking build results..." + +if [ ! -s "$RELEASE_DIR/master.db.zst" ]; then + echo "ERROR: master.db.zst missing or empty" >&2 + exit 1 +fi + +if [ ! -s "$RELEASE_DIR/manifest.json" ]; then + echo "ERROR: Release manifest missing" >&2 + exit 1 +fi + +# ================================================== +# Show results +# ================================================== + +echo "========================================" +echo "Build completed" +echo "========================================" + +ls -lh \ + "$RELEASE_DIR/master.db.zst" \ + "$RELEASE_DIR/manifest.json" + +find "$RELEASE_DIR" \ + -maxdepth 1 \ + -name 'delta-*.zst' \ + -printf '%f %s bytes\n' + +echo "SQLite build finished successfully" + From 3b4e755114544a89a8698a3c46aab9665aa1c805 Mon Sep 17 00:00:00 2001 From: Ronny Grobel Date: Wed, 11 Nov 2026 22:28:53 +0100 Subject: [PATCH 05/14] .gitea/workflows/build.yml aktualisiert --- .gitea/workflows/build.yml | 34 +++++++++++++++++++++++++++++++++- 1 file changed, 33 insertions(+), 1 deletion(-) diff --git a/.gitea/workflows/build.yml b/.gitea/workflows/build.yml index 4bf3616..6a9d7f0 100644 --- a/.gitea/workflows/build.yml +++ b/.gitea/workflows/build.yml @@ -44,5 +44,37 @@ jobs: git -C "$CACHE_ROOT/builder" fetch --depth 1 origin main git -C "$CACHE_ROOT/builder" reset --hard FETCH_HEAD fi - bash "$CACHE_ROOT/builder/scripts/run.sh" + - name: Load sources configuration + shell: bash + run: | + set -Eeuo pipefail + + mkdir -p "$CACHE_ROOT" + + python3 - <<'PY' + import os + import urllib.request + + base = os.environ["GITEA_BASE_URL"].rstrip("/") + token = os.environ["GITEA_JOB_TOKEN"] + root = os.environ["CACHE_ROOT"] + + url = ( + f"{base}/VULN/vuln-status/" + "raw/branch/main/sources.json" + ) + + req = urllib.request.Request( + url, + headers={"Authorization": f"token {token}"} + ) + + with urllib.request.urlopen(req, timeout=60) as response: + data = response.read() + + with open(f"{root}/sources.json", "wb") as f: + f.write(data) + + print("sources.json downloaded") + PY From 5620111766484bc71ffab1e79361e0c8a036e25f Mon Sep 17 00:00:00 2001 From: Ronny Grobel Date: Wed, 11 Nov 2026 22:34:45 +0100 Subject: [PATCH 06/14] .gitea/workflows/build.yml aktualisiert --- .gitea/workflows/build.yml | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/.gitea/workflows/build.yml b/.gitea/workflows/build.yml index 6a9d7f0..39894b8 100644 --- a/.gitea/workflows/build.yml +++ b/.gitea/workflows/build.yml @@ -29,6 +29,21 @@ jobs: exit 1 fi /usr/bin/flock --version + - name: Verify persistent cache + shell: bash + run: | + set -Eeuo pipefail + + echo "=== Cache Mount ===" + findmnt -T /vuln-cache || true + + echo "=== Schreibtest ===" + mkdir -p /vuln-cache/vuln-builder + echo "CACHE_TEST_OK" > /vuln-cache/vuln-builder/test.txt + + echo "=== Inhalt ===" + ls -lah /vuln-cache/vuln-builder + - name: Bootstrap and build shell: bash run: | From bb1825c2cc42f769416da610f4e81c29e5e88c25 Mon Sep 17 00:00:00 2001 From: Ronny Grobel Date: Wed, 11 Nov 2026 22:38:42 +0100 Subject: [PATCH 07/14] .gitea/workflows/build.yml aktualisiert --- .gitea/workflows/build.yml | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/.gitea/workflows/build.yml b/.gitea/workflows/build.yml index 39894b8..e68dc98 100644 --- a/.gitea/workflows/build.yml +++ b/.gitea/workflows/build.yml @@ -43,6 +43,13 @@ jobs: echo "=== Inhalt ===" ls -lah /vuln-cache/vuln-builder + - name: Test Gitea cache + shell: sh + run: | + set -eu + mkdir -p /vuln-cache/vuln-builder + echo "GITEA_JOB_OK" > /vuln-cache/vuln-builder/gitea-test.txt + ls -lah /vuln-cache/vuln-builder - name: Bootstrap and build shell: bash From 5c3c7e18b7e55af81d9d7781fa50ec307de54cb8 Mon Sep 17 00:00:00 2001 From: Ronny Grobel Date: Wed, 11 Nov 2026 22:43:46 +0100 Subject: [PATCH 08/14] .gitea/workflows/build.yml aktualisiert --- .gitea/workflows/build.yml | 272 ++++++++++++++++++++++++++++++++----- 1 file changed, 236 insertions(+), 36 deletions(-) diff --git a/.gitea/workflows/build.yml b/.gitea/workflows/build.yml index e68dc98..bfa4648 100644 --- a/.gitea/workflows/build.yml +++ b/.gitea/workflows/build.yml @@ -1,102 +1,302 @@ + name: Build SQLite vulnerability index + on: workflow_dispatch: schedule: - cron: "35 3 * * *" + concurrency: group: vuln-builder cancel-in-progress: false + jobs: build: runs-on: vuln-builder timeout-minutes: 720 + env: CACHE_ROOT: /vuln-cache/vuln-builder GITEA_BASE_URL: https://git.apex-project.de GITEA_JOB_TOKEN: ${{ secrets.GITEA_TOKEN }} + SOURCES_DIR: /vuln-cache/vuln-builder/sources + SOURCES_CONFIG: /vuln-cache/vuln-builder/sources.json + steps: + # ============================================ + # 1. Install dependencies + # ============================================ - name: Install tools shell: sh run: | set -eu + if command -v apk >/dev/null 2>&1; then - apk add --no-cache bash git python3 sqlite zstd util-linux ca-certificates coreutils + apk add --no-cache \ + bash git python3 sqlite zstd \ + util-linux ca-certificates coreutils + elif command -v apt-get >/dev/null 2>&1; then apt-get update - apt-get install -y --no-install-recommends bash git python3 sqlite3 zstd util-linux ca-certificates coreutils + apt-get install -y --no-install-recommends \ + bash git python3 sqlite3 zstd \ + util-linux ca-certificates coreutils else - echo 'Unsupported job image' >&2 + echo "Unsupported job image" >&2 exit 1 fi + + python3 --version + git --version /usr/bin/flock --version - - name: Verify persistent cache + + # ============================================ + # 2. Bootstrap builder + # ============================================ + - name: Bootstrap builder shell: bash run: | set -Eeuo pipefail + umask 077 - echo "=== Cache Mount ===" - findmnt -T /vuln-cache || true + : "${GITEA_JOB_TOKEN:?Missing Gitea token}" - echo "=== Schreibtest ===" - mkdir -p /vuln-cache/vuln-builder - echo "CACHE_TEST_OK" > /vuln-cache/vuln-builder/test.txt - - echo "=== Inhalt ===" - ls -lah /vuln-cache/vuln-builder - - name: Test Gitea cache - shell: sh - run: | - set -eu - mkdir -p /vuln-cache/vuln-builder - echo "GITEA_JOB_OK" > /vuln-cache/vuln-builder/gitea-test.txt - ls -lah /vuln-cache/vuln-builder - - - name: Bootstrap and build - shell: bash - run: | - set -Eeuo pipefail - : "${GITEA_JOB_TOKEN:?missing token}" mkdir -p "$CACHE_ROOT" + export GIT_CONFIG_COUNT=1 export GIT_CONFIG_KEY_0=http.extraHeader export GIT_CONFIG_VALUE_0="Authorization: Basic $(printf 'oauth2:%s' "$GITEA_JOB_TOKEN" | base64 | tr -d '\n')" + if [ ! -d "$CACHE_ROOT/builder/.git" ]; then - git clone --depth 1 --branch main "$GITEA_BASE_URL/VULN/vuln-builder.git" "$CACHE_ROOT/builder" + git clone --depth 1 --branch main \ + "$GITEA_BASE_URL/VULN/vuln-builder.git" \ + "$CACHE_ROOT/builder" else - git -C "$CACHE_ROOT/builder" fetch --depth 1 origin main - git -C "$CACHE_ROOT/builder" reset --hard FETCH_HEAD + git -C "$CACHE_ROOT/builder" \ + fetch --depth 1 origin main + + git -C "$CACHE_ROOT/builder" \ + reset --hard FETCH_HEAD fi + + echo "Builder ready" + + # ============================================ + # 3. Download sources.json + # ============================================ - name: Load sources configuration shell: bash run: | set -Eeuo pipefail - - mkdir -p "$CACHE_ROOT" + umask 077 python3 - <<'PY' + import json import os import urllib.request + from pathlib import Path base = os.environ["GITEA_BASE_URL"].rstrip("/") token = os.environ["GITEA_JOB_TOKEN"] - root = os.environ["CACHE_ROOT"] url = ( f"{base}/VULN/vuln-status/" "raw/branch/main/sources.json" ) - req = urllib.request.Request( + request = urllib.request.Request( url, headers={"Authorization": f"token {token}"} ) - with urllib.request.urlopen(req, timeout=60) as response: + with urllib.request.urlopen( + request, timeout=120 + ) as response: data = response.read() - with open(f"{root}/sources.json", "wb") as f: - f.write(data) + config = json.loads(data) - print("sources.json downloaded") + if not isinstance(config.get("sources"), list): + raise ValueError("Invalid sources.json") + + path = Path(os.environ["SOURCES_CONFIG"]) + path.parent.mkdir(parents=True, exist_ok=True) + path.write_bytes(data) + + print( + f"Loaded {len(config['sources'])} entries", + flush=True + ) PY + # ============================================ + # 4. Synchronize all repositories + # ============================================ + - name: Synchronize vulnerability repositories + shell: bash + run: | + set -Eeuo pipefail + umask 077 + + export GIT_CONFIG_COUNT=1 + export GIT_CONFIG_KEY_0=http.extraHeader + export GIT_CONFIG_VALUE_0="Authorization: Basic $(printf 'oauth2:%s' "$GITEA_JOB_TOKEN" | base64 | tr -d '\n')" + + python3 - <<'PY' + import json + import os + import re + import subprocess + from pathlib import Path + + root = Path(os.environ["CACHE_ROOT"]) + sources_dir = Path(os.environ["SOURCES_DIR"]) + base = os.environ["GITEA_BASE_URL"].rstrip("/") + + config = json.loads( + Path(os.environ["SOURCES_CONFIG"]) + .read_text(encoding="utf-8") + ) + + owner = config.get("owner", "VULN") + branch = config.get("branch", "main") + + valid_repo = re.compile(r"^vuln-[a-z0-9-]+$") + valid_owner = re.compile(r"^[A-Za-z0-9_-]+$") + valid_branch = re.compile(r"^[A-Za-z0-9_./-]+$") + + if not valid_owner.fullmatch(owner): + raise ValueError("Invalid owner") + + if ( + not valid_branch.fullmatch(branch) + or branch.startswith("-") + or ".." in branch + or branch.endswith("/") + ): + raise ValueError("Invalid branch") + + sources_dir.mkdir(parents=True, exist_ok=True) + + manifest = [] + seen = set() + + def git(*args, cwd=None): + subprocess.run( + ["git", *args], + cwd=cwd, + check=True + ) + + for entry in config["sources"]: + if entry.get("enabled", True) is False: + continue + + name = entry["repo"] + + if not valid_repo.fullmatch(name): + raise ValueError(f"Invalid repo: {name}") + + if name in seen: + raise ValueError(f"Duplicate repo: {name}") + + seen.add(name) + + repo_dir = sources_dir / name + repo_url = f"{base}/{owner}/{name}.git" + + print(f"\n=== SYNC {name} ===", flush=True) + + if not (repo_dir / ".git").is_dir(): + git( + "clone", + "--depth", "1", + "--single-branch", + "--branch", branch, + repo_url, + str(repo_dir) + ) + else: + git( + "remote", "set-url", + "origin", repo_url, + cwd=repo_dir + ) + git( + "fetch", + "--depth", "1", + "origin", branch, + cwd=repo_dir + ) + git( + "reset", "--hard", + "FETCH_HEAD", + cwd=repo_dir + ) + + commit = subprocess.check_output( + ["git", "rev-parse", "HEAD"], + cwd=repo_dir, + text=True + ).strip() + + manifest.append({ + "repo": name, + "commit": commit, + "path": str(repo_dir) + }) + + print( + f"OK {name}: {commit[:12]}", + flush=True + ) + + if not manifest: + raise RuntimeError( + "No enabled repositories found" + ) + + manifest_path = root / "source-manifest.json" + temp_path = root / "source-manifest.json.tmp" + + temp_path.write_text( + json.dumps(manifest, indent=2), + encoding="utf-8" + ) + temp_path.replace(manifest_path) + + print( + f"\nSynchronized: {len(manifest)} repositories", + flush=True + ) + PY + + # ============================================ + # 5. Build SQLite master and delta + # ============================================ + - name: Build SQLite database + shell: bash + run: | + set -Eeuo pipefail + + echo "Starting SQLite Builder" + + bash "$CACHE_ROOT/builder/scripts/run.sh" + + # ============================================ + # 6. Show results + # ============================================ + - name: Show build results + if: always() + shell: sh + run: | + echo "=== Cache usage ===" + du -sh "$CACHE_ROOT" 2>/dev/null || true + + echo "=== Sources ===" + ls -lah "$CACHE_ROOT/sources" \ + 2>/dev/null || true + + echo "=== Releases ===" + ls -lah "$CACHE_ROOT/releases" \ + 2>/dev/null || true From 7fcf3cf0f13ad6debc36a91c3c7d9305eb161250 Mon Sep 17 00:00:00 2001 From: Ronny Grobel Date: Thu, 12 Nov 2026 06:42:25 +0100 Subject: [PATCH 09/14] scripts/build.py aktualisiert --- scripts/build.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/scripts/build.py b/scripts/build.py index 7b83689..a6d6ba1 100644 --- a/scripts/build.py +++ b/scripts/build.py @@ -54,6 +54,11 @@ def upsert_vuln(db, vid, published=None, modified=None, state=None, summary=None def process_cve(db, obj): + if not isinstance(obj, dict): + raise ValueError( + f"Unexpected CVE JSON type: {type(obj).__name__}; " + f"sample={str(obj)[:300]}" + ) m = obj.get('cveMetadata') if not isinstance(m,dict): return False From 2bbf6f2ecb61ce6c4857d7ec4f73d221a970a736 Mon Sep 17 00:00:00 2001 From: Ronny Grobel Date: Thu, 12 Nov 2026 06:50:27 +0100 Subject: [PATCH 10/14] scripts/build.py aktualisiert --- scripts/build.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/scripts/build.py b/scripts/build.py index a6d6ba1..51bbbf8 100644 --- a/scripts/build.py +++ b/scripts/build.py @@ -38,6 +38,13 @@ def input_files(repo): d = repo / 'data' if not d.exists(): return + + # Nur tatsächliche CVE-Datensätze einlesen + if repo.name == 'vuln-cve': + yield from sorted(d.rglob('CVE-*.json')) + return + + # Andere Quellen unverändert verarbeiten yield from sorted(d.rglob('*.json')) From bed1f9ad8af0e47504b83c5e13f2d0b4897c5c33 Mon Sep 17 00:00:00 2001 From: Ronny Grobel Date: Thu, 12 Nov 2026 06:53:15 +0100 Subject: [PATCH 11/14] scripts/build.py aktualisiert --- scripts/build.py | 18 +++++++++++++----- 1 file changed, 13 insertions(+), 5 deletions(-) diff --git a/scripts/build.py b/scripts/build.py index 51bbbf8..1db10a8 100644 --- a/scripts/build.py +++ b/scripts/build.py @@ -36,15 +36,23 @@ def get_git_revision(path): def input_files(repo): d = repo / 'data' - if not d.exists(): - return - # Nur tatsächliche CVE-Datensätze einlesen + if not d.is_dir(): + raise RuntimeError( + f'Data directory missing: {d}' + ) + if repo.name == 'vuln-cve': - yield from sorted(d.rglob('CVE-*.json')) + files = sorted(d.rglob('CVE-*.json')) + + if not files: + raise RuntimeError( + f'No CVE records found in {d}' + ) + + yield from files return - # Andere Quellen unverändert verarbeiten yield from sorted(d.rglob('*.json')) From 2d6e8f4bdc5a33505b92cc8eab19f2c97940de8d Mon Sep 17 00:00:00 2001 From: Ronny Grobel Date: Thu, 12 Nov 2026 16:40:01 +0100 Subject: [PATCH 12/14] sql/schema.sql aktualisiert --- sql/schema.sql | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/sql/schema.sql b/sql/schema.sql index 5ab2592..76a1c6e 100644 --- a/sql/schema.sql +++ b/sql/schema.sql @@ -14,4 +14,24 @@ CREATE TABLE IF NOT EXISTS cpe_matches ( criteria TEXT, vulnerable INTEGER, version_start_including TEXT, version_start_excluding TEXT, version_end_including TEXT, version_end_excluding TEXT, source TEXT NOT NULL); CREATE INDEX IF NOT EXISTS idx_cpe_vuln ON cpe_matches(vulnerability_id); +CREATE TABLE IF NOT EXISTS cpe_nodes ( + id TEXT PRIMARY KEY, + vulnerability_id TEXT NOT NULL + REFERENCES vulnerabilities(id) + ON DELETE CASCADE, + parent_id TEXT REFERENCES cpe_nodes(id), + operator TEXT NOT NULL + CHECK (operator IN ('AND', 'OR')), + negate INTEGER NOT NULL DEFAULT 0 + CHECK (negate IN (0, 1)), + source TEXT NOT NULL +); + +CREATE INDEX IF NOT EXISTS idx_cpe_nodes_vuln +ON cpe_nodes(vulnerability_id); +ALTER TABLE cpe_matches +ADD COLUMN node_id TEXT REFERENCES cpe_nodes(id); + +CREATE INDEX IF NOT EXISTS idx_cpe_matches_node +ON cpe_matches(node_id); -- Conditions expressed as AND, OR and negate are deliberately NOT converted into independent vulnerability assertions. From e953869245c49629fa6edf1953fa17ce1ba5c052 Mon Sep 17 00:00:00 2001 From: Ronny Grobel Date: Thu, 12 Nov 2026 17:42:33 +0100 Subject: [PATCH 13/14] scripts/build.py aktualisiert --- scripts/build.py | 29 ++++++++++++----------------- 1 file changed, 12 insertions(+), 17 deletions(-) diff --git a/scripts/build.py b/scripts/build.py index 1db10a8..6763520 100644 --- a/scripts/build.py +++ b/scripts/build.py @@ -36,23 +36,15 @@ def get_git_revision(path): def input_files(repo): d = repo / 'data' - - if not d.is_dir(): - raise RuntimeError( - f'Data directory missing: {d}' - ) - - if repo.name == 'vuln-cve': - files = sorted(d.rglob('CVE-*.json')) - - if not files: - raise RuntimeError( - f'No CVE records found in {d}' - ) - - yield from files + if not d.exists(): return + # Nur tatsächliche CVE-Datensätze einlesen + if repo.name == 'vuln-cve': + yield from sorted(d.rglob('CVE-*.json')) + return + + # Andere Quellen unverändert verarbeiten yield from sorted(d.rglob('*.json')) @@ -102,9 +94,10 @@ def walk_cpe(node, allow_simple=True): def process_nvd(db,obj): - cve=obj.get('cve') - if not isinstance(cve,dict): + # NVD mirror stores the CVE object directly; API responses wrap it in {'cve': ...}. + if not isinstance(obj,dict): return False + cve=obj.get('cve') if isinstance(obj.get('cve'),dict) else obj vid=cve.get('id') if not isinstance(vid,str) or not CVE.match(vid): return False @@ -185,6 +178,8 @@ def stream_source(db,name,path): print(f'INVALID {name} {file}: {exc}',flush=True) if bad: raise RuntimeError(f'{name}: {bad} invalid records; database not published') + if name == 'vuln-nvd' and count == 0: + raise RuntimeError('vuln-nvd: no usable documents; check source data and JSON shape') return count From 990d1132470caade18fd8c7ca9382c74f2c27fde Mon Sep 17 00:00:00 2001 From: Ronny Grobel Date: Thu, 12 Nov 2026 20:49:50 +0100 Subject: [PATCH 14/14] scripts/build.py aktualisiert --- scripts/build.py | 48 +++++++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 47 insertions(+), 1 deletion(-) diff --git a/scripts/build.py b/scripts/build.py index 6763520..e2fb518 100644 --- a/scripts/build.py +++ b/scripts/build.py @@ -212,11 +212,41 @@ def compress(src,out): subprocess.run(['zstd','-q','-f','-T1','-19',str(src),'-o',str(out)],check=True) +def build_signature(source_revs, schema_path): + """Fingerprint inputs that can change the generated data, not wall-clock time.""" + scripts_dir = Path(__file__).resolve().parent + h = hashlib.sha256() + # Include every Python importer/helper alongside the builder, as well as SQL schema. + inputs = sorted(scripts_dir.glob('*.py')) + inputs.append(schema_path.resolve()) + for path in inputs: + if not path.is_file(): + raise RuntimeError(f'Missing build input: {path}') + logical_name = 'schema.sql' if path == schema_path.resolve() else f'scripts/{path.name}' + h.update(logical_name.encode('utf-8') + b'\x00') + h.update(hashlib.sha256(path.read_bytes()).digest()) + h.update(canonical(source_revs).encode('utf-8')) + return h.hexdigest() + + +def existing_release_ok(output, manifest): + """Only skip if current release artifacts exist and match recorded hash.""" + dbfile, packed = output / 'master.db', output / 'master.db.zst' + if not dbfile.is_file() or not packed.is_file() or not manifest.get('master_sha256'): + return False + h = hashlib.sha256() + with packed.open('rb') as stream: + for block in iter(lambda: stream.read(1024 * 1024), b''): + h.update(block) + return h.hexdigest() == manifest['master_sha256'] + + def main(): ap=argparse.ArgumentParser() ap.add_argument('--sources',type=Path,required=True) ap.add_argument('--output',type=Path,required=True) ap.add_argument('--schema',type=Path,default=Path(__file__).resolve().parents[1]/'sql/schema.sql') + ap.add_argument('--force', action='store_true', help='Force rebuild even when input signature is unchanged') args=ap.parse_args() args.output.mkdir(parents=True,exist_ok=True) active=[(p.name,p) for p in sorted(args.sources.iterdir()) if p.is_dir() and (p/'.git').exists()] @@ -227,7 +257,22 @@ def main(): if name not in [n for n,_ in active]: raise SystemExit(f'Missing core source: {name}') source_revs={name:get_git_revision(path) for name,path in active} - old_db=args.output/'master.db' + signature = build_signature(source_revs, args.schema) + manifest_path = args.output / 'manifest.json' + previous_manifest = None + if manifest_path.is_file(): + try: + previous_manifest = json.loads(manifest_path.read_text(encoding='utf-8')) + except (ValueError, UnicodeError) as exc: + raise RuntimeError(f'Invalid existing manifest, refusing to overwrite: {exc}') from exc + if (not args.force and previous_manifest + and previous_manifest.get('build_signature') == signature + and existing_release_ok(args.output, previous_manifest)): + print(f"NO_CHANGES: version={previous_manifest['version']} inputs unchanged; master and delta retained", flush=True) + return + if not args.force and previous_manifest and previous_manifest.get('build_signature') == signature: + print('WARNING: unchanged inputs but missing/invalid release artifact; rebuilding', flush=True) + old_db=args.output/'master.db' with tempfile.TemporaryDirectory(prefix='vuln-build-',dir=args.output) as td: tmp=Path(td); new=tmp/'master.db'; db=sqlite3.connect(new) db.executescript(args.schema.read_text(encoding='utf-8')) @@ -253,6 +298,7 @@ def main(): assert db.execute('PRAGMA integrity_check').fetchone()[0]=='ok' db.close() manifest={'version':version,'schema_version':1,'previous':old_version,'source_commits':source_revs, + 'build_signature':signature, 'indexed_documents':stats,'format':'sqlite+zstd','delta_format':'jsonl+zstd', 'note':'Only CVE/NVD/OSV indexed in v1; no complete version-matching engine'} compress(new,tmp/'master.db.zst')