303 lines
8.5 KiB
YAML
303 lines
8.5 KiB
YAML
|
|
name: Build SQLite vulnerability index
|
|
|
|
on:
|
|
workflow_dispatch:
|
|
schedule:
|
|
- cron: "35 3 * * *"
|
|
|
|
concurrency:
|
|
group: vuln-builder
|
|
cancel-in-progress: false
|
|
|
|
jobs:
|
|
build:
|
|
runs-on: vuln-builder
|
|
timeout-minutes: 720
|
|
|
|
env:
|
|
CACHE_ROOT: /vuln-cache/vuln-builder
|
|
GITEA_BASE_URL: https://git.apex-project.de
|
|
GITEA_JOB_TOKEN: ${{ secrets.GITEA_TOKEN }}
|
|
SOURCES_DIR: /vuln-cache/vuln-builder/sources
|
|
SOURCES_CONFIG: /vuln-cache/vuln-builder/sources.json
|
|
|
|
steps:
|
|
# ============================================
|
|
# 1. Install dependencies
|
|
# ============================================
|
|
- name: Install tools
|
|
shell: sh
|
|
run: |
|
|
set -eu
|
|
|
|
if command -v apk >/dev/null 2>&1; then
|
|
apk add --no-cache \
|
|
bash git python3 sqlite zstd \
|
|
util-linux ca-certificates coreutils
|
|
|
|
elif command -v apt-get >/dev/null 2>&1; then
|
|
apt-get update
|
|
apt-get install -y --no-install-recommends \
|
|
bash git python3 sqlite3 zstd \
|
|
util-linux ca-certificates coreutils
|
|
else
|
|
echo "Unsupported job image" >&2
|
|
exit 1
|
|
fi
|
|
|
|
python3 --version
|
|
git --version
|
|
/usr/bin/flock --version
|
|
|
|
# ============================================
|
|
# 2. Bootstrap builder
|
|
# ============================================
|
|
- name: Bootstrap builder
|
|
shell: bash
|
|
run: |
|
|
set -Eeuo pipefail
|
|
umask 077
|
|
|
|
: "${GITEA_JOB_TOKEN:?Missing Gitea token}"
|
|
|
|
mkdir -p "$CACHE_ROOT"
|
|
|
|
export GIT_CONFIG_COUNT=1
|
|
export GIT_CONFIG_KEY_0=http.extraHeader
|
|
export GIT_CONFIG_VALUE_0="Authorization: Basic $(printf 'oauth2:%s' "$GITEA_JOB_TOKEN" | base64 | tr -d '\n')"
|
|
|
|
if [ ! -d "$CACHE_ROOT/builder/.git" ]; then
|
|
git clone --depth 1 --branch main \
|
|
"$GITEA_BASE_URL/VULN/vuln-builder.git" \
|
|
"$CACHE_ROOT/builder"
|
|
else
|
|
git -C "$CACHE_ROOT/builder" \
|
|
fetch --depth 1 origin main
|
|
|
|
git -C "$CACHE_ROOT/builder" \
|
|
reset --hard FETCH_HEAD
|
|
fi
|
|
|
|
echo "Builder ready"
|
|
|
|
# ============================================
|
|
# 3. Download sources.json
|
|
# ============================================
|
|
- name: Load sources configuration
|
|
shell: bash
|
|
run: |
|
|
set -Eeuo pipefail
|
|
umask 077
|
|
|
|
python3 - <<'PY'
|
|
import json
|
|
import os
|
|
import urllib.request
|
|
from pathlib import Path
|
|
|
|
base = os.environ["GITEA_BASE_URL"].rstrip("/")
|
|
token = os.environ["GITEA_JOB_TOKEN"]
|
|
|
|
url = (
|
|
f"{base}/VULN/vuln-status/"
|
|
"raw/branch/main/sources.json"
|
|
)
|
|
|
|
request = urllib.request.Request(
|
|
url,
|
|
headers={"Authorization": f"token {token}"}
|
|
)
|
|
|
|
with urllib.request.urlopen(
|
|
request, timeout=120
|
|
) as response:
|
|
data = response.read()
|
|
|
|
config = json.loads(data)
|
|
|
|
if not isinstance(config.get("sources"), list):
|
|
raise ValueError("Invalid sources.json")
|
|
|
|
path = Path(os.environ["SOURCES_CONFIG"])
|
|
path.parent.mkdir(parents=True, exist_ok=True)
|
|
path.write_bytes(data)
|
|
|
|
print(
|
|
f"Loaded {len(config['sources'])} entries",
|
|
flush=True
|
|
)
|
|
PY
|
|
|
|
# ============================================
|
|
# 4. Synchronize all repositories
|
|
# ============================================
|
|
- name: Synchronize vulnerability repositories
|
|
shell: bash
|
|
run: |
|
|
set -Eeuo pipefail
|
|
umask 077
|
|
|
|
export GIT_CONFIG_COUNT=1
|
|
export GIT_CONFIG_KEY_0=http.extraHeader
|
|
export GIT_CONFIG_VALUE_0="Authorization: Basic $(printf 'oauth2:%s' "$GITEA_JOB_TOKEN" | base64 | tr -d '\n')"
|
|
|
|
python3 - <<'PY'
|
|
import json
|
|
import os
|
|
import re
|
|
import subprocess
|
|
from pathlib import Path
|
|
|
|
root = Path(os.environ["CACHE_ROOT"])
|
|
sources_dir = Path(os.environ["SOURCES_DIR"])
|
|
base = os.environ["GITEA_BASE_URL"].rstrip("/")
|
|
|
|
config = json.loads(
|
|
Path(os.environ["SOURCES_CONFIG"])
|
|
.read_text(encoding="utf-8")
|
|
)
|
|
|
|
owner = config.get("owner", "VULN")
|
|
branch = config.get("branch", "main")
|
|
|
|
valid_repo = re.compile(r"^vuln-[a-z0-9-]+$")
|
|
valid_owner = re.compile(r"^[A-Za-z0-9_-]+$")
|
|
valid_branch = re.compile(r"^[A-Za-z0-9_./-]+$")
|
|
|
|
if not valid_owner.fullmatch(owner):
|
|
raise ValueError("Invalid owner")
|
|
|
|
if (
|
|
not valid_branch.fullmatch(branch)
|
|
or branch.startswith("-")
|
|
or ".." in branch
|
|
or branch.endswith("/")
|
|
):
|
|
raise ValueError("Invalid branch")
|
|
|
|
sources_dir.mkdir(parents=True, exist_ok=True)
|
|
|
|
manifest = []
|
|
seen = set()
|
|
|
|
def git(*args, cwd=None):
|
|
subprocess.run(
|
|
["git", *args],
|
|
cwd=cwd,
|
|
check=True
|
|
)
|
|
|
|
for entry in config["sources"]:
|
|
if entry.get("enabled", True) is False:
|
|
continue
|
|
|
|
name = entry["repo"]
|
|
|
|
if not valid_repo.fullmatch(name):
|
|
raise ValueError(f"Invalid repo: {name}")
|
|
|
|
if name in seen:
|
|
raise ValueError(f"Duplicate repo: {name}")
|
|
|
|
seen.add(name)
|
|
|
|
repo_dir = sources_dir / name
|
|
repo_url = f"{base}/{owner}/{name}.git"
|
|
|
|
print(f"\n=== SYNC {name} ===", flush=True)
|
|
|
|
if not (repo_dir / ".git").is_dir():
|
|
git(
|
|
"clone",
|
|
"--depth", "1",
|
|
"--single-branch",
|
|
"--branch", branch,
|
|
repo_url,
|
|
str(repo_dir)
|
|
)
|
|
else:
|
|
git(
|
|
"remote", "set-url",
|
|
"origin", repo_url,
|
|
cwd=repo_dir
|
|
)
|
|
git(
|
|
"fetch",
|
|
"--depth", "1",
|
|
"origin", branch,
|
|
cwd=repo_dir
|
|
)
|
|
git(
|
|
"reset", "--hard",
|
|
"FETCH_HEAD",
|
|
cwd=repo_dir
|
|
)
|
|
|
|
commit = subprocess.check_output(
|
|
["git", "rev-parse", "HEAD"],
|
|
cwd=repo_dir,
|
|
text=True
|
|
).strip()
|
|
|
|
manifest.append({
|
|
"repo": name,
|
|
"commit": commit,
|
|
"path": str(repo_dir)
|
|
})
|
|
|
|
print(
|
|
f"OK {name}: {commit[:12]}",
|
|
flush=True
|
|
)
|
|
|
|
if not manifest:
|
|
raise RuntimeError(
|
|
"No enabled repositories found"
|
|
)
|
|
|
|
manifest_path = root / "source-manifest.json"
|
|
temp_path = root / "source-manifest.json.tmp"
|
|
|
|
temp_path.write_text(
|
|
json.dumps(manifest, indent=2),
|
|
encoding="utf-8"
|
|
)
|
|
temp_path.replace(manifest_path)
|
|
|
|
print(
|
|
f"\nSynchronized: {len(manifest)} repositories",
|
|
flush=True
|
|
)
|
|
PY
|
|
|
|
# ============================================
|
|
# 5. Build SQLite master and delta
|
|
# ============================================
|
|
- name: Build SQLite database
|
|
shell: bash
|
|
run: |
|
|
set -Eeuo pipefail
|
|
|
|
echo "Starting SQLite Builder"
|
|
|
|
bash "$CACHE_ROOT/builder/scripts/run.sh"
|
|
|
|
# ============================================
|
|
# 6. Show results
|
|
# ============================================
|
|
- name: Show build results
|
|
if: always()
|
|
shell: sh
|
|
run: |
|
|
echo "=== Cache usage ==="
|
|
du -sh "$CACHE_ROOT" 2>/dev/null || true
|
|
|
|
echo "=== Sources ==="
|
|
ls -lah "$CACHE_ROOT/sources" \
|
|
2>/dev/null || true
|
|
|
|
echo "=== Releases ==="
|
|
ls -lah "$CACHE_ROOT/releases" \
|
|
2>/dev/null || true
|