.gitea/workflows/build.yml aktualisiert
This commit is contained in:
RonnyG committed
1 parent
bb1825c2cc
commit
5c3c7e18b7
1 file changed
+236
-36
+236
-36
@@ -1,102 +1,302 @@
|
||||
|
||||
name: Build SQLite vulnerability index
|
||||
|
||||
on:
|
||||
workflow_dispatch:
|
||||
schedule:
|
||||
- cron: "35 3 * * *"
|
||||
|
||||
concurrency:
|
||||
group: vuln-builder
|
||||
cancel-in-progress: false
|
||||
|
||||
jobs:
|
||||
build:
|
||||
runs-on: vuln-builder
|
||||
timeout-minutes: 720
|
||||
|
||||
env:
|
||||
CACHE_ROOT: /vuln-cache/vuln-builder
|
||||
GITEA_BASE_URL: https://git.apex-project.de
|
||||
GITEA_JOB_TOKEN: ${{ secrets.GITEA_TOKEN }}
|
||||
SOURCES_DIR: /vuln-cache/vuln-builder/sources
|
||||
SOURCES_CONFIG: /vuln-cache/vuln-builder/sources.json
|
||||
|
||||
steps:
|
||||
# ============================================
|
||||
# 1. Install dependencies
|
||||
# ============================================
|
||||
- name: Install tools
|
||||
shell: sh
|
||||
run: |
|
||||
set -eu
|
||||
|
||||
if command -v apk >/dev/null 2>&1; then
|
||||
apk add --no-cache bash git python3 sqlite zstd util-linux ca-certificates coreutils
|
||||
apk add --no-cache \
|
||||
bash git python3 sqlite zstd \
|
||||
util-linux ca-certificates coreutils
|
||||
|
||||
elif command -v apt-get >/dev/null 2>&1; then
|
||||
apt-get update
|
||||
apt-get install -y --no-install-recommends bash git python3 sqlite3 zstd util-linux ca-certificates coreutils
|
||||
apt-get install -y --no-install-recommends \
|
||||
bash git python3 sqlite3 zstd \
|
||||
util-linux ca-certificates coreutils
|
||||
else
|
||||
echo 'Unsupported job image' >&2
|
||||
echo "Unsupported job image" >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
python3 --version
|
||||
git --version
|
||||
/usr/bin/flock --version
|
||||
- name: Verify persistent cache
|
||||
|
||||
# ============================================
|
||||
# 2. Bootstrap builder
|
||||
# ============================================
|
||||
- name: Bootstrap builder
|
||||
shell: bash
|
||||
run: |
|
||||
set -Eeuo pipefail
|
||||
umask 077
|
||||
|
||||
echo "=== Cache Mount ==="
|
||||
findmnt -T /vuln-cache || true
|
||||
: "${GITEA_JOB_TOKEN:?Missing Gitea token}"
|
||||
|
||||
echo "=== Schreibtest ==="
|
||||
mkdir -p /vuln-cache/vuln-builder
|
||||
echo "CACHE_TEST_OK" > /vuln-cache/vuln-builder/test.txt
|
||||
|
||||
echo "=== Inhalt ==="
|
||||
ls -lah /vuln-cache/vuln-builder
|
||||
- name: Test Gitea cache
|
||||
shell: sh
|
||||
run: |
|
||||
set -eu
|
||||
mkdir -p /vuln-cache/vuln-builder
|
||||
echo "GITEA_JOB_OK" > /vuln-cache/vuln-builder/gitea-test.txt
|
||||
ls -lah /vuln-cache/vuln-builder
|
||||
|
||||
- name: Bootstrap and build
|
||||
shell: bash
|
||||
run: |
|
||||
set -Eeuo pipefail
|
||||
: "${GITEA_JOB_TOKEN:?missing token}"
|
||||
mkdir -p "$CACHE_ROOT"
|
||||
|
||||
export GIT_CONFIG_COUNT=1
|
||||
export GIT_CONFIG_KEY_0=http.extraHeader
|
||||
export GIT_CONFIG_VALUE_0="Authorization: Basic $(printf 'oauth2:%s' "$GITEA_JOB_TOKEN" | base64 | tr -d '\n')"
|
||||
|
||||
if [ ! -d "$CACHE_ROOT/builder/.git" ]; then
|
||||
git clone --depth 1 --branch main "$GITEA_BASE_URL/VULN/vuln-builder.git" "$CACHE_ROOT/builder"
|
||||
git clone --depth 1 --branch main \
|
||||
"$GITEA_BASE_URL/VULN/vuln-builder.git" \
|
||||
"$CACHE_ROOT/builder"
|
||||
else
|
||||
git -C "$CACHE_ROOT/builder" fetch --depth 1 origin main
|
||||
git -C "$CACHE_ROOT/builder" reset --hard FETCH_HEAD
|
||||
git -C "$CACHE_ROOT/builder" \
|
||||
fetch --depth 1 origin main
|
||||
|
||||
git -C "$CACHE_ROOT/builder" \
|
||||
reset --hard FETCH_HEAD
|
||||
fi
|
||||
|
||||
echo "Builder ready"
|
||||
|
||||
# ============================================
|
||||
# 3. Download sources.json
|
||||
# ============================================
|
||||
- name: Load sources configuration
|
||||
shell: bash
|
||||
run: |
|
||||
set -Eeuo pipefail
|
||||
|
||||
mkdir -p "$CACHE_ROOT"
|
||||
umask 077
|
||||
|
||||
python3 - <<'PY'
|
||||
import json
|
||||
import os
|
||||
import urllib.request
|
||||
from pathlib import Path
|
||||
|
||||
base = os.environ["GITEA_BASE_URL"].rstrip("/")
|
||||
token = os.environ["GITEA_JOB_TOKEN"]
|
||||
root = os.environ["CACHE_ROOT"]
|
||||
|
||||
url = (
|
||||
f"{base}/VULN/vuln-status/"
|
||||
"raw/branch/main/sources.json"
|
||||
)
|
||||
|
||||
req = urllib.request.Request(
|
||||
request = urllib.request.Request(
|
||||
url,
|
||||
headers={"Authorization": f"token {token}"}
|
||||
)
|
||||
|
||||
with urllib.request.urlopen(req, timeout=60) as response:
|
||||
with urllib.request.urlopen(
|
||||
request, timeout=120
|
||||
) as response:
|
||||
data = response.read()
|
||||
|
||||
with open(f"{root}/sources.json", "wb") as f:
|
||||
f.write(data)
|
||||
config = json.loads(data)
|
||||
|
||||
print("sources.json downloaded")
|
||||
if not isinstance(config.get("sources"), list):
|
||||
raise ValueError("Invalid sources.json")
|
||||
|
||||
path = Path(os.environ["SOURCES_CONFIG"])
|
||||
path.parent.mkdir(parents=True, exist_ok=True)
|
||||
path.write_bytes(data)
|
||||
|
||||
print(
|
||||
f"Loaded {len(config['sources'])} entries",
|
||||
flush=True
|
||||
)
|
||||
PY
|
||||
|
||||
# ============================================
|
||||
# 4. Synchronize all repositories
|
||||
# ============================================
|
||||
- name: Synchronize vulnerability repositories
|
||||
shell: bash
|
||||
run: |
|
||||
set -Eeuo pipefail
|
||||
umask 077
|
||||
|
||||
export GIT_CONFIG_COUNT=1
|
||||
export GIT_CONFIG_KEY_0=http.extraHeader
|
||||
export GIT_CONFIG_VALUE_0="Authorization: Basic $(printf 'oauth2:%s' "$GITEA_JOB_TOKEN" | base64 | tr -d '\n')"
|
||||
|
||||
python3 - <<'PY'
|
||||
import json
|
||||
import os
|
||||
import re
|
||||
import subprocess
|
||||
from pathlib import Path
|
||||
|
||||
root = Path(os.environ["CACHE_ROOT"])
|
||||
sources_dir = Path(os.environ["SOURCES_DIR"])
|
||||
base = os.environ["GITEA_BASE_URL"].rstrip("/")
|
||||
|
||||
config = json.loads(
|
||||
Path(os.environ["SOURCES_CONFIG"])
|
||||
.read_text(encoding="utf-8")
|
||||
)
|
||||
|
||||
owner = config.get("owner", "VULN")
|
||||
branch = config.get("branch", "main")
|
||||
|
||||
valid_repo = re.compile(r"^vuln-[a-z0-9-]+$")
|
||||
valid_owner = re.compile(r"^[A-Za-z0-9_-]+$")
|
||||
valid_branch = re.compile(r"^[A-Za-z0-9_./-]+$")
|
||||
|
||||
if not valid_owner.fullmatch(owner):
|
||||
raise ValueError("Invalid owner")
|
||||
|
||||
if (
|
||||
not valid_branch.fullmatch(branch)
|
||||
or branch.startswith("-")
|
||||
or ".." in branch
|
||||
or branch.endswith("/")
|
||||
):
|
||||
raise ValueError("Invalid branch")
|
||||
|
||||
sources_dir.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
manifest = []
|
||||
seen = set()
|
||||
|
||||
def git(*args, cwd=None):
|
||||
subprocess.run(
|
||||
["git", *args],
|
||||
cwd=cwd,
|
||||
check=True
|
||||
)
|
||||
|
||||
for entry in config["sources"]:
|
||||
if entry.get("enabled", True) is False:
|
||||
continue
|
||||
|
||||
name = entry["repo"]
|
||||
|
||||
if not valid_repo.fullmatch(name):
|
||||
raise ValueError(f"Invalid repo: {name}")
|
||||
|
||||
if name in seen:
|
||||
raise ValueError(f"Duplicate repo: {name}")
|
||||
|
||||
seen.add(name)
|
||||
|
||||
repo_dir = sources_dir / name
|
||||
repo_url = f"{base}/{owner}/{name}.git"
|
||||
|
||||
print(f"\n=== SYNC {name} ===", flush=True)
|
||||
|
||||
if not (repo_dir / ".git").is_dir():
|
||||
git(
|
||||
"clone",
|
||||
"--depth", "1",
|
||||
"--single-branch",
|
||||
"--branch", branch,
|
||||
repo_url,
|
||||
str(repo_dir)
|
||||
)
|
||||
else:
|
||||
git(
|
||||
"remote", "set-url",
|
||||
"origin", repo_url,
|
||||
cwd=repo_dir
|
||||
)
|
||||
git(
|
||||
"fetch",
|
||||
"--depth", "1",
|
||||
"origin", branch,
|
||||
cwd=repo_dir
|
||||
)
|
||||
git(
|
||||
"reset", "--hard",
|
||||
"FETCH_HEAD",
|
||||
cwd=repo_dir
|
||||
)
|
||||
|
||||
commit = subprocess.check_output(
|
||||
["git", "rev-parse", "HEAD"],
|
||||
cwd=repo_dir,
|
||||
text=True
|
||||
).strip()
|
||||
|
||||
manifest.append({
|
||||
"repo": name,
|
||||
"commit": commit,
|
||||
"path": str(repo_dir)
|
||||
})
|
||||
|
||||
print(
|
||||
f"OK {name}: {commit[:12]}",
|
||||
flush=True
|
||||
)
|
||||
|
||||
if not manifest:
|
||||
raise RuntimeError(
|
||||
"No enabled repositories found"
|
||||
)
|
||||
|
||||
manifest_path = root / "source-manifest.json"
|
||||
temp_path = root / "source-manifest.json.tmp"
|
||||
|
||||
temp_path.write_text(
|
||||
json.dumps(manifest, indent=2),
|
||||
encoding="utf-8"
|
||||
)
|
||||
temp_path.replace(manifest_path)
|
||||
|
||||
print(
|
||||
f"\nSynchronized: {len(manifest)} repositories",
|
||||
flush=True
|
||||
)
|
||||
PY
|
||||
|
||||
# ============================================
|
||||
# 5. Build SQLite master and delta
|
||||
# ============================================
|
||||
- name: Build SQLite database
|
||||
shell: bash
|
||||
run: |
|
||||
set -Eeuo pipefail
|
||||
|
||||
echo "Starting SQLite Builder"
|
||||
|
||||
bash "$CACHE_ROOT/builder/scripts/run.sh"
|
||||
|
||||
# ============================================
|
||||
# 6. Show results
|
||||
# ============================================
|
||||
- name: Show build results
|
||||
if: always()
|
||||
shell: sh
|
||||
run: |
|
||||
echo "=== Cache usage ==="
|
||||
du -sh "$CACHE_ROOT" 2>/dev/null || true
|
||||
|
||||
echo "=== Sources ==="
|
||||
ls -lah "$CACHE_ROOT/sources" \
|
||||
2>/dev/null || true
|
||||
|
||||
echo "=== Releases ==="
|
||||
ls -lah "$CACHE_ROOT/releases" \
|
||||
2>/dev/null || true
|
||||
Reference in new issue
Block a user