From 2b662f7b42b3539f93d1e63d5ab1c8d65f69938e Mon Sep 17 00:00:00 2001 From: root Date: Tue, 10 Nov 2026 22:08:53 +0100 Subject: [PATCH] Initial Alpine security mirror --- .gitea/workflows/update.yml | 33 +++++++++ .gitignore | 4 ++ README.md | 47 ++++++++++++ data/secdb/.gitkeep | 0 scripts/run-cached.sh | 36 ++++++++++ scripts/sync.py | 140 ++++++++++++++++++++++++++++++++++++ scripts/validate.py | 22 ++++++ state/.gitkeep | 0 tests/test_sync.py | 16 +++++ 9 files changed, 298 insertions(+) create mode 100644 .gitea/workflows/update.yml create mode 100644 .gitignore create mode 100644 README.md create mode 100644 data/secdb/.gitkeep create mode 100644 scripts/run-cached.sh create mode 100644 scripts/sync.py create mode 100644 scripts/validate.py create mode 100644 state/.gitkeep create mode 100644 tests/test_sync.py diff --git a/.gitea/workflows/update.yml b/.gitea/workflows/update.yml new file mode 100644 index 0000000..65b1abe --- /dev/null +++ b/.gitea/workflows/update.yml @@ -0,0 +1,33 @@ +name: Alpine Security Database Mirror + +on: + schedule: + - cron: "07 */6 * * *" + workflow_dispatch: + +jobs: + mirror: + runs-on: vuln-runner + timeout-minutes: 180 + steps: + - name: Alpine secdb synchronisieren + env: + GITEA_JOB_TOKEN: ${{ secrets.GITEA_TOKEN }} + CACHE_ROOT: /var/lib/act_runner/cache/vuln-alpine + REPO_URL: https://git.apex-project.de/VULN/vuln-alpine.git + run: | + set -Eeuo pipefail + mkdir -p "$CACHE_ROOT" + # Bootstrap is embedded to avoid a second checkout just to read a shell script. + if [[ ! -d "$CACHE_ROOT/repo/.git" ]]; then + : "${GITEA_JOB_TOKEN:?GITEA_TOKEN fehlt}" + fi + AUTH_HEADER="Authorization: Basic $(printf 'oauth2:%s' "$GITEA_JOB_TOKEN" | base64 | tr -d '\n')" + export GIT_CONFIG_COUNT=1 + export GIT_CONFIG_KEY_0=http.extraheader + export GIT_CONFIG_VALUE_0="$AUTH_HEADER" + export GIT_TERMINAL_PROMPT=0 + if [[ ! -d "$CACHE_ROOT/repo/.git" ]]; then + git clone --depth 1 --branch main "$REPO_URL" "$CACHE_ROOT/repo" + fi + bash "$CACHE_ROOT/repo/scripts/run-cached.sh" diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..fa477f9 --- /dev/null +++ b/.gitignore @@ -0,0 +1,4 @@ +__pycache__/ +*.pyc +*.tmp +.pytest_cache/ diff --git a/README.md b/README.md new file mode 100644 index 0000000..4dd66f0 --- /dev/null +++ b/README.md @@ -0,0 +1,47 @@ +# vuln-alpine – Gitea mirror + +Vollständiger Mirror der offiziellen Alpine Security Database im unveränderten JSON-Format. + +## Offizielle Quelle + +https://secdb.alpinelinux.org/ – das ältere `alpinelinux/alpine-secdb` GitHub-Repository ist **deprecated**. + +Der Downloader entdeckt Verzeichnisse `v3.5` und neuer sowie `edge` automatisch und speichert – wenn im offiziellen Verzeichnis vorhanden – `main.json` und `community.json` unter `data/secdb//`. Der Start ab `v3.5` bewahrt historische Daten rund um 2017. + +## Installation in Gitea + +1. Leeres Repository `VULN/vuln-alpine` auf https://git.apex-project.de erstellen (Branch `main`). +2. Diese Dateien in dein neues Repository committen und pushen. +3. Auf der Gitea-VM: + +```bash +sudo apt-get install -y git python3 util-linux ca-certificates coreutils +sudo install -d -o act_runner -g act_runner -m 700 /var/lib/act_runner/cache/vuln-alpine +``` + +4. Sicherstellen, dass der Runner das Label `vuln-runner` besitzt und `GITEA_TOKEN` Schreibzugriff auf `VULN/vuln-alpine` hat. +5. Action **Alpine Security Database Mirror** starten. Danach automatischer Zeitplan alle sechs Stunden zur Minute 07 (Gitea-Zeitzone/CRON-Konfiguration beachten). + +## Verhalten + +- Kein `actions/checkout`, sondern persistenter Checkout in `/var/lib/act_runner/cache/vuln-alpine/repo`. +- HTTP ETag und Last-Modified im lokalen `http-cache.json` (nicht in Git). +- Bei jeder Synchronisierung wird der offizielle Verzeichnisindex abgerufen. Neue Versionen werden automatisch entdeckt. +- JSON wird geparst und strukturell validiert; Originalbytes werden unverändert gespeichert und per SHA256 im `state/manifest.json` dokumentiert. +- Bei HTTP-/Validierungsfehlern wird kein Git-Commit durchgeführt. Der nächste Lauf setzt erneut an. +- Historische Daten werden bewusst **nicht** gelöscht, wenn eine alte Version aus dem Index verschwindet. Datenkorrekturen aus vorhandenen Quellen werden synchronisiert. +- Bei unverändertem Inhalt entsteht kein Commit. +- Git-Zugangsdaten werden nur als kurzlebige Environment-Konfiguration an Git-Unterprozesse übergeben. + +## Tests + +```bash +python3 -m unittest discover -s tests -v +bash -n scripts/run-cached.sh +``` + +## Hinweis + +Runner läuft im `host`-Modus unter `act_runner`. Damit können Workflows beliebige Programme mit den Rechten dieses Benutzers auf dem Gitea-Host ausführen. Nur vertrauenswürdige Workflows zulassen. + +Lizenz/Weiterverteilung: Die Bedingungen der Datenquelle vor einer öffentlichen Veröffentlichung prüfen. diff --git a/data/secdb/.gitkeep b/data/secdb/.gitkeep new file mode 100644 index 0000000..e69de29 diff --git a/scripts/run-cached.sh b/scripts/run-cached.sh new file mode 100644 index 0000000..62b9685 --- /dev/null +++ b/scripts/run-cached.sh @@ -0,0 +1,36 @@ +#!/usr/bin/env bash +set -Eeuo pipefail +umask 077 +: "${GITEA_JOB_TOKEN:?GITEA_JOB_TOKEN fehlt}" +CACHE_ROOT="${CACHE_ROOT:-/var/lib/act_runner/cache/vuln-alpine}" +REMOTE="${REPO_URL:-https://git.apex-project.de/VULN/vuln-alpine.git}" +mkdir -p "$CACHE_ROOT" +exec 9>"$CACHE_ROOT/.sync.lock" +flock -w 3600 9 || { echo 'Another Alpine mirror job still running' >&2; exit 1; } +# Authentication only in the environment of git subprocesses, never in remote URLs or .git/config. +AUTH_HEADER="Authorization: Basic $(printf 'oauth2:%s' "$GITEA_JOB_TOKEN" | base64 | tr -d '\n')" +export GIT_CONFIG_COUNT=1 +export GIT_CONFIG_KEY_0=http.extraheader +export GIT_CONFIG_VALUE_0="$AUTH_HEADER" +export GIT_TERMINAL_PROMPT=0 +REPO="$CACHE_ROOT/repo" +if [[ ! -d "$REPO/.git" ]]; then + git clone --depth 1 --branch main "$REMOTE" "$REPO" +else + git -C "$REPO" remote set-url origin "$REMOTE" + git -C "$REPO" fetch --depth 1 origin main + git -C "$REPO" reset --hard FETCH_HEAD +fi +cd "$REPO" +python3 scripts/sync.py --repo "$REPO" --http-cache "$CACHE_ROOT/http-cache.json" --min-release 5 +python3 scripts/validate.py +git config user.name 'Vulnerability Bot' +git config user.email 'vuln-bot@users.noreply.local' +git add -A data/secdb state/manifest.json +if git diff --cached --quiet; then + echo 'No Alpine changes; nothing to push' + exit 0 +fi +git commit -m "Update Alpine secdb $(date -u +%Y-%m-%dT%H:%M:%SZ)" +# If another actor changed main during this run, do not force-push. +git push origin HEAD:main diff --git a/scripts/sync.py b/scripts/sync.py new file mode 100644 index 0000000..3303d3b --- /dev/null +++ b/scripts/sync.py @@ -0,0 +1,140 @@ +#!/usr/bin/env python3 +"""Mirror upstream Alpine secdb JSON files, preserving the upstream bytes.""" +import argparse +import hashlib +from html.parser import HTMLParser +import json +import os +from pathlib import Path +import re +import tempfile +import time +from urllib.error import HTTPError, URLError +from urllib.parse import urljoin, urlsplit +from urllib.request import Request, urlopen + +BASE='https://secdb.alpinelinux.org/' +RELEASE=re.compile(r'^v3\.(\d+)/$') +BRANCH_FILES={'main.json','community.json'} + +class Links(HTMLParser): + def __init__(self): + super().__init__(); self.hrefs=[] + def handle_starttag(self,tag,attrs): + if tag.lower()=='a': + for k,v in attrs: + if k.lower()=='href' and v is not None: self.hrefs.append(v) + +def fetch(url, cache=None, *, attempts=5): + """Returns (bytes | None for HTTP 304, headers).""" + if not url.startswith(BASE): raise ValueError('Untrusted origin: '+url) + headers={'User-Agent':'vuln-alpine-gitea-mirror/1.0','Accept':'application/json, text/html;q=0.8, */*;q=0.5'} + if cache: + if cache.get('etag'): headers['If-None-Match']=cache['etag'] + if cache.get('last_modified'): headers['If-Modified-Since']=cache['last_modified'] + for i in range(attempts): + try: + with urlopen(Request(url,headers=headers),timeout=75) as r: + data=r.read(15*1024*1024+1) + if len(data)>15*1024*1024: raise ValueError('Source file too large: '+url) + return data, {'etag':r.headers.get('ETag',''), 'last_modified':r.headers.get('Last-Modified','')} + except HTTPError as e: + if e.code==304: return None,cache or {} + if e.code not in (408,429,500,502,503,504) or i==attempts-1: raise + retry=e.headers.get('Retry-After','') + delay=min(int(retry),120) if retry.isdecimal() else min(2**i*3,60) + except (URLError, TimeoutError, ConnectionError): + if i==attempts-1: raise + delay=min(2**i*3,60) + time.sleep(delay) + raise RuntimeError('Unreachable') + +def links(blob): + parser=Links(); parser.feed(blob.decode('utf-8')); return parser.hrefs + +def release_links(root_html,min_release): + found=[] + for href in links(root_html): + m=RELEASE.fullmatch(href) + if m and int(m.group(1))>=min_release: found.append(href[:-1]) + elif href=='edge/': found.append('edge') + return sorted(set(found),key=lambda x:(x=='edge',int(x.split('.')[1]) if x.startswith('v3.') else 999)) + +def available_files(branch_html): + return sorted(set(h for h in links(branch_html) if h in BRANCH_FILES)) + +def validate_json(blob): + obj=json.loads(blob) + if not isinstance(obj,dict) or not isinstance(obj.get('packages'),list): + raise ValueError('Unexpected Alpine secdb JSON schema: expected packages array') + return len(obj['packages']) + +def atomic_write(path:Path,data:bytes): + path.parent.mkdir(parents=True,exist_ok=True) + fd,tmp=tempfile.mkstemp(prefix='.incoming-',dir=str(path.parent)) + try: + with os.fdopen(fd,'wb') as out: + out.write(data);out.flush();os.fsync(out.fileno()) + os.replace(tmp,path) + finally: + if os.path.exists(tmp): os.unlink(tmp) + +def run(args): + repo=Path(args.repo).resolve() + data_root=repo/'data'/'secdb' + state_dir=repo/'state';state_dir.mkdir(parents=True,exist_ok=True) + cachefile=Path(args.http_cache).resolve();cachefile.parent.mkdir(parents=True,exist_ok=True) + oldcache=json.loads(cachefile.read_text()) if cachefile.exists() else {} + newcache=dict(oldcache) + root_html,_=fetch(BASE) + branches=release_links(root_html,args.min_release) + if not branches: raise RuntimeError('No Alpine release branches discovered') + oldmanifest=state_dir/'manifest.json' + old=json.loads(oldmanifest.read_text()) if oldmanifest.exists() else {} + oldfiles=old.get('files',{}) + records={}; updates=0; skipped=0 + for branch in branches: + branch_url=urljoin(BASE,branch+'/') + branch_html,_=fetch(branch_url) + fnames=available_files(branch_html) + if not fnames: + raise RuntimeError('No secdb JSON files for '+branch) + for filename in fnames: + rel=f'{branch}/{filename}' + url=urljoin(branch_url,filename) + target=data_root/rel + cache=oldcache.get(rel) if target.is_file() else None + blob,meta=fetch(url,cache) + if blob is None: + if not target.is_file(): raise RuntimeError('304 without local file: '+rel) + digest=hashlib.sha256(target.read_bytes()).hexdigest() + count=oldfiles.get(rel,{}).get('packages') + if count is None: count=validate_json(target.read_bytes()) + skipped+=1 + else: + count=validate_json(blob) + digest=hashlib.sha256(blob).hexdigest() + if not target.is_file() or hashlib.sha256(target.read_bytes()).hexdigest()!=digest: + atomic_write(target,blob);updates+=1 + else: skipped+=1 + records[rel]={'sha256':digest,'packages':count,'source':url} + newcache[rel]=meta + print(f'{rel}: {"downloaded/changed" if blob is not None else "not modified"}; packages={count}',flush=True) + # Historical branches are retained, even if Alpine stops listing them. + # This prevents a temporary index regression from silently deleting data. + for rel,entry in oldfiles.items(): + if rel not in records and (data_root/rel).is_file(): records[rel]=entry + newmanifest={'source':BASE,'min_release':args.min_release,'files':dict(sorted(records.items()))} + encoded=(json.dumps(newmanifest,indent=2,sort_keys=True,ensure_ascii=False)+'\n').encode() + if not oldmanifest.exists() or oldmanifest.read_bytes()!=encoded: atomic_write(oldmanifest,encoded) + cached=json.dumps(newcache,indent=2,sort_keys=True).encode() + atomic_write(cachefile,cached) + print(f'OK: {len(records)} files, {updates} changed, {skipped} unchanged',flush=True) + +def main(): + ap=argparse.ArgumentParser() + ap.add_argument('--repo',default='.') + ap.add_argument('--http-cache',required=True,help='Persistent path outside Git') + ap.add_argument('--min-release',type=int,default=5,help='v3.5 onwards cover 2017 historical data') + args=ap.parse_args();run(args) +if __name__=='__main__': main() diff --git a/scripts/validate.py b/scripts/validate.py new file mode 100644 index 0000000..36f5a23 --- /dev/null +++ b/scripts/validate.py @@ -0,0 +1,22 @@ +#!/usr/bin/env python3 +"""Validate the complete Alpine mirror against the tracked manifest.""" +import hashlib,json,sys +from pathlib import Path + +def main(): + root=Path(__file__).resolve().parents[1] + manifest=json.loads((root/'state/manifest.json').read_text()) + files=manifest.get('files',{}) + if not files: raise ValueError('No manifest entries') + for rel,entry in files.items(): + p=(root/'data/secdb'/rel).resolve() + if not p.is_relative_to((root/'data/secdb').resolve()): raise ValueError('Unsafe path') + b=p.read_bytes() + if hashlib.sha256(b).hexdigest()!=entry['sha256']: raise ValueError('Hash mismatch: '+rel) + obj=json.loads(b) + if not isinstance(obj,dict) or not isinstance(obj.get('packages'),list): raise ValueError('Invalid secdb data: '+rel) + if len(obj['packages'])!=entry['packages']: raise ValueError('Package count mismatch: '+rel) + print(f'Validated {len(files)} Alpine security database files') +if __name__=='__main__': + try: main() + except Exception as e: print(f'Validation failed: {e}',file=sys.stderr);sys.exit(1) diff --git a/state/.gitkeep b/state/.gitkeep new file mode 100644 index 0000000..e69de29 diff --git a/tests/test_sync.py b/tests/test_sync.py new file mode 100644 index 0000000..0bf0aa1 --- /dev/null +++ b/tests/test_sync.py @@ -0,0 +1,16 @@ +import json,unittest,sys +from pathlib import Path +sys.path.insert(0,str(Path(__file__).resolve().parents[1]/'scripts')) +import sync + +class MirrorTests(unittest.TestCase): + def test_release_discovery(self): + html=b'oldv3.5newedge' + self.assertEqual(sync.release_links(html,5),['v3.5','v3.24','edge']) + def test_file_names(self): + h=b'maincommunitymain yaml' + self.assertEqual(sync.available_files(h),['community.json','main.json']) + def test_validate(self): + self.assertEqual(sync.validate_json(json.dumps({'packages':[{'pkg':{}}]}).encode()),1) + with self.assertRaises(ValueError): sync.validate_json(b'{}') +if __name__=='__main__':unittest.main()