Initial Alpine security mirror

2026-11-10 22:08:53 +01:00
commit 2b662f7b42
9 changed files with 298 additions and 0 deletions
@@ -0,0 +1,36 @@
+#!/usr/bin/env bash
+set -Eeuo pipefail
+umask 077
+: "${GITEA_JOB_TOKEN:?GITEA_JOB_TOKEN fehlt}"
+CACHE_ROOT="${CACHE_ROOT:-/var/lib/act_runner/cache/vuln-alpine}"
+REMOTE="${REPO_URL:-https://git.apex-project.de/VULN/vuln-alpine.git}"
+mkdir -p "$CACHE_ROOT"
+exec 9>"$CACHE_ROOT/.sync.lock"
+flock -w 3600 9 || { echo 'Another Alpine mirror job still running' >&2; exit 1; }
+# Authentication only in the environment of git subprocesses, never in remote URLs or .git/config.
+AUTH_HEADER="Authorization: Basic $(printf 'oauth2:%s' "$GITEA_JOB_TOKEN" | base64 | tr -d '\n')"
+export GIT_CONFIG_COUNT=1
+export GIT_CONFIG_KEY_0=http.extraheader
+export GIT_CONFIG_VALUE_0="$AUTH_HEADER"
+export GIT_TERMINAL_PROMPT=0
+REPO="$CACHE_ROOT/repo"
+if [[ ! -d "$REPO/.git" ]]; then
+  git clone --depth 1 --branch main "$REMOTE" "$REPO"
+else
+  git -C "$REPO" remote set-url origin "$REMOTE"
+  git -C "$REPO" fetch --depth 1 origin main
+  git -C "$REPO" reset --hard FETCH_HEAD
+fi
+cd "$REPO"
+python3 scripts/sync.py --repo "$REPO" --http-cache "$CACHE_ROOT/http-cache.json" --min-release 5
+python3 scripts/validate.py
+git config user.name 'Vulnerability Bot'
+git config user.email 'vuln-bot@users.noreply.local'
+git add -A data/secdb state/manifest.json
+if git diff --cached --quiet; then
+ echo 'No Alpine changes; nothing to push'
+ exit 0
+fi
+git commit -m "Update Alpine secdb $(date -u +%Y-%m-%dT%H:%M:%SZ)"
+# If another actor changed main during this run, do not force-push.
+git push origin HEAD:main
@@ -0,0 +1,140 @@
+#!/usr/bin/env python3
+"""Mirror upstream Alpine secdb JSON files, preserving the upstream bytes."""
+import argparse
+import hashlib
+from html.parser import HTMLParser
+import json
+import os
+from pathlib import Path
+import re
+import tempfile
+import time
+from urllib.error import HTTPError, URLError
+from urllib.parse import urljoin, urlsplit
+from urllib.request import Request, urlopen
+
+BASE='https://secdb.alpinelinux.org/'
+RELEASE=re.compile(r'^v3\.(\d+)/$')
+BRANCH_FILES={'main.json','community.json'}
+
+class Links(HTMLParser):
+    def __init__(self):
+        super().__init__(); self.hrefs=[]
+    def handle_starttag(self,tag,attrs):
+        if tag.lower()=='a':
+            for k,v in attrs:
+                if k.lower()=='href' and v is not None: self.hrefs.append(v)
+
+def fetch(url, cache=None, *, attempts=5):
+    """Returns (bytes | None for HTTP 304, headers)."""
+    if not url.startswith(BASE): raise ValueError('Untrusted origin: '+url)
+    headers={'User-Agent':'vuln-alpine-gitea-mirror/1.0','Accept':'application/json, text/html;q=0.8, */*;q=0.5'}
+    if cache:
+        if cache.get('etag'): headers['If-None-Match']=cache['etag']
+        if cache.get('last_modified'): headers['If-Modified-Since']=cache['last_modified']
+    for i in range(attempts):
+        try:
+            with urlopen(Request(url,headers=headers),timeout=75) as r:
+                data=r.read(15*1024*1024+1)
+                if len(data)>15*1024*1024: raise ValueError('Source file too large: '+url)
+                return data, {'etag':r.headers.get('ETag',''), 'last_modified':r.headers.get('Last-Modified','')}
+        except HTTPError as e:
+            if e.code==304: return None,cache or {}
+            if e.code not in (408,429,500,502,503,504) or i==attempts-1: raise
+            retry=e.headers.get('Retry-After','')
+            delay=min(int(retry),120) if retry.isdecimal() else min(2**i*3,60)
+        except (URLError, TimeoutError, ConnectionError):
+            if i==attempts-1: raise
+            delay=min(2**i*3,60)
+        time.sleep(delay)
+    raise RuntimeError('Unreachable')
+
+def links(blob):
+    parser=Links(); parser.feed(blob.decode('utf-8')); return parser.hrefs
+
+def release_links(root_html,min_release):
+    found=[]
+    for href in links(root_html):
+        m=RELEASE.fullmatch(href)
+        if m and int(m.group(1))>=min_release: found.append(href[:-1])
+        elif href=='edge/': found.append('edge')
+    return sorted(set(found),key=lambda x:(x=='edge',int(x.split('.')[1]) if x.startswith('v3.') else 999))
+
+def available_files(branch_html):
+    return sorted(set(h for h in links(branch_html) if h in BRANCH_FILES))
+
+def validate_json(blob):
+    obj=json.loads(blob)
+    if not isinstance(obj,dict) or not isinstance(obj.get('packages'),list):
+        raise ValueError('Unexpected Alpine secdb JSON schema: expected packages array')
+    return len(obj['packages'])
+
+def atomic_write(path:Path,data:bytes):
+    path.parent.mkdir(parents=True,exist_ok=True)
+    fd,tmp=tempfile.mkstemp(prefix='.incoming-',dir=str(path.parent))
+    try:
+        with os.fdopen(fd,'wb') as out:
+            out.write(data);out.flush();os.fsync(out.fileno())
+        os.replace(tmp,path)
+    finally:
+        if os.path.exists(tmp): os.unlink(tmp)
+
+def run(args):
+    repo=Path(args.repo).resolve()
+    data_root=repo/'data'/'secdb'
+    state_dir=repo/'state';state_dir.mkdir(parents=True,exist_ok=True)
+    cachefile=Path(args.http_cache).resolve();cachefile.parent.mkdir(parents=True,exist_ok=True)
+    oldcache=json.loads(cachefile.read_text()) if cachefile.exists() else {}
+    newcache=dict(oldcache)
+    root_html,_=fetch(BASE)
+    branches=release_links(root_html,args.min_release)
+    if not branches: raise RuntimeError('No Alpine release branches discovered')
+    oldmanifest=state_dir/'manifest.json'
+    old=json.loads(oldmanifest.read_text()) if oldmanifest.exists() else {}
+    oldfiles=old.get('files',{})
+    records={}; updates=0; skipped=0
+    for branch in branches:
+        branch_url=urljoin(BASE,branch+'/')
+        branch_html,_=fetch(branch_url)
+        fnames=available_files(branch_html)
+        if not fnames:
+            raise RuntimeError('No secdb JSON files for '+branch)
+        for filename in fnames:
+            rel=f'{branch}/{filename}'
+            url=urljoin(branch_url,filename)
+            target=data_root/rel
+            cache=oldcache.get(rel) if target.is_file() else None
+            blob,meta=fetch(url,cache)
+            if blob is None:
+                if not target.is_file(): raise RuntimeError('304 without local file: '+rel)
+                digest=hashlib.sha256(target.read_bytes()).hexdigest()
+                count=oldfiles.get(rel,{}).get('packages')
+                if count is None: count=validate_json(target.read_bytes())
+                skipped+=1
+            else:
+                count=validate_json(blob)
+                digest=hashlib.sha256(blob).hexdigest()
+                if not target.is_file() or hashlib.sha256(target.read_bytes()).hexdigest()!=digest:
+                    atomic_write(target,blob);updates+=1
+                else: skipped+=1
+            records[rel]={'sha256':digest,'packages':count,'source':url}
+            newcache[rel]=meta
+            print(f'{rel}: {"downloaded/changed" if blob is not None else "not modified"}; packages={count}',flush=True)
+    # Historical branches are retained, even if Alpine stops listing them.
+    # This prevents a temporary index regression from silently deleting data.
+    for rel,entry in oldfiles.items():
+        if rel not in records and (data_root/rel).is_file(): records[rel]=entry
+    newmanifest={'source':BASE,'min_release':args.min_release,'files':dict(sorted(records.items()))}
+    encoded=(json.dumps(newmanifest,indent=2,sort_keys=True,ensure_ascii=False)+'\n').encode()
+    if not oldmanifest.exists() or oldmanifest.read_bytes()!=encoded: atomic_write(oldmanifest,encoded)
+    cached=json.dumps(newcache,indent=2,sort_keys=True).encode()
+    atomic_write(cachefile,cached)
+    print(f'OK: {len(records)} files, {updates} changed, {skipped} unchanged',flush=True)
+
+def main():
+    ap=argparse.ArgumentParser()
+    ap.add_argument('--repo',default='.')
+    ap.add_argument('--http-cache',required=True,help='Persistent path outside Git')
+    ap.add_argument('--min-release',type=int,default=5,help='v3.5 onwards cover 2017 historical data')
+    args=ap.parse_args();run(args)
+if __name__=='__main__': main()
@@ -0,0 +1,22 @@
+#!/usr/bin/env python3
+"""Validate the complete Alpine mirror against the tracked manifest."""
+import hashlib,json,sys
+from pathlib import Path
+
+def main():
+ root=Path(__file__).resolve().parents[1]
+ manifest=json.loads((root/'state/manifest.json').read_text())
+ files=manifest.get('files',{})
+ if not files: raise ValueError('No manifest entries')
+ for rel,entry in files.items():
+  p=(root/'data/secdb'/rel).resolve()
+  if not p.is_relative_to((root/'data/secdb').resolve()): raise ValueError('Unsafe path')
+  b=p.read_bytes()
+  if hashlib.sha256(b).hexdigest()!=entry['sha256']: raise ValueError('Hash mismatch: '+rel)
+  obj=json.loads(b)
+  if not isinstance(obj,dict) or not isinstance(obj.get('packages'),list): raise ValueError('Invalid secdb data: '+rel)
+  if len(obj['packages'])!=entry['packages']: raise ValueError('Package count mismatch: '+rel)
+ print(f'Validated {len(files)} Alpine security database files')
+if __name__=='__main__':
+ try: main()
+ except Exception as e: print(f'Validation failed: {e}',file=sys.stderr);sys.exit(1)