Initial AlmaLinux security mirror

scripts/common.py

@@ -0,0 +1,29 @@
+import json, os, time, hashlib
+from pathlib import Path
+from urllib.request import Request, urlopen
+from urllib.error import HTTPError
+
+def request(url, headers=None, timeout=90, retries=4):
+    h={"User-Agent":"vulnerability-apex-project-mirror/1.0", "Accept":"application/json"}
+    h.update(headers or {})
+    for attempt in range(retries):
+        try:
+            with urlopen(Request(url,headers=h),timeout=timeout) as response:
+                return response.status, response.read(), dict(response.headers)
+        except HTTPError as err:
+            if err.code == 304: return 304,b'',dict(err.headers)
+            if err.code not in (429,500,502,503,504) or attempt==retries-1: raise
+        except (TimeoutError, OSError):
+            if attempt==retries-1: raise
+        time.sleep(min(30, 2**attempt*2))
+    raise RuntimeError('unreachable')
+
+def store(path, raw):
+    path=Path(path); path.parent.mkdir(parents=True,exist_ok=True)
+    if path.exists() and hashlib.sha256(path.read_bytes()).digest()==hashlib.sha256(raw).digest(): return False
+    temp=path.with_name(path.name+'.tmp')
+    temp.write_bytes(raw); temp.replace(path)
+    return True
+
+def state_file(path, value):
+    store(path,(json.dumps(value,indent=2,ensure_ascii=False,sort_keys=True)+'\n').encode())

scripts/run-cached.sh

@@ -0,0 +1,40 @@
+#!/usr/bin/env bash
+set -Eeuo pipefail
+umask 077
+: "${CACHE_ROOT:?CACHE_ROOT fehlt}"
+: "${REPO_URL:?REPO_URL fehlt}"
+: "${GITEA_JOB_TOKEN:?GITEA_JOB_TOKEN fehlt}"
+mkdir -p "$CACHE_ROOT"
+exec 9>"$CACHE_ROOT/.sync.lock"
+if command -v /usr/bin/flock >/dev/null 2>&1; then LOCK=/usr/bin/flock; else LOCK=$(command -v flock); fi
+"$LOCK" -n 9 || { echo 'Ein anderer Sync verwendet diesen Repository-Cache.' >&2; exit 1; }
+AUTH="Authorization: Basic $(printf 'oauth2:%s' "$GITEA_JOB_TOKEN" | base64 | tr -d '\n')"
+export GIT_CONFIG_COUNT=1 GIT_CONFIG_KEY_0=http.extraHeader GIT_CONFIG_VALUE_0="$AUTH"
+if [ ! -d "$CACHE_ROOT/repo/.git" ]; then
+  if [ -e "$CACHE_ROOT/repo" ]; then echo 'Repo-Verzeichnis existiert ohne .git' >&2; exit 1; fi
+  git clone --depth 1 --branch main "$REPO_URL" "$CACHE_ROOT/repo"
+fi
+cd "$CACHE_ROOT/repo"
+git remote set-url origin "$REPO_URL"
+git fetch origin main
+# Dirty data may be left behind by interrupted imports. Never silently reset it.
+if [ "$(git rev-parse HEAD)" != "$(git rev-parse origin/main)" ]; then
+  if [ -n "$(git status --porcelain)" ]; then
+    echo 'Remote wurde geändert und der Cache enthält ungespeicherte Dateien; kein Reset.' >&2
+    exit 1
+  fi
+  git merge --ff-only origin/main || { echo 'Lokale Commits weichen vom Remote ab; manuelle Prüfung erforderlich.' >&2; exit 1; }
+fi
+python3 scripts/sync.py
+python3 scripts/validate.py
+git config user.name 'Vulnerability Bot'
+git config user.email 'vulnerability-bot@users.noreply.local'
+git add -A data state
+if git diff --cached --quiet; then
+  echo 'Keine Datenänderungen'
+else
+  git -c gc.auto=0 commit -q -m "Update ${REPO_NAME:-vulnerability} $(date -u +%FT%TZ)"
+fi
+if [ "$(git rev-parse HEAD)" != "$(git rev-parse origin/main)" ]; then
+  git -c gc.auto=0 push origin HEAD:main
+fi

scripts/sync.py

@@ -0,0 +1,40 @@
+#!/usr/bin/env python3
+import json, os
+from datetime import datetime, timezone
+from pathlib import Path
+from common import request,store,state_file
+
+ROOT=Path(__file__).resolve().parents[1]
+URL='https://errata.almalinux.org/{major}/errata.full.json'
+
+def check(payload):
+    # Source can be a JSON object or an array; reject HTML/errors.
+    value=json.loads(payload)
+    if not isinstance(value,(dict,list)) or not value:
+        raise ValueError('Unexpected empty AlmaLinux errata response')
+    return value
+
+def main():
+    for major in (8,9,10):
+        dest=ROOT/f'data/almalinux/{major}/errata.full.json'
+        state=ROOT/f'state/{major}.json'
+        meta=json.loads(state.read_text()) if state.exists() else {}
+        headers={}
+        if dest.exists():
+            if meta.get('etag'): headers['If-None-Match']=meta['etag']
+            if meta.get('last_modified'): headers['If-Modified-Since']=meta['last_modified']
+        status,raw,h=request(URL.format(major=major),headers)
+        if status==304:
+            print(f'AlmaLinux {major}: unchanged (HTTP 304)'); continue
+        check(raw)
+        changed=store(dest,raw)
+        state_file(state,{
+            'source':URL.format(major=major),
+            'etag':h.get('ETag',meta.get('etag','')),
+            'last_modified':h.get('Last-Modified',meta.get('last_modified','')),
+            'last_success_utc':datetime.now(timezone.utc).isoformat(),
+            'sha256':__import__('hashlib').sha256(raw).hexdigest(),
+        })
+        print(f'AlmaLinux {major}: {len(raw)} bytes, changed={changed}')
+
+if __name__=='__main__': main()

scripts/validate.py

@@ -0,0 +1,10 @@
+#!/usr/bin/env python3
+import json
+from pathlib import Path
+root=Path(__file__).resolve().parents[1]
+found=0
+for f in sorted((root/'data/almalinux').glob('*/errata.full.json')):
+    x=json.loads(f.read_bytes()); assert isinstance(x,(list,dict)) and x, f
+    found+=1
+if found==0: raise SystemExit('No AlmaLinux files')
+print(f'Validated {found} AlmaLinux raw JSON files')