From 38edfb5aac89e65ce440faf637da612e292ed485 Mon Sep 17 00:00:00 2001 From: root Date: Wed, 11 Nov 2026 21:04:55 +0100 Subject: [PATCH] Initial AlmaLinux security mirror --- .gitea/workflows/update.yml | 57 +++++++++++++++++++++++++++++++++++++ .gitignore | 3 ++ README.md | 15 ++++++++++ data/.gitkeep | 0 scripts/common.py | 29 +++++++++++++++++++ scripts/run-cached.sh | 40 ++++++++++++++++++++++++++ scripts/sync.py | 40 ++++++++++++++++++++++++++ scripts/validate.py | 10 +++++++ state/.gitkeep | 0 tests/test_sync.py | 11 +++++++ 10 files changed, 205 insertions(+) create mode 100644 .gitea/workflows/update.yml create mode 100644 .gitignore create mode 100644 README.md create mode 100644 data/.gitkeep create mode 100644 scripts/common.py create mode 100644 scripts/run-cached.sh create mode 100644 scripts/sync.py create mode 100644 scripts/validate.py create mode 100644 state/.gitkeep create mode 100644 tests/test_sync.py diff --git a/.gitea/workflows/update.yml b/.gitea/workflows/update.yml new file mode 100644 index 0000000..94402d8 --- /dev/null +++ b/.gitea/workflows/update.yml @@ -0,0 +1,57 @@ +name: AlmaLinux mirror + +on: + workflow_dispatch: + schedule: + - cron: "37 */6 * * *" + +permissions: + contents: write + +concurrency: + group: vuln-almalinux-mirror + cancel-in-progress: false + +jobs: + mirror: + runs-on: vulnerability + timeout-minutes: 180 + env: + CACHE_ROOT: /vuln-cache/vuln-almalinux + REPO_URL: https://git.apex-project.de/VULN/vuln-almalinux.git + REPO_NAME: vuln-almalinux + GITEA_JOB_TOKEN: ${{ secrets.GITEA_TOKEN }} + steps: + - name: Prepare job dependencies + shell: sh + run: | + set -eu + if command -v apk >/dev/null 2>&1; then + apk add --no-cache bash git python3 util-linux ca-certificates coreutils + elif command -v apt-get >/dev/null 2>&1; then + apt-get update + apt-get install -y --no-install-recommends bash git python3 util-linux ca-certificates coreutils + else + echo 'No supported package manager' >&2 + exit 1 + fi + /usr/bin/flock --version + - name: Mirror using persistent checkout + shell: bash + run: | + set -Eeuo pipefail + mkdir -p "$CACHE_ROOT" + : "${GITEA_JOB_TOKEN:?GITEA_TOKEN fehlt}" + export GIT_CONFIG_COUNT=1 + export GIT_CONFIG_KEY_0=http.extraHeader + export GIT_CONFIG_VALUE_0="Authorization: Basic $(printf 'oauth2:%s' "$GITEA_JOB_TOKEN" | base64 | tr -d '\n')" + if [ ! -d "$CACHE_ROOT/repo/.git" ]; then + git clone --depth 1 --branch main "$REPO_URL" "$CACHE_ROOT/repo" + fi + # run-cached.sh owns the only flock lock + bash "$CACHE_ROOT/repo/scripts/run-cached.sh" + - name: Cache usage + if: always() + shell: sh + run: | + test ! -d "$CACHE_ROOT" || du -sh "$CACHE_ROOT" diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..c6a221a --- /dev/null +++ b/.gitignore @@ -0,0 +1,3 @@ +__pycache__/ +*.pyc +*.tmp diff --git a/README.md b/README.md new file mode 100644 index 0000000..6528573 --- /dev/null +++ b/README.md @@ -0,0 +1,15 @@ +# vuln-almalinux + +Official security advisory raw-data mirror for AlmaLinux. + +## Gitea Runner +`runs-on: vulnerability`; shared persistent volume `/vuln-cache/vuln-almalinux`. Single lock in `scripts/run-cached.sh`. + +## Workflow +Runs every 6 hours; see `.gitea/workflows/update.yml`. + +## Source +AlmaLinux official errata.full.json for releases 8, 9, 10. HTTP conditional requests, no deletion on source errors. + +## Limitations +Production full sync has not been tested. Raw data must be normalized for accurate package matching. diff --git a/data/.gitkeep b/data/.gitkeep new file mode 100644 index 0000000..e69de29 diff --git a/scripts/common.py b/scripts/common.py new file mode 100644 index 0000000..6227dfa --- /dev/null +++ b/scripts/common.py @@ -0,0 +1,29 @@ +import json, os, time, hashlib +from pathlib import Path +from urllib.request import Request, urlopen +from urllib.error import HTTPError + +def request(url, headers=None, timeout=90, retries=4): + h={"User-Agent":"vulnerability-apex-project-mirror/1.0", "Accept":"application/json"} + h.update(headers or {}) + for attempt in range(retries): + try: + with urlopen(Request(url,headers=h),timeout=timeout) as response: + return response.status, response.read(), dict(response.headers) + except HTTPError as err: + if err.code == 304: return 304,b'',dict(err.headers) + if err.code not in (429,500,502,503,504) or attempt==retries-1: raise + except (TimeoutError, OSError): + if attempt==retries-1: raise + time.sleep(min(30, 2**attempt*2)) + raise RuntimeError('unreachable') + +def store(path, raw): + path=Path(path); path.parent.mkdir(parents=True,exist_ok=True) + if path.exists() and hashlib.sha256(path.read_bytes()).digest()==hashlib.sha256(raw).digest(): return False + temp=path.with_name(path.name+'.tmp') + temp.write_bytes(raw); temp.replace(path) + return True + +def state_file(path, value): + store(path,(json.dumps(value,indent=2,ensure_ascii=False,sort_keys=True)+'\n').encode()) diff --git a/scripts/run-cached.sh b/scripts/run-cached.sh new file mode 100644 index 0000000..9d010fe --- /dev/null +++ b/scripts/run-cached.sh @@ -0,0 +1,40 @@ +#!/usr/bin/env bash +set -Eeuo pipefail +umask 077 +: "${CACHE_ROOT:?CACHE_ROOT fehlt}" +: "${REPO_URL:?REPO_URL fehlt}" +: "${GITEA_JOB_TOKEN:?GITEA_JOB_TOKEN fehlt}" +mkdir -p "$CACHE_ROOT" +exec 9>"$CACHE_ROOT/.sync.lock" +if command -v /usr/bin/flock >/dev/null 2>&1; then LOCK=/usr/bin/flock; else LOCK=$(command -v flock); fi +"$LOCK" -n 9 || { echo 'Ein anderer Sync verwendet diesen Repository-Cache.' >&2; exit 1; } +AUTH="Authorization: Basic $(printf 'oauth2:%s' "$GITEA_JOB_TOKEN" | base64 | tr -d '\n')" +export GIT_CONFIG_COUNT=1 GIT_CONFIG_KEY_0=http.extraHeader GIT_CONFIG_VALUE_0="$AUTH" +if [ ! -d "$CACHE_ROOT/repo/.git" ]; then + if [ -e "$CACHE_ROOT/repo" ]; then echo 'Repo-Verzeichnis existiert ohne .git' >&2; exit 1; fi + git clone --depth 1 --branch main "$REPO_URL" "$CACHE_ROOT/repo" +fi +cd "$CACHE_ROOT/repo" +git remote set-url origin "$REPO_URL" +git fetch origin main +# Dirty data may be left behind by interrupted imports. Never silently reset it. +if [ "$(git rev-parse HEAD)" != "$(git rev-parse origin/main)" ]; then + if [ -n "$(git status --porcelain)" ]; then + echo 'Remote wurde geändert und der Cache enthält ungespeicherte Dateien; kein Reset.' >&2 + exit 1 + fi + git merge --ff-only origin/main || { echo 'Lokale Commits weichen vom Remote ab; manuelle Prüfung erforderlich.' >&2; exit 1; } +fi +python3 scripts/sync.py +python3 scripts/validate.py +git config user.name 'Vulnerability Bot' +git config user.email 'vulnerability-bot@users.noreply.local' +git add -A data state +if git diff --cached --quiet; then + echo 'Keine Datenänderungen' +else + git -c gc.auto=0 commit -q -m "Update ${REPO_NAME:-vulnerability} $(date -u +%FT%TZ)" +fi +if [ "$(git rev-parse HEAD)" != "$(git rev-parse origin/main)" ]; then + git -c gc.auto=0 push origin HEAD:main +fi diff --git a/scripts/sync.py b/scripts/sync.py new file mode 100644 index 0000000..0b2c7ee --- /dev/null +++ b/scripts/sync.py @@ -0,0 +1,40 @@ +#!/usr/bin/env python3 +import json, os +from datetime import datetime, timezone +from pathlib import Path +from common import request,store,state_file + +ROOT=Path(__file__).resolve().parents[1] +URL='https://errata.almalinux.org/{major}/errata.full.json' + +def check(payload): + # Source can be a JSON object or an array; reject HTML/errors. + value=json.loads(payload) + if not isinstance(value,(dict,list)) or not value: + raise ValueError('Unexpected empty AlmaLinux errata response') + return value + +def main(): + for major in (8,9,10): + dest=ROOT/f'data/almalinux/{major}/errata.full.json' + state=ROOT/f'state/{major}.json' + meta=json.loads(state.read_text()) if state.exists() else {} + headers={} + if dest.exists(): + if meta.get('etag'): headers['If-None-Match']=meta['etag'] + if meta.get('last_modified'): headers['If-Modified-Since']=meta['last_modified'] + status,raw,h=request(URL.format(major=major),headers) + if status==304: + print(f'AlmaLinux {major}: unchanged (HTTP 304)'); continue + check(raw) + changed=store(dest,raw) + state_file(state,{ + 'source':URL.format(major=major), + 'etag':h.get('ETag',meta.get('etag','')), + 'last_modified':h.get('Last-Modified',meta.get('last_modified','')), + 'last_success_utc':datetime.now(timezone.utc).isoformat(), + 'sha256':__import__('hashlib').sha256(raw).hexdigest(), + }) + print(f'AlmaLinux {major}: {len(raw)} bytes, changed={changed}') + +if __name__=='__main__': main() diff --git a/scripts/validate.py b/scripts/validate.py new file mode 100644 index 0000000..c6d3629 --- /dev/null +++ b/scripts/validate.py @@ -0,0 +1,10 @@ +#!/usr/bin/env python3 +import json +from pathlib import Path +root=Path(__file__).resolve().parents[1] +found=0 +for f in sorted((root/'data/almalinux').glob('*/errata.full.json')): + x=json.loads(f.read_bytes()); assert isinstance(x,(list,dict)) and x, f + found+=1 +if found==0: raise SystemExit('No AlmaLinux files') +print(f'Validated {found} AlmaLinux raw JSON files') diff --git a/state/.gitkeep b/state/.gitkeep new file mode 100644 index 0000000..e69de29 diff --git a/tests/test_sync.py b/tests/test_sync.py new file mode 100644 index 0000000..eb838da --- /dev/null +++ b/tests/test_sync.py @@ -0,0 +1,11 @@ +import sys, unittest +from pathlib import Path +sys.path.insert(0,str(Path(__file__).resolve().parents[1]/'scripts')) +from sync import check +class TestData(unittest.TestCase): + def test_array(self): self.assertIsInstance(check(b'[{"id":"ALSA-2025:1"}]'),list) + def test_error_html(self): + with self.assertRaises(Exception): check(b'Bad Gateway') + def test_empty(self): + with self.assertRaises(ValueError): check(b'[]') +if __name__=='__main__':unittest.main()