Initial Rocky Linux security mirror

.gitea/workflows/update.yml

@@ -0,0 +1,57 @@
+name: Rocky Linux mirror
+
+on:
+  workflow_dispatch:
+  schedule:
+    - cron: "47 */6 * * *"
+
+permissions:
+  contents: write
+
+concurrency:
+  group: vuln-rocky-mirror
+  cancel-in-progress: false
+
+jobs:
+  mirror:
+    runs-on: vulnerability
+    timeout-minutes: 180
+    env:
+      CACHE_ROOT: /vuln-cache/vuln-rocky
+      REPO_URL: https://git.apex-project.de/VULN/vuln-rocky.git
+      REPO_NAME: vuln-rocky
+      GITEA_JOB_TOKEN: ${{ secrets.GITEA_TOKEN }}
+    steps:
+      - name: Prepare job dependencies
+        shell: sh
+        run: |
+          set -eu
+          if command -v apk >/dev/null 2>&1; then
+            apk add --no-cache bash git python3 util-linux ca-certificates coreutils
+          elif command -v apt-get >/dev/null 2>&1; then
+            apt-get update
+            apt-get install -y --no-install-recommends bash git python3 util-linux ca-certificates coreutils
+          else
+            echo 'No supported package manager' >&2
+            exit 1
+          fi
+          /usr/bin/flock --version
+      - name: Mirror using persistent checkout
+        shell: bash
+        run: |
+          set -Eeuo pipefail
+          mkdir -p "$CACHE_ROOT"
+          : "${GITEA_JOB_TOKEN:?GITEA_TOKEN fehlt}"
+          export GIT_CONFIG_COUNT=1
+          export GIT_CONFIG_KEY_0=http.extraHeader
+          export GIT_CONFIG_VALUE_0="Authorization: Basic $(printf 'oauth2:%s' "$GITEA_JOB_TOKEN" | base64 | tr -d '\n')"
+          if [ ! -d "$CACHE_ROOT/repo/.git" ]; then
+            git clone --depth 1 --branch main "$REPO_URL" "$CACHE_ROOT/repo"
+          fi
+          # run-cached.sh owns the only flock lock
+          bash "$CACHE_ROOT/repo/scripts/run-cached.sh"
+      - name: Cache usage
+        if: always()
+        shell: sh
+        run: |
+          test ! -d "$CACHE_ROOT" || du -sh "$CACHE_ROOT"

.gitignore

@@ -0,0 +1,3 @@
+__pycache__/
+*.pyc
+*.tmp

README.md

@@ -0,0 +1,15 @@
+# vuln-rocky
+
+Official security advisory raw-data mirror for Rocky Linux.
+
+## Gitea Runner
+`runs-on: vulnerability`; shared persistent volume `/vuln-cache/vuln-rocky`. Single lock in `scripts/run-cached.sh`.
+
+## Workflow
+Runs every 6 hours; see `.gitea/workflows/update.yml`.
+
+## Source
+Rocky Linux Apollo v2 API. Paginated sweep with detail fetch (RPM information); process 20 pages of 100 advisories per run by default, checkpoint state/sync.json. Does not infer deletions from missing entries.
+
+## Limitations
+Production full sync has not been tested. Raw data must be normalized for accurate package matching.

scripts/common.py

@@ -0,0 +1,29 @@
+import json, os, time, hashlib
+from pathlib import Path
+from urllib.request import Request, urlopen
+from urllib.error import HTTPError
+
+def request(url, headers=None, timeout=90, retries=4):
+    h={"User-Agent":"vulnerability-apex-project-mirror/1.0", "Accept":"application/json"}
+    h.update(headers or {})
+    for attempt in range(retries):
+        try:
+            with urlopen(Request(url,headers=h),timeout=timeout) as response:
+                return response.status, response.read(), dict(response.headers)
+        except HTTPError as err:
+            if err.code == 304: return 304,b'',dict(err.headers)
+            if err.code not in (429,500,502,503,504) or attempt==retries-1: raise
+        except (TimeoutError, OSError):
+            if attempt==retries-1: raise
+        time.sleep(min(30, 2**attempt*2))
+    raise RuntimeError('unreachable')
+
+def store(path, raw):
+    path=Path(path); path.parent.mkdir(parents=True,exist_ok=True)
+    if path.exists() and hashlib.sha256(path.read_bytes()).digest()==hashlib.sha256(raw).digest(): return False
+    temp=path.with_name(path.name+'.tmp')
+    temp.write_bytes(raw); temp.replace(path)
+    return True
+
+def state_file(path, value):
+    store(path,(json.dumps(value,indent=2,ensure_ascii=False,sort_keys=True)+'\n').encode())

scripts/run-cached.sh

@@ -0,0 +1,40 @@
+#!/usr/bin/env bash
+set -Eeuo pipefail
+umask 077
+: "${CACHE_ROOT:?CACHE_ROOT fehlt}"
+: "${REPO_URL:?REPO_URL fehlt}"
+: "${GITEA_JOB_TOKEN:?GITEA_JOB_TOKEN fehlt}"
+mkdir -p "$CACHE_ROOT"
+exec 9>"$CACHE_ROOT/.sync.lock"
+if command -v /usr/bin/flock >/dev/null 2>&1; then LOCK=/usr/bin/flock; else LOCK=$(command -v flock); fi
+"$LOCK" -n 9 || { echo 'Ein anderer Sync verwendet diesen Repository-Cache.' >&2; exit 1; }
+AUTH="Authorization: Basic $(printf 'oauth2:%s' "$GITEA_JOB_TOKEN" | base64 | tr -d '\n')"
+export GIT_CONFIG_COUNT=1 GIT_CONFIG_KEY_0=http.extraHeader GIT_CONFIG_VALUE_0="$AUTH"
+if [ ! -d "$CACHE_ROOT/repo/.git" ]; then
+  if [ -e "$CACHE_ROOT/repo" ]; then echo 'Repo-Verzeichnis existiert ohne .git' >&2; exit 1; fi
+  git clone --depth 1 --branch main "$REPO_URL" "$CACHE_ROOT/repo"
+fi
+cd "$CACHE_ROOT/repo"
+git remote set-url origin "$REPO_URL"
+git fetch origin main
+# Dirty data may be left behind by interrupted imports. Never silently reset it.
+if [ "$(git rev-parse HEAD)" != "$(git rev-parse origin/main)" ]; then
+  if [ -n "$(git status --porcelain)" ]; then
+    echo 'Remote wurde geändert und der Cache enthält ungespeicherte Dateien; kein Reset.' >&2
+    exit 1
+  fi
+  git merge --ff-only origin/main || { echo 'Lokale Commits weichen vom Remote ab; manuelle Prüfung erforderlich.' >&2; exit 1; }
+fi
+python3 scripts/sync.py
+python3 scripts/validate.py
+git config user.name 'Vulnerability Bot'
+git config user.email 'vulnerability-bot@users.noreply.local'
+git add -A data state
+if git diff --cached --quiet; then
+  echo 'Keine Datenänderungen'
+else
+  git -c gc.auto=0 commit -q -m "Update ${REPO_NAME:-vulnerability} $(date -u +%FT%TZ)"
+fi
+if [ "$(git rev-parse HEAD)" != "$(git rev-parse origin/main)" ]; then
+  git -c gc.auto=0 push origin HEAD:main
+fi

scripts/sync.py

@@ -0,0 +1,67 @@
+#!/usr/bin/env python3
+import json, os, re, time
+from datetime import datetime,timezone
+from pathlib import Path
+from urllib.parse import urlencode, quote
+from common import request,store,state_file
+
+ROOT=Path(__file__).resolve().parents[1]
+API=os.getenv('ROCKY_API','https://apollo.build.resf.org/v2/advisories').rstrip('/')
+PAGE_SIZE=int(os.getenv('ROCKY_PAGE_SIZE','100'))
+MAX_PAGES=int(os.getenv('ROCKY_MAX_PAGES','20'))
+DELAY=float(os.getenv('ROCKY_DELAY','0.15'))
+STATE=ROOT/'state/sync.json'
+
+def get_advisories(body):
+    obj=json.loads(body)
+    if not isinstance(obj,dict) or not isinstance(obj.get('advisories'),list):
+        raise ValueError('Unexpected Apollo list response')
+    total=int(obj['total']) if 'total' in obj else None
+    if total is None: raise ValueError('Apollo response lacks total count')
+    return obj['advisories'],total
+
+def identity(doc):
+    name=doc.get('name') or doc.get('id')
+    if not isinstance(name,str) or not re.fullmatch(r'[A-Za-z0-9:._-]+',name):
+        raise ValueError(f'Invalid advisory identity {name!r}')
+    return name
+
+def date_filter(doc):
+    return identity(doc).startswith(('RLSA','RXSA'))
+
+def main():
+    old=json.loads(STATE.read_text()) if STATE.exists() else {}
+    page=max(1,int(old.get('next_page',1)))
+    pages=0
+    while pages<MAX_PAGES:
+        q=urlencode({'page':page,'limit':PAGE_SIZE,'filters.type':'TYPE_SECURITY','filters.includeRpms':'false'})
+        _,body,_=request(API+'?'+q)
+        entries,total=get_advisories(body)
+        if not entries:
+            if page==1: raise RuntimeError('Apollo returned zero advisories')
+            page=1
+            break
+        count=0
+        for summary in entries:
+            if not date_filter(summary): continue
+            name=identity(summary)
+            # Individual API responses retain affected RPM mappings. Do not use only list summaries.
+            url=API+'/'+quote(name,safe='')
+            _,detail,_=request(url)
+            parsed=json.loads(detail)
+            advisory=parsed.get('advisory') if isinstance(parsed,dict) else None
+            if not isinstance(advisory,dict) or identity(advisory)!=name:
+                raise ValueError(f'Invalid detail for {name}')
+            year=re.search(r'(20\d\d)',name)
+            if not year: raise ValueError(f'No year for {name}')
+            dest=ROOT/'data/advisories'/year.group(1)/(name.replace(':','-')+'.json')
+            store(dest,detail)
+            count+=1
+            time.sleep(DELAY)
+        print(f'Rocky page {page}: {len(entries)} indexed, {count} details, total={total}',flush=True)
+        page+=1; pages+=1
+        if page>(total+PAGE_SIZE-1)//PAGE_SIZE: page=1
+        state_file(STATE,{'next_page':page,'last_success_utc':datetime.now(timezone.utc).isoformat(),'last_total':total})
+        if page==1: break
+
+if __name__=='__main__': main()

scripts/validate.py

@@ -0,0 +1,10 @@
+#!/usr/bin/env python3
+import json
+from pathlib import Path
+root=Path(__file__).resolve().parents[1]
+files=list((root/'data/advisories').glob('*/*.json'))
+if not files: raise SystemExit('No Rocky Linux advisories downloaded')
+for path in files:
+    obj=json.loads(path.read_bytes())
+    if not isinstance(obj,dict) or not isinstance(obj.get('advisory'),dict): raise ValueError(str(path))
+print(f'Validated {len(files)} Rocky advisory detail documents')

tests/test_sync.py

@@ -0,0 +1,14 @@
+import sys, unittest
+from pathlib import Path
+sys.path.insert(0,str(Path(__file__).resolve().parents[1]/'scripts'))
+from sync import get_advisories,identity,date_filter
+class TestData(unittest.TestCase):
+    def test_page(self):
+        items,total=get_advisories(b'{"advisories":[{"name":"RLSA-2026:1234"}],"total":"1"}')
+        self.assertEqual(total,1);self.assertTrue(date_filter(items[0]))
+    def test_identity(self): self.assertEqual(identity({'name':'RLSA-2026:1234'}),'RLSA-2026:1234')
+    def test_html(self):
+        with self.assertRaises(Exception):get_advisories(b'<html></html>')
+    def test_missing_total(self):
+        with self.assertRaises(ValueError):get_advisories(b'{"advisories":[]}')
+if __name__=='__main__':unittest.main()