Initial Microsoft MSRC mirror

.gitea/workflows/update.yml

@@ -0,0 +1,27 @@
+name: Microsoft MSRC Mirror
+
+on:
+  schedule:
+    - cron: '37 */4 * * *'
+  workflow_dispatch:
+
+permissions:
+  contents: write
+
+jobs:
+  mirror:
+    runs-on: vuln-runner
+    timeout-minutes: 180
+    env:
+      GITEA_JOB_TOKEN: ${{ secrets.GITEA_TOKEN }}
+      GITEA_REPO_URL: https://git.apex-project.de/VULN/vuln-microsoft.git
+      VULN_CACHE_BASE: /var/lib/act_runner/cache/vuln-microsoft
+    steps:
+      - name: Synchronize Microsoft advisory documents
+        run: |
+          if [ ! -f /var/lib/act_runner/cache/vuln-microsoft/repo/scripts/run-cached.sh ]; then
+            mkdir -p /var/lib/act_runner/cache/vuln-microsoft
+            AUTH="Authorization: Basic $(printf 'x-access-token:%s' "$GITEA_JOB_TOKEN" | base64 | tr -d '\n')"
+            git -c "http.extraHeader=$AUTH" clone --depth 1 --branch main "$GITEA_REPO_URL" /var/lib/act_runner/cache/vuln-microsoft/repo
+          fi
+          bash /var/lib/act_runner/cache/vuln-microsoft/repo/scripts/run-cached.sh

.gitignore

@@ -0,0 +1,5 @@
+__pycache__/
+*.pyc
+*.tmp
+.env
+.cache/

README.md

@@ -0,0 +1,36 @@
+# vuln-microsoft – Gitea MSRC CVRF mirror
+
+This repository mirrors **raw Microsoft Security Response Center CVRF 3.0 JSON** per monthly advisory document, starting in 2017. It is NOT a normalized CSAF/VEX database.
+
+- Source: `https://api.msrc.microsoft.com/cvrf/v3.0/updates`
+- Documents: `https://api.msrc.microsoft.com/cvrf/v3.0/cvrf/{YYYY-Mon}`
+- Full original source objects in `data/cvrf/YYYY/YYYY-Mon.json`
+- Full index in `data/index/updates.json`
+- Synchronization history/state in `state/sync.json`
+- Gitea runner executes every 4h at minute 37 UTC; also manually via Actions.
+
+## Setup
+
+1. Create an empty `VULN/vuln-microsoft` repository and push this folder as branch `main`.
+2. On the existing Linux Gitea host runner, install `git`, `python3`, `flock` (`util-linux`) and `base64` (`coreutils`).
+3. Create the persistent folder:
+
+   `sudo install -d -o act_runner -g act_runner -m 700 /var/lib/act_runner/cache/vuln-microsoft`
+
+4. The workflow bootstraps the checkout automatically on its first run. No manual clone is needed. Ensure the runner can write to the cache path.
+5. Gitea → repository → Settings → Actions: enable write permission for workflow token. `secrets.GITEA_TOKEN` is Gitea's native token; if your installation uses the custom `GITEATOKEN`, change only the workflow env line.
+6. Trigger the workflow manually for the initial import.
+
+## Manual operation
+
+`python3 scripts/sync.py --since-year 2017`
+
+`python3 scripts/sync.py --since-year 2017 --full`   (force all monthly documents to be rechecked)
+
+`python3 scripts/validate.py`
+
+## Incremental behavior
+
+The release listing is fetched every run. Missing/new documents, documents with changed release dates, and documents updated in the last 45 days are downloaded. A forced `--full` periodically helps reconcile older revisions whose metadata may not change. It does **not** delete unpublished documents automatically; deletion must be handled by a separately reviewed reconcile process.
+
+Security: Host runners execute repository code locally on the Gitea VM; protect `main` and restrict workflow edit rights.

scripts/run-cached.sh

@@ -0,0 +1,43 @@
+#!/usr/bin/env bash
+set -Eeuo pipefail
+umask 077
+
+BASE="${VULN_CACHE_BASE:-/var/lib/act_runner/cache/vuln-microsoft}"
+REPO="$BASE/repo"
+URL="${GITEA_REPO_URL:-https://git.apex-project.de/VULN/vuln-microsoft.git}"
+mkdir -p "$BASE"
+exec 9>"$BASE/.sync.lock"
+flock -w 60 9 || { echo 'Another sync holds the cache lock' >&2; exit 1; }
+
+if [[ -z "${GITEA_JOB_TOKEN:-}" ]]; then
+  echo 'GITEA_JOB_TOKEN missing; configure secret GITEA_TOKEN or GITEATOKEN' >&2
+  exit 1
+fi
+# No tokens in remotes, logs or files. Token is supplied only as a per-command HTTP header.
+AUTH="Authorization: Basic $(printf 'x-access-token:%s' "$GITEA_JOB_TOKEN" | base64 | tr -d '\n')"
+# Gitea job tokens may be accepted with x-access-token username; if unavailable, use a scoped PAT.
+git_auth() { git -c "http.extraHeader=$AUTH" "$@"; }
+
+if [[ ! -d "$REPO/.git" ]]; then
+  git_auth clone --branch main --depth 1 "$URL" "$REPO"
+fi
+cd "$REPO"
+git remote set-url origin "$URL"
+if [[ -n "$(git status --porcelain)" ]]; then
+  echo 'Persistent checkout has uncommitted changes; aborting without resetting data' >&2
+  exit 1
+fi
+git_auth fetch --depth 1 origin main
+git checkout main
+git merge --ff-only FETCH_HEAD
+
+python3 scripts/sync.py --since-year 2017
+python3 scripts/validate.py
+
+git add -A data/ state/
+if git diff --cached --quiet; then
+  echo 'No changes to commit'
+  exit 0
+fi
+git -c user.name='Vulnerability Bot' -c user.email='vulnerability-bot@users.noreply.local' commit -m "MSRC update $(date -u +%Y-%m-%dT%H:%M:%SZ)"
+git_auth push origin HEAD:main

scripts/sync.py

@@ -0,0 +1,127 @@
+#!/usr/bin/env python3
+"""Mirrors full MSRC CVRF JSON documents and the raw update index."""
+import argparse
+import datetime as dt
+import json
+import os
+from pathlib import Path
+import random
+import re
+import sys
+import time
+import urllib.error
+import urllib.request
+
+BASE = os.getenv('MSRC_BASE_URL', 'https://api.msrc.microsoft.com/cvrf/v3.0').rstrip('/')
+DOC_RE = re.compile(r'^([12][0-9]{3})-([A-Za-z]{3})$')
+HEADERS = {'Accept': 'application/json', 'User-Agent': 'apex-project-vuln-microsoft/1.0'}
+
+def fetch(url, attempts=5):
+    for n in range(attempts):
+        try:
+            with urllib.request.urlopen(urllib.request.Request(url, headers=HEADERS), timeout=90) as response:
+                data = response.read()
+            obj = json.loads(data)
+            if not isinstance(obj, (dict, list)):
+                raise ValueError('Unexpected response JSON type')
+            return obj
+        except (urllib.error.URLError, TimeoutError, json.JSONDecodeError, ValueError) as exc:
+            if n == attempts - 1:
+                raise RuntimeError(f'Download failed: {url}: {exc}') from exc
+            pause = min(60, 2 ** n * 3) + random.random()
+            print(f'Retrying {url}: {exc}; {pause:.1f}s', file=sys.stderr, flush=True)
+            time.sleep(pause)
+
+def atomic_json(path, obj):
+    path.parent.mkdir(parents=True, exist_ok=True)
+    content = (json.dumps(obj, ensure_ascii=False, sort_keys=True, separators=(',', ':')) + '\n').encode()
+    if path.exists() and path.read_bytes() == content:
+        return False
+    temp = path.with_suffix(path.suffix + '.tmp')
+    temp.write_bytes(content)
+    os.replace(temp, path)
+    return True
+
+def index_rows(index):
+    if isinstance(index, list):
+        rows = index
+    elif isinstance(index, dict):
+        rows = index.get('value') or index.get('Value')
+    else:
+        rows = None
+    if not isinstance(rows, list) or not rows:
+        raise ValueError('Microsoft update index has no entries; aborting safely')
+    return rows
+
+def normalize_id(row):
+    value = row.get('ID') or row.get('Id') or row.get('id')
+    if not isinstance(value, str) or not DOC_RE.fullmatch(value):
+        return None
+    return value
+
+def release_date(row):
+    value = row.get('CurrentReleaseDate') or row.get('currentReleaseDate') or row.get('InitialReleaseDate') or row.get('initialReleaseDate')
+    if not value:
+        return None
+    try:
+        return dt.datetime.fromisoformat(str(value).replace('Z', '+00:00')).astimezone(dt.timezone.utc)
+    except ValueError:
+        return None
+
+def sync(root, since_year, days, full, delay):
+    data_root = root / 'data'
+    state_file = root / 'state' / 'sync.json'
+    today = dt.datetime.now(dt.timezone.utc)
+    index = fetch(BASE + '/updates')
+    rows = index_rows(index)
+    filtered = {}
+    for row in rows:
+        if not isinstance(row, dict):
+            raise ValueError('Non-object update entry')
+        docid = normalize_id(row)
+        if docid and int(docid[:4]) >= since_year:
+            filtered[docid] = row
+    if not filtered:
+        raise ValueError('No documents after year filter; aborting')
+    previous = json.loads(state_file.read_text()) if state_file.exists() else {}
+    prev_dates = previous.get('release_dates', {})
+    selected = []
+    for docid, row in sorted(filtered.items()):
+        target = data_root / 'cvrf' / docid[:4] / (docid + '.json')
+        date_val = row.get('CurrentReleaseDate') or row.get('currentReleaseDate') or row.get('InitialReleaseDate') or row.get('initialReleaseDate')
+        date_key = str(date_val or '')
+        recent = (release_date(row) or dt.datetime(1970, 1, 1, tzinfo=dt.timezone.utc)) >= today - dt.timedelta(days=days)
+        if full or not target.exists() or prev_dates.get(docid) != date_key or recent:
+            selected.append((docid, target))
+    print(f'MSRC documents listed={len(filtered)}, download={len(selected)}, full={full}', flush=True)
+    updated = 0
+    for idx, (docid, target) in enumerate(selected, start=1):
+        doc = fetch(BASE + '/cvrf/' + docid)
+        if not isinstance(doc, dict) or not doc:
+            raise ValueError(f'Empty or invalid CVRF document {docid}')
+        updated += atomic_json(target, doc)
+        if idx % 20 == 0 or idx == len(selected):
+            print(f'processed {idx}/{len(selected)}, changed={updated}', flush=True)
+        if delay and idx < len(selected):
+            time.sleep(delay)
+    atomic_json(data_root / 'index' / 'updates.json', index)
+    atomic_json(state_file, {
+        'source': BASE,
+        'since_year': since_year,
+        'last_success_utc': today.isoformat(),
+        'documents': len(filtered),
+        'processed': len(selected),
+        'changed': updated,
+        'release_dates': {id_: str(row.get('CurrentReleaseDate') or row.get('currentReleaseDate') or row.get('InitialReleaseDate') or row.get('initialReleaseDate') or '') for id_, row in filtered.items()}
+    })
+    print('SUCCESS', flush=True)
+
+if __name__ == '__main__':
+    ap = argparse.ArgumentParser()
+    ap.add_argument('--root', type=Path, default=Path('.'))
+    ap.add_argument('--since-year', type=int, default=2017)
+    ap.add_argument('--recheck-days', type=int, default=45)
+    ap.add_argument('--full', action='store_true')
+    ap.add_argument('--delay', type=float, default=0.5)
+    args = ap.parse_args()
+    sync(args.root.resolve(), args.since_year, args.recheck_days, args.full, args.delay)

scripts/validate.py

@@ -0,0 +1,23 @@
+#!/usr/bin/env python3
+import json
+import re
+from pathlib import Path
+
+root = Path(__file__).resolve().parents[1]
+index = root / 'data/index/updates.json'
+if not index.is_file():
+    raise SystemExit('Missing data/index/updates.json')
+obj = json.loads(index.read_text())
+rows = obj if isinstance(obj, list) else obj.get('value')
+if not isinstance(rows, list) or not rows:
+    raise SystemExit('Empty or invalid updates index')
+files = list((root/'data/cvrf').glob('*/*.json'))
+if not files:
+    raise SystemExit('No CVRF documents present')
+for f in files:
+    if not re.fullmatch(r'\d{4}-[A-Za-z]{3}\.json', f.name) or f.parent.name != f.name[:4]:
+        raise SystemExit(f'Unexpected file path: {f}')
+    document = json.loads(f.read_text())
+    if not isinstance(document, dict) or not document:
+        raise SystemExit(f'Invalid document {f}')
+print(f'OK: {len(rows)} index entries and {len(files)} CVRF JSON files')

tests/test_sync.py

@@ -0,0 +1,25 @@
+import sys
+import tempfile
+import unittest
+from pathlib import Path
+from unittest.mock import patch
+
+sys.path.insert(0, str(Path(__file__).resolve().parents[1] / 'scripts'))
+import sync
+
+class MirrorTests(unittest.TestCase):
+    def test_index(self):
+        self.assertEqual(len(sync.index_rows({'value': [{'ID': '2026-Nov'}]})), 1)
+        with self.assertRaises(ValueError):
+            sync.index_rows({'value': []})
+    def test_update_and_idempotency(self):
+        index = {'value': [{'ID': '2026-Nov', 'CurrentReleaseDate': '2026-11-10T00:00:00Z'}]}
+        with tempfile.TemporaryDirectory() as t:
+            with patch.object(sync, 'fetch', side_effect=[index, {'DocumentTitle': {'Value': 'test'}}]):
+                sync.sync(Path(t), 2017, 45, False, 0)
+            target = Path(t)/'data/cvrf/2026/2026-Nov.json'
+            self.assertTrue(target.exists())
+            with patch.object(sync, 'fetch', side_effect=[index, {'DocumentTitle': {'Value': 'test'}}]):
+                sync.sync(Path(t), 2017, 45, False, 0)
+            self.assertEqual(__import__('json').loads((Path(t)/'state/sync.json').read_text())['changed'], 0)
+if __name__ == '__main__': unittest.main()