From 88dbaef3617dd84026085dbe4bc7ffc3163c304a Mon Sep 17 00:00:00 2001 From: root Date: Tue, 10 Nov 2026 21:06:18 +0100 Subject: [PATCH] Initial Microsoft MSRC mirror --- .gitea/workflows/update.yml | 27 ++++++++ .gitignore | 5 ++ README.md | 36 ++++++++++ scripts/run-cached.sh | 43 ++++++++++++ scripts/sync.py | 127 ++++++++++++++++++++++++++++++++++++ scripts/validate.py | 23 +++++++ tests/test_sync.py | 25 +++++++ 7 files changed, 286 insertions(+) create mode 100644 .gitea/workflows/update.yml create mode 100644 .gitignore create mode 100644 README.md create mode 100644 scripts/run-cached.sh create mode 100644 scripts/sync.py create mode 100644 scripts/validate.py create mode 100644 tests/test_sync.py diff --git a/.gitea/workflows/update.yml b/.gitea/workflows/update.yml new file mode 100644 index 0000000..74d3c01 --- /dev/null +++ b/.gitea/workflows/update.yml @@ -0,0 +1,27 @@ +name: Microsoft MSRC Mirror + +on: + schedule: + - cron: '37 */4 * * *' + workflow_dispatch: + +permissions: + contents: write + +jobs: + mirror: + runs-on: vuln-runner + timeout-minutes: 180 + env: + GITEA_JOB_TOKEN: ${{ secrets.GITEA_TOKEN }} + GITEA_REPO_URL: https://git.apex-project.de/VULN/vuln-microsoft.git + VULN_CACHE_BASE: /var/lib/act_runner/cache/vuln-microsoft + steps: + - name: Synchronize Microsoft advisory documents + run: | + if [ ! -f /var/lib/act_runner/cache/vuln-microsoft/repo/scripts/run-cached.sh ]; then + mkdir -p /var/lib/act_runner/cache/vuln-microsoft + AUTH="Authorization: Basic $(printf 'x-access-token:%s' "$GITEA_JOB_TOKEN" | base64 | tr -d '\n')" + git -c "http.extraHeader=$AUTH" clone --depth 1 --branch main "$GITEA_REPO_URL" /var/lib/act_runner/cache/vuln-microsoft/repo + fi + bash /var/lib/act_runner/cache/vuln-microsoft/repo/scripts/run-cached.sh diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..b51d1fd --- /dev/null +++ b/.gitignore @@ -0,0 +1,5 @@ +__pycache__/ +*.pyc +*.tmp +.env +.cache/ diff --git a/README.md b/README.md new file mode 100644 index 0000000..a4c2f22 --- /dev/null +++ b/README.md @@ -0,0 +1,36 @@ +# vuln-microsoft – Gitea MSRC CVRF mirror + +This repository mirrors **raw Microsoft Security Response Center CVRF 3.0 JSON** per monthly advisory document, starting in 2017. It is NOT a normalized CSAF/VEX database. + +- Source: `https://api.msrc.microsoft.com/cvrf/v3.0/updates` +- Documents: `https://api.msrc.microsoft.com/cvrf/v3.0/cvrf/{YYYY-Mon}` +- Full original source objects in `data/cvrf/YYYY/YYYY-Mon.json` +- Full index in `data/index/updates.json` +- Synchronization history/state in `state/sync.json` +- Gitea runner executes every 4h at minute 37 UTC; also manually via Actions. + +## Setup + +1. Create an empty `VULN/vuln-microsoft` repository and push this folder as branch `main`. +2. On the existing Linux Gitea host runner, install `git`, `python3`, `flock` (`util-linux`) and `base64` (`coreutils`). +3. Create the persistent folder: + + `sudo install -d -o act_runner -g act_runner -m 700 /var/lib/act_runner/cache/vuln-microsoft` + +4. The workflow bootstraps the checkout automatically on its first run. No manual clone is needed. Ensure the runner can write to the cache path. +5. Gitea → repository → Settings → Actions: enable write permission for workflow token. `secrets.GITEA_TOKEN` is Gitea's native token; if your installation uses the custom `GITEATOKEN`, change only the workflow env line. +6. Trigger the workflow manually for the initial import. + +## Manual operation + +`python3 scripts/sync.py --since-year 2017` + +`python3 scripts/sync.py --since-year 2017 --full` (force all monthly documents to be rechecked) + +`python3 scripts/validate.py` + +## Incremental behavior + +The release listing is fetched every run. Missing/new documents, documents with changed release dates, and documents updated in the last 45 days are downloaded. A forced `--full` periodically helps reconcile older revisions whose metadata may not change. It does **not** delete unpublished documents automatically; deletion must be handled by a separately reviewed reconcile process. + +Security: Host runners execute repository code locally on the Gitea VM; protect `main` and restrict workflow edit rights. diff --git a/scripts/run-cached.sh b/scripts/run-cached.sh new file mode 100644 index 0000000..87ca307 --- /dev/null +++ b/scripts/run-cached.sh @@ -0,0 +1,43 @@ +#!/usr/bin/env bash +set -Eeuo pipefail +umask 077 + +BASE="${VULN_CACHE_BASE:-/var/lib/act_runner/cache/vuln-microsoft}" +REPO="$BASE/repo" +URL="${GITEA_REPO_URL:-https://git.apex-project.de/VULN/vuln-microsoft.git}" +mkdir -p "$BASE" +exec 9>"$BASE/.sync.lock" +flock -w 60 9 || { echo 'Another sync holds the cache lock' >&2; exit 1; } + +if [[ -z "${GITEA_JOB_TOKEN:-}" ]]; then + echo 'GITEA_JOB_TOKEN missing; configure secret GITEA_TOKEN or GITEATOKEN' >&2 + exit 1 +fi +# No tokens in remotes, logs or files. Token is supplied only as a per-command HTTP header. +AUTH="Authorization: Basic $(printf 'x-access-token:%s' "$GITEA_JOB_TOKEN" | base64 | tr -d '\n')" +# Gitea job tokens may be accepted with x-access-token username; if unavailable, use a scoped PAT. +git_auth() { git -c "http.extraHeader=$AUTH" "$@"; } + +if [[ ! -d "$REPO/.git" ]]; then + git_auth clone --branch main --depth 1 "$URL" "$REPO" +fi +cd "$REPO" +git remote set-url origin "$URL" +if [[ -n "$(git status --porcelain)" ]]; then + echo 'Persistent checkout has uncommitted changes; aborting without resetting data' >&2 + exit 1 +fi +git_auth fetch --depth 1 origin main +git checkout main +git merge --ff-only FETCH_HEAD + +python3 scripts/sync.py --since-year 2017 +python3 scripts/validate.py + +git add -A data/ state/ +if git diff --cached --quiet; then + echo 'No changes to commit' + exit 0 +fi +git -c user.name='Vulnerability Bot' -c user.email='vulnerability-bot@users.noreply.local' commit -m "MSRC update $(date -u +%Y-%m-%dT%H:%M:%SZ)" +git_auth push origin HEAD:main diff --git a/scripts/sync.py b/scripts/sync.py new file mode 100644 index 0000000..6da0588 --- /dev/null +++ b/scripts/sync.py @@ -0,0 +1,127 @@ +#!/usr/bin/env python3 +"""Mirrors full MSRC CVRF JSON documents and the raw update index.""" +import argparse +import datetime as dt +import json +import os +from pathlib import Path +import random +import re +import sys +import time +import urllib.error +import urllib.request + +BASE = os.getenv('MSRC_BASE_URL', 'https://api.msrc.microsoft.com/cvrf/v3.0').rstrip('/') +DOC_RE = re.compile(r'^([12][0-9]{3})-([A-Za-z]{3})$') +HEADERS = {'Accept': 'application/json', 'User-Agent': 'apex-project-vuln-microsoft/1.0'} + +def fetch(url, attempts=5): + for n in range(attempts): + try: + with urllib.request.urlopen(urllib.request.Request(url, headers=HEADERS), timeout=90) as response: + data = response.read() + obj = json.loads(data) + if not isinstance(obj, (dict, list)): + raise ValueError('Unexpected response JSON type') + return obj + except (urllib.error.URLError, TimeoutError, json.JSONDecodeError, ValueError) as exc: + if n == attempts - 1: + raise RuntimeError(f'Download failed: {url}: {exc}') from exc + pause = min(60, 2 ** n * 3) + random.random() + print(f'Retrying {url}: {exc}; {pause:.1f}s', file=sys.stderr, flush=True) + time.sleep(pause) + +def atomic_json(path, obj): + path.parent.mkdir(parents=True, exist_ok=True) + content = (json.dumps(obj, ensure_ascii=False, sort_keys=True, separators=(',', ':')) + '\n').encode() + if path.exists() and path.read_bytes() == content: + return False + temp = path.with_suffix(path.suffix + '.tmp') + temp.write_bytes(content) + os.replace(temp, path) + return True + +def index_rows(index): + if isinstance(index, list): + rows = index + elif isinstance(index, dict): + rows = index.get('value') or index.get('Value') + else: + rows = None + if not isinstance(rows, list) or not rows: + raise ValueError('Microsoft update index has no entries; aborting safely') + return rows + +def normalize_id(row): + value = row.get('ID') or row.get('Id') or row.get('id') + if not isinstance(value, str) or not DOC_RE.fullmatch(value): + return None + return value + +def release_date(row): + value = row.get('CurrentReleaseDate') or row.get('currentReleaseDate') or row.get('InitialReleaseDate') or row.get('initialReleaseDate') + if not value: + return None + try: + return dt.datetime.fromisoformat(str(value).replace('Z', '+00:00')).astimezone(dt.timezone.utc) + except ValueError: + return None + +def sync(root, since_year, days, full, delay): + data_root = root / 'data' + state_file = root / 'state' / 'sync.json' + today = dt.datetime.now(dt.timezone.utc) + index = fetch(BASE + '/updates') + rows = index_rows(index) + filtered = {} + for row in rows: + if not isinstance(row, dict): + raise ValueError('Non-object update entry') + docid = normalize_id(row) + if docid and int(docid[:4]) >= since_year: + filtered[docid] = row + if not filtered: + raise ValueError('No documents after year filter; aborting') + previous = json.loads(state_file.read_text()) if state_file.exists() else {} + prev_dates = previous.get('release_dates', {}) + selected = [] + for docid, row in sorted(filtered.items()): + target = data_root / 'cvrf' / docid[:4] / (docid + '.json') + date_val = row.get('CurrentReleaseDate') or row.get('currentReleaseDate') or row.get('InitialReleaseDate') or row.get('initialReleaseDate') + date_key = str(date_val or '') + recent = (release_date(row) or dt.datetime(1970, 1, 1, tzinfo=dt.timezone.utc)) >= today - dt.timedelta(days=days) + if full or not target.exists() or prev_dates.get(docid) != date_key or recent: + selected.append((docid, target)) + print(f'MSRC documents listed={len(filtered)}, download={len(selected)}, full={full}', flush=True) + updated = 0 + for idx, (docid, target) in enumerate(selected, start=1): + doc = fetch(BASE + '/cvrf/' + docid) + if not isinstance(doc, dict) or not doc: + raise ValueError(f'Empty or invalid CVRF document {docid}') + updated += atomic_json(target, doc) + if idx % 20 == 0 or idx == len(selected): + print(f'processed {idx}/{len(selected)}, changed={updated}', flush=True) + if delay and idx < len(selected): + time.sleep(delay) + atomic_json(data_root / 'index' / 'updates.json', index) + atomic_json(state_file, { + 'source': BASE, + 'since_year': since_year, + 'last_success_utc': today.isoformat(), + 'documents': len(filtered), + 'processed': len(selected), + 'changed': updated, + 'release_dates': {id_: str(row.get('CurrentReleaseDate') or row.get('currentReleaseDate') or row.get('InitialReleaseDate') or row.get('initialReleaseDate') or '') for id_, row in filtered.items()} + }) + print('SUCCESS', flush=True) + +if __name__ == '__main__': + ap = argparse.ArgumentParser() + ap.add_argument('--root', type=Path, default=Path('.')) + ap.add_argument('--since-year', type=int, default=2017) + ap.add_argument('--recheck-days', type=int, default=45) + ap.add_argument('--full', action='store_true') + ap.add_argument('--delay', type=float, default=0.5) + args = ap.parse_args() + sync(args.root.resolve(), args.since_year, args.recheck_days, args.full, args.delay) diff --git a/scripts/validate.py b/scripts/validate.py new file mode 100644 index 0000000..924c74e --- /dev/null +++ b/scripts/validate.py @@ -0,0 +1,23 @@ +#!/usr/bin/env python3 +import json +import re +from pathlib import Path + +root = Path(__file__).resolve().parents[1] +index = root / 'data/index/updates.json' +if not index.is_file(): + raise SystemExit('Missing data/index/updates.json') +obj = json.loads(index.read_text()) +rows = obj if isinstance(obj, list) else obj.get('value') +if not isinstance(rows, list) or not rows: + raise SystemExit('Empty or invalid updates index') +files = list((root/'data/cvrf').glob('*/*.json')) +if not files: + raise SystemExit('No CVRF documents present') +for f in files: + if not re.fullmatch(r'\d{4}-[A-Za-z]{3}\.json', f.name) or f.parent.name != f.name[:4]: + raise SystemExit(f'Unexpected file path: {f}') + document = json.loads(f.read_text()) + if not isinstance(document, dict) or not document: + raise SystemExit(f'Invalid document {f}') +print(f'OK: {len(rows)} index entries and {len(files)} CVRF JSON files') diff --git a/tests/test_sync.py b/tests/test_sync.py new file mode 100644 index 0000000..0a4c57c --- /dev/null +++ b/tests/test_sync.py @@ -0,0 +1,25 @@ +import sys +import tempfile +import unittest +from pathlib import Path +from unittest.mock import patch + +sys.path.insert(0, str(Path(__file__).resolve().parents[1] / 'scripts')) +import sync + +class MirrorTests(unittest.TestCase): + def test_index(self): + self.assertEqual(len(sync.index_rows({'value': [{'ID': '2026-Nov'}]})), 1) + with self.assertRaises(ValueError): + sync.index_rows({'value': []}) + def test_update_and_idempotency(self): + index = {'value': [{'ID': '2026-Nov', 'CurrentReleaseDate': '2026-11-10T00:00:00Z'}]} + with tempfile.TemporaryDirectory() as t: + with patch.object(sync, 'fetch', side_effect=[index, {'DocumentTitle': {'Value': 'test'}}]): + sync.sync(Path(t), 2017, 45, False, 0) + target = Path(t)/'data/cvrf/2026/2026-Nov.json' + self.assertTrue(target.exists()) + with patch.object(sync, 'fetch', side_effect=[index, {'DocumentTitle': {'Value': 'test'}}]): + sync.sync(Path(t), 2017, 45, False, 0) + self.assertEqual(__import__('json').loads((Path(t)/'state/sync.json').read_text())['changed'], 0) +if __name__ == '__main__': unittest.main()