From 02387e8c1723648f03890e742945a00bd52d4b87 Mon Sep 17 00:00:00 2001 From: root Date: Wed, 11 Nov 2026 19:02:58 +0100 Subject: [PATCH] Initial NVD CPE mirror --- .gitea/workflows/update.yml | 31 ++++++++ .gitignore | 4 + README.md | 30 ++++++++ data/cpematch/.gitkeep | 0 data/cpes/.gitkeep | 0 scripts/run-cached.sh | 38 ++++++++++ scripts/sync.py | 145 ++++++++++++++++++++++++++++++++++++ scripts/validate.py | 32 ++++++++ state/.gitkeep | 0 tests/test_sync.py | 37 +++++++++ 10 files changed, 317 insertions(+) create mode 100644 .gitea/workflows/update.yml create mode 100644 .gitignore create mode 100644 README.md create mode 100644 data/cpematch/.gitkeep create mode 100644 data/cpes/.gitkeep create mode 100644 scripts/run-cached.sh create mode 100644 scripts/sync.py create mode 100644 scripts/validate.py create mode 100644 state/.gitkeep create mode 100644 tests/test_sync.py diff --git a/.gitea/workflows/update.yml b/.gitea/workflows/update.yml new file mode 100644 index 0000000000..2744e6f75c --- /dev/null +++ b/.gitea/workflows/update.yml @@ -0,0 +1,31 @@ +name: NVD CPE and Match Criteria Mirror +on: + workflow_dispatch: + schedule: + - cron: "19 */4 * * *" + +jobs: + update: + runs-on: vuln-runner + timeout-minutes: 240 + env: + CACHE_ROOT: /var/lib/act_runner/cache/vuln-cpe + REPO_URL: https://git.apex-project.de/VULN/vuln-cpe.git + GITEA_JOB_TOKEN: ${{ secrets.GITEA_TOKEN }} + NVD_API_KEY: ${{ secrets.NVD_API_KEY }} + steps: + - name: Cached sync + shell: bash + run: | + set -Eeuo pipefail + : "${GITEA_JOB_TOKEN:?Gitea Job Token fehlt}" + mkdir -p "$CACHE_ROOT" + exec 9>"$CACHE_ROOT/.sync.lock" + flock -w 300 9 + export GIT_CONFIG_COUNT=1 + export GIT_CONFIG_KEY_0=http.extraHeader + export GIT_CONFIG_VALUE_0="Authorization: Basic $(printf 'oauth2:%s' "$GITEA_JOB_TOKEN" | base64 | tr -d '\n')" + if [ ! -d "$CACHE_ROOT/repo/.git" ]; then + git clone --depth 1 --branch main "$REPO_URL" "$CACHE_ROOT/repo" + fi + bash "$CACHE_ROOT/repo/scripts/run-cached.sh" diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000000..fa477f9aa9 --- /dev/null +++ b/.gitignore @@ -0,0 +1,4 @@ +__pycache__/ +*.pyc +*.tmp +.pytest_cache/ diff --git a/README.md b/README.md new file mode 100644 index 0000000000..c877f39c25 --- /dev/null +++ b/README.md @@ -0,0 +1,30 @@ +# vuln-cpe – NVD Official CPE Dictionary + CPE Match Criteria + +Standalone raw-data mirror for Gitea Actions, using a persistent Git checkout on the existing `vuln-runner` host. + +## Sources +- `https://services.nvd.nist.gov/rest/json/cpes/2.0` — product CPE dictionary; max 10,000/page. +- `https://services.nvd.nist.gov/rest/json/cpematch/2.0` — match criteria and version ranges; max 500/page. + +The mirror stores one full API record wrapper per UUID at `data/cpes//.json` and `data/cpematch//.json`. No flattening or interpretation is done. Deprecated records are retained as such. Source entries removed without a modification event require a periodically scheduled full reconciliation; this implementation does **not** delete absent entries automatically. + +## Gitea setup +1. Create empty `VULN/vuln-cpe` repo and push these files to `main`. +2. Create `/var/lib/act_runner/cache/vuln-cpe` owned by `act_runner` (`0700`). +3. Runner label `vuln-runner`; normal Gitea `secrets.GITEA_TOKEN` needs write permission. +4. Put the same `NVD_API_KEY` as for `vuln-nvd` in the org Actions Secrets (recommended). Without key the requests are slower. +5. Manually start the workflow, then it runs every four hours (UTC cron). + +## Bootstrap and updates +Both feeds are separate, checkpointed and capped at 20 pages per run. The initial full import takes multiple runs, committing progress after every run; it does **not** mean all data are available after the first successful action. Watch `state/cpes.json` and `state/cpematch.json`: both must show `"mode":"delta"` before declaring the initial mirror complete. + +After a full import, changes since the bootstrap start are replayed in 110-day windows with a 2-minute safety lag, to avoid gaps. Individual pages have a durable checkpoint; a crash during a page causes harmless replay on the next run. The HTTP API window is limited to 120 days. No data is deleted merely because it wasn't in a page. + +## Test +`python3 -m unittest discover -s tests -v` + +## Important limitations +- The NVD API imposes request limits; repeated runs for a large bootstrap can take days. Do not run multiple jobs in parallel with the same API key. +- A shallow Git checkout with thousands of records still requires disk space and pushes can be large. +- NVD CVE applicability tree (`AND`, `OR`, negate) remains in `vuln-nvd`; this repository stores only match criteria and CPE dictionary, not the applicability tree itself. +- A standalone full reconciliation/retirement audit should be scheduled periodically. NVD deprecations are retained with their status. diff --git a/data/cpematch/.gitkeep b/data/cpematch/.gitkeep new file mode 100644 index 0000000000..e69de29bb2 diff --git a/data/cpes/.gitkeep b/data/cpes/.gitkeep new file mode 100644 index 0000000000..e69de29bb2 diff --git a/scripts/run-cached.sh b/scripts/run-cached.sh new file mode 100644 index 0000000000..a60f230dfe --- /dev/null +++ b/scripts/run-cached.sh @@ -0,0 +1,38 @@ +#!/usr/bin/env bash +set -Eeuo pipefail +umask 077 +: "${CACHE_ROOT:?CACHE_ROOT fehlt}" +: "${REPO_URL:?REPO_URL fehlt}" +cd "$CACHE_ROOT/repo" +git remote set-url origin "$REPO_URL" +git fetch origin main +# Never destroy uncommitted downloaded records. A pending local commit is pushed first. +LOCAL=$(git rev-parse HEAD) +REMOTE=$(git rev-parse FETCH_HEAD) +if [[ "$LOCAL" != "$REMOTE" ]]; then + if git merge-base --is-ancestor "$REMOTE" "$LOCAL"; then + if [[ -n "$(git status --porcelain)" ]]; then + echo 'Local pending commit plus uncommitted files; manual review needed' >&2; exit 1 + fi + git push origin HEAD:main + git fetch origin main + elif [[ -z "$(git status --porcelain)" ]]; then + # Clean checkout: resetting to the fetched head cannot discard local data. + git reset --hard FETCH_HEAD + else + echo 'Remote changed while cached checkout has local data. Refusing destructive reset.' >&2; exit 1 + fi +fi +# Dirty data/state is an interrupted import and is safe to continue. Other changes are not. +if git status --porcelain | awk '{print substr($0,4)}' | grep -Ev '^(data/|state/)' | grep -q .; then + echo 'Unexpected changes outside data/ and state/; aborting' >&2; exit 1 +fi +python3 scripts/sync.py --root . --max-pages "${NVD_MAX_PAGES:-20}" +python3 scripts/validate.py --changed +git add -A -- data state +if git diff --cached --quiet; then + echo 'No changes'; exit 0 +fi +git -c user.name='Vulnerability Bot' -c user.email='vulnerability-bot@users.noreply.local' -c gc.auto=0 commit -qm "Update NVD CPE $(date -u +%FT%TZ)" +git -c gc.auto=0 push origin HEAD:main +echo "Updated $(git rev-parse --short HEAD)" diff --git a/scripts/sync.py b/scripts/sync.py new file mode 100644 index 0000000000..63b2df7aa0 --- /dev/null +++ b/scripts/sync.py @@ -0,0 +1,145 @@ +#!/usr/bin/env python3 +"""Mirror raw NVD CPE dictionary and match criteria using resumable page checkpoints. +Requires Python 3.9+; no third-party dependencies. +""" +import argparse +import datetime as dt +import json +import os +from pathlib import Path +import random +import time +import urllib.error +import urllib.parse +import urllib.request + +UTC = dt.timezone.utc +SOURCES = { + "cpes": ("https://services.nvd.nist.gov/rest/json/cpes/2.0", "products", "cpe", "cpeNameId", 2000), + "cpematch": ("https://services.nvd.nist.gov/rest/json/cpematch/2.0", "matchStrings", "matchString", "matchCriteriaId", 500), +} + +def stamp(t): + return t.astimezone(UTC).strftime("%Y-%m-%dT%H:%M:%S.000Z") + +def parse_stamp(s): + return dt.datetime.fromisoformat(s.replace("Z", "+00:00")) + +def save_atomic(path, obj): + path.parent.mkdir(parents=True, exist_ok=True) + payload = (json.dumps(obj, ensure_ascii=False, separators=(",", ":"), sort_keys=True) + "\n").encode("utf-8") + if path.exists() and path.read_bytes() == payload: + return False + temp = path.with_suffix(path.suffix + ".tmp") + temp.write_bytes(payload) + os.replace(temp, path) + return True + +def fetch(url, params, api_key, delay, retries=5): + query = urllib.parse.urlencode(params) + request = urllib.request.Request(url + "?" + query, headers={ + "User-Agent": "vuln-cpe-mirror/1.0 (NVD API)", "Accept": "application/json", + **({"apiKey": api_key} if api_key else {}) + }) + for attempt in range(retries): + time.sleep(delay) + try: + with urllib.request.urlopen(request, timeout=120) as response: + result = json.load(response) + if not isinstance(result, dict): + raise ValueError("NVD response not an object") + return result + except (urllib.error.URLError, TimeoutError, json.JSONDecodeError) as err: + if isinstance(err, urllib.error.HTTPError) and err.code not in (403, 429, 500, 502, 503, 504): + raise + if attempt == retries - 1: + raise + wait = min(120, 2 ** (attempt + 2)) + random.random() + print(f"Retry {attempt + 1}: {err}, sleep {wait:.1f}s", flush=True) + time.sleep(wait) + +def validate_page(page, key, nested, id_field): + rows = page.get(key) + if not isinstance(rows, list) or not isinstance(page.get("totalResults"), int): + raise ValueError(f"Invalid NVD page envelope: {key}") + if not isinstance(page.get("startIndex"), int): + raise ValueError("Missing page startIndex") + for wrapper in rows: + if not isinstance(wrapper, dict) or not isinstance(wrapper.get(nested), dict): + raise ValueError("Missing record object") + obj = wrapper[nested] + value = obj.get(id_field) + if not isinstance(value, str) or not value: + raise ValueError(f"Missing {id_field}") + return rows + +def process(name, root, api_key, max_pages, delay): + endpoint, field, nested, id_field, page_size = SOURCES[name] + state_path = root / "state" / f"{name}.json" + state = json.loads(state_path.read_text()) if state_path.exists() else {"mode": "full", "offset": 0} + now = dt.datetime.now(UTC) + # First full sync runs across many jobs; delta begins at the *start* of the full scan. + if state.get("mode") == "full" and not state.get("bootstrap_started"): + state["bootstrap_started"] = stamp(now) + save_atomic(state_path, state) + processed = written = 0 + for _ in range(max_pages): + if state["mode"] == "delta" and not state.get("window_end"): + start = parse_stamp(state["last_completed"]) + end = min(start + dt.timedelta(days=110), dt.datetime.now(UTC) - dt.timedelta(minutes=2)) + if start >= end: + break + state["window_start"] = stamp(start) + state["window_end"] = stamp(end) + state["offset"] = 0 + params = {"startIndex": state.get("offset", 0), "resultsPerPage": page_size} + if state["mode"] == "delta": + params.update(lastModStartDate=state["window_start"], lastModEndDate=state["window_end"]) + result = fetch(endpoint, params, api_key, delay) + offset = state.get("offset", 0) + if result["startIndex"] != offset: + raise RuntimeError(f"Unexpected pagination offset {result['startIndex']} instead of {offset}") + rows = validate_page(result, field, nested, id_field) + if not rows and offset < result["totalResults"]: + raise RuntimeError("NVD returned an empty page before totalResults") + for wrapper in rows: + obj = wrapper[nested] + rec_id = obj[id_field].lower() + # NVD IDs are UUIDs; don't permit untrusted path separators. + import uuid + try: uid = str(uuid.UUID(rec_id)) + except ValueError as exc: raise ValueError(f"Bad {id_field}: {rec_id}") from exc + path = root / "data" / name / uid[:2] / f"{uid}.json" + written += save_atomic(path, wrapper) + processed += 1 + next_offset = offset + len(rows) + if next_offset >= result["totalResults"]: + if state["mode"] == "full": + state = {"mode": "delta", "last_completed": state["bootstrap_started"], "offset": 0} + else: + state = {"mode": "delta", "last_completed": state["window_end"], "offset": 0} + else: + state["offset"] = next_offset + # State only moves forward after every returned record is durably written. + save_atomic(state_path, state) + print(f"{name}: +{len(rows)} records, written={written}, total={result['totalResults']}, phase={state['mode']}", flush=True) + if state["mode"] == "delta" and not state.get("window_end"): + # Only one delta window per run, never spin indefinitely. + break + return processed, written, state["mode"] + +def main(): + ap = argparse.ArgumentParser() + ap.add_argument("--root", type=Path, default=Path(".")) + ap.add_argument("--max-pages", type=int, default=20) + ap.add_argument("--delay", type=float) + args = ap.parse_args() + if not 1 <= args.max_pages <= 500: + ap.error("--max-pages must be between 1 and 500") + key = os.environ.get("NVD_API_KEY", "").strip() + delay = args.delay if args.delay is not None else (0.7 if key else 6.2) + for name in SOURCES: + print(f"Start {name}", flush=True) + print(process(name, args.root, key, args.max_pages, delay), flush=True) + +if __name__ == "__main__": main() diff --git a/scripts/validate.py b/scripts/validate.py new file mode 100644 index 0000000000..39839d2523 --- /dev/null +++ b/scripts/validate.py @@ -0,0 +1,32 @@ +#!/usr/bin/env python3 +import argparse +import json +from pathlib import Path +import subprocess + +def validate(path): + obj = json.loads(path.read_text(encoding="utf-8")) + if "products" in obj or "matchStrings" in obj: + raise ValueError("Unexpected page envelope in record") + expected = "cpe" if "/cpes/" in "/" + path.as_posix() else "matchString" + field = "cpeNameId" if expected == "cpe" else "matchCriteriaId" + if not isinstance(obj.get(expected), dict) or not obj[expected].get(field): + raise ValueError(f"Invalid record {path}") + +def main(): + p = argparse.ArgumentParser() + p.add_argument("--changed", action="store_true") + args = p.parse_args() + if args.changed: + result = subprocess.run(["git", "status", "--porcelain", "--untracked-files=all", "--", "data"], capture_output=True, text=True, check=True) + files = [Path(line[3:]) for line in result.stdout.splitlines() if len(line) > 3 and line[:2] != " D" and line[:2] != "D "] + else: + files = list(Path("data").rglob("*.json")) + count = 0 + for path in files: + if not path.is_file() or path.suffix != ".json": continue + validate(path) + count += 1 + print(f"VALID: {count} CPE / match records", flush=True) + +if __name__ == "__main__": main() diff --git a/state/.gitkeep b/state/.gitkeep new file mode 100644 index 0000000000..e69de29bb2 diff --git a/tests/test_sync.py b/tests/test_sync.py new file mode 100644 index 0000000000..5bde8e2ebc --- /dev/null +++ b/tests/test_sync.py @@ -0,0 +1,37 @@ +import json +import tempfile +import unittest +from pathlib import Path +from unittest.mock import patch +from scripts.sync import process, validate_page, save_atomic, stamp +import datetime as dt + +A = "12345678-1234-1234-1234-123456789abc" +B = "87654321-4321-4321-4321-cba987654321" + +class SyncTest(unittest.TestCase): + def test_validation(self): + page = {"startIndex":0,"totalResults":1,"products":[{"cpe":{"cpeNameId":A,"cpeName":"cpe:2.3:a:example:test:1:*:*:*:*:*:*:*"}}]} + self.assertEqual(len(validate_page(page,"products","cpe","cpeNameId")),1) + with self.assertRaises(ValueError): + validate_page({"startIndex":0,"totalResults":1,"products":[{}]},"products","cpe","cpeNameId") + def test_resume_full_then_delta(self): + with tempfile.TemporaryDirectory() as t: + root = Path(t) + def fake_fetch(url, params, key, delay): + if "lastModStartDate" in params: return {"startIndex":0,"totalResults":0,"products":[]} + index=params["startIndex"] + return {"startIndex":index,"totalResults":2,"products":[{"cpe":{"cpeNameId": [A,B][index]}}]} + with patch("scripts.sync.fetch", fake_fetch): + processed,_,mode=process("cpes",root,"",1,0) + self.assertEqual((processed,mode),(1,"full")) + processed,_,mode=process("cpes",root,"",1,0) + self.assertEqual((processed,mode),(1,"delta")) + self.assertEqual(len(list((root/"data/cpes").rglob("*.json"))),2) + def test_atomic_equal(self): + with tempfile.TemporaryDirectory() as t: + p = Path(t)/"x.json" + self.assertTrue(save_atomic(p,{"a":1})) + self.assertFalse(save_atomic(p,{"a":1})) + +if __name__ == "__main__": unittest.main()