diff --git a/.gitea/workflows/build.yml b/.gitea/workflows/build.yml new file mode 100644 index 0000000..a1b75a3 --- /dev/null +++ b/.gitea/workflows/build.yml @@ -0,0 +1,47 @@ +name: Build SQLite vulnerability index +on: + workflow_dispatch: + schedule: + - cron: "35 3 * * *" +concurrency: + group: vuln-builder + cancel-in-progress: false +jobs: + build: + runs-on: vuln-builder + timeout-minutes: 720 + env: + CACHE_ROOT: /vuln-cache/vuln-builder + GITEA_BASE_URL: https://git.apex-project.de + GITEA_JOB_TOKEN: ${{ secrets.GITEA_TOKEN }} + steps: + - name: Install tools + shell: sh + run: | + set -eu + if command -v apk >/dev/null 2>&1; then + apk add --no-cache bash git python3 sqlite zstd util-linux ca-certificates coreutils + elif command -v apt-get >/dev/null 2>&1; then + apt-get update + apt-get install -y --no-install-recommends bash git python3 sqlite3 zstd util-linux ca-certificates coreutils + else + echo 'Unsupported job image' >&2 + exit 1 + fi + /usr/bin/flock --version + - name: Bootstrap and build + shell: bash + run: | + set -Eeuo pipefail + : "${GITEA_JOB_TOKEN:?missing token}" + mkdir -p "$CACHE_ROOT" + export GIT_CONFIG_COUNT=1 + export GIT_CONFIG_KEY_0=http.extraHeader + export GIT_CONFIG_VALUE_0="Authorization: Basic $(printf 'oauth2:%s' "$GITEA_JOB_TOKEN" | base64 | tr -d '\n')" + if [ ! -d "$CACHE_ROOT/builder/.git" ]; then + git clone --depth 1 --branch main "$GITEA_BASE_URL/VULN/vuln-builder.git" "$CACHE_ROOT/builder" + else + git -C "$CACHE_ROOT/builder" fetch --depth 1 origin main + git -C "$CACHE_ROOT/builder" reset --hard FETCH_HEAD + fi + bash "$CACHE_ROOT/builder/scripts/run.sh" diff --git a/README.md b/README.md index 3c55095..27d975d 100644 --- a/README.md +++ b/README.md @@ -1,20 +1,45 @@ -# vuln-builder – CVE-Referenzimplementierung +# vuln-builder – SQLite Starter (v1) -Erstellt `master.db.zst`, `manifest.json` und (wenn `--previous` existiert) `delta.jsonl.zst` aus dem Repository `vuln-cve`. Das SQLite-Schema hält **CVE-ID, Status, Aktualisierungsdatum und verlustfrei die originale CNA-affected-Liste** vor. Es ist ein kompakter **CVE-Index**, **kein fertiger Scanner**: CNA-affected-Angaben sind nicht allein ausreichend für genaue CPE-, PURL-, Vendor- oder Distro-Versionsabgleiche; diese benötigen quellenspezifische Parser und AND/OR/NOT-Bedingungen. +First functional, **conservative** builder for the Gitea organization `VULN`. -## Lokal +## Scope -```bash -python3 -m pip install -r requirements.txt -python3 scripts/build.py --source ../vuln-cve/data --output dist -# Später mit vorheriger Datei für Delta: -python3 scripts/build.py --source ../vuln-cve/data --previous /pfad/alte/master.db --output dist -# Delta auf vorhandene Kopie anwenden: -python3 scripts/apply_delta.py --database /pfad/alte/master.db --delta dist/delta.jsonl.zst +- Clones/fetches all named repositories in a persistent Docker volume. +- Keeps the source Git commit for **every** mirror in `sources`. +- Indexes **CVEProject (`cveMetadata`)**, **NVD (`cve`)** and **OSV (`id`/`affected`)** into SQLite. +- Saves OSV package/version range events without guessing at version comparisons. +- Saves only simple non-negated NVD CPE disjunctions; complex AND/NOT expressions are **not** converted into potentially incorrect matches. +- Other vendors and distro data are **not yet normalized**. This is not yet a complete Trivy-compatible scanner DB. +- Writes `master.db` (local staging), compressed `master.db.zst`, and on subsequent builds `delta-OLD-to-NEW.jsonl.zst`, plus `manifest.json`. +- **No automated release publishing yet**: files remain on the Docker host volume at `/vuln-cache/vuln-builder/releases`. + +## Install + +1. Create empty `VULN/vuln-builder` in Gitea, then push this folder to branch `main`. +2. Ensure Docker runner with label `vulnerability` is online and the named volume `gitea-vuln-cache` is mounted in **job** containers at `/vuln-cache`. +3. Configure Gitea Actions read permission for the Gitea token across all `VULN` repositories (a job token may be scoped to a single repo; if cross-repo reads fail, create a separate read-only organization credential named `VULN_READ_TOKEN`, and change only the workflow's secret reference). Do not log credentials. +4. Start `.gitea/workflows/build.yml`. Cron: daily 03:35 UTC. Initial download can be *very* large. Make sure all source mirrors finished their first import. + +## Manual release inspection + +On Docker host: + +```sh +docker run --rm -v gitea-vuln-cache:/vuln-cache:ro alpine ls -lh /vuln-cache/vuln-builder/releases ``` -## GitHub Actions +## Safety and limitations -Die Build-Action checkt `vuln-cve` aus und lädt die vorherige veröffentlichte `master.db.zst` per `gh release download`. Sie erstellt eine neue GitHub Release mit Master und ggf. Delta. `VULN_READ_TOKEN` muss Leserechte auf `vuln-cve` haben (Fine-grained PAT oder GitHub-App-Token); der `GITHUB_TOKEN` des Builder-Repos braucht `contents: write`. Private Quellen, Branch-Schutzregeln und Self-hosted-Runner sind entsprechend einzurichten. +- Input source repositories need valid Git checkout and the three core sources must exist. +- A successful first-build **does not** imply all feeds are fully imported; check their source-specific completion markers before starting. +- SQLite v1 is an index, not a final vulnerability scanner: no Debian/RPM version comparison, no complete CPE boolean-expression evaluator, no CSAF product-tree traversal yet. +- Delta JSONL updates have `{table, op, key}` for deletion and `{table, op, row}` for upserts. A client must verify the starting version, apply within one SQLite transaction with foreign key constraints managed correctly, and only mark the new version after integrity checks. This package does **not** yet implement that client. +- The source data can require many GB in the persistent Git cache. Avoid resetting/deleting the volume to save time. +- `master.db` remains uncompressed on the server for diffing against the next build. +- The master artifact and manifest are replaced one after another; consumers must only use `manifest.json` after it is written and verify the published SHA-256. -**Grenzen für den Produktivbetrieb:** Die GitHub-Release-Dateigrößen und Repo-Quotas müssen überprüft werden. Der Build erstellt bisher nur einen CVE-Datensatz-Index; NVD/OSV/Hersteller/KEV/EPSS-Adapter sowie die Zusammenführung von Advisories und Versionsbereichen sind noch nicht implementiert. Deltas müssen sequenziell und unverändert veröffentlicht werden; GitHub `latest` ersetzt keine versionierte Delta-Historie. Releases dürfen nicht überschrieben werden. Für größere Datenmengen sind ein persistenter Runner, inkrementelle Builds und Release-Retention-Strategie sinnvoll. +## Tests + +```sh +python3 -m unittest discover -s tests -v +``` diff --git a/scripts/build.py b/scripts/build.py index 7f85605..7b83689 100644 --- a/scripts/build.py +++ b/scripts/build.py @@ -1,113 +1,263 @@ #!/usr/bin/env python3 -"""Build compact CVE SQLite plus optionally incremental row-level delta (CVE slice).""" +"""Conservative SQLite index from CVEProject, NVD and OSV raw JSON. +Do not mistake this index for a complete vulnerability matching engine. +""" import argparse -import base64 -import datetime as dt import hashlib import json +import os +import re +import shutil import sqlite3 -import zlib -from pathlib import Path import subprocess +import tempfile +from datetime import datetime, timezone +from pathlib import Path -SCHEMA = '''CREATE TABLE cve ( - cve_id TEXT PRIMARY KEY, - state TEXT NOT NULL, - date_updated TEXT, - affected BLOB NOT NULL, - row_hash TEXT NOT NULL -) WITHOUT ROWID; -CREATE INDEX idx_cve_state ON cve(state);''' +CVE = re.compile(r'^CVE-\d{4}-\d{4,}$', re.I) +FILES = ('vulnerabilities','aliases','affected_packages','cpe_matches') -def encode_record(doc): - meta = doc['cveMetadata'] - cna = doc.get('containers', {}).get('cna', {}) - # Preserve entire CNA affected array as structured JSON. Do NOT infer ranges. - affected = json.dumps(cna.get('affected', []), sort_keys=True, separators=(',', ':'), ensure_ascii=False).encode('utf-8') - blob = zlib.compress(affected, level=9) - row = (meta['cveId'], meta.get('state', 'UNKNOWN'), meta.get('dateUpdated'), blob) - digest = hashlib.sha256(json.dumps([row[0], row[1], row[2]], separators=(',', ':')).encode() + blob).hexdigest() - return (*row, digest) -def fingerprint(conn): - h = hashlib.sha256() - for cve_id, row_hash in conn.execute('SELECT cve_id,row_hash FROM cve ORDER BY cve_id'): - h.update(cve_id.encode() + b'\x00' + row_hash.encode() + b'\n') - return h.hexdigest() +def canonical(obj): + return json.dumps(obj, ensure_ascii=False, sort_keys=True, separators=(',', ':')) + + +def sha(s): + return hashlib.sha256(s.encode('utf-8')).hexdigest()[:32] + + +def date(value): + return value if isinstance(value, str) and value else None + + +def get_git_revision(path): + return subprocess.check_output(['git', '-C', str(path), 'rev-parse', 'HEAD'], text=True).strip() + + +def input_files(repo): + d = repo / 'data' + if not d.exists(): + return + yield from sorted(d.rglob('*.json')) + + +def upsert_vuln(db, vid, published=None, modified=None, state=None, summary=None): + if not vid or not (CVE.match(vid) or vid.startswith(('GHSA-','OSV-','PYSEC-','RUSTSEC-','GO-','DSA-','USN-'))): + return + db.execute('''INSERT INTO vulnerabilities(id,published,modified,state,summary) VALUES(?,?,?,?,?) + ON CONFLICT(id) DO UPDATE SET + published=COALESCE(excluded.published,vulnerabilities.published), + modified=COALESCE(excluded.modified,vulnerabilities.modified), + state=COALESCE(excluded.state,vulnerabilities.state), + summary=COALESCE(excluded.summary,vulnerabilities.summary)''', + (vid, published, modified, state, summary[:500] if isinstance(summary,str) else None)) + + +def process_cve(db, obj): + m = obj.get('cveMetadata') + if not isinstance(m,dict): + return False + vid=m.get('cveId') + if not isinstance(vid,str) or not CVE.match(vid): + return False + upsert_vuln(db,vid,m.get('datePublished'),m.get('dateUpdated'),m.get('state')) + return True + + +def walk_cpe(node, allow_simple=True): + """Only emit independent leaf matches for simple non-negated OR nodes. + Complex AND/NOT expressions must remain unsupported rather than cause false positives. + """ + if not isinstance(node,dict) or node.get('negate'): + return + if node.get('operator','OR').upper() != 'OR': + return + matches=node.get('cpeMatch') or [] + if isinstance(matches,list): + for match in matches: + if isinstance(match,dict): + yield match + for child in node.get('children') or []: + yield from walk_cpe(child) + + +def process_nvd(db,obj): + cve=obj.get('cve') + if not isinstance(cve,dict): + return False + vid=cve.get('id') + if not isinstance(vid,str) or not CVE.match(vid): + return False + upsert_vuln(db,vid,cve.get('published'),cve.get('lastModified'),cve.get('vulnStatus')) + db.execute('DELETE FROM cpe_matches WHERE vulnerability_id=? AND source=?',(vid,'vuln-nvd')) + for node in cve.get('configurations') or []: + # NVD root configurations may contain nodes; a configuration with several + # nodes can be an AND and is not safe to flatten. + if not isinstance(node,dict) or node.get('operator','OR').upper() != 'OR' or node.get('negate'): + continue + for n in node.get('nodes') or []: + for match in walk_cpe(n): + criteria=match.get('criteria') + if not isinstance(criteria,str): + continue + vals=(criteria, int(bool(match.get('vulnerable',False))), + match.get('versionStartIncluding'), match.get('versionStartExcluding'), + match.get('versionEndIncluding'),match.get('versionEndExcluding')) + key=sha(canonical((vid,vals))) + db.execute('''INSERT OR REPLACE INTO cpe_matches + (id,vulnerability_id,criteria,vulnerable,version_start_including,version_start_excluding,version_end_including,version_end_excluding,source) + VALUES(?,?,?,?,?,?,?,?,?)''',(key,vid,*vals,'vuln-nvd')) + return True + + +def process_osv(db,obj): + vid=obj.get('id') + if not isinstance(vid,str) or not vid: + return False + # Only capture OSV ids with a known prefix or valid CVE. + if not (CVE.match(vid) or vid.startswith(('GHSA-','OSV-','PYSEC-','RUSTSEC-','GO-','DSA-','USN-'))): + return False + upsert_vuln(db,vid,date(obj.get('published')),date(obj.get('modified')), + 'WITHDRAWN' if obj.get('withdrawn') else None,obj.get('summary')) + db.execute('DELETE FROM affected_packages WHERE vulnerability_id=? AND source=?',(vid,'vuln-osv')) + db.execute('DELETE FROM aliases WHERE vulnerability_id=?',(vid,)) + for a in obj.get('aliases') or []: + if isinstance(a,str) and a: + db.execute('INSERT OR IGNORE INTO aliases(alias,vulnerability_id) VALUES(?,?)',(a,vid)) + for i,a in enumerate(obj.get('affected') or []): + if not isinstance(a,dict): + continue + pkg=a.get('package') or {} + if not isinstance(pkg,dict): + continue + eco=pkg.get('ecosystem'); name=pkg.get('name') + ranges=a.get('ranges') or [] + versions=a.get('versions') or [] + if not isinstance(ranges,list) or not isinstance(versions,list): + continue + if not isinstance(eco,str) or not isinstance(name,str): + continue + # Preserve version/range event semantics; no lexicographic comparisons. + key=sha(canonical((vid,i,eco,name,ranges,versions))) + db.execute('INSERT OR REPLACE INTO affected_packages VALUES(?,?,?,?,?,?,?)', + (key,vid,eco,name,canonical(ranges),canonical(versions),'vuln-osv')) + return True + + +def stream_source(db,name,path): + count=0; bad=0 + for file in input_files(path): + try: + with file.open('r',encoding='utf-8') as f: + obj=json.load(f) + if name=='vuln-cve': + accepted=process_cve(db,obj) + elif name=='vuln-nvd': + accepted=process_nvd(db,obj) + elif name=='vuln-osv': + accepted=process_osv(db,obj) + else: + accepted=False + count+=int(accepted) + except (ValueError,UnicodeError,TypeError,sqlite3.Error) as exc: + bad+=1 + if bad<=5: + print(f'INVALID {name} {file}: {exc}',flush=True) + if bad: + raise RuntimeError(f'{name}: {bad} invalid records; database not published') + return count + + +def dump_table(db,table): + # memory efficient cursor; a canonical row object for deltas + cols=[r[1] for r in db.execute(f'PRAGMA table_info({table})')] + for row in db.execute(f'SELECT * FROM {table} ORDER BY 1'): + yield row[0],dict(zip(cols,row)) + + +def make_delta(old_path,new_db,path): + old=sqlite3.connect(old_path) + with path.open('w',encoding='utf-8') as f: + for table in FILES: + prev=dump_table(old,table); curr=dump_table(new_db,table) + a=next(prev,None); b=next(curr,None) + while a is not None or b is not None: + if a is not None and (b is None or a[0]"$CACHE_ROOT/.build.lock" +/usr/bin/flock -w 1800 9 || { echo 'Another builder run in progress' >&2; exit 1; } +export GIT_CONFIG_COUNT=1 GIT_CONFIG_KEY_0=http.extraHeader +GIT_CONFIG_VALUE_0="Authorization: Basic $(printf 'oauth2:%s' "$GITEA_JOB_TOKEN" | base64 | tr -d '\n')" +export GIT_CONFIG_VALUE_0 +# Configurable list; extra feeds tracked in manifest but not yet indexed. +SOURCES="${VULN_SOURCES:-vuln-cve vuln-nvd vuln-osv vuln-cpe vuln-exploits vuln-microsoft vuln-redhat vuln-oracle vuln-bsi vuln-debian vuln-ubuntu vuln-alpine vuln-suse vuln-suse-vex vuln-almalinux vuln-rocky}" +for name in $SOURCES; do + [[ "$name" =~ ^vuln-[a-z0-9-]+$ ]] || { echo "Invalid source name: $name" >&2; exit 1; } + repo="$CACHE_ROOT/sources/$name" + url="${GITEA_BASE_URL%/}/VULN/$name.git" + if [ ! -d "$repo/.git" ]; then + echo "Cloning $name" + git clone --quiet --depth 1 --single-branch --branch main "$url" "$repo" + else + echo "Updating $name" + git -C "$repo" fetch --quiet --depth 1 origin main + git -C "$repo" reset --hard -q FETCH_HEAD + git -C "$repo" clean -fdq + fi +done +unset GIT_CONFIG_COUNT GIT_CONFIG_KEY_0 GIT_CONFIG_VALUE_0 +cd "$(dirname "$0")/.." +python3 scripts/build.py --sources "$CACHE_ROOT/sources" --output "$CACHE_ROOT/releases" +echo 'Release files:' +ls -lh "$CACHE_ROOT/releases"/*.zst "$CACHE_ROOT/releases"/manifest.json diff --git a/sql/schema.sql b/sql/schema.sql index 0ea20e4..5ab2592 100644 --- a/sql/schema.sql +++ b/sql/schema.sql @@ -1,9 +1,17 @@ --- CVE-Metadatenindex. Keine CPE-/Paket-Matching-Engine! -CREATE TABLE cve ( - cve_id TEXT PRIMARY KEY, - state TEXT NOT NULL, - date_updated TEXT, - affected BLOB NOT NULL, -- zlib-komprimierte JSON-CNA-affected-Liste - row_hash TEXT NOT NULL -) WITHOUT ROWID; -CREATE INDEX idx_cve_state ON cve(state); +PRAGMA foreign_keys=ON; +CREATE TABLE IF NOT EXISTS meta (key TEXT PRIMARY KEY,value TEXT NOT NULL); +CREATE TABLE IF NOT EXISTS sources (name TEXT PRIMARY KEY, revision TEXT NOT NULL, indexed INTEGER NOT NULL DEFAULT 0, documents INTEGER NOT NULL DEFAULT 0); +CREATE TABLE IF NOT EXISTS vulnerabilities (id TEXT PRIMARY KEY, published TEXT, modified TEXT, state TEXT, summary TEXT); +CREATE TABLE IF NOT EXISTS aliases (alias TEXT NOT NULL, vulnerability_id TEXT NOT NULL REFERENCES vulnerabilities(id) ON DELETE CASCADE, PRIMARY KEY(alias,vulnerability_id)); +CREATE INDEX IF NOT EXISTS idx_aliases_vuln ON aliases(vulnerability_id); +CREATE TABLE IF NOT EXISTS affected_packages ( + id TEXT PRIMARY KEY, vulnerability_id TEXT NOT NULL REFERENCES vulnerabilities(id) ON DELETE CASCADE, + ecosystem TEXT, package_name TEXT, ranges_json TEXT NOT NULL, versions_json TEXT NOT NULL, source TEXT NOT NULL); +CREATE INDEX IF NOT EXISTS idx_affected_vuln ON affected_packages(vulnerability_id); +CREATE INDEX IF NOT EXISTS idx_affected_package ON affected_packages(ecosystem,package_name); +CREATE TABLE IF NOT EXISTS cpe_matches ( + id TEXT PRIMARY KEY, vulnerability_id TEXT NOT NULL REFERENCES vulnerabilities(id) ON DELETE CASCADE, + criteria TEXT, vulnerable INTEGER, version_start_including TEXT, version_start_excluding TEXT, + version_end_including TEXT, version_end_excluding TEXT, source TEXT NOT NULL); +CREATE INDEX IF NOT EXISTS idx_cpe_vuln ON cpe_matches(vulnerability_id); +-- Conditions expressed as AND, OR and negate are deliberately NOT converted into independent vulnerability assertions. diff --git a/tests/test_build.py b/tests/test_build.py new file mode 100644 index 0000000..328aae4 --- /dev/null +++ b/tests/test_build.py @@ -0,0 +1,34 @@ +import json +import sqlite3 +import sys +import tempfile +import unittest +from pathlib import Path +sys.path.insert(0,str(Path(__file__).resolve().parents[1]/'scripts')) +from build import process_cve,process_nvd,process_osv,walk_cpe + +class TestBuild(unittest.TestCase): + def setUp(self): + self.conn=sqlite3.connect(':memory:') + self.conn.executescript((Path(__file__).resolve().parents[1]/'sql/schema.sql').read_text()) + def tearDown(self): + self.conn.close() + def test_cve(self): + self.assertTrue(process_cve(self.conn,{'cveMetadata':{'cveId':'CVE-2026-12345','state':'PUBLISHED'}})) + self.assertEqual(self.conn.execute('select state from vulnerabilities').fetchone()[0],'PUBLISHED') + def test_osv_range_preserved(self): + self.assertTrue(process_osv(self.conn,{'id':'GHSA-abcd-efgh-ijkl','aliases':['CVE-2026-12345'], + 'affected':[{'package':{'ecosystem':'PyPI','name':'foo'},'ranges':[{'type':'ECOSYSTEM','events':[{'introduced':'0'},{'fixed':'2.0'}]}]}]})) + row=self.conn.execute('select ranges_json from affected_packages').fetchone()[0] + self.assertIn('fixed',row) + self.assertEqual(self.conn.execute('select count(*) from aliases').fetchone()[0],1) + def test_nvd_ignores_and(self): + self.assertTrue(process_nvd(self.conn,{'cve':{'id':'CVE-2026-12345','configurations':[ + {'nodes':[{'operator':'AND','cpeMatch':[{'criteria':'cpe:2.3:a:foo'}]}]}]}})) + self.assertEqual(self.conn.execute('select count(*) from cpe_matches').fetchone()[0],0) + def test_simple_nvd(self): + self.assertTrue(process_nvd(self.conn,{'cve':{'id':'CVE-2026-12345','configurations':[ + {'nodes':[{'operator':'OR','cpeMatch':[{'criteria':'cpe:2.3:a:vendor:product:*:*:*:*:*:*:*:*','vulnerable':True}]}]}]}})) + self.assertEqual(self.conn.execute('select count(*) from cpe_matches').fetchone()[0],1) + +if __name__=='__main__': unittest.main()