diff --git a/scripts/build.py b/scripts/build.py index 6763520..e2fb518 100644 --- a/scripts/build.py +++ b/scripts/build.py @@ -212,11 +212,41 @@ def compress(src,out): subprocess.run(['zstd','-q','-f','-T1','-19',str(src),'-o',str(out)],check=True) +def build_signature(source_revs, schema_path): + """Fingerprint inputs that can change the generated data, not wall-clock time.""" + scripts_dir = Path(__file__).resolve().parent + h = hashlib.sha256() + # Include every Python importer/helper alongside the builder, as well as SQL schema. + inputs = sorted(scripts_dir.glob('*.py')) + inputs.append(schema_path.resolve()) + for path in inputs: + if not path.is_file(): + raise RuntimeError(f'Missing build input: {path}') + logical_name = 'schema.sql' if path == schema_path.resolve() else f'scripts/{path.name}' + h.update(logical_name.encode('utf-8') + b'\x00') + h.update(hashlib.sha256(path.read_bytes()).digest()) + h.update(canonical(source_revs).encode('utf-8')) + return h.hexdigest() + + +def existing_release_ok(output, manifest): + """Only skip if current release artifacts exist and match recorded hash.""" + dbfile, packed = output / 'master.db', output / 'master.db.zst' + if not dbfile.is_file() or not packed.is_file() or not manifest.get('master_sha256'): + return False + h = hashlib.sha256() + with packed.open('rb') as stream: + for block in iter(lambda: stream.read(1024 * 1024), b''): + h.update(block) + return h.hexdigest() == manifest['master_sha256'] + + def main(): ap=argparse.ArgumentParser() ap.add_argument('--sources',type=Path,required=True) ap.add_argument('--output',type=Path,required=True) ap.add_argument('--schema',type=Path,default=Path(__file__).resolve().parents[1]/'sql/schema.sql') + ap.add_argument('--force', action='store_true', help='Force rebuild even when input signature is unchanged') args=ap.parse_args() args.output.mkdir(parents=True,exist_ok=True) active=[(p.name,p) for p in sorted(args.sources.iterdir()) if p.is_dir() and (p/'.git').exists()] @@ -227,7 +257,22 @@ def main(): if name not in [n for n,_ in active]: raise SystemExit(f'Missing core source: {name}') source_revs={name:get_git_revision(path) for name,path in active} - old_db=args.output/'master.db' + signature = build_signature(source_revs, args.schema) + manifest_path = args.output / 'manifest.json' + previous_manifest = None + if manifest_path.is_file(): + try: + previous_manifest = json.loads(manifest_path.read_text(encoding='utf-8')) + except (ValueError, UnicodeError) as exc: + raise RuntimeError(f'Invalid existing manifest, refusing to overwrite: {exc}') from exc + if (not args.force and previous_manifest + and previous_manifest.get('build_signature') == signature + and existing_release_ok(args.output, previous_manifest)): + print(f"NO_CHANGES: version={previous_manifest['version']} inputs unchanged; master and delta retained", flush=True) + return + if not args.force and previous_manifest and previous_manifest.get('build_signature') == signature: + print('WARNING: unchanged inputs but missing/invalid release artifact; rebuilding', flush=True) + old_db=args.output/'master.db' with tempfile.TemporaryDirectory(prefix='vuln-build-',dir=args.output) as td: tmp=Path(td); new=tmp/'master.db'; db=sqlite3.connect(new) db.executescript(args.schema.read_text(encoding='utf-8')) @@ -253,6 +298,7 @@ def main(): assert db.execute('PRAGMA integrity_check').fetchone()[0]=='ok' db.close() manifest={'version':version,'schema_version':1,'previous':old_version,'source_commits':source_revs, + 'build_signature':signature, 'indexed_documents':stats,'format':'sqlite+zstd','delta_format':'jsonl+zstd', 'note':'Only CVE/NVD/OSV indexed in v1; no complete version-matching engine'} compress(new,tmp/'master.db.zst')