#!/usr/bin/env python3
"""Mirror upstream Alpine secdb JSON files, preserving the upstream bytes."""
import argparse
import hashlib
from html.parser import HTMLParser
import json
import os
from pathlib import Path
import re
import tempfile
import time
from urllib.error import HTTPError, URLError
from urllib.parse import urljoin, urlsplit
from urllib.request import Request, urlopen

BASE='https://secdb.alpinelinux.org/'
RELEASE=re.compile(r'^v3\.(\d+)/$')
BRANCH_FILES={'main.json','community.json'}

class Links(HTMLParser):
    def __init__(self):
        super().__init__(); self.hrefs=[]
    def handle_starttag(self,tag,attrs):
        if tag.lower()=='a':
            for k,v in attrs:
                if k.lower()=='href' and v is not None: self.hrefs.append(v)

def fetch(url, cache=None, *, attempts=5):
    """Returns (bytes | None for HTTP 304, headers)."""
    if not url.startswith(BASE): raise ValueError('Untrusted origin: '+url)
    headers={'User-Agent':'vuln-alpine-gitea-mirror/1.0','Accept':'application/json, text/html;q=0.8, */*;q=0.5'}
    if cache:
        if cache.get('etag'): headers['If-None-Match']=cache['etag']
        if cache.get('last_modified'): headers['If-Modified-Since']=cache['last_modified']
    for i in range(attempts):
        try:
            with urlopen(Request(url,headers=headers),timeout=75) as r:
                data=r.read(15*1024*1024+1)
                if len(data)>15*1024*1024: raise ValueError('Source file too large: '+url)
                return data, {'etag':r.headers.get('ETag',''), 'last_modified':r.headers.get('Last-Modified','')}
        except HTTPError as e:
            if e.code==304: return None,cache or {}
            if e.code not in (408,429,500,502,503,504) or i==attempts-1: raise
            retry=e.headers.get('Retry-After','')
            delay=min(int(retry),120) if retry.isdecimal() else min(2**i*3,60)
        except (URLError, TimeoutError, ConnectionError):
            if i==attempts-1: raise
            delay=min(2**i*3,60)
        time.sleep(delay)
    raise RuntimeError('Unreachable')

def links(blob):
    parser=Links(); parser.feed(blob.decode('utf-8')); return parser.hrefs

def release_links(root_html,min_release):
    found=[]
    for href in links(root_html):
        m=RELEASE.fullmatch(href)
        if m and int(m.group(1))>=min_release: found.append(href[:-1])
        elif href=='edge/': found.append('edge')
    return sorted(set(found),key=lambda x:(x=='edge',int(x.split('.')[1]) if x.startswith('v3.') else 999))

def available_files(branch_html):
    return sorted(set(h for h in links(branch_html) if h in BRANCH_FILES))

def validate_json(blob):
    obj=json.loads(blob)
    if not isinstance(obj,dict) or not isinstance(obj.get('packages'),list):
        raise ValueError('Unexpected Alpine secdb JSON schema: expected packages array')
    return len(obj['packages'])

def atomic_write(path:Path,data:bytes):
    path.parent.mkdir(parents=True,exist_ok=True)
    fd,tmp=tempfile.mkstemp(prefix='.incoming-',dir=str(path.parent))
    try:
        with os.fdopen(fd,'wb') as out:
            out.write(data);out.flush();os.fsync(out.fileno())
        os.replace(tmp,path)
    finally:
        if os.path.exists(tmp): os.unlink(tmp)

def run(args):
    repo=Path(args.repo).resolve()
    data_root=repo/'data'/'secdb'
    state_dir=repo/'state';state_dir.mkdir(parents=True,exist_ok=True)
    cachefile=Path(args.http_cache).resolve();cachefile.parent.mkdir(parents=True,exist_ok=True)
    oldcache=json.loads(cachefile.read_text()) if cachefile.exists() else {}
    newcache=dict(oldcache)
    root_html,_=fetch(BASE)
    branches=release_links(root_html,args.min_release)
    if not branches: raise RuntimeError('No Alpine release branches discovered')
    oldmanifest=state_dir/'manifest.json'
    old=json.loads(oldmanifest.read_text()) if oldmanifest.exists() else {}
    oldfiles=old.get('files',{})
    records={}; updates=0; skipped=0
    for branch in branches:
        branch_url=urljoin(BASE,branch+'/')
        branch_html,_=fetch(branch_url)
        fnames=available_files(branch_html)
        if not fnames:
            raise RuntimeError('No secdb JSON files for '+branch)
        for filename in fnames:
            rel=f'{branch}/{filename}'
            url=urljoin(branch_url,filename)
            target=data_root/rel
            cache=oldcache.get(rel) if target.is_file() else None
            blob,meta=fetch(url,cache)
            if blob is None:
                if not target.is_file(): raise RuntimeError('304 without local file: '+rel)
                digest=hashlib.sha256(target.read_bytes()).hexdigest()
                count=oldfiles.get(rel,{}).get('packages')
                if count is None: count=validate_json(target.read_bytes())
                skipped+=1
            else:
                count=validate_json(blob)
                digest=hashlib.sha256(blob).hexdigest()
                if not target.is_file() or hashlib.sha256(target.read_bytes()).hexdigest()!=digest:
                    atomic_write(target,blob);updates+=1
                else: skipped+=1
            records[rel]={'sha256':digest,'packages':count,'source':url}
            newcache[rel]=meta
            print(f'{rel}: {"downloaded/changed" if blob is not None else "not modified"}; packages={count}',flush=True)
    # Historical branches are retained, even if Alpine stops listing them.
    # This prevents a temporary index regression from silently deleting data.
    for rel,entry in oldfiles.items():
        if rel not in records and (data_root/rel).is_file(): records[rel]=entry
    newmanifest={'source':BASE,'min_release':args.min_release,'files':dict(sorted(records.items()))}
    encoded=(json.dumps(newmanifest,indent=2,sort_keys=True,ensure_ascii=False)+'\n').encode()
    if not oldmanifest.exists() or oldmanifest.read_bytes()!=encoded: atomic_write(oldmanifest,encoded)
    cached=json.dumps(newcache,indent=2,sort_keys=True).encode()
    atomic_write(cachefile,cached)
    print(f'OK: {len(records)} files, {updates} changed, {skipped} unchanged',flush=True)

def main():
    ap=argparse.ArgumentParser()
    ap.add_argument('--repo',default='.')
    ap.add_argument('--http-cache',required=True,help='Persistent path outside Git')
    ap.add_argument('--min-release',type=int,default=5,help='v3.5 onwards cover 2017 historical data')
    args=ap.parse_args();run(args)
if __name__=='__main__': main()