141 lines
6.3 KiB
Python
141 lines
6.3 KiB
Python
#!/usr/bin/env python3
|
|
"""Mirror upstream Alpine secdb JSON files, preserving the upstream bytes."""
|
|
import argparse
|
|
import hashlib
|
|
from html.parser import HTMLParser
|
|
import json
|
|
import os
|
|
from pathlib import Path
|
|
import re
|
|
import tempfile
|
|
import time
|
|
from urllib.error import HTTPError, URLError
|
|
from urllib.parse import urljoin, urlsplit
|
|
from urllib.request import Request, urlopen
|
|
|
|
BASE='https://secdb.alpinelinux.org/'
|
|
RELEASE=re.compile(r'^v3\.(\d+)/$')
|
|
BRANCH_FILES={'main.json','community.json'}
|
|
|
|
class Links(HTMLParser):
|
|
def __init__(self):
|
|
super().__init__(); self.hrefs=[]
|
|
def handle_starttag(self,tag,attrs):
|
|
if tag.lower()=='a':
|
|
for k,v in attrs:
|
|
if k.lower()=='href' and v is not None: self.hrefs.append(v)
|
|
|
|
def fetch(url, cache=None, *, attempts=5):
|
|
"""Returns (bytes | None for HTTP 304, headers)."""
|
|
if not url.startswith(BASE): raise ValueError('Untrusted origin: '+url)
|
|
headers={'User-Agent':'vuln-alpine-gitea-mirror/1.0','Accept':'application/json, text/html;q=0.8, */*;q=0.5'}
|
|
if cache:
|
|
if cache.get('etag'): headers['If-None-Match']=cache['etag']
|
|
if cache.get('last_modified'): headers['If-Modified-Since']=cache['last_modified']
|
|
for i in range(attempts):
|
|
try:
|
|
with urlopen(Request(url,headers=headers),timeout=75) as r:
|
|
data=r.read(15*1024*1024+1)
|
|
if len(data)>15*1024*1024: raise ValueError('Source file too large: '+url)
|
|
return data, {'etag':r.headers.get('ETag',''), 'last_modified':r.headers.get('Last-Modified','')}
|
|
except HTTPError as e:
|
|
if e.code==304: return None,cache or {}
|
|
if e.code not in (408,429,500,502,503,504) or i==attempts-1: raise
|
|
retry=e.headers.get('Retry-After','')
|
|
delay=min(int(retry),120) if retry.isdecimal() else min(2**i*3,60)
|
|
except (URLError, TimeoutError, ConnectionError):
|
|
if i==attempts-1: raise
|
|
delay=min(2**i*3,60)
|
|
time.sleep(delay)
|
|
raise RuntimeError('Unreachable')
|
|
|
|
def links(blob):
|
|
parser=Links(); parser.feed(blob.decode('utf-8')); return parser.hrefs
|
|
|
|
def release_links(root_html,min_release):
|
|
found=[]
|
|
for href in links(root_html):
|
|
m=RELEASE.fullmatch(href)
|
|
if m and int(m.group(1))>=min_release: found.append(href[:-1])
|
|
elif href=='edge/': found.append('edge')
|
|
return sorted(set(found),key=lambda x:(x=='edge',int(x.split('.')[1]) if x.startswith('v3.') else 999))
|
|
|
|
def available_files(branch_html):
|
|
return sorted(set(h for h in links(branch_html) if h in BRANCH_FILES))
|
|
|
|
def validate_json(blob):
|
|
obj=json.loads(blob)
|
|
if not isinstance(obj,dict) or not isinstance(obj.get('packages'),list):
|
|
raise ValueError('Unexpected Alpine secdb JSON schema: expected packages array')
|
|
return len(obj['packages'])
|
|
|
|
def atomic_write(path:Path,data:bytes):
|
|
path.parent.mkdir(parents=True,exist_ok=True)
|
|
fd,tmp=tempfile.mkstemp(prefix='.incoming-',dir=str(path.parent))
|
|
try:
|
|
with os.fdopen(fd,'wb') as out:
|
|
out.write(data);out.flush();os.fsync(out.fileno())
|
|
os.replace(tmp,path)
|
|
finally:
|
|
if os.path.exists(tmp): os.unlink(tmp)
|
|
|
|
def run(args):
|
|
repo=Path(args.repo).resolve()
|
|
data_root=repo/'data'/'secdb'
|
|
state_dir=repo/'state';state_dir.mkdir(parents=True,exist_ok=True)
|
|
cachefile=Path(args.http_cache).resolve();cachefile.parent.mkdir(parents=True,exist_ok=True)
|
|
oldcache=json.loads(cachefile.read_text()) if cachefile.exists() else {}
|
|
newcache=dict(oldcache)
|
|
root_html,_=fetch(BASE)
|
|
branches=release_links(root_html,args.min_release)
|
|
if not branches: raise RuntimeError('No Alpine release branches discovered')
|
|
oldmanifest=state_dir/'manifest.json'
|
|
old=json.loads(oldmanifest.read_text()) if oldmanifest.exists() else {}
|
|
oldfiles=old.get('files',{})
|
|
records={}; updates=0; skipped=0
|
|
for branch in branches:
|
|
branch_url=urljoin(BASE,branch+'/')
|
|
branch_html,_=fetch(branch_url)
|
|
fnames=available_files(branch_html)
|
|
if not fnames:
|
|
raise RuntimeError('No secdb JSON files for '+branch)
|
|
for filename in fnames:
|
|
rel=f'{branch}/{filename}'
|
|
url=urljoin(branch_url,filename)
|
|
target=data_root/rel
|
|
cache=oldcache.get(rel) if target.is_file() else None
|
|
blob,meta=fetch(url,cache)
|
|
if blob is None:
|
|
if not target.is_file(): raise RuntimeError('304 without local file: '+rel)
|
|
digest=hashlib.sha256(target.read_bytes()).hexdigest()
|
|
count=oldfiles.get(rel,{}).get('packages')
|
|
if count is None: count=validate_json(target.read_bytes())
|
|
skipped+=1
|
|
else:
|
|
count=validate_json(blob)
|
|
digest=hashlib.sha256(blob).hexdigest()
|
|
if not target.is_file() or hashlib.sha256(target.read_bytes()).hexdigest()!=digest:
|
|
atomic_write(target,blob);updates+=1
|
|
else: skipped+=1
|
|
records[rel]={'sha256':digest,'packages':count,'source':url}
|
|
newcache[rel]=meta
|
|
print(f'{rel}: {"downloaded/changed" if blob is not None else "not modified"}; packages={count}',flush=True)
|
|
# Historical branches are retained, even if Alpine stops listing them.
|
|
# This prevents a temporary index regression from silently deleting data.
|
|
for rel,entry in oldfiles.items():
|
|
if rel not in records and (data_root/rel).is_file(): records[rel]=entry
|
|
newmanifest={'source':BASE,'min_release':args.min_release,'files':dict(sorted(records.items()))}
|
|
encoded=(json.dumps(newmanifest,indent=2,sort_keys=True,ensure_ascii=False)+'\n').encode()
|
|
if not oldmanifest.exists() or oldmanifest.read_bytes()!=encoded: atomic_write(oldmanifest,encoded)
|
|
cached=json.dumps(newcache,indent=2,sort_keys=True).encode()
|
|
atomic_write(cachefile,cached)
|
|
print(f'OK: {len(records)} files, {updates} changed, {skipped} unchanged',flush=True)
|
|
|
|
def main():
|
|
ap=argparse.ArgumentParser()
|
|
ap.add_argument('--repo',default='.')
|
|
ap.add_argument('--http-cache',required=True,help='Persistent path outside Git')
|
|
ap.add_argument('--min-release',type=int,default=5,help='v3.5 onwards cover 2017 historical data')
|
|
args=ap.parse_args();run(args)
|
|
if __name__=='__main__': main()
|