import hashlib, json, pathlib, sys
from datetime import datetime
manifest, target = map(pathlib.Path, sys.argv[1:3])
entries = json.loads(manifest.read_text(encoding='utf-8'))
if not isinstance(entries, list) or not 1 <= len(entries) <= 3:
raise ValueError('Supply 1–3 saved pages; this helper makes no requests')
action = 'wechat_search.v2.fetch_search'
items, sources, blobs, seen, request_ids = [], [], [], set(), set()
base, expected, previous_time, stop = None, None, None, 'saved_pages_end'
for n, entry in enumerate(entries, 1):
p = entry['params']
if entry['action'] != action or p.get('business_type') != 'article' or p.get('raw') is not False:
raise ValueError('Use article with explicit raw=false')
if set(p) - {'keyword','business_type','publish_time','sort','offset','cursor','raw'}:
raise ValueError('Unexpected parameter')
if p.get('publish_time') != 'week' or p.get('sort') != 'latest' or type(p.get('offset')) is not int or p['offset'] != 0:
raise ValueError('This workflow uses week/latest/offset=0 throughout')
if not isinstance(p.get('keyword'), str) or not 1 <= len(p['keyword'].strip()) <= 100:
raise ValueError('Invalid keyword')
core = {k:v for k,v in p.items() if k != 'cursor'}
if base is None: base = core
if core != base or (n == 1 and p.get('cursor') not in (None,'')) or (n > 1 and p.get('cursor') != expected):
raise ValueError('Query changed or cursor chain broken')
ts = datetime.fromisoformat(entry['collected_at'].replace('Z','+00:00'))
if ts.tzinfo is None or (previous_time is not None and ts < previous_time):
raise ValueError('Use chronological timezone-aware collection times')
previous_time = ts
status = entry['http_status']
if type(status) is not int or not 100 <= status <= 599: raise ValueError('Invalid HTTP status')
raw = (manifest.parent / entry['response_file']).read_bytes()
body = json.loads(raw)
if not isinstance(body, dict): raise ValueError('Expected gateway object')
error = body.get('error')
ok = status == 200 and body.get('status') == 'succeeded' and not error
if not ok and not (isinstance(error,dict) and isinstance(error.get('code'),str) and error['code'].strip()):
raise ValueError('Malformed or unrecognized failure envelope')
rid = body.get('request_id') or (error.get('request_id') if isinstance(error,dict) else None)
if ok and (body.get('action') != action or not isinstance(rid,str) or not rid or rid in request_ids):
raise ValueError('Mismatched action or missing/repeated request ID')
if ok: request_ids.add(rid)
raw_name = 'raw-%03d.json' % n
record = {'page':n,'request_id':rid,'params':p,'collected_at':entry['collected_at'],
'http_status':status,'error_code':error.get('code') if isinstance(error,dict) else None,'raw_file':raw_name,'sha256':hashlib.sha256(raw).hexdigest()}
blobs.append((raw_name, raw)); sources.append(record)
if not ok:
stop = 'gateway_failed'
else:
data = body.get('data')
if not isinstance(data,dict) or not isinstance(data.get('items'),list) or not all(isinstance(x,dict) for x in data['items']):
raise ValueError('Expected data.items object array')
if data.get('keyword') != p['keyword'].strip(): raise ValueError('Keyword echo mismatch')
vertical = data.get('business_type')
if (isinstance(vertical,str) and vertical != 'article') or vertical == 33554499:
raise ValueError('Response explicitly reports a non-article vertical')
record.update({k:data.get(k) for k in ('count','total','business_type','no_more','continue_flag','cursor','offset')})
for i,item in enumerate(data['items']):
items.append({'page':n,'item_index':i,'request_id':rid,'item':item})
more, cursor = data.get('continue_flag'), data.get('cursor')
if more is False or (type(more) is int and more == 0): stop = 'provider_end'
elif not (more is True or (type(more) is int and more == 1)): stop = 'unsupported_more_flag'
elif not data['items']: stop = 'empty_page_with_more'
elif not isinstance(cursor,str) or not cursor: stop = 'missing_cursor'
elif cursor in seen: stop = 'repeated_cursor'
else:
seen.add(cursor); expected = cursor; stop = 'saved_pages_end'
if stop != 'saved_pages_end' and n != len(entries):
raise ValueError('Manifest continues after a terminal or invalid page')
if stop == 'saved_pages_end' and len(entries) == 3: stop = 'helper_page_cap'
report = {'query':base,'pages_saved':len(entries),'item_observations':len(items),
'stop_reason':stop,'complete_search':False,'deduplicated_articles':False,'sources':sources}
target.mkdir(parents=True,exist_ok=False)
for name,raw in blobs: (target/name).write_bytes(raw)
(target/'items.jsonl').write_text(''.join(json.dumps(x,ensure_ascii=False)+'\n' for x in items),encoding='utf-8')
(target/'coverage.json').write_text(json.dumps(report,ensure_ascii=False,indent=2),encoding='utf-8')
if stop == 'saved_pages_end':
(target/'next-request.json').write_text(json.dumps({'action':action,'params':{**base,'cursor':expected}},ensure_ascii=False,indent=2),encoding='utf-8')
print(json.dumps({'pages_saved':len(entries),'item_observations':len(items),'stop_reason':stop}))