import csv, datetime, hashlib, json, pathlib, re, sys
manifest=pathlib.Path(sys.argv[1]);manifest_bytes=manifest.read_bytes();pages=json.loads(manifest_bytes)
if not isinstance(pages,list) or not 1<=len(pages)<=20: raise ValueError('Use 1..20 saved article-list pages')
ACTION='wechat_mp.v2.fetch_account_articles'
rows,unknown,seen,requests,archives,sources=[],[],{},set(),[],[]
account=None;next_offset=None;last_time=None;used_offsets=set()
def identifier(value):
if type(value) is int and 1<=value<=9007199254740991:return str(value)
if isinstance(value,str) and re.fullmatch(r'[0-9]+',value) and int(value)>0:return str(int(value))
raise ValueError('Use original positive digit strings or safe integers')
for page_index,page in enumerate(pages):
params=page['params'];username=params.get('username')
if not isinstance(username,str) or not re.fullmatch(r'gh_[A-Za-z0-9_]+',username):raise ValueError('Use the canonical account username')
if account is None:account=username
if username!=account or params.get('raw') is not False or type(params.get('item_show_type')) is not int or params['item_show_type']!=0:raise ValueError('Same account, explicit raw=false and article tab 0 required')
if page_index and (not isinstance(next_offset,str) or not next_offset or params.get('offset')!=next_offset):raise ValueError('Saved pages do not follow the previous cursor')
offset=params.get('offset','')
if not isinstance(offset,str) or offset in used_offsets:raise ValueError('Repeated or invalid requested cursor')
used_offsets.add(offset)
time=datetime.datetime.fromisoformat(page['collected_at'].replace('Z','+00:00'))
if time.utcoffset() is None or (last_time is not None and time<last_time):raise ValueError('Use ordered actual collection times with timezone')
last_time=time
if page.get('action')!=ACTION or type(page.get('http_status')) is not int or page['http_status']!=200:raise ValueError('Expected successful recorded article-list request')
raw=(manifest.parent/page['file']).read_bytes();response=json.loads(raw)
rid=response.get('request_id')
if not isinstance(rid,str) or not rid.strip() or rid in requests:raise ValueError('Missing/repeated request ID')
requests.add(rid)
if response.get('status')!='succeeded' or response.get('error') is not None or response.get('action')!=ACTION:raise ValueError('Failed or wrong response')
data=response['data']
if data.get('biz_username')!=account or not isinstance(data.get('articles'),list):raise ValueError('Wrong account or simplified article shape missing')
if type(data.get('is_end')) is not int or data['is_end'] not in (0,1):raise ValueError('Invalid end marker')
next_offset=None if data['is_end']==1 else data.get('next_offset')
if next_offset is not None and (not isinstance(next_offset,str) or not next_offset or next_offset in used_offsets):raise ValueError('Repeated or invalid continuation cursor')
name='raw-'+str(page_index+1).zfill(3)+'.json';digest=hashlib.sha256(raw).hexdigest();archives.append((name,raw));sources.append({'file':name,'sha256':digest,'request_id':rid,'collected_at':page['collected_at']})
for position,article in enumerate(data['articles'],1):
if not isinstance(article,dict):raise ValueError('Unreadable article object')
appid=identifier(article.get('app_msg_id'))
idx=None if article.get('idx') is None else int(identifier(article['idx']))
values={k:article.get(k) for k in ['title','url','create_time']}
if values['title'] is not None and not isinstance(values['title'],str):raise ValueError('Invalid title')
if values['url'] is not None and not isinstance(values['url'],str):raise ValueError('Invalid article URL field')
published=values['create_time']
if published is not None and not ((type(published) is int and 0<=published<=9007199254740991) or (isinstance(published,str) and re.fullmatch(r'[0-9]+',published))):raise ValueError('Invalid raw publication timestamp')
row={'account':account,'app_msg_id':appid,'idx':idx,'placement':'unknown' if idx is None else 'headline' if idx==1 else 'secondary',
**values,'request_id':rid,'collected_at':page['collected_at'],'source_file':name,'source_sha256':digest,'item_position_on_page':position}
if idx is None:unknown.append(row);continue
key=(appid,idx)
if key in seen:
if any(seen[key][k]!=row[k] for k in values):raise ValueError('Conflicting duplicate article metadata; split observations for review')
continue
seen[key]=row;rows.append(row)
out=pathlib.Path(sys.argv[2]);out.mkdir()
for name,raw in archives:(out/name).write_bytes(raw)
(out/'input-manifest.json').write_bytes(manifest_bytes)
for name,value in [('articles.json',rows),('unknown-position.json',unknown),('coverage.json',{'account':account,
'known_position_articles':len(rows),'headlines':sum(r['idx']==1 for r in rows),'secondary_articles':sum(r['idx']>1 for r in rows),
'unknown_position_observations':len(unknown),'sources':sources,'manifest_sha256':hashlib.sha256(manifest_bytes).hexdigest(),
'observed_groups':[{'app_msg_id':appid,'positions':sorted(r['idx'] for r in rows if r['app_msg_id']==appid),'complete_group_verified':False} for appid in sorted({r['app_msg_id'] for r in rows})],
'complete_history':False,'executed_calls':0})]:
(out/name).write_text(json.dumps(value,ensure_ascii=False,indent=2),encoding='utf-8')
fields=['account','app_msg_id','idx','placement','title','url','create_time','collected_at','request_id','source_file','source_sha256']
for filename,selected in [('headlines.csv',[r for r in rows if r['idx']==1]),('secondary.csv',[r for r in rows if r['idx']>1])]:
with (out/filename).open('w',encoding='utf-8-sig',newline='') as f:
w=csv.DictWriter(f,fieldnames=fields);w.writeheader()
for row in selected:
cells={k:'' if row[k] is None else str(row[k]) for k in fields};w.writerow({k:"'"+v if v.lstrip().startswith(('=','+','-','@')) else v for k,v in cells.items()})
print(json.dumps({'headlines':sum(r['idx']==1 for r in rows),'secondary':sum(r['idx']>1 for r in rows),'unknown':len(unknown),'executed_calls':0}))