import collections,csv,hashlib,json,pathlib,sys
from datetime import datetime
from urllib.parse import parse_qs,urlsplit
manifest,out=map(pathlib.Path,sys.argv[1:3]);manifest_bytes=manifest.read_bytes();entries=json.loads(manifest_bytes)
if not isinstance(entries,list) or not 1<=len(entries)<=50:raise ValueError('Supply 1-50 saved observations')
def article_url(value):
if not isinstance(value,str):raise ValueError('Expected requested article URL')
u=urlsplit(value)
if u.scheme!='https' or u.hostname!='mp.weixin.qq.com' or u.username is not None or u.password is not None or u.port not in (None,443):raise ValueError('Expected HTTPS WeChat article')
if u.path=='/s':
q=parse_qs(u.query,keep_blank_values=True)
if not all(len(q.get(k,[]))==1 and q[k][0].strip() for k in ('__biz','mid','idx')):raise ValueError('Long URL lacks article identity')
if not q['mid'][0].isascii() or not q['mid'][0].isdigit() or not q['idx'][0].isascii() or not q['idx'][0].isdigit() or int(q['idx'][0])<1:raise ValueError('Bad article position')
elif not u.path.startswith('/s/') or not u.path[3:] or '/' in u.path[3:]:raise ValueError('Expected article path')
return value
def field(content,key):
if key not in content:return None,'absent'
value=content[key]
if value is None:return None,'null'
if not isinstance(value,str):raise ValueError('Location must be a string or missing')
return value,('reported' if value.strip() else 'empty')
rows=[];raws=[];seen_requests=set();seen_urls=set();countries=collections.Counter();provinces=collections.Counter()
for n,entry in enumerate(entries,1):
url=article_url(entry['url']);stamp=datetime.fromisoformat(entry['collected_at'].replace('Z','+00:00'))
if stamp.utcoffset() is None:raise ValueError('Collection time needs a timezone')
if url in seen_urls:raise ValueError('Choose one observation per exact requested URL')
seen_urls.add(url);path=manifest.parent/entry['file'];raw=path.read_bytes();r=json.loads(raw)
if not isinstance(r,dict) or r.get('status')!='succeeded' or r.get('action')!='wechat_mp.v2.fetch_article_detail_h5' or r.get('error') is not None:raise ValueError('Expected successful article detail')
rid=r.get('request_id');data=r.get('data')
if not isinstance(rid,str) or not rid.strip() or rid in seen_requests or not isinstance(data,dict) or not isinstance(data.get('content'),dict):raise ValueError('Missing/repeated provenance or content')
seen_requests.add(rid)
if data.get('url') is not None and data['url']!=url:raise ValueError('Response request URL differs')
content=data['content'];country,cs=field(content,'country_name');province,ps=field(content,'province_name')
if cs=='reported':countries[country]+=1
if ps=='reported':provinces[(country if cs=='reported' else None,province)]+=1
row={'requested_url':url,'collected_at':entry['collected_at'],'country_name':country,'country_state':cs,'province_name':province,'province_state':ps,'request_id':rid,'request_url_echo_verified':data.get('url')==url,'source_sha256':hashlib.sha256(raw).hexdigest(),'source_file':str(path.resolve()),'archive_file':'raw-%02d.json'%n}
rows.append(row);raws.append(raw)
def safe(v):
if v is None:return ''
s=str(v)
return "'"+s if s.lstrip().startswith(('=','+','-','@')) or s.startswith(('\t','\r','\n')) else s
report={'saved_observations':len(rows),'unique_request_urls':len(seen_urls),'url_aliases_resolved':False,'country_states':dict(collections.Counter(x['country_state'] for x in rows)),'province_states':dict(collections.Counter(x['province_state'] for x in rows)),'countries':[{'country_name':k,'observations':v} for k,v in sorted(countries.items())],'province_groups':[{'country_name':k[0],'province_name':k[1],'observations':v} for k,v in sorted(provinces.items(),key=lambda item:json.dumps(item[0],ensure_ascii=False))],'manifest_sha256':hashlib.sha256(manifest_bytes).hexdigest(),'scope':'Selected saved article observations; not readers, unique authors, account addresses or audience geography'}
out.mkdir();(out/'manifest.json').write_bytes(manifest_bytes)
for row,raw in zip(rows,raws):(out/row['archive_file']).write_bytes(raw)
with (out/'publication-locations.csv').open('x',encoding='utf-8-sig',newline='') as f:
w=csv.DictWriter(f,fieldnames=list(rows[0]));w.writeheader();w.writerows({k:safe(v) for k,v in row.items()} for row in rows)
(out/'publication-locations.json').write_text(json.dumps(rows,ensure_ascii=False,indent=2),encoding='utf-8')
(out/'coverage.json').write_text(json.dumps(report,ensure_ascii=False,indent=2),encoding='utf-8')
print(str(out))