import csv,hashlib,json,pathlib,re,sys
from datetime import datetime
manifest,target=map(pathlib.Path,sys.argv[1:]);manifest_bytes=manifest.read_bytes();entries=json.loads(manifest_bytes)
if not isinstance(entries,list) or not 1<=len(entries)<=20:raise ValueError('Provide 1..20 saved direct pages')
ACTION='twitter.web.fetch_user_post_tweet';expected_cursor=None;expected_id=None;last_time=None;request_ids=set();used_cursors=set();seen={};rows=[];observations=[];sources=[];raw_files=[]
def ident(v):return isinstance(v,str) and v.isascii() and v.isdigit() and int(v)>0
def cell(v):
text='' if v is None else str(v)
return "'"+text if text.startswith(('=','+','-','@','\t','\r','\n')) else text
for page,e in enumerate(entries,1):
params=e['params']
if not isinstance(params,dict):raise ValueError('Request params must be an object')
if e.get('action')!=ACTION or type(e.get('http_status')) is not int or e['http_status']!=200:raise ValueError('Require successful direct account pages')
if 'screen_name' in params:
handle=params['screen_name'];account=e.get('verified_author_rest_id');evidence=e.get('identity_evidence')
if set(params)-{'screen_name','cursor'} or not isinstance(handle,str) or not re.fullmatch(r'[A-Za-z0-9_]{1,15}',handle):raise ValueError('Use a bare handle only, not a URL or mixed identity')
if not ident(account) or not isinstance(evidence,str) or not evidence.strip():raise ValueError('Provide the previously verified string author ID and its identity evidence')
identity=('screen_name',handle.lower(),account,evidence)
else:
account=params.get('rest_id')
if set(params)-{'rest_id','cursor'} or type(account) is not int or not 0<account<=9007199254740991:raise ValueError('Use a verified safe-range integer rest_id')
account=str(account);identity=('rest_id',account)
if expected_id is None:expected_id=account;expected_identity=identity
if identity!=expected_identity:raise ValueError('Mixed account identity or changed verification evidence')
cursor=params.get('cursor')
if page==1:
if cursor not in (None,''):raise ValueError('Start this bounded run at the first page')
elif not expected_cursor or cursor!=expected_cursor:raise ValueError('Continue with the exact preceding cursor')
if cursor:
if cursor in used_cursors:raise ValueError('Repeated cursor')
used_cursors.add(cursor)
stamp=datetime.fromisoformat(e['collected_at'].replace('Z','+00:00'))
if stamp.utcoffset() is None or last_time is not None and stamp<last_time:raise ValueError('Use chronological timezone-aware observations')
last_time=stamp;path=manifest.parent/e['response_file'];raw=path.read_bytes();response=json.loads(raw)
if response.get('action')!=ACTION or response.get('status')!='succeeded' or response.get('error') is not None:raise ValueError('Unsuccessful or wrong-action response')
rid=response.get('request_id')
if not isinstance(rid,str) or not rid.strip() or rid in request_ids:raise ValueError('Missing or repeated request ID')
request_ids.add(rid);data=response.get('data')
if not isinstance(data,dict) or not isinstance(data.get('timeline'),list):raise ValueError('Expected complete direct data.timeline')
next_cursor=data.get('next_cursor')
if next_cursor is not None and not isinstance(next_cursor,str):raise ValueError('Invalid next cursor')
if next_cursor and (next_cursor==cursor or next_cursor in used_cursors):raise ValueError('Cursor loop')
expected_cursor=next_cursor;archive=f'raw-{page:03}.json';digest=hashlib.sha256(raw).hexdigest();raw_files.append((archive,raw))
sources.append({'file':archive,'sha256':digest,'page':page,'request_id':rid,'params':params,'collected_at':e['collected_at'],'next_cursor':next_cursor,'pinned_present':data.get('pinned') is not None})
for index,post in enumerate(data['timeline']):
if not isinstance(post,dict) or not ident(post.get('tweet_id')):raise ValueError('Original string tweet_id required')
author=post.get('author')
if author is not None and not isinstance(author,dict):raise ValueError('Invalid author')
aid=(author or {}).get('rest_id')
if aid is not None and not ident(aid):raise ValueError('Original string author ID required')
original=post.get('retweeted_tweet')
if original is not None and not isinstance(original,dict):raise ValueError('Unexpected retweeted_tweet type')
for field in ['text','created_at','reply_to']:
if post.get(field) is not None and not isinstance(post[field],str):raise ValueError('Unexpected post text/date/reply field')
classification='repost_object_present' if original is not None else ('author_missing' if aid is None else ('other_author' if aid!=expected_id else 'matching_author_no_repost_object'))
signature=[aid,classification,post.get('text'),post.get('created_at'),post.get('reply_to')]
duplicate=post['tweet_id'] in seen
if duplicate and seen[post['tweet_id']]!=signature:raise ValueError('Conflicting identity/content for repeated post ID')
seen[post['tweet_id']]=signature
row={'tweet_id':post['tweet_id'],'author_rest_id':aid,'target_rest_id':expected_id,'classification':classification,'duplicate_id':duplicate,'page':page,'index':index,'text':post.get('text'),'created_at':post.get('created_at'),'reply_to':post.get('reply_to'),'reply_indicator_present':bool(post.get('reply_to')),'source_file':archive,'source_sha256':digest,'request_id':rid,'collected_at':e['collected_at']}
observations.append(row)
if classification=='matching_author_no_repost_object' and not duplicate:rows.append(row)
target.mkdir()
(target/'input-manifest.json').write_bytes(manifest_bytes)
for name,raw in raw_files:(target/name).write_bytes(raw)
for name,value in [('selected-posts.json',rows),('observations.json',observations),('coverage.json',{'target_rest_id':expected_id,'selected_unique_posts':len(rows),'observations':len(observations),'sources':sources,'manifest_sha256':hashlib.sha256(manifest_bytes).hexdigest(),'next_cursor':expected_cursor,'history_complete':False,'pinned_included':False,'source_identity_independently_authenticated':False,'network_calls':0})]:
(target/name).write_text(json.dumps(value,ensure_ascii=False,indent=2),encoding='utf-8')
fields=['tweet_id','author_rest_id','text','created_at','reply_to','reply_indicator_present','source_file','request_id','collected_at']
with (target/'selected-posts.csv').open('w',encoding='utf-8-sig',newline='') as f:
writer=csv.writer(f);writer.writerow(fields)
for row in rows:writer.writerow([cell(row[k]) for k in fields])
print(json.dumps({'selected_unique_posts':len(rows),'observations':len(observations),'network_calls':0}))