import csv, datetime, hashlib, json, pathlib, sys
manifest = pathlib.Path(sys.argv[1]); manifest_bytes = manifest.read_bytes()
config = json.loads(manifest_bytes)
targets, runs = config['targets'], config['runs']
if not isinstance(targets, dict) or not targets or not isinstance(runs,list) or not runs: raise ValueError('Targets and saved runs required')
for keyword, ids in targets.items():
if not keyword.strip() or not isinstance(ids,list) or not ids or any(not isinstance(x,str) or not x.strip() for x in ids) or len(ids)!=len(set(ids)): raise ValueError('Each keyword needs unique original note IDs')
def stamp(value):
d=datetime.datetime.fromisoformat(value.replace('Z','+00:00'))
if d.utcoffset() is None: raise ValueError('Record actual query time with timezone')
return d
rows, archives, sources, scopes, moments, request_ids = [], [], [], {}, set(), set()
for run in runs:
pages=run['pages']
if not isinstance(pages,list) or not pages: raise ValueError('Saved pages required')
first=pages[0]['params']; keyword=first.get('keyword')
if keyword not in targets: raise ValueError('No targets for this keyword')
scope={k:v for k,v in first.items() if k not in ('page','search_id','search_session_id')}
if keyword in scopes and scopes[keyword]!=scope: raise ValueError('Use identical explicit query/filter parameters across observations')
scopes[keyword]=scope
observed=pages[0]['collected_at']; moment=stamp(observed)
if (keyword,moment) in moments: raise ValueError('Duplicate observation time for keyword')
moments.add((keyword,moment))
found, note_slot, total_items, next_page, session, last_time, seen, can_continue = {}, 0, 0, 1, None, None, set(), True
for page in pages:
params=page['params']; number=params.get('page')
if not can_continue: raise ValueError('Do not extend an empty/repeated page')
if type(number) is not int or number!=next_page: raise ValueError('Start at page one and follow the observed next_page without gaps')
if {k:v for k,v in params.items() if k not in ('page','search_id','search_session_id')}!=scope: raise ValueError('Mixed query/filter run')
time=stamp(page['collected_at'])
if last_time is not None and time<last_time: raise ValueError('Page collection times go backwards')
last_time=time
raw=(manifest.parent/page['file']).read_bytes(); response=json.loads(raw); rid=response.get('request_id')
if not isinstance(rid,str) or not rid.strip() or rid in request_ids: raise ValueError('Missing/repeated request ID')
request_ids.add(rid)
if response.get('status')!='succeeded' or response.get('error') is not None or response.get('action')!='xiaohongshu.app_v2.search_notes': raise ValueError('Failed or mismatched search')
body=response['data']
if body.get('success') is not True or type(body.get('code')) is not int or body['code']!=0 or type(body.get('page')) is not int or body['page']!=number: raise ValueError('Wrong page or business error')
returned=(body.get('search_id'),body.get('search_session_id'))
if any(not isinstance(x,str) or not x.strip() for x in returned): raise ValueError('Missing session identity')
if session is None:
if params.get('search_id') not in (None,'') or params.get('search_session_id') not in (None,''): raise ValueError('Start a fresh page-one observation without prior session IDs')
session=returned
elif returned!=session or (params.get('search_id'),params.get('search_session_id'))!=session: raise ValueError('Changed search session')
items=body['data']['items']
if not isinstance(items,list): raise ValueError('Missing result items')
name='raw-'+str(len(archives)+1).zfill(3)+'.json';digest=hashlib.sha256(raw).hexdigest();archives.append((name,raw))
sources.append({'file':name,'sha256':digest,'request_id':rid,'keyword':keyword,'page':number,'collected_at':page['collected_at']})
fresh=0
for position,item in enumerate(items,1):
if not isinstance(item,dict): raise ValueError('Invalid result item')
note=item.get('note')
if note is None: continue
if not isinstance(note,dict): raise ValueError('Unreadable note object; do not report it as not found')
nid=note.get('id')
if not isinstance(nid,str) or not nid.strip(): raise ValueError('Missing note ID')
note_slot+=1
if nid not in seen: fresh+=1;seen.add(nid)
if nid in targets[keyword] and nid not in found:
found[nid]={'page':number,'item_position_on_page':position,'note_slot_in_saved_run':note_slot,
'request_id':rid,'source_file':name,'source_sha256':digest}
total_items+=len(items);can_continue=fresh>0
next_page=body.get('next_page')
if next_page is not None and (type(next_page) is not int or next_page<=number): next_page=None
for nid in targets[keyword]:
row={'keyword':keyword,'note_id':nid,'observed_at':observed,'last_page_at':pages[-1]['collected_at'],
'pages_scanned':len(pages),'items_scanned':total_items,'note_slots_scanned':note_slot,'unique_notes_scanned':len(seen),
'state':'found' if nid in found else 'not_found_in_scanned_pages','scope':scope,
'page':None,'item_position_on_page':None,'note_slot_in_saved_run':None,
'continuation_available':can_continue and next_page is not None,'request_id':None,'source_file':None,'source_sha256':None}
row.update(found.get(nid,{}));rows.append(row)
missing=set(targets)-{row['keyword'] for row in rows}
if missing: raise ValueError('Every target keyword needs at least one saved run')
rows.sort(key=lambda r:(r['keyword'],r['note_id'],stamp(r['observed_at'])))
previous={};changes=[]
for row in rows:
key=(row['keyword'],row['note_id']);before=previous.get(key)
if before:
both=before['state']=='found' and row['state']=='found'
changes.append({'keyword':key[0],'note_id':key[1],'before':before['observed_at'],'after':row['observed_at'],
'before_state':before['state'],'after_state':row['state'],
'observed_note_slots_up':before['note_slot_in_saved_run']-row['note_slot_in_saved_run'] if both else None,
'equal_pages_scanned':before['pages_scanned']==row['pages_scanned'],
'scope':'Change in saved API ordering only; not universal app ranking or causal SEO impact'})
previous[key]=row
out=pathlib.Path(sys.argv[2]);out.mkdir()
for name,raw in archives:(out/name).write_bytes(raw)
(out/'input-manifest.json').write_bytes(manifest_bytes)
for name,value in [('observations.json',rows),('changes.json',changes),('coverage.json',{'sources':sources,
'manifest_sha256':hashlib.sha256(manifest_bytes).hexdigest(),'saved_runs':len(runs),'executed_calls':0,
'position_definition':'Raw item position on page and note slots across consecutive saved pages; repeated notes occupy slots; first appearance retained'})]:
(out/name).write_text(json.dumps(value,ensure_ascii=False,indent=2),encoding='utf-8')
fields=['keyword','note_id','observed_at','state','page','item_position_on_page','note_slot_in_saved_run','pages_scanned','items_scanned','unique_notes_scanned','continuation_available','request_id','source_file','source_sha256']
with (out/'observations.csv').open('w',encoding='utf-8-sig',newline='') as f:
writer=csv.DictWriter(f,fieldnames=fields);writer.writeheader()
for row in rows:
cells={k:'' if row.get(k) is None else str(row[k]) for k in fields}
writer.writerow({k:"'"+v if v.lstrip().startswith(('=','+','-','@')) else v for k,v in cells.items()})
print(json.dumps({'saved_runs':len(runs),'target_observations':len(rows),'comparisons':len(changes),'executed_calls':0}))