import csv, hashlib, json, pathlib, sys
from datetime import datetime
previous_package, manifest_path, target = sys.argv[1:4]
previous = "-" if previous_package == "-" else str(pathlib.Path(previous_package) / "state.json")
restart = len(sys.argv) == 5 and sys.argv[4] == "--restart"
if len(sys.argv) not in (4, 5) or len(sys.argv) == 5 and not restart:
raise ValueError("Optional final argument is --restart")
entries = json.loads(pathlib.Path(manifest_path).read_text(encoding="utf-8"))
if not isinstance(entries, list) or not 1 <= len(entries) <= 20:
raise ValueError("Supply 1–20 saved pages for this history chunk")
tab = entries[0]["params"].get("item_show_type")
if type(tab) is not int or tab not in (0, 5, 7, 8):
raise ValueError("Select one explicit content tab: 0, 5, 7 or 8")
checkpoint = None if previous_package == "-" else json.loads((pathlib.Path(previous_package) / "checkpoint.json").read_text(encoding="utf-8"))
if checkpoint is not None and restart:
checkpoint = {"version": 1, "resume_params": {"username": json.loads(pathlib.Path(previous).read_text(encoding="utf-8"))["username"], "raw": False, "item_show_type": tab, "offset": ""}}
if checkpoint is not None and (checkpoint.get("version") != 1 or not isinstance(checkpoint.get("resume_params"), dict)):
raise ValueError("History checkpoint is terminal or requires investigation; do not issue another page")
manifest_path, target = pathlib.Path(manifest_path), pathlib.Path(target)
ACTION = "wechat_mp.v2.fetch_account_articles"
state = {"version": 1, "username": None, "item_show_type": tab, "articles": []} if previous == "-" else json.loads(pathlib.Path(previous).read_text(encoding="utf-8"))
if state.get("version") != 1 or (type(state.get("item_show_type")) is not int or state["item_show_type"] != tab) or not isinstance(state.get("articles"), list):
raise ValueError("Expected state for this same content tab")
def number(v):
if type(v) is int and v > 9007199254740991:
raise ValueError("Unsafe numeric ID; require an originally string-encoded ID")
if isinstance(v, bool) or not isinstance(v, (int, str)) or not str(v).isdigit() or int(v) < 1:
raise ValueError("Expected positive article ID/position")
return str(int(v))
def stamp(v):
t = datetime.fromisoformat(v.replace("Z", "+00:00"))
if t.tzinfo is None:
raise ValueError("Collection times must include timezone")
return t
def key(row):
return number(row["app_msg_id"]) + ":" + number(row["idx"])
records = {}
for row in state["articles"]:
k = key(row)
if k in records:
raise ValueError("Duplicate article identity in old state")
stamp(row["first_seen_at"]); stamp(row["last_seen_at"])
records[k] = row
initial = set(records)
new, changed, observed, request_ids, cursors = set(), set(), set(), set(), set()
if checkpoint is not None and not restart:
prior_cursors = checkpoint.get("seen_next_offsets", [])
if not isinstance(prior_cursors, list) or any(not isinstance(c, str) or not c for c in prior_cursors):
raise ValueError("Invalid history cursor ledger")
cursors.update(prior_cursors)
resumed_offset = checkpoint["resume_params"].get("offset")
if resumed_offset:
cursors.add(resumed_offset)
pages, raw_pages, observations = [], [], []
expected, username, last_time = None, state["username"], None
stop = "saved_pages_exhausted"
watermark = stamp(state["last_collected_at"]) if state.get("last_collected_at") else None
processed = state.get("processed_requests", {})
if not isinstance(processed, dict):
raise ValueError("Invalid processed-request ledger")
metadata = ("title", "digest", "url", "create_time", "update_time")
for index, entry in enumerate(entries):
if stop != "saved_pages_exhausted":
raise ValueError("Manifest contains pages after terminal response")
params = entry["params"]
if entry.get("action") != ACTION or params.get("raw") is not False or (type(params.get("item_show_type")) is not int or params["item_show_type"] != tab):
raise ValueError("Keep the same explicit content tab and raw=false")
if set(params) - {"username", "raw", "item_show_type", "offset", "page_size"}:
raise ValueError("Unexpected parameters")
account = params["username"]
if not isinstance(account, str) or not account.startswith("gh_"):
raise ValueError("This helper uses the account's verified gh_ username")
if username is None:
username = account
if account != username:
raise ValueError("Mixed account or old state belongs to another account")
if index == 0:
if checkpoint is None and params.get("offset") not in (None, ""):
raise ValueError("The first history chunk starts at the beginning")
if checkpoint is not None:
resume = checkpoint["resume_params"]
if any(params.get(field) != resume.get(field) for field in ("username", "raw", "item_show_type")) or (params.get("offset") or "") != (resume.get("offset") or ""):
raise ValueError("First page does not match the saved history checkpoint")
if index > 0 and params.get("offset") != expected:
raise ValueError("Broken offset chain")
current_time = stamp(entry["collected_at"])
if last_time and current_time < last_time:
raise ValueError("Manifest times must be chronological")
last_time = current_time
http = entry["http_status"]
if type(http) is not int or not 100 <= http <= 599:
raise ValueError("Record the actual integer HTTP status")
path = manifest_path.parent / entry["response_file"]
raw = path.read_bytes(); raw_pages.append(raw)
result = json.loads(raw)
page = {"page": index + 1, "params": params, "collected_at": entry["collected_at"], "http_status": http,
"source": str(path), "sha256": hashlib.sha256(raw).hexdigest()}
pages.append(page)
if http != 200:
error = result.get("error", {})
if not isinstance(error, dict) or not isinstance(error.get("code"), str) or not error["code"]:
raise ValueError("Expected gateway error envelope")
page.update(error_code=error["code"], request_id=error.get("request_id"))
stop = "gateway_failed"
continue
if result.get("action") != ACTION or result.get("status") != "succeeded":
raise ValueError("Wrong action or unsuccessful response")
request = result.get("request_id")
if not isinstance(request, str) or not request.strip() or request in request_ids:
raise ValueError("Missing/repeated request ID")
request_ids.add(request); page["request_id"] = request
replayed = request in processed
signature = {"sha256": page["sha256"], "params": params, "collected_at": entry["collected_at"]}
if replayed and processed[request] != signature:
raise ValueError("Processed request was changed")
if not replayed and watermark and current_time < watermark:
raise ValueError("Do not apply older unseen observations")
page["replayed"] = replayed
data = result.get("data")
if not isinstance(data, dict) or data.get("biz_username") != username or not isinstance(data.get("articles"), list):
raise ValueError("Expected simplified article list for the same account")
for article in data["articles"]:
k = key(article)
if not isinstance(article.get("title"), str) or not isinstance(article.get("url"), str) or not article["url"]:
raise ValueError("Article requires title and URL")
for field in metadata:
if article.get(field) is not None and (isinstance(article[field], bool) or not isinstance(article[field], (str, int))):
raise ValueError("Unexpected article field type")
values = {field: article.get(field) for field in metadata}
observations.append({"article_key": k, "page": index + 1, "request_id": request, "article": article})
observed.add(k)
if replayed:
if k not in records:
raise ValueError("Processed request refers to a missing stored article")
continue
prior = records.get(k)
if prior is None:
new.add(k)
elif k in initial and any(prior.get(field) != values[field] for field in metadata):
changed.add(k)
record = {"app_msg_id": number(article["app_msg_id"]), "idx": number(article["idx"]), **values,
"first_seen_at": prior["first_seen_at"] if prior else entry["collected_at"],
"last_seen_at": entry["collected_at"], "last_request_id": request}
records[k] = record
if not replayed:
processed[request] = signature
state["last_collected_at"] = entry["collected_at"]
watermark = current_time
page["returned_objects"] = len(data["articles"])
end, cursor = data.get("is_end"), data.get("next_offset")
if type(end) not in (bool, int) or end not in (0, 1):
stop = "missing_end_flag"
elif end == 1:
stop = "provider_end"
elif not data["articles"]:
stop = "empty_page_before_end"
elif not isinstance(cursor, str) or not cursor:
stop = "missing_cursor"
elif cursor in cursors:
stop = "repeated_cursor"
else:
cursors.add(cursor); expected = cursor
resume_params = None
if stop == "saved_pages_exhausted":
resume_params = {"username": username, "raw": False, "item_show_type": tab, "offset": expected}
stop = "history_chunk_saved"
elif stop == "gateway_failed":
resume_params = dict(entries[-1]["params"])
next_request = {"action": ACTION, "params": resume_params} if resume_params is not None else None
state.update(username=username, articles=[records[k] for k in sorted(records)], processed_requests=processed)
target.mkdir(exist_ok=False)
def safe(v):
if v is None:
return ""
value = str(v)
return "'" + value if value.startswith(("=", "+", "-", "@", "\t", "\r", "\n")) else value
columns = ["app_msg_id", "idx", *metadata, "first_seen_at", "last_seen_at", "last_request_id"]
for name, keys in [("new-articles.csv", new), ("changed-articles.csv", changed)]:
with (target / name).open("x", encoding="utf-8-sig", newline="") as f:
writer = csv.DictWriter(f, fieldnames=columns); writer.writeheader()
writer.writerows({field: safe(records[k].get(field)) for field in columns} for k in sorted(keys))
(target / "state.json").write_text(json.dumps(state, ensure_ascii=False, indent=2), encoding="utf-8")
with (target / "observations.jsonl").open("x", encoding="utf-8") as f:
for item in observations:
f.write(json.dumps(item, ensure_ascii=False) + "\n")
for index, raw in enumerate(raw_pages):
(target / ("raw-page-%02d.json" % (index + 1))).write_bytes(raw)
report = {"username": username, "requested_tab": tab, "new_articles": len(new), "changed_known_articles": len(changed),
"observed_keys": len(observed), "stored_articles": len(records), "stop_reason": stop,
"complete_history": False, "visible_history_end_observed": stop == "provider_end", "publication_dates_inferred": False, "pages": pages}
(target / "coverage.json").write_text(json.dumps(report, ensure_ascii=False, indent=2), encoding="utf-8")
if next_request:
(target / "next-request.json").write_text(json.dumps(next_request, ensure_ascii=False, indent=2), encoding="utf-8")
print(json.dumps({"new": len(new), "changed": len(changed), "stop": stop}))
(target / "checkpoint.json").write_text(json.dumps({"version": 1, "username": username, "requested_tab": tab,
"stop_reason": stop, "resume_params": resume_params, "seen_next_offsets": sorted(cursors), "state_file": "state.json",
"prior_package": previous_package, "warning": "A cursor is not a permanent checkpoint; preserve all chunks and restart from page one if invalid"}, ensure_ascii=False, indent=2), encoding="utf-8")