feat(stage1): register primary outcome candidates
This commit is contained in:
Jinotech
@@ -0,0 +1,280 @@
|
||||
"""Build a provisional, source-linked registry for the three primary outcomes.
|
||||
|
||||
This script deliberately does not approve cross-survey equivalence, recode
|
||||
responses, or substitute AI review for the two required human reviewers.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import html
|
||||
import json
|
||||
from collections import Counter
|
||||
from pathlib import Path
|
||||
|
||||
import pandas as pd
|
||||
import pyarrow.parquet as pq
|
||||
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[2]
|
||||
STAGE_DIR = ROOT / "research" / "stage1"
|
||||
GSHS_BOOK = ROOT / "Dataset" / "GSHS-全球学生健康调查数据" / "GSHS" / "02_documentation" / "GSHS_数据字典.xlsx"
|
||||
YRBS_DIR = ROOT / "Dataset" / "可直接分析数据包_NSDUH_YRBS" / "YRBS"
|
||||
NSDUH_DIR = ROOT / "Dataset" / "可直接分析数据包_NSDUH_YRBS" / "NSDUH"
|
||||
OUT_JSON = STAGE_DIR / "primary_outcome_registry.json"
|
||||
OUT_MD = STAGE_DIR / "primary_outcome_registry.md"
|
||||
|
||||
PRIMARY_CONSTRUCTS = {"suicide_ideation", "suicide_plan", "suicide_attempt"}
|
||||
GSHS_CONSTRUCT = {
|
||||
"raw_mh_considersui": "suicide_ideation",
|
||||
"raw_mh_b_considersui": "suicide_ideation",
|
||||
"raw_mh_plansui": "suicide_plan",
|
||||
"raw_mh_b_plansui": "suicide_plan",
|
||||
"raw_mh_attemptsui": "suicide_attempt",
|
||||
"raw_mh_b_attemptsui": "suicide_attempt",
|
||||
}
|
||||
NSDUH_YOUTH_RAW = {"YUSUITHK": "suicide_ideation", "YUSUIPLN": "suicide_plan"}
|
||||
NSDUH_YOUTH_DERIVED = {"YUSUITHKYR": "suicide_ideation", "YUSUIPLNYR": "suicide_plan"}
|
||||
|
||||
|
||||
def clean_text(value: object) -> str:
|
||||
return html.unescape(str(value or "")).strip()
|
||||
|
||||
|
||||
def parse_json_object(value: object) -> object:
|
||||
text = clean_text(value)
|
||||
if not text:
|
||||
return None
|
||||
try:
|
||||
return json.loads(text)
|
||||
except json.JSONDecodeError:
|
||||
return {"unparsed_source_text": text}
|
||||
|
||||
|
||||
def parquet_columns(path: Path) -> set[str]:
|
||||
return set(pq.ParquetFile(path).schema_arrow.names)
|
||||
|
||||
|
||||
def build_gshs() -> list[dict]:
|
||||
frame = pd.read_excel(GSHS_BOOK, sheet_name="Master_Columns", dtype=str, keep_default_na=False)
|
||||
frame = frame[frame["column_name"].isin(GSHS_CONSTRUCT)].copy()
|
||||
records = []
|
||||
for row in frame.sort_values("ordinal").to_dict("records"):
|
||||
records.append(
|
||||
{
|
||||
"survey": "GSHS",
|
||||
"year": None,
|
||||
"source_variable": row["column_name"],
|
||||
"construct": GSHS_CONSTRUCT[row["column_name"]],
|
||||
"record_type": "raw_candidate",
|
||||
"population": "school-attending students; exact component ages pending source review",
|
||||
"time_window": "past 12 months (from source label; questionnaire verification pending)",
|
||||
"question_text": None,
|
||||
"source_label": clean_text(row["label_variants"]),
|
||||
"answer_options": parse_json_object(row["value_labels_example"]),
|
||||
"source_file": str(GSHS_BOOK.relative_to(ROOT)).replace("\\", "/"),
|
||||
"source_url": None,
|
||||
"page_or_section": "Master_Columns",
|
||||
"present_in_local_data": True,
|
||||
"source_verification_status": "pending_questionnaire_review",
|
||||
"review_status": "provisional",
|
||||
"reviewer_1": None,
|
||||
"reviewer_2": None,
|
||||
"link_status": "not_evaluated",
|
||||
"notes": "The `_b_` and unprefixed forms remain separate item versions; labels alone do not prove equivalence.",
|
||||
}
|
||||
)
|
||||
return records
|
||||
|
||||
|
||||
def build_yrbs() -> list[dict]:
|
||||
source = YRBS_DIR / "yrbs_selected_variable_dictionary.csv"
|
||||
frame = pd.read_csv(source, dtype=str, keep_default_na=False)
|
||||
frame = frame[frame["construct"].isin(PRIMARY_CONSTRUCTS)].copy()
|
||||
records = []
|
||||
for row in frame.sort_values(["year", "variable"]).to_dict("records"):
|
||||
year = int(row["year"])
|
||||
records.append(
|
||||
{
|
||||
"survey": "YRBS",
|
||||
"year": year,
|
||||
"source_variable": row["variable"],
|
||||
"construct": row["construct"],
|
||||
"record_type": "raw_candidate",
|
||||
"population": "national high-school students; exact eligibility pending annual source review",
|
||||
"time_window": clean_text(row["time_window"]),
|
||||
"question_text": clean_text(row["original_question"]),
|
||||
"answer_options": parse_json_object(row["answer_options_json"]),
|
||||
"source_file": str(source.relative_to(ROOT)).replace("\\", "/"),
|
||||
"source_url": clean_text(row["official_url"]),
|
||||
"page_or_section": row["questionnaire_version"],
|
||||
"present_in_local_data": year <= 2019,
|
||||
"data_status": "available_selected_table" if year <= 2019 else "quarantined_pending_import_validation",
|
||||
"source_verification_status": "pending_direct_pdf_review",
|
||||
"review_status": "provisional",
|
||||
"reviewer_1": None,
|
||||
"reviewer_2": None,
|
||||
"link_status": "not_evaluated",
|
||||
"notes": "Repository dictionary candidate only. Stale source-machine absolute paths are intentionally not copied.",
|
||||
}
|
||||
)
|
||||
return records
|
||||
|
||||
|
||||
def build_nsduh() -> tuple[list[dict], list[dict], list[dict]]:
|
||||
source = NSDUH_DIR / "nsduh_complete_variable_dictionary.csv"
|
||||
frame = pd.read_csv(source, dtype=str, keep_default_na=False)
|
||||
full_cols = parquet_columns(NSDUH_DIR / "nsduh_2021_2024_full.parquet")
|
||||
selected_cols = parquet_columns(NSDUH_DIR / "nsduh_2021_2024_selected.parquet")
|
||||
raw_records: list[dict] = []
|
||||
derived_records: list[dict] = []
|
||||
excluded_records: list[dict] = []
|
||||
|
||||
def common(row: dict, construct: str, record_type: str) -> dict:
|
||||
variable = row["variable"]
|
||||
return {
|
||||
"survey": "NSDUH",
|
||||
"year": int(row["year"]),
|
||||
"source_variable": variable,
|
||||
"construct": construct,
|
||||
"record_type": record_type,
|
||||
"population": "youth module; exact age/eligibility and skip logic pending codebook review",
|
||||
"time_window": "past 12 months",
|
||||
"question_text": clean_text(row["original_question_or_definition"]) or None,
|
||||
"source_label": clean_text(row["official_title"]),
|
||||
"source_file": str(source.relative_to(ROOT)).replace("\\", "/"),
|
||||
"source_url": clean_text(row["official_source"]),
|
||||
"page_or_section": variable,
|
||||
"present_in_full_table": variable in full_cols,
|
||||
"present_in_selected_table": variable in selected_cols,
|
||||
"source_verification_status": "official_api_metadata_reviewed_codebook_pending",
|
||||
"review_status": "provisional",
|
||||
"reviewer_1": None,
|
||||
"reviewer_2": None,
|
||||
"link_status": "not_evaluated",
|
||||
}
|
||||
|
||||
for row in frame[frame["variable"].isin(NSDUH_YOUTH_RAW)].to_dict("records"):
|
||||
item = common(row, NSDUH_YOUTH_RAW[row["variable"]], "raw_candidate")
|
||||
item["notes"] = "Keep 'not sure' and refusal/nonresponse distinct until skip and missing rules are verified."
|
||||
raw_records.append(item)
|
||||
|
||||
for row in frame[frame["variable"].isin(NSDUH_YOUTH_DERIVED)].to_dict("records"):
|
||||
item = common(row, NSDUH_YOUTH_DERIVED[row["variable"]], "derived_youth_indicator")
|
||||
item["question_text"] = None
|
||||
item["notes"] = "Derived/recode variable; do not count as an independent administered item."
|
||||
derived_records.append(item)
|
||||
|
||||
exclusion_mask = frame["variable"].str.match(r"^(SUICTHNK|SUIPLANYR|IRSUITRYYR|IRCOSUITRYYR|YUCOSUI|COSUI|ADSUIT)")
|
||||
for row in frame[exclusion_mask].to_dict("records"):
|
||||
variable = row["variable"]
|
||||
reason = "adult_module"
|
||||
if "COSUI" in variable:
|
||||
reason = "covid_conditioned"
|
||||
elif variable.startswith(("IR", "AD")):
|
||||
reason = "adult_derived_or_composite"
|
||||
excluded_records.append(
|
||||
{
|
||||
"survey": "NSDUH",
|
||||
"year": int(row["year"]),
|
||||
"source_variable": variable,
|
||||
"source_label": clean_text(row["official_title"]),
|
||||
"exclusion_reason": reason,
|
||||
"status": "excluded_from_primary_youth_item_candidates",
|
||||
}
|
||||
)
|
||||
return raw_records, derived_records, excluded_records
|
||||
|
||||
|
||||
def render_markdown(registry: dict) -> str:
|
||||
records = registry["candidate_records"]
|
||||
counts = Counter((r["survey"], r["construct"]) for r in records)
|
||||
lines = [
|
||||
"# Primary outcome source registry",
|
||||
"",
|
||||
"Version: 1.0 | Status: provisional | Generated: 2026-09-20",
|
||||
"",
|
||||
"## Scope and decision boundary",
|
||||
"",
|
||||
"This registry captures source-linked candidates for suicide ideation, plan, and attempt. It is a development artifact, not an approved item bank. No row has completed the two-human-reviewer requirement, no cross-survey link is approved, and no missing or skip code has been canonically recoded.",
|
||||
"",
|
||||
"## Candidate coverage",
|
||||
"",
|
||||
"| Survey | Ideation | Plan | Attempt | Current evidence boundary |",
|
||||
"|---|---:|---:|---:|---|",
|
||||
]
|
||||
for survey in ["GSHS", "YRBS", "NSDUH"]:
|
||||
boundary = {
|
||||
"GSHS": "Workbook labels/options only; component questionnaires still require direct review.",
|
||||
"YRBS": "51 annual candidates; 2021/2023 data remain quarantined; annual PDFs require direct review.",
|
||||
"NSDUH": "Official API metadata supports youth ideation/plan only; no youth attempt field appears in the local full dictionary.",
|
||||
}[survey]
|
||||
lines.append(
|
||||
f"| {survey} | {counts[(survey, 'suicide_ideation')]} | {counts[(survey, 'suicide_plan')]} | {counts[(survey, 'suicide_attempt')]} | {boundary} |"
|
||||
)
|
||||
lines += [
|
||||
"",
|
||||
"Counts are source-record counts, not distinct item families. YRBS counts include annual versions; NSDUH counts include 2021–2024 raw youth records only.",
|
||||
"",
|
||||
"## Material conflicts and exclusions",
|
||||
"",
|
||||
"- GSHS exposes both `_b_` and unprefixed variants. They differ in naming and, for attempt, response form (binary versus frequency). They remain separate until questionnaire/component provenance is verified.",
|
||||
"- GSHS self-harm variables are not included as suicide-attempt items.",
|
||||
"- YRBS metadata contains stale absolute paths from another machine and some automatically extracted neighboring-text contamination. Only exact primary-construct rows were retained, and their direct PDF review is still pending.",
|
||||
"- YRBS 2021 and 2023 item metadata are retained for future migration checks, but their respondent data are quarantined pending layout/import validation.",
|
||||
"- NSDUH adult, COVID-conditioned, imputed, recoded, and composite variables are not treated as primary raw youth items.",
|
||||
"- NSDUH youth attempt questions `YSUI03`–`YSUI05` are mentioned in notes, but no corresponding youth attempt variable appears in the local complete dictionary. Attempt coverage is therefore recorded as a source gap, not inferred from ideation wording.",
|
||||
"",
|
||||
"## Gate 1 status",
|
||||
"",
|
||||
"Gate 1 remains **in progress**. Before any candidate can enter confirmatory analysis, the project still needs direct official questionnaire/codebook review, exact population and skip-logic verification, canonical response/missing rules, component-specific GSHS provenance, and two actual human reviewers for primary outcomes.",
|
||||
"",
|
||||
"## Reproduction",
|
||||
"",
|
||||
"Run `research/stage1/build_primary_outcome_registry.py` in the locked research environment. The script reads repository-local dictionaries and Parquet schemas and rewrites this report plus `primary_outcome_registry.json`.",
|
||||
"",
|
||||
]
|
||||
return "\n".join(lines)
|
||||
|
||||
|
||||
def main() -> None:
|
||||
gshs = build_gshs()
|
||||
yrbs = build_yrbs()
|
||||
nsduh_raw, nsduh_derived, nsduh_excluded = build_nsduh()
|
||||
registry = {
|
||||
"schema_version": "1.0",
|
||||
"generated_on": "2026-09-20",
|
||||
"status": "provisional",
|
||||
"allowed_uses": ["pipeline_development", "source_review_queue"],
|
||||
"prohibited_uses": ["confirmatory_analysis", "approved_anchor_links", "claim_of_human_double_review"],
|
||||
"candidate_records": gshs + yrbs + nsduh_raw,
|
||||
"derived_records_not_independent_items": nsduh_derived,
|
||||
"excluded_nsduh_records": nsduh_excluded,
|
||||
"source_gaps": [
|
||||
{
|
||||
"survey": "NSDUH",
|
||||
"construct": "suicide_attempt",
|
||||
"status": "not_found_in_local_complete_dictionary",
|
||||
"evidence": "YSUI03–YSUI05 are named in 2022–2024 module-move notes, but no youth attempt variable is exposed in the local 2021–2024 dictionary.",
|
||||
"next_check": "Review annual youth questionnaires/codebooks and public-use suppression/derivation notes.",
|
||||
}
|
||||
],
|
||||
"review_policy": {
|
||||
"human_reviewers_required": 2,
|
||||
"ai_review_is_human_review": False,
|
||||
"default_review_status": "provisional",
|
||||
"cross_survey_equivalence_default": "not_evaluated",
|
||||
},
|
||||
}
|
||||
OUT_JSON.write_text(json.dumps(registry, ensure_ascii=False, indent=2) + "\n", encoding="utf-8")
|
||||
OUT_MD.write_text(render_markdown(registry), encoding="utf-8")
|
||||
print(json.dumps({
|
||||
"candidate_records": len(registry["candidate_records"]),
|
||||
"derived_records": len(nsduh_derived),
|
||||
"excluded_nsduh_records": len(nsduh_excluded),
|
||||
"json": str(OUT_JSON.relative_to(ROOT)),
|
||||
"report": str(OUT_MD.relative_to(ROOT)),
|
||||
}, ensure_ascii=False))
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,34 @@
|
||||
# Primary outcome source registry
|
||||
|
||||
Version: 1.0 | Status: provisional | Generated: 2026-09-20
|
||||
|
||||
## Scope and decision boundary
|
||||
|
||||
This registry captures source-linked candidates for suicide ideation, plan, and attempt. It is a development artifact, not an approved item bank. No row has completed the two-human-reviewer requirement, no cross-survey link is approved, and no missing or skip code has been canonically recoded.
|
||||
|
||||
## Candidate coverage
|
||||
|
||||
| Survey | Ideation | Plan | Attempt | Current evidence boundary |
|
||||
|---|---:|---:|---:|---|
|
||||
| GSHS | 2 | 2 | 2 | Workbook labels/options only; component questionnaires still require direct review. |
|
||||
| YRBS | 17 | 17 | 17 | 51 annual candidates; 2021/2023 data remain quarantined; annual PDFs require direct review. |
|
||||
| NSDUH | 4 | 4 | 0 | Official API metadata supports youth ideation/plan only; no youth attempt field appears in the local full dictionary. |
|
||||
|
||||
Counts are source-record counts, not distinct item families. YRBS counts include annual versions; NSDUH counts include 2021–2024 raw youth records only.
|
||||
|
||||
## Material conflicts and exclusions
|
||||
|
||||
- GSHS exposes both `_b_` and unprefixed variants. They differ in naming and, for attempt, response form (binary versus frequency). They remain separate until questionnaire/component provenance is verified.
|
||||
- GSHS self-harm variables are not included as suicide-attempt items.
|
||||
- YRBS metadata contains stale absolute paths from another machine and some automatically extracted neighboring-text contamination. Only exact primary-construct rows were retained, and their direct PDF review is still pending.
|
||||
- YRBS 2021 and 2023 item metadata are retained for future migration checks, but their respondent data are quarantined pending layout/import validation.
|
||||
- NSDUH adult, COVID-conditioned, imputed, recoded, and composite variables are not treated as primary raw youth items.
|
||||
- NSDUH youth attempt questions `YSUI03`–`YSUI05` are mentioned in notes, but no corresponding youth attempt variable appears in the local complete dictionary. Attempt coverage is therefore recorded as a source gap, not inferred from ideation wording.
|
||||
|
||||
## Gate 1 status
|
||||
|
||||
Gate 1 remains **in progress**. Before any candidate can enter confirmatory analysis, the project still needs direct official questionnaire/codebook review, exact population and skip-logic verification, canonical response/missing rules, component-specific GSHS provenance, and two actual human reviewers for primary outcomes.
|
||||
|
||||
## Reproduction
|
||||
|
||||
Run `research/stage1/build_primary_outcome_registry.py` in the locked research environment. The script reads repository-local dictionaries and Parquet schemas and rewrites this report plus `primary_outcome_registry.json`.
|
||||
Reference in New Issue
Block a user