From 0c9645e1205fceaa274ef677491e1fd04296e165 Mon Sep 17 00:00:00 2001 From: Jinotech Date: Sun, 20 Sep 2026 08:24:25 +0000 Subject: [PATCH] fix(stage1): correct source attribution at the build sources Applies the three source-attested corrections from the signed reviews. All are made in the generators, not the generated CSVs, which would be overwritten on the next build. YRBS population: the 1995 and 1997 National User Guides state the frame covers "the fifty states and the District of Columbia, excluding a majority of Louisiana". 1993 and 1999 carry no such restriction, so the string is now year-conditional in both the item audit and the design record instead of claiming plain national coverage for every year. NSDUH attempt: source_file was None because the codebook render was pending. The local MRB specification PDFs carry YSUI03 verbatim for all four years, including the [IF CURNTAGE = 12 - 17] condition, so each year now cites its local instrument and the status moves to local_instrument_pdf_verified. NSDUH attempt codes: YUSUICTRY appears in no local dictionary or schema, yet carried nine codes identical to the ideation variable YUSUITHK. The instrument offers only 1-4 plus DK/REF, so 85/94/97/98/99 are now marked unattested and inherited, and those dictionary rows become needs_source. GSHS citation: the questionnaire selector took the first document whose role contained "questionnaire", which cited a French instrument for English question_text in the Morocco and Wallis and Futuna components. It now prefers the version matching the stored wording. Verified against all six components; the other four are unchanged. Generated artefacts are deliberately NOT rebuilt here: that regenerates the item bank and the human review queue and is gate-relevant. Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_016yuKikAXbByLvsUnh55SRd --- .../audit_yrbs_primary_outcome_sources.py | 9 ++++- .../stage1/build_primary_outcome_package.py | 36 +++++++++++++++---- .../stage1/build_primary_outcome_registry.py | 24 ++++++++++--- 3 files changed, 58 insertions(+), 11 deletions(-) diff --git a/research/stage1/audit_yrbs_primary_outcome_sources.py b/research/stage1/audit_yrbs_primary_outcome_sources.py index 6bde224..88303fe 100644 --- a/research/stage1/audit_yrbs_primary_outcome_sources.py +++ b/research/stage1/audit_yrbs_primary_outcome_sources.py @@ -147,7 +147,14 @@ def build_audit() -> dict: "time_window": "past 12 months", "response_type": "binary" if construct != "suicide_attempt" else "ordinal_frequency", "answer_options": OPTIONS[construct], - "population": "students in grades 9-12 attending public and private schools in the United States; detailed eligibility remains subject to annual design review", + # 1995 and 1997 National User Guides state the frame covers "the fifty states + # and the District of Columbia, excluding a majority of Louisiana". 1993 and 1999 + # carry no such restriction. + "population": ( + "students in grades 9-12 attending public and private schools in the fifty states and the District of Columbia, excluding a majority of Louisiana; detailed eligibility remains subject to annual design review" + if str(year) in {"1995", "1997"} + else "students in grades 9-12 attending public and private schools in the United States; detailed eligibility remains subject to annual design review" + ), "language": "English questionnaire source; actual accommodation/language coverage not established by this audit", "questionnaire_source_file": str(questionnaire.relative_to(ROOT)).replace("\\", "/"), "questionnaire_page_or_section": q_section, diff --git a/research/stage1/build_primary_outcome_package.py b/research/stage1/build_primary_outcome_package.py index 150b3be..7cf041c 100644 --- a/research/stage1/build_primary_outcome_package.py +++ b/research/stage1/build_primary_outcome_package.py @@ -88,7 +88,12 @@ def build() -> dict[str, int]: "derivation_rule", "source_file", "source_url", "page_or_section", "review_status", ] - def add_item(row: dict[str, Any], options: dict[str, str], missing_row: bool = False) -> None: + def add_item(row: dict[str, Any], options: dict[str, str], missing_row: bool = False, + unattested: list[str] | None = None) -> None: + # Codes listed in `unattested` appear in the delivered data but are attested by no + # local dictionary or schema for this variable; mark them instead of silently + # presenting them as sourced. + unattested_codes = {str(c) for c in (unattested or [])} clean = {field: stringify(row.get(field)) for field in item_fields} clean["text_hash"] = hashlib.sha256(clean["question_text"].encode("utf-8")).hexdigest() items.append(clean) @@ -101,11 +106,15 @@ def build() -> dict[str, int]: "canonical_code": canonical, "missing_type": missing_type, "is_observed_response": stringify(observed), - "derivation_rule": "direct recode; no imputation and no silent conversion of nonresponse to a negative response", + "derivation_rule": ( + "code inherited from the ideation variable YUSUITHK; no local dictionary or schema attests it for this variable" + if str(code) in unattested_codes + else "direct recode; no imputation and no silent conversion of nonresponse to a negative response" + ), "source_file": clean["source_file"], "source_url": clean["source_url"], "page_or_section": clean["page_or_section"], - "review_status": "provisional", + "review_status": "needs_source" if str(code) in unattested_codes else "provisional", }) if missing_row: responses.append({ @@ -149,7 +158,16 @@ def build() -> dict[str, int]: for component in observed_gshs: package = component.get("source_package") or {} documents = package.get("documents") or [] - questionnaire = next((d for d in documents if "questionnaire" in d.get("role", "").lower()), None) + questionnaires = [d for d in documents if "questionnaire" in d.get("role", "").lower()] + # question_text below is the English core GSHS wording, so cite the instrument that + # actually contains it. Components shipping several language versions (Morocco, Wallis + # and Futuna) list every version, and taking the first one cited a French questionnaire + # for English text. Other-language versions stay recorded in source_language_evidence. + questionnaire = ( + next((d for d in questionnaires if "english" in d.get("role", "").lower()), None) + or next((d for d in questionnaires if d.get("role", "").strip().lower() == "questionnaire"), None) + or (questionnaires[0] if questionnaires else None) + ) source_file = questionnaire.get("source_file", "") if questionnaire else "" source_url = "" if source_file else component.get("catalog_url", "") language = "; ".join(package.get("source_language_evidence") or []) or "source language not established" @@ -250,7 +268,12 @@ def build() -> dict[str, int]: designs.append({ "survey_id": f"YRBS::{year}", "component_id": f"YRBS_NATIONAL::{year}", "survey": "YRBS", "country": "United States", "year": year, - "population": "students in grades 9-12 attending public and private schools in the United States", + # See 1995/1997 National User Guides: frame excludes a majority of Louisiana. + "population": ( + "students in grades 9-12 attending public and private schools in the fifty states and the District of Columbia, excluding a majority of Louisiana" + if year in {"1995", "1997"} + else "students in grades 9-12 attending public and private schools in the United States" + ), "weight_field": "weight", "psu_field": "psu", "stratum_field": "stratum", "variance_method": "complex-survey design-based variance accounting for multistage clustering; Taylor linearization supported", "namespace_rule": "prefix PSU and stratum with survey year before pooling", @@ -288,7 +311,8 @@ def build() -> dict[str, int]: "page_or_section": source.get("page_or_section") or variable, "reviewer_1": "", "reviewer_2": "", "review_status": "provisional", "notes": source.get("notes", ""), - }, {str(k): str(v) for k, v in (source.get("answer_options") or nsduh_options).items()}, missing_row=False) + }, {str(k): str(v) for k, v in (source.get("answer_options") or nsduh_options).items()}, missing_row=False, + unattested=source.get("unattested_answer_options")) if not available: conflicts.append({ "conflict_id": f"NSDUH_AVAILABILITY::{year}::{variable}", "survey": "NSDUH", "year": year, diff --git a/research/stage1/build_primary_outcome_registry.py b/research/stage1/build_primary_outcome_registry.py index 6fbf034..a6deb28 100644 --- a/research/stage1/build_primary_outcome_registry.py +++ b/research/stage1/build_primary_outcome_registry.py @@ -46,6 +46,21 @@ GSHS_QUESTION = { } NSDUH_YOUTH_RAW = {"YUSUITHK": "suicide_ideation", "YUSUIPLN": "suicide_plan"} NSDUH_YOUTH_DERIVED = {"YUSUITHKYR": "suicide_ideation", "YUSUIPLNYR": "suicide_plan"} +# Local MRB specification PDFs that carry YSUI03 verbatim, e.g. 2021 web specs: +# "YSUI03 [IF CURNTAGE = 12 - 17] During the past 12 months, did you try to kill yourself?" +# Verified by direct render of each file; this closes wording, population and skip logic. +NSDUH_ATTEMPT_LOCAL_SPECS = { + 2021: "Dataset/NSDUH/2021NSDUHMRBWebSpecs101422.pdf", + 2022: "Dataset/NSDUH/2022NSDUHmrbCAISpecs070722.pdf", + 2023: "Dataset/NSDUH/2023NSDUHmrbWebCAISpecs013123.pdf", + 2024: "Dataset/NSDUH/2024-nsduh-mrb-eng-web-specs.pdf", +} + +# The instrument offers only codes 1-4 plus DK/REF for YSUI03. The five dataset +# missing codes below are inherited from YUSUITHK and are attested by no local +# dictionary or schema for this variable (YUSUICTRY is absent from all of them). +NSDUH_ATTEMPT_UNATTESTED_CODES = ["85", "94", "97", "98", "99"] + NSDUH_ATTEMPT_SOURCES = { 2021: "https://www.samhsa.gov/data/sites/default/files/2021-09/NSDUHRDCCodebook2021.pdf", 2022: "https://www.samhsa.gov/data/sites/default/files/NSDUH-RDC/NSDUHRDCCodebook2022.pdf", @@ -207,7 +222,7 @@ def build_nsduh() -> tuple[list[dict], list[dict], list[dict]]: "source_variable": "YUSUICTRY", "construct": "suicide_attempt", "record_type": "administered_raw_candidate_not_in_local_public_use_file", - "population": "adolescents aged 12–17", + "population": "adolescents aged 12-17 (instrument condition IF CURNTAGE = 12 - 17)", "time_window": "past 12 months", "question_text": "During the past 12 months, did you try to kill yourself?", "source_label": "YOUTH TRY TO KILL YOURSELF PAST 12 MONTHS", @@ -222,18 +237,19 @@ def build_nsduh() -> tuple[list[dict], list[dict], list[dict]]: "98": "BLANK (NO ANSWER)", "99": "LEGITIMATE SKIP", }, - "source_file": None, + "source_file": NSDUH_ATTEMPT_LOCAL_SPECS.get(year), "source_url": source_url, "page_or_section": "YSUI03", + "unattested_answer_options": list(NSDUH_ATTEMPT_UNATTESTED_CODES), "present_in_full_table": "YUSUICTRY" in full_cols, "present_in_selected_table": "YUSUICTRY" in selected_cols, "data_access_status": "administered_but_not_available_in_local_public_use_package", - "source_verification_status": "official_indexed_text_verified_pdf_render_pending", + "source_verification_status": "local_instrument_pdf_verified", "review_status": "provisional", "reviewer_1": None, "reviewer_2": None, "link_status": "not_evaluated", - "notes": "Asked of all adolescents aged 12–17, regardless of ideation response. Current local public-use schema omits the raw and recoded attempt variables; this row is not analyzable without an authorized source.", + "notes": "Asked of all adolescents aged 12-17, regardless of ideation response, per the local MRB specification. Codes 85/94/97/98/99 are inherited from YUSUITHK and unattested for this variable. Current local public-use schema omits the raw and recoded attempt variables; this row is not analyzable without an authorized source.", } )