fix(stage1): correct source attribution at the build sources

Applies the three source-attested corrections from the signed reviews.
All are made in the generators, not the generated CSVs, which would be
overwritten on the next build.

YRBS population: the 1995 and 1997 National User Guides state the frame
covers "the fifty states and the District of Columbia, excluding a
majority of Louisiana". 1993 and 1999 carry no such restriction, so the
string is now year-conditional in both the item audit and the design
record instead of claiming plain national coverage for every year.

NSDUH attempt: source_file was None because the codebook render was
pending. The local MRB specification PDFs carry YSUI03 verbatim for all
four years, including the [IF CURNTAGE = 12 - 17] condition, so each
year now cites its local instrument and the status moves to
local_instrument_pdf_verified.

NSDUH attempt codes: YUSUICTRY appears in no local dictionary or schema,
yet carried nine codes identical to the ideation variable YUSUITHK. The
instrument offers only 1-4 plus DK/REF, so 85/94/97/98/99 are now marked
unattested and inherited, and those dictionary rows become needs_source.

GSHS citation: the questionnaire selector took the first document whose
role contained "questionnaire", which cited a French instrument for
English question_text in the Morocco and Wallis and Futuna components.
It now prefers the version matching the stored wording. Verified against
all six components; the other four are unchanged.

Generated artefacts are deliberately NOT rebuilt here: that regenerates
the item bank and the human review queue and is gate-relevant.

Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_016yuKikAXbByLvsUnh55SRd

research/stage1/audit_yrbs_primary_outcome_sources.py

@@ -147,7 +147,14 @@ def build_audit() -> dict:
                     "time_window": "past 12 months",
                     "response_type": "binary" if construct != "suicide_attempt" else "ordinal_frequency",
                     "answer_options": OPTIONS[construct],
-                    "population": "students in grades 9-12 attending public and private schools in the United States; detailed eligibility remains subject to annual design review",
+                    # 1995 and 1997 National User Guides state the frame covers "the fifty states
+                    # and the District of Columbia, excluding a majority of Louisiana". 1993 and 1999
+                    # carry no such restriction.
+                    "population": (
+                        "students in grades 9-12 attending public and private schools in the fifty states and the District of Columbia, excluding a majority of Louisiana; detailed eligibility remains subject to annual design review"
+                        if str(year) in {"1995", "1997"}
+                        else "students in grades 9-12 attending public and private schools in the United States; detailed eligibility remains subject to annual design review"
+                    ),
                     "language": "English questionnaire source; actual accommodation/language coverage not established by this audit",
                     "questionnaire_source_file": str(questionnaire.relative_to(ROOT)).replace("\\", "/"),
                     "questionnaire_page_or_section": q_section,

research/stage1/build_primary_outcome_package.py

@@ -88,7 +88,12 @@ def build() -> dict[str, int]:
         "derivation_rule", "source_file", "source_url", "page_or_section", "review_status",
     ]
 
-    def add_item(row: dict[str, Any], options: dict[str, str], missing_row: bool = False) -> None:
+    def add_item(row: dict[str, Any], options: dict[str, str], missing_row: bool = False,
+                 unattested: list[str] | None = None) -> None:
+        # Codes listed in `unattested` appear in the delivered data but are attested by no
+        # local dictionary or schema for this variable; mark them instead of silently
+        # presenting them as sourced.
+        unattested_codes = {str(c) for c in (unattested or [])}
         clean = {field: stringify(row.get(field)) for field in item_fields}
         clean["text_hash"] = hashlib.sha256(clean["question_text"].encode("utf-8")).hexdigest()
         items.append(clean)
@@ -101,11 +106,15 @@ def build() -> dict[str, int]:
                 "canonical_code": canonical,
                 "missing_type": missing_type,
                 "is_observed_response": stringify(observed),
-                "derivation_rule": "direct recode; no imputation and no silent conversion of nonresponse to a negative response",
+                "derivation_rule": (
+                    "code inherited from the ideation variable YUSUITHK; no local dictionary or schema attests it for this variable"
+                    if str(code) in unattested_codes
+                    else "direct recode; no imputation and no silent conversion of nonresponse to a negative response"
+                ),
                 "source_file": clean["source_file"],
                 "source_url": clean["source_url"],
                 "page_or_section": clean["page_or_section"],
-                "review_status": "provisional",
+                "review_status": "needs_source" if str(code) in unattested_codes else "provisional",
             })
         if missing_row:
             responses.append({
@@ -149,7 +158,16 @@ def build() -> dict[str, int]:
     for component in observed_gshs:
         package = component.get("source_package") or {}
         documents = package.get("documents") or []
-        questionnaire = next((d for d in documents if "questionnaire" in d.get("role", "").lower()), None)
+        questionnaires = [d for d in documents if "questionnaire" in d.get("role", "").lower()]
+        # question_text below is the English core GSHS wording, so cite the instrument that
+        # actually contains it. Components shipping several language versions (Morocco, Wallis
+        # and Futuna) list every version, and taking the first one cited a French questionnaire
+        # for English text. Other-language versions stay recorded in source_language_evidence.
+        questionnaire = (
+            next((d for d in questionnaires if "english" in d.get("role", "").lower()), None)
+            or next((d for d in questionnaires if d.get("role", "").strip().lower() == "questionnaire"), None)
+            or (questionnaires[0] if questionnaires else None)
+        )
         source_file = questionnaire.get("source_file", "") if questionnaire else ""
         source_url = "" if source_file else component.get("catalog_url", "")
         language = "; ".join(package.get("source_language_evidence") or []) or "source language not established"
@@ -250,7 +268,12 @@ def build() -> dict[str, int]:
         designs.append({
             "survey_id": f"YRBS::{year}", "component_id": f"YRBS_NATIONAL::{year}", "survey": "YRBS",
             "country": "United States", "year": year,
-            "population": "students in grades 9-12 attending public and private schools in the United States",
+            # See 1995/1997 National User Guides: frame excludes a majority of Louisiana.
+            "population": (
+                "students in grades 9-12 attending public and private schools in the fifty states and the District of Columbia, excluding a majority of Louisiana"
+                if year in {"1995", "1997"}
+                else "students in grades 9-12 attending public and private schools in the United States"
+            ),
             "weight_field": "weight", "psu_field": "psu", "stratum_field": "stratum",
             "variance_method": "complex-survey design-based variance accounting for multistage clustering; Taylor linearization supported",
             "namespace_rule": "prefix PSU and stratum with survey year before pooling",
@@ -288,7 +311,8 @@ def build() -> dict[str, int]:
             "page_or_section": source.get("page_or_section") or variable,
             "reviewer_1": "", "reviewer_2": "", "review_status": "provisional",
             "notes": source.get("notes", ""),
-        }, {str(k): str(v) for k, v in (source.get("answer_options") or nsduh_options).items()}, missing_row=False)
+        }, {str(k): str(v) for k, v in (source.get("answer_options") or nsduh_options).items()}, missing_row=False,
+            unattested=source.get("unattested_answer_options"))
         if not available:
             conflicts.append({
                 "conflict_id": f"NSDUH_AVAILABILITY::{year}::{variable}", "survey": "NSDUH", "year": year,

research/stage1/build_primary_outcome_registry.py

@@ -46,6 +46,21 @@ GSHS_QUESTION = {
 }
 NSDUH_YOUTH_RAW = {"YUSUITHK": "suicide_ideation", "YUSUIPLN": "suicide_plan"}
 NSDUH_YOUTH_DERIVED = {"YUSUITHKYR": "suicide_ideation", "YUSUIPLNYR": "suicide_plan"}
+# Local MRB specification PDFs that carry YSUI03 verbatim, e.g. 2021 web specs:
+#   "YSUI03 [IF CURNTAGE = 12 - 17] During the past 12 months, did you try to kill yourself?"
+# Verified by direct render of each file; this closes wording, population and skip logic.
+NSDUH_ATTEMPT_LOCAL_SPECS = {
+    2021: "Dataset/NSDUH/2021NSDUHMRBWebSpecs101422.pdf",
+    2022: "Dataset/NSDUH/2022NSDUHmrbCAISpecs070722.pdf",
+    2023: "Dataset/NSDUH/2023NSDUHmrbWebCAISpecs013123.pdf",
+    2024: "Dataset/NSDUH/2024-nsduh-mrb-eng-web-specs.pdf",
+}
+
+# The instrument offers only codes 1-4 plus DK/REF for YSUI03. The five dataset
+# missing codes below are inherited from YUSUITHK and are attested by no local
+# dictionary or schema for this variable (YUSUICTRY is absent from all of them).
+NSDUH_ATTEMPT_UNATTESTED_CODES = ["85", "94", "97", "98", "99"]
+
 NSDUH_ATTEMPT_SOURCES = {
     2021: "https://www.samhsa.gov/data/sites/default/files/2021-09/NSDUHRDCCodebook2021.pdf",
     2022: "https://www.samhsa.gov/data/sites/default/files/NSDUH-RDC/NSDUHRDCCodebook2022.pdf",
@@ -207,7 +222,7 @@ def build_nsduh() -> tuple[list[dict], list[dict], list[dict]]:
                 "source_variable": "YUSUICTRY",
                 "construct": "suicide_attempt",
                 "record_type": "administered_raw_candidate_not_in_local_public_use_file",
-                "population": "adolescents aged 12–17",
+                "population": "adolescents aged 12-17 (instrument condition IF CURNTAGE = 12 - 17)",
                 "time_window": "past 12 months",
                 "question_text": "During the past 12 months, did you try to kill yourself?",
                 "source_label": "YOUTH TRY TO KILL YOURSELF PAST 12 MONTHS",
@@ -222,18 +237,19 @@ def build_nsduh() -> tuple[list[dict], list[dict], list[dict]]:
                     "98": "BLANK (NO ANSWER)",
                     "99": "LEGITIMATE SKIP",
                 },
-                "source_file": None,
+                "source_file": NSDUH_ATTEMPT_LOCAL_SPECS.get(year),
                 "source_url": source_url,
                 "page_or_section": "YSUI03",
+                "unattested_answer_options": list(NSDUH_ATTEMPT_UNATTESTED_CODES),
                 "present_in_full_table": "YUSUICTRY" in full_cols,
                 "present_in_selected_table": "YUSUICTRY" in selected_cols,
                 "data_access_status": "administered_but_not_available_in_local_public_use_package",
-                "source_verification_status": "official_indexed_text_verified_pdf_render_pending",
+                "source_verification_status": "local_instrument_pdf_verified",
                 "review_status": "provisional",
                 "reviewer_1": None,
                 "reviewer_2": None,
                 "link_status": "not_evaluated",
-                "notes": "Asked of all adolescents aged 12–17, regardless of ideation response. Current local public-use schema omits the raw and recoded attempt variables; this row is not analyzable without an authorized source.",
+                "notes": "Asked of all adolescents aged 12-17, regardless of ideation response, per the local MRB specification. Codes 85/94/97/98/99 are inherited from YUSUITHK and unattested for this variable. Current local public-use schema omits the raw and recoded attempt variables; this row is not analyzable without an authorized source.",
             }
         )