Jinovo/language-conditioned-psychometric-harmonization-model
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat(stage1): verify NSDUH youth attempt availability

Browse Source
This commit is contained in:
Jinotech
2026-09-20 08:16:33 +12:00
parent c84ff48688
commit 7d50ae61ca
8 changed files with 534 additions and 22 deletions
Download Patch File Download Diff File Expand all files Collapse all files
research/stage1/build_primary_outcome_registry.py
+53 -8
View File
@@ -34,6 +34,12 @@ GSHS_CONSTRUCT = {
}
NSDUH_YOUTH_RAW = {"YUSUITHK": "suicide_ideation", "YUSUIPLN": "suicide_plan"}
NSDUH_YOUTH_DERIVED = {"YUSUITHKYR": "suicide_ideation", "YUSUIPLNYR": "suicide_plan"}
NSDUH_ATTEMPT_SOURCES = {
2021: "https://www.samhsa.gov/data/sites/default/files/2021-09/NSDUHRDCCodebook2021.pdf",
2022: "https://www.samhsa.gov/data/sites/default/files/NSDUH-RDC/NSDUHRDCCodebook2022.pdf",
2023: "https://www.samhsa.gov/data/system/files/media-puf-file/2023-nsduh-rdc-codebook.pdf",
2024: "https://www.samhsa.gov/data/sites/default/files/reports/rpt44494/2024-nsduh-mrb-eng-web-specs.pdf",
}
def clean_text(value: object) -> str:
@@ -158,6 +164,44 @@ def build_nsduh() -> tuple[list[dict], list[dict], list[dict]]:
item["notes"] = "Keep 'not sure' and refusal/nonresponse distinct until skip and missing rules are verified."
raw_records.append(item)
for year, source_url in NSDUH_ATTEMPT_SOURCES.items():
raw_records.append(
{
"survey": "NSDUH",
"year": year,
"source_variable": "YUSUICTRY",
"construct": "suicide_attempt",
"record_type": "administered_raw_candidate_not_in_local_public_use_file",
"population": "adolescents aged 12–17",
"time_window": "past 12 months",
"question_text": "During the past 12 months, did you try to kill yourself?",
"source_label": "YOUTH TRY TO KILL YOURSELF PAST 12 MONTHS",
"answer_options": {
"1": "Yes",
"2": "No",
"3": "I'm not sure",
"4": "I don't want to answer",
"85": "BAD DATA Logically assigned",
"94": "DON'T KNOW",
"97": "REFUSED",
"98": "BLANK (NO ANSWER)",
"99": "LEGITIMATE SKIP",
},
"source_file": None,
"source_url": source_url,
"page_or_section": "YSUI03",
"present_in_full_table": "YUSUICTRY" in full_cols,
"present_in_selected_table": "YUSUICTRY" in selected_cols,
"data_access_status": "administered_but_not_available_in_local_public_use_package",
"source_verification_status": "official_indexed_text_verified_pdf_render_pending",
"review_status": "provisional",
"reviewer_1": None,
"reviewer_2": None,
"link_status": "not_evaluated",
"notes": "Asked of all adolescents aged 12–17, regardless of ideation response. Current local public-use schema omits the raw and recoded attempt variables; this row is not analyzable without an authorized source.",
}
)
for row in frame[frame["variable"].isin(NSDUH_YOUTH_DERIVED)].to_dict("records"):
item = common(row, NSDUH_YOUTH_DERIVED[row["variable"]], "derived_youth_indicator")
item["question_text"] = None
@@ -191,7 +235,7 @@ def render_markdown(registry: dict) -> str:
lines = [
"# Primary outcome source registry",
"",
"Version: 1.0 | Status: provisional | Generated: 2026-09-20",
"Version: 1.1 | Status: provisional | Generated: 2026-09-20",
"",
"## Scope and decision boundary",
"",
@@ -206,7 +250,7 @@ def render_markdown(registry: dict) -> str:
boundary = {
"GSHS": "Workbook labels/options only; component questionnaires still require direct review.",
"YRBS": "51 annual candidates; 2021/2023 data remain quarantined; annual PDFs require direct review.",
"NSDUH": "Official API metadata supports youth ideation/plan only; no youth attempt field appears in the local full dictionary.",
"NSDUH": "Attempt is administered as YSUI03/YUSUICTRY, but the raw and recoded attempt fields are absent from the local public-use package.",
}[survey]
lines.append(
f"| {survey} | {counts[(survey, 'suicide_ideation')]} | {counts[(survey, 'suicide_plan')]} | {counts[(survey, 'suicide_attempt')]} | {boundary} |"
@@ -222,7 +266,8 @@ def render_markdown(registry: dict) -> str:
"- YRBS metadata contains stale absolute paths from another machine and some automatically extracted neighboring-text contamination. Only exact primary-construct rows were retained, and their direct PDF review is still pending.",
"- YRBS 2021 and 2023 item metadata are retained for future migration checks, but their respondent data are quarantined pending layout/import validation.",
"- NSDUH adult, COVID-conditioned, imputed, recoded, and composite variables are not treated as primary raw youth items.",
"- NSDUH youth attempt questions `YSUI03`–`YSUI05` are mentioned in notes, but no corresponding youth attempt variable appears in the local complete dictionary. Attempt coverage is therefore recorded as a source gap, not inferred from ideation wording.",
"- Official restricted-use codebooks and the 2024 questionnaire verify `YSUI03`/`YUSUICTRY` as the youth attempt item. It is administered independently of the ideation response, but is absent from the local public-use data and cannot be analyzed from the current package.",
"- `YSUI04`/`YUSUICMED` and `YSUI05`/`YUSUIHOSP` are conditional severity/follow-up items, not substitutes for the primary attempt item.",
"",
"## Gate 1 status",
"",
@@ -241,7 +286,7 @@ def main() -> None:
yrbs = build_yrbs()
nsduh_raw, nsduh_derived, nsduh_excluded = build_nsduh()
registry = {
"schema_version": "1.0",
"schema_version": "1.1",
"generated_on": "2026-09-20",
"status": "provisional",
"allowed_uses": ["pipeline_development", "source_review_queue"],
@@ -249,13 +294,13 @@ def main() -> None:
"candidate_records": gshs + yrbs + nsduh_raw,
"derived_records_not_independent_items": nsduh_derived,
"excluded_nsduh_records": nsduh_excluded,
"source_gaps": [
"availability_constraints": [
{
"survey": "NSDUH",
"construct": "suicide_attempt",
"status": "not_found_in_local_complete_dictionary",
"evidence": "YSUI03–YSUI05 are named in 2022–2024 module-move notes, but no youth attempt variable is exposed in the local 2021–2024 dictionary.",
"next_check": "Review annual youth questionnaires/codebooks and public-use suppression/derivation notes.",
"status": "administered_but_unavailable_in_local_public_use_package",
"evidence": "Official restricted-use codebooks map YSUI03 to YUSUICTRY; the 2024 public-use guide states that the recoded youth attempt variable is excluded due to disclosure risk; neither raw nor recoded attempt fields occur in the local full/selected schemas.",
"analysis_consequence": "NSDUH can externally validate youth ideation and plan with current local microdata, but not youth attempt. Attempt analysis requires authorized restricted-use access or a separately justified aggregate-data design.",
}
],
"review_policy": {
research/stage1/nsduh_youth_attempt_source_audit.json
+97
View File
@@ -0,0 +1,97 @@
{
"schema_version": "1.0",
"generated_on": "2026-09-20",
"status": "provisional_source_verification",
"workflow_stage": 1,
"official_sources": [
{
"year": 2021,
"type": "restricted_use_codebook",
"url": "https://www.samhsa.gov/data/sites/default/files/2021-09/NSDUHRDCCodebook2021.pdf",
"verified_claim": "YSUI03 maps to YUSUICTRY and asks whether the youth tried to kill themself in the past 12 months."
},
{
"year": 2022,
"type": "restricted_use_codebook",
"url": "https://www.samhsa.gov/data/sites/default/files/NSDUH-RDC/NSDUHRDCCodebook2022.pdf",
"verified_claim": "YSUI03/YUSUICTRY is the attempt item; YSUI04/YUSUICMED and YSUI05/YUSUIHOSP are conditional follow-ups."
},
{
"year": 2023,
"type": "restricted_use_codebook",
"url": "https://www.samhsa.gov/data/system/files/media-puf-file/2023-nsduh-rdc-codebook.pdf",
"verified_claim": "YSUI03 maps to YUSUICTRY; the analytic recode is documented as YUSUICTRYYR while composite formulas reference YUSUITRYYR, requiring name reconciliation before restricted-use analysis."
},
{
"year": 2024,
"type": "questionnaire_web_specification",
"url": "https://www.samhsa.gov/data/sites/default/files/reports/rpt44494/2024-nsduh-mrb-eng-web-specs.pdf",
"verified_claim": "YSUI03 is asked of respondents aged 12–17; YSUI04 is routed only when YSUI03=1 and YSUI05 only when YSUI04=1."
},
{
"year": 2024,
"type": "methodological_summary",
"url": "https://www.samhsa.gov/data/sites/default/files/reports/rpt56485/2024-nsduh-method-summ-defs/2024-nsduh-method-summ-defs/2024-nsduh-method-summary-defs.htm",
"verified_claim": "All adolescents aged 12–17 are asked plan and attempt regardless of serious-thought response; youth variables are not statistically imputed."
},
{
"year": 2024,
"type": "public_use_data_users_guide",
"url": "https://www.samhsa.gov/data/sites/default/files/reports/rpt56775/2024-nsduh-puf-data-users-guide.pdf",
"verified_claim": "The past-year youth suicide-attempt variable YUSUITRYYR is excluded from the public-use file due to disclosure risk."
}
],
"local_availability": [
{
"variable": "YUSUICTRY",
"in_complete_dictionary": false,
"in_full_parquet": false,
"in_selected_parquet": false
},
{
"variable": "YUSUITRYYR",
"in_complete_dictionary": false,
"in_full_parquet": false,
"in_selected_parquet": false
},
{
"variable": "YUSUICTRYYR",
"in_complete_dictionary": false,
"in_full_parquet": false,
"in_selected_parquet": false
},
{
"variable": "YUSUICMED",
"in_complete_dictionary": false,
"in_full_parquet": false,
"in_selected_parquet": false
},
{
"variable": "YUSUIMEDYR",
"in_complete_dictionary": false,
"in_full_parquet": false,
"in_selected_parquet": false
},
{
"variable": "YUSUIHOSP",
"in_complete_dictionary": false,
"in_full_parquet": false,
"in_selected_parquet": false
},
{
"variable": "YUSUIHOSYR",
"in_complete_dictionary": false,
"in_full_parquet": false,
"in_selected_parquet": false
}
],
"verified_item_logic": {
"YSUI03": "Asked of all respondents aged 12–17; not conditional on YSUI01 or YSUI02.",
"YSUI04": "Asked only if YSUI03=1; medical-attention follow-up.",
"YSUI05": "Asked only if YSUI04=1; overnight-hospital follow-up.",
"module_location": "Youth mental health service utilization in 2021; youth experiences from 2022 onward."
},
"conclusion": "The attempt item was administered, but no raw, recoded, medical-attention, or hospital-follow-up youth-attempt fields are available in the current local public-use package.",
"analysis_decision": "Do not use NSDUH as respondent-level external validation for youth suicide attempt with the current data. Retain ideation and plan only; pursue restricted-use access or an aggregate-only alternative if attempt validation is required.",
"remaining_review": "Direct PDF page rendering was blocked by HTTP 403. Exact indexed official text and HTML were verified; two-human-reviewer sign-off remains outstanding."
}
research/stage1/nsduh_youth_attempt_source_audit.md
+41
View File
@@ -0,0 +1,41 @@
# NSDUH youth suicide-attempt source audit
Version: 1.0 | Status: provisional source verification | Date: 2026-09-20
## Result
Official NSDUH materials verify that `YSUI03` asks adolescents aged 12–17 whether they tried to kill themselves in the past 12 months. The source variable is `YUSUICTRY`; `YSUI04`/`YUSUICMED` and `YSUI05`/`YUSUIHOSP` are conditional severity follow-ups. Plan and attempt are asked regardless of the answer to the ideation item.
None of the attempt or follow-up fields is present in the current local public-use dictionary or Parquet schemas:
| Variable | Complete dictionary | Full Parquet | Selected Parquet |
|---|---:|---:|---:|
| `YUSUICTRY` | false | false | false |
| `YUSUITRYYR` | false | false | false |
| `YUSUICTRYYR` | false | false | false |
| `YUSUICMED` | false | false | false |
| `YUSUIMEDYR` | false | false | false |
| `YUSUIHOSP` | false | false | false |
| `YUSUIHOSYR` | false | false | false |
The 2024 public-use guide explicitly states that the recoded youth attempt variable `YUSUITRYYR` is excluded because of disclosure risk. The current local package is sourced from the official public-use DAS API, which explains why its complete dictionary ends after youth ideation and plan rather than proving the attempt item was never administered.
## Stage 1 decision
- Register `YSUI03`/`YUSUICTRY` as an administered, source-verified, but locally unavailable candidate.
- Do not derive attempt from ideation wording, plan responses, COVID-conditioned questions, or medical/hospital follow-ups.
- With current microdata, NSDUH may validate youth ideation and plan only. Youth attempt requires authorized restricted-use data or a separately designed aggregate-data analysis.
- Keep all records `provisional`; direct PDF page rendering and two-human-reviewer approval remain outstanding.
## Official sources
- [2021 restricted-use codebook](https://www.samhsa.gov/data/sites/default/files/2021-09/NSDUHRDCCodebook2021.pdf)
- [2022 restricted-use codebook](https://www.samhsa.gov/data/sites/default/files/NSDUH-RDC/NSDUHRDCCodebook2022.pdf)
- [2023 restricted-use codebook](https://www.samhsa.gov/data/system/files/media-puf-file/2023-nsduh-rdc-codebook.pdf)
- [2024 questionnaire web specification](https://www.samhsa.gov/data/sites/default/files/reports/rpt44494/2024-nsduh-mrb-eng-web-specs.pdf)
- [2024 methodological summary](https://www.samhsa.gov/data/sites/default/files/reports/rpt56485/2024-nsduh-method-summ-defs/2024-nsduh-method-summ-defs/2024-nsduh-method-summary-defs.htm)
- [2024 public-use data users' guide](https://www.samhsa.gov/data/sites/default/files/reports/rpt56775/2024-nsduh-puf-data-users-guide.pdf)
## Reproducibility and limitation
`verify_nsduh_youth_attempt.py` rechecks the repository-local dictionary and both Parquet schemas. SAMHSA's PDF endpoints returned HTTP 403 to direct readers during this action, so the official documents were verified through their indexed official text and the accessible SAMHSA HTML material; visual page-level PDF verification is not claimed.
research/stage1/primary_outcome_registry.json
+141 -5
View File
@@ -1,5 +1,5 @@
{
"schema_version": "1.0",
"schema_version": "1.1",
"generated_on": "2026-09-20",
"status": "provisional",
"allowed_uses": [
@@ -2168,6 +2168,142 @@
"reviewer_2": null,
"link_status": "not_evaluated",
"notes": "Keep 'not sure' and refusal/nonresponse distinct until skip and missing rules are verified."
},
{
"survey": "NSDUH",
"year": 2021,
"source_variable": "YUSUICTRY",
"construct": "suicide_attempt",
"record_type": "administered_raw_candidate_not_in_local_public_use_file",
"population": "adolescents aged 12–17",
"time_window": "past 12 months",
"question_text": "During the past 12 months, did you try to kill yourself?",
"source_label": "YOUTH TRY TO KILL YOURSELF PAST 12 MONTHS",
"answer_options": {
"1": "Yes",
"2": "No",
"3": "I'm not sure",
"4": "I don't want to answer",
"85": "BAD DATA Logically assigned",
"94": "DON'T KNOW",
"97": "REFUSED",
"98": "BLANK (NO ANSWER)",
"99": "LEGITIMATE SKIP"
},
"source_file": null,
"source_url": "https://www.samhsa.gov/data/sites/default/files/2021-09/NSDUHRDCCodebook2021.pdf",
"page_or_section": "YSUI03",
"present_in_full_table": false,
"present_in_selected_table": false,
"data_access_status": "administered_but_not_available_in_local_public_use_package",
"source_verification_status": "official_indexed_text_verified_pdf_render_pending",
"review_status": "provisional",
"reviewer_1": null,
"reviewer_2": null,
"link_status": "not_evaluated",
"notes": "Asked of all adolescents aged 12–17, regardless of ideation response. Current local public-use schema omits the raw and recoded attempt variables; this row is not analyzable without an authorized source."
},
{
"survey": "NSDUH",
"year": 2022,
"source_variable": "YUSUICTRY",
"construct": "suicide_attempt",
"record_type": "administered_raw_candidate_not_in_local_public_use_file",
"population": "adolescents aged 12–17",
"time_window": "past 12 months",
"question_text": "During the past 12 months, did you try to kill yourself?",
"source_label": "YOUTH TRY TO KILL YOURSELF PAST 12 MONTHS",
"answer_options": {
"1": "Yes",
"2": "No",
"3": "I'm not sure",
"4": "I don't want to answer",
"85": "BAD DATA Logically assigned",
"94": "DON'T KNOW",
"97": "REFUSED",
"98": "BLANK (NO ANSWER)",
"99": "LEGITIMATE SKIP"
},
"source_file": null,
"source_url": "https://www.samhsa.gov/data/sites/default/files/NSDUH-RDC/NSDUHRDCCodebook2022.pdf",
"page_or_section": "YSUI03",
"present_in_full_table": false,
"present_in_selected_table": false,
"data_access_status": "administered_but_not_available_in_local_public_use_package",
"source_verification_status": "official_indexed_text_verified_pdf_render_pending",
"review_status": "provisional",
"reviewer_1": null,
"reviewer_2": null,
"link_status": "not_evaluated",
"notes": "Asked of all adolescents aged 12–17, regardless of ideation response. Current local public-use schema omits the raw and recoded attempt variables; this row is not analyzable without an authorized source."
},
{
"survey": "NSDUH",
"year": 2023,
"source_variable": "YUSUICTRY",
"construct": "suicide_attempt",
"record_type": "administered_raw_candidate_not_in_local_public_use_file",
"population": "adolescents aged 12–17",
"time_window": "past 12 months",
"question_text": "During the past 12 months, did you try to kill yourself?",
"source_label": "YOUTH TRY TO KILL YOURSELF PAST 12 MONTHS",
"answer_options": {
"1": "Yes",
"2": "No",
"3": "I'm not sure",
"4": "I don't want to answer",
"85": "BAD DATA Logically assigned",
"94": "DON'T KNOW",
"97": "REFUSED",
"98": "BLANK (NO ANSWER)",
"99": "LEGITIMATE SKIP"
},
"source_file": null,
"source_url": "https://www.samhsa.gov/data/system/files/media-puf-file/2023-nsduh-rdc-codebook.pdf",
"page_or_section": "YSUI03",
"present_in_full_table": false,
"present_in_selected_table": false,
"data_access_status": "administered_but_not_available_in_local_public_use_package",
"source_verification_status": "official_indexed_text_verified_pdf_render_pending",
"review_status": "provisional",
"reviewer_1": null,
"reviewer_2": null,
"link_status": "not_evaluated",
"notes": "Asked of all adolescents aged 12–17, regardless of ideation response. Current local public-use schema omits the raw and recoded attempt variables; this row is not analyzable without an authorized source."
},
{
"survey": "NSDUH",
"year": 2024,
"source_variable": "YUSUICTRY",
"construct": "suicide_attempt",
"record_type": "administered_raw_candidate_not_in_local_public_use_file",
"population": "adolescents aged 12–17",
"time_window": "past 12 months",
"question_text": "During the past 12 months, did you try to kill yourself?",
"source_label": "YOUTH TRY TO KILL YOURSELF PAST 12 MONTHS",
"answer_options": {
"1": "Yes",
"2": "No",
"3": "I'm not sure",
"4": "I don't want to answer",
"85": "BAD DATA Logically assigned",
"94": "DON'T KNOW",
"97": "REFUSED",
"98": "BLANK (NO ANSWER)",
"99": "LEGITIMATE SKIP"
},
"source_file": null,
"source_url": "https://www.samhsa.gov/data/sites/default/files/reports/rpt44494/2024-nsduh-mrb-eng-web-specs.pdf",
"page_or_section": "YSUI03",
"present_in_full_table": false,
"present_in_selected_table": false,
"data_access_status": "administered_but_not_available_in_local_public_use_package",
"source_verification_status": "official_indexed_text_verified_pdf_render_pending",
"review_status": "provisional",
"reviewer_1": null,
"reviewer_2": null,
"link_status": "not_evaluated",
"notes": "Asked of all adolescents aged 12–17, regardless of ideation response. Current local public-use schema omits the raw and recoded attempt variables; this row is not analyzable without an authorized source."
}
],
"derived_records_not_independent_items": [
@@ -2670,13 +2806,13 @@
"status": "excluded_from_primary_youth_item_candidates"
}
],
"source_gaps": [
"availability_constraints": [
{
"survey": "NSDUH",
"construct": "suicide_attempt",
"status": "not_found_in_local_complete_dictionary",
"evidence": "YSUI03–YSUI05 are named in 2022–2024 module-move notes, but no youth attempt variable is exposed in the local 2021–2024 dictionary.",
"next_check": "Review annual youth questionnaires/codebooks and public-use suppression/derivation notes."
"status": "administered_but_unavailable_in_local_public_use_package",
"evidence": "Official restricted-use codebooks map YSUI03 to YUSUICTRY; the 2024 public-use guide states that the recoded youth attempt variable is excluded due to disclosure risk; neither raw nor recoded attempt fields occur in the local full/selected schemas.",
"analysis_consequence": "NSDUH can externally validate youth ideation and plan with current local microdata, but not youth attempt. Attempt analysis requires authorized restricted-use access or a separately justified aggregate-data design."
}
],
"review_policy": {
research/stage1/primary_outcome_registry.md
+4 -3
View File
@@ -1,6 +1,6 @@
# Primary outcome source registry
Version: 1.0 | Status: provisional | Generated: 2026-09-20
Version: 1.1 | Status: provisional | Generated: 2026-09-20
## Scope and decision boundary
@@ -12,7 +12,7 @@ This registry captures source-linked candidates for suicide ideation, plan, and
|---|---:|---:|---:|---|
| GSHS | 2 | 2 | 2 | Workbook labels/options only; component questionnaires still require direct review. |
| YRBS | 17 | 17 | 17 | 51 annual candidates; 2021/2023 data remain quarantined; annual PDFs require direct review. |
| NSDUH | 4 | 4 | 0 | Official API metadata supports youth ideation/plan only; no youth attempt field appears in the local full dictionary. |
| NSDUH | 4 | 4 | 4 | Attempt is administered as YSUI03/YUSUICTRY, but the raw and recoded attempt fields are absent from the local public-use package. |
Counts are source-record counts, not distinct item families. YRBS counts include annual versions; NSDUH counts include 2021–2024 raw youth records only.
@@ -23,7 +23,8 @@ Counts are source-record counts, not distinct item families. YRBS counts include
- YRBS metadata contains stale absolute paths from another machine and some automatically extracted neighboring-text contamination. Only exact primary-construct rows were retained, and their direct PDF review is still pending.
- YRBS 2021 and 2023 item metadata are retained for future migration checks, but their respondent data are quarantined pending layout/import validation.
- NSDUH adult, COVID-conditioned, imputed, recoded, and composite variables are not treated as primary raw youth items.
- NSDUH youth attempt questions `YSUI03`–`YSUI05` are mentioned in notes, but no corresponding youth attempt variable appears in the local complete dictionary. Attempt coverage is therefore recorded as a source gap, not inferred from ideation wording.
- Official restricted-use codebooks and the 2024 questionnaire verify `YSUI03`/`YUSUICTRY` as the youth attempt item. It is administered independently of the ideation response, but is absent from the local public-use data and cannot be analyzed from the current package.
- `YSUI04`/`YUSUICMED` and `YSUI05`/`YUSUIHOSP` are conditional severity/follow-up items, not substitutes for the primary attempt item.
## Gate 1 status
research/stage1/verify_nsduh_youth_attempt.py
+148
View File
@@ -0,0 +1,148 @@
"""Verify local NSDUH youth-attempt availability against frozen official evidence."""
from __future__ import annotations
import json
from pathlib import Path
import pandas as pd
import pyarrow.parquet as pq
ROOT = Path(__file__).resolve().parents[2]
NSDUH = ROOT / "Dataset" / "可直接分析数据包_NSDUH_YRBS" / "NSDUH"
OUT_JSON = ROOT / "research" / "stage1" / "nsduh_youth_attempt_source_audit.json"
OUT_MD = ROOT / "research" / "stage1" / "nsduh_youth_attempt_source_audit.md"
TARGETS = [
"YUSUICTRY",
"YUSUITRYYR",
"YUSUICTRYYR",
"YUSUICMED",
"YUSUIMEDYR",
"YUSUIHOSP",
"YUSUIHOSYR",
]
OFFICIAL_SOURCES = [
{
"year": 2021,
"type": "restricted_use_codebook",
"url": "https://www.samhsa.gov/data/sites/default/files/2021-09/NSDUHRDCCodebook2021.pdf",
"verified_claim": "YSUI03 maps to YUSUICTRY and asks whether the youth tried to kill themself in the past 12 months.",
},
{
"year": 2022,
"type": "restricted_use_codebook",
"url": "https://www.samhsa.gov/data/sites/default/files/NSDUH-RDC/NSDUHRDCCodebook2022.pdf",
"verified_claim": "YSUI03/YUSUICTRY is the attempt item; YSUI04/YUSUICMED and YSUI05/YUSUIHOSP are conditional follow-ups.",
},
{
"year": 2023,
"type": "restricted_use_codebook",
"url": "https://www.samhsa.gov/data/system/files/media-puf-file/2023-nsduh-rdc-codebook.pdf",
"verified_claim": "YSUI03 maps to YUSUICTRY; the analytic recode is documented as YUSUICTRYYR while composite formulas reference YUSUITRYYR, requiring name reconciliation before restricted-use analysis.",
},
{
"year": 2024,
"type": "questionnaire_web_specification",
"url": "https://www.samhsa.gov/data/sites/default/files/reports/rpt44494/2024-nsduh-mrb-eng-web-specs.pdf",
"verified_claim": "YSUI03 is asked of respondents aged 12–17; YSUI04 is routed only when YSUI03=1 and YSUI05 only when YSUI04=1.",
},
{
"year": 2024,
"type": "methodological_summary",
"url": "https://www.samhsa.gov/data/sites/default/files/reports/rpt56485/2024-nsduh-method-summ-defs/2024-nsduh-method-summ-defs/2024-nsduh-method-summary-defs.htm",
"verified_claim": "All adolescents aged 12–17 are asked plan and attempt regardless of serious-thought response; youth variables are not statistically imputed.",
},
{
"year": 2024,
"type": "public_use_data_users_guide",
"url": "https://www.samhsa.gov/data/sites/default/files/reports/rpt56775/2024-nsduh-puf-data-users-guide.pdf",
"verified_claim": "The past-year youth suicide-attempt variable YUSUITRYYR is excluded from the public-use file due to disclosure risk.",
},
]
def columns(path: Path) -> set[str]:
return set(pq.ParquetFile(path).schema_arrow.names)
def main() -> None:
dictionary = pd.read_csv(NSDUH / "nsduh_complete_variable_dictionary.csv", dtype=str, keep_default_na=False)
dictionary_vars = set(dictionary["variable"])
full_vars = columns(NSDUH / "nsduh_2021_2024_full.parquet")
selected_vars = columns(NSDUH / "nsduh_2021_2024_selected.parquet")
availability = [
{
"variable": variable,
"in_complete_dictionary": variable in dictionary_vars,
"in_full_parquet": variable in full_vars,
"in_selected_parquet": variable in selected_vars,
}
for variable in TARGETS
]
result = {
"schema_version": "1.0",
"generated_on": "2026-09-20",
"status": "provisional_source_verification",
"workflow_stage": 1,
"official_sources": OFFICIAL_SOURCES,
"local_availability": availability,
"verified_item_logic": {
"YSUI03": "Asked of all respondents aged 12–17; not conditional on YSUI01 or YSUI02.",
"YSUI04": "Asked only if YSUI03=1; medical-attention follow-up.",
"YSUI05": "Asked only if YSUI04=1; overnight-hospital follow-up.",
"module_location": "Youth mental health service utilization in 2021; youth experiences from 2022 onward.",
},
"conclusion": "The attempt item was administered, but no raw, recoded, medical-attention, or hospital-follow-up youth-attempt fields are available in the current local public-use package.",
"analysis_decision": "Do not use NSDUH as respondent-level external validation for youth suicide attempt with the current data. Retain ideation and plan only; pursue restricted-use access or an aggregate-only alternative if attempt validation is required.",
"remaining_review": "Direct PDF page rendering was blocked by HTTP 403. Exact indexed official text and HTML were verified; two-human-reviewer sign-off remains outstanding.",
}
OUT_JSON.write_text(json.dumps(result, ensure_ascii=False, indent=2) + "\n", encoding="utf-8")
table = ["| Variable | Complete dictionary | Full Parquet | Selected Parquet |", "|---|---:|---:|---:|"]
for row in availability:
table.append(
f"| `{row['variable']}` | {str(row['in_complete_dictionary']).lower()} | {str(row['in_full_parquet']).lower()} | {str(row['in_selected_parquet']).lower()} |"
)
report = f"""# NSDUH youth suicide-attempt source audit
Version: 1.0 | Status: provisional source verification | Date: 2026-09-20
## Result
Official NSDUH materials verify that `YSUI03` asks adolescents aged 12–17 whether they tried to kill themselves in the past 12 months. The source variable is `YUSUICTRY`; `YSUI04`/`YUSUICMED` and `YSUI05`/`YUSUIHOSP` are conditional severity follow-ups. Plan and attempt are asked regardless of the answer to the ideation item.
None of the attempt or follow-up fields is present in the current local public-use dictionary or Parquet schemas:
{chr(10).join(table)}
The 2024 public-use guide explicitly states that the recoded youth attempt variable `YUSUITRYYR` is excluded because of disclosure risk. The current local package is sourced from the official public-use DAS API, which explains why its complete dictionary ends after youth ideation and plan rather than proving the attempt item was never administered.
## Stage 1 decision
- Register `YSUI03`/`YUSUICTRY` as an administered, source-verified, but locally unavailable candidate.
- Do not derive attempt from ideation wording, plan responses, COVID-conditioned questions, or medical/hospital follow-ups.
- With current microdata, NSDUH may validate youth ideation and plan only. Youth attempt requires authorized restricted-use data or a separately designed aggregate-data analysis.
- Keep all records `provisional`; direct PDF page rendering and two-human-reviewer approval remain outstanding.
## Official sources
- [2021 restricted-use codebook]({OFFICIAL_SOURCES[0]['url']})
- [2022 restricted-use codebook]({OFFICIAL_SOURCES[1]['url']})
- [2023 restricted-use codebook]({OFFICIAL_SOURCES[2]['url']})
- [2024 questionnaire web specification]({OFFICIAL_SOURCES[3]['url']})
- [2024 methodological summary]({OFFICIAL_SOURCES[4]['url']})
- [2024 public-use data users' guide]({OFFICIAL_SOURCES[5]['url']})
## Reproducibility and limitation
`verify_nsduh_youth_attempt.py` rechecks the repository-local dictionary and both Parquet schemas. SAMHSA's PDF endpoints returned HTTP 403 to direct readers during this action, so the official documents were verified through their indexed official text and the accessible SAMHSA HTML material; visual page-level PDF verification is not claimed.
"""
OUT_MD.write_text(report, encoding="utf-8")
print(json.dumps({"variables_checked": len(availability), "all_absent": all(not any(v for k, v in row.items() if k != "variable") for row in availability)}, ensure_ascii=False))
if __name__ == "__main__":
main()