fix(stage1): verify survey design fields

research/stage1/audit_input_sources.py

@@ -166,9 +166,10 @@ def build_audit() -> dict[str, Any]:
             ),
         },
         "limitations": [
-            "YRBS 2021 and 2023 are not included in the selected analytic Parquet and remain quarantined pending layout verification.",
-            "GSHS design fields are candidates and require survey-specific source verification.",
-            "NSDUH year-specific weight selection and variance usage require confirmation against the combined-file guide.",
+            "YRBS 2021 and 2023 have validated annual full Parquet files but are not appended to the legacy 1991-2019 selected table.",
+            "One YRBS 1997 record lacks weight, stratum, and PSU; 2017/2019 design fields were restored from official fixed-width locations.",
+            "GSHS design-field identities are verified for the seven primary-outcome components; cross-component identifiers require namespacing.",
+            "NSDUH uses ANALWT2_C1 for single-year estimates; C2/C3/C4 are exact 2/3/4-year pooled annual-average weights. Domain variance analysis must retain the full design.",
             "This audit does not approve item wording, response recodes, missingness rules, constructs, or cross-survey links."
         ]
     }

research/stage1/build_primary_outcome_package.py

@@ -197,10 +197,10 @@ def build() -> dict[str, int]:
             "survey_id": survey_id, "component_id": component["component_id"], "survey": "GSHS",
             "country": component.get("country", ""), "year": str(component.get("collection_start", ""))[:4],
             "population": component.get("population", ""), "weight_field": "sample_weight", "psu_field": "sampling_psu",
-            "stratum_field": "sampling_stratum", "variance_method": "pending official component design review",
+            "stratum_field": "sampling_stratum", "variance_method": "complex-survey design using weight, stratum, and PSU; Taylor linearization planned",
             "namespace_rule": "prefix PSU and stratum with component_id before pooling",
-            "design_source": source_file or source_url, "design_status": "provisional",
-            "notes": "Canonical fields are present; observed full GSHS table has 20 missing values in each design field overall. Component-specific design validity remains to be reviewed.",
+            "design_source": f"{component.get('catalog_url', '')}; WHO GSHS methodology and data catalog", "design_status": "source_verified_fields",
+            "notes": "For all 21,533 respondents in the seven primary-outcome components, canonical weight/PSU/stratum fields are complete, positive, and numerically identical to raw_weight/raw_psu/raw_stratum. Namespace by component before pooling.",
         })
 
     # YRBS: source-audited annual items.
@@ -238,20 +238,20 @@ def build() -> dict[str, int]:
             })
 
     for year in sorted({str(r["year"]) for r in yrbs_rows}):
-        if year in {"2017", "2019"}:
-            status = "provisional_design_incomplete"
+        if year == "1997":
+            status = "source_verified_one_missing_record"
         else:
-            status = "provisional"
+            status = "source_verified"
         designs.append({
             "survey_id": f"YRBS::{year}", "component_id": f"YRBS_NATIONAL::{year}", "survey": "YRBS",
             "country": "United States", "year": year,
             "population": "students in grades 9-12 attending public and private schools in the United States",
             "weight_field": "weight", "psu_field": "psu", "stratum_field": "stratum",
-            "variance_method": "Taylor linearization candidate; annual official design confirmation pending",
+            "variance_method": "complex-survey design-based variance accounting for multistage clustering; Taylor linearization supported",
             "namespace_rule": "prefix PSU and stratum with survey year before pooling",
             "design_source": next(r["data_guide_source_file"] for r in yrbs_rows if str(r["year"]) == year),
             "design_status": status,
-            "notes": "2017/2019 local PSU and stratum fields are entirely missing and weights are partial; 1997 has one row missing all three design fields; 2021/2023 official-layout imports have complete design fields.",
+            "notes": "Annual guide identifies WEIGHT, STRATUM, and PSU. Official-layout repairs restored complete 2017/2019 fields; 2021/2023 imports are complete. One 1997 record lacks all three design fields and requires explicit exclusion/handling.",
         })
 
     # NSDUH: four youth-module years. Attempt is documented but unavailable locally.
@@ -299,12 +299,12 @@ def build() -> dict[str, int]:
         designs.append({
             "survey_id": f"NSDUH::{year}", "component_id": f"NSDUH_YOUTH::{year}", "survey": "NSDUH",
             "country": "United States", "year": year, "population": "adolescents aged 12-17",
-            "weight_field": "ANALWT2_C (year-specific C1-C4 mapping unresolved)", "psu_field": "VEREP",
-            "stratum_field": "VESTR_C", "variance_method": "pending official combined-file guide review",
-            "namespace_rule": "prefix variance units with survey year before pooling",
-            "design_source": "Dataset/可直接分析数据包_NSDUH_YRBS/README.md; official annual/combined-file guide pending",
-            "design_status": "provisional_weight_mapping_unresolved",
-            "notes": "All four ANALWT2_C1-C4 fields are positive in every local year; do not select a weight by suffix without the official year-specific mapping.",
+            "weight_field": "ANALWT2_C1 (single-year); C2/C3/C4 are C1 divided by 2/3/4 for pooled annual averages", "psu_field": "VEREP",
+            "stratum_field": "VESTR_C", "variance_method": "Taylor series linearization; 50 df for 2014 onward national PUF estimates",
+            "namespace_rule": "retain full-sample design and specify CATAG2=1 as a survey domain; do not physically subset before variance estimation",
+            "design_source": "Dataset/NSDUH/2023-nsduh-puf-data-users-guide.pdf sections 6, 8, 9, 10 and 12; official annual API metadata",
+            "design_status": "source_verified",
+            "notes": "Local C2=C1/2, C3=C1/3, and C4=C1/4 exactly for every row. Use C1 for a single year. VESTR_C and VEREP are complete; domain analysis must retain all PSUs.",
         })
 
     links = []

research/stage1/input_structure_audit.json

@@ -27,7 +27,7 @@
     },
     "YRBS_selected_1991_2019": {
       "path": "Dataset/可直接分析数据包_NSDUH_YRBS/YRBS/yrbs_1991_2019_selected.parquet",
-      "bytes": 2576853,
+      "bytes": 2595834,
       "rows": 217341,
       "columns": 79,
       "row_groups": 1,
@@ -49,9 +49,9 @@
         "2019": 13677
       },
       "design_non_null": {
-        "weight": 210769,
-        "stratum": 188898,
-        "psu": 188898
+        "weight": 217340,
+        "stratum": 217340,
+        "psu": 217340
       },
       "design_non_null_by_year": {
         "1991": {
@@ -120,10 +120,14 @@
           "psu": 15624
         },
         "2017": {
-          "weight": 13026
+          "weight": 14765,
+          "stratum": 14765,
+          "psu": 14765
         },
         "2019": {
-          "weight": 8845
+          "weight": 13677,
+          "stratum": 13677,
+          "psu": 13677
         }
       },
       "design_positive_by_year": {
@@ -193,10 +197,14 @@
           "psu": 15624
         },
         "2017": {
-          "weight": 13026
+          "weight": 14765,
+          "stratum": 14765,
+          "psu": 14765
         },
         "2019": {
-          "weight": 8845
+          "weight": 13677,
+          "stratum": 13677,
+          "psu": 13677
         }
       },
       "required_columns_present": true,

research/stage1/repair_yrbs_2017_2019_design.py

@@ -0,0 +1,134 @@
+"""Restore YRBS 2017/2019 design fields from official fixed-width locations."""
+
+from __future__ import annotations
+
+import hashlib
+import json
+from pathlib import Path
+
+import pyarrow as pa
+import pyarrow.csv as pacsv
+import pyarrow.parquet as pq
+
+
+ROOT = Path(__file__).resolve().parents[2]
+RAW = ROOT / "Dataset" / "YRBS_National_1991_2023" / "raw"
+GUIDES = ROOT / "Dataset" / "YRBS_National_1991_2023" / "documentation" / "UserGuide"
+FULL = ROOT / "Dataset" / "可直接分析数据包_NSDUH_YRBS" / "YRBS" / "full_by_year"
+STAGE1 = ROOT / "research" / "stage1"
+SELECTED_PARQUET = FULL.parent / "yrbs_1991_2019_selected.parquet"
+SELECTED_CSV = FULL.parent / "yrbs_1991_2019_selected.csv"
+SPECS = {
+    2017: {
+        "raw": RAW / "XXH2017_YRBS_Data.dat",
+        "guide": GUIDES / "2017_YRBS_Data_Users_Guide.pdf",
+        "page": "PDF page 102 / printed page 96",
+        "positions": {"weight": (387, 396), "stratum": (397, 399), "psu": (400, 405)},
+    },
+    2019: {
+        "raw": RAW / "XXH2019_YRBS_Data.dat",
+        "guide": GUIDES / "2019_National_YRBS_Data_Users_Guide.pdf",
+        "page": "PDF page 89 / printed page 86",
+        "positions": {"weight": (386, 395), "stratum": (396, 398), "psu": (399, 404)},
+    },
+}
+
+
+def digest(path: Path) -> str:
+    h = hashlib.sha256()
+    with path.open("rb") as handle:
+        for block in iter(lambda: handle.read(1024 * 1024), b""):
+            h.update(block)
+    return h.hexdigest()
+
+
+def parse_numeric(line: str, start: int, end: int) -> float | None:
+    text = line[start - 1 : end].strip()
+    return None if not text or text == "." else float(text)
+
+
+def main() -> None:
+    results = []
+    repaired_values = {}
+    for year, spec in SPECS.items():
+        parquet = FULL / f"yrbs_{year}_full.parquet"
+        table = pq.read_table(parquet)
+        source_rows = table.column("source_row").to_pylist()
+        if source_rows != list(range(1, table.num_rows + 1)):
+            raise AssertionError(f"{year}: source_row is not sequential; cannot prove raw-row alignment")
+        values = {name: [] for name in spec["positions"]}
+        lengths = set()
+        with spec["raw"].open(encoding="ascii", newline="") as handle:
+            for line in handle:
+                line = line.rstrip("\r\n")
+                lengths.add(len(line))
+                for name, (start, end) in spec["positions"].items():
+                    values[name].append(parse_numeric(line, start, end))
+        if any(len(column) != table.num_rows for column in values.values()):
+            raise AssertionError(f"{year}: raw/Parquet row counts differ")
+        for name, column in values.items():
+            index = table.schema.get_field_index(name)
+            table = table.set_column(index, name, pa.array(column, type=pa.float64()))
+        temp = parquet.with_suffix(".repair.tmp.parquet")
+        pq.write_table(table, temp, compression="zstd")
+        check = pq.read_table(temp, columns=["source_row", "weight", "stratum", "psu"])
+        nonmissing = {name: check.column(name).null_count for name in ("weight", "stratum", "psu")}
+        if any(nonmissing.values()):
+            raise AssertionError(f"{year}: design fields still contain nulls: {nonmissing}")
+        if min(check.column("weight").to_pylist()) <= 0:
+            raise AssertionError(f"{year}: nonpositive weight")
+        temp.replace(parquet)
+        repaired_values[year] = values
+        weight_sum = sum(values["weight"])
+        results.append({
+            "year": year, "rows": table.num_rows, "raw_record_lengths": sorted(lengths),
+            "raw_file": str(spec["raw"].relative_to(ROOT)).replace("\\", "/"),
+            "raw_sha256": digest(spec["raw"]),
+            "guide_file": str(spec["guide"].relative_to(ROOT)).replace("\\", "/"),
+            "guide_sha256": digest(spec["guide"]), "page_or_section": spec["page"],
+            "positions": spec["positions"], "design_nulls_after_repair": nonmissing,
+            "weight_min": min(values["weight"]), "weight_max": max(values["weight"]),
+            "weight_sum": weight_sum,
+            "weight_sum_minus_sample_size": weight_sum - table.num_rows,
+            "output_file": str(parquet.relative_to(ROOT)).replace("\\", "/"),
+            "status": "passed",
+        })
+    selected = pq.read_table(SELECTED_PARQUET)
+    years = selected.column("survey_year").to_pylist()
+    source_rows = selected.column("source_row").to_pylist()
+    for name in ("weight", "stratum", "psu"):
+        existing = selected.column(name).to_pylist()
+        repaired = [
+            repaired_values[int(year)][name][int(source_row) - 1]
+            if int(year) in repaired_values else old
+            for old, year, source_row in zip(existing, years, source_rows)
+        ]
+        selected = selected.set_column(selected.schema.get_field_index(name), name, pa.array(repaired, type=pa.float64()))
+    selected_temp = SELECTED_PARQUET.with_suffix(".repair.tmp.parquet")
+    pq.write_table(selected, selected_temp, compression="zstd")
+    selected_temp.replace(SELECTED_PARQUET)
+    pacsv.write_csv(selected, SELECTED_CSV)
+    for result in results:
+        year = result["year"]
+        mask = [i for i, value in enumerate(years) if int(value) == year]
+        result["selected_table_design_nulls_after_repair"] = {
+            name: sum(selected.column(name)[i].as_py() is None for i in mask)
+            for name in ("weight", "stratum", "psu")
+        }
+    audit = {
+        "schema_version": "1.0", "generated_on": "2026-09-20", "status": "passed",
+        "purpose": "Restore YRBS 2017/2019 weight, stratum, and PSU from the official guide's fixed-width locations.",
+        "years": results,
+        "decision": "2017 and 2019 design fields are no longer missing in the annual full Parquet files; use WEIGHT with STRATUM and PSU in complex-survey procedures.",
+    }
+    (STAGE1 / "yrbs_2017_2019_design_repair_audit.json").write_text(json.dumps(audit, indent=2, ensure_ascii=False) + "\n", encoding="utf-8")
+    lines = ["# YRBS 2017/2019 design-field repair audit", "", audit["decision"], ""]
+    for row in results:
+        lines += [f"## {row['year']}", "", f"- rows: {row['rows']:,}", f"- official locations: `{row['positions']}` ({row['page_or_section']})", f"- design nulls after repair: `{row['design_nulls_after_repair']}`", f"- weight range/sum: {row['weight_min']:.4f}–{row['weight_max']:.4f}; sum {row['weight_sum']:.4f}", ""]
+    lines += ["The repair changes only derived Parquet files, which remain reproducible and excluded from Git. Official raw files are never modified.", ""]
+    (STAGE1 / "yrbs_2017_2019_design_repair_audit.md").write_text("\n".join(lines), encoding="utf-8")
+    print(json.dumps(audit, indent=2, ensure_ascii=False))
+
+
+if __name__ == "__main__":
+    main()

research/stage1/survey_design.csv

@@ -1,29 +1,29 @@
 survey_id,component_id,survey,country,year,population,weight_field,psu_field,stratum_field,variance_method,namespace_rule,design_source,design_status,notes
-GSHS::2023_MAR_GSHS_v01,2023_MAR_GSHS_v01::mar_fez2023,GSHS,Morocco,2023,school-attending students; exact sampled ages/grades remain pending component source review,sample_weight,sampling_psu,sampling_stratum,pending official component design review,prefix PSU and stratum with component_id before pooling,Dataset/GSHS-全球学生健康调查数据/GSHS/Questionnaire/GSHS Per Country/Morocco 2023/Fez_2022_GSHS_Questionnaire_Final_FRENCH.pdf,provisional,Canonical fields are present; observed full GSHS table has 20 missing values in each design field overall. Component-specific design validity remains to be reviewed.
-GSHS::GHA_2022_GSHS_v01,GHA_2022_GSHS_v01::gha_sek_tak2022,GSHS,Ghana,2022,school-attending students; exact sampled ages/grades remain pending component source review,sample_weight,sampling_psu,sampling_stratum,pending official component design review,prefix PSU and stratum with component_id before pooling,Dataset/GSHS-全球学生健康调查数据/GSHS/Questionnaire/GSHS Per Country/Ghana 2022/Sekondi-Takoradi-2022_GSHS_Questionnaire-Final.pdf,provisional,Canonical fields are present; observed full GSHS table has 20 missing values in each design field overall. Component-specific design validity remains to be reviewed.
-GSHS::IND_2022_GSHS_v01,IND_2022_GSHS_v01::ind_jaipur_2022,GSHS,India,2022,school-attending students; exact sampled ages/grades remain pending component source review,sample_weight,sampling_psu,sampling_stratum,pending official component design review,prefix PSU and stratum with component_id before pooling,Dataset/GSHS-全球学生健康调查数据/GSHS/Questionnaire/GSHS Per Country/India 2022/Jaipur-2022_GSHS_Questionnaire-Final.docx,provisional,Canonical fields are present; observed full GSHS table has 20 missing values in each design field overall. Component-specific design validity remains to be reviewed.
-GSHS::JAM_2023_GSHS_v01,JAM_2023_GSHS_v01::jam_st_cath_2023,GSHS,Jamaica,2023,school-attending students; exact sampled ages/grades remain pending component source review,sample_weight,sampling_psu,sampling_stratum,pending official component design review,prefix PSU and stratum with component_id before pooling,Dataset/GSHS-全球学生健康调查数据/GSHS/Questionnaire/GSHS Per Country/Jamaica 2023/FINAL_St. Catherine-2023-GSHS-Questionnaire.docx,provisional,Canonical fields are present; observed full GSHS table has 20 missing values in each design field overall. Component-specific design validity remains to be reviewed.
-GSHS::MNG_2023_GSHS_v01,MNG_2023_GSHS_v01::mng2023,GSHS,Mongolia,2023,school-attending students; exact sampled ages/grades remain pending component source review,sample_weight,sampling_psu,sampling_stratum,pending official component design review,prefix PSU and stratum with component_id before pooling,Dataset/GSHS-全球学生健康调查数据/GSHS/Questionnaire/GSHS Per Country/Mongolia 2023/Mongolia-GSHS-Questionnaire-ENGLISH.docx,provisional,Canonical fields are present; observed full GSHS table has 20 missing values in each design field overall. Component-specific design validity remains to be reviewed.
-GSHS::SLB_2023_GSHS_v01,SLB_2023_GSHS_v01::slb_2023,GSHS,Solomon Islands,2023,school-going adolescents aged 13-17 years; Form 1-Form 6; all students in selected classes eligible,sample_weight,sampling_psu,sampling_stratum,pending official component design review,prefix PSU and stratum with component_id before pooling,https://extranet.who.int/ncdsmicrodata/index.php/catalog/992,provisional,Canonical fields are present; observed full GSHS table has 20 missing values in each design field overall. Component-specific design validity remains to be reviewed.
-GSHS::WLF_2023_GSHS_v01,WLF_2023_GSHS_v01::wlf_2023,GSHS,Wallis and Futuna,2023,school-attending students; exact sampled ages/grades remain pending component source review,sample_weight,sampling_psu,sampling_stratum,pending official component design review,prefix PSU and stratum with component_id before pooling,Dataset/GSHS-全球学生健康调查数据/GSHS/Questionnaire/GSHS Per Country/Wallis and Futuna 2023/Wallis and Futuna - GSHS Questionnaire - Final - French.docx,provisional,Canonical fields are present; observed full GSHS table has 20 missing values in each design field overall. Component-specific design validity remains to be reviewed.
-YRBS::1991,YRBS_NATIONAL::1991,YRBS,United States,1991,students in grades 9-12 attending public and private schools in the United States,weight,psu,stratum,Taylor linearization candidate; annual official design confirmation pending,prefix PSU and stratum with survey year before pooling,Dataset/YRBS_National_1991_2023/documentation/UserGuide/YRBS_1991_National_User_Guide.pdf,provisional,2017/2019 local PSU and stratum fields are entirely missing and weights are partial; 1997 has one row missing all three design fields; 2021/2023 official-layout imports have complete design fields.
-YRBS::1993,YRBS_NATIONAL::1993,YRBS,United States,1993,students in grades 9-12 attending public and private schools in the United States,weight,psu,stratum,Taylor linearization candidate; annual official design confirmation pending,prefix PSU and stratum with survey year before pooling,Dataset/YRBS_National_1991_2023/documentation/UserGuide/YRBS_1993_National_User_Guide.pdf,provisional,2017/2019 local PSU and stratum fields are entirely missing and weights are partial; 1997 has one row missing all three design fields; 2021/2023 official-layout imports have complete design fields.
-YRBS::1995,YRBS_NATIONAL::1995,YRBS,United States,1995,students in grades 9-12 attending public and private schools in the United States,weight,psu,stratum,Taylor linearization candidate; annual official design confirmation pending,prefix PSU and stratum with survey year before pooling,Dataset/YRBS_National_1991_2023/documentation/UserGuide/YRBS_1995_National_User_Guide.pdf,provisional,2017/2019 local PSU and stratum fields are entirely missing and weights are partial; 1997 has one row missing all three design fields; 2021/2023 official-layout imports have complete design fields.
-YRBS::1997,YRBS_NATIONAL::1997,YRBS,United States,1997,students in grades 9-12 attending public and private schools in the United States,weight,psu,stratum,Taylor linearization candidate; annual official design confirmation pending,prefix PSU and stratum with survey year before pooling,Dataset/YRBS_National_1991_2023/documentation/UserGuide/YRBS_1997_National_User_Guide.pdf,provisional,2017/2019 local PSU and stratum fields are entirely missing and weights are partial; 1997 has one row missing all three design fields; 2021/2023 official-layout imports have complete design fields.
-YRBS::1999,YRBS_NATIONAL::1999,YRBS,United States,1999,students in grades 9-12 attending public and private schools in the United States,weight,psu,stratum,Taylor linearization candidate; annual official design confirmation pending,prefix PSU and stratum with survey year before pooling,Dataset/YRBS_National_1991_2023/documentation/UserGuide/YRBS_1999_National_User_Guide.pdf,provisional,2017/2019 local PSU and stratum fields are entirely missing and weights are partial; 1997 has one row missing all three design fields; 2021/2023 official-layout imports have complete design fields.
-YRBS::2001,YRBS_NATIONAL::2001,YRBS,United States,2001,students in grades 9-12 attending public and private schools in the United States,weight,psu,stratum,Taylor linearization candidate; annual official design confirmation pending,prefix PSU and stratum with survey year before pooling,Dataset/YRBS_National_1991_2023/documentation/UserGuide/YRBS_2001_National_User_Guide.pdf,provisional,2017/2019 local PSU and stratum fields are entirely missing and weights are partial; 1997 has one row missing all three design fields; 2021/2023 official-layout imports have complete design fields.
-YRBS::2003,YRBS_NATIONAL::2003,YRBS,United States,2003,students in grades 9-12 attending public and private schools in the United States,weight,psu,stratum,Taylor linearization candidate; annual official design confirmation pending,prefix PSU and stratum with survey year before pooling,Dataset/YRBS_National_1991_2023/documentation/UserGuide/YRBS_2003_National_User_Guide.pdf,provisional,2017/2019 local PSU and stratum fields are entirely missing and weights are partial; 1997 has one row missing all three design fields; 2021/2023 official-layout imports have complete design fields.
-YRBS::2005,YRBS_NATIONAL::2005,YRBS,United States,2005,students in grades 9-12 attending public and private schools in the United States,weight,psu,stratum,Taylor linearization candidate; annual official design confirmation pending,prefix PSU and stratum with survey year before pooling,Dataset/YRBS_National_1991_2023/documentation/UserGuide/YRBS_2005_National_User_Guide.pdf,provisional,2017/2019 local PSU and stratum fields are entirely missing and weights are partial; 1997 has one row missing all three design fields; 2021/2023 official-layout imports have complete design fields.
-YRBS::2007,YRBS_NATIONAL::2007,YRBS,United States,2007,students in grades 9-12 attending public and private schools in the United States,weight,psu,stratum,Taylor linearization candidate; annual official design confirmation pending,prefix PSU and stratum with survey year before pooling,Dataset/YRBS_National_1991_2023/documentation/UserGuide/YRBS_2007_National_User_Guide.pdf,provisional,2017/2019 local PSU and stratum fields are entirely missing and weights are partial; 1997 has one row missing all three design fields; 2021/2023 official-layout imports have complete design fields.
-YRBS::2009,YRBS_NATIONAL::2009,YRBS,United States,2009,students in grades 9-12 attending public and private schools in the United States,weight,psu,stratum,Taylor linearization candidate; annual official design confirmation pending,prefix PSU and stratum with survey year before pooling,Dataset/YRBS_National_1991_2023/documentation/UserGuide/YRBS_2009_National_User_Guide.pdf,provisional,2017/2019 local PSU and stratum fields are entirely missing and weights are partial; 1997 has one row missing all three design fields; 2021/2023 official-layout imports have complete design fields.
-YRBS::2011,YRBS_NATIONAL::2011,YRBS,United States,2011,students in grades 9-12 attending public and private schools in the United States,weight,psu,stratum,Taylor linearization candidate; annual official design confirmation pending,prefix PSU and stratum with survey year before pooling,Dataset/YRBS_National_1991_2023/documentation/UserGuide/YRBS_2011_National_User_Guide.pdf,provisional,2017/2019 local PSU and stratum fields are entirely missing and weights are partial; 1997 has one row missing all three design fields; 2021/2023 official-layout imports have complete design fields.
-YRBS::2013,YRBS_NATIONAL::2013,YRBS,United States,2013,students in grades 9-12 attending public and private schools in the United States,weight,psu,stratum,Taylor linearization candidate; annual official design confirmation pending,prefix PSU and stratum with survey year before pooling,Dataset/YRBS_National_1991_2023/documentation/UserGuide/YRBS_2013_National_User_Guide.pdf,provisional,2017/2019 local PSU and stratum fields are entirely missing and weights are partial; 1997 has one row missing all three design fields; 2021/2023 official-layout imports have complete design fields.
-YRBS::2015,YRBS_NATIONAL::2015,YRBS,United States,2015,students in grades 9-12 attending public and private schools in the United States,weight,psu,stratum,Taylor linearization candidate; annual official design confirmation pending,prefix PSU and stratum with survey year before pooling,Dataset/YRBS_National_1991_2023/documentation/UserGuide/2015_yrbs-data-users_guide_smy_combined.pdf,provisional,2017/2019 local PSU and stratum fields are entirely missing and weights are partial; 1997 has one row missing all three design fields; 2021/2023 official-layout imports have complete design fields.
-YRBS::2017,YRBS_NATIONAL::2017,YRBS,United States,2017,students in grades 9-12 attending public and private schools in the United States,weight,psu,stratum,Taylor linearization candidate; annual official design confirmation pending,prefix PSU and stratum with survey year before pooling,Dataset/YRBS_National_1991_2023/documentation/UserGuide/2017_YRBS_Data_Users_Guide.pdf,provisional_design_incomplete,2017/2019 local PSU and stratum fields are entirely missing and weights are partial; 1997 has one row missing all three design fields; 2021/2023 official-layout imports have complete design fields.
-YRBS::2019,YRBS_NATIONAL::2019,YRBS,United States,2019,students in grades 9-12 attending public and private schools in the United States,weight,psu,stratum,Taylor linearization candidate; annual official design confirmation pending,prefix PSU and stratum with survey year before pooling,Dataset/YRBS_National_1991_2023/documentation/UserGuide/2019_National_YRBS_Data_Users_Guide.pdf,provisional_design_incomplete,2017/2019 local PSU and stratum fields are entirely missing and weights are partial; 1997 has one row missing all three design fields; 2021/2023 official-layout imports have complete design fields.
-YRBS::2021,YRBS_NATIONAL::2021,YRBS,United States,2021,students in grades 9-12 attending public and private schools in the United States,weight,psu,stratum,Taylor linearization candidate; annual official design confirmation pending,prefix PSU and stratum with survey year before pooling,Dataset/YRBS_National_1991_2023/documentation/UserGuide/2021_YRBS_Data_Users_Guide_508.pdf,provisional,2017/2019 local PSU and stratum fields are entirely missing and weights are partial; 1997 has one row missing all three design fields; 2021/2023 official-layout imports have complete design fields.
-YRBS::2023,YRBS_NATIONAL::2023,YRBS,United States,2023,students in grades 9-12 attending public and private schools in the United States,weight,psu,stratum,Taylor linearization candidate; annual official design confirmation pending,prefix PSU and stratum with survey year before pooling,Dataset/YRBS_National_1991_2023/documentation/UserGuide/2023_National_YRBS_Data_Users_Guide508.pdf,provisional,2017/2019 local PSU and stratum fields are entirely missing and weights are partial; 1997 has one row missing all three design fields; 2021/2023 official-layout imports have complete design fields.
-NSDUH::2021,NSDUH_YOUTH::2021,NSDUH,United States,2021,adolescents aged 12-17,ANALWT2_C (year-specific C1-C4 mapping unresolved),VEREP,VESTR_C,pending official combined-file guide review,prefix variance units with survey year before pooling,Dataset/可直接分析数据包_NSDUH_YRBS/README.md; official annual/combined-file guide pending,provisional_weight_mapping_unresolved,All four ANALWT2_C1-C4 fields are positive in every local year; do not select a weight by suffix without the official year-specific mapping.
-NSDUH::2022,NSDUH_YOUTH::2022,NSDUH,United States,2022,adolescents aged 12-17,ANALWT2_C (year-specific C1-C4 mapping unresolved),VEREP,VESTR_C,pending official combined-file guide review,prefix variance units with survey year before pooling,Dataset/可直接分析数据包_NSDUH_YRBS/README.md; official annual/combined-file guide pending,provisional_weight_mapping_unresolved,All four ANALWT2_C1-C4 fields are positive in every local year; do not select a weight by suffix without the official year-specific mapping.
-NSDUH::2023,NSDUH_YOUTH::2023,NSDUH,United States,2023,adolescents aged 12-17,ANALWT2_C (year-specific C1-C4 mapping unresolved),VEREP,VESTR_C,pending official combined-file guide review,prefix variance units with survey year before pooling,Dataset/可直接分析数据包_NSDUH_YRBS/README.md; official annual/combined-file guide pending,provisional_weight_mapping_unresolved,All four ANALWT2_C1-C4 fields are positive in every local year; do not select a weight by suffix without the official year-specific mapping.
-NSDUH::2024,NSDUH_YOUTH::2024,NSDUH,United States,2024,adolescents aged 12-17,ANALWT2_C (year-specific C1-C4 mapping unresolved),VEREP,VESTR_C,pending official combined-file guide review,prefix variance units with survey year before pooling,Dataset/可直接分析数据包_NSDUH_YRBS/README.md; official annual/combined-file guide pending,provisional_weight_mapping_unresolved,All four ANALWT2_C1-C4 fields are positive in every local year; do not select a weight by suffix without the official year-specific mapping.
+GSHS::2023_MAR_GSHS_v01,2023_MAR_GSHS_v01::mar_fez2023,GSHS,Morocco,2023,school-attending students; exact sampled ages/grades remain pending component source review,sample_weight,sampling_psu,sampling_stratum,"complex-survey design using weight, stratum, and PSU; Taylor linearization planned",prefix PSU and stratum with component_id before pooling,https://extranet.who.int/ncdsmicrodata/index.php/catalog/982; WHO GSHS methodology and data catalog,source_verified_fields,"For all 21,533 respondents in the seven primary-outcome components, canonical weight/PSU/stratum fields are complete, positive, and numerically identical to raw_weight/raw_psu/raw_stratum. Namespace by component before pooling."
+GSHS::GHA_2022_GSHS_v01,GHA_2022_GSHS_v01::gha_sek_tak2022,GSHS,Ghana,2022,school-attending students; exact sampled ages/grades remain pending component source review,sample_weight,sampling_psu,sampling_stratum,"complex-survey design using weight, stratum, and PSU; Taylor linearization planned",prefix PSU and stratum with component_id before pooling,https://extranet.who.int/ncdsmicrodata/index.php/catalog/981; WHO GSHS methodology and data catalog,source_verified_fields,"For all 21,533 respondents in the seven primary-outcome components, canonical weight/PSU/stratum fields are complete, positive, and numerically identical to raw_weight/raw_psu/raw_stratum. Namespace by component before pooling."
+GSHS::IND_2022_GSHS_v01,IND_2022_GSHS_v01::ind_jaipur_2022,GSHS,India,2022,school-attending students; exact sampled ages/grades remain pending component source review,sample_weight,sampling_psu,sampling_stratum,"complex-survey design using weight, stratum, and PSU; Taylor linearization planned",prefix PSU and stratum with component_id before pooling,https://extranet.who.int/ncdsmicrodata/index.php/catalog/984; WHO GSHS methodology and data catalog,source_verified_fields,"For all 21,533 respondents in the seven primary-outcome components, canonical weight/PSU/stratum fields are complete, positive, and numerically identical to raw_weight/raw_psu/raw_stratum. Namespace by component before pooling."
+GSHS::JAM_2023_GSHS_v01,JAM_2023_GSHS_v01::jam_st_cath_2023,GSHS,Jamaica,2023,school-attending students; exact sampled ages/grades remain pending component source review,sample_weight,sampling_psu,sampling_stratum,"complex-survey design using weight, stratum, and PSU; Taylor linearization planned",prefix PSU and stratum with component_id before pooling,https://extranet.who.int/ncdsmicrodata/index.php/catalog/983; WHO GSHS methodology and data catalog,source_verified_fields,"For all 21,533 respondents in the seven primary-outcome components, canonical weight/PSU/stratum fields are complete, positive, and numerically identical to raw_weight/raw_psu/raw_stratum. Namespace by component before pooling."
+GSHS::MNG_2023_GSHS_v01,MNG_2023_GSHS_v01::mng2023,GSHS,Mongolia,2023,school-attending students; exact sampled ages/grades remain pending component source review,sample_weight,sampling_psu,sampling_stratum,"complex-survey design using weight, stratum, and PSU; Taylor linearization planned",prefix PSU and stratum with component_id before pooling,https://extranet.who.int/ncdsmicrodata/index.php/catalog/990; WHO GSHS methodology and data catalog,source_verified_fields,"For all 21,533 respondents in the seven primary-outcome components, canonical weight/PSU/stratum fields are complete, positive, and numerically identical to raw_weight/raw_psu/raw_stratum. Namespace by component before pooling."
+GSHS::SLB_2023_GSHS_v01,SLB_2023_GSHS_v01::slb_2023,GSHS,Solomon Islands,2023,school-going adolescents aged 13-17 years; Form 1-Form 6; all students in selected classes eligible,sample_weight,sampling_psu,sampling_stratum,"complex-survey design using weight, stratum, and PSU; Taylor linearization planned",prefix PSU and stratum with component_id before pooling,https://extranet.who.int/ncdsmicrodata/index.php/catalog/992; WHO GSHS methodology and data catalog,source_verified_fields,"For all 21,533 respondents in the seven primary-outcome components, canonical weight/PSU/stratum fields are complete, positive, and numerically identical to raw_weight/raw_psu/raw_stratum. Namespace by component before pooling."
+GSHS::WLF_2023_GSHS_v01,WLF_2023_GSHS_v01::wlf_2023,GSHS,Wallis and Futuna,2023,school-attending students; exact sampled ages/grades remain pending component source review,sample_weight,sampling_psu,sampling_stratum,"complex-survey design using weight, stratum, and PSU; Taylor linearization planned",prefix PSU and stratum with component_id before pooling,https://extranet.who.int/ncdsmicrodata/index.php/catalog/994; WHO GSHS methodology and data catalog,source_verified_fields,"For all 21,533 respondents in the seven primary-outcome components, canonical weight/PSU/stratum fields are complete, positive, and numerically identical to raw_weight/raw_psu/raw_stratum. Namespace by component before pooling."
+YRBS::1991,YRBS_NATIONAL::1991,YRBS,United States,1991,students in grades 9-12 attending public and private schools in the United States,weight,psu,stratum,complex-survey design-based variance accounting for multistage clustering; Taylor linearization supported,prefix PSU and stratum with survey year before pooling,Dataset/YRBS_National_1991_2023/documentation/UserGuide/YRBS_1991_National_User_Guide.pdf,source_verified,"Annual guide identifies WEIGHT, STRATUM, and PSU. Official-layout repairs restored complete 2017/2019 fields; 2021/2023 imports are complete. One 1997 record lacks all three design fields and requires explicit exclusion/handling."
+YRBS::1993,YRBS_NATIONAL::1993,YRBS,United States,1993,students in grades 9-12 attending public and private schools in the United States,weight,psu,stratum,complex-survey design-based variance accounting for multistage clustering; Taylor linearization supported,prefix PSU and stratum with survey year before pooling,Dataset/YRBS_National_1991_2023/documentation/UserGuide/YRBS_1993_National_User_Guide.pdf,source_verified,"Annual guide identifies WEIGHT, STRATUM, and PSU. Official-layout repairs restored complete 2017/2019 fields; 2021/2023 imports are complete. One 1997 record lacks all three design fields and requires explicit exclusion/handling."
+YRBS::1995,YRBS_NATIONAL::1995,YRBS,United States,1995,students in grades 9-12 attending public and private schools in the United States,weight,psu,stratum,complex-survey design-based variance accounting for multistage clustering; Taylor linearization supported,prefix PSU and stratum with survey year before pooling,Dataset/YRBS_National_1991_2023/documentation/UserGuide/YRBS_1995_National_User_Guide.pdf,source_verified,"Annual guide identifies WEIGHT, STRATUM, and PSU. Official-layout repairs restored complete 2017/2019 fields; 2021/2023 imports are complete. One 1997 record lacks all three design fields and requires explicit exclusion/handling."
+YRBS::1997,YRBS_NATIONAL::1997,YRBS,United States,1997,students in grades 9-12 attending public and private schools in the United States,weight,psu,stratum,complex-survey design-based variance accounting for multistage clustering; Taylor linearization supported,prefix PSU and stratum with survey year before pooling,Dataset/YRBS_National_1991_2023/documentation/UserGuide/YRBS_1997_National_User_Guide.pdf,source_verified_one_missing_record,"Annual guide identifies WEIGHT, STRATUM, and PSU. Official-layout repairs restored complete 2017/2019 fields; 2021/2023 imports are complete. One 1997 record lacks all three design fields and requires explicit exclusion/handling."
+YRBS::1999,YRBS_NATIONAL::1999,YRBS,United States,1999,students in grades 9-12 attending public and private schools in the United States,weight,psu,stratum,complex-survey design-based variance accounting for multistage clustering; Taylor linearization supported,prefix PSU and stratum with survey year before pooling,Dataset/YRBS_National_1991_2023/documentation/UserGuide/YRBS_1999_National_User_Guide.pdf,source_verified,"Annual guide identifies WEIGHT, STRATUM, and PSU. Official-layout repairs restored complete 2017/2019 fields; 2021/2023 imports are complete. One 1997 record lacks all three design fields and requires explicit exclusion/handling."
+YRBS::2001,YRBS_NATIONAL::2001,YRBS,United States,2001,students in grades 9-12 attending public and private schools in the United States,weight,psu,stratum,complex-survey design-based variance accounting for multistage clustering; Taylor linearization supported,prefix PSU and stratum with survey year before pooling,Dataset/YRBS_National_1991_2023/documentation/UserGuide/YRBS_2001_National_User_Guide.pdf,source_verified,"Annual guide identifies WEIGHT, STRATUM, and PSU. Official-layout repairs restored complete 2017/2019 fields; 2021/2023 imports are complete. One 1997 record lacks all three design fields and requires explicit exclusion/handling."
+YRBS::2003,YRBS_NATIONAL::2003,YRBS,United States,2003,students in grades 9-12 attending public and private schools in the United States,weight,psu,stratum,complex-survey design-based variance accounting for multistage clustering; Taylor linearization supported,prefix PSU and stratum with survey year before pooling,Dataset/YRBS_National_1991_2023/documentation/UserGuide/YRBS_2003_National_User_Guide.pdf,source_verified,"Annual guide identifies WEIGHT, STRATUM, and PSU. Official-layout repairs restored complete 2017/2019 fields; 2021/2023 imports are complete. One 1997 record lacks all three design fields and requires explicit exclusion/handling."
+YRBS::2005,YRBS_NATIONAL::2005,YRBS,United States,2005,students in grades 9-12 attending public and private schools in the United States,weight,psu,stratum,complex-survey design-based variance accounting for multistage clustering; Taylor linearization supported,prefix PSU and stratum with survey year before pooling,Dataset/YRBS_National_1991_2023/documentation/UserGuide/YRBS_2005_National_User_Guide.pdf,source_verified,"Annual guide identifies WEIGHT, STRATUM, and PSU. Official-layout repairs restored complete 2017/2019 fields; 2021/2023 imports are complete. One 1997 record lacks all three design fields and requires explicit exclusion/handling."
+YRBS::2007,YRBS_NATIONAL::2007,YRBS,United States,2007,students in grades 9-12 attending public and private schools in the United States,weight,psu,stratum,complex-survey design-based variance accounting for multistage clustering; Taylor linearization supported,prefix PSU and stratum with survey year before pooling,Dataset/YRBS_National_1991_2023/documentation/UserGuide/YRBS_2007_National_User_Guide.pdf,source_verified,"Annual guide identifies WEIGHT, STRATUM, and PSU. Official-layout repairs restored complete 2017/2019 fields; 2021/2023 imports are complete. One 1997 record lacks all three design fields and requires explicit exclusion/handling."
+YRBS::2009,YRBS_NATIONAL::2009,YRBS,United States,2009,students in grades 9-12 attending public and private schools in the United States,weight,psu,stratum,complex-survey design-based variance accounting for multistage clustering; Taylor linearization supported,prefix PSU and stratum with survey year before pooling,Dataset/YRBS_National_1991_2023/documentation/UserGuide/YRBS_2009_National_User_Guide.pdf,source_verified,"Annual guide identifies WEIGHT, STRATUM, and PSU. Official-layout repairs restored complete 2017/2019 fields; 2021/2023 imports are complete. One 1997 record lacks all three design fields and requires explicit exclusion/handling."
+YRBS::2011,YRBS_NATIONAL::2011,YRBS,United States,2011,students in grades 9-12 attending public and private schools in the United States,weight,psu,stratum,complex-survey design-based variance accounting for multistage clustering; Taylor linearization supported,prefix PSU and stratum with survey year before pooling,Dataset/YRBS_National_1991_2023/documentation/UserGuide/YRBS_2011_National_User_Guide.pdf,source_verified,"Annual guide identifies WEIGHT, STRATUM, and PSU. Official-layout repairs restored complete 2017/2019 fields; 2021/2023 imports are complete. One 1997 record lacks all three design fields and requires explicit exclusion/handling."
+YRBS::2013,YRBS_NATIONAL::2013,YRBS,United States,2013,students in grades 9-12 attending public and private schools in the United States,weight,psu,stratum,complex-survey design-based variance accounting for multistage clustering; Taylor linearization supported,prefix PSU and stratum with survey year before pooling,Dataset/YRBS_National_1991_2023/documentation/UserGuide/YRBS_2013_National_User_Guide.pdf,source_verified,"Annual guide identifies WEIGHT, STRATUM, and PSU. Official-layout repairs restored complete 2017/2019 fields; 2021/2023 imports are complete. One 1997 record lacks all three design fields and requires explicit exclusion/handling."
+YRBS::2015,YRBS_NATIONAL::2015,YRBS,United States,2015,students in grades 9-12 attending public and private schools in the United States,weight,psu,stratum,complex-survey design-based variance accounting for multistage clustering; Taylor linearization supported,prefix PSU and stratum with survey year before pooling,Dataset/YRBS_National_1991_2023/documentation/UserGuide/2015_yrbs-data-users_guide_smy_combined.pdf,source_verified,"Annual guide identifies WEIGHT, STRATUM, and PSU. Official-layout repairs restored complete 2017/2019 fields; 2021/2023 imports are complete. One 1997 record lacks all three design fields and requires explicit exclusion/handling."
+YRBS::2017,YRBS_NATIONAL::2017,YRBS,United States,2017,students in grades 9-12 attending public and private schools in the United States,weight,psu,stratum,complex-survey design-based variance accounting for multistage clustering; Taylor linearization supported,prefix PSU and stratum with survey year before pooling,Dataset/YRBS_National_1991_2023/documentation/UserGuide/2017_YRBS_Data_Users_Guide.pdf,source_verified,"Annual guide identifies WEIGHT, STRATUM, and PSU. Official-layout repairs restored complete 2017/2019 fields; 2021/2023 imports are complete. One 1997 record lacks all three design fields and requires explicit exclusion/handling."
+YRBS::2019,YRBS_NATIONAL::2019,YRBS,United States,2019,students in grades 9-12 attending public and private schools in the United States,weight,psu,stratum,complex-survey design-based variance accounting for multistage clustering; Taylor linearization supported,prefix PSU and stratum with survey year before pooling,Dataset/YRBS_National_1991_2023/documentation/UserGuide/2019_National_YRBS_Data_Users_Guide.pdf,source_verified,"Annual guide identifies WEIGHT, STRATUM, and PSU. Official-layout repairs restored complete 2017/2019 fields; 2021/2023 imports are complete. One 1997 record lacks all three design fields and requires explicit exclusion/handling."
+YRBS::2021,YRBS_NATIONAL::2021,YRBS,United States,2021,students in grades 9-12 attending public and private schools in the United States,weight,psu,stratum,complex-survey design-based variance accounting for multistage clustering; Taylor linearization supported,prefix PSU and stratum with survey year before pooling,Dataset/YRBS_National_1991_2023/documentation/UserGuide/2021_YRBS_Data_Users_Guide_508.pdf,source_verified,"Annual guide identifies WEIGHT, STRATUM, and PSU. Official-layout repairs restored complete 2017/2019 fields; 2021/2023 imports are complete. One 1997 record lacks all three design fields and requires explicit exclusion/handling."
+YRBS::2023,YRBS_NATIONAL::2023,YRBS,United States,2023,students in grades 9-12 attending public and private schools in the United States,weight,psu,stratum,complex-survey design-based variance accounting for multistage clustering; Taylor linearization supported,prefix PSU and stratum with survey year before pooling,Dataset/YRBS_National_1991_2023/documentation/UserGuide/2023_National_YRBS_Data_Users_Guide508.pdf,source_verified,"Annual guide identifies WEIGHT, STRATUM, and PSU. Official-layout repairs restored complete 2017/2019 fields; 2021/2023 imports are complete. One 1997 record lacks all three design fields and requires explicit exclusion/handling."
+NSDUH::2021,NSDUH_YOUTH::2021,NSDUH,United States,2021,adolescents aged 12-17,ANALWT2_C1 (single-year); C2/C3/C4 are C1 divided by 2/3/4 for pooled annual averages,VEREP,VESTR_C,Taylor series linearization; 50 df for 2014 onward national PUF estimates,retain full-sample design and specify CATAG2=1 as a survey domain; do not physically subset before variance estimation,"Dataset/NSDUH/2023-nsduh-puf-data-users-guide.pdf sections 6, 8, 9, 10 and 12; official annual API metadata",source_verified,"Local C2=C1/2, C3=C1/3, and C4=C1/4 exactly for every row. Use C1 for a single year. VESTR_C and VEREP are complete; domain analysis must retain all PSUs."
+NSDUH::2022,NSDUH_YOUTH::2022,NSDUH,United States,2022,adolescents aged 12-17,ANALWT2_C1 (single-year); C2/C3/C4 are C1 divided by 2/3/4 for pooled annual averages,VEREP,VESTR_C,Taylor series linearization; 50 df for 2014 onward national PUF estimates,retain full-sample design and specify CATAG2=1 as a survey domain; do not physically subset before variance estimation,"Dataset/NSDUH/2023-nsduh-puf-data-users-guide.pdf sections 6, 8, 9, 10 and 12; official annual API metadata",source_verified,"Local C2=C1/2, C3=C1/3, and C4=C1/4 exactly for every row. Use C1 for a single year. VESTR_C and VEREP are complete; domain analysis must retain all PSUs."
+NSDUH::2023,NSDUH_YOUTH::2023,NSDUH,United States,2023,adolescents aged 12-17,ANALWT2_C1 (single-year); C2/C3/C4 are C1 divided by 2/3/4 for pooled annual averages,VEREP,VESTR_C,Taylor series linearization; 50 df for 2014 onward national PUF estimates,retain full-sample design and specify CATAG2=1 as a survey domain; do not physically subset before variance estimation,"Dataset/NSDUH/2023-nsduh-puf-data-users-guide.pdf sections 6, 8, 9, 10 and 12; official annual API metadata",source_verified,"Local C2=C1/2, C3=C1/3, and C4=C1/4 exactly for every row. Use C1 for a single year. VESTR_C and VEREP are complete; domain analysis must retain all PSUs."
+NSDUH::2024,NSDUH_YOUTH::2024,NSDUH,United States,2024,adolescents aged 12-17,ANALWT2_C1 (single-year); C2/C3/C4 are C1 divided by 2/3/4 for pooled annual averages,VEREP,VESTR_C,Taylor series linearization; 50 df for 2014 onward national PUF estimates,retain full-sample design and specify CATAG2=1 as a survey domain; do not physically subset before variance estimation,"Dataset/NSDUH/2023-nsduh-puf-data-users-guide.pdf sections 6, 8, 9, 10 and 12; official annual API metadata",source_verified,"Local C2=C1/2, C3=C1/3, and C4=C1/4 exactly for every row. Use C1 for a single year. VESTR_C and VEREP are complete; domain analysis must retain all PSUs."

research/stage1/survey_design_source_audit.md

@@ -0,0 +1,25 @@
+# Survey-design source audit
+
+Status: **source fields verified; Gate 1 human item review still pending**.
+
+## GSHS
+
+WHO catalog metadata identifies `weight`, `stratum`, and `psu` as the overall analysis weight, sampling stratum, and primary sampling unit. In the seven components containing the primary outcomes, all 21,533 respondents have positive canonical design fields. `sample_weight`, `sampling_psu`, and `sampling_stratum` are numerically identical to `raw_weight`, `raw_psu`, and `raw_stratum`. PSU and stratum must be namespaced by component before pooling.
+
+Sources: WHO GSHS methodology and NCD Microdata catalog; component catalog URLs are retained in `survey_design.csv`.
+
+## YRBS
+
+Annual CDC guides define `WEIGHT`, `STRATUM`, and `PSU` and require software that accounts for the multistage cluster sample. Official fixed-width locations restored the incomplete 2017 and 2019 derived files. The 2017 locations are PDF page 102 (printed page 96); the 2019 locations are PDF page 89 (printed page 86). Both annual full files and the combined selected files now have complete positive design fields for those years. The only remaining local design missingness is one 1997 record with all three fields absent. The 2021 and 2023 official-layout imports are complete.
+
+## NSDUH
+
+The official public-use guide specifies `ANALWT2_C` as the final person-level weight, `VESTR_C` as the pseudo-stratum, `VEREP` as the pseudo-PSU, Taylor series linearization, and 50 degrees of freedom for 2014 onward national PUF estimates. It requires domain analysis without physically dropping non-domain respondents.
+
+The local combined package stores the single-year weight as `ANALWT2_C1`; across all 232,441 rows, `C2=C1/2`, `C3=C1/3`, and `C4=C1/4` exactly, matching pooled annual-average rules. All weight and variance fields are complete and positive.
+
+Source: `Dataset/NSDUH/2023-nsduh-puf-data-users-guide.pdf`, sections 6, 8, 9, 10 and 12.
+
+## Boundary
+
+This audit approves the design-field identities and roles in `survey_design.csv`. It does not approve cross-survey pooling, item equivalence, or human item-review decisions. The single YRBS 1997 design-incomplete record must not silently enter weighted variance estimation.

research/stage1/yrbs_2017_2019_design_repair_audit.json

@@ -0,0 +1,93 @@
+{
+  "schema_version": "1.0",
+  "generated_on": "2026-09-20",
+  "status": "passed",
+  "purpose": "Restore YRBS 2017/2019 weight, stratum, and PSU from the official guide's fixed-width locations.",
+  "years": [
+    {
+      "year": 2017,
+      "rows": 14765,
+      "raw_record_lengths": [
+        427
+      ],
+      "raw_file": "Dataset/YRBS_National_1991_2023/raw/XXH2017_YRBS_Data.dat",
+      "raw_sha256": "99f7a87f1f936b21674603064118222247e797323377de7c7365e75a26c6f4b9",
+      "guide_file": "Dataset/YRBS_National_1991_2023/documentation/UserGuide/2017_YRBS_Data_Users_Guide.pdf",
+      "guide_sha256": "3ac5a8b3e62cef9007c5c86f4b8a6eaba15ebd82ef77332335e6670e5387370a",
+      "page_or_section": "PDF page 102 / printed page 96",
+      "positions": {
+        "weight": [
+          387,
+          396
+        ],
+        "stratum": [
+          397,
+          399
+        ],
+        "psu": [
+          400,
+          405
+        ]
+      },
+      "design_nulls_after_repair": {
+        "weight": 0,
+        "stratum": 0,
+        "psu": 0
+      },
+      "weight_min": 0.0442,
+      "weight_max": 4.6923,
+      "weight_sum": 14765.0049,
+      "weight_sum_minus_sample_size": 0.004899999999906868,
+      "output_file": "Dataset/可直接分析数据包_NSDUH_YRBS/YRBS/full_by_year/yrbs_2017_full.parquet",
+      "status": "passed",
+      "selected_table_design_nulls_after_repair": {
+        "weight": 0,
+        "stratum": 0,
+        "psu": 0
+      }
+    },
+    {
+      "year": 2019,
+      "rows": 13677,
+      "raw_record_lengths": [
+        419
+      ],
+      "raw_file": "Dataset/YRBS_National_1991_2023/raw/XXH2019_YRBS_Data.dat",
+      "raw_sha256": "9bfb3e87c519418eb6bd5b491ecfc723c1b21b85dbe796816fcce6b3cea8ca26",
+      "guide_file": "Dataset/YRBS_National_1991_2023/documentation/UserGuide/2019_National_YRBS_Data_Users_Guide.pdf",
+      "guide_sha256": "b60d9e9571939a47fab597448cda537e12e8c712cd20cd332d2ebe6c0fcd25b1",
+      "page_or_section": "PDF page 89 / printed page 86",
+      "positions": {
+        "weight": [
+          386,
+          395
+        ],
+        "stratum": [
+          396,
+          398
+        ],
+        "psu": [
+          399,
+          404
+        ]
+      },
+      "design_nulls_after_repair": {
+        "weight": 0,
+        "stratum": 0,
+        "psu": 0
+      },
+      "weight_min": 0.065,
+      "weight_max": 4.8641,
+      "weight_sum": 13677.0071,
+      "weight_sum_minus_sample_size": 0.007100000000718865,
+      "output_file": "Dataset/可直接分析数据包_NSDUH_YRBS/YRBS/full_by_year/yrbs_2019_full.parquet",
+      "status": "passed",
+      "selected_table_design_nulls_after_repair": {
+        "weight": 0,
+        "stratum": 0,
+        "psu": 0
+      }
+    }
+  ],
+  "decision": "2017 and 2019 design fields are no longer missing in the annual full Parquet files; use WEIGHT with STRATUM and PSU in complex-survey procedures."
+}

research/stage1/yrbs_2017_2019_design_repair_audit.md

@@ -0,0 +1,19 @@
+# YRBS 2017/2019 design-field repair audit
+
+2017 and 2019 design fields are no longer missing in the annual full Parquet files; use WEIGHT with STRATUM and PSU in complex-survey procedures.
+
+## 2017
+
+- rows: 14,765
+- official locations: `{'weight': (387, 396), 'stratum': (397, 399), 'psu': (400, 405)}` (PDF page 102 / printed page 96)
+- design nulls after repair: `{'weight': 0, 'stratum': 0, 'psu': 0}`
+- weight range/sum: 0.0442–4.6923; sum 14765.0049
+
+## 2019
+
+- rows: 13,677
+- official locations: `{'weight': (386, 395), 'stratum': (396, 398), 'psu': (399, 404)}` (PDF page 89 / printed page 86)
+- design nulls after repair: `{'weight': 0, 'stratum': 0, 'psu': 0}`
+- weight range/sum: 0.0650–4.8641; sum 13677.0071
+
+The repair changes only derived Parquet files, which remain reproducible and excluded from Git. Official raw files are never modified.