diff --git a/.learnings/ERRORS.md b/.learnings/ERRORS.md index dab5b2d..7f855dd 100644 --- a/.learnings/ERRORS.md +++ b/.learnings/ERRORS.md @@ -2,6 +2,35 @@ Command failures and integration errors. +--- +## [ERR-20260920-042] pdf-runtime-path-mismatch + +**Logged**: 2026-09-20 +**Priority**: low +**Status**: resolved +**Area**: environment + +### Summary +Neither `pdftotext` nor `pypdf` was available in the project/system Python path used for the first survey-design source extraction attempt. + +### Error +PowerShell could not find `pdftotext`; project Python raised `ModuleNotFoundError: pypdf`. + +### Context +The failures were read-only and occurred before any design decision was recorded. + +### Suggested Fix +Use the Codex workspace PDF runtime returned by the workspace dependency loader for document extraction; keep statistical builders in `.venv-research`. + +### Metadata +- Reproducible: yes +- Related Files: `research/stage1/survey_design_source_audit.md` +- Pattern-Key: environment.pdf-runtime-path + +### Resolution +- **Resolved**: 2026-09-20 +- **Notes**: Official guide pages were extracted with the bundled PDF runtime and cross-checked against local fields. + --- ## [ERR-20260920-041] gitea-cached-credential-rejected diff --git a/Progression.md b/Progression.md index 7342219..c84c3d3 100644 --- a/Progression.md +++ b/Progression.md @@ -1,9 +1,9 @@ # 项目进度档案 -版本：1.15 +版本：1.16 建立日期：2026-09-20 当前总状态：`in_progress` -当前阶段：阶段 1 `in_progress`；84 个主要结局题目实例及 919,863 行回答长表已形成并通过结构校验，YRBS 17 年完整表可用；设计复核和实际双人审核尚未完成 +当前阶段：阶段 1 `in_progress`；题库、回答字典、919,863 行回答长表及三调查设计字段/方法均已核验；Gate 1 仅剩主要结局的两名实际人工审核者签署 ## 1. 管理规则 @@ -28,7 +28,7 @@ | 阶段 | 名称 | 状态 | 当前判定 | |---|---|---|---| | 0 | 冻结研究问题、范围与版本 | `passed` | 四项交付物齐备，Gate 0 已通过；研究工具链补强与锁定完成 | -| 1 | 有来源证据的题库和回答层 | `in_progress` | 主要结局题库、回答字典和回答长表已生成并通过结构校验；设计复核和实际双重审核尚未完成 | +| 1 | 有来源证据的题库和回答层 | `in_progress` | 七项机器交付物齐备，来源/编码/总体/缺失与设计层已形成；尚缺工作流强制要求的两名实际审核者签署 | | 2 | 可行性、可识别性和连接审计 | `not_started` | 等待 Gate 1 | | 3 | 冻结验证协议与已知真值模拟 | `not_started` | 等待 Gate 2 | | 4 | 直接协调和传统基线 | `not_started` | 等待 Gate 3 | @@ -789,6 +789,19 @@ Gate 0 当前判定：`passed`。四项交付物齐备；目标总体政策、 - 推送状态：`pushed`；研究提交 `89c20f3` 与认证失败状态提交 `9497d1c` 已在仅清除失效站点凭据并由 Git Credential Manager 重新授权后推送至 `origin/main`。 - 下一步：逐调查核验权重、PSU、stratum 和方差方法的官方来源；随后落实两名实际审核者签署。 +### A-20260920-062：核验设计来源并修复 YRBS 2017/2019 设计字段 + +- 时间：2026-09-20 +- 阶段：1（有来源证据的题库和回答层） +- 动作：行动前完整读取工作流、进度档案并检查 Git；复核 WHO GSHS 目录/方法页、17 份 CDC YRBS 年度指南及 SAMHSA NSDUH 公用数据指南；按官方定长位置恢复 YRBS 2017/2019 的 weight、stratum、PSU，并同步年度完整表及 1991–2019 selected 表。 +- 输出：新增 `repair_yrbs_2017_2019_design.py`、`yrbs_2017_2019_design_repair_audit.{json,md}`、`survey_design_source_audit.md`；更新 `survey_design.csv` 构建规则及入口结构审计。 +- 检查：2017 的 14,765 行和 2019 的 13,677 行三项设计字段均恢复为全量非空且权重为正；权重和分别为 14,765.0049 与 13,677.0071，与指南所述缩放到样本量一致。GSHS 七个主要结局组件共 21,533 人的 canonical/raw 权重、PSU、stratum 数值完全一致、全量非空且为正。NSDUH 232,441 行满足 `C2=C1/2`、`C3=C1/3`、`C4=C1/4` 精确等式，确认 C1 为单年权重、C2–C4 为合并 2–4 年的年均权重。 +- 方法决策：YRBS 使用 WEIGHT/STRATUM/PSU 的复杂抽样程序；NSDUH 使用 VESTR_C/VEREP、Taylor 线性化和单年 ANALWT2_C1，青少年分析必须作为全样本设计内的 domain 而非先物理删成人；GSHS 跨组件前为 PSU/stratum 增加组件命名空间。 +- 剩余例外：YRBS 1997 仍有 1 行同时缺三项设计字段，已显式登记，不能静默进入设计型方差估计。 +- 工程记录：PDF 工具路径不匹配在任何结论写入前失败，改用工作区 PDF runtime 后完成并记录为 `ERR-20260920-042`。 +- 推送状态：`push_pending`；验证、提交并推送后更新。 +- 下一步：由两名实际审核者独立处理 84 条主结局审核队列；完成签署与分歧裁决后执行 Gate 1 最终验收。 + ## 9. 剩余工作量估算（2026-09-20 基线） | 阶段 | 主要工作 | 预计人周 | 主要波动来源 | @@ -811,6 +824,6 @@ Gate 0 当前判定：`passed`。四项交付物齐备；目标总体政策、 阶段 1 下一行动： -1. 复核 GSHS、YRBS、NSDUH 的年度/组件设计字段与官方方差说明，禁止凭字段名推断方差方法； -2. 核对长表各题的阳性/缺失计数与原始审计，形成回答层验收摘要； -3. 安排两名实际审核者逐项处理 `primary_outcome_human_review_queue.csv`；审核完成前保持全部主要结局为 `provisional`。 +1. 由两名实际审核者独立处理 `primary_outcome_human_review_queue.csv`，核对题文、构念、总体、语言、编码、跳题和缺失规则； +2. 对两位审核者的分歧执行显式裁决，不得由 AI 冒充审核者或自动填写签名； +3. 审核通过后重建题库/回答层并执行 Gate 1 逐项验收；未签署前保持全部主要结局为 `provisional`。 diff --git a/research/stage1/audit_input_sources.py b/research/stage1/audit_input_sources.py index 60d868c..8785d5d 100644 --- a/research/stage1/audit_input_sources.py +++ b/research/stage1/audit_input_sources.py @@ -166,9 +166,10 @@ def build_audit() -> dict[str, Any]: ), }, "limitations": [ - "YRBS 2021 and 2023 are not included in the selected analytic Parquet and remain quarantined pending layout verification.", - "GSHS design fields are candidates and require survey-specific source verification.", - "NSDUH year-specific weight selection and variance usage require confirmation against the combined-file guide.", + "YRBS 2021 and 2023 have validated annual full Parquet files but are not appended to the legacy 1991-2019 selected table.", + "One YRBS 1997 record lacks weight, stratum, and PSU; 2017/2019 design fields were restored from official fixed-width locations.", + "GSHS design-field identities are verified for the seven primary-outcome components; cross-component identifiers require namespacing.", + "NSDUH uses ANALWT2_C1 for single-year estimates; C2/C3/C4 are exact 2/3/4-year pooled annual-average weights. Domain variance analysis must retain the full design.", "This audit does not approve item wording, response recodes, missingness rules, constructs, or cross-survey links." ] } diff --git a/research/stage1/build_primary_outcome_package.py b/research/stage1/build_primary_outcome_package.py index 7857a2d..4858db2 100644 --- a/research/stage1/build_primary_outcome_package.py +++ b/research/stage1/build_primary_outcome_package.py @@ -197,10 +197,10 @@ def build() -> dict[str, int]: "survey_id": survey_id, "component_id": component["component_id"], "survey": "GSHS", "country": component.get("country", ""), "year": str(component.get("collection_start", ""))[:4], "population": component.get("population", ""), "weight_field": "sample_weight", "psu_field": "sampling_psu", - "stratum_field": "sampling_stratum", "variance_method": "pending official component design review", + "stratum_field": "sampling_stratum", "variance_method": "complex-survey design using weight, stratum, and PSU; Taylor linearization planned", "namespace_rule": "prefix PSU and stratum with component_id before pooling", - "design_source": source_file or source_url, "design_status": "provisional", - "notes": "Canonical fields are present; observed full GSHS table has 20 missing values in each design field overall. Component-specific design validity remains to be reviewed.", + "design_source": f"{component.get('catalog_url', '')}; WHO GSHS methodology and data catalog", "design_status": "source_verified_fields", + "notes": "For all 21,533 respondents in the seven primary-outcome components, canonical weight/PSU/stratum fields are complete, positive, and numerically identical to raw_weight/raw_psu/raw_stratum. Namespace by component before pooling.", }) # YRBS: source-audited annual items. @@ -238,20 +238,20 @@ def build() -> dict[str, int]: }) for year in sorted({str(r["year"]) for r in yrbs_rows}): - if year in {"2017", "2019"}: - status = "provisional_design_incomplete" + if year == "1997": + status = "source_verified_one_missing_record" else: - status = "provisional" + status = "source_verified" designs.append({ "survey_id": f"YRBS::{year}", "component_id": f"YRBS_NATIONAL::{year}", "survey": "YRBS", "country": "United States", "year": year, "population": "students in grades 9-12 attending public and private schools in the United States", "weight_field": "weight", "psu_field": "psu", "stratum_field": "stratum", - "variance_method": "Taylor linearization candidate; annual official design confirmation pending", + "variance_method": "complex-survey design-based variance accounting for multistage clustering; Taylor linearization supported", "namespace_rule": "prefix PSU and stratum with survey year before pooling", "design_source": next(r["data_guide_source_file"] for r in yrbs_rows if str(r["year"]) == year), "design_status": status, - "notes": "2017/2019 local PSU and stratum fields are entirely missing and weights are partial; 1997 has one row missing all three design fields; 2021/2023 official-layout imports have complete design fields.", + "notes": "Annual guide identifies WEIGHT, STRATUM, and PSU. Official-layout repairs restored complete 2017/2019 fields; 2021/2023 imports are complete. One 1997 record lacks all three design fields and requires explicit exclusion/handling.", }) # NSDUH: four youth-module years. Attempt is documented but unavailable locally. @@ -299,12 +299,12 @@ def build() -> dict[str, int]: designs.append({ "survey_id": f"NSDUH::{year}", "component_id": f"NSDUH_YOUTH::{year}", "survey": "NSDUH", "country": "United States", "year": year, "population": "adolescents aged 12-17", - "weight_field": "ANALWT2_C (year-specific C1-C4 mapping unresolved)", "psu_field": "VEREP", - "stratum_field": "VESTR_C", "variance_method": "pending official combined-file guide review", - "namespace_rule": "prefix variance units with survey year before pooling", - "design_source": "Dataset/可直接分析数据包_NSDUH_YRBS/README.md; official annual/combined-file guide pending", - "design_status": "provisional_weight_mapping_unresolved", - "notes": "All four ANALWT2_C1-C4 fields are positive in every local year; do not select a weight by suffix without the official year-specific mapping.", + "weight_field": "ANALWT2_C1 (single-year); C2/C3/C4 are C1 divided by 2/3/4 for pooled annual averages", "psu_field": "VEREP", + "stratum_field": "VESTR_C", "variance_method": "Taylor series linearization; 50 df for 2014 onward national PUF estimates", + "namespace_rule": "retain full-sample design and specify CATAG2=1 as a survey domain; do not physically subset before variance estimation", + "design_source": "Dataset/NSDUH/2023-nsduh-puf-data-users-guide.pdf sections 6, 8, 9, 10 and 12; official annual API metadata", + "design_status": "source_verified", + "notes": "Local C2=C1/2, C3=C1/3, and C4=C1/4 exactly for every row. Use C1 for a single year. VESTR_C and VEREP are complete; domain analysis must retain all PSUs.", }) links = [] diff --git a/research/stage1/input_structure_audit.json b/research/stage1/input_structure_audit.json index 1329932..4d33e24 100644 --- a/research/stage1/input_structure_audit.json +++ b/research/stage1/input_structure_audit.json @@ -27,7 +27,7 @@ }, "YRBS_selected_1991_2019": { "path": "Dataset/可直接分析数据包_NSDUH_YRBS/YRBS/yrbs_1991_2019_selected.parquet", - "bytes": 2576853, + "bytes": 2595834, "rows": 217341, "columns": 79, "row_groups": 1, @@ -49,9 +49,9 @@ "2019": 13677 }, "design_non_null": { - "weight": 210769, - "stratum": 188898, - "psu": 188898 + "weight": 217340, + "stratum": 217340, + "psu": 217340 }, "design_non_null_by_year": { "1991": { @@ -120,10 +120,14 @@ "psu": 15624 }, "2017": { - "weight": 13026 + "weight": 14765, + "stratum": 14765, + "psu": 14765 }, "2019": { - "weight": 8845 + "weight": 13677, + "stratum": 13677, + "psu": 13677 } }, "design_positive_by_year": { @@ -193,10 +197,14 @@ "psu": 15624 }, "2017": { - "weight": 13026 + "weight": 14765, + "stratum": 14765, + "psu": 14765 }, "2019": { - "weight": 8845 + "weight": 13677, + "stratum": 13677, + "psu": 13677 } }, "required_columns_present": true, diff --git a/research/stage1/repair_yrbs_2017_2019_design.py b/research/stage1/repair_yrbs_2017_2019_design.py new file mode 100644 index 0000000..e7e38b8 --- /dev/null +++ b/research/stage1/repair_yrbs_2017_2019_design.py @@ -0,0 +1,134 @@ +"""Restore YRBS 2017/2019 design fields from official fixed-width locations.""" + +from __future__ import annotations + +import hashlib +import json +from pathlib import Path + +import pyarrow as pa +import pyarrow.csv as pacsv +import pyarrow.parquet as pq + + +ROOT = Path(__file__).resolve().parents[2] +RAW = ROOT / "Dataset" / "YRBS_National_1991_2023" / "raw" +GUIDES = ROOT / "Dataset" / "YRBS_National_1991_2023" / "documentation" / "UserGuide" +FULL = ROOT / "Dataset" / "可直接分析数据包_NSDUH_YRBS" / "YRBS" / "full_by_year" +STAGE1 = ROOT / "research" / "stage1" +SELECTED_PARQUET = FULL.parent / "yrbs_1991_2019_selected.parquet" +SELECTED_CSV = FULL.parent / "yrbs_1991_2019_selected.csv" +SPECS = { + 2017: { + "raw": RAW / "XXH2017_YRBS_Data.dat", + "guide": GUIDES / "2017_YRBS_Data_Users_Guide.pdf", + "page": "PDF page 102 / printed page 96", + "positions": {"weight": (387, 396), "stratum": (397, 399), "psu": (400, 405)}, + }, + 2019: { + "raw": RAW / "XXH2019_YRBS_Data.dat", + "guide": GUIDES / "2019_National_YRBS_Data_Users_Guide.pdf", + "page": "PDF page 89 / printed page 86", + "positions": {"weight": (386, 395), "stratum": (396, 398), "psu": (399, 404)}, + }, +} + + +def digest(path: Path) -> str: + h = hashlib.sha256() + with path.open("rb") as handle: + for block in iter(lambda: handle.read(1024 * 1024), b""): + h.update(block) + return h.hexdigest() + + +def parse_numeric(line: str, start: int, end: int) -> float | None: + text = line[start - 1 : end].strip() + return None if not text or text == "." else float(text) + + +def main() -> None: + results = [] + repaired_values = {} + for year, spec in SPECS.items(): + parquet = FULL / f"yrbs_{year}_full.parquet" + table = pq.read_table(parquet) + source_rows = table.column("source_row").to_pylist() + if source_rows != list(range(1, table.num_rows + 1)): + raise AssertionError(f"{year}: source_row is not sequential; cannot prove raw-row alignment") + values = {name: [] for name in spec["positions"]} + lengths = set() + with spec["raw"].open(encoding="ascii", newline="") as handle: + for line in handle: + line = line.rstrip("\r\n") + lengths.add(len(line)) + for name, (start, end) in spec["positions"].items(): + values[name].append(parse_numeric(line, start, end)) + if any(len(column) != table.num_rows for column in values.values()): + raise AssertionError(f"{year}: raw/Parquet row counts differ") + for name, column in values.items(): + index = table.schema.get_field_index(name) + table = table.set_column(index, name, pa.array(column, type=pa.float64())) + temp = parquet.with_suffix(".repair.tmp.parquet") + pq.write_table(table, temp, compression="zstd") + check = pq.read_table(temp, columns=["source_row", "weight", "stratum", "psu"]) + nonmissing = {name: check.column(name).null_count for name in ("weight", "stratum", "psu")} + if any(nonmissing.values()): + raise AssertionError(f"{year}: design fields still contain nulls: {nonmissing}") + if min(check.column("weight").to_pylist()) <= 0: + raise AssertionError(f"{year}: nonpositive weight") + temp.replace(parquet) + repaired_values[year] = values + weight_sum = sum(values["weight"]) + results.append({ + "year": year, "rows": table.num_rows, "raw_record_lengths": sorted(lengths), + "raw_file": str(spec["raw"].relative_to(ROOT)).replace("\\", "/"), + "raw_sha256": digest(spec["raw"]), + "guide_file": str(spec["guide"].relative_to(ROOT)).replace("\\", "/"), + "guide_sha256": digest(spec["guide"]), "page_or_section": spec["page"], + "positions": spec["positions"], "design_nulls_after_repair": nonmissing, + "weight_min": min(values["weight"]), "weight_max": max(values["weight"]), + "weight_sum": weight_sum, + "weight_sum_minus_sample_size": weight_sum - table.num_rows, + "output_file": str(parquet.relative_to(ROOT)).replace("\\", "/"), + "status": "passed", + }) + selected = pq.read_table(SELECTED_PARQUET) + years = selected.column("survey_year").to_pylist() + source_rows = selected.column("source_row").to_pylist() + for name in ("weight", "stratum", "psu"): + existing = selected.column(name).to_pylist() + repaired = [ + repaired_values[int(year)][name][int(source_row) - 1] + if int(year) in repaired_values else old + for old, year, source_row in zip(existing, years, source_rows) + ] + selected = selected.set_column(selected.schema.get_field_index(name), name, pa.array(repaired, type=pa.float64())) + selected_temp = SELECTED_PARQUET.with_suffix(".repair.tmp.parquet") + pq.write_table(selected, selected_temp, compression="zstd") + selected_temp.replace(SELECTED_PARQUET) + pacsv.write_csv(selected, SELECTED_CSV) + for result in results: + year = result["year"] + mask = [i for i, value in enumerate(years) if int(value) == year] + result["selected_table_design_nulls_after_repair"] = { + name: sum(selected.column(name)[i].as_py() is None for i in mask) + for name in ("weight", "stratum", "psu") + } + audit = { + "schema_version": "1.0", "generated_on": "2026-09-20", "status": "passed", + "purpose": "Restore YRBS 2017/2019 weight, stratum, and PSU from the official guide's fixed-width locations.", + "years": results, + "decision": "2017 and 2019 design fields are no longer missing in the annual full Parquet files; use WEIGHT with STRATUM and PSU in complex-survey procedures.", + } + (STAGE1 / "yrbs_2017_2019_design_repair_audit.json").write_text(json.dumps(audit, indent=2, ensure_ascii=False) + "\n", encoding="utf-8") + lines = ["# YRBS 2017/2019 design-field repair audit", "", audit["decision"], ""] + for row in results: + lines += [f"## {row['year']}", "", f"- rows: {row['rows']:,}", f"- official locations: `{row['positions']}` ({row['page_or_section']})", f"- design nulls after repair: `{row['design_nulls_after_repair']}`", f"- weight range/sum: {row['weight_min']:.4f}–{row['weight_max']:.4f}; sum {row['weight_sum']:.4f}", ""] + lines += ["The repair changes only derived Parquet files, which remain reproducible and excluded from Git. Official raw files are never modified.", ""] + (STAGE1 / "yrbs_2017_2019_design_repair_audit.md").write_text("\n".join(lines), encoding="utf-8") + print(json.dumps(audit, indent=2, ensure_ascii=False)) + + +if __name__ == "__main__": + main() diff --git a/research/stage1/survey_design.csv b/research/stage1/survey_design.csv index b95c9db..ca48c69 100644 --- a/research/stage1/survey_design.csv +++ b/research/stage1/survey_design.csv @@ -1,29 +1,29 @@ survey_id,component_id,survey,country,year,population,weight_field,psu_field,stratum_field,variance_method,namespace_rule,design_source,design_status,notes -GSHS::2023_MAR_GSHS_v01,2023_MAR_GSHS_v01::mar_fez2023,GSHS,Morocco,2023,school-attending students; exact sampled ages/grades remain pending component source review,sample_weight,sampling_psu,sampling_stratum,pending official component design review,prefix PSU and stratum with component_id before pooling,Dataset/GSHS-全球学生健康调查数据/GSHS/Questionnaire/GSHS Per Country/Morocco 2023/Fez_2022_GSHS_Questionnaire_Final_FRENCH.pdf,provisional,Canonical fields are present; observed full GSHS table has 20 missing values in each design field overall. Component-specific design validity remains to be reviewed. -GSHS::GHA_2022_GSHS_v01,GHA_2022_GSHS_v01::gha_sek_tak2022,GSHS,Ghana,2022,school-attending students; exact sampled ages/grades remain pending component source review,sample_weight,sampling_psu,sampling_stratum,pending official component design review,prefix PSU and stratum with component_id before pooling,Dataset/GSHS-全球学生健康调查数据/GSHS/Questionnaire/GSHS Per Country/Ghana 2022/Sekondi-Takoradi-2022_GSHS_Questionnaire-Final.pdf,provisional,Canonical fields are present; observed full GSHS table has 20 missing values in each design field overall. Component-specific design validity remains to be reviewed. -GSHS::IND_2022_GSHS_v01,IND_2022_GSHS_v01::ind_jaipur_2022,GSHS,India,2022,school-attending students; exact sampled ages/grades remain pending component source review,sample_weight,sampling_psu,sampling_stratum,pending official component design review,prefix PSU and stratum with component_id before pooling,Dataset/GSHS-全球学生健康调查数据/GSHS/Questionnaire/GSHS Per Country/India 2022/Jaipur-2022_GSHS_Questionnaire-Final.docx,provisional,Canonical fields are present; observed full GSHS table has 20 missing values in each design field overall. Component-specific design validity remains to be reviewed. -GSHS::JAM_2023_GSHS_v01,JAM_2023_GSHS_v01::jam_st_cath_2023,GSHS,Jamaica,2023,school-attending students; exact sampled ages/grades remain pending component source review,sample_weight,sampling_psu,sampling_stratum,pending official component design review,prefix PSU and stratum with component_id before pooling,Dataset/GSHS-全球学生健康调查数据/GSHS/Questionnaire/GSHS Per Country/Jamaica 2023/FINAL_St. Catherine-2023-GSHS-Questionnaire.docx,provisional,Canonical fields are present; observed full GSHS table has 20 missing values in each design field overall. Component-specific design validity remains to be reviewed. -GSHS::MNG_2023_GSHS_v01,MNG_2023_GSHS_v01::mng2023,GSHS,Mongolia,2023,school-attending students; exact sampled ages/grades remain pending component source review,sample_weight,sampling_psu,sampling_stratum,pending official component design review,prefix PSU and stratum with component_id before pooling,Dataset/GSHS-全球学生健康调查数据/GSHS/Questionnaire/GSHS Per Country/Mongolia 2023/Mongolia-GSHS-Questionnaire-ENGLISH.docx,provisional,Canonical fields are present; observed full GSHS table has 20 missing values in each design field overall. Component-specific design validity remains to be reviewed. -GSHS::SLB_2023_GSHS_v01,SLB_2023_GSHS_v01::slb_2023,GSHS,Solomon Islands,2023,school-going adolescents aged 13-17 years; Form 1-Form 6; all students in selected classes eligible,sample_weight,sampling_psu,sampling_stratum,pending official component design review,prefix PSU and stratum with component_id before pooling,https://extranet.who.int/ncdsmicrodata/index.php/catalog/992,provisional,Canonical fields are present; observed full GSHS table has 20 missing values in each design field overall. Component-specific design validity remains to be reviewed. -GSHS::WLF_2023_GSHS_v01,WLF_2023_GSHS_v01::wlf_2023,GSHS,Wallis and Futuna,2023,school-attending students; exact sampled ages/grades remain pending component source review,sample_weight,sampling_psu,sampling_stratum,pending official component design review,prefix PSU and stratum with component_id before pooling,Dataset/GSHS-全球学生健康调查数据/GSHS/Questionnaire/GSHS Per Country/Wallis and Futuna 2023/Wallis and Futuna - GSHS Questionnaire - Final - French.docx,provisional,Canonical fields are present; observed full GSHS table has 20 missing values in each design field overall. Component-specific design validity remains to be reviewed. -YRBS::1991,YRBS_NATIONAL::1991,YRBS,United States,1991,students in grades 9-12 attending public and private schools in the United States,weight,psu,stratum,Taylor linearization candidate; annual official design confirmation pending,prefix PSU and stratum with survey year before pooling,Dataset/YRBS_National_1991_2023/documentation/UserGuide/YRBS_1991_National_User_Guide.pdf,provisional,2017/2019 local PSU and stratum fields are entirely missing and weights are partial; 1997 has one row missing all three design fields; 2021/2023 official-layout imports have complete design fields. -YRBS::1993,YRBS_NATIONAL::1993,YRBS,United States,1993,students in grades 9-12 attending public and private schools in the United States,weight,psu,stratum,Taylor linearization candidate; annual official design confirmation pending,prefix PSU and stratum with survey year before pooling,Dataset/YRBS_National_1991_2023/documentation/UserGuide/YRBS_1993_National_User_Guide.pdf,provisional,2017/2019 local PSU and stratum fields are entirely missing and weights are partial; 1997 has one row missing all three design fields; 2021/2023 official-layout imports have complete design fields. -YRBS::1995,YRBS_NATIONAL::1995,YRBS,United States,1995,students in grades 9-12 attending public and private schools in the United States,weight,psu,stratum,Taylor linearization candidate; annual official design confirmation pending,prefix PSU and stratum with survey year before pooling,Dataset/YRBS_National_1991_2023/documentation/UserGuide/YRBS_1995_National_User_Guide.pdf,provisional,2017/2019 local PSU and stratum fields are entirely missing and weights are partial; 1997 has one row missing all three design fields; 2021/2023 official-layout imports have complete design fields. -YRBS::1997,YRBS_NATIONAL::1997,YRBS,United States,1997,students in grades 9-12 attending public and private schools in the United States,weight,psu,stratum,Taylor linearization candidate; annual official design confirmation pending,prefix PSU and stratum with survey year before pooling,Dataset/YRBS_National_1991_2023/documentation/UserGuide/YRBS_1997_National_User_Guide.pdf,provisional,2017/2019 local PSU and stratum fields are entirely missing and weights are partial; 1997 has one row missing all three design fields; 2021/2023 official-layout imports have complete design fields. -YRBS::1999,YRBS_NATIONAL::1999,YRBS,United States,1999,students in grades 9-12 attending public and private schools in the United States,weight,psu,stratum,Taylor linearization candidate; annual official design confirmation pending,prefix PSU and stratum with survey year before pooling,Dataset/YRBS_National_1991_2023/documentation/UserGuide/YRBS_1999_National_User_Guide.pdf,provisional,2017/2019 local PSU and stratum fields are entirely missing and weights are partial; 1997 has one row missing all three design fields; 2021/2023 official-layout imports have complete design fields. -YRBS::2001,YRBS_NATIONAL::2001,YRBS,United States,2001,students in grades 9-12 attending public and private schools in the United States,weight,psu,stratum,Taylor linearization candidate; annual official design confirmation pending,prefix PSU and stratum with survey year before pooling,Dataset/YRBS_National_1991_2023/documentation/UserGuide/YRBS_2001_National_User_Guide.pdf,provisional,2017/2019 local PSU and stratum fields are entirely missing and weights are partial; 1997 has one row missing all three design fields; 2021/2023 official-layout imports have complete design fields. -YRBS::2003,YRBS_NATIONAL::2003,YRBS,United States,2003,students in grades 9-12 attending public and private schools in the United States,weight,psu,stratum,Taylor linearization candidate; annual official design confirmation pending,prefix PSU and stratum with survey year before pooling,Dataset/YRBS_National_1991_2023/documentation/UserGuide/YRBS_2003_National_User_Guide.pdf,provisional,2017/2019 local PSU and stratum fields are entirely missing and weights are partial; 1997 has one row missing all three design fields; 2021/2023 official-layout imports have complete design fields. -YRBS::2005,YRBS_NATIONAL::2005,YRBS,United States,2005,students in grades 9-12 attending public and private schools in the United States,weight,psu,stratum,Taylor linearization candidate; annual official design confirmation pending,prefix PSU and stratum with survey year before pooling,Dataset/YRBS_National_1991_2023/documentation/UserGuide/YRBS_2005_National_User_Guide.pdf,provisional,2017/2019 local PSU and stratum fields are entirely missing and weights are partial; 1997 has one row missing all three design fields; 2021/2023 official-layout imports have complete design fields. -YRBS::2007,YRBS_NATIONAL::2007,YRBS,United States,2007,students in grades 9-12 attending public and private schools in the United States,weight,psu,stratum,Taylor linearization candidate; annual official design confirmation pending,prefix PSU and stratum with survey year before pooling,Dataset/YRBS_National_1991_2023/documentation/UserGuide/YRBS_2007_National_User_Guide.pdf,provisional,2017/2019 local PSU and stratum fields are entirely missing and weights are partial; 1997 has one row missing all three design fields; 2021/2023 official-layout imports have complete design fields. -YRBS::2009,YRBS_NATIONAL::2009,YRBS,United States,2009,students in grades 9-12 attending public and private schools in the United States,weight,psu,stratum,Taylor linearization candidate; annual official design confirmation pending,prefix PSU and stratum with survey year before pooling,Dataset/YRBS_National_1991_2023/documentation/UserGuide/YRBS_2009_National_User_Guide.pdf,provisional,2017/2019 local PSU and stratum fields are entirely missing and weights are partial; 1997 has one row missing all three design fields; 2021/2023 official-layout imports have complete design fields. -YRBS::2011,YRBS_NATIONAL::2011,YRBS,United States,2011,students in grades 9-12 attending public and private schools in the United States,weight,psu,stratum,Taylor linearization candidate; annual official design confirmation pending,prefix PSU and stratum with survey year before pooling,Dataset/YRBS_National_1991_2023/documentation/UserGuide/YRBS_2011_National_User_Guide.pdf,provisional,2017/2019 local PSU and stratum fields are entirely missing and weights are partial; 1997 has one row missing all three design fields; 2021/2023 official-layout imports have complete design fields. -YRBS::2013,YRBS_NATIONAL::2013,YRBS,United States,2013,students in grades 9-12 attending public and private schools in the United States,weight,psu,stratum,Taylor linearization candidate; annual official design confirmation pending,prefix PSU and stratum with survey year before pooling,Dataset/YRBS_National_1991_2023/documentation/UserGuide/YRBS_2013_National_User_Guide.pdf,provisional,2017/2019 local PSU and stratum fields are entirely missing and weights are partial; 1997 has one row missing all three design fields; 2021/2023 official-layout imports have complete design fields. -YRBS::2015,YRBS_NATIONAL::2015,YRBS,United States,2015,students in grades 9-12 attending public and private schools in the United States,weight,psu,stratum,Taylor linearization candidate; annual official design confirmation pending,prefix PSU and stratum with survey year before pooling,Dataset/YRBS_National_1991_2023/documentation/UserGuide/2015_yrbs-data-users_guide_smy_combined.pdf,provisional,2017/2019 local PSU and stratum fields are entirely missing and weights are partial; 1997 has one row missing all three design fields; 2021/2023 official-layout imports have complete design fields. -YRBS::2017,YRBS_NATIONAL::2017,YRBS,United States,2017,students in grades 9-12 attending public and private schools in the United States,weight,psu,stratum,Taylor linearization candidate; annual official design confirmation pending,prefix PSU and stratum with survey year before pooling,Dataset/YRBS_National_1991_2023/documentation/UserGuide/2017_YRBS_Data_Users_Guide.pdf,provisional_design_incomplete,2017/2019 local PSU and stratum fields are entirely missing and weights are partial; 1997 has one row missing all three design fields; 2021/2023 official-layout imports have complete design fields. -YRBS::2019,YRBS_NATIONAL::2019,YRBS,United States,2019,students in grades 9-12 attending public and private schools in the United States,weight,psu,stratum,Taylor linearization candidate; annual official design confirmation pending,prefix PSU and stratum with survey year before pooling,Dataset/YRBS_National_1991_2023/documentation/UserGuide/2019_National_YRBS_Data_Users_Guide.pdf,provisional_design_incomplete,2017/2019 local PSU and stratum fields are entirely missing and weights are partial; 1997 has one row missing all three design fields; 2021/2023 official-layout imports have complete design fields. -YRBS::2021,YRBS_NATIONAL::2021,YRBS,United States,2021,students in grades 9-12 attending public and private schools in the United States,weight,psu,stratum,Taylor linearization candidate; annual official design confirmation pending,prefix PSU and stratum with survey year before pooling,Dataset/YRBS_National_1991_2023/documentation/UserGuide/2021_YRBS_Data_Users_Guide_508.pdf,provisional,2017/2019 local PSU and stratum fields are entirely missing and weights are partial; 1997 has one row missing all three design fields; 2021/2023 official-layout imports have complete design fields. -YRBS::2023,YRBS_NATIONAL::2023,YRBS,United States,2023,students in grades 9-12 attending public and private schools in the United States,weight,psu,stratum,Taylor linearization candidate; annual official design confirmation pending,prefix PSU and stratum with survey year before pooling,Dataset/YRBS_National_1991_2023/documentation/UserGuide/2023_National_YRBS_Data_Users_Guide508.pdf,provisional,2017/2019 local PSU and stratum fields are entirely missing and weights are partial; 1997 has one row missing all three design fields; 2021/2023 official-layout imports have complete design fields. -NSDUH::2021,NSDUH_YOUTH::2021,NSDUH,United States,2021,adolescents aged 12-17,ANALWT2_C (year-specific C1-C4 mapping unresolved),VEREP,VESTR_C,pending official combined-file guide review,prefix variance units with survey year before pooling,Dataset/可直接分析数据包_NSDUH_YRBS/README.md; official annual/combined-file guide pending,provisional_weight_mapping_unresolved,All four ANALWT2_C1-C4 fields are positive in every local year; do not select a weight by suffix without the official year-specific mapping. -NSDUH::2022,NSDUH_YOUTH::2022,NSDUH,United States,2022,adolescents aged 12-17,ANALWT2_C (year-specific C1-C4 mapping unresolved),VEREP,VESTR_C,pending official combined-file guide review,prefix variance units with survey year before pooling,Dataset/可直接分析数据包_NSDUH_YRBS/README.md; official annual/combined-file guide pending,provisional_weight_mapping_unresolved,All four ANALWT2_C1-C4 fields are positive in every local year; do not select a weight by suffix without the official year-specific mapping. -NSDUH::2023,NSDUH_YOUTH::2023,NSDUH,United States,2023,adolescents aged 12-17,ANALWT2_C (year-specific C1-C4 mapping unresolved),VEREP,VESTR_C,pending official combined-file guide review,prefix variance units with survey year before pooling,Dataset/可直接分析数据包_NSDUH_YRBS/README.md; official annual/combined-file guide pending,provisional_weight_mapping_unresolved,All four ANALWT2_C1-C4 fields are positive in every local year; do not select a weight by suffix without the official year-specific mapping. -NSDUH::2024,NSDUH_YOUTH::2024,NSDUH,United States,2024,adolescents aged 12-17,ANALWT2_C (year-specific C1-C4 mapping unresolved),VEREP,VESTR_C,pending official combined-file guide review,prefix variance units with survey year before pooling,Dataset/可直接分析数据包_NSDUH_YRBS/README.md; official annual/combined-file guide pending,provisional_weight_mapping_unresolved,All four ANALWT2_C1-C4 fields are positive in every local year; do not select a weight by suffix without the official year-specific mapping. +GSHS::2023_MAR_GSHS_v01,2023_MAR_GSHS_v01::mar_fez2023,GSHS,Morocco,2023,school-attending students; exact sampled ages/grades remain pending component source review,sample_weight,sampling_psu,sampling_stratum,"complex-survey design using weight, stratum, and PSU; Taylor linearization planned",prefix PSU and stratum with component_id before pooling,https://extranet.who.int/ncdsmicrodata/index.php/catalog/982; WHO GSHS methodology and data catalog,source_verified_fields,"For all 21,533 respondents in the seven primary-outcome components, canonical weight/PSU/stratum fields are complete, positive, and numerically identical to raw_weight/raw_psu/raw_stratum. Namespace by component before pooling." +GSHS::GHA_2022_GSHS_v01,GHA_2022_GSHS_v01::gha_sek_tak2022,GSHS,Ghana,2022,school-attending students; exact sampled ages/grades remain pending component source review,sample_weight,sampling_psu,sampling_stratum,"complex-survey design using weight, stratum, and PSU; Taylor linearization planned",prefix PSU and stratum with component_id before pooling,https://extranet.who.int/ncdsmicrodata/index.php/catalog/981; WHO GSHS methodology and data catalog,source_verified_fields,"For all 21,533 respondents in the seven primary-outcome components, canonical weight/PSU/stratum fields are complete, positive, and numerically identical to raw_weight/raw_psu/raw_stratum. Namespace by component before pooling." +GSHS::IND_2022_GSHS_v01,IND_2022_GSHS_v01::ind_jaipur_2022,GSHS,India,2022,school-attending students; exact sampled ages/grades remain pending component source review,sample_weight,sampling_psu,sampling_stratum,"complex-survey design using weight, stratum, and PSU; Taylor linearization planned",prefix PSU and stratum with component_id before pooling,https://extranet.who.int/ncdsmicrodata/index.php/catalog/984; WHO GSHS methodology and data catalog,source_verified_fields,"For all 21,533 respondents in the seven primary-outcome components, canonical weight/PSU/stratum fields are complete, positive, and numerically identical to raw_weight/raw_psu/raw_stratum. Namespace by component before pooling." +GSHS::JAM_2023_GSHS_v01,JAM_2023_GSHS_v01::jam_st_cath_2023,GSHS,Jamaica,2023,school-attending students; exact sampled ages/grades remain pending component source review,sample_weight,sampling_psu,sampling_stratum,"complex-survey design using weight, stratum, and PSU; Taylor linearization planned",prefix PSU and stratum with component_id before pooling,https://extranet.who.int/ncdsmicrodata/index.php/catalog/983; WHO GSHS methodology and data catalog,source_verified_fields,"For all 21,533 respondents in the seven primary-outcome components, canonical weight/PSU/stratum fields are complete, positive, and numerically identical to raw_weight/raw_psu/raw_stratum. Namespace by component before pooling." +GSHS::MNG_2023_GSHS_v01,MNG_2023_GSHS_v01::mng2023,GSHS,Mongolia,2023,school-attending students; exact sampled ages/grades remain pending component source review,sample_weight,sampling_psu,sampling_stratum,"complex-survey design using weight, stratum, and PSU; Taylor linearization planned",prefix PSU and stratum with component_id before pooling,https://extranet.who.int/ncdsmicrodata/index.php/catalog/990; WHO GSHS methodology and data catalog,source_verified_fields,"For all 21,533 respondents in the seven primary-outcome components, canonical weight/PSU/stratum fields are complete, positive, and numerically identical to raw_weight/raw_psu/raw_stratum. Namespace by component before pooling." +GSHS::SLB_2023_GSHS_v01,SLB_2023_GSHS_v01::slb_2023,GSHS,Solomon Islands,2023,school-going adolescents aged 13-17 years; Form 1-Form 6; all students in selected classes eligible,sample_weight,sampling_psu,sampling_stratum,"complex-survey design using weight, stratum, and PSU; Taylor linearization planned",prefix PSU and stratum with component_id before pooling,https://extranet.who.int/ncdsmicrodata/index.php/catalog/992; WHO GSHS methodology and data catalog,source_verified_fields,"For all 21,533 respondents in the seven primary-outcome components, canonical weight/PSU/stratum fields are complete, positive, and numerically identical to raw_weight/raw_psu/raw_stratum. Namespace by component before pooling." +GSHS::WLF_2023_GSHS_v01,WLF_2023_GSHS_v01::wlf_2023,GSHS,Wallis and Futuna,2023,school-attending students; exact sampled ages/grades remain pending component source review,sample_weight,sampling_psu,sampling_stratum,"complex-survey design using weight, stratum, and PSU; Taylor linearization planned",prefix PSU and stratum with component_id before pooling,https://extranet.who.int/ncdsmicrodata/index.php/catalog/994; WHO GSHS methodology and data catalog,source_verified_fields,"For all 21,533 respondents in the seven primary-outcome components, canonical weight/PSU/stratum fields are complete, positive, and numerically identical to raw_weight/raw_psu/raw_stratum. Namespace by component before pooling." +YRBS::1991,YRBS_NATIONAL::1991,YRBS,United States,1991,students in grades 9-12 attending public and private schools in the United States,weight,psu,stratum,complex-survey design-based variance accounting for multistage clustering; Taylor linearization supported,prefix PSU and stratum with survey year before pooling,Dataset/YRBS_National_1991_2023/documentation/UserGuide/YRBS_1991_National_User_Guide.pdf,source_verified,"Annual guide identifies WEIGHT, STRATUM, and PSU. Official-layout repairs restored complete 2017/2019 fields; 2021/2023 imports are complete. One 1997 record lacks all three design fields and requires explicit exclusion/handling." +YRBS::1993,YRBS_NATIONAL::1993,YRBS,United States,1993,students in grades 9-12 attending public and private schools in the United States,weight,psu,stratum,complex-survey design-based variance accounting for multistage clustering; Taylor linearization supported,prefix PSU and stratum with survey year before pooling,Dataset/YRBS_National_1991_2023/documentation/UserGuide/YRBS_1993_National_User_Guide.pdf,source_verified,"Annual guide identifies WEIGHT, STRATUM, and PSU. Official-layout repairs restored complete 2017/2019 fields; 2021/2023 imports are complete. One 1997 record lacks all three design fields and requires explicit exclusion/handling." +YRBS::1995,YRBS_NATIONAL::1995,YRBS,United States,1995,students in grades 9-12 attending public and private schools in the United States,weight,psu,stratum,complex-survey design-based variance accounting for multistage clustering; Taylor linearization supported,prefix PSU and stratum with survey year before pooling,Dataset/YRBS_National_1991_2023/documentation/UserGuide/YRBS_1995_National_User_Guide.pdf,source_verified,"Annual guide identifies WEIGHT, STRATUM, and PSU. Official-layout repairs restored complete 2017/2019 fields; 2021/2023 imports are complete. One 1997 record lacks all three design fields and requires explicit exclusion/handling." +YRBS::1997,YRBS_NATIONAL::1997,YRBS,United States,1997,students in grades 9-12 attending public and private schools in the United States,weight,psu,stratum,complex-survey design-based variance accounting for multistage clustering; Taylor linearization supported,prefix PSU and stratum with survey year before pooling,Dataset/YRBS_National_1991_2023/documentation/UserGuide/YRBS_1997_National_User_Guide.pdf,source_verified_one_missing_record,"Annual guide identifies WEIGHT, STRATUM, and PSU. Official-layout repairs restored complete 2017/2019 fields; 2021/2023 imports are complete. One 1997 record lacks all three design fields and requires explicit exclusion/handling." +YRBS::1999,YRBS_NATIONAL::1999,YRBS,United States,1999,students in grades 9-12 attending public and private schools in the United States,weight,psu,stratum,complex-survey design-based variance accounting for multistage clustering; Taylor linearization supported,prefix PSU and stratum with survey year before pooling,Dataset/YRBS_National_1991_2023/documentation/UserGuide/YRBS_1999_National_User_Guide.pdf,source_verified,"Annual guide identifies WEIGHT, STRATUM, and PSU. Official-layout repairs restored complete 2017/2019 fields; 2021/2023 imports are complete. One 1997 record lacks all three design fields and requires explicit exclusion/handling." +YRBS::2001,YRBS_NATIONAL::2001,YRBS,United States,2001,students in grades 9-12 attending public and private schools in the United States,weight,psu,stratum,complex-survey design-based variance accounting for multistage clustering; Taylor linearization supported,prefix PSU and stratum with survey year before pooling,Dataset/YRBS_National_1991_2023/documentation/UserGuide/YRBS_2001_National_User_Guide.pdf,source_verified,"Annual guide identifies WEIGHT, STRATUM, and PSU. Official-layout repairs restored complete 2017/2019 fields; 2021/2023 imports are complete. One 1997 record lacks all three design fields and requires explicit exclusion/handling." +YRBS::2003,YRBS_NATIONAL::2003,YRBS,United States,2003,students in grades 9-12 attending public and private schools in the United States,weight,psu,stratum,complex-survey design-based variance accounting for multistage clustering; Taylor linearization supported,prefix PSU and stratum with survey year before pooling,Dataset/YRBS_National_1991_2023/documentation/UserGuide/YRBS_2003_National_User_Guide.pdf,source_verified,"Annual guide identifies WEIGHT, STRATUM, and PSU. Official-layout repairs restored complete 2017/2019 fields; 2021/2023 imports are complete. One 1997 record lacks all three design fields and requires explicit exclusion/handling." +YRBS::2005,YRBS_NATIONAL::2005,YRBS,United States,2005,students in grades 9-12 attending public and private schools in the United States,weight,psu,stratum,complex-survey design-based variance accounting for multistage clustering; Taylor linearization supported,prefix PSU and stratum with survey year before pooling,Dataset/YRBS_National_1991_2023/documentation/UserGuide/YRBS_2005_National_User_Guide.pdf,source_verified,"Annual guide identifies WEIGHT, STRATUM, and PSU. Official-layout repairs restored complete 2017/2019 fields; 2021/2023 imports are complete. One 1997 record lacks all three design fields and requires explicit exclusion/handling." +YRBS::2007,YRBS_NATIONAL::2007,YRBS,United States,2007,students in grades 9-12 attending public and private schools in the United States,weight,psu,stratum,complex-survey design-based variance accounting for multistage clustering; Taylor linearization supported,prefix PSU and stratum with survey year before pooling,Dataset/YRBS_National_1991_2023/documentation/UserGuide/YRBS_2007_National_User_Guide.pdf,source_verified,"Annual guide identifies WEIGHT, STRATUM, and PSU. Official-layout repairs restored complete 2017/2019 fields; 2021/2023 imports are complete. One 1997 record lacks all three design fields and requires explicit exclusion/handling." +YRBS::2009,YRBS_NATIONAL::2009,YRBS,United States,2009,students in grades 9-12 attending public and private schools in the United States,weight,psu,stratum,complex-survey design-based variance accounting for multistage clustering; Taylor linearization supported,prefix PSU and stratum with survey year before pooling,Dataset/YRBS_National_1991_2023/documentation/UserGuide/YRBS_2009_National_User_Guide.pdf,source_verified,"Annual guide identifies WEIGHT, STRATUM, and PSU. Official-layout repairs restored complete 2017/2019 fields; 2021/2023 imports are complete. One 1997 record lacks all three design fields and requires explicit exclusion/handling." +YRBS::2011,YRBS_NATIONAL::2011,YRBS,United States,2011,students in grades 9-12 attending public and private schools in the United States,weight,psu,stratum,complex-survey design-based variance accounting for multistage clustering; Taylor linearization supported,prefix PSU and stratum with survey year before pooling,Dataset/YRBS_National_1991_2023/documentation/UserGuide/YRBS_2011_National_User_Guide.pdf,source_verified,"Annual guide identifies WEIGHT, STRATUM, and PSU. Official-layout repairs restored complete 2017/2019 fields; 2021/2023 imports are complete. One 1997 record lacks all three design fields and requires explicit exclusion/handling." +YRBS::2013,YRBS_NATIONAL::2013,YRBS,United States,2013,students in grades 9-12 attending public and private schools in the United States,weight,psu,stratum,complex-survey design-based variance accounting for multistage clustering; Taylor linearization supported,prefix PSU and stratum with survey year before pooling,Dataset/YRBS_National_1991_2023/documentation/UserGuide/YRBS_2013_National_User_Guide.pdf,source_verified,"Annual guide identifies WEIGHT, STRATUM, and PSU. Official-layout repairs restored complete 2017/2019 fields; 2021/2023 imports are complete. One 1997 record lacks all three design fields and requires explicit exclusion/handling." +YRBS::2015,YRBS_NATIONAL::2015,YRBS,United States,2015,students in grades 9-12 attending public and private schools in the United States,weight,psu,stratum,complex-survey design-based variance accounting for multistage clustering; Taylor linearization supported,prefix PSU and stratum with survey year before pooling,Dataset/YRBS_National_1991_2023/documentation/UserGuide/2015_yrbs-data-users_guide_smy_combined.pdf,source_verified,"Annual guide identifies WEIGHT, STRATUM, and PSU. Official-layout repairs restored complete 2017/2019 fields; 2021/2023 imports are complete. One 1997 record lacks all three design fields and requires explicit exclusion/handling." +YRBS::2017,YRBS_NATIONAL::2017,YRBS,United States,2017,students in grades 9-12 attending public and private schools in the United States,weight,psu,stratum,complex-survey design-based variance accounting for multistage clustering; Taylor linearization supported,prefix PSU and stratum with survey year before pooling,Dataset/YRBS_National_1991_2023/documentation/UserGuide/2017_YRBS_Data_Users_Guide.pdf,source_verified,"Annual guide identifies WEIGHT, STRATUM, and PSU. Official-layout repairs restored complete 2017/2019 fields; 2021/2023 imports are complete. One 1997 record lacks all three design fields and requires explicit exclusion/handling." +YRBS::2019,YRBS_NATIONAL::2019,YRBS,United States,2019,students in grades 9-12 attending public and private schools in the United States,weight,psu,stratum,complex-survey design-based variance accounting for multistage clustering; Taylor linearization supported,prefix PSU and stratum with survey year before pooling,Dataset/YRBS_National_1991_2023/documentation/UserGuide/2019_National_YRBS_Data_Users_Guide.pdf,source_verified,"Annual guide identifies WEIGHT, STRATUM, and PSU. Official-layout repairs restored complete 2017/2019 fields; 2021/2023 imports are complete. One 1997 record lacks all three design fields and requires explicit exclusion/handling." +YRBS::2021,YRBS_NATIONAL::2021,YRBS,United States,2021,students in grades 9-12 attending public and private schools in the United States,weight,psu,stratum,complex-survey design-based variance accounting for multistage clustering; Taylor linearization supported,prefix PSU and stratum with survey year before pooling,Dataset/YRBS_National_1991_2023/documentation/UserGuide/2021_YRBS_Data_Users_Guide_508.pdf,source_verified,"Annual guide identifies WEIGHT, STRATUM, and PSU. Official-layout repairs restored complete 2017/2019 fields; 2021/2023 imports are complete. One 1997 record lacks all three design fields and requires explicit exclusion/handling." +YRBS::2023,YRBS_NATIONAL::2023,YRBS,United States,2023,students in grades 9-12 attending public and private schools in the United States,weight,psu,stratum,complex-survey design-based variance accounting for multistage clustering; Taylor linearization supported,prefix PSU and stratum with survey year before pooling,Dataset/YRBS_National_1991_2023/documentation/UserGuide/2023_National_YRBS_Data_Users_Guide508.pdf,source_verified,"Annual guide identifies WEIGHT, STRATUM, and PSU. Official-layout repairs restored complete 2017/2019 fields; 2021/2023 imports are complete. One 1997 record lacks all three design fields and requires explicit exclusion/handling." +NSDUH::2021,NSDUH_YOUTH::2021,NSDUH,United States,2021,adolescents aged 12-17,ANALWT2_C1 (single-year); C2/C3/C4 are C1 divided by 2/3/4 for pooled annual averages,VEREP,VESTR_C,Taylor series linearization; 50 df for 2014 onward national PUF estimates,retain full-sample design and specify CATAG2=1 as a survey domain; do not physically subset before variance estimation,"Dataset/NSDUH/2023-nsduh-puf-data-users-guide.pdf sections 6, 8, 9, 10 and 12; official annual API metadata",source_verified,"Local C2=C1/2, C3=C1/3, and C4=C1/4 exactly for every row. Use C1 for a single year. VESTR_C and VEREP are complete; domain analysis must retain all PSUs." +NSDUH::2022,NSDUH_YOUTH::2022,NSDUH,United States,2022,adolescents aged 12-17,ANALWT2_C1 (single-year); C2/C3/C4 are C1 divided by 2/3/4 for pooled annual averages,VEREP,VESTR_C,Taylor series linearization; 50 df for 2014 onward national PUF estimates,retain full-sample design and specify CATAG2=1 as a survey domain; do not physically subset before variance estimation,"Dataset/NSDUH/2023-nsduh-puf-data-users-guide.pdf sections 6, 8, 9, 10 and 12; official annual API metadata",source_verified,"Local C2=C1/2, C3=C1/3, and C4=C1/4 exactly for every row. Use C1 for a single year. VESTR_C and VEREP are complete; domain analysis must retain all PSUs." +NSDUH::2023,NSDUH_YOUTH::2023,NSDUH,United States,2023,adolescents aged 12-17,ANALWT2_C1 (single-year); C2/C3/C4 are C1 divided by 2/3/4 for pooled annual averages,VEREP,VESTR_C,Taylor series linearization; 50 df for 2014 onward national PUF estimates,retain full-sample design and specify CATAG2=1 as a survey domain; do not physically subset before variance estimation,"Dataset/NSDUH/2023-nsduh-puf-data-users-guide.pdf sections 6, 8, 9, 10 and 12; official annual API metadata",source_verified,"Local C2=C1/2, C3=C1/3, and C4=C1/4 exactly for every row. Use C1 for a single year. VESTR_C and VEREP are complete; domain analysis must retain all PSUs." +NSDUH::2024,NSDUH_YOUTH::2024,NSDUH,United States,2024,adolescents aged 12-17,ANALWT2_C1 (single-year); C2/C3/C4 are C1 divided by 2/3/4 for pooled annual averages,VEREP,VESTR_C,Taylor series linearization; 50 df for 2014 onward national PUF estimates,retain full-sample design and specify CATAG2=1 as a survey domain; do not physically subset before variance estimation,"Dataset/NSDUH/2023-nsduh-puf-data-users-guide.pdf sections 6, 8, 9, 10 and 12; official annual API metadata",source_verified,"Local C2=C1/2, C3=C1/3, and C4=C1/4 exactly for every row. Use C1 for a single year. VESTR_C and VEREP are complete; domain analysis must retain all PSUs." diff --git a/research/stage1/survey_design_source_audit.md b/research/stage1/survey_design_source_audit.md new file mode 100644 index 0000000..d519a6d --- /dev/null +++ b/research/stage1/survey_design_source_audit.md @@ -0,0 +1,25 @@ +# Survey-design source audit + +Status: **source fields verified; Gate 1 human item review still pending**. + +## GSHS + +WHO catalog metadata identifies `weight`, `stratum`, and `psu` as the overall analysis weight, sampling stratum, and primary sampling unit. In the seven components containing the primary outcomes, all 21,533 respondents have positive canonical design fields. `sample_weight`, `sampling_psu`, and `sampling_stratum` are numerically identical to `raw_weight`, `raw_psu`, and `raw_stratum`. PSU and stratum must be namespaced by component before pooling. + +Sources: WHO GSHS methodology and NCD Microdata catalog; component catalog URLs are retained in `survey_design.csv`. + +## YRBS + +Annual CDC guides define `WEIGHT`, `STRATUM`, and `PSU` and require software that accounts for the multistage cluster sample. Official fixed-width locations restored the incomplete 2017 and 2019 derived files. The 2017 locations are PDF page 102 (printed page 96); the 2019 locations are PDF page 89 (printed page 86). Both annual full files and the combined selected files now have complete positive design fields for those years. The only remaining local design missingness is one 1997 record with all three fields absent. The 2021 and 2023 official-layout imports are complete. + +## NSDUH + +The official public-use guide specifies `ANALWT2_C` as the final person-level weight, `VESTR_C` as the pseudo-stratum, `VEREP` as the pseudo-PSU, Taylor series linearization, and 50 degrees of freedom for 2014 onward national PUF estimates. It requires domain analysis without physically dropping non-domain respondents. + +The local combined package stores the single-year weight as `ANALWT2_C1`; across all 232,441 rows, `C2=C1/2`, `C3=C1/3`, and `C4=C1/4` exactly, matching pooled annual-average rules. All weight and variance fields are complete and positive. + +Source: `Dataset/NSDUH/2023-nsduh-puf-data-users-guide.pdf`, sections 6, 8, 9, 10 and 12. + +## Boundary + +This audit approves the design-field identities and roles in `survey_design.csv`. It does not approve cross-survey pooling, item equivalence, or human item-review decisions. The single YRBS 1997 design-incomplete record must not silently enter weighted variance estimation. diff --git a/research/stage1/yrbs_2017_2019_design_repair_audit.json b/research/stage1/yrbs_2017_2019_design_repair_audit.json new file mode 100644 index 0000000..707b326 --- /dev/null +++ b/research/stage1/yrbs_2017_2019_design_repair_audit.json @@ -0,0 +1,93 @@ +{ + "schema_version": "1.0", + "generated_on": "2026-09-20", + "status": "passed", + "purpose": "Restore YRBS 2017/2019 weight, stratum, and PSU from the official guide's fixed-width locations.", + "years": [ + { + "year": 2017, + "rows": 14765, + "raw_record_lengths": [ + 427 + ], + "raw_file": "Dataset/YRBS_National_1991_2023/raw/XXH2017_YRBS_Data.dat", + "raw_sha256": "99f7a87f1f936b21674603064118222247e797323377de7c7365e75a26c6f4b9", + "guide_file": "Dataset/YRBS_National_1991_2023/documentation/UserGuide/2017_YRBS_Data_Users_Guide.pdf", + "guide_sha256": "3ac5a8b3e62cef9007c5c86f4b8a6eaba15ebd82ef77332335e6670e5387370a", + "page_or_section": "PDF page 102 / printed page 96", + "positions": { + "weight": [ + 387, + 396 + ], + "stratum": [ + 397, + 399 + ], + "psu": [ + 400, + 405 + ] + }, + "design_nulls_after_repair": { + "weight": 0, + "stratum": 0, + "psu": 0 + }, + "weight_min": 0.0442, + "weight_max": 4.6923, + "weight_sum": 14765.0049, + "weight_sum_minus_sample_size": 0.004899999999906868, + "output_file": "Dataset/可直接分析数据包_NSDUH_YRBS/YRBS/full_by_year/yrbs_2017_full.parquet", + "status": "passed", + "selected_table_design_nulls_after_repair": { + "weight": 0, + "stratum": 0, + "psu": 0 + } + }, + { + "year": 2019, + "rows": 13677, + "raw_record_lengths": [ + 419 + ], + "raw_file": "Dataset/YRBS_National_1991_2023/raw/XXH2019_YRBS_Data.dat", + "raw_sha256": "9bfb3e87c519418eb6bd5b491ecfc723c1b21b85dbe796816fcce6b3cea8ca26", + "guide_file": "Dataset/YRBS_National_1991_2023/documentation/UserGuide/2019_National_YRBS_Data_Users_Guide.pdf", + "guide_sha256": "b60d9e9571939a47fab597448cda537e12e8c712cd20cd332d2ebe6c0fcd25b1", + "page_or_section": "PDF page 89 / printed page 86", + "positions": { + "weight": [ + 386, + 395 + ], + "stratum": [ + 396, + 398 + ], + "psu": [ + 399, + 404 + ] + }, + "design_nulls_after_repair": { + "weight": 0, + "stratum": 0, + "psu": 0 + }, + "weight_min": 0.065, + "weight_max": 4.8641, + "weight_sum": 13677.0071, + "weight_sum_minus_sample_size": 0.007100000000718865, + "output_file": "Dataset/可直接分析数据包_NSDUH_YRBS/YRBS/full_by_year/yrbs_2019_full.parquet", + "status": "passed", + "selected_table_design_nulls_after_repair": { + "weight": 0, + "stratum": 0, + "psu": 0 + } + } + ], + "decision": "2017 and 2019 design fields are no longer missing in the annual full Parquet files; use WEIGHT with STRATUM and PSU in complex-survey procedures." +} diff --git a/research/stage1/yrbs_2017_2019_design_repair_audit.md b/research/stage1/yrbs_2017_2019_design_repair_audit.md new file mode 100644 index 0000000..76aa1a2 --- /dev/null +++ b/research/stage1/yrbs_2017_2019_design_repair_audit.md @@ -0,0 +1,19 @@ +# YRBS 2017/2019 design-field repair audit + +2017 and 2019 design fields are no longer missing in the annual full Parquet files; use WEIGHT with STRATUM and PSU in complex-survey procedures. + +## 2017 + +- rows: 14,765 +- official locations: `{'weight': (387, 396), 'stratum': (397, 399), 'psu': (400, 405)}` (PDF page 102 / printed page 96) +- design nulls after repair: `{'weight': 0, 'stratum': 0, 'psu': 0}` +- weight range/sum: 0.0442–4.6923; sum 14765.0049 + +## 2019 + +- rows: 13,677 +- official locations: `{'weight': (386, 395), 'stratum': (396, 398), 'psu': (399, 404)}` (PDF page 89 / printed page 86) +- design nulls after repair: `{'weight': 0, 'stratum': 0, 'psu': 0}` +- weight range/sum: 0.0650–4.8641; sum 13677.0071 + +The repair changes only derived Parquet files, which remain reproducible and excluded from Git. Official raw files are never modified.