diff --git a/.learnings/ERRORS.md b/.learnings/ERRORS.md index 458aba9..c5b8976 100644 --- a/.learnings/ERRORS.md +++ b/.learnings/ERRORS.md @@ -2,6 +2,64 @@ Command failures and integration errors. +--- +## [ERR-20260920-028] python-assignment-expression-comprehension + +**Logged**: 2026-09-20 +**Priority**: low +**Status**: resolved +**Area**: tests + +### Summary +An inline audit command used an assignment expression in a comprehension iterable, which Python rejects. + +### Error +`SyntaxError: assignment expression cannot be used in a comprehension iterable expression`. + +### Context +The command was a read-only attempt to aggregate GSHS design-field coverage; no data or project file changed. + +### Suggested Fix +Use an ordinary loop in a maintained audit script instead of compressing stateful aggregation into a one-liner. + +### Metadata +- Reproducible: yes +- Related Files: `research/stage1/audit_input_sources.py` +- Pattern-Key: runtime.python-syntax + +### Resolution +- **Resolved**: 2026-09-20 +- **Notes**: Replaced the inline expression with a reusable streaming audit script. + +--- +## [ERR-20260920-029] pandas-csv-memory-exhaustion + +**Logged**: 2026-09-20 +**Priority**: medium +**Status**: resolved +**Area**: tests + +### Summary +Pandas exhausted memory while parsing the 0.77 GB, 637-column GSHS CSV even though only 12 columns were requested. + +### Error +`pandas.errors.ParserError: Error tokenizing data. C error: out of memory`. + +### Context +The failed read was non-mutating. The GSHS file is wide enough that the C parser's tokenization buffers remain material despite `usecols`. + +### Suggested Fix +Stream rows with the standard CSV reader, map only required column indexes, and retain aggregate counters rather than a DataFrame. + +### Metadata +- Reproducible: yes +- Related Files: `research/stage1/audit_input_sources.py` +- Pattern-Key: runtime.memory-exhaustion + +### Resolution +- **Resolved**: 2026-09-20 +- **Notes**: The streaming implementation completed the full 673,499-row audit without loading the dataset into memory. + --- ## [ERR-20260920-024] git-push-repository-not-found diff --git a/Progression.md b/Progression.md index c0148df..b821234 100644 --- a/Progression.md +++ b/Progression.md @@ -1,9 +1,9 @@ # 项目进度档案 -版本：1.6 +版本：1.7 建立日期：2026-09-20 当前总状态：`in_progress` -当前阶段：阶段 0 已完成并完成环境补强；下一行动进入阶段 1 +当前阶段：阶段 1 `in_progress`；输入结构与数据契约首轮审计完成 ## 1. 管理规则 @@ -28,7 +28,7 @@ | 阶段 | 名称 | 状态 | 当前判定 | |---|---|---|---| | 0 | 冻结研究问题、范围与版本 | `passed` | 四项交付物齐备，Gate 0 已通过；研究工具链补强与锁定完成 | -| 1 | 有来源证据的题库和回答层 | `not_started` | 环境阻断已解除；下一行动启动阶段 1 | +| 1 | 有来源证据的题库和回答层 | `in_progress` | 数据契约与三套候选入口结构审计完成；题目来源登记和人工复核尚未完成 | | 2 | 可行性、可识别性和连接审计 | `not_started` | 等待 Gate 1 | | 3 | 冻结验证协议与已知真值模拟 | `not_started` | 等待 Gate 2 | | 4 | 直接协调和传统基线 | `not_started` | 等待 Gate 3 | @@ -662,6 +662,20 @@ Gate 0 当前判定：`passed`。四项交付物齐备；目标总体政策、 - 错误状态：`ERR-20260920-024`、`ERR-20260920-025`、`ERR-20260920-026` 均已解决；一次无副作用的补丁上下文失败记为已解决的 `ERR-20260920-027`。 - 下一步：提交并推送本行动记录；随后按阶段 1 计划建立数据实体 schema，并只读核验 GSHS、YRBS、NSDUH 候选入口。 +### A-20260920-053：冻结阶段 1 数据契约并完成候选入口结构审计 + +- 时间：2026-09-20 +- 阶段：1（有来源证据的题库和回答层） +- 动作：完整读取工作流与进度档案；按来源验证边界建立阶段 1 机器可读数据契约；流式核验 GSHS 主表，并只读核验 YRBS/NSDUH Parquet 的行列、年度和设计字段覆盖。 +- 输入：GSHS `GSHS.csv`；YRBS `yrbs_1991_2019_selected.parquet`；NSDUH 2021–2024 full/selected Parquet；现有 README、变量注意事项与质量检查。 +- 输出：`research/stage1/stage1_data_contract.json`、`audit_input_sources.py`、`input_structure_audit.json`、`input_structure_audit.md`。 +- 检查：两份 JSON 均通过解析；GSHS 实测 673,499×637、191 个组件键；YRBS 实测 217,341×79；NSDUH 实测 232,441 行、full 2,638 列、selected 151 列；所有预期入口字段存在。 +- 新证据：GSHS 三个设计候选字段各缺 20 行；YRBS 1997 有 1 行设计字段缺失，2017/2019 的 PSU/stratum 全缺且权重仅部分存在；NSDUH 四个权重列在全部年度均非空且为正，需要官方指南解释适用语义。 +- 质量边界：所有审核状态默认 `provisional`；本次未批准题目等价、构念、缺失重编码、锚题或复杂抽样方差方法；Gate 1 未通过。 +- 工程修复：内联聚合的 Python 语法失败和 pandas 内存耗尽分别记录为已解决的 `ERR-20260920-028`、`ERR-20260920-029`；最终脚本使用标准 CSV 流式扫描。 +- 推送状态：提交后立即推送。 +- 下一步：建立主要结局来源登记骨架；优先核验 GSHS/NSDUH 的意念、计划、尝试题与适用总体，并追查 YRBS 2017/2019 设计字段缺口。 + ## 9. 剩余工作量估算（2026-09-20 基线） | 阶段 | 主要工作 | 预计人周 | 主要波动来源 | diff --git a/research/stage1/audit_input_sources.py b/research/stage1/audit_input_sources.py new file mode 100644 index 0000000..60d868c --- /dev/null +++ b/research/stage1/audit_input_sources.py @@ -0,0 +1,192 @@ +from __future__ import annotations + +import argparse +import csv +import json +from collections import Counter +from pathlib import Path +from typing import Any + +import pyarrow.parquet as pq + + +ROOT = Path(__file__).resolve().parents[2] + +GSHS_PATH = ROOT / "Dataset" / "GSHS-全球学生健康调查数据" / "GSHS" / "01_data" / "GSHS.csv" +YRBS_PATH = ROOT / "Dataset" / "可直接分析数据包_NSDUH_YRBS" / "YRBS" / "yrbs_1991_2019_selected.parquet" +NSDUH_FULL_PATH = ROOT / "Dataset" / "可直接分析数据包_NSDUH_YRBS" / "NSDUH" / "nsduh_2021_2024_full.parquet" +NSDUH_SELECTED_PATH = ROOT / "Dataset" / "可直接分析数据包_NSDUH_YRBS" / "NSDUH" / "nsduh_2021_2024_selected.parquet" + + +def nonempty(value: str) -> bool: + return value.strip() != "" + + +def audit_gshs(path: Path) -> dict[str, Any]: + required = [ + "country", + "iso3", + "survey_year", + "survey_scope", + "survey_component", + "school_level", + "sample_weight", + "sampling_psu", + "sampling_stratum", + "scope_review_flag", + "study_id", + "dataset_id", + ] + row_count = 0 + survey_keys: set[tuple[str, str, str]] = set() + years: set[int] = set() + scopes: Counter[str] = Counter() + design_non_null: Counter[str] = Counter() + flagged_keys: set[tuple[str, str, str]] = set() + + with path.open("r", encoding="utf-8-sig", newline="") as handle: + reader = csv.reader(handle) + header = next(reader) + missing = sorted(set(required) - set(header)) + if missing: + raise ValueError(f"GSHS is missing required audit columns: {missing}") + index = {name: header.index(name) for name in required} + + for row in reader: + row_count += 1 + country = row[index["country"]] + year_text = row[index["survey_year"]] + component = row[index["survey_component"]] + key = (country, year_text, component) + survey_keys.add(key) + if year_text: + years.add(int(float(year_text))) + scopes[row[index["survey_scope"]] or "missing"] += 1 + for field in ("sample_weight", "sampling_psu", "sampling_stratum"): + if nonempty(row[index[field]]): + design_non_null[field] += 1 + if row[index["scope_review_flag"]].strip() == "1": + flagged_keys.add(key) + + return { + "path": path.relative_to(ROOT).as_posix(), + "bytes": path.stat().st_size, + "rows": row_count, + "columns": len(header), + "survey_component_keys": len(survey_keys), + "year_min": min(years), + "year_max": max(years), + "scope_row_counts": dict(sorted(scopes.items())), + "design_non_null": dict(design_non_null), + "scope_review_flagged_keys": len(flagged_keys), + "required_columns_present": True, + } + + +def audit_parquet( + path: Path, + year_field: str, + design_fields: list[str], + expected_columns: int, +) -> dict[str, Any]: + parquet_file = pq.ParquetFile(path) + names = parquet_file.schema_arrow.names + required = [year_field, *design_fields] + missing = sorted(set(required) - set(names)) + if missing: + raise ValueError(f"{path.name} is missing required audit columns: {missing}") + + table = parquet_file.read(columns=required) + year_values = table[year_field].to_pylist() + design_values = {field: table[field].to_pylist() for field in design_fields} + year_counts = Counter(str(value) for value in year_values if value is not None) + design_non_null = { + field: table.num_rows - table[field].null_count + for field in design_fields + } + design_non_null_by_year: dict[str, Counter[str]] = {} + design_positive_by_year: dict[str, Counter[str]] = {} + for row_index, year_value in enumerate(year_values): + if year_value is None: + continue + year = str(year_value) + design_non_null_by_year.setdefault(year, Counter()) + design_positive_by_year.setdefault(year, Counter()) + for field in design_fields: + value = design_values[field][row_index] + if value is not None: + design_non_null_by_year[year][field] += 1 + if isinstance(value, (int, float)) and value > 0: + design_positive_by_year[year][field] += 1 + return { + "path": path.relative_to(ROOT).as_posix(), + "bytes": path.stat().st_size, + "rows": parquet_file.metadata.num_rows, + "columns": len(names), + "row_groups": parquet_file.metadata.num_row_groups, + "year_counts": dict(sorted(year_counts.items(), key=lambda pair: int(float(pair[0])))), + "design_non_null": design_non_null, + "design_non_null_by_year": { + year: dict(counts) + for year, counts in sorted(design_non_null_by_year.items(), key=lambda pair: int(float(pair[0]))) + }, + "design_positive_by_year": { + year: dict(counts) + for year, counts in sorted(design_positive_by_year.items(), key=lambda pair: int(float(pair[0]))) + }, + "required_columns_present": True, + "expected_column_count_matches": len(names) == expected_columns, + } + + +def build_audit() -> dict[str, Any]: + return { + "audit_version": "0.1.0", + "status": "provisional", + "scope": "Read-only structure and design-field audit; no item equivalence or coding approval.", + "sources": { + "GSHS": audit_gshs(GSHS_PATH), + "YRBS_selected_1991_2019": audit_parquet( + YRBS_PATH, + "survey_year", + ["weight", "stratum", "psu"], + 79, + ), + "NSDUH_full_2021_2024": audit_parquet( + NSDUH_FULL_PATH, + "YEAR", + ["VESTR_C", "VEREP", "ANALWT2_C1", "ANALWT2_C2", "ANALWT2_C3", "ANALWT2_C4"], + 2638, + ), + "NSDUH_selected_2021_2024": audit_parquet( + NSDUH_SELECTED_PATH, + "YEAR", + ["VESTR_C", "VEREP", "ANALWT2_C1", "ANALWT2_C2", "ANALWT2_C3", "ANALWT2_C4"], + 151, + ), + }, + "limitations": [ + "YRBS 2021 and 2023 are not included in the selected analytic Parquet and remain quarantined pending layout verification.", + "GSHS design fields are candidates and require survey-specific source verification.", + "NSDUH year-specific weight selection and variance usage require confirmation against the combined-file guide.", + "This audit does not approve item wording, response recodes, missingness rules, constructs, or cross-survey links." + ] + } + + +def main() -> None: + parser = argparse.ArgumentParser() + parser.add_argument( + "--output", + type=Path, + default=Path(__file__).with_name("input_structure_audit.json"), + ) + args = parser.parse_args() + result = build_audit() + args.output.parent.mkdir(parents=True, exist_ok=True) + args.output.write_text(json.dumps(result, ensure_ascii=False, indent=2) + "\n", encoding="utf-8") + print(json.dumps(result, ensure_ascii=False, indent=2)) + + +if __name__ == "__main__": + main() diff --git a/research/stage1/input_structure_audit.json b/research/stage1/input_structure_audit.json new file mode 100644 index 0000000..1329932 --- /dev/null +++ b/research/stage1/input_structure_audit.json @@ -0,0 +1,394 @@ +{ + "audit_version": "0.1.0", + "status": "provisional", + "scope": "Read-only structure and design-field audit; no item equivalence or coding approval.", + "sources": { + "GSHS": { + "path": "Dataset/GSHS-全球学生健康调查数据/GSHS/01_data/GSHS.csv", + "bytes": 766920275, + "rows": 673499, + "columns": 637, + "survey_component_keys": 191, + "year_min": 2003, + "year_max": 2023, + "scope_row_counts": { + "component": 167067, + "national": 471274, + "special_school": 6501, + "subnational": 28657 + }, + "design_non_null": { + "sample_weight": 673479, + "sampling_psu": 673479, + "sampling_stratum": 673479 + }, + "scope_review_flagged_keys": 36, + "required_columns_present": true + }, + "YRBS_selected_1991_2019": { + "path": "Dataset/可直接分析数据包_NSDUH_YRBS/YRBS/yrbs_1991_2019_selected.parquet", + "bytes": 2576853, + "rows": 217341, + "columns": 79, + "row_groups": 1, + "year_counts": { + "1991": 12272, + "1993": 16296, + "1995": 10904, + "1997": 16263, + "1999": 15349, + "2001": 13601, + "2003": 15214, + "2005": 13917, + "2007": 14041, + "2009": 16410, + "2011": 15425, + "2013": 13583, + "2015": 15624, + "2017": 14765, + "2019": 13677 + }, + "design_non_null": { + "weight": 210769, + "stratum": 188898, + "psu": 188898 + }, + "design_non_null_by_year": { + "1991": { + "weight": 12272, + "stratum": 12272, + "psu": 12272 + }, + "1993": { + "weight": 16296, + "stratum": 16296, + "psu": 16296 + }, + "1995": { + "weight": 10904, + "stratum": 10904, + "psu": 10904 + }, + "1997": { + "weight": 16262, + "stratum": 16262, + "psu": 16262 + }, + "1999": { + "weight": 15349, + "stratum": 15349, + "psu": 15349 + }, + "2001": { + "weight": 13601, + "stratum": 13601, + "psu": 13601 + }, + "2003": { + "weight": 15214, + "stratum": 15214, + "psu": 15214 + }, + "2005": { + "weight": 13917, + "stratum": 13917, + "psu": 13917 + }, + "2007": { + "weight": 14041, + "stratum": 14041, + "psu": 14041 + }, + "2009": { + "weight": 16410, + "stratum": 16410, + "psu": 16410 + }, + "2011": { + "weight": 15425, + "stratum": 15425, + "psu": 15425 + }, + "2013": { + "weight": 13583, + "stratum": 13583, + "psu": 13583 + }, + "2015": { + "weight": 15624, + "stratum": 15624, + "psu": 15624 + }, + "2017": { + "weight": 13026 + }, + "2019": { + "weight": 8845 + } + }, + "design_positive_by_year": { + "1991": { + "weight": 12272, + "stratum": 12272, + "psu": 12272 + }, + "1993": { + "weight": 16296, + "stratum": 16296, + "psu": 16296 + }, + "1995": { + "weight": 10904, + "stratum": 10904, + "psu": 10904 + }, + "1997": { + "weight": 16262, + "stratum": 16262, + "psu": 16262 + }, + "1999": { + "weight": 15349, + "stratum": 15349, + "psu": 15349 + }, + "2001": { + "weight": 13601, + "stratum": 13601, + "psu": 13601 + }, + "2003": { + "weight": 15214, + "stratum": 15214, + "psu": 15214 + }, + "2005": { + "weight": 13917, + "stratum": 13917, + "psu": 13917 + }, + "2007": { + "weight": 14041, + "stratum": 14041, + "psu": 14041 + }, + "2009": { + "weight": 16410, + "stratum": 16410, + "psu": 16410 + }, + "2011": { + "weight": 15425, + "stratum": 15425, + "psu": 15425 + }, + "2013": { + "weight": 13583, + "stratum": 13583, + "psu": 13583 + }, + "2015": { + "weight": 15624, + "stratum": 15624, + "psu": 15624 + }, + "2017": { + "weight": 13026 + }, + "2019": { + "weight": 8845 + } + }, + "required_columns_present": true, + "expected_column_count_matches": true + }, + "NSDUH_full_2021_2024": { + "path": "Dataset/可直接分析数据包_NSDUH_YRBS/NSDUH/nsduh_2021_2024_full.parquet", + "bytes": 90527632, + "rows": 232441, + "columns": 2638, + "row_groups": 1, + "year_counts": { + "2021": 58034, + "2022": 59069, + "2023": 56705, + "2024": 58633 + }, + "design_non_null": { + "VESTR_C": 232441, + "VEREP": 232441, + "ANALWT2_C1": 232441, + "ANALWT2_C2": 232441, + "ANALWT2_C3": 232441, + "ANALWT2_C4": 232441 + }, + "design_non_null_by_year": { + "2021": { + "VESTR_C": 58034, + "VEREP": 58034, + "ANALWT2_C1": 58034, + "ANALWT2_C2": 58034, + "ANALWT2_C3": 58034, + "ANALWT2_C4": 58034 + }, + "2022": { + "VESTR_C": 59069, + "VEREP": 59069, + "ANALWT2_C1": 59069, + "ANALWT2_C2": 59069, + "ANALWT2_C3": 59069, + "ANALWT2_C4": 59069 + }, + "2023": { + "VESTR_C": 56705, + "VEREP": 56705, + "ANALWT2_C1": 56705, + "ANALWT2_C2": 56705, + "ANALWT2_C3": 56705, + "ANALWT2_C4": 56705 + }, + "2024": { + "VESTR_C": 58633, + "VEREP": 58633, + "ANALWT2_C1": 58633, + "ANALWT2_C2": 58633, + "ANALWT2_C3": 58633, + "ANALWT2_C4": 58633 + } + }, + "design_positive_by_year": { + "2021": { + "VESTR_C": 58034, + "VEREP": 58034, + "ANALWT2_C1": 58034, + "ANALWT2_C2": 58034, + "ANALWT2_C3": 58034, + "ANALWT2_C4": 58034 + }, + "2022": { + "VESTR_C": 59069, + "VEREP": 59069, + "ANALWT2_C1": 59069, + "ANALWT2_C2": 59069, + "ANALWT2_C3": 59069, + "ANALWT2_C4": 59069 + }, + "2023": { + "VESTR_C": 56705, + "VEREP": 56705, + "ANALWT2_C1": 56705, + "ANALWT2_C2": 56705, + "ANALWT2_C3": 56705, + "ANALWT2_C4": 56705 + }, + "2024": { + "VESTR_C": 58633, + "VEREP": 58633, + "ANALWT2_C1": 58633, + "ANALWT2_C2": 58633, + "ANALWT2_C3": 58633, + "ANALWT2_C4": 58633 + } + }, + "required_columns_present": true, + "expected_column_count_matches": true + }, + "NSDUH_selected_2021_2024": { + "path": "Dataset/可直接分析数据包_NSDUH_YRBS/NSDUH/nsduh_2021_2024_selected.parquet", + "bytes": 14550549, + "rows": 232441, + "columns": 151, + "row_groups": 1, + "year_counts": { + "2021": 58034, + "2022": 59069, + "2023": 56705, + "2024": 58633 + }, + "design_non_null": { + "VESTR_C": 232441, + "VEREP": 232441, + "ANALWT2_C1": 232441, + "ANALWT2_C2": 232441, + "ANALWT2_C3": 232441, + "ANALWT2_C4": 232441 + }, + "design_non_null_by_year": { + "2021": { + "VESTR_C": 58034, + "VEREP": 58034, + "ANALWT2_C1": 58034, + "ANALWT2_C2": 58034, + "ANALWT2_C3": 58034, + "ANALWT2_C4": 58034 + }, + "2022": { + "VESTR_C": 59069, + "VEREP": 59069, + "ANALWT2_C1": 59069, + "ANALWT2_C2": 59069, + "ANALWT2_C3": 59069, + "ANALWT2_C4": 59069 + }, + "2023": { + "VESTR_C": 56705, + "VEREP": 56705, + "ANALWT2_C1": 56705, + "ANALWT2_C2": 56705, + "ANALWT2_C3": 56705, + "ANALWT2_C4": 56705 + }, + "2024": { + "VESTR_C": 58633, + "VEREP": 58633, + "ANALWT2_C1": 58633, + "ANALWT2_C2": 58633, + "ANALWT2_C3": 58633, + "ANALWT2_C4": 58633 + } + }, + "design_positive_by_year": { + "2021": { + "VESTR_C": 58034, + "VEREP": 58034, + "ANALWT2_C1": 58034, + "ANALWT2_C2": 58034, + "ANALWT2_C3": 58034, + "ANALWT2_C4": 58034 + }, + "2022": { + "VESTR_C": 59069, + "VEREP": 59069, + "ANALWT2_C1": 59069, + "ANALWT2_C2": 59069, + "ANALWT2_C3": 59069, + "ANALWT2_C4": 59069 + }, + "2023": { + "VESTR_C": 56705, + "VEREP": 56705, + "ANALWT2_C1": 56705, + "ANALWT2_C2": 56705, + "ANALWT2_C3": 56705, + "ANALWT2_C4": 56705 + }, + "2024": { + "VESTR_C": 58633, + "VEREP": 58633, + "ANALWT2_C1": 58633, + "ANALWT2_C2": 58633, + "ANALWT2_C3": 58633, + "ANALWT2_C4": 58633 + } + }, + "required_columns_present": true, + "expected_column_count_matches": true + } + }, + "limitations": [ + "YRBS 2021 and 2023 are not included in the selected analytic Parquet and remain quarantined pending layout verification.", + "GSHS design fields are candidates and require survey-specific source verification.", + "NSDUH year-specific weight selection and variance usage require confirmation against the combined-file guide.", + "This audit does not approve item wording, response recodes, missingness rules, constructs, or cross-survey links." + ] +} diff --git a/research/stage1/input_structure_audit.md b/research/stage1/input_structure_audit.md new file mode 100644 index 0000000..3cdde4d --- /dev/null +++ b/research/stage1/input_structure_audit.md @@ -0,0 +1,54 @@ +# 阶段 1 输入结构审计 + +版本：0.1.0 +日期：2026-09-20 +状态：`provisional` + +## 审计范围 + +本次只读核验三个候选数据入口的实际行列数、年度覆盖和复杂抽样候选字段。结果不构成题目等价、构念归类、缺失值重编码、锚题或 Gate 1 的批准。 + +机器可读结果见 `input_structure_audit.json`；复现脚本为 `audit_input_sources.py`；实体和审核状态契约见 `stage1_data_contract.json`。 + +## GSHS + +- 入口：`Dataset/GSHS-全球学生健康调查数据/GSHS/01_data/GSHS.csv` +- 实测规模：673,499 行、637 列；2003–2023；191 个国家×年份×组件键。 +- 范围行数：national 471,274；component 167,067；subnational 28,657；special_school 6,501。 +- `sample_weight`、`sampling_psu`、`sampling_stratum` 各有 673,479 个非空值，即各缺 20 行。 +- `scope_review_flag=1` 覆盖 36 个组件键。既有质量报告写的是 10 项调查；两者统计单位不同，进入分析前需建立调查—组件映射并解释差异。 + +判定：GSHS 继续作为主开发候选，但设计字段只能视为候选；必须按组件回到问卷、codebook 和设计来源核验。 + +## YRBS + +- 入口：`Dataset/可直接分析数据包_NSDUH_YRBS/YRBS/yrbs_1991_2019_selected.parquet` +- 实测规模：217,341 行、79 列；覆盖 1991–2019 的 15 个隔年调查。 +- 1991–2015 的设计字段基本完整；1997 年 16,263 行中有 1 行同时缺 weight、stratum 和 psu。 +- 2017 年 14,765 行中仅 13,026 行有 weight，stratum/psu 全缺。 +- 2019 年 13,677 行中仅 8,845 行有 weight，stratum/psu 全缺。 +- 2021、2023 不在该可分析 Parquet 中，继续保持 layout 验证隔离状态。 + +判定：当前选表不能直接支持 2017/2019 的完整复杂抽样分析；不得把 1991–2019 整体标记为设计就绪。后续应对照官方 SAS 输入/格式程序修复或另建受限分析分支。 + +## NSDUH + +- 全表入口：`Dataset/可直接分析数据包_NSDUH_YRBS/NSDUH/nsduh_2021_2024_full.parquet`，232,441 行、2,638 列。 +- 选表入口：同目录 `nsduh_2021_2024_selected.parquet`，232,441 行、151 列。 +- 年度行数：2021 58,034；2022 59,069；2023 56,705；2024 58,633。 +- `VESTR_C`、`VEREP`、`ANALWT2_C1`–`ANALWT2_C4` 在全表和选表中均无空值且均为正值。 + +判定：结构完整不等于权重适用规则已确认。四个权重列在四个年度均为正，需对照官方 combined-file guide 核实每列的估计目标、年度组合和方差方法；青少年总体、题目适用条件及成人/青少年题分离仍未完成。 + +## 当前结论 + +1. 三个候选入口均可读，实测规模与现有交付说明一致。 +2. 阶段 1 已有机器可读实体契约，所有审核状态默认 `provisional`，AI 检查不能冒充双人审核。 +3. YRBS 设计变量缺口是当前最明确的数据阻断；GSHS 组件范围和设计来源、NSDUH 权重语义仍需官方证据。 +4. Gate 1 保持 `not passed`。下一增量应建立主要结局的来源登记骨架，并优先核验 GSHS/NSDUH 的自杀意念、计划、尝试题及 YRBS 设计字段缺口。 + +## 限制 + +- 本次未读取或解释个体回答值，也未计算任何患病率或模型结果。 +- 本次未逐页复核 PDF/DOCX 问卷和 codebook。 +- 行数、字段存在和非空性不能证明编码正确、测量等价或复杂抽样方差可用。 diff --git a/research/stage1/stage1_data_contract.json b/research/stage1/stage1_data_contract.json new file mode 100644 index 0000000..413a5aa --- /dev/null +++ b/research/stage1/stage1_data_contract.json @@ -0,0 +1,146 @@ +{ + "schema_version": "0.1.0", + "status": "provisional", + "purpose": "Stage 1 contract for source-traceable survey items, responses, design variables, and review state.", + "review_status": [ + "provisional", + "under_review", + "human_reviewed_single", + "human_reviewed_double", + "approved", + "conflicted", + "excluded" + ], + "missing_type": [ + "observed", + "not_administered", + "inapplicable", + "legitimate_skip", + "refused", + "dont_know", + "ordinary_missing", + "data_error", + "unknown" + ], + "entities": { + "survey_component": { + "primary_key": ["survey_id", "component_id"], + "required_fields": [ + "survey_id", + "component_id", + "country", + "year", + "scope", + "population", + "language", + "questionnaire_version", + "source_file", + "source_url", + "review_status" + ] + }, + "item_instance": { + "primary_key": ["survey_id", "component_id", "item_version_id"], + "required_fields": [ + "item_version_id", + "item_family_id", + "source_variable", + "question_text", + "construct", + "time_window", + "response_type", + "language", + "survey_id", + "component_id", + "source_file", + "source_url", + "page_or_section", + "text_hash", + "review_status" + ] + }, + "response_code": { + "primary_key": ["item_version_id", "raw_code"], + "required_fields": [ + "item_version_id", + "raw_code", + "raw_label", + "canonical_code", + "missing_type", + "derivation_rule", + "source_file", + "page_or_section", + "review_status" + ] + }, + "response_long": { + "primary_key": ["respondent_id", "survey_id", "component_id", "item_version_id"], + "required_fields": [ + "respondent_id", + "survey_id", + "component_id", + "item_version_id", + "raw_response", + "canonical_response", + "eligibility", + "observed_status" + ] + }, + "survey_design": { + "primary_key": ["survey_id", "component_id"], + "required_fields": [ + "survey_id", + "component_id", + "weight_field", + "psu_field", + "stratum_field", + "variance_method", + "design_source", + "review_status" + ] + }, + "source_review": { + "primary_key": ["survey_id", "component_id", "item_version_id", "source_file"], + "required_fields": [ + "source_file", + "source_url", + "page_or_section", + "text_hash", + "reviewer_1", + "reviewer_2", + "review_status" + ], + "constraints": [ + "reviewer_1 and reviewer_2 must identify actual human reviewers before review_status can be human_reviewed_double or approved", + "AI checks may be recorded separately but must not populate reviewer_1 or reviewer_2" + ] + }, + "candidate_link": { + "primary_key": ["left_item_version_id", "right_item_version_id"], + "required_fields": [ + "left_item_version_id", + "right_item_version_id", + "relation_type", + "semantic_evidence", + "population_overlap", + "anchor_status", + "exclusion_reason", + "review_status" + ], + "allowed_anchor_status": [ + "candidate", + "training_only", + "approved_partial_invariance", + "rejected" + ] + } + }, + "global_constraints": [ + "All records default to provisional until source evidence is checked.", + "A shared source variable name does not establish item equivalence.", + "Derived indicators and raw responses must remain distinguishable.", + "Legitimate skips may be recoded as negative only when an explicit official rule supports the derivation.", + "PSU and stratum identifiers require survey or component namespaces before pooling.", + "Semantic similarity may nominate a candidate link but cannot establish measurement invariance." + ] +}