Jinovo/language-conditioned-psychometric-harmonization-model
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat(stage1): audit survey input structures

Browse Source
This commit is contained in:
Jinotech
2026-09-20 07:18:51 +12:00
parent f34add281f
commit 5c31417935
6 changed files with 861 additions and 3 deletions
Download Patch File Download Diff File Expand all files Collapse all files
research/stage1/audit_input_sources.py
+192
View File
@@ -0,0 +1,192 @@
from __future__ import annotations
import argparse
import csv
import json
from collections import Counter
from pathlib import Path
from typing import Any
import pyarrow.parquet as pq
ROOT = Path(__file__).resolve().parents[2]
GSHS_PATH = ROOT / "Dataset" / "GSHS-全球学生健康调查数据" / "GSHS" / "01_data" / "GSHS.csv"
YRBS_PATH = ROOT / "Dataset" / "可直接分析数据包_NSDUH_YRBS" / "YRBS" / "yrbs_1991_2019_selected.parquet"
NSDUH_FULL_PATH = ROOT / "Dataset" / "可直接分析数据包_NSDUH_YRBS" / "NSDUH" / "nsduh_2021_2024_full.parquet"
NSDUH_SELECTED_PATH = ROOT / "Dataset" / "可直接分析数据包_NSDUH_YRBS" / "NSDUH" / "nsduh_2021_2024_selected.parquet"
def nonempty(value: str) -> bool:
return value.strip() != ""
def audit_gshs(path: Path) -> dict[str, Any]:
required = [
"country",
"iso3",
"survey_year",
"survey_scope",
"survey_component",
"school_level",
"sample_weight",
"sampling_psu",
"sampling_stratum",
"scope_review_flag",
"study_id",
"dataset_id",
]
row_count = 0
survey_keys: set[tuple[str, str, str]] = set()
years: set[int] = set()
scopes: Counter[str] = Counter()
design_non_null: Counter[str] = Counter()
flagged_keys: set[tuple[str, str, str]] = set()
with path.open("r", encoding="utf-8-sig", newline="") as handle:
reader = csv.reader(handle)
header = next(reader)
missing = sorted(set(required) - set(header))
if missing:
raise ValueError(f"GSHS is missing required audit columns: {missing}")
index = {name: header.index(name) for name in required}
for row in reader:
row_count += 1
country = row[index["country"]]
year_text = row[index["survey_year"]]
component = row[index["survey_component"]]
key = (country, year_text, component)
survey_keys.add(key)
if year_text:
years.add(int(float(year_text)))
scopes[row[index["survey_scope"]] or "missing"] += 1
for field in ("sample_weight", "sampling_psu", "sampling_stratum"):
if nonempty(row[index[field]]):
design_non_null[field] += 1
if row[index["scope_review_flag"]].strip() == "1":
flagged_keys.add(key)
return {
"path": path.relative_to(ROOT).as_posix(),
"bytes": path.stat().st_size,
"rows": row_count,
"columns": len(header),
"survey_component_keys": len(survey_keys),
"year_min": min(years),
"year_max": max(years),
"scope_row_counts": dict(sorted(scopes.items())),
"design_non_null": dict(design_non_null),
"scope_review_flagged_keys": len(flagged_keys),
"required_columns_present": True,
}
def audit_parquet(
path: Path,
year_field: str,
design_fields: list[str],
expected_columns: int,
) -> dict[str, Any]:
parquet_file = pq.ParquetFile(path)
names = parquet_file.schema_arrow.names
required = [year_field, *design_fields]
missing = sorted(set(required) - set(names))
if missing:
raise ValueError(f"{path.name} is missing required audit columns: {missing}")
table = parquet_file.read(columns=required)
year_values = table[year_field].to_pylist()
design_values = {field: table[field].to_pylist() for field in design_fields}
year_counts = Counter(str(value) for value in year_values if value is not None)
design_non_null = {
field: table.num_rows - table[field].null_count
for field in design_fields
}
design_non_null_by_year: dict[str, Counter[str]] = {}
design_positive_by_year: dict[str, Counter[str]] = {}
for row_index, year_value in enumerate(year_values):
if year_value is None:
continue
year = str(year_value)
design_non_null_by_year.setdefault(year, Counter())
design_positive_by_year.setdefault(year, Counter())
for field in design_fields:
value = design_values[field][row_index]
if value is not None:
design_non_null_by_year[year][field] += 1
if isinstance(value, (int, float)) and value > 0:
design_positive_by_year[year][field] += 1
return {
"path": path.relative_to(ROOT).as_posix(),
"bytes": path.stat().st_size,
"rows": parquet_file.metadata.num_rows,
"columns": len(names),
"row_groups": parquet_file.metadata.num_row_groups,
"year_counts": dict(sorted(year_counts.items(), key=lambda pair: int(float(pair[0])))),
"design_non_null": design_non_null,
"design_non_null_by_year": {
year: dict(counts)
for year, counts in sorted(design_non_null_by_year.items(), key=lambda pair: int(float(pair[0])))
},
"design_positive_by_year": {
year: dict(counts)
for year, counts in sorted(design_positive_by_year.items(), key=lambda pair: int(float(pair[0])))
},
"required_columns_present": True,
"expected_column_count_matches": len(names) == expected_columns,
}
def build_audit() -> dict[str, Any]:
return {
"audit_version": "0.1.0",
"status": "provisional",
"scope": "Read-only structure and design-field audit; no item equivalence or coding approval.",
"sources": {
"GSHS": audit_gshs(GSHS_PATH),
"YRBS_selected_1991_2019": audit_parquet(
YRBS_PATH,
"survey_year",
["weight", "stratum", "psu"],
79,
),
"NSDUH_full_2021_2024": audit_parquet(
NSDUH_FULL_PATH,
"YEAR",
["VESTR_C", "VEREP", "ANALWT2_C1", "ANALWT2_C2", "ANALWT2_C3", "ANALWT2_C4"],
2638,
),
"NSDUH_selected_2021_2024": audit_parquet(
NSDUH_SELECTED_PATH,
"YEAR",
["VESTR_C", "VEREP", "ANALWT2_C1", "ANALWT2_C2", "ANALWT2_C3", "ANALWT2_C4"],
151,
),
},
"limitations": [
"YRBS 2021 and 2023 are not included in the selected analytic Parquet and remain quarantined pending layout verification.",
"GSHS design fields are candidates and require survey-specific source verification.",
"NSDUH year-specific weight selection and variance usage require confirmation against the combined-file guide.",
"This audit does not approve item wording, response recodes, missingness rules, constructs, or cross-survey links."
]
}
def main() -> None:
parser = argparse.ArgumentParser()
parser.add_argument(
"--output",
type=Path,
default=Path(__file__).with_name("input_structure_audit.json"),
)
args = parser.parse_args()
result = build_audit()
args.output.parent.mkdir(parents=True, exist_ok=True)
args.output.write_text(json.dumps(result, ensure_ascii=False, indent=2) + "\n", encoding="utf-8")
print(json.dumps(result, ensure_ascii=False, indent=2))
if __name__ == "__main__":
main()
research/stage1/input_structure_audit.json
+394
View File
@@ -0,0 +1,394 @@
{
"audit_version": "0.1.0",
"status": "provisional",
"scope": "Read-only structure and design-field audit; no item equivalence or coding approval.",
"sources": {
"GSHS": {
"path": "Dataset/GSHS-全球学生健康调查数据/GSHS/01_data/GSHS.csv",
"bytes": 766920275,
"rows": 673499,
"columns": 637,
"survey_component_keys": 191,
"year_min": 2003,
"year_max": 2023,
"scope_row_counts": {
"component": 167067,
"national": 471274,
"special_school": 6501,
"subnational": 28657
},
"design_non_null": {
"sample_weight": 673479,
"sampling_psu": 673479,
"sampling_stratum": 673479
},
"scope_review_flagged_keys": 36,
"required_columns_present": true
},
"YRBS_selected_1991_2019": {
"path": "Dataset/可直接分析数据包_NSDUH_YRBS/YRBS/yrbs_1991_2019_selected.parquet",
"bytes": 2576853,
"rows": 217341,
"columns": 79,
"row_groups": 1,
"year_counts": {
"1991": 12272,
"1993": 16296,
"1995": 10904,
"1997": 16263,
"1999": 15349,
"2001": 13601,
"2003": 15214,
"2005": 13917,
"2007": 14041,
"2009": 16410,
"2011": 15425,
"2013": 13583,
"2015": 15624,
"2017": 14765,
"2019": 13677
},
"design_non_null": {
"weight": 210769,
"stratum": 188898,
"psu": 188898
},
"design_non_null_by_year": {
"1991": {
"weight": 12272,
"stratum": 12272,
"psu": 12272
},
"1993": {
"weight": 16296,
"stratum": 16296,
"psu": 16296
},
"1995": {
"weight": 10904,
"stratum": 10904,
"psu": 10904
},
"1997": {
"weight": 16262,
"stratum": 16262,
"psu": 16262
},
"1999": {
"weight": 15349,
"stratum": 15349,
"psu": 15349
},
"2001": {
"weight": 13601,
"stratum": 13601,
"psu": 13601
},
"2003": {
"weight": 15214,
"stratum": 15214,
"psu": 15214
},
"2005": {
"weight": 13917,
"stratum": 13917,
"psu": 13917
},
"2007": {
"weight": 14041,
"stratum": 14041,
"psu": 14041
},
"2009": {
"weight": 16410,
"stratum": 16410,
"psu": 16410
},
"2011": {
"weight": 15425,
"stratum": 15425,
"psu": 15425
},
"2013": {
"weight": 13583,
"stratum": 13583,
"psu": 13583
},
"2015": {
"weight": 15624,
"stratum": 15624,
"psu": 15624
},
"2017": {
"weight": 13026
},
"2019": {
"weight": 8845
}
},
"design_positive_by_year": {
"1991": {
"weight": 12272,
"stratum": 12272,
"psu": 12272
},
"1993": {
"weight": 16296,
"stratum": 16296,
"psu": 16296
},
"1995": {
"weight": 10904,
"stratum": 10904,
"psu": 10904
},
"1997": {
"weight": 16262,
"stratum": 16262,
"psu": 16262
},
"1999": {
"weight": 15349,
"stratum": 15349,
"psu": 15349
},
"2001": {
"weight": 13601,
"stratum": 13601,
"psu": 13601
},
"2003": {
"weight": 15214,
"stratum": 15214,
"psu": 15214
},
"2005": {
"weight": 13917,
"stratum": 13917,
"psu": 13917
},
"2007": {
"weight": 14041,
"stratum": 14041,
"psu": 14041
},
"2009": {
"weight": 16410,
"stratum": 16410,
"psu": 16410
},
"2011": {
"weight": 15425,
"stratum": 15425,
"psu": 15425
},
"2013": {
"weight": 13583,
"stratum": 13583,
"psu": 13583
},
"2015": {
"weight": 15624,
"stratum": 15624,
"psu": 15624
},
"2017": {
"weight": 13026
},
"2019": {
"weight": 8845
}
},
"required_columns_present": true,
"expected_column_count_matches": true
},
"NSDUH_full_2021_2024": {
"path": "Dataset/可直接分析数据包_NSDUH_YRBS/NSDUH/nsduh_2021_2024_full.parquet",
"bytes": 90527632,
"rows": 232441,
"columns": 2638,
"row_groups": 1,
"year_counts": {
"2021": 58034,
"2022": 59069,
"2023": 56705,
"2024": 58633
},
"design_non_null": {
"VESTR_C": 232441,
"VEREP": 232441,
"ANALWT2_C1": 232441,
"ANALWT2_C2": 232441,
"ANALWT2_C3": 232441,
"ANALWT2_C4": 232441
},
"design_non_null_by_year": {
"2021": {
"VESTR_C": 58034,
"VEREP": 58034,
"ANALWT2_C1": 58034,
"ANALWT2_C2": 58034,
"ANALWT2_C3": 58034,
"ANALWT2_C4": 58034
},
"2022": {
"VESTR_C": 59069,
"VEREP": 59069,
"ANALWT2_C1": 59069,
"ANALWT2_C2": 59069,
"ANALWT2_C3": 59069,
"ANALWT2_C4": 59069
},
"2023": {
"VESTR_C": 56705,
"VEREP": 56705,
"ANALWT2_C1": 56705,
"ANALWT2_C2": 56705,
"ANALWT2_C3": 56705,
"ANALWT2_C4": 56705
},
"2024": {
"VESTR_C": 58633,
"VEREP": 58633,
"ANALWT2_C1": 58633,
"ANALWT2_C2": 58633,
"ANALWT2_C3": 58633,
"ANALWT2_C4": 58633
}
},
"design_positive_by_year": {
"2021": {
"VESTR_C": 58034,
"VEREP": 58034,
"ANALWT2_C1": 58034,
"ANALWT2_C2": 58034,
"ANALWT2_C3": 58034,
"ANALWT2_C4": 58034
},
"2022": {
"VESTR_C": 59069,
"VEREP": 59069,
"ANALWT2_C1": 59069,
"ANALWT2_C2": 59069,
"ANALWT2_C3": 59069,
"ANALWT2_C4": 59069
},
"2023": {
"VESTR_C": 56705,
"VEREP": 56705,
"ANALWT2_C1": 56705,
"ANALWT2_C2": 56705,
"ANALWT2_C3": 56705,
"ANALWT2_C4": 56705
},
"2024": {
"VESTR_C": 58633,
"VEREP": 58633,
"ANALWT2_C1": 58633,
"ANALWT2_C2": 58633,
"ANALWT2_C3": 58633,
"ANALWT2_C4": 58633
}
},
"required_columns_present": true,
"expected_column_count_matches": true
},
"NSDUH_selected_2021_2024": {
"path": "Dataset/可直接分析数据包_NSDUH_YRBS/NSDUH/nsduh_2021_2024_selected.parquet",
"bytes": 14550549,
"rows": 232441,
"columns": 151,
"row_groups": 1,
"year_counts": {
"2021": 58034,
"2022": 59069,
"2023": 56705,
"2024": 58633
},
"design_non_null": {
"VESTR_C": 232441,
"VEREP": 232441,
"ANALWT2_C1": 232441,
"ANALWT2_C2": 232441,
"ANALWT2_C3": 232441,
"ANALWT2_C4": 232441
},
"design_non_null_by_year": {
"2021": {
"VESTR_C": 58034,
"VEREP": 58034,
"ANALWT2_C1": 58034,
"ANALWT2_C2": 58034,
"ANALWT2_C3": 58034,
"ANALWT2_C4": 58034
},
"2022": {
"VESTR_C": 59069,
"VEREP": 59069,
"ANALWT2_C1": 59069,
"ANALWT2_C2": 59069,
"ANALWT2_C3": 59069,
"ANALWT2_C4": 59069
},
"2023": {
"VESTR_C": 56705,
"VEREP": 56705,
"ANALWT2_C1": 56705,
"ANALWT2_C2": 56705,
"ANALWT2_C3": 56705,
"ANALWT2_C4": 56705
},
"2024": {
"VESTR_C": 58633,
"VEREP": 58633,
"ANALWT2_C1": 58633,
"ANALWT2_C2": 58633,
"ANALWT2_C3": 58633,
"ANALWT2_C4": 58633
}
},
"design_positive_by_year": {
"2021": {
"VESTR_C": 58034,
"VEREP": 58034,
"ANALWT2_C1": 58034,
"ANALWT2_C2": 58034,
"ANALWT2_C3": 58034,
"ANALWT2_C4": 58034
},
"2022": {
"VESTR_C": 59069,
"VEREP": 59069,
"ANALWT2_C1": 59069,
"ANALWT2_C2": 59069,
"ANALWT2_C3": 59069,
"ANALWT2_C4": 59069
},
"2023": {
"VESTR_C": 56705,
"VEREP": 56705,
"ANALWT2_C1": 56705,
"ANALWT2_C2": 56705,
"ANALWT2_C3": 56705,
"ANALWT2_C4": 56705
},
"2024": {
"VESTR_C": 58633,
"VEREP": 58633,
"ANALWT2_C1": 58633,
"ANALWT2_C2": 58633,
"ANALWT2_C3": 58633,
"ANALWT2_C4": 58633
}
},
"required_columns_present": true,
"expected_column_count_matches": true
}
},
"limitations": [
"YRBS 2021 and 2023 are not included in the selected analytic Parquet and remain quarantined pending layout verification.",
"GSHS design fields are candidates and require survey-specific source verification.",
"NSDUH year-specific weight selection and variance usage require confirmation against the combined-file guide.",
"This audit does not approve item wording, response recodes, missingness rules, constructs, or cross-survey links."
]
}
research/stage1/input_structure_audit.md
+54
View File
@@ -0,0 +1,54 @@
# 阶段 1 输入结构审计
版本：0.1.0
日期：2026-09-20
状态：`provisional`
## 审计范围
本次只读核验三个候选数据入口的实际行列数、年度覆盖和复杂抽样候选字段。结果不构成题目等价、构念归类、缺失值重编码、锚题或 Gate 1 的批准。
机器可读结果见 `input_structure_audit.json`；复现脚本为 `audit_input_sources.py`；实体和审核状态契约见 `stage1_data_contract.json`。
## GSHS
- 入口：`Dataset/GSHS-全球学生健康调查数据/GSHS/01_data/GSHS.csv`
- 实测规模：673,499 行、637 列；2003–2023；191 个国家×年份×组件键。
- 范围行数：national 471,274；component 167,067；subnational 28,657；special_school 6,501。
- `sample_weight`、`sampling_psu`、`sampling_stratum` 各有 673,479 个非空值，即各缺 20 行。
- `scope_review_flag=1` 覆盖 36 个组件键。既有质量报告写的是 10 项调查；两者统计单位不同，进入分析前需建立调查—组件映射并解释差异。
判定：GSHS 继续作为主开发候选，但设计字段只能视为候选；必须按组件回到问卷、codebook 和设计来源核验。
## YRBS
- 入口：`Dataset/可直接分析数据包_NSDUH_YRBS/YRBS/yrbs_1991_2019_selected.parquet`
- 实测规模：217,341 行、79 列；覆盖 1991–2019 的 15 个隔年调查。
- 1991–2015 的设计字段基本完整；1997 年 16,263 行中有 1 行同时缺 weight、stratum 和 psu。
- 2017 年 14,765 行中仅 13,026 行有 weight，stratum/psu 全缺。
- 2019 年 13,677 行中仅 8,845 行有 weight，stratum/psu 全缺。
- 2021、2023 不在该可分析 Parquet 中，继续保持 layout 验证隔离状态。
判定：当前选表不能直接支持 2017/2019 的完整复杂抽样分析；不得把 1991–2019 整体标记为设计就绪。后续应对照官方 SAS 输入/格式程序修复或另建受限分析分支。
## NSDUH
- 全表入口：`Dataset/可直接分析数据包_NSDUH_YRBS/NSDUH/nsduh_2021_2024_full.parquet`，232,441 行、2,638 列。
- 选表入口：同目录 `nsduh_2021_2024_selected.parquet`，232,441 行、151 列。
- 年度行数：2021 58,034；2022 59,069；2023 56,705；2024 58,633。
- `VESTR_C`、`VEREP`、`ANALWT2_C1`–`ANALWT2_C4` 在全表和选表中均无空值且均为正值。
判定：结构完整不等于权重适用规则已确认。四个权重列在四个年度均为正，需对照官方 combined-file guide 核实每列的估计目标、年度组合和方差方法；青少年总体、题目适用条件及成人/青少年题分离仍未完成。
## 当前结论
1. 三个候选入口均可读，实测规模与现有交付说明一致。
2. 阶段 1 已有机器可读实体契约，所有审核状态默认 `provisional`，AI 检查不能冒充双人审核。
3. YRBS 设计变量缺口是当前最明确的数据阻断；GSHS 组件范围和设计来源、NSDUH 权重语义仍需官方证据。
4. Gate 1 保持 `not passed`。下一增量应建立主要结局的来源登记骨架，并优先核验 GSHS/NSDUH 的自杀意念、计划、尝试题及 YRBS 设计字段缺口。
## 限制
- 本次未读取或解释个体回答值，也未计算任何患病率或模型结果。
- 本次未逐页复核 PDF/DOCX 问卷和 codebook。
- 行数、字段存在和非空性不能证明编码正确、测量等价或复杂抽样方差可用。
research/stage1/stage1_data_contract.json
+146
View File
@@ -0,0 +1,146 @@
{
"schema_version": "0.1.0",
"status": "provisional",
"purpose": "Stage 1 contract for source-traceable survey items, responses, design variables, and review state.",
"review_status": [
"provisional",
"under_review",
"human_reviewed_single",
"human_reviewed_double",
"approved",
"conflicted",
"excluded"
],
"missing_type": [
"observed",
"not_administered",
"inapplicable",
"legitimate_skip",
"refused",
"dont_know",
"ordinary_missing",
"data_error",
"unknown"
],
"entities": {
"survey_component": {
"primary_key": ["survey_id", "component_id"],
"required_fields": [
"survey_id",
"component_id",
"country",
"year",
"scope",
"population",
"language",
"questionnaire_version",
"source_file",
"source_url",
"review_status"
]
},
"item_instance": {
"primary_key": ["survey_id", "component_id", "item_version_id"],
"required_fields": [
"item_version_id",
"item_family_id",
"source_variable",
"question_text",
"construct",
"time_window",
"response_type",
"language",
"survey_id",
"component_id",
"source_file",
"source_url",
"page_or_section",
"text_hash",
"review_status"
]
},
"response_code": {
"primary_key": ["item_version_id", "raw_code"],
"required_fields": [
"item_version_id",
"raw_code",
"raw_label",
"canonical_code",
"missing_type",
"derivation_rule",
"source_file",
"page_or_section",
"review_status"
]
},
"response_long": {
"primary_key": ["respondent_id", "survey_id", "component_id", "item_version_id"],
"required_fields": [
"respondent_id",
"survey_id",
"component_id",
"item_version_id",
"raw_response",
"canonical_response",
"eligibility",
"observed_status"
]
},
"survey_design": {
"primary_key": ["survey_id", "component_id"],
"required_fields": [
"survey_id",
"component_id",
"weight_field",
"psu_field",
"stratum_field",
"variance_method",
"design_source",
"review_status"
]
},
"source_review": {
"primary_key": ["survey_id", "component_id", "item_version_id", "source_file"],
"required_fields": [
"source_file",
"source_url",
"page_or_section",
"text_hash",
"reviewer_1",
"reviewer_2",
"review_status"
],
"constraints": [
"reviewer_1 and reviewer_2 must identify actual human reviewers before review_status can be human_reviewed_double or approved",
"AI checks may be recorded separately but must not populate reviewer_1 or reviewer_2"
]
},
"candidate_link": {
"primary_key": ["left_item_version_id", "right_item_version_id"],
"required_fields": [
"left_item_version_id",
"right_item_version_id",
"relation_type",
"semantic_evidence",
"population_overlap",
"anchor_status",
"exclusion_reason",
"review_status"
],
"allowed_anchor_status": [
"candidate",
"training_only",
"approved_partial_invariance",
"rejected"
]
}
},
"global_constraints": [
"All records default to provisional until source evidence is checked.",
"A shared source variable name does not establish item equivalence.",
"Derived indicators and raw responses must remain distinguishable.",
"Legitimate skips may be recoded as negative only when an explicit official rule supports the derivation.",
"PSU and stratum identifiers require survey or component namespaces before pooling.",
"Semantic similarity may nominate a candidate link but cannot establish measurement invariance."
]
}