Jinovo/language-conditioned-psychometric-harmonization-model
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

docs(stage1): source Solomon GSHS outcomes

Browse Source
This commit is contained in:
Jinotech
2026-09-20 08:49:27 +12:00
parent 300e2df619
commit e5e301ef54
7 changed files with 107 additions and 44 deletions
Download Patch File Download Diff File Expand all files Collapse all files
research/stage1/audit_gshs_primary_outcome_components.py
+34 -9
View File
@@ -136,6 +136,14 @@ SOURCE_PACKAGES = {
},
}
SOLOMON_COMPONENT = "SLB_2023_GSHS_v01::slb_2023"
SOLOMON_CATALOG_SOURCES = [
("https://extranet.who.int/ncdsmicrodata/index.php/catalog/992/variable/V39", "ideation variable metadata"),
("https://extranet.who.int/ncdsmicrodata/index.php/catalog/992/variable/V40", "plan variable metadata"),
("https://extranet.who.int/ncdsmicrodata/index.php/catalog/992/variable/V41", "attempt variable metadata"),
("https://extranet.who.int/ncdsmicrodata/index.php/catalog/992", "study universe and sampling metadata"),
]
def sha256(path: Path) -> str:
digest = hashlib.sha256()
@@ -187,6 +195,17 @@ def scan_data() -> dict:
def source_package(component_id: str) -> dict:
package = SOURCE_PACKAGES.get(component_id)
if package is None:
if component_id == SOLOMON_COMPONENT:
return {
"status": "official_catalog_item_metadata_verified",
"documents": [
{"source_url": url, "role": role, "page_or_section": "WHO NCD Microdata Repository catalog"}
for url, role in SOLOMON_CATALOG_SOURCES
],
"question_numbers": None,
"source_language_evidence": ["English catalog literal-question metadata"],
"note": "The official catalog provides literal questions, response categories, case counts, universe, and sampling metadata. A country questionnaire file and actual administration-language evidence were not located.",
}
return {
"status": "pending_local_direct_source",
"documents": [],
@@ -254,7 +273,11 @@ def build_audit() -> dict:
"collection_start": meta["collection_start"],
"survey_component": meta["survey_component"],
"survey_scope": meta["survey_scope"],
"population": "school-attending students; exact sampled ages/grades remain pending component source review",
"population": (
"school-going adolescents aged 13-17 years; Form 1-Form 6; all students in selected classes eligible"
if component_id == SOLOMON_COMPONENT
else "school-attending students; exact sampled ages/grades remain pending component source review"
),
"observed_rows": observed_rows,
"canonical_relative_path": meta["canonical_relative_path"],
"catalog_url": meta["catalog_url"],
@@ -290,11 +313,10 @@ def build_audit() -> dict:
)
available = [component for component in components if component["primary_outcomes_observed"]]
direct_sources = [
component for component in available if component["source_package"]["status"] == "local_questionnaire_and_codebooks_located"
]
local_sources = [component for component in available if component["source_package"]["status"] == "local_questionnaire_and_codebooks_located"]
direct_sources = [component for component in available if component["source_package"]["status"] != "pending_local_direct_source"]
return {
"schema_version": "1.0",
"schema_version": "1.1",
"generated_on": "2026-09-20",
"status": "provisional",
"purpose": "Gate 1 component-level coverage and provenance audit for GSHS suicide ideation, plan, and attempt.",
@@ -308,8 +330,11 @@ def build_audit() -> dict:
"selected_components": len(components),
"components_with_primary_outcomes": len(available),
"components_without_primary_outcomes": len(components) - len(available),
"components_with_local_direct_sources": len(direct_sources),
"components_pending_local_direct_source": len(available) - len(direct_sources),
"components_with_direct_item_sources": len(direct_sources),
"components_with_local_direct_sources": len(local_sources),
"components_with_local_questionnaire_packages": len(local_sources),
"components_with_official_catalog_only": len(direct_sources) - len(local_sources),
"components_pending_direct_item_source": len(available) - len(direct_sources),
"observed_rows": sum(scan["row_counts"].values()),
"row_count_mismatches": len(row_count_mismatches),
},
@@ -372,13 +397,13 @@ def render_markdown(audit: dict) -> str:
"",
"## Source boundary",
"",
f"Local questionnaires plus raw and binary codebooks were located for {audit['summary']['components_with_local_direct_sources']} of the 7 components: Ghana, India, Jamaica, Mongolia, Morocco, and Wallis and Futuna. No country-specific Solomon Islands 2023 questionnaire/codebook was found locally, so that component remains pending direct-source acquisition.",
f"Local questionnaires plus raw and binary codebooks were located for {audit['summary']['components_with_local_questionnaire_packages']} components: Ghana, India, Jamaica, Mongolia, Morocco, and Wallis and Futuna. For Solomon Islands, the official WHO catalog directly provides the three literal questions, response categories, case counts, universe, and sampling metadata. A country questionnaire file and administration-language evidence were not located, so those narrower provenance fields remain pending.",
"",
"Available English/French/Mongolian files establish wording evidence but do not by themselves prove the language administered to every respondent. Exact sampled ages/grades, component-specific eligibility, and canonical missing reasons also remain unresolved.",
"",
"## Gate 1 decision",
"",
"Gate 1 remains **in progress**. This audit resolves the 191-component coverage question and the raw-versus-`_b_` representation issue, but it does not satisfy the population/missing-rule or two-human-reviewer requirements. All records remain provisional and are not approved for confirmatory analysis or anchor construction.",
"Gate 1 remains **in progress**. This audit resolves the 191-component coverage question, direct item-source coverage, and the raw-versus-`_b_` representation issue, but it does not satisfy all population/language/missing-rule or two-human-reviewer requirements. All records remain provisional and are not approved for confirmatory analysis or anchor construction.",
"",
"## Reproduction",
"",
research/stage1/build_primary_outcome_registry.py
+6 -6
View File
@@ -92,7 +92,7 @@ def build_gshs() -> tuple[list[dict], list[dict]]:
"source_url": None,
"page_or_section": "Master_Columns plus component-specific questionnaire/codebook references in component audit",
"present_in_local_data": True,
"source_verification_status": "direct_local_sources_verified_for_six_of_seven_observed_components",
"source_verification_status": "direct_sources_verified_for_all_seven_observed_components_six_local_packages_one_official_catalog",
"review_status": "provisional",
"reviewer_1": None,
"reviewer_2": None,
@@ -104,7 +104,7 @@ def build_gshs() -> tuple[list[dict], list[dict]]:
"construct": GSHS_CONSTRUCT[variable],
"record_type": "administered_raw_candidate",
"question_text": GSHS_QUESTION[variable],
"notes": "Country questionnaires and ordinary codebooks identify this as an administered item. Solomon Islands 2023 direct local source remains pending; population and missing rules are not approved.",
"notes": "Country questionnaires/ordinary codebooks cover six components; official WHO catalog item metadata covers Solomon Islands 2023. Administration language and missing rules are not approved.",
}
)
records.append(common)
@@ -270,7 +270,7 @@ def render_markdown(registry: dict) -> str:
lines = [
"# Primary outcome source registry",
"",
"Version: 1.3 | Status: provisional | Generated: 2026-09-20",
"Version: 1.4 | Status: provisional | Generated: 2026-09-20",
"",
"## Scope and decision boundary",
"",
@@ -283,7 +283,7 @@ def render_markdown(registry: dict) -> str:
]
for survey in ["GSHS", "YRBS", "NSDUH"]:
boundary = {
"GSHS": "Three administered items occur in 7/191 components; local direct sources cover 6/7, with Solomon Islands pending.",
"GSHS": "Three administered items occur in 7/191 components; six local source packages plus WHO catalog metadata cover all seven.",
"YRBS": "51 annual candidates directly cross-checked; 21 prior mappings corrected; 2021/2023 data remain quarantined.",
"NSDUH": "Attempt is administered as YSUI03/YUSUICTRY, but the raw and recoded attempt fields are absent from the local public-use package.",
}[survey]
@@ -306,7 +306,7 @@ def render_markdown(registry: dict) -> str:
"",
"## Gate 1 status",
"",
"Gate 1 remains **in progress**. Before any candidate can enter confirmatory analysis, the project still needs the Solomon Islands GSHS source, exact population and skip-logic verification, canonical response/missing rules, validated YRBS 2021/2023 imports, and two actual human reviewers for primary outcomes.",
"Gate 1 remains **in progress**. Before any candidate can enter confirmatory analysis, the project still needs exact remaining population/language and skip-logic verification, canonical response/missing rules, validated YRBS 2021/2023 imports, and two actual human reviewers for primary outcomes.",
"",
"## Reproduction",
"",
@@ -321,7 +321,7 @@ def main() -> None:
yrbs = build_yrbs()
nsduh_raw, nsduh_derived, nsduh_excluded = build_nsduh()
registry = {
"schema_version": "1.3",
"schema_version": "1.4",
"generated_on": "2026-09-20",
"status": "provisional",
"allowed_uses": ["pipeline_development", "source_review_queue"],
research/stage1/gshs_primary_outcome_component_audit.json
+33 -7
View File
@@ -1,5 +1,5 @@
{
"schema_version": "1.0",
"schema_version": "1.1",
"generated_on": "2026-09-20",
"status": "provisional",
"purpose": "Gate 1 component-level coverage and provenance audit for GSHS suicide ideation, plan, and attempt.",
@@ -13,8 +13,11 @@
"selected_components": 191,
"components_with_primary_outcomes": 7,
"components_without_primary_outcomes": 184,
"components_with_direct_item_sources": 7,
"components_with_local_direct_sources": 6,
"components_pending_local_direct_source": 1,
"components_with_local_questionnaire_packages": 6,
"components_with_official_catalog_only": 1,
"components_pending_direct_item_source": 0,
"observed_rows": 673499,
"row_count_mismatches": 0
},
@@ -6870,7 +6873,7 @@
"collection_start": "2023-08",
"survey_component": "",
"survey_scope": "component",
"population": "school-attending students; exact sampled ages/grades remain pending component source review",
"population": "school-going adolescents aged 13-17 years; Form 1-Form 6; all students in selected classes eligible",
"observed_rows": 1995,
"canonical_relative_path": "raw/microdata/SLB_2023_GSHS_v01/SLB-2023.dta",
"catalog_url": "https://extranet.who.int/ncdsmicrodata/index.php/catalog/992",
@@ -6923,11 +6926,34 @@
}
},
"source_package": {
"status": "pending_local_direct_source",
"documents": [],
"status": "official_catalog_item_metadata_verified",
"documents": [
{
"source_url": "https://extranet.who.int/ncdsmicrodata/index.php/catalog/992/variable/V39",
"role": "ideation variable metadata",
"page_or_section": "WHO NCD Microdata Repository catalog"
},
{
"source_url": "https://extranet.who.int/ncdsmicrodata/index.php/catalog/992/variable/V40",
"role": "plan variable metadata",
"page_or_section": "WHO NCD Microdata Repository catalog"
},
{
"source_url": "https://extranet.who.int/ncdsmicrodata/index.php/catalog/992/variable/V41",
"role": "attempt variable metadata",
"page_or_section": "WHO NCD Microdata Repository catalog"
},
{
"source_url": "https://extranet.who.int/ncdsmicrodata/index.php/catalog/992",
"role": "study universe and sampling metadata",
"page_or_section": "WHO NCD Microdata Repository catalog"
}
],
"question_numbers": null,
"source_language_evidence": [],
"note": "No country questionnaire or codebook for this component was found in the local source package."
"source_language_evidence": [
"English catalog literal-question metadata"
],
"note": "The official catalog provides literal questions, response categories, case counts, universe, and sampling metadata. A country questionnaire file and actual administration-language evidence were not located."
},
"review_status": "provisional",
"reviewer_1": null,
research/stage1/gshs_primary_outcome_component_audit.md
+3 -3
View File
@@ -17,7 +17,7 @@ The country questionnaires and ordinary codebooks identify three administered it
| `IND_2022_GSHS_v01::ind_jaipur_2022` | India | 4,299 | 4,174 | 4,106 | 4,228 | local_questionnaire_and_codebooks_located |
| `JAM_2023_GSHS_v01::jam_st_cath_2023` | Jamaica | 3,041 | 2,879 | 2,850 | 2,936 | local_questionnaire_and_codebooks_located |
| `MNG_2023_GSHS_v01::mng2023` | Mongolia | 3,311 | 3,274 | 3,258 | 3,238 | local_questionnaire_and_codebooks_located |
| `SLB_2023_GSHS_v01::slb_2023` | Solomon Islands | 1,995 | 1,890 | 1,953 | 1,963 | pending_local_direct_source |
| `SLB_2023_GSHS_v01::slb_2023` | Solomon Islands | 1,995 | 1,890 | 1,953 | 1,963 | official_catalog_item_metadata_verified |
| `WLF_2023_GSHS_v01::wlf_2023` | Wallis and Futuna | 919 | 894 | 907 | 905 | local_questionnaire_and_codebooks_located |
## Representation checks
@@ -30,13 +30,13 @@ The country questionnaires and ordinary codebooks identify three administered it
## Source boundary
Local questionnaires plus raw and binary codebooks were located for 6 of the 7 components: Ghana, India, Jamaica, Mongolia, Morocco, and Wallis and Futuna. No country-specific Solomon Islands 2023 questionnaire/codebook was found locally, so that component remains pending direct-source acquisition.
Local questionnaires plus raw and binary codebooks were located for 6 components: Ghana, India, Jamaica, Mongolia, Morocco, and Wallis and Futuna. For Solomon Islands, the official WHO catalog directly provides the three literal questions, response categories, case counts, universe, and sampling metadata. A country questionnaire file and administration-language evidence were not located, so those narrower provenance fields remain pending.
Available English/French/Mongolian files establish wording evidence but do not by themselves prove the language administered to every respondent. Exact sampled ages/grades, component-specific eligibility, and canonical missing reasons also remain unresolved.
## Gate 1 decision
Gate 1 remains **in progress**. This audit resolves the 191-component coverage question and the raw-versus-`_b_` representation issue, but it does not satisfy the population/missing-rule or two-human-reviewer requirements. All records remain provisional and are not approved for confirmatory analysis or anchor construction.
Gate 1 remains **in progress**. This audit resolves the 191-component coverage question, direct item-source coverage, and the raw-versus-`_b_` representation issue, but it does not satisfy all population/language/missing-rule or two-human-reviewer requirements. All records remain provisional and are not approved for confirmatory analysis or anchor construction.
## Reproduction
research/stage1/primary_outcome_registry.json
+10 -10
View File
@@ -1,5 +1,5 @@
{
"schema_version": "1.3",
"schema_version": "1.4",
"generated_on": "2026-09-20",
"status": "provisional",
"allowed_uses": [
@@ -31,7 +31,7 @@
"source_url": null,
"page_or_section": "Master_Columns plus component-specific questionnaire/codebook references in component audit",
"present_in_local_data": true,
"source_verification_status": "direct_local_sources_verified_for_six_of_seven_observed_components",
"source_verification_status": "direct_sources_verified_for_all_seven_observed_components_six_local_packages_one_official_catalog",
"review_status": "provisional",
"reviewer_1": null,
"reviewer_2": null,
@@ -39,7 +39,7 @@
"construct": "suicide_attempt",
"record_type": "administered_raw_candidate",
"question_text": "During the past 12 months, how many times did you attempt suicide?",
"notes": "Country questionnaires and ordinary codebooks identify this as an administered item. Solomon Islands 2023 direct local source remains pending; population and missing rules are not approved."
"notes": "Country questionnaires/ordinary codebooks cover six components; official WHO catalog item metadata covers Solomon Islands 2023. Administration language and missing rules are not approved."
},
{
"survey": "GSHS",
@@ -57,7 +57,7 @@
"source_url": null,
"page_or_section": "Master_Columns plus component-specific questionnaire/codebook references in component audit",
"present_in_local_data": true,
"source_verification_status": "direct_local_sources_verified_for_six_of_seven_observed_components",
"source_verification_status": "direct_sources_verified_for_all_seven_observed_components_six_local_packages_one_official_catalog",
"review_status": "provisional",
"reviewer_1": null,
"reviewer_2": null,
@@ -65,7 +65,7 @@
"construct": "suicide_ideation",
"record_type": "administered_raw_candidate",
"question_text": "During the past 12 months, did you seriously consider attempting suicide?",
"notes": "Country questionnaires and ordinary codebooks identify this as an administered item. Solomon Islands 2023 direct local source remains pending; population and missing rules are not approved."
"notes": "Country questionnaires/ordinary codebooks cover six components; official WHO catalog item metadata covers Solomon Islands 2023. Administration language and missing rules are not approved."
},
{
"survey": "GSHS",
@@ -83,7 +83,7 @@
"source_url": null,
"page_or_section": "Master_Columns plus component-specific questionnaire/codebook references in component audit",
"present_in_local_data": true,
"source_verification_status": "direct_local_sources_verified_for_six_of_seven_observed_components",
"source_verification_status": "direct_sources_verified_for_all_seven_observed_components_six_local_packages_one_official_catalog",
"review_status": "provisional",
"reviewer_1": null,
"reviewer_2": null,
@@ -91,7 +91,7 @@
"construct": "suicide_plan",
"record_type": "administered_raw_candidate",
"question_text": "During the past 12 months, did you make a plan about how you would attempt suicide?",
"notes": "Country questionnaires and ordinary codebooks identify this as an administered item. Solomon Islands 2023 direct local source remains pending; population and missing rules are not approved."
"notes": "Country questionnaires/ordinary codebooks cover six components; official WHO catalog item metadata covers Solomon Islands 2023. Administration language and missing rules are not approved."
},
{
"survey": "YRBS",
@@ -2004,7 +2004,7 @@
"source_url": null,
"page_or_section": "Master_Columns plus component-specific questionnaire/codebook references in component audit",
"present_in_local_data": true,
"source_verification_status": "direct_local_sources_verified_for_six_of_seven_observed_components",
"source_verification_status": "direct_sources_verified_for_all_seven_observed_components_six_local_packages_one_official_catalog",
"review_status": "provisional",
"reviewer_1": null,
"reviewer_2": null,
@@ -2032,7 +2032,7 @@
"source_url": null,
"page_or_section": "Master_Columns plus component-specific questionnaire/codebook references in component audit",
"present_in_local_data": true,
"source_verification_status": "direct_local_sources_verified_for_six_of_seven_observed_components",
"source_verification_status": "direct_sources_verified_for_all_seven_observed_components_six_local_packages_one_official_catalog",
"review_status": "provisional",
"reviewer_1": null,
"reviewer_2": null,
@@ -2060,7 +2060,7 @@
"source_url": null,
"page_or_section": "Master_Columns plus component-specific questionnaire/codebook references in component audit",
"present_in_local_data": true,
"source_verification_status": "direct_local_sources_verified_for_six_of_seven_observed_components",
"source_verification_status": "direct_sources_verified_for_all_seven_observed_components_six_local_packages_one_official_catalog",
"review_status": "provisional",
"reviewer_1": null,
"reviewer_2": null,
research/stage1/primary_outcome_registry.md
+3 -3
View File
@@ -1,6 +1,6 @@
# Primary outcome source registry
Version: 1.3 | Status: provisional | Generated: 2026-09-20
Version: 1.4 | Status: provisional | Generated: 2026-09-20
## Scope and decision boundary
@@ -10,7 +10,7 @@ This registry captures source-linked candidates for suicide ideation, plan, and
| Survey | Ideation | Plan | Attempt | Current evidence boundary |
|---|---:|---:|---:|---|
| GSHS | 1 | 1 | 1 | Three administered items occur in 7/191 components; local direct sources cover 6/7, with Solomon Islands pending. |
| GSHS | 1 | 1 | 1 | Three administered items occur in 7/191 components; six local source packages plus WHO catalog metadata cover all seven. |
| YRBS | 17 | 17 | 17 | 51 annual candidates directly cross-checked; 21 prior mappings corrected; 2021/2023 data remain quarantined. |
| NSDUH | 4 | 4 | 4 | Attempt is administered as YSUI03/YUSUICTRY, but the raw and recoded attempt fields are absent from the local public-use package. |
@@ -28,7 +28,7 @@ Counts are source-record counts, not distinct item families. YRBS counts include
## Gate 1 status
Gate 1 remains **in progress**. Before any candidate can enter confirmatory analysis, the project still needs the Solomon Islands GSHS source, exact population and skip-logic verification, canonical response/missing rules, validated YRBS 2021/2023 imports, and two actual human reviewers for primary outcomes.
Gate 1 remains **in progress**. Before any candidate can enter confirmatory analysis, the project still needs exact remaining population/language and skip-logic verification, canonical response/missing rules, validated YRBS 2021/2023 imports, and two actual human reviewers for primary outcomes.
## Reproduction