Feat/quality based upsampling

config_files/data_preparation/quality/annealing_registry.yaml

@@ -0,0 +1,305 @@
+# Corpus registry for the annealing blend.
+#
+# Every join key below was verified by sampling real keys out of the JSONL and scanning
+# the propella `id` columns for them. Coverage measured 2026-08-17 against the caches
+# named in `annotation_root`:
+#
+#   finewiki (all 5 languages)                100%
+#   hplt-4-unfiltered (de/fr/it/es)           100%
+#   nemotron-cc/high-actual                   100%  (also covers the synthetic
+#                                                    high_diverse_qa_pairs subdirectory,
+#                                                    which shares warc_record_id)
+#   nemotron-climbmix                         100%  (English, and German via the source)
+#   finepdfs (eng/deu/fra/spa)                10-34%  only 1-5 shards fetched per split
+#
+# Datasets with no `annotation_split` have no propella corpus at all and can only be
+# shaped by their native metrics.
+
+annotation_root: /data/michael.fromm/hf-cache/datasets--openeurollm--propella-annotations/snapshots/e80fc1407801a15b956f40c642d3709b528abbc9/data/propella-1-4b
+extra_annotation_roots:
+  # The HPLT shards were fetched into a second cache; both are searched.
+  - /data/alex.jude/.cache/huggingface/datasets--openeurollm--propella-annotations/snapshots/9e9c5083f81dc4bbd2708b65816a4dc41f59b911/data/propella-1-4b
+
+datasets:
+  # ---------------------------------------------------------------- FineWiki
+  # Ids look like `dewiki/6851323` and are used verbatim on both sides.
+  - name: finewiki-en
+    jsonl_root: /data/annealing/english/Finewiki
+    annotation_split: finewiki
+    key: {kind: field, field: id}
+    native_metrics:
+      - {name: bytes_html, jq_pattern: .bytes_html}
+  - name: finewiki-de
+    jsonl_root: /data/annealing/german/Finewiki
+    annotation_split: finewiki
+    key: {kind: field, field: id}
+  - name: finewiki-fr
+    jsonl_root: /data/annealing/french/Finewiki
+    annotation_split: finewiki
+    key: {kind: field, field: id}
+  - name: finewiki-it
+    jsonl_root: /data/annealing/italian/Finewiki
+    annotation_split: finewiki
+    key: {kind: field, field: id}
+  - name: finewiki-es
+    jsonl_root: /data/annealing/spanish/Finewiki
+    annotation_split: finewiki
+    key: {kind: field, field: id}
+
+  # ---------------------------------------------------------------- HPLT v4
+  # 32-hex ids, used verbatim. No quality score of any kind in the records, so propella
+  # is the only quality signal available for these 14 TB.
+  - name: hplt-de
+    jsonl_root: /data/annealing/german/hpltv4
+    annotation_split: hplt-4-unfiltered/deu_Latn
+    key: {kind: field, field: id}
+    native_metrics:
+      - {name: lid_prob, jq_pattern: ".\"openlid-v3\".prob", aggregation: max}
+      - {name: cluster_size, jq_pattern: .cluster_size}
+  - name: hplt-fr
+    jsonl_root: /data/annealing/french/hpltv4
+    annotation_split: hplt-4-unfiltered/fra_Latn
+    key: {kind: field, field: id}
+    native_metrics:
+      - {name: lid_prob, jq_pattern: ".\"openlid-v3\".prob", aggregation: max}
+      - {name: cluster_size, jq_pattern: .cluster_size}
+  - name: hplt-it
+    jsonl_root: /data/annealing/italian/hpltv4
+    annotation_split: hplt-4-unfiltered/ita_Latn
+    key: {kind: field, field: id}
+    native_metrics:
+      - {name: lid_prob, jq_pattern: ".\"openlid-v3\".prob", aggregation: max}
+      - {name: cluster_size, jq_pattern: .cluster_size}
+  - name: hplt-es
+    jsonl_root: /data/annealing/spanish/hpltv4
+    annotation_split: hplt-4-unfiltered/spa_Latn
+    key: {kind: field, field: id}
+    native_metrics:
+      - {name: lid_prob, jq_pattern: ".\"openlid-v3\".prob", aggregation: max}
+      - {name: cluster_size, jq_pattern: .cluster_size}
+
+  # ---------------------------------------------------------------- Nemotron-CC
+  # No `id` field at all; `warc_record_id` is the annotation key. The annotation ids are
+  # not unique -- roughly 4% recur -- so the join keeps the first occurrence. This is not
+  # specific to this split: finewiki measured 868,586 duplicate keys in 43.1 M rows (2.0%)
+  # on a real run, so expect the join to report duplicates for most splits.
+  # The 2026-08 delivery renamed `warc_record_id` to `id`, keeping the same bare UUIDs.
+  # The old name silently produced a null join key for every document and 0% coverage.
+  # Both subdirectories -- high_actual and high_diverse_qa_pairs -- are covered by the
+  # high-actual annotation split; 400/400 sampled ids from each were found in it.
+  - name: nemotron-cc
+    jsonl_root: /data/annealing/english/Nemotron-CC
+    annotation_split: nemotron-cc/high-actual
+    key: {kind: field, field: id}
+
+  # ---------------------------------------------------------------- ClimbMix
+  # No identifier of any kind; the annotation key is the SHA-256 of the exact text.
+  - name: climbmix-en
+    jsonl_root: /data/annealing/english/Climbmix
+    annotation_split: nemotron-climbmix
+    key: {kind: sha256_text}
+
+  # German KletterMix is a translation of ClimbMix. Its own text is German, so hashing
+  # it matches nothing -- the annotation belongs to the English original, which the id
+  # points at as `<file>/<line>`, zero-indexed.
+  - name: klettermix-de
+    jsonl_root: /data/annealing/german/AIML-TUDA-KletterMix-filtered
+    glob: "*.jsonl"
+    annotation_split: nemotron-climbmix
+    key:
+      kind: source_pointer
+      field: id
+      source_root: /data/annealing_unused/Nemotron-ClimbMix
+      source_line_offset: 0
+    native_metrics:
+      - {name: proxy_score, jq_pattern: .proxy_score}
+      - {name: token_count, jq_pattern: .token_count}
+
+  # ---------------------------------------------------------------- FinePDFs
+  # Ids are UUIDs stored in two forms -- `<urn:uuid:...>` and bare -- mixed within
+  # single files on both sides, so both sides are normalised before comparing.
+  - name: finepdfs-en
+    jsonl_root: /data/annealing/english/Finepdfs
+    annotation_split: finepdfs/eng_Latn
+    key: {kind: urn_uuid_field, field: id}
+    note: "annotation coverage was 34% on 2026-08-17; finish the fetch before relying on a propella predicate"
+    native_metrics:
+      - {name: fw_edu, jq_pattern: .fw_edu_scores, aggregation: max}
+      - {name: dclm, jq_pattern: .dclm_scores, aggregation: max}
+      - {name: ocr_quality, jq_pattern: .ocr_quality_scores, aggregation: min}
+      - {name: full_doc_lid_score, jq_pattern: .full_doc_lid_score}
+      - {name: duplicate_count, jq_pattern: .duplicate_count}
+  - name: finepdfs-de
+    jsonl_root: /data/annealing/german/Finepdfs
+    annotation_split: finepdfs/deu_Latn
+    key: {kind: urn_uuid_field, field: id}
+    native_metrics:
+      - {name: fw_edu, jq_pattern: .fw_edu_scores, aggregation: max}
+      - {name: full_doc_lid_score, jq_pattern: .full_doc_lid_score}
+  - name: finepdfs-fr
+    jsonl_root: /data/annealing/french/Finepdfs
+    annotation_split: finepdfs/fra_Latn
+    key: {kind: urn_uuid_field, field: id}
+    native_metrics:
+      - {name: fw_edu, jq_pattern: .fw_edu_scores, aggregation: max}
+      - {name: full_doc_lid_score, jq_pattern: .full_doc_lid_score}
+  - name: finepdfs-es
+    jsonl_root: /data/annealing/spanish/Finepdfs
+    annotation_split: finepdfs/spa_Latn
+    key: {kind: urn_uuid_field, field: id}
+    native_metrics:
+      - {name: fw_edu, jq_pattern: .fw_edu_scores, aggregation: max}
+      - {name: full_doc_lid_score, jq_pattern: .full_doc_lid_score}
+
+  # `italian/Finepdfs` holds English data, byte-identical to `english/Finepdfs` in 578
+  # of its 580 files. Enabling it would count the same 5.4 TB of English documents
+  # twice, so it stays declared and disabled until that is resolved.
+  - name: finepdfs-it
+    jsonl_root: /data/annealing/italian/Finepdfs
+    annotation_split: finepdfs/eng_Latn
+    key: {kind: urn_uuid_field, field: id}
+    enabled: false
+    note: "English data in the Italian folder; duplicates finepdfs-en"
+
+  # -------------------------------------------- no propella corpus, native only
+  # The largest subset of the blend. Every candidate annotation key returns 0%, so it
+  # can only be shaped by its category directory and native fields.
+  - name: nemotron-cc-v2
+    jsonl_root: /data/annealing/english/Nemotron-CC-v2
+    native_metrics: []
+  - name: finephrase
+    jsonl_root: /data/annealing/english/Finephrase
+    native_metrics:
+      - {name: fw_edu, jq_pattern: .score}
+      - {name: language_score, jq_pattern: .language_score}
+      - {name: token_count, jq_pattern: .token_count}
+  - name: dolmino
+    jsonl_root: /data/annealing/english/Dolmino
+    note: "dclm_plus2 exists only under stem-heavy-crawl; it was null in all of 400 sampled sidecar parts, so the cube drops it"
+    native_metrics:
+      - {name: dclm_plus2, jq_pattern: '.metadata.dclm_plus2."__label__1"'}
+      - {name: len_cl100k_base, jq_pattern: .metadata.len_cl100k_base}
+
+  # ==================================================================== added scope
+  # Everything in /data/annealing that was not previously registered, so the cubes cover
+  # the whole tree rather than the 20 datasets of the first blend.
+  #
+  # None of these has a propella annotation split -- the two annotation roots hold only
+  # finewiki, finepdfs, hplt-4-unfiltered, nemotron-cc, nemotron-climbmix and
+  # german-commons -- so none declares a `key`, and the join stage skips them all. Their
+  # cubes are built from native metrics where the records carry any, and 24 of the 29 carry
+  # none, so those cubes report exact document and token counts with no dimension to
+  # threshold on. That is still what a blend is budgeted from.
+  #
+  # Two directory trees are deliberately absent: `math_epoch_2` and
+  # `code/swallow-code-v2_stage5_epoch_2` are byte-identical copies of `math/` and
+  # `..._epoch_1` (one 95 GB pair confirmed by full hash), 1.94 TB in total.
+
+  - name: nemotron-code-v1
+    jsonl_root: /data/annealing/code/Nemotron-Pretraining-Code-v1
+    native_metrics: []
+  - name: nemotron-code-v2
+    jsonl_root: /data/annealing/code/Nemotron-Pretraining-Code-v2
+    native_metrics: []
+  - name: nemotron-code-v3
+    jsonl_root: /data/annealing/code/Nemotron-Pretraining-Code-v3
+    native_metrics: []
+  - name: dolmino-stackedu-fim
+    jsonl_root: /data/annealing/code/dolmino_stackedu_fim
+    native_metrics:
+      - {name: score, jq_pattern: '.metadata.score'}
+      - {name: int_score, jq_pattern: '.metadata.int_score'}
+      - {name: length_bytes, jq_pattern: '.metadata.length_bytes'}
+  - name: swallow-code-v2
+    jsonl_root: /data/annealing/code/swallow-code-v2_stage5_epoch_1
+    note: "epoch_2 alongside this is a byte-identical copy (verified by full hash) and is deliberately not registered"
+    native_metrics:
+      - {name: score, jq_pattern: '.metadata.score'}
+      - {name: star_events, jq_pattern: '.metadata.star_events_count'}
+      - {name: fork_events, jq_pattern: '.metadata.fork_events_count'}
+      - {name: length_bytes, jq_pattern: '.metadata.length_bytes'}
+  - name: nemotron-cc-v2.1
+    jsonl_root: /data/annealing/english/Nemotron-CC-v2.1
+    native_metrics: []
+  - name: nemotron-legal-v1
+    jsonl_root: /data/annealing/english/Nemotron-Pretraining-Legal-v1
+    native_metrics: []
+  - name: qa-base-en
+    jsonl_root: /data/annealing/english/QA_base
+    note: "git-lfs style checkout; carries .gitattributes files"
+    native_metrics: []
+  - name: qa-base-fr
+    jsonl_root: /data/annealing/french/QA_base
+    native_metrics: []
+  - name: nemotron-qa-fr
+    jsonl_root: /data/annealing/french/nemotron-cc-v2-translated-diverse-qa
+    native_metrics: []
+  - name: qa-base-de
+    jsonl_root: /data/annealing/german/QA_base_data
+    native_metrics: []
+  - name: soofi-think-de
+    jsonl_root: /data/annealing/german/Soofi-Think
+    native_metrics: []
+  - name: court-decisions-de
+    jsonl_root: /data/annealing/german/court-decisions-germany
+    native_metrics: []
+  - name: qa-base-it
+    jsonl_root: /data/annealing/italian/QA_base
+    native_metrics: []
+  - name: nemotron-qa-it
+    jsonl_root: /data/annealing/italian/nemotron-cc-v2-translated-diverse-qa
+    native_metrics: []
+  - name: nemotron-math-v1
+    jsonl_root: /data/annealing/math/Nemotron-CC-Math-v1
+    note: "math_epoch_2 holds a copy of the whole math/ tree and is deliberately not registered"
+    native_metrics:
+      - {name: finemath, jq_pattern: '.metadata.finemath_scores'}
+      - {name: nemocurator, jq_pattern: '.metadata.nemocurator_scores'}
+      - {name: finemath_int, jq_pattern: '.metadata.finemath_int_scores'}
+  - name: ultradata-math
+    jsonl_root: /data/annealing/math/UltraData-Math
+    native_metrics: []
+  - name: dolmino-math
+    jsonl_root: /data/annealing/math/dolmino_math
+    native_metrics: []
+  - name: mt-reasoning
+    jsonl_root: /data/annealing/reasoning/MT-Reasoning
+    native_metrics:
+      - {name: prompt_tokens, jq_pattern: '.prompt_tokens'}
+      - {name: reasoning_tokens, jq_pattern: '.reasoning_tokens'}
+      - {name: response_tokens, jq_pattern: '.response_tokens'}
+      - {name: total_tokens, jq_pattern: '.total_tokens'}
+  - name: nemotron-specialized-v1
+    jsonl_root: /data/annealing/reasoning/Nemotron-Pretraining-Specialized-v1
+    native_metrics: []
+  - name: nemotron-specialized-v1.1
+    jsonl_root: /data/annealing/reasoning/Nemotron-Pretraining-Specialized-v1.1
+    native_metrics: []
+  - name: nemotron-specialized-v1.2
+    jsonl_root: /data/annealing/reasoning/Nemotron-Pretraining-Specialized-v1.2
+    native_metrics: []
+  - name: dolmino-reasoning
+    jsonl_root: /data/annealing/reasoning/dolmino_pool_reasoning
+    native_metrics: []
+  - name: acereason
+    jsonl_root: /data/annealing/sft/AceReason
+    native_metrics: []
+  - name: nemotron-sft-v1
+    jsonl_root: /data/annealing/sft/Nemotron-Pretraining-SFT-v1
+    native_metrics: []
+  - name: dolmino-sft
+    jsonl_root: /data/annealing/sft/dolmino
+    native_metrics: []
+  - name: sft-posttraining
+    jsonl_root: /data/annealing/sft/sft-posttraining_2
+    note: "the _2 delivery; sft-posttraining/ shares 710 of its 715 files by name and size and is not registered"
+    native_metrics:
+      - {name: lang_conf, jq_pattern: '.lang_conf'}
+      - {name: est_tokens_native, jq_pattern: '.est_tokens'}
+      - {name: fertility_factor, jq_pattern: '.fertility_factor'}
+  - name: qa-base-es
+    jsonl_root: /data/annealing/spanish/QA_base
+    native_metrics: []
+  - name: nemotron-qa-es
+    jsonl_root: /data/annealing/spanish/nemotron-cc-v2-translated-diverse-qa
+    native_metrics: []