LENGTH-POLICY-CONTRACT.json

Extracted facts (0)

No facts are anchored to this source yet.

Full text (2 KB)

{
  "schema_version": 1,
  "study_id": "kuku-yalanji-v25.0-decoding-length-policy",
  "frozen_at": "2026-07-16T08:10:00Z",
  "question": "How much of the measured sentence error is caused by the frozen 64-source/32-target token limits, and what greedy length policy is noninferior across all four sentence diagnostics?",
  "models": [
    "C_J69",
    "A1_GVN",
    "A2_GVN_TRANSLATE"
  ],
  "policies": {
    "S64_T32": {"max_source_length": 64, "max_new_tokens": 32},
    "S64_T64": {"max_source_length": 64, "max_new_tokens": 64},
    "S64_T128": {"max_source_length": 64, "max_new_tokens": 128},
    "S128_T32": {"max_source_length": 128, "max_new_tokens": 32},
    "S128_T64": {"max_source_length": 128, "max_new_tokens": 64},
    "S128_T128": {"max_source_length": 128, "max_new_tokens": 128}
  },
  "sentence_endpoints": [
    "synthetic_dev",
    "natural_dev_text36",
    "usage_diagnostic",
    "elder_diagnostic"
  ],
  "fixed_factors": {
    "direction": "eng-gvn",
    "source_lang": "eng_Latn",
    "target_lang": "gvn_Latn",
    "batch_size": 64,
    "num_beams": 1,
    "do_sample": false,
    "no_repeat_ngram_size": 0,
    "repetition_penalty": 1.0,
    "length_penalty": 1.0,
    "dtype": "float32",
    "decoder_start": "EOS",
    "forced_bos": "gvn_Latn",
    "seed": 0,
    "deterministic_algorithms": true,
    "tf32": false
  },
  "policy_rule": {
    "control": "S64_T32",
    "endpoint_chrf_noninferiority_margin": 0.5,
    "zero_empty_outputs": true,
    "no_increase_in_repeated_segment_rows_at_least_10": true,
    "selection_order": [
      "all hard checks pass",
      "highest unweighted mean chrF++ delta over the four endpoints",
      "best worst-endpoint chrF++ delta",
      "shortest target budget",
      "shortest source budget"
    ]
  },
  "claim_boundary": [
    "This is a decoder-length study, not a beam-search study and not evidence from new linguistic data.",
    "The natural endpoints are small, clustered, repeatedly inspected diagnostics.",
    "A better length policy cannot authorize a reliable free-form translation claim."
  ]
}