{
  "experiment": "commerce-001",
  "model": "JevK5 v0.3 4B",
  "hardware": "RTX 4090",
  "precision": "BF16 for both models",
  "labels": [
    "E",
    "S",
    "C",
    "I"
  ],
  "data": {
    "train": {
      "sha256": "18bc59659563239bfb2311566a82825978beec2a1f740c85865ae5ddf07daa3c",
      "pairs": 2048,
      "queries": 1991,
      "labels": {
        "C": 512,
        "I": 512,
        "S": 512,
        "E": 512
      },
      "max_tokens": 543
    },
    "calibration": {
      "sha256": "c998fd679fa9f0d5a3155ab2537d1c03d5d88fb1d1811c5e0c95b9f6713e0322",
      "pairs": 256,
      "queries": 32,
      "labels": {
        "S": 54,
        "E": 166,
        "C": 16,
        "I": 20
      },
      "max_tokens": 438
    },
    "test": {
      "sha256": "01e0bb6a4a70226ebc5c726ce79a0b7bc4b026401edb25db95c2a8aed9efdec5",
      "pairs": 512,
      "queries": 64,
      "labels": {
        "E": 371,
        "S": 100,
        "I": 35,
        "C": 6
      },
      "max_tokens": 461
    }
  },
  "protocol_sha256": "e312741c6ba5df8f32a02eaa912c5f9976010a16d8c459d5b97175fa94b831cf",
  "analysis_sha256": "f659140e42023489306b171fea93418fc8c8ff14202644cec409513f0a3ef737",
  "models": {
    "shared": {
      "default_temperature": 1.22,
      "fitted_temperature": 1.4142135623730951,
      "default": {
        "cases": 512,
        "correct": 347,
        "accuracy": 0.677734375,
        "macro_f1": 0.4039473684210526,
        "brier": 0.4381323017514104,
        "nll": 0.7976819946987379,
        "confident_errors": 7,
        "automated_cases_at_90": 60,
        "review_rate_at_90": 0.8828125,
        "automated_accuracy_at_90": 0.8833333333333333,
        "per_class_f1": {
          "E": 0.8,
          "S": 0.42105263157894735,
          "C": 0.0,
          "I": 0.39473684210526316
        },
        "confusion_matrix": [
          [
            284,
            72,
            2,
            13
          ],
          [
            39,
            48,
            0,
            13
          ],
          [
            5,
            1,
            0,
            0
          ],
          [
            11,
            7,
            2,
            15
          ]
        ]
      },
      "calibrated": {
        "cases": 512,
        "correct": 347,
        "accuracy": 0.677734375,
        "macro_f1": 0.4039473684210526,
        "brier": 0.4380560589024908,
        "nll": 0.8009620201265379,
        "confident_errors": 2,
        "automated_cases_at_90": 14,
        "review_rate_at_90": 0.97265625,
        "automated_accuracy_at_90": 0.8571428571428571,
        "per_class_f1": {
          "E": 0.8,
          "S": 0.42105263157894735,
          "C": 0.0,
          "I": 0.39473684210526316
        },
        "confusion_matrix": [
          [
            284,
            72,
            2,
            13
          ],
          [
            39,
            48,
            0,
            13
          ],
          [
            5,
            1,
            0,
            0
          ],
          [
            11,
            7,
            2,
            15
          ]
        ]
      },
      "median_ms": 107.11989300000369,
      "p95_ms": 130.80297800001972
    },
    "fine_tuned": {
      "default_temperature": 1.22,
      "fitted_temperature": 1.4640856959456252,
      "default": {
        "cases": 512,
        "correct": 308,
        "accuracy": 0.6015625,
        "macro_f1": 0.4571929240449647,
        "brier": 0.5514884163119048,
        "nll": 1.0536824554661133,
        "confident_errors": 0,
        "automated_cases_at_90": 0,
        "review_rate_at_90": 1.0,
        "automated_accuracy_at_90": null,
        "per_class_f1": {
          "E": 0.7203947368421053,
          "S": 0.45353159851301117,
          "C": 0.16,
          "I": 0.4948453608247423
        },
        "confusion_matrix": [
          [
            219,
            102,
            29,
            21
          ],
          [
            15,
            61,
            7,
            17
          ],
          [
            2,
            0,
            4,
            0
          ],
          [
            1,
            6,
            4,
            24
          ]
        ]
      },
      "calibrated": {
        "cases": 512,
        "correct": 308,
        "accuracy": 0.6015625,
        "macro_f1": 0.4571929240449647,
        "brier": 0.563442963128493,
        "nll": 1.0694060074448093,
        "confident_errors": 0,
        "automated_cases_at_90": 0,
        "review_rate_at_90": 1.0,
        "automated_accuracy_at_90": null,
        "per_class_f1": {
          "E": 0.7203947368421053,
          "S": 0.45353159851301117,
          "C": 0.16,
          "I": 0.4948453608247423
        },
        "confusion_matrix": [
          [
            219,
            102,
            29,
            21
          ],
          [
            15,
            61,
            7,
            17
          ],
          [
            2,
            0,
            4,
            0
          ],
          [
            1,
            6,
            4,
            24
          ]
        ]
      },
      "median_ms": 70.11431849991823,
      "p95_ms": 75.40016799998739
    }
  },
  "primary_metric": "macro_f1",
  "paired_changes": {
    "fixed": 36,
    "regressed": 75
  },
  "macro_f1_difference": {
    "estimate": 0.05324555562391209,
    "query_bootstrap_95_interval": [
      -0.019793778063703582,
      0.11923298175685593
    ],
    "resamples": 2000,
    "groups": 64
  },
  "release": {
    "base_revision": "c4f7fdb3aeab5582336406e78d3bef11bf98833d",
    "base_model_sha256": "13824e47f2e40fe052f06943976cf742cb366ba305741a111e75a8ebae907a9c",
    "adapter_hashes": {
      "adapter_model.safetensors": "2f761c5abbcb6da1936ed3fe8c81aa12d52b34e09485460709cbbac693ec2f50",
      "README.md": "f1d28b92ae9a4c78230b419097dfac98073309642e2c1b997ec376909119d6dc",
      "adapter_config.json": "41b246e384d21b88f12d3d5b49ec774c93e8c57408d0691c3d265db771876af8"
    },
    "training_seconds": 1281.6666562139999,
    "peak_gpu_allocated_mb": 8646.2314453125,
    "protocol_sha256": "e312741c6ba5df8f32a02eaa912c5f9976010a16d8c459d5b97175fa94b831cf",
    "dataset_manifest_sha256": "840e729ba6176087c691bfc003252cd3288c59f5749ced003dfae7121a9858b2"
  },
  "environment": {
    "python": "3.13.15",
    "os": "Linux",
    "architecture": "x86_64",
    "packages": {
      "torch": "2.8.0",
      "transformers": "5.17.0",
      "peft": "0.21.0",
      "accelerate": "1.15.0",
      "huggingface-hub": "1.33.0",
      "tokenizers": "0.23.2",
      "safetensors": "0.8.0",
      "numpy": "2.5.3",
      "pandas": "3.0.6",
      "pyarrow": "25.0.1"
    },
    "torch_cuda": "12.8",
    "gpu": "NVIDIA GeForce RTX 4090",
    "device_total_memory_bytes": 24146083840,
    "driver": "595.79",
    "kernels": "Torch reference gated delta attention; SDPA; no FLA, flash-attn or causal-conv1d",
    "executed_source_sha256": {
      "study.py": "e6d300cd13a45e4f40afe5ca26c49c41b6748761fcd992e70191e7f390262b68",
      "model.py": "7cc233eeb8bcef91bd55a492edee664e03354e48722e57d2a1d96a1dfe0b387e",
      "prepare.py": "46b48d849c3ffd46b522795955907df2352a14cd402a8cef73f067af0f320e74",
      "run_experiment.py": "7d1e3bb63e2c9ed44a8f1fedc93d0a410ffeea738b3968cbb816f9078702bed7"
    }
  },
  "verification": {
    "metric-verification": {
      "passed": true,
      "implementation": "Independent NumPy softmax and scikit-learn metrics",
      "absolute_tolerance": 1e-12,
      "scores": {
        "shared_default": {
          "accuracy": 0.677734375,
          "macro_f1": 0.4039473684210526,
          "nll": 0.7976819946987379,
          "brier": 0.4381323017514104,
          "confident_errors": 7.0,
          "automated_cases_at_90": 60.0
        },
        "shared_calibrated": {
          "accuracy": 0.677734375,
          "macro_f1": 0.4039473684210526,
          "nll": 0.8009620201265379,
          "brier": 0.4380560589024908,
          "confident_errors": 2.0,
          "automated_cases_at_90": 14.0
        },
        "fine_tuned_default": {
          "accuracy": 0.6015625,
          "macro_f1": 0.45719292404496464,
          "nll": 1.0536824554661133,
          "brier": 0.5514884163119048,
          "confident_errors": 0.0,
          "automated_cases_at_90": 0.0
        },
        "fine_tuned_calibrated": {
          "accuracy": 0.6015625,
          "macro_f1": 0.45719292404496464,
          "nll": 1.0694060074448093,
          "brier": 0.563442963128493,
          "confident_errors": 0.0,
          "automated_cases_at_90": 0.0
        }
      }
    },
    "reload-verification": {
      "passed": true,
      "forwards": 32,
      "max_absolute_logit_difference": 0.0,
      "base_and_adapter_hashes_verified": true,
      "comparisons": [
        {
          "model": "shared",
          "split": "calibration",
          "id": "esci-1953035",
          "max_absolute_logit_difference": 0.0
        },
        {
          "model": "fine_tuned",
          "split": "calibration",
          "id": "esci-1953035",
          "max_absolute_logit_difference": 0.0
        },
        {
          "model": "shared",
          "split": "calibration",
          "id": "esci-536052",
          "max_absolute_logit_difference": 0.0
        },
        {
          "model": "fine_tuned",
          "split": "calibration",
          "id": "esci-536052",
          "max_absolute_logit_difference": 0.0
        },
        {
          "model": "shared",
          "split": "calibration",
          "id": "esci-290147",
          "max_absolute_logit_difference": 0.0
        },
        {
          "model": "fine_tuned",
          "split": "calibration",
          "id": "esci-290147",
          "max_absolute_logit_difference": 0.0
        },
        {
          "model": "shared",
          "split": "calibration",
          "id": "esci-475200",
          "max_absolute_logit_difference": 0.0
        },
        {
          "model": "fine_tuned",
          "split": "calibration",
          "id": "esci-475200",
          "max_absolute_logit_difference": 0.0
        },
        {
          "model": "shared",
          "split": "calibration",
          "id": "esci-518791",
          "max_absolute_logit_difference": 0.0
        },
        {
          "model": "fine_tuned",
          "split": "calibration",
          "id": "esci-518791",
          "max_absolute_logit_difference": 0.0
        },
        {
          "model": "shared",
          "split": "calibration",
          "id": "esci-1288970",
          "max_absolute_logit_difference": 0.0
        },
        {
          "model": "fine_tuned",
          "split": "calibration",
          "id": "esci-1288970",
          "max_absolute_logit_difference": 0.0
        },
        {
          "model": "shared",
          "split": "calibration",
          "id": "esci-475201",
          "max_absolute_logit_difference": 0.0
        },
        {
          "model": "fine_tuned",
          "split": "calibration",
          "id": "esci-475201",
          "max_absolute_logit_difference": 0.0
        },
        {
          "model": "shared",
          "split": "calibration",
          "id": "esci-1447904",
          "max_absolute_logit_difference": 0.0
        },
        {
          "model": "fine_tuned",
          "split": "calibration",
          "id": "esci-1447904",
          "max_absolute_logit_difference": 0.0
        },
        {
          "model": "shared",
          "split": "test",
          "id": "esci-1168311",
          "max_absolute_logit_difference": 0.0
        },
        {
          "model": "fine_tuned",
          "split": "test",
          "id": "esci-1168311",
          "max_absolute_logit_difference": 0.0
        },
        {
          "model": "shared",
          "split": "test",
          "id": "esci-1394000",
          "max_absolute_logit_difference": 0.0
        },
        {
          "model": "fine_tuned",
          "split": "test",
          "id": "esci-1394000",
          "max_absolute_logit_difference": 0.0
        },
        {
          "model": "shared",
          "split": "test",
          "id": "esci-2227764",
          "max_absolute_logit_difference": 0.0
        },
        {
          "model": "fine_tuned",
          "split": "test",
          "id": "esci-2227764",
          "max_absolute_logit_difference": 0.0
        },
        {
          "model": "shared",
          "split": "test",
          "id": "esci-68811",
          "max_absolute_logit_difference": 0.0
        },
        {
          "model": "fine_tuned",
          "split": "test",
          "id": "esci-68811",
          "max_absolute_logit_difference": 0.0
        },
        {
          "model": "shared",
          "split": "test",
          "id": "esci-1432747",
          "max_absolute_logit_difference": 0.0
        },
        {
          "model": "fine_tuned",
          "split": "test",
          "id": "esci-1432747",
          "max_absolute_logit_difference": 0.0
        },
        {
          "model": "shared",
          "split": "test",
          "id": "esci-1918402",
          "max_absolute_logit_difference": 0.0
        },
        {
          "model": "fine_tuned",
          "split": "test",
          "id": "esci-1918402",
          "max_absolute_logit_difference": 0.0
        },
        {
          "model": "shared",
          "split": "test",
          "id": "esci-1018759",
          "max_absolute_logit_difference": 0.0
        },
        {
          "model": "fine_tuned",
          "split": "test",
          "id": "esci-1018759",
          "max_absolute_logit_difference": 0.0
        },
        {
          "model": "shared",
          "split": "test",
          "id": "esci-106035",
          "max_absolute_logit_difference": 0.0
        },
        {
          "model": "fine_tuned",
          "split": "test",
          "id": "esci-106035",
          "max_absolute_logit_difference": 0.0
        }
      ]
    },
    "reproduction-verification": {
      "passed": true,
      "method": "Public preparation scripts, same pinned inputs and recorded GPU-host environment; cached downloads reused; no retraining.",
      "exact_sha256_match": {
        "train.jsonl": true,
        "calibration.jsonl": true,
        "test.jsonl": true,
        "dataset-manifest.json": true
      }
    }
  },
  "manifest": {
    "protocol_sha256": "e312741c6ba5df8f32a02eaa912c5f9976010a16d8c459d5b97175fa94b831cf",
    "source_revision": "7916cdf6ab75a462e77f20ab40428a10923998d5",
    "sources": {
      "shopping_queries_dataset_examples.parquet": "4a735b693b4a424a6fc67f5be6e4c811495c488bbf66d02a602d308b2744263a",
      "shopping_queries_dataset_products.parquet": "25124442d064d64b26f74082d6fa09438d679efc0c183cf28d19064a2b65a265"
    },
    "splits": {
      "train": {
        "sha256": "18bc59659563239bfb2311566a82825978beec2a1f740c85865ae5ddf07daa3c",
        "pairs": 2048,
        "queries": 1991,
        "labels": {
          "C": 512,
          "I": 512,
          "S": 512,
          "E": 512
        },
        "max_tokens": 543
      },
      "calibration": {
        "sha256": "c998fd679fa9f0d5a3155ab2537d1c03d5d88fb1d1811c5e0c95b9f6713e0322",
        "pairs": 256,
        "queries": 32,
        "labels": {
          "S": 54,
          "E": 166,
          "C": 16,
          "I": 20
        },
        "max_tokens": 438
      },
      "test": {
        "sha256": "01e0bb6a4a70226ebc5c726ce79a0b7bc4b026401edb25db95c2a8aed9efdec5",
        "pairs": 512,
        "queries": 64,
        "labels": {
          "E": 371,
          "S": 100,
          "I": 35,
          "C": 6
        },
        "max_tokens": 461
      }
    },
    "excluded": {
      "train": 0,
      "calibration": 0,
      "test": 0
    },
    "demo_queries": [
      "mermaid party",
      "womens running shoes wide",
      "light blue curtains",
      "youth golf",
      "teen girl clothes 13 years old",
      "seachem flourish excel",
      "microsoft xbox one controller",
      "outdoor urns planters large"
    ],
    "demo_ids": [
      "esci-1357344",
      "esci-2206375",
      "esci-2206362",
      "esci-1227774",
      "esci-1507800",
      "esci-1357335",
      "esci-1227767",
      "esci-2206370",
      "esci-1357337",
      "esci-1343181",
      "esci-2245683",
      "esci-1788200",
      "esci-2245689",
      "esci-1227775",
      "esci-1977961",
      "esci-2206363",
      "esci-1343182",
      "esci-2245687",
      "esci-1977967",
      "esci-1343168",
      "esci-1977956",
      "esci-1227772",
      "esci-1357336",
      "esci-1357341",
      "esci-1788197",
      "esci-1788201",
      "esci-1788189",
      "esci-1343175",
      "esci-1343169",
      "esci-1977957",
      "esci-1343174",
      "esci-1343171",
      "esci-1227769",
      "esci-1507795",
      "esci-1788188",
      "esci-2245692",
      "esci-1507806",
      "esci-2206372",
      "esci-2206369",
      "esci-1343167",
      "esci-1788198",
      "esci-1788202",
      "esci-1977966",
      "esci-1977960",
      "esci-1977959",
      "esci-1507792",
      "esci-1507803",
      "esci-2206364",
      "esci-1357334",
      "esci-1227764",
      "esci-2206366",
      "esci-1507793",
      "esci-2245679",
      "esci-1507798",
      "esci-1227766",
      "esci-1507799",
      "esci-1357339",
      "esci-2245685",
      "esci-1357340",
      "esci-2245688",
      "esci-2245684",
      "esci-1788192",
      "esci-1227777",
      "esci-1977958"
    ],
    "product_overlap_train_test": 4
  }
}