Emil_Shanaty/micro-scout
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

docs: publish laptop training results and model card

Browse Source
This commit is contained in:
emil28092005
2026-09-16 04:31:50 +03:00
parent 424aba1d5f
commit f49400932b
12 changed files with 734 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files
reports/laptop-v1/README.md
+16
View File
@@ -0,0 +1,16 @@
# Laptop experiment v1
Recorded September 16, 2026. See the [model card](../../docs/MODEL_CARD.md) for interpretation and limits.
- `data-manifest.json`: pinned sources, split sizes, rejection counts, checksums, overlap audit.
- `data-quality.json`: preparation checks and training-only sample statistics.
- `frozen-experiment.json`: checkpoint and retrieval settings frozen before test evaluation.
- `training-result.json`, `training-curve.jsonl`: actual local run and checkpoint-selection history.
- `validation-metrics.json`, `test-metrics.json`: same-candidate comparison of all five retrieval variants.
- `validation-bm25.json`: earlier standalone baseline, consistent with the complete validation comparison.
- `environment.json`: laptop hardware and installed package versions.
- `index-build.json`, `latency-cpu.json`, `mcp-smoke.json`: development-index timing and actual MCP integration checks.
Public reports replace the local repository root with `.`. Per-query evaluation ranks, raw data, logs, and checkpoints remain in ignored local directories. They are regenerated by the documented commands; binary weights are not included in Git.
The training result's elapsed time begins after model loading and tokenization. Latency is for a small repository and a warm sequential workload. Neither number estimates full setup time or performance under concurrent load.
reports/laptop-v1/environment.json
+25
View File
@@ -0,0 +1,25 @@
{
"cpu": "Intel Core i7-12650H",
"logical_cpus": 16,
"ram_gib": 30.965343475341797,
"gpu": "NVIDIA GeForce RTX 3050 Laptop GPU",
"gpu_vram_mib": 4096,
"cuda_driver": "580.178.04",
"training_cuda_runtime": "12.6",
"python": "3.12.13",
"packages": {
"torch": "2.7.1",
"transformers": "4.57.6",
"numpy": "2.5.3",
"safetensors": "0.8.0",
"mcp": "1.30.0",
"pyarrow": "23.0.1",
"huggingface-hub": "0.36.2",
"pytest": "9.1.1",
"ruff": "0.16.7"
},
"lockfile": "uv.lock",
"teacher_api_calls": 0,
"training_runs": 1,
"evaluation_note": "One seed; repository-cluster bootstrap does not estimate training-seed variation"
}
reports/laptop-v1/frozen-experiment.json
+52
View File
@@ -0,0 +1,52 @@
{
"frozen_at_utc": "2026-09-16T01:26:51.217051+00:00",
"source_commit": "424aba1d5f50d1aa316009333a2c5cb09b6ca965",
"selected_checkpoint": "runs/minilm-v1/best",
"selected_batch_step": 1750,
"selection": "maximum MRR on fixed 512 validation pairs",
"default_retrieval_mode": "dense",
"default_mode_reason": "Highest full-validation MRR; fixed 50/50 hybrid underperformed dense",
"hybrid": {
"dense_weight": 0.5,
"per_method_limit": 100,
"k": 60,
"tuned": false
},
"training_runs": 1,
"seeds": [
17
],
"test_evaluated_before_freeze": false,
"validation_sha256": "efc252634008c16dcf720033a2d5eda7c03629cd372b00960e173cbdc970de2e",
"model_fingerprint": "2bb5197791e4a0caf85af4e92c7ad9f2394cc4c01dafa0b0b18d7e32f0fc5697",
"checkpoint_files": {
"config.json": {
"sha256": "4599e8a5d74ed192b70919aa02124c2d68580070b22e89db956c0353618bccd9",
"bytes": 611
},
"model.safetensors": {
"sha256": "3141c072bf1f4d0ba9bfcf10f1c1744f3174993989c5a2728ec3e63bb86a4450",
"bytes": 90864192
},
"scout_config.json": {
"sha256": "40c2ddc9a67adb72d64e0d0d4108b64259e22deb3b3d482b9039963752590442",
"bytes": 289
},
"special_tokens_map.json": {
"sha256": "5d5b662e421ea9fac075174bb0688ee0d9431699900b90662acd44b2a350503a",
"bytes": 695
},
"tokenizer.json": {
"sha256": "851ca67100d372ca3ae031a6abd168f53489eebfd7d89523f35c5c9b4d372c3c",
"bytes": 711649
},
"tokenizer_config.json": {
"sha256": "ccb4eb21a03e1442ee5c3f85431b9c307960a04579942537d74778dc8080a48c",
"bytes": 1464
},
"vocab.txt": {
"sha256": "07eced375cec144d27c900241f3e339478dec958f92fddbc551f295c992038a3",
"bytes": 231508
}
}
}
reports/laptop-v1/index-build.json
+14
View File
@@ -0,0 +1,14 @@
{
"schema_version": 1,
"root": ".",
"snapshot": "e532e1cccfaf6de0f186e39768e035f0f2982121762a5fc8b6cc1c681c6f0f3d",
"created_at_unix": 1789522014.2341716,
"files": 27,
"symbols": 199,
"edges": 93,
"encoder_fingerprint": "2bb5197791e4a0caf85af4e92c7ad9f2394cc4c01dafa0b0b18d7e32f0fc5697",
"dimension": 384,
"reused_embeddings": 0,
"warnings": [],
"build_seconds": 13.303038476999063
}
reports/laptop-v1/latency-cpu.json
+17
View File
@@ -0,0 +1,17 @@
{
"scope": "warm complete harness search including selected-file verification",
"iterations": 100,
"query_count": 5,
"mode": "dense",
"device": "cpu",
"threads": 4,
"symbols": 199,
"startup_ms": 6352.995459998056,
"median_ms": 18.2878574996721,
"p95_ms": 23.77733434896072,
"min_ms": 14.119207000476308,
"max_ms": 29.591208000056213,
"snapshot": "e532e1cccfaf6de0f186e39768e035f0f2982121762a5fc8b6cc1c681c6f0f3d",
"model_fingerprint": "2bb5197791e4a0caf85af4e92c7ad9f2394cc4c01dafa0b0b18d7e32f0fc5697",
"note": "Five repeated development queries; not a production workload estimate"
}
reports/laptop-v1/mcp-smoke.json
+105
View File
@@ -0,0 +1,105 @@
{
"scope": "Author-written own-repository integration smoke; not an independent quality benchmark",
"device": "cpu",
"queries": [
{
"query": "validate the source file hash before returning a code reference",
"expected_path": "src/micro_scout/scout.py",
"expected_file_rank_in_top_6": 4,
"rpc_ms": 31.78067699991516,
"harness_ms": 21.872625999094453,
"references": [
"src/micro_scout/data.py:102-133",
"tests/test_retrieval.py:242-255",
"src/micro_scout/download.py:41-71",
"src/micro_scout/scout.py:43-72",
"src/micro_scout/train.py:72-103",
"src/micro_scout/text.py:56-76"
]
},
{
"query": "compute normalized embeddings with attention masked mean pooling",
"expected_path": "src/micro_scout/encoder.py",
"expected_file_rank_in_top_6": 1,
"rpc_ms": 34.01597700212733,
"harness_ms": 24.61087999836309,
"references": [
"src/micro_scout/encoder.py:95-103",
"src/micro_scout/encoder.py:105-118",
"tests/test_encoder.py:61-71",
"src/micro_scout/encoder.py:41-72",
"src/micro_scout/evaluate.py:90-121",
"src/micro_scout/lexical.py:51-65"
]
},
{
"query": "save training checkpoint with optimizer and random generator states",
"expected_path": "src/micro_scout/train.py",
"expected_file_rank_in_top_6": 1,
"rpc_ms": 32.459801997902105,
"harness_ms": 22.04264600004535,
"references": [
"src/micro_scout/train.py:144-175",
"src/micro_scout/train.py:240-271",
"src/micro_scout/train.py:192-223",
"src/micro_scout/encoder.py:120-126",
"src/micro_scout/train.py:24-55",
"tests/test_encoder.py:74-121"
]
},
{
"query": "remove docstrings and comments from Python code",
"expected_path": "src/micro_scout/text.py",
"expected_file_rank_in_top_6": 1,
"rpc_ms": 30.592275001254166,
"harness_ms": 20.372359002067242,
"references": [
"src/micro_scout/text.py:17-53",
"tests/test_text.py:21-30",
"tests/test_text.py:33-35",
"src/micro_scout/text.py:79-85",
"tests/test_text.py:7-18",
"src/micro_scout/encoder.py:41-72"
]
},
{
"query": "combine lexical and neural search rankings",
"expected_path": "src/micro_scout/lexical.py",
"expected_file_rank_in_top_6": 2,
"rpc_ms": 34.242352998262504,
"harness_ms": 23.09402000173577,
"references": [
"src/micro_scout/scout.py:126-157",
"src/micro_scout/lexical.py:51-65",
"src/micro_scout/lexical.py:35-41",
"src/micro_scout/evaluate.py:90-121",
"tests/test_retrieval.py:50-53",
"src/micro_scout/cli.py:60-91"
]
}
],
"process_start_to_initialized_ms": 7326.935010001762,
"tools": [
"scout_search",
"scout_read",
"scout_status",
"scout_feedback"
],
"lexical_mode_override_passed": true,
"metadata": {
"schema_version": 1,
"root": ".",
"snapshot": "e532e1cccfaf6de0f186e39768e035f0f2982121762a5fc8b6cc1c681c6f0f3d",
"created_at_unix": 1789522014.2341716,
"files": 27,
"symbols": 199,
"edges": 93,
"encoder_fingerprint": "2bb5197791e4a0caf85af4e92c7ad9f2394cc4c01dafa0b0b18d7e32f0fc5697",
"dimension": 384,
"reused_embeddings": 0,
"warnings": [],
"build_seconds": 13.303038476999063
},
"child_peak_rss_mib": 656.5625,
"all_returned_references_verified": true
}
reports/laptop-v1/test-metrics.json
+126
View File
@@ -0,0 +1,126 @@
{
"split": "test",
"queries": 3000,
"candidates_per_query": 3000,
"repositories": 444,
"dataset_sha256": "b9e150337e195838efb5fe3f4ecafd63dfa99a55cf3635764c893912f4747596",
"evaluator_source_sha256": "29c942b9cbfff21ba8514511e27752527f0ec5e2cad5fbf57ec3b220fc7acd0b",
"fusion": {
"dense_weight": 0.5,
"per_method_limit": 100,
"k": 60
},
"evaluation_ids_sha256": "27c4062864258ea8f93c3e35b1ef6f65038dc24d15673afc930168dec349b3ba",
"protocol": "one labeled positive per query; all split snippets are candidates; docstrings removed",
"limitations": [
"Other semantically correct snippets may be counted as negatives",
"This is function retrieval, not a bug-fixing or multi-file context benchmark",
"No downstream Astra task-success evaluation has been run",
"Base-model pretraining contamination cannot be excluded"
],
"models": {
"pretrained": {
"fingerprint": "78f9f86bd7f71186560e631f0ed43678868c5d39ec07512adc52bc9185009f82",
"parameters": 22713216,
"config": {
"format_version": 1,
"base_model": "sentence-transformers/all-MiniLM-L6-v2",
"revision": "1110a243fdf4706b3f48f1d95db1a4f5529b4d41",
"max_length": 256,
"query_length": 96,
"pooling": "attention-masked-mean-l2",
"code_normalization": "strip-python-docstrings-comments-v1"
},
"embedding_seconds": 14.739996934000374
},
"trained": {
"fingerprint": "2bb5197791e4a0caf85af4e92c7ad9f2394cc4c01dafa0b0b18d7e32f0fc5697",
"parameters": 22713216,
"config": {
"format_version": 1,
"base_model": "sentence-transformers/all-MiniLM-L6-v2",
"revision": "1110a243fdf4706b3f48f1d95db1a4f5529b4d41",
"max_length": 256,
"query_length": 96,
"pooling": "attention-masked-mean-l2",
"code_normalization": "strip-python-docstrings-comments-v1"
},
"embedding_seconds": 14.511412101001042
}
},
"metrics": {
"bm25": {
"mrr": 0.559536732255293,
"mrr_at_10": 0.5529096560846561,
"recall_at_1": 0.4673333333333333,
"recall_at_5": 0.668,
"recall_at_10": 0.7263333333333334,
"median_rank": 2.0,
"macro_repository_mrr": 0.5716105643053901
},
"pretrained_dense": {
"mrr": 0.6819148989769989,
"mrr_at_10": 0.6770041005291004,
"recall_at_1": 0.587,
"recall_at_5": 0.7976666666666666,
"recall_at_10": 0.8523333333333334,
"median_rank": 1.0,
"macro_repository_mrr": 0.6956670635572176
},
"pretrained_hybrid": {
"mrr": 0.6879723629119775,
"mrr_at_10": 0.6825018518518519,
"recall_at_1": 0.5963333333333334,
"recall_at_5": 0.796,
"recall_at_10": 0.8496666666666667,
"median_rank": 1.0,
"macro_repository_mrr": 0.6997776730651677
},
"trained_dense": {
"mrr": 0.7784403489523588,
"mrr_at_10": 0.7752448412698412,
"recall_at_1": 0.701,
"recall_at_5": 0.8746666666666667,
"recall_at_10": 0.9156666666666666,
"median_rank": 1.0,
"macro_repository_mrr": 0.7791868690119261
},
"trained_hybrid": {
"mrr": 0.7220843512885813,
"mrr_at_10": 0.7164558201058201,
"recall_at_1": 0.6406666666666667,
"recall_at_5": 0.815,
"recall_at_10": 0.8663333333333333,
"median_rank": 1.0,
"macro_repository_mrr": 0.7273184680219402
}
},
"trained_vs_pretrained_dense_mrr": {
"delta": 0.09652544997535988,
"ci95": [
0.0848868558350596,
0.10748017710804628
],
"method": "paired repository-cluster bootstrap, 2000 resamples",
"repository_clusters": 444
},
"trained_hybrid_vs_bm25_mrr": {
"delta": 0.16254761903328815,
"ci95": [
0.150834640125844,
0.1763703573684562
],
"method": "paired repository-cluster bootstrap, 2000 resamples",
"repository_clusters": 444
},
"trained_dense_vs_bm25_mrr": {
"delta": 0.21890361669706576,
"ci95": [
0.2004503288862072,
0.23732116908748563
],
"method": "paired repository-cluster bootstrap, 2000 resamples",
"repository_clusters": 444
},
"elapsed_seconds": 39.51678221300244
}
reports/laptop-v1/training-curve.jsonl
+114
View File
@@ -0,0 +1,114 @@
{"step": 0, "elapsed_seconds": 2.688, "event": "validation", "candidates": 512, "mrr": 0.8158749848852709, "mrr_at_10": 0.8132843501984127, "recall_at_1": 0.7421875, "recall_at_5": 0.900390625, "recall_at_10": 0.93359375, "median_rank": 1.0}
{"step": 25, "elapsed_seconds": 6.592, "event": "train", "epoch": 1, "loss": 0.2999372160434723, "learning_rate": 2.0000000000000003e-06, "peak_vram_mb": 1167.20947265625, "optimizer_steps": 25, "skipped_optimizer_steps": 0}
{"step": 50, "elapsed_seconds": 9.662, "event": "train", "epoch": 1, "loss": 0.2673264843225479, "learning_rate": 4.000000000000001e-06, "peak_vram_mb": 1167.20947265625, "optimizer_steps": 50, "skipped_optimizer_steps": 0}
{"step": 75, "elapsed_seconds": 12.727, "event": "train", "epoch": 1, "loss": 0.19812003761529923, "learning_rate": 6e-06, "peak_vram_mb": 1167.20947265625, "optimizer_steps": 75, "skipped_optimizer_steps": 0}
{"step": 100, "elapsed_seconds": 15.81, "event": "train", "epoch": 1, "loss": 0.2650404736399651, "learning_rate": 8.000000000000001e-06, "peak_vram_mb": 1167.20947265625, "optimizer_steps": 100, "skipped_optimizer_steps": 0}
{"step": 125, "elapsed_seconds": 18.924, "event": "train", "epoch": 1, "loss": 0.21082983404397965, "learning_rate": 1e-05, "peak_vram_mb": 1167.20947265625, "optimizer_steps": 125, "skipped_optimizer_steps": 0}
{"step": 150, "elapsed_seconds": 21.962, "event": "train", "epoch": 1, "loss": 0.15117341600358486, "learning_rate": 1.2e-05, "peak_vram_mb": 1167.20947265625, "optimizer_steps": 150, "skipped_optimizer_steps": 0}
{"step": 175, "elapsed_seconds": 25.019, "event": "train", "epoch": 1, "loss": 0.17314316660165788, "learning_rate": 1.4e-05, "peak_vram_mb": 1167.20947265625, "optimizer_steps": 175, "skipped_optimizer_steps": 0}
{"step": 200, "elapsed_seconds": 28.109, "event": "train", "epoch": 1, "loss": 0.1510033105313778, "learning_rate": 1.6000000000000003e-05, "peak_vram_mb": 1167.20947265625, "optimizer_steps": 200, "skipped_optimizer_steps": 0}
{"step": 225, "elapsed_seconds": 31.203, "event": "train", "epoch": 1, "loss": 0.1837605682015419, "learning_rate": 1.8e-05, "peak_vram_mb": 1167.20947265625, "optimizer_steps": 225, "skipped_optimizer_steps": 0}
{"step": 250, "elapsed_seconds": 34.245, "event": "train", "epoch": 1, "loss": 0.1343583283573389, "learning_rate": 2e-05, "peak_vram_mb": 1167.20947265625, "optimizer_steps": 250, "skipped_optimizer_steps": 0}
{"step": 250, "elapsed_seconds": 36.339, "event": "validation", "candidates": 512, "mrr": 0.8700161731325496, "mrr_at_10": 0.868243892609127, "recall_at_1": 0.81640625, "recall_at_5": 0.931640625, "recall_at_10": 0.958984375, "median_rank": 1.0}
{"step": 275, "elapsed_seconds": 40.52, "event": "train", "epoch": 1, "loss": 0.15882841497659683, "learning_rate": 1.977777777777778e-05, "peak_vram_mb": 1167.20947265625, "optimizer_steps": 275, "skipped_optimizer_steps": 0}
{"step": 300, "elapsed_seconds": 43.643, "event": "train", "epoch": 1, "loss": 0.17688855156302452, "learning_rate": 1.9555555555555557e-05, "peak_vram_mb": 1167.20947265625, "optimizer_steps": 300, "skipped_optimizer_steps": 0}
{"step": 325, "elapsed_seconds": 46.748, "event": "train", "epoch": 1, "loss": 0.18821210134774446, "learning_rate": 1.9333333333333333e-05, "peak_vram_mb": 1167.20947265625, "optimizer_steps": 325, "skipped_optimizer_steps": 0}
{"step": 350, "elapsed_seconds": 49.885, "event": "train", "epoch": 1, "loss": 0.17427126467227935, "learning_rate": 1.9111111111111113e-05, "peak_vram_mb": 1167.20947265625, "optimizer_steps": 350, "skipped_optimizer_steps": 0}
{"step": 375, "elapsed_seconds": 53.029, "event": "train", "epoch": 1, "loss": 0.14787629939615726, "learning_rate": 1.888888888888889e-05, "peak_vram_mb": 1167.20947265625, "optimizer_steps": 375, "skipped_optimizer_steps": 0}
{"step": 400, "elapsed_seconds": 56.159, "event": "train", "epoch": 1, "loss": 0.1515346374362707, "learning_rate": 1.866666666666667e-05, "peak_vram_mb": 1167.20947265625, "optimizer_steps": 400, "skipped_optimizer_steps": 0}
{"step": 425, "elapsed_seconds": 59.313, "event": "train", "epoch": 1, "loss": 0.1394197326898575, "learning_rate": 1.8444444444444448e-05, "peak_vram_mb": 1167.20947265625, "optimizer_steps": 425, "skipped_optimizer_steps": 0}
{"step": 450, "elapsed_seconds": 62.471, "event": "train", "epoch": 1, "loss": 0.16173378251492976, "learning_rate": 1.8222222222222224e-05, "peak_vram_mb": 1167.20947265625, "optimizer_steps": 450, "skipped_optimizer_steps": 0}
{"step": 475, "elapsed_seconds": 65.579, "event": "train", "epoch": 1, "loss": 0.1569698416441679, "learning_rate": 1.8e-05, "peak_vram_mb": 1167.20947265625, "optimizer_steps": 475, "skipped_optimizer_steps": 0}
{"step": 500, "elapsed_seconds": 68.708, "event": "train", "epoch": 1, "loss": 0.17313105337321757, "learning_rate": 1.7777777777777777e-05, "peak_vram_mb": 1167.20947265625, "optimizer_steps": 500, "skipped_optimizer_steps": 0}
{"step": 500, "elapsed_seconds": 70.821, "event": "validation", "candidates": 512, "mrr": 0.8770863142570198, "mrr_at_10": 0.8752534412202382, "recall_at_1": 0.828125, "recall_at_5": 0.935546875, "recall_at_10": 0.958984375, "median_rank": 1.0}
{"step": 525, "elapsed_seconds": 75.108, "event": "train", "epoch": 1, "loss": 0.15202117685228586, "learning_rate": 1.7555555555555556e-05, "peak_vram_mb": 1167.20947265625, "optimizer_steps": 525, "skipped_optimizer_steps": 0}
{"step": 550, "elapsed_seconds": 78.161, "event": "train", "epoch": 1, "loss": 0.19238240577280522, "learning_rate": 1.7333333333333336e-05, "peak_vram_mb": 1167.20947265625, "optimizer_steps": 550, "skipped_optimizer_steps": 0}
{"step": 575, "elapsed_seconds": 81.259, "event": "train", "epoch": 1, "loss": 0.107438475638628, "learning_rate": 1.7111111111111112e-05, "peak_vram_mb": 1167.20947265625, "optimizer_steps": 575, "skipped_optimizer_steps": 0}
{"step": 600, "elapsed_seconds": 84.394, "event": "train", "epoch": 1, "loss": 0.16651105839759112, "learning_rate": 1.688888888888889e-05, "peak_vram_mb": 1167.20947265625, "optimizer_steps": 600, "skipped_optimizer_steps": 0}
{"step": 625, "elapsed_seconds": 87.529, "event": "train", "epoch": 1, "loss": 0.16286010384559632, "learning_rate": 1.6666666666666667e-05, "peak_vram_mb": 1167.20947265625, "optimizer_steps": 625, "skipped_optimizer_steps": 0}
{"step": 650, "elapsed_seconds": 90.641, "event": "train", "epoch": 1, "loss": 0.17467676237225532, "learning_rate": 1.6444444444444444e-05, "peak_vram_mb": 1167.20947265625, "optimizer_steps": 650, "skipped_optimizer_steps": 0}
{"step": 675, "elapsed_seconds": 93.744, "event": "train", "epoch": 1, "loss": 0.1901424279808998, "learning_rate": 1.6222222222222223e-05, "peak_vram_mb": 1167.20947265625, "optimizer_steps": 675, "skipped_optimizer_steps": 0}
{"step": 700, "elapsed_seconds": 96.891, "event": "train", "epoch": 1, "loss": 0.17884070463478566, "learning_rate": 1.6000000000000003e-05, "peak_vram_mb": 1167.20947265625, "optimizer_steps": 700, "skipped_optimizer_steps": 0}
{"step": 725, "elapsed_seconds": 99.989, "event": "train", "epoch": 1, "loss": 0.16399568542838097, "learning_rate": 1.577777777777778e-05, "peak_vram_mb": 1167.20947265625, "optimizer_steps": 725, "skipped_optimizer_steps": 0}
{"step": 750, "elapsed_seconds": 103.036, "event": "train", "epoch": 1, "loss": 0.18295324414968492, "learning_rate": 1.555555555555556e-05, "peak_vram_mb": 1167.20947265625, "optimizer_steps": 750, "skipped_optimizer_steps": 0}
{"step": 750, "elapsed_seconds": 105.154, "event": "validation", "candidates": 512, "mrr": 0.8834634371924863, "mrr_at_10": 0.881622798859127, "recall_at_1": 0.837890625, "recall_at_5": 0.9375, "recall_at_10": 0.9609375, "median_rank": 1.0}
{"step": 775, "elapsed_seconds": 110.203, "event": "train", "epoch": 1, "loss": 0.14178925782442092, "learning_rate": 1.5333333333333334e-05, "peak_vram_mb": 1167.20947265625, "optimizer_steps": 775, "skipped_optimizer_steps": 0}
{"step": 800, "elapsed_seconds": 113.413, "event": "train", "epoch": 1, "loss": 0.1517408673465252, "learning_rate": 1.5111111111111112e-05, "peak_vram_mb": 1167.20947265625, "optimizer_steps": 800, "skipped_optimizer_steps": 0}
{"step": 825, "elapsed_seconds": 116.497, "event": "train", "epoch": 1, "loss": 0.11933087974786759, "learning_rate": 1.488888888888889e-05, "peak_vram_mb": 1167.20947265625, "optimizer_steps": 825, "skipped_optimizer_steps": 0}
{"step": 850, "elapsed_seconds": 119.608, "event": "train", "epoch": 1, "loss": 0.18476217068731784, "learning_rate": 1.4666666666666666e-05, "peak_vram_mb": 1167.20947265625, "optimizer_steps": 850, "skipped_optimizer_steps": 0}
{"step": 875, "elapsed_seconds": 122.727, "event": "train", "epoch": 1, "loss": 0.15981429502367972, "learning_rate": 1.4444444444444446e-05, "peak_vram_mb": 1167.20947265625, "optimizer_steps": 875, "skipped_optimizer_steps": 0}
{"step": 900, "elapsed_seconds": 125.873, "event": "train", "epoch": 1, "loss": 0.13738767936825752, "learning_rate": 1.4222222222222224e-05, "peak_vram_mb": 1167.20947265625, "optimizer_steps": 900, "skipped_optimizer_steps": 0}
{"step": 925, "elapsed_seconds": 128.995, "event": "train", "epoch": 1, "loss": 0.15457843832671642, "learning_rate": 1.4e-05, "peak_vram_mb": 1167.20947265625, "optimizer_steps": 925, "skipped_optimizer_steps": 0}
{"step": 950, "elapsed_seconds": 132.124, "event": "train", "epoch": 1, "loss": 0.12165915101766586, "learning_rate": 1.377777777777778e-05, "peak_vram_mb": 1167.20947265625, "optimizer_steps": 950, "skipped_optimizer_steps": 0}
{"step": 975, "elapsed_seconds": 135.283, "event": "train", "epoch": 1, "loss": 0.13803807649761438, "learning_rate": 1.3555555555555557e-05, "peak_vram_mb": 1167.20947265625, "optimizer_steps": 975, "skipped_optimizer_steps": 0}
{"step": 1000, "elapsed_seconds": 138.438, "event": "train", "epoch": 1, "loss": 0.11398605667054654, "learning_rate": 1.3333333333333333e-05, "peak_vram_mb": 1167.20947265625, "optimizer_steps": 1000, "skipped_optimizer_steps": 0}
{"step": 1000, "elapsed_seconds": 140.603, "event": "validation", "candidates": 512, "mrr": 0.8916671465499062, "mrr_at_10": 0.8894763764880953, "recall_at_1": 0.849609375, "recall_at_5": 0.943359375, "recall_at_10": 0.95703125, "median_rank": 1.0}
{"step": 1025, "elapsed_seconds": 145.349, "event": "train", "epoch": 1, "loss": 0.13659145399928094, "learning_rate": 1.3111111111111113e-05, "peak_vram_mb": 1167.20947265625, "optimizer_steps": 1025, "skipped_optimizer_steps": 0}
{"step": 1050, "elapsed_seconds": 148.455, "event": "train", "epoch": 1, "loss": 0.15747894018888473, "learning_rate": 1.288888888888889e-05, "peak_vram_mb": 1167.20947265625, "optimizer_steps": 1050, "skipped_optimizer_steps": 0}
{"step": 1075, "elapsed_seconds": 151.535, "event": "train", "epoch": 1, "loss": 0.1223088663816452, "learning_rate": 1.2666666666666667e-05, "peak_vram_mb": 1167.20947265625, "optimizer_steps": 1075, "skipped_optimizer_steps": 0}
{"step": 1100, "elapsed_seconds": 154.591, "event": "train", "epoch": 1, "loss": 0.11804232776165008, "learning_rate": 1.2444444444444446e-05, "peak_vram_mb": 1167.20947265625, "optimizer_steps": 1100, "skipped_optimizer_steps": 0}
{"step": 1125, "elapsed_seconds": 157.667, "event": "train", "epoch": 1, "loss": 0.1450914055109024, "learning_rate": 1.2222222222222224e-05, "peak_vram_mb": 1167.20947265625, "optimizer_steps": 1125, "skipped_optimizer_steps": 0}
{"step": 1150, "elapsed_seconds": 160.717, "event": "train", "epoch": 1, "loss": 0.17949952512979508, "learning_rate": 1.2e-05, "peak_vram_mb": 1167.20947265625, "optimizer_steps": 1150, "skipped_optimizer_steps": 0}
{"step": 1175, "elapsed_seconds": 163.807, "event": "train", "epoch": 1, "loss": 0.17356954403221608, "learning_rate": 1.177777777777778e-05, "peak_vram_mb": 1167.20947265625, "optimizer_steps": 1175, "skipped_optimizer_steps": 0}
{"step": 1200, "elapsed_seconds": 166.905, "event": "train", "epoch": 1, "loss": 0.17497427016496658, "learning_rate": 1.1555555555555556e-05, "peak_vram_mb": 1167.20947265625, "optimizer_steps": 1200, "skipped_optimizer_steps": 0}
{"step": 1225, "elapsed_seconds": 170.051, "event": "train", "epoch": 1, "loss": 0.1488281112909317, "learning_rate": 1.1333333333333334e-05, "peak_vram_mb": 1167.20947265625, "optimizer_steps": 1225, "skipped_optimizer_steps": 0}
{"step": 1250, "elapsed_seconds": 173.234, "event": "train", "epoch": 1, "loss": 0.1613723538815975, "learning_rate": 1.1111111111111113e-05, "peak_vram_mb": 1167.20947265625, "optimizer_steps": 1250, "skipped_optimizer_steps": 0}
{"step": 1250, "elapsed_seconds": 175.354, "event": "validation", "candidates": 512, "mrr": 0.8890644173403563, "mrr_at_10": 0.8870628720238095, "recall_at_1": 0.845703125, "recall_at_5": 0.939453125, "recall_at_10": 0.95703125, "median_rank": 1.0}
{"step": 1250, "elapsed_seconds": 178.559, "event": "validation", "candidates": 512, "mrr": 0.8890644173403563, "mrr_at_10": 0.8870628720238095, "recall_at_1": 0.845703125, "recall_at_5": 0.939453125, "recall_at_10": 0.95703125, "median_rank": 1.0}
{"step": 1275, "elapsed_seconds": 182.828, "event": "train", "epoch": 2, "loss": 0.09954056203365326, "learning_rate": 1.088888888888889e-05, "peak_vram_mb": 1167.20947265625, "optimizer_steps": 1275, "skipped_optimizer_steps": 0}
{"step": 1300, "elapsed_seconds": 185.942, "event": "train", "epoch": 2, "loss": 0.07810468420386314, "learning_rate": 1.0666666666666667e-05, "peak_vram_mb": 1167.20947265625, "optimizer_steps": 1300, "skipped_optimizer_steps": 0}
{"step": 1325, "elapsed_seconds": 189.074, "event": "train", "epoch": 2, "loss": 0.09669223334640265, "learning_rate": 1.0444444444444445e-05, "peak_vram_mb": 1167.20947265625, "optimizer_steps": 1325, "skipped_optimizer_steps": 0}
{"step": 1350, "elapsed_seconds": 192.15, "event": "train", "epoch": 2, "loss": 0.0731896192021668, "learning_rate": 1.0222222222222223e-05, "peak_vram_mb": 1167.20947265625, "optimizer_steps": 1350, "skipped_optimizer_steps": 0}
{"step": 1375, "elapsed_seconds": 195.288, "event": "train", "epoch": 2, "loss": 0.10116158414632082, "learning_rate": 1e-05, "peak_vram_mb": 1167.20947265625, "optimizer_steps": 1375, "skipped_optimizer_steps": 0}
{"step": 1400, "elapsed_seconds": 198.387, "event": "train", "epoch": 2, "loss": 0.08757915049791336, "learning_rate": 9.777777777777779e-06, "peak_vram_mb": 1167.20947265625, "optimizer_steps": 1400, "skipped_optimizer_steps": 0}
{"step": 1425, "elapsed_seconds": 201.434, "event": "train", "epoch": 2, "loss": 0.1208132740110159, "learning_rate": 9.555555555555556e-06, "peak_vram_mb": 1167.20947265625, "optimizer_steps": 1425, "skipped_optimizer_steps": 0}
{"step": 1450, "elapsed_seconds": 204.596, "event": "train", "epoch": 2, "loss": 0.10880279898643494, "learning_rate": 9.333333333333334e-06, "peak_vram_mb": 1167.20947265625, "optimizer_steps": 1450, "skipped_optimizer_steps": 0}
{"step": 1475, "elapsed_seconds": 207.713, "event": "train", "epoch": 2, "loss": 0.09981458291411399, "learning_rate": 9.111111111111112e-06, "peak_vram_mb": 1167.20947265625, "optimizer_steps": 1475, "skipped_optimizer_steps": 0}
{"step": 1500, "elapsed_seconds": 210.837, "event": "train", "epoch": 2, "loss": 0.12432493068277836, "learning_rate": 8.888888888888888e-06, "peak_vram_mb": 1167.20947265625, "optimizer_steps": 1500, "skipped_optimizer_steps": 0}
{"step": 1500, "elapsed_seconds": 212.954, "event": "validation", "candidates": 512, "mrr": 0.8906939674739834, "mrr_at_10": 0.8888826884920635, "recall_at_1": 0.8515625, "recall_at_5": 0.94140625, "recall_at_10": 0.9609375, "median_rank": 1.0}
{"step": 1525, "elapsed_seconds": 217.014, "event": "train", "epoch": 2, "loss": 0.10201668687164783, "learning_rate": 8.666666666666668e-06, "peak_vram_mb": 1167.20947265625, "optimizer_steps": 1525, "skipped_optimizer_steps": 0}
{"step": 1550, "elapsed_seconds": 220.125, "event": "train", "epoch": 2, "loss": 0.1103788635879755, "learning_rate": 8.444444444444446e-06, "peak_vram_mb": 1167.20947265625, "optimizer_steps": 1550, "skipped_optimizer_steps": 0}
{"step": 1575, "elapsed_seconds": 223.222, "event": "train", "epoch": 2, "loss": 0.06910347186028958, "learning_rate": 8.222222222222222e-06, "peak_vram_mb": 1167.20947265625, "optimizer_steps": 1575, "skipped_optimizer_steps": 0}
{"step": 1600, "elapsed_seconds": 226.334, "event": "train", "epoch": 2, "loss": 0.0826541879028082, "learning_rate": 8.000000000000001e-06, "peak_vram_mb": 1167.20947265625, "optimizer_steps": 1600, "skipped_optimizer_steps": 0}
{"step": 1625, "elapsed_seconds": 229.511, "event": "train", "epoch": 2, "loss": 0.10087273364886641, "learning_rate": 7.77777777777778e-06, "peak_vram_mb": 1167.20947265625, "optimizer_steps": 1625, "skipped_optimizer_steps": 0}
{"step": 1650, "elapsed_seconds": 232.693, "event": "train", "epoch": 2, "loss": 0.10273163206875324, "learning_rate": 7.555555555555556e-06, "peak_vram_mb": 1167.20947265625, "optimizer_steps": 1650, "skipped_optimizer_steps": 0}
{"step": 1675, "elapsed_seconds": 235.795, "event": "train", "epoch": 2, "loss": 0.12350528098642827, "learning_rate": 7.333333333333333e-06, "peak_vram_mb": 1167.20947265625, "optimizer_steps": 1675, "skipped_optimizer_steps": 0}
{"step": 1700, "elapsed_seconds": 238.889, "event": "train", "epoch": 2, "loss": 0.10550767021253706, "learning_rate": 7.111111111111112e-06, "peak_vram_mb": 1167.20947265625, "optimizer_steps": 1700, "skipped_optimizer_steps": 0}
{"step": 1725, "elapsed_seconds": 242.033, "event": "train", "epoch": 2, "loss": 0.08353679748252034, "learning_rate": 6.88888888888889e-06, "peak_vram_mb": 1167.20947265625, "optimizer_steps": 1725, "skipped_optimizer_steps": 0}
{"step": 1750, "elapsed_seconds": 245.136, "event": "train", "epoch": 2, "loss": 0.09644823379814625, "learning_rate": 6.666666666666667e-06, "peak_vram_mb": 1167.20947265625, "optimizer_steps": 1750, "skipped_optimizer_steps": 0}
{"step": 1750, "elapsed_seconds": 247.273, "event": "validation", "candidates": 512, "mrr": 0.8944014436603791, "mrr_at_10": 0.8926750062003967, "recall_at_1": 0.8515625, "recall_at_5": 0.9453125, "recall_at_10": 0.9609375, "median_rank": 1.0}
{"step": 1775, "elapsed_seconds": 251.808, "event": "train", "epoch": 2, "loss": 0.07751014851033687, "learning_rate": 6.444444444444445e-06, "peak_vram_mb": 1167.20947265625, "optimizer_steps": 1775, "skipped_optimizer_steps": 0}
{"step": 1800, "elapsed_seconds": 254.935, "event": "train", "epoch": 2, "loss": 0.11908595895394683, "learning_rate": 6.222222222222223e-06, "peak_vram_mb": 1167.20947265625, "optimizer_steps": 1800, "skipped_optimizer_steps": 0}
{"step": 1825, "elapsed_seconds": 258.091, "event": "train", "epoch": 2, "loss": 0.10733085677027702, "learning_rate": 6e-06, "peak_vram_mb": 1167.20947265625, "optimizer_steps": 1825, "skipped_optimizer_steps": 0}
{"step": 1850, "elapsed_seconds": 261.215, "event": "train", "epoch": 2, "loss": 0.08080575112253427, "learning_rate": 5.777777777777778e-06, "peak_vram_mb": 1167.20947265625, "optimizer_steps": 1850, "skipped_optimizer_steps": 0}
{"step": 1875, "elapsed_seconds": 264.365, "event": "train", "epoch": 2, "loss": 0.09689645860344172, "learning_rate": 5.555555555555557e-06, "peak_vram_mb": 1167.20947265625, "optimizer_steps": 1875, "skipped_optimizer_steps": 0}
{"step": 1900, "elapsed_seconds": 267.447, "event": "train", "epoch": 2, "loss": 0.10643041014671326, "learning_rate": 5.333333333333334e-06, "peak_vram_mb": 1167.20947265625, "optimizer_steps": 1900, "skipped_optimizer_steps": 0}
{"step": 1925, "elapsed_seconds": 270.514, "event": "train", "epoch": 2, "loss": 0.09937104240059852, "learning_rate": 5.1111111111111115e-06, "peak_vram_mb": 1167.20947265625, "optimizer_steps": 1925, "skipped_optimizer_steps": 0}
{"step": 1950, "elapsed_seconds": 273.57, "event": "train", "epoch": 2, "loss": 0.09365111857652664, "learning_rate": 4.888888888888889e-06, "peak_vram_mb": 1167.20947265625, "optimizer_steps": 1950, "skipped_optimizer_steps": 0}
{"step": 1975, "elapsed_seconds": 276.575, "event": "train", "epoch": 2, "loss": 0.10431641653180122, "learning_rate": 4.666666666666667e-06, "peak_vram_mb": 1167.20947265625, "optimizer_steps": 1975, "skipped_optimizer_steps": 0}
{"step": 2000, "elapsed_seconds": 279.615, "event": "train", "epoch": 2, "loss": 0.12314280480146408, "learning_rate": 4.444444444444444e-06, "peak_vram_mb": 1167.20947265625, "optimizer_steps": 2000, "skipped_optimizer_steps": 0}
{"step": 2000, "elapsed_seconds": 281.676, "event": "validation", "candidates": 512, "mrr": 0.8878709272821281, "mrr_at_10": 0.8856112041170634, "recall_at_1": 0.845703125, "recall_at_5": 0.9375, "recall_at_10": 0.955078125, "median_rank": 1.0}
{"step": 2025, "elapsed_seconds": 285.755, "event": "train", "epoch": 2, "loss": 0.10427925728261471, "learning_rate": 4.2311111111111114e-06, "peak_vram_mb": 1167.20947265625, "optimizer_steps": 2024, "skipped_optimizer_steps": 1}
{"step": 2050, "elapsed_seconds": 288.887, "event": "train", "epoch": 2, "loss": 0.07830743595957756, "learning_rate": 4.008888888888889e-06, "peak_vram_mb": 1167.20947265625, "optimizer_steps": 2049, "skipped_optimizer_steps": 1}
{"step": 2075, "elapsed_seconds": 292.038, "event": "train", "epoch": 2, "loss": 0.0889523448050022, "learning_rate": 3.7866666666666667e-06, "peak_vram_mb": 1167.20947265625, "optimizer_steps": 2074, "skipped_optimizer_steps": 1}
{"step": 2100, "elapsed_seconds": 295.165, "event": "train", "epoch": 2, "loss": 0.09483716983348131, "learning_rate": 3.564444444444445e-06, "peak_vram_mb": 1167.20947265625, "optimizer_steps": 2099, "skipped_optimizer_steps": 1}
{"step": 2125, "elapsed_seconds": 298.279, "event": "train", "epoch": 2, "loss": 0.1344515113532543, "learning_rate": 3.3422222222222224e-06, "peak_vram_mb": 1167.20947265625, "optimizer_steps": 2124, "skipped_optimizer_steps": 1}
{"step": 2150, "elapsed_seconds": 301.42, "event": "train", "epoch": 2, "loss": 0.10280320219695568, "learning_rate": 3.12e-06, "peak_vram_mb": 1167.20947265625, "optimizer_steps": 2149, "skipped_optimizer_steps": 1}
{"step": 2175, "elapsed_seconds": 304.531, "event": "train", "epoch": 2, "loss": 0.0891904953122139, "learning_rate": 2.8977777777777785e-06, "peak_vram_mb": 1167.20947265625, "optimizer_steps": 2174, "skipped_optimizer_steps": 1}
{"step": 2200, "elapsed_seconds": 307.664, "event": "train", "epoch": 2, "loss": 0.09797250606119633, "learning_rate": 2.675555555555556e-06, "peak_vram_mb": 1167.20947265625, "optimizer_steps": 2199, "skipped_optimizer_steps": 1}
{"step": 2225, "elapsed_seconds": 310.739, "event": "train", "epoch": 2, "loss": 0.10983087375760078, "learning_rate": 2.4533333333333333e-06, "peak_vram_mb": 1167.20947265625, "optimizer_steps": 2224, "skipped_optimizer_steps": 1}
{"step": 2250, "elapsed_seconds": 313.826, "event": "train", "epoch": 2, "loss": 0.0706938909366727, "learning_rate": 2.2311111111111115e-06, "peak_vram_mb": 1167.20947265625, "optimizer_steps": 2249, "skipped_optimizer_steps": 1}
{"step": 2250, "elapsed_seconds": 316.019, "event": "validation", "candidates": 512, "mrr": 0.8916258561959278, "mrr_at_10": 0.8894880022321429, "recall_at_1": 0.84765625, "recall_at_5": 0.9453125, "recall_at_10": 0.958984375, "median_rank": 1.0}
{"step": 2275, "elapsed_seconds": 320.204, "event": "train", "epoch": 2, "loss": 0.11725177850574255, "learning_rate": 2.008888888888889e-06, "peak_vram_mb": 1167.20947265625, "optimizer_steps": 2274, "skipped_optimizer_steps": 1}
{"step": 2300, "elapsed_seconds": 323.407, "event": "train", "epoch": 2, "loss": 0.08511088168248535, "learning_rate": 1.7866666666666668e-06, "peak_vram_mb": 1167.20947265625, "optimizer_steps": 2299, "skipped_optimizer_steps": 1}
{"step": 2325, "elapsed_seconds": 326.555, "event": "train", "epoch": 2, "loss": 0.092320506144315, "learning_rate": 1.5644444444444446e-06, "peak_vram_mb": 1167.20947265625, "optimizer_steps": 2324, "skipped_optimizer_steps": 1}
{"step": 2350, "elapsed_seconds": 329.743, "event": "train", "epoch": 2, "loss": 0.07981016110628843, "learning_rate": 1.3422222222222222e-06, "peak_vram_mb": 1167.20947265625, "optimizer_steps": 2349, "skipped_optimizer_steps": 1}
{"step": 2375, "elapsed_seconds": 332.908, "event": "train", "epoch": 2, "loss": 0.09189343268983066, "learning_rate": 1.12e-06, "peak_vram_mb": 1167.20947265625, "optimizer_steps": 2374, "skipped_optimizer_steps": 1}
{"step": 2400, "elapsed_seconds": 335.958, "event": "train", "epoch": 2, "loss": 0.09500313613563777, "learning_rate": 8.977777777777778e-07, "peak_vram_mb": 1167.20947265625, "optimizer_steps": 2399, "skipped_optimizer_steps": 1}
{"step": 2425, "elapsed_seconds": 339.022, "event": "train", "epoch": 2, "loss": 0.08238222647458315, "learning_rate": 6.755555555555555e-07, "peak_vram_mb": 1167.20947265625, "optimizer_steps": 2424, "skipped_optimizer_steps": 1}
{"step": 2450, "elapsed_seconds": 342.162, "event": "train", "epoch": 2, "loss": 0.1382678287103772, "learning_rate": 4.533333333333334e-07, "peak_vram_mb": 1167.20947265625, "optimizer_steps": 2449, "skipped_optimizer_steps": 1}
{"step": 2475, "elapsed_seconds": 345.253, "event": "train", "epoch": 2, "loss": 0.10618807382881641, "learning_rate": 2.3111111111111112e-07, "peak_vram_mb": 1167.20947265625, "optimizer_steps": 2474, "skipped_optimizer_steps": 1}
{"step": 2500, "elapsed_seconds": 348.357, "event": "train", "epoch": 2, "loss": 0.09265013325959444, "learning_rate": 8.88888888888889e-09, "peak_vram_mb": 1167.20947265625, "optimizer_steps": 2499, "skipped_optimizer_steps": 1}
{"step": 2500, "elapsed_seconds": 350.536, "event": "validation", "candidates": 512, "mrr": 0.8918919949815127, "mrr_at_10": 0.8899817088293651, "recall_at_1": 0.84765625, "recall_at_5": 0.94140625, "recall_at_10": 0.9609375, "median_rank": 1.0}
{"step": 2500, "elapsed_seconds": 353.779, "event": "validation", "candidates": 512, "mrr": 0.8918919949815127, "mrr_at_10": 0.8899817088293651, "recall_at_1": 0.84765625, "recall_at_5": 0.94140625, "recall_at_10": 0.9609375, "median_rank": 1.0}
{"step": 2500, "elapsed_seconds": 354.815, "event": "complete", "best_validation_mrr": 0.8944014436603791}
reports/laptop-v1/training-result.json
+45
View File
@@ -0,0 +1,45 @@
{
"config": {
"base_model": "sentence-transformers/all-MiniLM-L6-v2",
"revision": "1110a243fdf4706b3f48f1d95db1a4f5529b4d41",
"max_length": 256,
"query_length": 96,
"batch_size": 24,
"epochs": 2,
"learning_rate": 2e-05,
"weight_decay": 0.01,
"temperature": 0.05,
"warmup_ratio": 0.1,
"eval_every": 250,
"seed": 17,
"threads": 4,
"max_minutes": 100,
"mixed_precision": true
},
"dataset_manifest_sha256": "b4a39e8f44d21e3758413a61e5099374dcb90658646b32c39e03d13f9d4f6f85",
"dataset_file_sha256": {
"train": "3eeed3185b74f934462aaccbd3c567f8db20754a1bf6f5f3686779f8b39f4338",
"validation": "e1b934c33f12322e4a56d0d17a987966c40961b035e15939fd716b76d4e017a2"
},
"source_commit": "abffddaa5e54c78a5f86308c4a31bb5cc225c077",
"training_source_sha256": "cf20f6189967cfc88f3849bc3a225f2680c42026d53f204d9d5afc05fe88a2d6",
"training_pairs": 30000,
"validation_pairs_for_selection": 512,
"parameters": 22713216,
"dimension": 384,
"python": "3.12.13",
"torch": "2.7.1+cu126",
"device": "cuda",
"gpu": "NVIDIA GeForce RTX 3050 Laptop GPU",
"objective": "symmetric in-batch contrastive cross-entropy",
"test_set_used_for_selection": false,
"code_at_token_limit_fraction": 0.19296666666666668,
"query_at_token_limit_fraction": 0.008366666666666666,
"batch_steps": 2500,
"optimizer_steps": 2499,
"skipped_optimizer_steps": 1,
"best_validation_mrr": 0.8944014436603791,
"elapsed_seconds": 354.81236612299836,
"stopped_early": false,
"peak_vram_mb": 1167.20947265625
}
reports/laptop-v1/validation-metrics.json
+126
View File
@@ -0,0 +1,126 @@
{
"split": "validation",
"queries": 2000,
"candidates_per_query": 2000,
"repositories": 397,
"dataset_sha256": "e1b934c33f12322e4a56d0d17a987966c40961b035e15939fd716b76d4e017a2",
"evaluator_source_sha256": "29c942b9cbfff21ba8514511e27752527f0ec5e2cad5fbf57ec3b220fc7acd0b",
"fusion": {
"dense_weight": 0.5,
"per_method_limit": 100,
"k": 60
},
"evaluation_ids_sha256": "37d3ee7d31626ab63e5746faac6e8ff2bb117506ddd682b2a71405682196f2ef",
"protocol": "one labeled positive per query; all split snippets are candidates; docstrings removed",
"limitations": [
"Other semantically correct snippets may be counted as negatives",
"This is function retrieval, not a bug-fixing or multi-file context benchmark",
"No downstream Astra task-success evaluation has been run",
"Base-model pretraining contamination cannot be excluded"
],
"models": {
"pretrained": {
"fingerprint": "78f9f86bd7f71186560e631f0ed43678868c5d39ec07512adc52bc9185009f82",
"parameters": 22713216,
"config": {
"format_version": 1,
"base_model": "sentence-transformers/all-MiniLM-L6-v2",
"revision": "1110a243fdf4706b3f48f1d95db1a4f5529b4d41",
"max_length": 256,
"query_length": 96,
"pooling": "attention-masked-mean-l2",
"code_normalization": "strip-python-docstrings-comments-v1"
},
"embedding_seconds": 9.731017789999896
},
"trained": {
"fingerprint": "2bb5197791e4a0caf85af4e92c7ad9f2394cc4c01dafa0b0b18d7e32f0fc5697",
"parameters": 22713216,
"config": {
"format_version": 1,
"base_model": "sentence-transformers/all-MiniLM-L6-v2",
"revision": "1110a243fdf4706b3f48f1d95db1a4f5529b4d41",
"max_length": 256,
"query_length": 96,
"pooling": "attention-masked-mean-l2",
"code_normalization": "strip-python-docstrings-comments-v1"
},
"embedding_seconds": 9.365606817002117
}
},
"metrics": {
"bm25": {
"mrr": 0.6081888214934283,
"mrr_at_10": 0.6021025793650794,
"recall_at_1": 0.526,
"recall_at_5": 0.7045,
"recall_at_10": 0.755,
"median_rank": 1.0,
"macro_repository_mrr": 0.6114859318187233
},
"pretrained_dense": {
"mrr": 0.7306130564474945,
"mrr_at_10": 0.7263926587301587,
"recall_at_1": 0.648,
"recall_at_5": 0.829,
"recall_at_10": 0.8685,
"median_rank": 1.0,
"macro_repository_mrr": 0.7451453403171922
},
"pretrained_hybrid": {
"mrr": 0.7231443528469972,
"mrr_at_10": 0.7179565476190476,
"recall_at_1": 0.6435,
"recall_at_5": 0.812,
"recall_at_10": 0.8645,
"median_rank": 1.0,
"macro_repository_mrr": 0.7220954273009865
},
"trained_dense": {
"mrr": 0.8026987712292828,
"mrr_at_10": 0.7997367063492063,
"recall_at_1": 0.7345,
"recall_at_5": 0.8845,
"recall_at_10": 0.918,
"median_rank": 1.0,
"macro_repository_mrr": 0.8070166379212605
},
"trained_hybrid": {
"mrr": 0.7488568869258339,
"mrr_at_10": 0.7437049603174603,
"recall_at_1": 0.675,
"recall_at_5": 0.8375,
"recall_at_10": 0.878,
"median_rank": 1.0,
"macro_repository_mrr": 0.7540302206437699
}
},
"trained_vs_pretrained_dense_mrr": {
"delta": 0.07208571478178834,
"ci95": [
0.05819084812420347,
0.08860171812961275
],
"method": "paired repository-cluster bootstrap, 2000 resamples",
"repository_clusters": 397
},
"trained_hybrid_vs_bm25_mrr": {
"delta": 0.1406680654324055,
"ci95": [
0.12797679548634003,
0.15351894949147968
],
"method": "paired repository-cluster bootstrap, 2000 resamples",
"repository_clusters": 397
},
"trained_dense_vs_bm25_mrr": {
"delta": 0.19450994973585453,
"ci95": [
0.1733663076176514,
0.21632412756364125
],
"method": "paired repository-cluster bootstrap, 2000 resamples",
"repository_clusters": 397
},
"elapsed_seconds": 28.711440942999616
}