127 lines
4.1 KiB
JSON
127 lines
4.1 KiB
JSON
{
|
|
"split": "test",
|
|
"queries": 3000,
|
|
"candidates_per_query": 3000,
|
|
"repositories": 444,
|
|
"dataset_sha256": "b9e150337e195838efb5fe3f4ecafd63dfa99a55cf3635764c893912f4747596",
|
|
"evaluator_source_sha256": "29c942b9cbfff21ba8514511e27752527f0ec5e2cad5fbf57ec3b220fc7acd0b",
|
|
"fusion": {
|
|
"dense_weight": 0.5,
|
|
"per_method_limit": 100,
|
|
"k": 60
|
|
},
|
|
"evaluation_ids_sha256": "27c4062864258ea8f93c3e35b1ef6f65038dc24d15673afc930168dec349b3ba",
|
|
"protocol": "one labeled positive per query; all split snippets are candidates; docstrings removed",
|
|
"limitations": [
|
|
"Other semantically correct snippets may be counted as negatives",
|
|
"This is function retrieval, not a bug-fixing or multi-file context benchmark",
|
|
"No downstream Astra task-success evaluation has been run",
|
|
"Base-model pretraining contamination cannot be excluded"
|
|
],
|
|
"models": {
|
|
"pretrained": {
|
|
"fingerprint": "78f9f86bd7f71186560e631f0ed43678868c5d39ec07512adc52bc9185009f82",
|
|
"parameters": 22713216,
|
|
"config": {
|
|
"format_version": 1,
|
|
"base_model": "sentence-transformers/all-MiniLM-L6-v2",
|
|
"revision": "1110a243fdf4706b3f48f1d95db1a4f5529b4d41",
|
|
"max_length": 256,
|
|
"query_length": 96,
|
|
"pooling": "attention-masked-mean-l2",
|
|
"code_normalization": "strip-python-docstrings-comments-v1"
|
|
},
|
|
"embedding_seconds": 14.739996934000374
|
|
},
|
|
"trained": {
|
|
"fingerprint": "2bb5197791e4a0caf85af4e92c7ad9f2394cc4c01dafa0b0b18d7e32f0fc5697",
|
|
"parameters": 22713216,
|
|
"config": {
|
|
"format_version": 1,
|
|
"base_model": "sentence-transformers/all-MiniLM-L6-v2",
|
|
"revision": "1110a243fdf4706b3f48f1d95db1a4f5529b4d41",
|
|
"max_length": 256,
|
|
"query_length": 96,
|
|
"pooling": "attention-masked-mean-l2",
|
|
"code_normalization": "strip-python-docstrings-comments-v1"
|
|
},
|
|
"embedding_seconds": 14.511412101001042
|
|
}
|
|
},
|
|
"metrics": {
|
|
"bm25": {
|
|
"mrr": 0.559536732255293,
|
|
"mrr_at_10": 0.5529096560846561,
|
|
"recall_at_1": 0.4673333333333333,
|
|
"recall_at_5": 0.668,
|
|
"recall_at_10": 0.7263333333333334,
|
|
"median_rank": 2.0,
|
|
"macro_repository_mrr": 0.5716105643053901
|
|
},
|
|
"pretrained_dense": {
|
|
"mrr": 0.6819148989769989,
|
|
"mrr_at_10": 0.6770041005291004,
|
|
"recall_at_1": 0.587,
|
|
"recall_at_5": 0.7976666666666666,
|
|
"recall_at_10": 0.8523333333333334,
|
|
"median_rank": 1.0,
|
|
"macro_repository_mrr": 0.6956670635572176
|
|
},
|
|
"pretrained_hybrid": {
|
|
"mrr": 0.6879723629119775,
|
|
"mrr_at_10": 0.6825018518518519,
|
|
"recall_at_1": 0.5963333333333334,
|
|
"recall_at_5": 0.796,
|
|
"recall_at_10": 0.8496666666666667,
|
|
"median_rank": 1.0,
|
|
"macro_repository_mrr": 0.6997776730651677
|
|
},
|
|
"trained_dense": {
|
|
"mrr": 0.7784403489523588,
|
|
"mrr_at_10": 0.7752448412698412,
|
|
"recall_at_1": 0.701,
|
|
"recall_at_5": 0.8746666666666667,
|
|
"recall_at_10": 0.9156666666666666,
|
|
"median_rank": 1.0,
|
|
"macro_repository_mrr": 0.7791868690119261
|
|
},
|
|
"trained_hybrid": {
|
|
"mrr": 0.7220843512885813,
|
|
"mrr_at_10": 0.7164558201058201,
|
|
"recall_at_1": 0.6406666666666667,
|
|
"recall_at_5": 0.815,
|
|
"recall_at_10": 0.8663333333333333,
|
|
"median_rank": 1.0,
|
|
"macro_repository_mrr": 0.7273184680219402
|
|
}
|
|
},
|
|
"trained_vs_pretrained_dense_mrr": {
|
|
"delta": 0.09652544997535988,
|
|
"ci95": [
|
|
0.0848868558350596,
|
|
0.10748017710804628
|
|
],
|
|
"method": "paired repository-cluster bootstrap, 2000 resamples",
|
|
"repository_clusters": 444
|
|
},
|
|
"trained_hybrid_vs_bm25_mrr": {
|
|
"delta": 0.16254761903328815,
|
|
"ci95": [
|
|
0.150834640125844,
|
|
0.1763703573684562
|
|
],
|
|
"method": "paired repository-cluster bootstrap, 2000 resamples",
|
|
"repository_clusters": 444
|
|
},
|
|
"trained_dense_vs_bm25_mrr": {
|
|
"delta": 0.21890361669706576,
|
|
"ci95": [
|
|
0.2004503288862072,
|
|
0.23732116908748563
|
|
],
|
|
"method": "paired repository-cluster bootstrap, 2000 resamples",
|
|
"repository_clusters": 444
|
|
},
|
|
"elapsed_seconds": 39.51678221300244
|
|
}
|