127 lines
4.0 KiB
JSON
127 lines
4.0 KiB
JSON
{
|
|
"split": "validation",
|
|
"queries": 2000,
|
|
"candidates_per_query": 2000,
|
|
"repositories": 397,
|
|
"dataset_sha256": "e1b934c33f12322e4a56d0d17a987966c40961b035e15939fd716b76d4e017a2",
|
|
"evaluator_source_sha256": "29c942b9cbfff21ba8514511e27752527f0ec5e2cad5fbf57ec3b220fc7acd0b",
|
|
"fusion": {
|
|
"dense_weight": 0.5,
|
|
"per_method_limit": 100,
|
|
"k": 60
|
|
},
|
|
"evaluation_ids_sha256": "37d3ee7d31626ab63e5746faac6e8ff2bb117506ddd682b2a71405682196f2ef",
|
|
"protocol": "one labeled positive per query; all split snippets are candidates; docstrings removed",
|
|
"limitations": [
|
|
"Other semantically correct snippets may be counted as negatives",
|
|
"This is function retrieval, not a bug-fixing or multi-file context benchmark",
|
|
"No downstream Astra task-success evaluation has been run",
|
|
"Base-model pretraining contamination cannot be excluded"
|
|
],
|
|
"models": {
|
|
"pretrained": {
|
|
"fingerprint": "78f9f86bd7f71186560e631f0ed43678868c5d39ec07512adc52bc9185009f82",
|
|
"parameters": 22713216,
|
|
"config": {
|
|
"format_version": 1,
|
|
"base_model": "sentence-transformers/all-MiniLM-L6-v2",
|
|
"revision": "1110a243fdf4706b3f48f1d95db1a4f5529b4d41",
|
|
"max_length": 256,
|
|
"query_length": 96,
|
|
"pooling": "attention-masked-mean-l2",
|
|
"code_normalization": "strip-python-docstrings-comments-v1"
|
|
},
|
|
"embedding_seconds": 9.731017789999896
|
|
},
|
|
"trained": {
|
|
"fingerprint": "2bb5197791e4a0caf85af4e92c7ad9f2394cc4c01dafa0b0b18d7e32f0fc5697",
|
|
"parameters": 22713216,
|
|
"config": {
|
|
"format_version": 1,
|
|
"base_model": "sentence-transformers/all-MiniLM-L6-v2",
|
|
"revision": "1110a243fdf4706b3f48f1d95db1a4f5529b4d41",
|
|
"max_length": 256,
|
|
"query_length": 96,
|
|
"pooling": "attention-masked-mean-l2",
|
|
"code_normalization": "strip-python-docstrings-comments-v1"
|
|
},
|
|
"embedding_seconds": 9.365606817002117
|
|
}
|
|
},
|
|
"metrics": {
|
|
"bm25": {
|
|
"mrr": 0.6081888214934283,
|
|
"mrr_at_10": 0.6021025793650794,
|
|
"recall_at_1": 0.526,
|
|
"recall_at_5": 0.7045,
|
|
"recall_at_10": 0.755,
|
|
"median_rank": 1.0,
|
|
"macro_repository_mrr": 0.6114859318187233
|
|
},
|
|
"pretrained_dense": {
|
|
"mrr": 0.7306130564474945,
|
|
"mrr_at_10": 0.7263926587301587,
|
|
"recall_at_1": 0.648,
|
|
"recall_at_5": 0.829,
|
|
"recall_at_10": 0.8685,
|
|
"median_rank": 1.0,
|
|
"macro_repository_mrr": 0.7451453403171922
|
|
},
|
|
"pretrained_hybrid": {
|
|
"mrr": 0.7231443528469972,
|
|
"mrr_at_10": 0.7179565476190476,
|
|
"recall_at_1": 0.6435,
|
|
"recall_at_5": 0.812,
|
|
"recall_at_10": 0.8645,
|
|
"median_rank": 1.0,
|
|
"macro_repository_mrr": 0.7220954273009865
|
|
},
|
|
"trained_dense": {
|
|
"mrr": 0.8026987712292828,
|
|
"mrr_at_10": 0.7997367063492063,
|
|
"recall_at_1": 0.7345,
|
|
"recall_at_5": 0.8845,
|
|
"recall_at_10": 0.918,
|
|
"median_rank": 1.0,
|
|
"macro_repository_mrr": 0.8070166379212605
|
|
},
|
|
"trained_hybrid": {
|
|
"mrr": 0.7488568869258339,
|
|
"mrr_at_10": 0.7437049603174603,
|
|
"recall_at_1": 0.675,
|
|
"recall_at_5": 0.8375,
|
|
"recall_at_10": 0.878,
|
|
"median_rank": 1.0,
|
|
"macro_repository_mrr": 0.7540302206437699
|
|
}
|
|
},
|
|
"trained_vs_pretrained_dense_mrr": {
|
|
"delta": 0.07208571478178834,
|
|
"ci95": [
|
|
0.05819084812420347,
|
|
0.08860171812961275
|
|
],
|
|
"method": "paired repository-cluster bootstrap, 2000 resamples",
|
|
"repository_clusters": 397
|
|
},
|
|
"trained_hybrid_vs_bm25_mrr": {
|
|
"delta": 0.1406680654324055,
|
|
"ci95": [
|
|
0.12797679548634003,
|
|
0.15351894949147968
|
|
],
|
|
"method": "paired repository-cluster bootstrap, 2000 resamples",
|
|
"repository_clusters": 397
|
|
},
|
|
"trained_dense_vs_bm25_mrr": {
|
|
"delta": 0.19450994973585453,
|
|
"ci95": [
|
|
0.1733663076176514,
|
|
0.21632412756364125
|
|
],
|
|
"method": "paired repository-cluster bootstrap, 2000 resamples",
|
|
"repository_clusters": 397
|
|
},
|
|
"elapsed_seconds": 28.711440942999616
|
|
}
|