--- tags: - mteb model-index: - name: mist-zh results: - task: type: STS dataset: type: C-MTEB/AFQMC name: MTEB AFQMC config: default split: validation revision: None metrics: - type: cos_sim_pearson value: 44.80910972039708 - type: cos_sim_spearman value: 46.97947004057185 - type: euclidean_pearson value: 45.36774158404125 - type: euclidean_spearman value: 46.97947004232487 - type: manhattan_pearson value: 45.23486628014998 - type: manhattan_spearman value: 46.87721960765866 - task: type: STS dataset: type: C-MTEB/ATEC name: MTEB ATEC config: default split: test revision: None metrics: - type: cos_sim_pearson value: 49.5294624928126 - type: cos_sim_spearman value: 51.34771777448503 - type: euclidean_pearson value: 53.56859824288157 - type: euclidean_spearman value: 51.34771439634126 - type: manhattan_pearson value: 53.581640877132685 - type: manhattan_spearman value: 51.349656519071274 - task: type: Classification dataset: type: mteb/amazon_reviews_multi name: MTEB AmazonReviewsClassification (zh) config: zh split: test revision: 1399c76144fd37290681b995c656ef9b2e06e26d metrics: - type: accuracy value: 39.318 - type: f1 value: 37.37720144558489 - task: type: STS dataset: type: C-MTEB/BQ name: MTEB BQ config: default split: test revision: None metrics: - type: cos_sim_pearson value: 62.12016334764962 - type: cos_sim_spearman value: 65.08208654969742 - type: euclidean_pearson value: 63.53078822303454 - type: euclidean_spearman value: 65.0820865487212 - type: manhattan_pearson value: 63.510532363654725 - type: manhattan_spearman value: 65.06622789125241 - task: type: Clustering dataset: type: C-MTEB/CLSClusteringP2P name: MTEB CLSClusteringP2P config: default split: test revision: None metrics: - type: v_measure value: 39.5071157612481 - task: type: Clustering dataset: type: C-MTEB/CLSClusteringS2S name: MTEB CLSClusteringS2S config: default split: test revision: None metrics: - type: v_measure value: 37.99964332311132 - task: type: Reranking dataset: type: C-MTEB/CMedQAv1-reranking name: MTEB CMedQAv1 config: default split: test revision: None metrics: - type: map value: 84.67010533089491 - type: mrr value: 86.99488095238095 - task: type: Reranking dataset: type: C-MTEB/CMedQAv2-reranking name: MTEB CMedQAv2 config: default split: test revision: None metrics: - type: map value: 85.27288868896477 - type: mrr value: 87.5929761904762 - task: type: Retrieval dataset: type: C-MTEB/CmedqaRetrieval name: MTEB CmedqaRetrieval config: default split: dev revision: None metrics: - type: map_at_1 value: 23.949 - type: map_at_10 value: 35.394 - type: map_at_100 value: 37.235 - type: map_at_1000 value: 37.364999999999995 - type: map_at_3 value: 31.433 - type: map_at_5 value: 33.668 - type: mrr_at_1 value: 36.834 - type: mrr_at_10 value: 44.451 - type: mrr_at_100 value: 45.445 - type: mrr_at_1000 value: 45.501000000000005 - type: mrr_at_3 value: 42.010999999999996 - type: mrr_at_5 value: 43.34 - type: ndcg_at_1 value: 36.834 - type: ndcg_at_10 value: 41.803000000000004 - type: ndcg_at_100 value: 49.091 - type: ndcg_at_1000 value: 51.474 - type: ndcg_at_3 value: 36.736000000000004 - type: ndcg_at_5 value: 38.868 - type: precision_at_1 value: 36.834 - type: precision_at_10 value: 9.354999999999999 - type: precision_at_100 value: 1.5310000000000001 - type: precision_at_1000 value: 0.183 - type: precision_at_3 value: 20.78 - type: precision_at_5 value: 15.238999999999999 - type: recall_at_1 value: 23.949 - type: recall_at_10 value: 51.68000000000001 - type: recall_at_100 value: 81.938 - type: recall_at_1000 value: 98.091 - type: recall_at_3 value: 36.408 - type: recall_at_5 value: 42.952 - task: type: PairClassification dataset: type: C-MTEB/CMNLI name: MTEB Cmnli config: default split: validation revision: None metrics: - type: cos_sim_accuracy value: 76.24774503908598 - type: cos_sim_ap value: 84.76081551540754 - type: cos_sim_f1 value: 77.76321537789427 - type: cos_sim_precision value: 72.96577167452347 - type: cos_sim_recall value: 83.23591302314706 - type: dot_accuracy value: 76.24774503908598 - type: dot_ap value: 84.75968761251127 - type: dot_f1 value: 77.76321537789427 - type: dot_precision value: 72.96577167452347 - type: dot_recall value: 83.23591302314706 - type: euclidean_accuracy value: 76.24774503908598 - type: euclidean_ap value: 84.7608250840413 - type: euclidean_f1 value: 77.76321537789427 - type: euclidean_precision value: 72.96577167452347 - type: euclidean_recall value: 83.23591302314706 - type: manhattan_accuracy value: 76.19963920625375 - type: manhattan_ap value: 84.76313920535411 - type: manhattan_f1 value: 77.74253527288636 - type: manhattan_precision value: 73.0374023838882 - type: manhattan_recall value: 83.09562777647884 - type: max_accuracy value: 76.24774503908598 - type: max_ap value: 84.76313920535411 - type: max_f1 value: 77.76321537789427 - task: type: Retrieval dataset: type: C-MTEB/CovidRetrieval name: MTEB CovidRetrieval config: default split: dev revision: None metrics: - type: map_at_1 value: 66.149 - type: map_at_10 value: 75.22999999999999 - type: map_at_100 value: 75.536 - type: map_at_1000 value: 75.542 - type: map_at_3 value: 73.384 - type: map_at_5 value: 74.459 - type: mrr_at_1 value: 66.28 - type: mrr_at_10 value: 75.232 - type: mrr_at_100 value: 75.52799999999999 - type: mrr_at_1000 value: 75.534 - type: mrr_at_3 value: 73.446 - type: mrr_at_5 value: 74.473 - type: ndcg_at_1 value: 66.386 - type: ndcg_at_10 value: 79.295 - type: ndcg_at_100 value: 80.741 - type: ndcg_at_1000 value: 80.891 - type: ndcg_at_3 value: 75.613 - type: ndcg_at_5 value: 77.46300000000001 - type: precision_at_1 value: 66.386 - type: precision_at_10 value: 9.283 - type: precision_at_100 value: 0.996 - type: precision_at_1000 value: 0.101 - type: precision_at_3 value: 27.503 - type: precision_at_5 value: 17.408 - type: recall_at_1 value: 66.149 - type: recall_at_10 value: 91.886 - type: recall_at_100 value: 98.52499999999999 - type: recall_at_1000 value: 99.684 - type: recall_at_3 value: 81.849 - type: recall_at_5 value: 86.275 - task: type: Retrieval dataset: type: C-MTEB/DuRetrieval name: MTEB DuRetrieval config: default split: dev revision: None metrics: - type: map_at_1 value: 25.166 - type: map_at_10 value: 78.805 - type: map_at_100 value: 81.782 - type: map_at_1000 value: 81.818 - type: map_at_3 value: 54.226 - type: map_at_5 value: 68.783 - type: mrr_at_1 value: 88.6 - type: mrr_at_10 value: 92.244 - type: mrr_at_100 value: 92.31899999999999 - type: mrr_at_1000 value: 92.321 - type: mrr_at_3 value: 91.867 - type: mrr_at_5 value: 92.119 - type: ndcg_at_1 value: 88.6 - type: ndcg_at_10 value: 86.432 - type: ndcg_at_100 value: 89.357 - type: ndcg_at_1000 value: 89.688 - type: ndcg_at_3 value: 84.90299999999999 - type: ndcg_at_5 value: 84.137 - type: precision_at_1 value: 88.6 - type: precision_at_10 value: 41.685 - type: precision_at_100 value: 4.811 - type: precision_at_1000 value: 0.48900000000000005 - type: precision_at_3 value: 76.44999999999999 - type: precision_at_5 value: 64.87 - type: recall_at_1 value: 25.166 - type: recall_at_10 value: 88.227 - type: recall_at_100 value: 97.597 - type: recall_at_1000 value: 99.359 - type: recall_at_3 value: 56.946 - type: recall_at_5 value: 74.261 - task: type: Retrieval dataset: type: C-MTEB/EcomRetrieval name: MTEB EcomRetrieval config: default split: dev revision: None metrics: - type: map_at_1 value: 48.3 - type: map_at_10 value: 57.635999999999996 - type: map_at_100 value: 58.306000000000004 - type: map_at_1000 value: 58.326 - type: map_at_3 value: 54.900000000000006 - type: map_at_5 value: 56.620000000000005 - type: mrr_at_1 value: 48.3 - type: mrr_at_10 value: 57.635999999999996 - type: mrr_at_100 value: 58.306000000000004 - type: mrr_at_1000 value: 58.326 - type: mrr_at_3 value: 54.900000000000006 - type: mrr_at_5 value: 56.620000000000005 - type: ndcg_at_1 value: 48.3 - type: ndcg_at_10 value: 62.638000000000005 - type: ndcg_at_100 value: 65.726 - type: ndcg_at_1000 value: 66.253 - type: ndcg_at_3 value: 57.081 - type: ndcg_at_5 value: 60.217 - type: precision_at_1 value: 48.3 - type: precision_at_10 value: 7.85 - type: precision_at_100 value: 0.9249999999999999 - type: precision_at_1000 value: 0.097 - type: precision_at_3 value: 21.133 - type: precision_at_5 value: 14.219999999999999 - type: recall_at_1 value: 48.3 - type: recall_at_10 value: 78.5 - type: recall_at_100 value: 92.5 - type: recall_at_1000 value: 96.6 - type: recall_at_3 value: 63.4 - type: recall_at_5 value: 71.1 - task: type: Classification dataset: type: C-MTEB/IFlyTek-classification name: MTEB IFlyTek config: default split: validation revision: None metrics: - type: accuracy value: 47.9646017699115 - type: f1 value: 35.03552351349023 - task: type: Classification dataset: type: C-MTEB/JDReview-classification name: MTEB JDReview config: default split: test revision: None metrics: - type: accuracy value: 84.8968105065666 - type: ap value: 52.564605306946774 - type: f1 value: 79.59880155481291 - task: type: STS dataset: type: C-MTEB/LCQMC name: MTEB LCQMC config: default split: test revision: None metrics: - type: cos_sim_pearson value: 70.03662039861051 - type: cos_sim_spearman value: 76.9642260444222 - type: euclidean_pearson value: 75.47376966815843 - type: euclidean_spearman value: 76.9642282583736 - type: manhattan_pearson value: 75.45535385433548 - type: manhattan_spearman value: 76.94609742735338 - task: type: Retrieval dataset: type: C-MTEB/MMarcoRetrieval name: MTEB MMarcoRetrieval config: default split: dev revision: None metrics: - type: map_at_1 value: 65.604 - type: map_at_10 value: 74.522 - type: map_at_100 value: 74.878 - type: map_at_1000 value: 74.889 - type: map_at_3 value: 72.61 - type: map_at_5 value: 73.882 - type: mrr_at_1 value: 67.75099999999999 - type: mrr_at_10 value: 75.08399999999999 - type: mrr_at_100 value: 75.402 - type: mrr_at_1000 value: 75.412 - type: mrr_at_3 value: 73.446 - type: mrr_at_5 value: 74.531 - type: ndcg_at_1 value: 67.75099999999999 - type: ndcg_at_10 value: 78.172 - type: ndcg_at_100 value: 79.753 - type: ndcg_at_1000 value: 80.06400000000001 - type: ndcg_at_3 value: 74.607 - type: ndcg_at_5 value: 76.728 - type: precision_at_1 value: 67.75099999999999 - type: precision_at_10 value: 9.443999999999999 - type: precision_at_100 value: 1.023 - type: precision_at_1000 value: 0.105 - type: precision_at_3 value: 28.009 - type: precision_at_5 value: 17.934 - type: recall_at_1 value: 65.604 - type: recall_at_10 value: 88.84100000000001 - type: recall_at_100 value: 95.954 - type: recall_at_1000 value: 98.425 - type: recall_at_3 value: 79.497 - type: recall_at_5 value: 84.515 - task: type: Classification dataset: type: mteb/amazon_massive_intent name: MTEB MassiveIntentClassification (zh-CN) config: zh-CN split: test revision: 31efe3c427b0bae9c22cbb560b8f15491cc6bed7 metrics: - type: accuracy value: 67.64963012777405 - type: f1 value: 65.01092085388518 - task: type: Classification dataset: type: mteb/amazon_massive_scenario name: MTEB MassiveScenarioClassification (zh-CN) config: zh-CN split: test revision: 7d571f92784cd94a019292a1f45445077d0ef634 metrics: - type: accuracy value: 72.9724277067922 - type: f1 value: 72.48003852874602 - task: type: Retrieval dataset: type: C-MTEB/MedicalRetrieval name: MTEB MedicalRetrieval config: default split: dev revision: None metrics: - type: map_at_1 value: 48.9 - type: map_at_10 value: 55.189 - type: map_at_100 value: 55.687 - type: map_at_1000 value: 55.74400000000001 - type: map_at_3 value: 53.75 - type: map_at_5 value: 54.555 - type: mrr_at_1 value: 49.1 - type: mrr_at_10 value: 55.289 - type: mrr_at_100 value: 55.788000000000004 - type: mrr_at_1000 value: 55.845 - type: mrr_at_3 value: 53.849999999999994 - type: mrr_at_5 value: 54.655 - type: ndcg_at_1 value: 48.9 - type: ndcg_at_10 value: 58.275 - type: ndcg_at_100 value: 60.980000000000004 - type: ndcg_at_1000 value: 62.672000000000004 - type: ndcg_at_3 value: 55.282 - type: ndcg_at_5 value: 56.749 - type: precision_at_1 value: 48.9 - type: precision_at_10 value: 6.800000000000001 - type: precision_at_100 value: 0.8130000000000001 - type: precision_at_1000 value: 0.095 - type: precision_at_3 value: 19.900000000000002 - type: precision_at_5 value: 12.659999999999998 - type: recall_at_1 value: 48.9 - type: recall_at_10 value: 68.0 - type: recall_at_100 value: 81.3 - type: recall_at_1000 value: 95.0 - type: recall_at_3 value: 59.699999999999996 - type: recall_at_5 value: 63.3 - task: type: Classification dataset: type: C-MTEB/MultilingualSentiment-classification name: MTEB MultilingualSentiment config: default split: validation revision: None metrics: - type: accuracy value: 71.53666666666668 - type: f1 value: 70.74267338218574 - task: type: PairClassification dataset: type: C-MTEB/OCNLI name: MTEB Ocnli config: default split: validation revision: None metrics: - type: cos_sim_accuracy value: 70.43854899837575 - type: cos_sim_ap value: 75.25713109733296 - type: cos_sim_f1 value: 73.18777292576418 - type: cos_sim_precision value: 62.397617274758 - type: cos_sim_recall value: 88.48996832101372 - type: dot_accuracy value: 70.43854899837575 - type: dot_ap value: 75.25713109733296 - type: dot_f1 value: 73.18777292576418 - type: dot_precision value: 62.397617274758 - type: dot_recall value: 88.48996832101372 - type: euclidean_accuracy value: 70.43854899837575 - type: euclidean_ap value: 75.25713109733296 - type: euclidean_f1 value: 73.18777292576418 - type: euclidean_precision value: 62.397617274758 - type: euclidean_recall value: 88.48996832101372 - type: manhattan_accuracy value: 70.60097455332972 - type: manhattan_ap value: 75.22177995740668 - type: manhattan_f1 value: 73.13750532141337 - type: manhattan_precision value: 61.26961483594865 - type: manhattan_recall value: 90.70749736008447 - type: max_accuracy value: 70.60097455332972 - type: max_ap value: 75.25713109733296 - type: max_f1 value: 73.18777292576418 - task: type: Classification dataset: type: C-MTEB/OnlineShopping-classification name: MTEB OnlineShopping config: default split: test revision: None metrics: - type: accuracy value: 91.3 - type: ap value: 89.03601366589187 - type: f1 value: 91.28612226957141 - task: type: STS dataset: type: C-MTEB/PAWSX name: MTEB PAWSX config: default split: test revision: None metrics: - type: cos_sim_pearson value: 24.254041798082984 - type: cos_sim_spearman value: 30.029755057178846 - type: euclidean_pearson value: 30.394005237465905 - type: euclidean_spearman value: 30.029751825186153 - type: manhattan_pearson value: 30.400683181995863 - type: manhattan_spearman value: 29.981240616043326 - task: type: STS dataset: type: C-MTEB/QBQTC name: MTEB QBQTC config: default split: test revision: None metrics: - type: cos_sim_pearson value: 35.09911024323138 - type: cos_sim_spearman value: 37.49790006053554 - type: euclidean_pearson value: 35.65689785105493 - type: euclidean_spearman value: 37.498032509597344 - type: manhattan_pearson value: 35.68350134483341 - type: manhattan_spearman value: 37.54046578100128 - task: type: STS dataset: type: mteb/sts22-crosslingual-sts name: MTEB STS22 (zh) config: zh split: test revision: 6d1ba47164174a496b7fa5d3569dae26a6813b80 metrics: - type: cos_sim_pearson value: 68.26707578158273 - type: cos_sim_spearman value: 69.19741429899995 - type: euclidean_pearson value: 68.53026048034656 - type: euclidean_spearman value: 69.1974135636389 - type: manhattan_pearson value: 70.02306646353263 - type: manhattan_spearman value: 70.46158498712836 - task: type: STS dataset: type: C-MTEB/STSB name: MTEB STSB config: default split: test revision: None metrics: - type: cos_sim_pearson value: 78.88749955421177 - type: cos_sim_spearman value: 79.56695106617856 - type: euclidean_pearson value: 79.13787024514338 - type: euclidean_spearman value: 79.56690827015423 - type: manhattan_pearson value: 79.08154812411563 - type: manhattan_spearman value: 79.52391077945943 - task: type: Reranking dataset: type: C-MTEB/T2Reranking name: MTEB T2Reranking config: default split: dev revision: None metrics: - type: map value: 65.78663254562939 - type: mrr value: 74.9786877626248 - task: type: Retrieval dataset: type: C-MTEB/T2Retrieval name: MTEB T2Retrieval config: default split: dev revision: None metrics: - type: map_at_1 value: 26.169999999999998 - type: map_at_10 value: 74.009 - type: map_at_100 value: 77.788 - type: map_at_1000 value: 77.866 - type: map_at_3 value: 51.861000000000004 - type: map_at_5 value: 63.775000000000006 - type: mrr_at_1 value: 87.748 - type: mrr_at_10 value: 90.737 - type: mrr_at_100 value: 90.84400000000001 - type: mrr_at_1000 value: 90.849 - type: mrr_at_3 value: 90.257 - type: mrr_at_5 value: 90.54299999999999 - type: ndcg_at_1 value: 87.748 - type: ndcg_at_10 value: 82.114 - type: ndcg_at_100 value: 86.148 - type: ndcg_at_1000 value: 86.913 - type: ndcg_at_3 value: 83.54599999999999 - type: ndcg_at_5 value: 81.987 - type: precision_at_1 value: 87.748 - type: precision_at_10 value: 41.076 - type: precision_at_100 value: 4.976 - type: precision_at_1000 value: 0.515 - type: precision_at_3 value: 73.282 - type: precision_at_5 value: 61.351 - type: recall_at_1 value: 26.169999999999998 - type: recall_at_10 value: 81.292 - type: recall_at_100 value: 94.285 - type: recall_at_1000 value: 98.221 - type: recall_at_3 value: 53.824000000000005 - type: recall_at_5 value: 67.547 - task: type: Classification dataset: type: C-MTEB/TNews-classification name: MTEB TNews config: default split: validation revision: None metrics: - type: accuracy value: 51.564 - type: f1 value: 49.711462885083286 - task: type: Clustering dataset: type: C-MTEB/ThuNewsClusteringP2P name: MTEB ThuNewsClusteringP2P config: default split: test revision: None metrics: - type: v_measure value: 62.57078038998942 - task: type: Clustering dataset: type: C-MTEB/ThuNewsClusteringS2S name: MTEB ThuNewsClusteringS2S config: default split: test revision: None metrics: - type: v_measure value: 57.842602165392144 - task: type: Retrieval dataset: type: C-MTEB/VideoRetrieval name: MTEB VideoRetrieval config: default split: dev revision: None metrics: - type: map_at_1 value: 52.0 - type: map_at_10 value: 62.932 - type: map_at_100 value: 63.471999999999994 - type: map_at_1000 value: 63.483999999999995 - type: map_at_3 value: 60.516999999999996 - type: map_at_5 value: 62.097 - type: mrr_at_1 value: 52.0 - type: mrr_at_10 value: 62.932 - type: mrr_at_100 value: 63.471999999999994 - type: mrr_at_1000 value: 63.483999999999995 - type: mrr_at_3 value: 60.516999999999996 - type: mrr_at_5 value: 62.097 - type: ndcg_at_1 value: 52.0 - type: ndcg_at_10 value: 67.963 - type: ndcg_at_100 value: 70.598 - type: ndcg_at_1000 value: 70.896 - type: ndcg_at_3 value: 63.144 - type: ndcg_at_5 value: 65.988 - type: precision_at_1 value: 52.0 - type: precision_at_10 value: 8.36 - type: precision_at_100 value: 0.959 - type: precision_at_1000 value: 0.098 - type: precision_at_3 value: 23.567 - type: precision_at_5 value: 15.52 - type: recall_at_1 value: 52.0 - type: recall_at_10 value: 83.6 - type: recall_at_100 value: 95.89999999999999 - type: recall_at_1000 value: 98.2 - type: recall_at_3 value: 70.7 - type: recall_at_5 value: 77.60000000000001 - task: type: Classification dataset: type: C-MTEB/waimai-classification name: MTEB Waimai config: default split: test revision: None metrics: - type: accuracy value: 86.65999999999998 - type: ap value: 69.91988858863054 - type: f1 value: 84.92982698422784 - task: type: Reranking dataset: type: C-MTEB/Mmarco-reranking name: MTEB MMarcoReranking config: default split: dev revision: None metrics: - type: map value: 27.838972963193315 - type: mrr value: 26.65238095238095 ---