diff --git a/.gitattributes b/.gitattributes index a6344aac8c09253b3b630fb776ae94478aa0275b..52fa044089d473e25f3d017cbd4dfbc990156277 100644 --- a/.gitattributes +++ b/.gitattributes @@ -33,3 +33,5 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text *.zip filter=lfs diff=lfs merge=lfs -text *.zst filter=lfs diff=lfs merge=lfs -text *tfevents* filter=lfs diff=lfs merge=lfs -text +tokenizer.json filter=lfs diff=lfs merge=lfs -text +unigram.json filter=lfs diff=lfs merge=lfs -text diff --git a/1_Pooling/config.json b/1_Pooling/config.json new file mode 100644 index 0000000000000000000000000000000000000000..d1514c3162bbe87b343f565fadc62e6c06f04f03 --- /dev/null +++ b/1_Pooling/config.json @@ -0,0 +1,7 @@ +{ + "word_embedding_dimension": 384, + "pooling_mode_cls_token": false, + "pooling_mode_mean_tokens": true, + "pooling_mode_max_tokens": false, + "pooling_mode_mean_sqrt_len_tokens": false +} \ No newline at end of file diff --git a/README.md b/README.md new file mode 100644 index 0000000000000000000000000000000000000000..16239aae0767f0c27d2f20ed433a129a68af6159 --- /dev/null +++ b/README.md @@ -0,0 +1,424 @@ +--- +library_name: setfit +tags: +- setfit +- sentence-transformers +- text-classification +- generated_from_setfit_trainer +metrics: +- accuracy +widget: +- text: The support and confidence measures were applied to identify frequent itemsets + and association rules in the given large database. +- text: 'The present thesis is organized into three sections: Section 1 provides an + overview of physical therapy interventions for enhancing cardiorespiratory fitness, + Section 2 focuses on musculoskeletal fitness, and Section 3 explores neuromotor + fitness.' +- text: The authors of this paper build upon the work of Kitchin (2014) and Mills + (2017), who emphasized the importance of understanding the social and ethical + implications of big data in the field of data science. +- text: This study adds to the existing literature on late-onset sepsis in very low + birth weight neonates by providing insights into the clinical characteristics, + microbiological etiologies, and outcomes of these infections based on a large, + multicenter database. +- text: The model-based clustering algorithm, specifically the Gaussian Mixture Model, + effectively identified distinct clusters in the data with high accuracy. +pipeline_tag: text-classification +inference: true +base_model: sentence-transformers/paraphrase-multilingual-MiniLM-L12-v2 +model-index: +- name: SetFit with sentence-transformers/paraphrase-multilingual-MiniLM-L12-v2 + results: + - task: + type: text-classification + name: Text Classification + dataset: + name: Unknown + type: unknown + split: test + metrics: + - type: accuracy + value: 0.6514563106796116 + name: Accuracy +--- + +# SetFit with sentence-transformers/paraphrase-multilingual-MiniLM-L12-v2 + +This is a [SetFit](https://github.com/huggingface/setfit) model that can be used for Text Classification. This SetFit model uses [sentence-transformers/paraphrase-multilingual-MiniLM-L12-v2](https://huggingface.co/sentence-transformers/paraphrase-multilingual-MiniLM-L12-v2) as the Sentence Transformer embedding model. A [LogisticRegression](https://scikit-learn.org/stable/modules/generated/sklearn.linear_model.LogisticRegression.html) instance is used for classification. + +The model has been trained using an efficient few-shot learning technique that involves: + +1. Fine-tuning a [Sentence Transformer](https://www.sbert.net) with contrastive learning. +2. Training a classification head with features from the fine-tuned Sentence Transformer. + +## Model Details + +### Model Description +- **Model Type:** SetFit +- **Sentence Transformer body:** [sentence-transformers/paraphrase-multilingual-MiniLM-L12-v2](https://huggingface.co/sentence-transformers/paraphrase-multilingual-MiniLM-L12-v2) +- **Classification head:** a [LogisticRegression](https://scikit-learn.org/stable/modules/generated/sklearn.linear_model.LogisticRegression.html) instance +- **Maximum Sequence Length:** 128 tokens +- **Number of Classes:** 103 classes + + + + +### Model Sources + +- **Repository:** [SetFit on GitHub](https://github.com/huggingface/setfit) +- **Paper:** [Efficient Few-Shot Learning Without Prompts](https://arxiv.org/abs/2209.11055) +- **Blogpost:** [SetFit: Efficient Few-Shot Learning Without Prompts](https://huggingface.co/blog/setfit) + +### Model Labels +| Label | Examples | +|:-------------------------------------------------------------------------------------------------------------------------|:-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| +| Acknowledging limitation(s) whilst stating a finding or contribution | | +| Advising cautious interpretation of the findings | | +| Commenting on the findings | | +| Commenting on the strengths of the current study | | +| Comparing the result: contradicting previous findings | | +| Comparing the result: supporting previous findings | | +| Contrasting sources with ‘however’ for emphasis | | +| Describing previously used methods | | +| Describing questionnaire design | | +| Describing the characteristics of the participants | | +| Describing the limitations of the current study | | +| Describing the process: adverbs of manner | | +| Describing the process: expressing purpose with for | | +| Describing the process: infinitive of purpose | | +| Describing the process: sequence words | | +| Describing the process: statistical procedures | | +| Describing the process: typical verbs in the passive form | | +| Describing the process: using + instrument | | +| Describing the research design and the methods used | | +| Describing what other writers do in their published work | | +| Detailing specific limitations | | +| Establishing the importance of the topic for the discipline | | +| Establishing the importance of the topic for the discipline: time frame given | | +| Establishing the importance of the topic for the world or society | | +| Establishing the importance of the topic for the world or society: time frame given | | +| Establising the importance of the topic as a problem to be addressed | | +| Explaining keywords (also refer to Defining Terms) | | +| Explaining the provenance of articles for review | | +| Explaining the provenance of the participants | | +| Explaining the significance of the current study | | +| Explaining the significance of the findings or contribution of the study | | +| General comments on the relevant literature | | +| General reference to previous research or scholarship: highlighting negative outcomes | | +| Giving reasons for personal interest in the research (sometimes found in the humanities, and the applied human sciences) | | +| Giving reasons why a particular method was adopted | | +| Giving reasons why a particular method was rejected | | +| Highlighting inadequacies or weaknesses of previous studies (also refer to Being Critical) | | +| Highlighting interesting or surprising results | | +| Highlighting significant data in a table or chart | | +| Identifying a controversy within the field of study | | +| Identifying a knowledge gap in the field of study | | +| Implications and/or recommendations for practice or policy | | +| Indicating an expected outcome | | +| Indicating an unexpected outcome | | +| Indicating criteria for selection or inclusion in the study | | +| Indicating methodological problems or limitations | | +| Indicating missing, weak, or contradictory evidence | | +| Indicating the methodology for the current research | | +| Indicating the use of an established method | | +| Introducing the limitations of the current study | | +| Making recommendations for further research work | | +| Noting implications of the findings | | +| Noting the lack of or paucity of previous research | | +| Offering an explanation for the findings | | +| Outlining the structure of a short paper | | +| Outlining the structure of a thesis or dissertation | | +| Pointing out interesting or important findings | | +| Previewing a chapter | | +| Previous research: A historic perspective | | +| Previous research: Approaches taken | | +| Previous research: What has been established or proposed | | +| Previous research: area investigated as the sentence object | | +| Previous research: area investigated as the sentence subject | | +| Previous research: highlighting negative outcomes | | +| Providing background information: reference to the literature | | +| Providing background information: reference to the purpose of the study | | +| Reference to previous research: important studies | | +| Referring back to the purpose of the paper or study | | +| Referring back to the research aims or procedures | | +| Referring to a single investigation in the past: investigation prominent | | +| Referring to a single investigation in the past: researcher prominent | | +| Referring to another writer’s idea(s) or position | | +| Referring to data in a table or chart | | +| Referring to important texts in the area of interest | | +| Referring to previous work to establish what is already known | | +| Referring to secondary sources | | +| Referring to the literature to justify a method or approach | | +| Reporting positive and negative reactions | | +| Restating a result or one of several results | | +| Setting out the research questions or hypotheses | | +| Some ways of introducing quotations | | +| Stating a negative result | | +| Stating a positive result | | +| Stating purpose of the current research with reference to gaps or issues in the literature | | +| Stating the aims of the current research (note frequent use of past tense) | | +| Stating the focus, aim, or argument of a short paper | | +| Stating the purpose of the thesis, dissertation, or research article (note use of present tense) | | +| Stating what is currently known about the topic | | +| Suggesting general hypotheses | | +| Suggesting implications for what is already known | | +| Suggestions for future work | | +| Summarising the literature review | | +| Summarising the main research findings | | +| Summarising the results section | | +| Summarising the studies reviewed | | +| Surveys and interviews: Introducing excerpts from interview data | | +| Surveys and interviews: Reporting participants’ views | | +| Surveys and interviews: Reporting proportions | | +| Surveys and interviews: Reporting response rates | | +| Surveys and interviews: Reporting themes | | +| Synthesising sources: contrasting evidence or ideas | | +| Synthesising sources: supporting evidence or ideas | | +| Transition: moving to the next result | | + +## Evaluation + +### Metrics +| Label | Accuracy | +|:--------|:---------| +| **all** | 0.6515 | + +## Uses + +### Direct Use for Inference + +First install the SetFit library: + +```bash +pip install setfit +``` + +Then you can load this model and run inference. + +```python +from setfit import SetFitModel + +# Download from the 🤗 Hub +model = SetFitModel.from_pretrained("Corran/SciGenSetfit") +# Run inference +preds = model("The support and confidence measures were applied to identify frequent itemsets and association rules in the given large database.") +``` + + + + + + + + + +## Training Details + +### Training Set Metrics +| Training set | Min | Median | Max | +|:-------------|:----|:--------|:----| +| Word count | 1 | 26.9658 | 61 | + +| Label | Training Sample Count | +|:-------------------------------------------------------------------------------------------------------------------------|:----------------------| +| Acknowledging limitation(s) whilst stating a finding or contribution | 50 | +| Advising cautious interpretation of the findings | 50 | +| Commenting on the findings | 50 | +| Commenting on the strengths of the current study | 50 | +| Comparing the result: contradicting previous findings | 50 | +| Comparing the result: supporting previous findings | 50 | +| Contrasting sources with ‘however’ for emphasis | 50 | +| Describing previously used methods | 50 | +| Describing questionnaire design | 50 | +| Describing the characteristics of the participants | 50 | +| Describing the limitations of the current study | 50 | +| Describing the process: adverbs of manner | 50 | +| Describing the process: expressing purpose with for | 50 | +| Describing the process: infinitive of purpose | 50 | +| Describing the process: sequence words | 50 | +| Describing the process: statistical procedures | 50 | +| Describing the process: typical verbs in the passive form | 50 | +| Describing the process: using + instrument | 50 | +| Describing the research design and the methods used | 50 | +| Describing what other writers do in their published work | 50 | +| Detailing specific limitations | 50 | +| Establishing the importance of the topic for the discipline | 50 | +| Establishing the importance of the topic for the discipline: time frame given | 50 | +| Establishing the importance of the topic for the world or society | 50 | +| Establishing the importance of the topic for the world or society: time frame given | 50 | +| Establising the importance of the topic as a problem to be addressed | 50 | +| Explaining keywords (also refer to Defining Terms) | 50 | +| Explaining the provenance of articles for review | 50 | +| Explaining the provenance of the participants | 50 | +| Explaining the significance of the current study | 50 | +| Explaining the significance of the findings or contribution of the study | 50 | +| General comments on the relevant literature | 50 | +| General reference to previous research or scholarship: highlighting negative outcomes | 50 | +| Giving reasons for personal interest in the research (sometimes found in the humanities, and the applied human sciences) | 50 | +| Giving reasons why a particular method was adopted | 50 | +| Giving reasons why a particular method was rejected | 50 | +| Highlighting inadequacies or weaknesses of previous studies (also refer to Being Critical) | 50 | +| Highlighting interesting or surprising results | 50 | +| Highlighting significant data in a table or chart | 50 | +| Identifying a controversy within the field of study | 50 | +| Identifying a knowledge gap in the field of study | 50 | +| Implications and/or recommendations for practice or policy | 50 | +| Indicating an expected outcome | 50 | +| Indicating an unexpected outcome | 50 | +| Indicating criteria for selection or inclusion in the study | 50 | +| Indicating methodological problems or limitations | 50 | +| Indicating missing, weak, or contradictory evidence | 50 | +| Indicating the methodology for the current research | 50 | +| Indicating the use of an established method | 50 | +| Introducing the limitations of the current study | 50 | +| Making recommendations for further research work | 50 | +| Noting implications of the findings | 50 | +| Noting the lack of or paucity of previous research | 50 | +| Offering an explanation for the findings | 50 | +| Outlining the structure of a short paper | 50 | +| Outlining the structure of a thesis or dissertation | 50 | +| Pointing out interesting or important findings | 50 | +| Previewing a chapter | 50 | +| Previous research: A historic perspective | 50 | +| Previous research: Approaches taken | 50 | +| Previous research: What has been established or proposed | 50 | +| Previous research: area investigated as the sentence object | 50 | +| Previous research: area investigated as the sentence subject | 50 | +| Previous research: highlighting negative outcomes | 50 | +| Providing background information: reference to the literature | 50 | +| Providing background information: reference to the purpose of the study | 50 | +| Reference to previous research: important studies | 50 | +| Referring back to the purpose of the paper or study | 50 | +| Referring back to the research aims or procedures | 50 | +| Referring to a single investigation in the past: investigation prominent | 50 | +| Referring to a single investigation in the past: researcher prominent | 50 | +| Referring to another writer’s idea(s) or position | 50 | +| Referring to data in a table or chart | 50 | +| Referring to important texts in the area of interest | 50 | +| Referring to previous work to establish what is already known | 50 | +| Referring to secondary sources | 50 | +| Referring to the literature to justify a method or approach | 50 | +| Reporting positive and negative reactions | 50 | +| Restating a result or one of several results | 50 | +| Setting out the research questions or hypotheses | 50 | +| Some ways of introducing quotations | 50 | +| Stating a negative result | 50 | +| Stating a positive result | 50 | +| Stating purpose of the current research with reference to gaps or issues in the literature | 50 | +| Stating the aims of the current research (note frequent use of past tense) | 50 | +| Stating the focus, aim, or argument of a short paper | 50 | +| Stating the purpose of the thesis, dissertation, or research article (note use of present tense) | 50 | +| Stating what is currently known about the topic | 50 | +| Suggesting general hypotheses | 50 | +| Suggesting implications for what is already known | 50 | +| Suggestions for future work | 50 | +| Summarising the literature review | 50 | +| Summarising the main research findings | 50 | +| Summarising the results section | 50 | +| Summarising the studies reviewed | 50 | +| Surveys and interviews: Introducing excerpts from interview data | 50 | +| Surveys and interviews: Reporting participants’ views | 50 | +| Surveys and interviews: Reporting proportions | 50 | +| Surveys and interviews: Reporting response rates | 50 | +| Surveys and interviews: Reporting themes | 50 | +| Synthesising sources: contrasting evidence or ideas | 50 | +| Synthesising sources: supporting evidence or ideas | 50 | +| Transition: moving to the next result | 50 | + +### Training Hyperparameters +- batch_size: (256, 256) +- num_epochs: (1, 1) +- max_steps: -1 +- sampling_strategy: oversampling +- num_iterations: 20 +- body_learning_rate: (2e-05, 1e-05) +- head_learning_rate: 0.01 +- loss: CosineSimilarityLoss +- distance_metric: cosine_distance +- margin: 0.25 +- end_to_end: False +- use_amp: False +- warmup_proportion: 0.1 +- seed: 42 +- eval_max_steps: -1 +- load_best_model_at_end: False + +### Training Results +| Epoch | Step | Training Loss | Validation Loss | +|:------:|:----:|:-------------:|:---------------:| +| 0.0012 | 1 | 0.3446 | - | +| 0.0621 | 50 | 0.219 | - | +| 0.1242 | 100 | 0.1466 | - | +| 0.1863 | 150 | 0.1332 | - | +| 0.2484 | 200 | 0.1396 | - | +| 0.3106 | 250 | 0.1113 | - | +| 0.3727 | 300 | 0.1226 | - | +| 0.4348 | 350 | 0.1141 | - | +| 0.4969 | 400 | 0.1051 | - | +| 0.5590 | 450 | 0.0988 | - | +| 0.6211 | 500 | 0.0999 | - | +| 0.6832 | 550 | 0.1125 | - | +| 0.7453 | 600 | 0.1187 | - | +| 0.8075 | 650 | 0.0801 | - | +| 0.8696 | 700 | 0.1132 | - | +| 0.9317 | 750 | 0.1158 | - | +| 0.9938 | 800 | 0.0995 | - | + +### Framework Versions +- Python: 3.10.12 +- SetFit: 1.0.2 +- Sentence Transformers: 2.2.2 +- Transformers: 4.35.2 +- PyTorch: 2.1.0+cu121 +- Datasets: 2.16.1 +- Tokenizers: 0.15.0 + +## Citation + +### BibTeX +```bibtex +@article{https://doi.org/10.48550/arxiv.2209.11055, + doi = {10.48550/ARXIV.2209.11055}, + url = {https://arxiv.org/abs/2209.11055}, + author = {Tunstall, Lewis and Reimers, Nils and Jo, Unso Eun Seo and Bates, Luke and Korat, Daniel and Wasserblat, Moshe and Pereg, Oren}, + keywords = {Computation and Language (cs.CL), FOS: Computer and information sciences, FOS: Computer and information sciences}, + title = {Efficient Few-Shot Learning Without Prompts}, + publisher = {arXiv}, + year = {2022}, + copyright = {Creative Commons Attribution 4.0 International} +} +``` + + + + + + \ No newline at end of file diff --git a/config.json b/config.json new file mode 100644 index 0000000000000000000000000000000000000000..85a91f7446d015897493e9d7caa0868d3a3155a8 --- /dev/null +++ b/config.json @@ -0,0 +1,26 @@ +{ + "_name_or_path": "/root/.cache/torch/sentence_transformers/sentence-transformers_paraphrase-multilingual-MiniLM-L12-v2/", + "architectures": [ + "BertModel" + ], + "attention_probs_dropout_prob": 0.1, + "classifier_dropout": null, + "gradient_checkpointing": false, + "hidden_act": "gelu", + "hidden_dropout_prob": 0.1, + "hidden_size": 384, + "initializer_range": 0.02, + "intermediate_size": 1536, + "layer_norm_eps": 1e-12, + "max_position_embeddings": 512, + "model_type": "bert", + "num_attention_heads": 12, + "num_hidden_layers": 12, + "pad_token_id": 0, + "position_embedding_type": "absolute", + "torch_dtype": "float32", + "transformers_version": "4.35.2", + "type_vocab_size": 2, + "use_cache": true, + "vocab_size": 250037 +} diff --git a/config_sentence_transformers.json b/config_sentence_transformers.json new file mode 100644 index 0000000000000000000000000000000000000000..b974b349cb2d419ada11181750a733ff82f291ad --- /dev/null +++ b/config_sentence_transformers.json @@ -0,0 +1,7 @@ +{ + "__version__": { + "sentence_transformers": "2.0.0", + "transformers": "4.7.0", + "pytorch": "1.9.0+cu102" + } +} \ No newline at end of file diff --git a/config_setfit.json b/config_setfit.json new file mode 100644 index 0000000000000000000000000000000000000000..a9efd89f0e8b18c3cf3235e671e0378e3c9ed14a --- /dev/null +++ b/config_setfit.json @@ -0,0 +1,4 @@ +{ + "labels": null, + "normalize_embeddings": false +} \ No newline at end of file diff --git a/model.safetensors b/model.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..5afb3344655eb7dc0435c94a94998cdc9e6a1f15 --- /dev/null +++ b/model.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d0d8ea4d945097515c2274602b2e9693d59877338f11ff29ae9fe3ef40f644c1 +size 470637416 diff --git a/model_head.pkl b/model_head.pkl new file mode 100644 index 0000000000000000000000000000000000000000..cc63a061b3c552076385c20361be5b5755d37fd8 --- /dev/null +++ b/model_head.pkl @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b67d3087acb12ace79c6145dc8e30dfcfe28712cc5497490f6f42271394fdaec +size 367503 diff --git a/modules.json b/modules.json new file mode 100644 index 0000000000000000000000000000000000000000..f7640f94e81bb7f4f04daf1668850b38763a13d9 --- /dev/null +++ b/modules.json @@ -0,0 +1,14 @@ +[ + { + "idx": 0, + "name": "0", + "path": "", + "type": "sentence_transformers.models.Transformer" + }, + { + "idx": 1, + "name": "1", + "path": "1_Pooling", + "type": "sentence_transformers.models.Pooling" + } +] \ No newline at end of file diff --git a/sentence_bert_config.json b/sentence_bert_config.json new file mode 100644 index 0000000000000000000000000000000000000000..5fd10429389515d3e5cccdeda08cae5fea1ae82e --- /dev/null +++ b/sentence_bert_config.json @@ -0,0 +1,4 @@ +{ + "max_seq_length": 128, + "do_lower_case": false +} \ No newline at end of file diff --git a/special_tokens_map.json b/special_tokens_map.json new file mode 100644 index 0000000000000000000000000000000000000000..d5698132694f4f1bcff08fa7d937b1701812598e --- /dev/null +++ b/special_tokens_map.json @@ -0,0 +1,15 @@ +{ + "bos_token": "", + "cls_token": "", + "eos_token": "", + "mask_token": { + "content": "", + "lstrip": true, + "normalized": false, + "rstrip": false, + "single_word": false + }, + "pad_token": "", + "sep_token": "", + "unk_token": "" +} diff --git a/tokenizer.json b/tokenizer.json new file mode 100644 index 0000000000000000000000000000000000000000..22623d38d9b8f6d777608968cb3530e963df53c8 --- /dev/null +++ b/tokenizer.json @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:fa685fc160bbdbab64058d4fc91b60e62d207e8dc60b9af5c002c5ab946ded00 +size 17083009 diff --git a/tokenizer_config.json b/tokenizer_config.json new file mode 100644 index 0000000000000000000000000000000000000000..6af3e8bba20e4425103afb1ae3dee3cacdbe7afb --- /dev/null +++ b/tokenizer_config.json @@ -0,0 +1,64 @@ +{ + "added_tokens_decoder": { + "0": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "1": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "2": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "3": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "250001": { + "content": "", + "lstrip": true, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + } + }, + "bos_token": "", + "clean_up_tokenization_spaces": true, + "cls_token": "", + "do_lower_case": true, + "eos_token": "", + "mask_token": "", + "max_length": 128, + "model_max_length": 512, + "pad_to_multiple_of": null, + "pad_token": "", + "pad_token_type_id": 0, + "padding_side": "right", + "sep_token": "", + "stride": 0, + "strip_accents": null, + "tokenize_chinese_chars": true, + "tokenizer_class": "BertTokenizer", + "truncation_side": "right", + "truncation_strategy": "longest_first", + "unk_token": "" +} diff --git a/unigram.json b/unigram.json new file mode 100644 index 0000000000000000000000000000000000000000..2faa9ec874108d53a017ff2c7ab98d155fb21a82 --- /dev/null +++ b/unigram.json @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:da145b5e7700ae40f16691ec32a0b1fdc1ee3298db22a31ea55f57a966c4a65d +size 14763260