Fix errors in load data

spanish_medica_llm.py

@@ -682,14 +682,13 @@ def run_training_process():
     login(token = os.environ.get('HG_FACE_TOKEN'))
     os.environ['WANDB_DISABLED'] = 'true'
     tokenizer = loadSpanishTokenizer()
-    medicalSpanishDataset = applyChatInstructFormat( loadSpanishDatasetFinnetuning())
-    medicalSpanishDataset = medicalSpanishDataset.train_test_split(0.2, seed=203984)
 
-    # train_dataset, eval_dataset, test_dataset = splitDatasetInTestValid(
-    #     getTokenizedDataset( medicalSpanishDataset, tokenizer)
-    #    )
-    
-    
+    medicalSpanishDataset =  loadSpanishDataset()
+    train_dataset, eval_dataset, test_dataset = splitDatasetInTestValid(
+        getTokenizedDataset( medicalSpanishDataset, tokenizer)
+       )
+   
+   
     train_dataset, eval_dataset, test_dataset = splitDatasetInTestValid(  medicalSpanishDataset )
 
     base_model =  loadBaseModel(MISTRAL_BASE_MODEL_ID)
@@ -702,11 +701,10 @@ def run_finnetuning_process():
     login(token = os.environ.get('HG_FACE_TOKEN'))
     os.environ['WANDB_DISABLED'] = 'true'
     tokenizer = loadSpanishTokenizer()
-    medicalSpanishDataset =  loadSpanishDataset()
-    train_dataset, eval_dataset, test_dataset = splitDatasetInTestValid(
-        getTokenizedDataset( medicalSpanishDataset, tokenizer)
-       )
-
+    medicalSpanishDataset = applyChatInstructFormat( loadSpanishDatasetFinnetuning())
+    medicalSpanishDataset = medicalSpanishDataset.train_test_split(0.2, seed=203984)
+    train_dataset, eval_dataset, test_dataset = splitDatasetInTestValid(  medicalSpanishDataset )
+    
     base_model =  loadBaseModel(HUB_MODEL_ID)
 
     configAndRunFineTuning(base_model,train_dataset, eval_dataset, tokenizer)