Lisibonny commited on
Commit
c8e2bd5
1 Parent(s): 083cdef

Update app.py

Browse files
Files changed (1) hide show
  1. app.py +4 -8
app.py CHANGED
@@ -1,6 +1,7 @@
1
  import pandas as pd
2
  import streamlit as st
3
  from transformers import pipeline
 
4
  from carga_articulos import cargar_articulos
5
  from preprocesamiento_articulos import limpieza_articulos
6
  from entrenamiento_modelo import term_document_matrix, tf_idf_score
@@ -16,14 +17,9 @@ def crear_indice():
16
  td_idf_matrix.to_csv('articulos_indexados.csv')
17
 
18
  def load_qa_model():
19
- model = pipeline(
20
- 'question-answering',
21
- model='mrm8488/distill-bert-base-spanish-wwm-cased-finetuned-spa-squad2-es',
22
- tokenizer=(
23
- 'mrm8488/distill-bert-base-spanish-wwm-cased-finetuned-spa-squad2-es',
24
- {"use_fast": False} # Este modelo aun no es compatible con fast tokenizer
25
- )
26
- )
27
  return model
28
 
29
  # 4. Use streamlit to create a web app
 
1
  import pandas as pd
2
  import streamlit as st
3
  from transformers import pipeline
4
+ from transformers import AutoTokenizer, DistilBertForQuestionAnswering
5
  from carga_articulos import cargar_articulos
6
  from preprocesamiento_articulos import limpieza_articulos
7
  from entrenamiento_modelo import term_document_matrix, tf_idf_score
 
17
  td_idf_matrix.to_csv('articulos_indexados.csv')
18
 
19
  def load_qa_model():
20
+
21
+ tokenizer = AutoTokenizer.from_pretrained('mrm8488/distill-bert-base-spanish-wwm-cased-finetuned-spa-squad2-es', use_fast="false")
22
+ model = DistilBertForQuestionAnswering.from_pretrained("mrm8488/distill-bert-base-spanish-wwm-cased-finetuned-spa-squad2-es", device_map="auto", trust_remote_code=True, torch_dtype=torch.bfloat16)
 
 
 
 
 
23
  return model
24
 
25
  # 4. Use streamlit to create a web app