taka-yamakoshi commited on
Commit
6751661
1 Parent(s): ef0b5c6

fix a minor bug

Browse files
Files changed (1) hide show
  1. app.py +2 -2
app.py CHANGED
@@ -95,7 +95,7 @@ if __name__=='__main__':
95
 
96
  # Title
97
  st.markdown(generate_markdown('WordPiece Explorer',size=32), unsafe_allow_html=True)
98
- st.markdown(generate_markdown('quick and easy way to explore how tokenizers work',size=24), unsafe_allow_html=True)
99
 
100
  # Select and load the tokenizer
101
  st.sidebar.write('1. Choose the tokenizer from below')
@@ -139,7 +139,7 @@ if __name__=='__main__':
139
  default_tokens = tokenizer('Tokenizers decompose bigger words into smaller tokens')['input_ids']
140
  else:
141
  default_tokens = tokenizer('Tokenizers decompose bigger words into smaller tokens')['input_ids'][1:-1]
142
- sentence = st.text_input(f'Tokenized IDs',value=' '.join(default_tokens))
143
  num_tokens = DeTokenizeText(sentence)
144
  else:
145
  sentence = st.text_input(f'Text',value='Tokenizers decompose bigger words into smaller tokens')
 
95
 
96
  # Title
97
  st.markdown(generate_markdown('WordPiece Explorer',size=32), unsafe_allow_html=True)
98
+ st.markdown(generate_markdown('- quick and easy way to explore how tokenizers work -',size=24), unsafe_allow_html=True)
99
 
100
  # Select and load the tokenizer
101
  st.sidebar.write('1. Choose the tokenizer from below')
 
139
  default_tokens = tokenizer('Tokenizers decompose bigger words into smaller tokens')['input_ids']
140
  else:
141
  default_tokens = tokenizer('Tokenizers decompose bigger words into smaller tokens')['input_ids'][1:-1]
142
+ sentence = st.text_input(f'Tokenized IDs',value=' '.join([str(token) for token in default_tokens]))
143
  num_tokens = DeTokenizeText(sentence)
144
  else:
145
  sentence = st.text_input(f'Text',value='Tokenizers decompose bigger words into smaller tokens')