tokenizer.py

1fromtypingimportCallable23fromlabml.configsimportBaseConfigs,option

6classTokenizerConfigs(BaseConfigs):

13tokenizer:Callable='character'

15def\_\_init\_\_(self):16super().\_\_init\_\_(\_primary='tokenizer')

'tokenizer': 'basic_english'

19@option(TokenizerConfigs.tokenizer)20defbasic\_english():

34fromtorchtext.dataimportget\_tokenizer35returnget\_tokenizer('basic\_english')

38defcharacter\_tokenizer(x:str):

42returnlist(x)

45@option(TokenizerConfigs.tokenizer)46defcharacter():

50returncharacter\_tokenizer