Updatre charabia v0.8.9

This commit is contained in:
ManyTheFish 2024-04-18 11:38:26 +02:00
parent 4a8459b799
commit c71b5d09ff
4 changed files with 205 additions and 47 deletions

View file

@ -44,6 +44,7 @@ all-tokenizations = ["milli/all-tokenizations"]
# chinese specialized tokenization
chinese = ["milli/chinese"]
chinese-pinyin = ["milli/chinese-pinyin"]
# hebrew specialized tokenization
hebrew = ["milli/hebrew"]
# japanese specialized tokenization