update charabia

This commit is contained in:
ManyTheFish 2023-10-26 17:01:10 +02:00
parent ca52021079
commit 4c6fddb1cb
4 changed files with 761 additions and 32 deletions

View file

@ -50,6 +50,7 @@ hebrew = ["milli/hebrew"]
japanese = ["milli/japanese"]
# thai specialized tokenization
thai = ["milli/thai"]
# allow greek specialized tokenization
greek = ["milli/greek"]
# allow khmer specialized tokenization
khmer = ["milli/khmer"]