merge main

This commit is contained in:
ManyTheFish 2024-07-01 11:50:36 +02:00
commit 015d90a962
38 changed files with 792 additions and 205 deletions

View file

@ -54,6 +54,8 @@ chinese-pinyin = ["milli/chinese-pinyin"]
hebrew = ["milli/hebrew"]
# japanese specialized tokenization
japanese = ["milli/japanese"]
# korean specialized tokenization
korean = ["milli/korean"]
# thai specialized tokenization
thai = ["milli/thai"]
# allow greek specialized tokenization