This commit is contained in:
ManyTheFish 2024-01-25 18:58:52 +01:00
parent 15dafde21d
commit 55e942cd45
4 changed files with 21 additions and 642 deletions

View file

@ -54,3 +54,5 @@ thai = ["milli/thai"]
greek = ["milli/greek"]
# allow khmer specialized tokenization
khmer = ["milli/khmer"]
# allow vietnamese specialized tokenization
vietnamese = ["milli/vietnamese"]