Add german tokenization pipeline

This commit is contained in:
ManyTheFish 2024-09-19 13:30:07 +02:00
parent f77661ec44
commit 7d6768e4c4
3 changed files with 7 additions and 0 deletions

View file

@ -66,3 +66,5 @@ khmer = ["milli/khmer"]
vietnamese = ["milli/vietnamese"]
# force swedish character recomposition
swedish-recomposition = ["milli/swedish-recomposition"]
# force german character recomposition
german = ["milli/german"]