searchengines-web/MeiliSearch
searchengines-web/MeiliSearch
1
0
Fork 0
mirror of https://github.com/meilisearch/MeiliSearch synced 2024-12-03 18:15:46 +01:00
Code Issues Releases Wiki Activity

Truncate HuggingFace vectors that are too long

Browse Source
This commit is contained in:
Louis Dureuil 2024-01-29 11:22:24 +01:00
parent d77df4ecdb
commit cbd065ed46
No known key found for this signature in database
1 changed files with 2 additions and 1 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
milli/src/vector/hf.rs
View File

@ -145,7 +145,8 @@ impl Embedder {
let token_ids = tokens let token_ids = tokens
.iter() .iter()
.map(|tokens| { .map(|tokens| {
let tokens = tokens.get_ids().to_vec(); let mut tokens = tokens.get_ids().to_vec();
tokens.truncate(512);
Tensor::new(tokens.as_slice(), &self.model.device).map_err(EmbedError::tensor_shape) Tensor::new(tokens.as_slice(), &self.model.device).map_err(EmbedError::tensor_shape)
}) })
.collect::<Result<Vec<_>, EmbedError>>()?; .collect::<Result<Vec<_>, EmbedError>>()?;