searchengines-web/MeiliSearch
searchengines-web/MeiliSearch
1
0
Fork 0
mirror of https://github.com/meilisearch/MeiliSearch synced 2024-11-22 21:04:27 +01:00
Code Issues Releases Wiki Activity

WIP

Browse Source
This commit is contained in:
Louis Dureuil 2023-12-07 17:35:45 +01:00
parent cb4ebe163e
commit fb539f61fe
No known key found for this signature in database
2 changed files with 19 additions and 16 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

28
milli/src/update/index_documents/extract/mod.rs
View File

@ -295,7 +295,7 @@ fn send_original_documents_data(
let (embedder, prompt) = embedders.get("default").cloned().unzip(); let (embedder, prompt) = embedders.get("default").cloned().unzip();
let result = let result =
extract_vector_points(documents_chunk_cloned, indexer, field_id_map, prompt.as_deref()); extract_vector_points(documents_chunk_cloned, indexer, field_id_map, prompt.as_deref());
let _ = match result { match result {
Ok(ExtractedVectorPoints { manual_vectors, remove_vectors, prompts }) => { Ok(ExtractedVectorPoints { manual_vectors, remove_vectors, prompts }) => {
/// FIXME: support multiple embedders /// FIXME: support multiple embedders
let results = embedder.and_then(|embedder| { let results = embedder.and_then(|embedder| {
@ -309,15 +309,25 @@ fn send_original_documents_data(
}); });
let (embeddings, expected_dimension) = results.unzip(); let (embeddings, expected_dimension) = results.unzip();
let expected_dimension = expected_dimension.flatten(); let expected_dimension = expected_dimension.flatten();
lmdb_writer_sx_cloned.send(Ok(TypedChunk::VectorPoints { if !(remove_vectors.is_empty()
remove_vectors, && manual_vectors.is_empty()
embeddings, && embeddings.as_ref().map_or(true, |e| e.is_empty()))
/// FIXME: compute an expected dimension from the manual vectors if any {
expected_dimension: expected_dimension.unwrap(), /// FIXME FIXME FIXME
manual_vectors, if expected_dimension.is_some() {
})) let _ = lmdb_writer_sx_cloned.send(Ok(TypedChunk::VectorPoints {
remove_vectors,
embeddings,
/// FIXME: compute an expected dimension from the manual vectors if any
expected_dimension: expected_dimension.unwrap(),
manual_vectors,
}));
}
}
}
Err(error) => {
let _ = lmdb_writer_sx_cloned.send(Err(error));
} }
Err(error) => lmdb_writer_sx_cloned.send(Err(error)),
}; };
}); });

7
milli/src/update/index_documents/typed_chunk.rs
View File

@ -361,13 +361,6 @@ pub(crate) fn write_typed_chunk_into_index(
embeddings, embeddings,
expected_dimension, expected_dimension,
} => { } => {
if remove_vectors.is_empty()
&& manual_vectors.is_empty()
&& embeddings.as_ref().map_or(true, |e| e.is_empty())
{
return Ok((RoaringBitmap::new(), is_merged_database));
}
/// FIXME: allow customizing distance /// FIXME: allow customizing distance
let writers: std::result::Result<Vec<_>, _> = (0..=u8::MAX) let writers: std::result::Result<Vec<_>, _> = (0..=u8::MAX)
.map(|k| { .map(|k| {