searchengines-web/MeiliSearch
searchengines-web/MeiliSearch
1
0
Fork 0
mirror of https://github.com/meilisearch/MeiliSearch synced 2024-12-31 17:01:39 +01:00
Code Issues Releases Wiki Activity

WIP

Browse Source
This commit is contained in:
Louis Dureuil 2023-12-07 17:35:45 +01:00
parent cb4ebe163e
commit fb539f61fe
No known key found for this signature in database
2 changed files with 19 additions and 16 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

28
milli/src/update/index_documents/extract/mod.rs
View File

@ -295,7 +295,7 @@ fn send_original_documents_data(
let (embedder, prompt) = embedders.get("default").cloned().unzip();
let result =
extract_vector_points(documents_chunk_cloned, indexer, field_id_map, prompt.as_deref());
let _ = match result {
match result {
Ok(ExtractedVectorPoints { manual_vectors, remove_vectors, prompts }) => {
/// FIXME: support multiple embedders
let results = embedder.and_then(|embedder| {
@ -309,15 +309,25 @@ fn send_original_documents_data(
});
let (embeddings, expected_dimension) = results.unzip();
let expected_dimension = expected_dimension.flatten();
lmdb_writer_sx_cloned.send(Ok(TypedChunk::VectorPoints {
remove_vectors,
embeddings,
/// FIXME: compute an expected dimension from the manual vectors if any
expected_dimension: expected_dimension.unwrap(),
manual_vectors,
}))
if !(remove_vectors.is_empty()
&& manual_vectors.is_empty()
&& embeddings.as_ref().map_or(true, |e| e.is_empty()))
{
/// FIXME FIXME FIXME
if expected_dimension.is_some() {
let _ = lmdb_writer_sx_cloned.send(Ok(TypedChunk::VectorPoints {
remove_vectors,
embeddings,
/// FIXME: compute an expected dimension from the manual vectors if any
expected_dimension: expected_dimension.unwrap(),
manual_vectors,
}));
}
}
}
Err(error) => {
let _ = lmdb_writer_sx_cloned.send(Err(error));
}
Err(error) => lmdb_writer_sx_cloned.send(Err(error)),
};
});

7
milli/src/update/index_documents/typed_chunk.rs
View File

@ -361,13 +361,6 @@ pub(crate) fn write_typed_chunk_into_index(
embeddings,
expected_dimension,
} => {
if remove_vectors.is_empty()
&& manual_vectors.is_empty()
&& embeddings.as_ref().map_or(true, |e| e.is_empty())
{
return Ok((RoaringBitmap::new(), is_merged_database));
}
/// FIXME: allow customizing distance
let writers: std::result::Result<Vec<_>, _> = (0..=u8::MAX)
.map(|k| {