searchengines-web/MeiliSearch
searchengines-web/MeiliSearch
1
0
Fork 0
mirror of https://github.com/meilisearch/MeiliSearch synced 2024-07-02 00:59:07 +02:00
Code Issues Releases Wiki Activity

user_provided => regenerate

Browse Source
This commit is contained in:
Louis Dureuil 2024-06-12 18:11:11 +02:00
parent a89eea233b
commit 3bc8f81abc
No known key found for this signature in database
6 changed files with 62 additions and 48 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
index-scheduler/src/batch.rs
View File

@ -958,10 +958,10 @@ impl IndexScheduler {
.is_some_and(|conf| conf.user_provided.contains(id));
let embeddings = ExplicitVectors {
embeddings: VectorOrArrayOfVectors::from_array_of_vectors(
embeddings,
embeddings: Some(
VectorOrArrayOfVectors::from_array_of_vectors(embeddings),
),
user_provided,
regenerate: !user_provided,
};
vectors.insert(
embedder_name,

5
meilisearch/src/routes/indexes/documents.rs
View File

@ -625,7 +625,10 @@ fn some_documents<'a, 't: 'a>(
.iter()
.find(|conf| conf.name == name)
.is_some_and(|conf| conf.user_provided.contains(key));
let embeddings = ExplicitVectors { embeddings: vector.into(), user_provided };
let embeddings = ExplicitVectors {
embeddings: Some(vector.into()),
regenerate: !user_provided,
};
vectors.insert(
name,
serde_json::to_value(embeddings).map_err(MeilisearchHttpError::from)?,

3
meilisearch/src/search.rs
View File

@ -1072,7 +1072,8 @@ fn make_hits(
.iter()
.find(|conf| conf.name == name)
.is_some_and(|conf| conf.user_provided.contains(id));
let embeddings = ExplicitVectors { embeddings: vector.into(), user_provided };
let embeddings =
ExplicitVectors { embeddings: Some(vector.into()), regenerate: !user_provided };
vectors.insert(name, serde_json::to_value(embeddings)?);
}
document.insert("_vectors".into(), vectors.into());

56
milli/src/update/index_documents/extract/extract_vector_points.rs
View File

@ -260,28 +260,33 @@ pub fn extract_vector_points<R: io::Read + io::Seek>(
// 2. an existing embedder changed so that it must regenerate all generated embeddings.
// For a new embedder, there can be `_vectors.embedder` embeddings to add to the DB
VectorState::Inline(vectors) => {
if vectors.is_user_provided() {
if !vectors.must_regenerate() {
add_to_user_provided.insert(docid);
}
let add_vectors = vectors.into_array_of_vectors();
if add_vectors.len() > usize::from(u8::MAX) {
return Err(crate::Error::UserError(crate::UserError::TooManyVectors(
document_id().to_string(),
add_vectors.len(),
)));
match vectors.into_array_of_vectors() {
Some(add_vectors) => {
if add_vectors.len() > usize::from(u8::MAX) {
return Err(crate::Error::UserError(
crate::UserError::TooManyVectors(
document_id().to_string(),
add_vectors.len(),
),
));
}
VectorStateDelta::NowManual(add_vectors)
}
None => VectorStateDelta::NoChange,
}
VectorStateDelta::NowManual(add_vectors)
}
// this happens only when an existing embedder changed. We cannot regenerate userProvided vectors
VectorState::InDb => VectorStateDelta::NoChange,
VectorState::Manual => VectorStateDelta::NoChange,
// generated vectors must be regenerated
VectorState::Generated => regenerate_prompt(obkv, prompt, new_fields_ids_map)?,
},
// prompt regeneration is only triggered for existing embedders
ExtractionAction::SettingsRegeneratePrompts { old_prompt } => {
if !old.is_user_provided() {
if old.must_regenerate() {
regenerate_if_prompt_changed(
obkv,
(old_prompt, prompt),
@ -362,31 +367,32 @@ fn extract_vector_document_diff(
(old_fields_ids_map, new_fields_ids_map): (&FieldsIdsMap, &FieldsIdsMap),
document_id: impl Fn() -> Value,
) -> Result<VectorStateDelta> {
match (old.is_user_provided(), new.is_user_provided()) {
match (old.must_regenerate(), new.must_regenerate()) {
(true, true) | (false, false) => {}
(true, false) => {
remove_from_user_provided.insert(docid);
add_to_user_provided.insert(docid);
}
(false, true) => {
add_to_user_provided.insert(docid);
remove_from_user_provided.insert(docid);
}
}
let delta = match (old, new) {
// regardless of the previous state, if a document now contains inline _vectors, they must
// be extracted manually
(_old, VectorState::Inline(new)) => {
let add_vectors = new.into_array_of_vectors();
(_old, VectorState::Inline(new)) => match new.into_array_of_vectors() {
Some(add_vectors) => {
if add_vectors.len() > usize::from(u8::MAX) {
return Err(crate::Error::UserError(crate::UserError::TooManyVectors(
document_id().to_string(),
add_vectors.len(),
)));
}
if add_vectors.len() > usize::from(u8::MAX) {
return Err(crate::Error::UserError(crate::UserError::TooManyVectors(
document_id().to_string(),
add_vectors.len(),
)));
VectorStateDelta::NowManual(add_vectors)
}
VectorStateDelta::NowManual(add_vectors)
}
None => VectorStateDelta::NoChange,
},
// no `_vectors` anywhere, we check for document removal and otherwise we regenerate the prompt if the
// document changed
(VectorState::Generated, VectorState::Generated) => {
@ -437,7 +443,7 @@ fn extract_vector_document_diff(
VectorStateDelta::NowRemoved
}
}
(_old, VectorState::InDb) => {
(_old, VectorState::Manual) => {
// Do we keep this document?
let document_is_kept = obkv
.iter()

6
milli/src/update/index_documents/transform.rs
View File

@ -1068,8 +1068,10 @@ impl<'a, 'i> Transform<'a, 'i> {
Some(Ok((
name.to_string(),
serde_json::to_value(ExplicitVectors {
embeddings: VectorOrArrayOfVectors::from_array_of_vectors(vectors),
user_provided: true,
embeddings: Some(VectorOrArrayOfVectors::from_array_of_vectors(
vectors,
)),
regenerate: false,
})
.unwrap(),
)))

34
milli/src/vector/parsed_vectors.rs
View File

@ -18,18 +18,20 @@ pub enum Vectors {
}
impl Vectors {
pub fn is_user_provided(&self) -> bool {
pub fn must_regenerate(&self) -> bool {
match self {
Vectors::ImplicitlyUserProvided(_) => true,
Vectors::Explicit(ExplicitVectors { user_provided, .. }) => *user_provided,
Vectors::ImplicitlyUserProvided(_) => false,
Vectors::Explicit(ExplicitVectors { regenerate, .. }) => *regenerate,
}
}
pub fn into_array_of_vectors(self) -> Vec<Embedding> {
pub fn into_array_of_vectors(self) -> Option<Vec<Embedding>> {
match self {
Vectors::ImplicitlyUserProvided(embeddings)
| Vectors::Explicit(ExplicitVectors { embeddings, user_provided: _ }) => {
embeddings.into_array_of_vectors().unwrap_or_default()
Vectors::ImplicitlyUserProvided(embeddings) => {
Some(embeddings.into_array_of_vectors().unwrap_or_default())
}
Vectors::Explicit(ExplicitVectors { embeddings, regenerate: _ }) => {
embeddings.map(|embeddings| embeddings.into_array_of_vectors().unwrap_or_default())
}
}
}
@ -38,22 +40,22 @@ impl Vectors {
#[derive(serde::Serialize, serde::Deserialize, Debug)]
#[serde(rename_all = "camelCase")]
pub struct ExplicitVectors {
pub embeddings: VectorOrArrayOfVectors,
pub user_provided: bool,
pub embeddings: Option<VectorOrArrayOfVectors>,
pub regenerate: bool,
}
pub enum VectorState {
Inline(Vectors),
InDb,
Manual,
Generated,
}
impl VectorState {
pub fn is_user_provided(&self) -> bool {
pub fn must_regenerate(&self) -> bool {
match self {
VectorState::Inline(vectors) => vectors.is_user_provided(),
VectorState::InDb => true,
VectorState::Generated => false,
VectorState::Inline(vectors) => vectors.must_regenerate(),
VectorState::Manual => false,
VectorState::Generated => true,
}
}
}
@ -96,7 +98,7 @@ impl ParsedVectorsDiff {
.flatten().map_or(BTreeMap::default(), |del| del.into_iter().map(|(name, vec)| (name, VectorState::Inline(vec))).collect());
for embedding_config in embedders_configs {
if embedding_config.user_provided.contains(docid) {
old.entry(embedding_config.name.to_string()).or_insert(VectorState::InDb);
old.entry(embedding_config.name.to_string()).or_insert(VectorState::Manual);
}
}
@ -121,7 +123,7 @@ impl ParsedVectorsDiff {
let old = self.old.remove(embedder_name).unwrap_or(VectorState::Generated);
let state_from_old = match old {
// assume a userProvided is still userProvided
VectorState::InDb => VectorState::InDb,
VectorState::Manual => VectorState::Manual,
// generated is still generated
VectorState::Generated => VectorState::Generated,
// weird case that shouldn't happen were the previous docs version is inline,