searchengines-web/MeiliSearch
searchengines-web/MeiliSearch
1
0
Fork 0
mirror of https://github.com/meilisearch/MeiliSearch synced 2025-07-03 20:07:09 +02:00
Code Issues Releases Wiki Activity

Support no pattern when exporting

Browse source
This commit is contained in:
Clément Renault 2025-06-16 15:50:32 +02:00 • committed by Kerollmops
parent bc08cd0deb
commit 3329248a84
No known key found for this signature in database
GPG key ID: F250A4C4E3AE5F5F
3 changed files with 54 additions and 59 deletions
Download patch file Download diff file Expand all files Collapse all files

89
crates/index-scheduler/src/scheduler/process_export.rs
View file

@ -54,7 +54,7 @@ impl IndexScheduler {
indexes.len() as u32,
));
let ExportIndexSettings { skip_embeddings, filter } = settings;
let ExportIndexSettings { filter } = settings;
let index = self.index(uid)?;
let index_rtxn = index.read_txn()?;
@ -131,56 +131,53 @@ impl IndexScheduler {
.map_err(|e| Error::from_milli(e, Some(uid.to_string())))?;
// TODO definitely factorize this code
if !*skip_embeddings {
'inject_vectors: {
let embeddings = index
.embeddings(&index_rtxn, docid)
.map_err(|e| Error::from_milli(e, Some(uid.to_string())))?;
'inject_vectors: {
let embeddings = index
.embeddings(&index_rtxn, docid)
.map_err(|e| Error::from_milli(e, Some(uid.to_string())))?;
if embeddings.is_empty() {
break 'inject_vectors;
}
if embeddings.is_empty() {
break 'inject_vectors;
}
let vectors = document
.entry(RESERVED_VECTORS_FIELD_NAME)
.or_insert(serde_json::Value::Object(Default::default()));
let vectors = document
.entry(RESERVED_VECTORS_FIELD_NAME)
.or_insert(serde_json::Value::Object(Default::default()));
let serde_json::Value::Object(vectors) = vectors else {
return Err(Error::from_milli(
meilisearch_types::milli::Error::UserError(
meilisearch_types::milli::UserError::InvalidVectorsMapType {
document_id: {
if let Ok(Some(Ok(index))) = index
.external_id_of(&index_rtxn, std::iter::once(docid))
.map(|it| it.into_iter().next())
{
index
} else {
format!("internal docid={docid}")
}
},
value: vectors.clone(),
let serde_json::Value::Object(vectors) = vectors else {
return Err(Error::from_milli(
meilisearch_types::milli::Error::UserError(
meilisearch_types::milli::UserError::InvalidVectorsMapType {
document_id: {
if let Ok(Some(Ok(index))) = index
.external_id_of(&index_rtxn, std::iter::once(docid))
.map(|it| it.into_iter().next())
{
index
} else {
format!("internal docid={docid}")
}
},
),
Some(uid.to_string()),
));
value: vectors.clone(),
},
),
Some(uid.to_string()),
));
};
for (embedder_name, embeddings) in embeddings {
let user_provided = embedding_configs
.iter()
.find(|conf| conf.name == embedder_name)
.is_some_and(|conf| conf.user_provided.contains(docid));
let embeddings = ExplicitVectors {
embeddings: Some(VectorOrArrayOfVectors::from_array_of_vectors(
embeddings,
)),
regenerate: !user_provided,
};
for (embedder_name, embeddings) in embeddings {
let user_provided = embedding_configs
.iter()
.find(|conf| conf.name == embedder_name)
.is_some_and(|conf| conf.user_provided.contains(docid));
let embeddings = ExplicitVectors {
embeddings: Some(VectorOrArrayOfVectors::from_array_of_vectors(
embeddings,
)),
regenerate: !user_provided,
};
vectors
.insert(embedder_name, serde_json::to_value(embeddings).unwrap());
}
vectors.insert(embedder_name, serde_json::to_value(embeddings).unwrap());
}
}

3
crates/meilisearch-types/src/tasks.rs
View file

@ -171,10 +171,9 @@ pub struct IndexSwap {
pub indexes: (String, String),
}
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize, ToSchema)]
#[derive(Debug, Default, Clone, PartialEq, Eq, Serialize, Deserialize, ToSchema)]
#[serde(rename_all = "camelCase")]
pub struct ExportIndexSettings {
pub skip_embeddings: bool,
pub filter: Option<String>,
}

21
crates/meilisearch/src/routes/export.rs
View file

@ -72,16 +72,19 @@ async fn export(
debug!(returns = ?export, "Trigger export");
let Export { url, api_key, indexes } = export;
let task = KindWithContent::Export {
url,
api_key,
indexes: indexes
let indexes = if indexes.is_empty() {
BTreeMap::from([(IndexUidPattern::new_unchecked("*"), DbExportIndexSettings::default())])
} else {
indexes
.into_iter()
.map(|(pattern, ExportIndexSettings { skip_embeddings, filter })| {
(pattern, DbExportIndexSettings { skip_embeddings, filter })
.map(|(pattern, ExportIndexSettings { filter })| {
(pattern, DbExportIndexSettings { filter })
})
.collect(),
.collect()
};
let task = KindWithContent::Export { url, api_key, indexes };
let uid = get_task_id(&req, &opt)?;
let dry_run = is_dry_run(&req, &opt)?;
let task: SummarizedTaskView =
@ -116,10 +119,6 @@ pub struct Export {
#[serde(rename_all = "camelCase")]
#[schema(rename_all = "camelCase")]
pub struct ExportIndexSettings {
#[schema(value_type = Option<bool>, example = json!("true"))]
#[serde(default)]
#[deserr(default, error = DeserrJsonError<InvalidExportIndexSkipEmbeddings>)]
pub skip_embeddings: bool,
#[schema(value_type = Option<String>, example = json!("genres = action"))]
#[serde(default)]
#[deserr(default, error = DeserrJsonError<InvalidExportIndexFilter>)]