mirror of
https://github.com/meilisearch/MeiliSearch
synced 2024-11-25 22:34:28 +01:00
Merge #4801
4801: AI quality-of-life improvements r=irevoire a=dureuill # Pull Request ## Related issue Fixes #4802 ## What does this PR do? This PR implements several quality-of-life improvements described in the [public usage](https://meilisearch.notion.site/v1-10-AI-search-changes-737c9d7d010d4dd685582bf5dab579e2#ece824a1814e47a0a986d786baff1be9) Co-authored-by: Louis Dureuil <louis@meilisearch.com>
This commit is contained in:
commit
7a292b572a
@ -415,7 +415,9 @@ impl ErrorCode for milli::Error {
|
||||
Code::InvalidSettingsTypoTolerance
|
||||
}
|
||||
UserError::InvalidEmbedder(_) => Code::InvalidEmbedder,
|
||||
UserError::VectorEmbeddingError(_) => Code::VectorEmbeddingError,
|
||||
UserError::VectorEmbeddingError(_) | UserError::DocumentEmbeddingError(_) => {
|
||||
Code::VectorEmbeddingError
|
||||
}
|
||||
UserError::DocumentEditionCannotModifyPrimaryKey
|
||||
| UserError::DocumentEditionDocumentMustBeObject
|
||||
| UserError::DocumentEditionRuntimeError(_)
|
||||
|
@ -645,7 +645,12 @@ async fn get_document_with_vectors() {
|
||||
{
|
||||
"id": 1,
|
||||
"name": "echo",
|
||||
"_vectors": {}
|
||||
"_vectors": {
|
||||
"manual": {
|
||||
"embeddings": [],
|
||||
"regenerate": false
|
||||
}
|
||||
}
|
||||
}
|
||||
],
|
||||
"offset": 0,
|
||||
@ -701,7 +706,12 @@ async fn get_document_with_vectors() {
|
||||
},
|
||||
{
|
||||
"name": "echo",
|
||||
"_vectors": {}
|
||||
"_vectors": {
|
||||
"manual": {
|
||||
"embeddings": [],
|
||||
"regenerate": false
|
||||
}
|
||||
}
|
||||
}
|
||||
],
|
||||
"offset": 0,
|
||||
|
@ -120,7 +120,12 @@ async fn add_remove_user_provided() {
|
||||
{
|
||||
"id": 1,
|
||||
"name": "echo",
|
||||
"_vectors": {}
|
||||
"_vectors": {
|
||||
"manual": {
|
||||
"embeddings": [],
|
||||
"regenerate": false
|
||||
}
|
||||
}
|
||||
}
|
||||
],
|
||||
"offset": 0,
|
||||
@ -142,7 +147,12 @@ async fn add_remove_user_provided() {
|
||||
{
|
||||
"id": 1,
|
||||
"name": "echo",
|
||||
"_vectors": {}
|
||||
"_vectors": {
|
||||
"manual": {
|
||||
"embeddings": [],
|
||||
"regenerate": false
|
||||
}
|
||||
}
|
||||
}
|
||||
],
|
||||
"offset": 0,
|
||||
@ -471,6 +481,99 @@ async fn user_provided_embeddings_error() {
|
||||
"###);
|
||||
}
|
||||
|
||||
#[actix_rt::test]
|
||||
async fn user_provided_vectors_error() {
|
||||
let server = Server::new().await;
|
||||
let index = generate_default_user_provided_documents(&server).await;
|
||||
|
||||
// First case, we forget to specify `_vectors`
|
||||
let documents = json!({"id": 42, "name": "kefir"});
|
||||
let (value, code) = index.add_documents(documents, None).await;
|
||||
snapshot!(code, @"202 Accepted");
|
||||
let task = index.wait_task(value.uid()).await;
|
||||
snapshot!(task, @r###"
|
||||
{
|
||||
"uid": 2,
|
||||
"indexUid": "doggo",
|
||||
"status": "failed",
|
||||
"type": "documentAdditionOrUpdate",
|
||||
"canceledBy": null,
|
||||
"details": {
|
||||
"receivedDocuments": 1,
|
||||
"indexedDocuments": 0
|
||||
},
|
||||
"error": {
|
||||
"message": "While embedding documents for embedder `manual`: user error: attempt to embed the following text in a configuration where embeddings must be user provided: \" id: 42\\n name: kefir\\n _vectors: \\n _vectors.manual: \\n _vectors.manual.regenerate: \\n _vectors.manual.embeddings: \\n\"\n- Note: `manual` has `source: userProvided`, so documents must provide embeddings as an array in `_vectors.manual`.\n- Hint: opt-out for a document with `_vectors.manual: null`",
|
||||
"code": "vector_embedding_error",
|
||||
"type": "invalid_request",
|
||||
"link": "https://docs.meilisearch.com/errors#vector_embedding_error"
|
||||
},
|
||||
"duration": "[duration]",
|
||||
"enqueuedAt": "[date]",
|
||||
"startedAt": "[date]",
|
||||
"finishedAt": "[date]"
|
||||
}
|
||||
"###);
|
||||
|
||||
// Second case, we provide `_vectors` with a typo
|
||||
let documents = json!({"id": 42, "name": "kefir", "_vector": { "manaul": [0, 0, 0] }});
|
||||
let (value, code) = index.add_documents(documents, None).await;
|
||||
snapshot!(code, @"202 Accepted");
|
||||
let task = index.wait_task(value.uid()).await;
|
||||
snapshot!(task, @r###"
|
||||
{
|
||||
"uid": 3,
|
||||
"indexUid": "doggo",
|
||||
"status": "failed",
|
||||
"type": "documentAdditionOrUpdate",
|
||||
"canceledBy": null,
|
||||
"details": {
|
||||
"receivedDocuments": 1,
|
||||
"indexedDocuments": 0
|
||||
},
|
||||
"error": {
|
||||
"message": "While embedding documents for embedder `manual`: user error: attempt to embed the following text in a configuration where embeddings must be user provided: \" id: 42\\n name: kefir\\n _vectors: \\n _vectors.manual: \\n _vectors.manual.regenerate: \\n _vectors.manual.embeddings: \\n _vector: manaul000\\n _vector.manaul: \\n\"\n- Note: `manual` has `source: userProvided`, so documents must provide embeddings as an array in `_vectors.manual`.\n- Hint: try replacing `_vector` by `_vectors` in 1 document(s).",
|
||||
"code": "vector_embedding_error",
|
||||
"type": "invalid_request",
|
||||
"link": "https://docs.meilisearch.com/errors#vector_embedding_error"
|
||||
},
|
||||
"duration": "[duration]",
|
||||
"enqueuedAt": "[date]",
|
||||
"startedAt": "[date]",
|
||||
"finishedAt": "[date]"
|
||||
}
|
||||
"###);
|
||||
|
||||
// Third case, we specify the embedder with a typo
|
||||
let documents = json!({"id": 42, "name": "kefir", "_vectors": { "manaul": [0, 0, 0] }});
|
||||
let (value, code) = index.add_documents(documents, None).await;
|
||||
snapshot!(code, @"202 Accepted");
|
||||
let task = index.wait_task(value.uid()).await;
|
||||
snapshot!(task, @r###"
|
||||
{
|
||||
"uid": 4,
|
||||
"indexUid": "doggo",
|
||||
"status": "failed",
|
||||
"type": "documentAdditionOrUpdate",
|
||||
"canceledBy": null,
|
||||
"details": {
|
||||
"receivedDocuments": 1,
|
||||
"indexedDocuments": 0
|
||||
},
|
||||
"error": {
|
||||
"message": "While embedding documents for embedder `manual`: user error: attempt to embed the following text in a configuration where embeddings must be user provided: \" id: 42\\n name: kefir\\n _vectors: manaul000\\n _vectors.manual: \\n _vectors.manual.regenerate: \\n _vectors.manual.embeddings: \\n _vectors.manaul: \\n\"\n- Note: `manual` has `source: userProvided`, so documents must provide embeddings as an array in `_vectors.manual`.\n- Hint: try replacing `_vectors.manaul` by `_vectors.manual` in 1 document(s).",
|
||||
"code": "vector_embedding_error",
|
||||
"type": "invalid_request",
|
||||
"link": "https://docs.meilisearch.com/errors#vector_embedding_error"
|
||||
},
|
||||
"duration": "[duration]",
|
||||
"enqueuedAt": "[date]",
|
||||
"startedAt": "[date]",
|
||||
"finishedAt": "[date]"
|
||||
}
|
||||
"###);
|
||||
}
|
||||
|
||||
#[actix_rt::test]
|
||||
async fn clear_documents() {
|
||||
let server = Server::new().await;
|
||||
@ -580,7 +683,12 @@ async fn add_remove_one_vector_4588() {
|
||||
{
|
||||
"id": 0,
|
||||
"name": "kefir",
|
||||
"_vectors": {}
|
||||
"_vectors": {
|
||||
"manual": {
|
||||
"embeddings": [],
|
||||
"regenerate": false
|
||||
}
|
||||
}
|
||||
}
|
||||
],
|
||||
"offset": 0,
|
||||
|
@ -268,15 +268,17 @@ only composed of alphanumeric characters (a-z A-Z 0-9), hyphens (-) and undersco
|
||||
DocumentEditionRuntimeError(Box<EvalAltResult>),
|
||||
#[error("Document edition runtime error encountered while compiling the function: {0}")]
|
||||
DocumentEditionCompilationError(rhai::ParseError),
|
||||
#[error("{0}")]
|
||||
DocumentEmbeddingError(String),
|
||||
}
|
||||
|
||||
impl From<crate::vector::Error> for Error {
|
||||
fn from(value: crate::vector::Error) -> Self {
|
||||
match value.fault() {
|
||||
FaultSource::User => Error::UserError(value.into()),
|
||||
FaultSource::Runtime => Error::InternalError(value.into()),
|
||||
FaultSource::Runtime => Error::UserError(value.into()),
|
||||
FaultSource::Bug => Error::InternalError(value.into()),
|
||||
FaultSource::Undecided => Error::InternalError(value.into()),
|
||||
FaultSource::Undecided => Error::UserError(value.into()),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
@ -1691,9 +1691,7 @@ impl Index {
|
||||
}
|
||||
}
|
||||
|
||||
if !embeddings.is_empty() {
|
||||
res.insert(embedder_name.to_owned(), embeddings);
|
||||
}
|
||||
res.insert(embedder_name.to_owned(), embeddings);
|
||||
}
|
||||
Ok(res)
|
||||
}
|
||||
|
@ -13,13 +13,15 @@ use roaring::RoaringBitmap;
|
||||
use serde_json::Value;
|
||||
|
||||
use super::helpers::{create_writer, writer_into_reader, GrenadParameters};
|
||||
use crate::error::FaultSource;
|
||||
use crate::index::IndexEmbeddingConfig;
|
||||
use crate::prompt::Prompt;
|
||||
use crate::update::del_add::{DelAdd, KvReaderDelAdd, KvWriterDelAdd};
|
||||
use crate::update::settings::InnerIndexSettingsDiff;
|
||||
use crate::vector::error::{EmbedErrorKind, PossibleEmbeddingMistakes, UnusedVectorsDistribution};
|
||||
use crate::vector::parsed_vectors::{ParsedVectorsDiff, VectorState, RESERVED_VECTORS_FIELD_NAME};
|
||||
use crate::vector::settings::{EmbedderAction, ReindexAction};
|
||||
use crate::vector::Embedder;
|
||||
use crate::vector::{Embedder, Embeddings};
|
||||
use crate::{try_split_array_at, DocumentId, FieldId, FieldsIdsMap, Result, ThreadPoolNoAbort};
|
||||
|
||||
/// The length of the elements that are always in the buffer when inserting new values.
|
||||
@ -102,7 +104,8 @@ pub fn extract_vector_points<R: io::Read + io::Seek>(
|
||||
indexer: GrenadParameters,
|
||||
embedders_configs: &[IndexEmbeddingConfig],
|
||||
settings_diff: &InnerIndexSettingsDiff,
|
||||
) -> Result<Vec<ExtractedVectorPoints>> {
|
||||
) -> Result<(Vec<ExtractedVectorPoints>, UnusedVectorsDistribution)> {
|
||||
let mut unused_vectors_distribution = UnusedVectorsDistribution::new();
|
||||
let reindex_vectors = settings_diff.reindex_vectors();
|
||||
|
||||
let old_fields_ids_map = &settings_diff.old.fields_ids_map;
|
||||
@ -319,6 +322,8 @@ pub fn extract_vector_points<R: io::Read + io::Seek>(
|
||||
delta,
|
||||
)?;
|
||||
}
|
||||
|
||||
unused_vectors_distribution.append(parsed_vectors);
|
||||
}
|
||||
|
||||
let mut results = Vec::new();
|
||||
@ -355,7 +360,7 @@ pub fn extract_vector_points<R: io::Read + io::Seek>(
|
||||
})
|
||||
}
|
||||
|
||||
Ok(results)
|
||||
Ok((results, unused_vectors_distribution))
|
||||
}
|
||||
|
||||
fn extract_vector_document_diff(
|
||||
@ -547,6 +552,9 @@ pub fn extract_embeddings<R: io::Read + io::Seek>(
|
||||
prompt_reader: grenad::Reader<R>,
|
||||
indexer: GrenadParameters,
|
||||
embedder: Arc<Embedder>,
|
||||
embedder_name: &str,
|
||||
possible_embedding_mistakes: &PossibleEmbeddingMistakes,
|
||||
unused_vectors_distribution: &UnusedVectorsDistribution,
|
||||
request_threads: &ThreadPoolNoAbort,
|
||||
) -> Result<grenad::Reader<BufReader<File>>> {
|
||||
let n_chunks = embedder.chunk_count_hint(); // chunk level parallelism
|
||||
@ -583,13 +591,14 @@ pub fn extract_embeddings<R: io::Read + io::Seek>(
|
||||
current_chunk_ids.push(docid);
|
||||
|
||||
if chunks.len() == chunks.capacity() {
|
||||
let chunked_embeds = embedder
|
||||
.embed_chunks(
|
||||
std::mem::replace(&mut chunks, Vec::with_capacity(n_chunks)),
|
||||
request_threads,
|
||||
)
|
||||
.map_err(crate::vector::Error::from)
|
||||
.map_err(crate::Error::from)?;
|
||||
let chunked_embeds = embed_chunks(
|
||||
&embedder,
|
||||
std::mem::replace(&mut chunks, Vec::with_capacity(n_chunks)),
|
||||
embedder_name,
|
||||
possible_embedding_mistakes,
|
||||
unused_vectors_distribution,
|
||||
request_threads,
|
||||
)?;
|
||||
|
||||
for (docid, embeddings) in chunks_ids
|
||||
.iter()
|
||||
@ -604,10 +613,14 @@ pub fn extract_embeddings<R: io::Read + io::Seek>(
|
||||
|
||||
// send last chunk
|
||||
if !chunks.is_empty() {
|
||||
let chunked_embeds = embedder
|
||||
.embed_chunks(std::mem::take(&mut chunks), request_threads)
|
||||
.map_err(crate::vector::Error::from)
|
||||
.map_err(crate::Error::from)?;
|
||||
let chunked_embeds = embed_chunks(
|
||||
&embedder,
|
||||
std::mem::take(&mut chunks),
|
||||
embedder_name,
|
||||
possible_embedding_mistakes,
|
||||
unused_vectors_distribution,
|
||||
request_threads,
|
||||
)?;
|
||||
for (docid, embeddings) in chunks_ids
|
||||
.iter()
|
||||
.flat_map(|docids| docids.iter())
|
||||
@ -618,10 +631,14 @@ pub fn extract_embeddings<R: io::Read + io::Seek>(
|
||||
}
|
||||
|
||||
if !current_chunk.is_empty() {
|
||||
let embeds = embedder
|
||||
.embed_chunks(vec![std::mem::take(&mut current_chunk)], request_threads)
|
||||
.map_err(crate::vector::Error::from)
|
||||
.map_err(crate::Error::from)?;
|
||||
let embeds = embed_chunks(
|
||||
&embedder,
|
||||
vec![std::mem::take(&mut current_chunk)],
|
||||
embedder_name,
|
||||
possible_embedding_mistakes,
|
||||
unused_vectors_distribution,
|
||||
request_threads,
|
||||
)?;
|
||||
|
||||
if let Some(embeds) = embeds.first() {
|
||||
for (docid, embeddings) in current_chunk_ids.iter().zip(embeds.iter()) {
|
||||
@ -632,3 +649,57 @@ pub fn extract_embeddings<R: io::Read + io::Seek>(
|
||||
|
||||
writer_into_reader(state_writer)
|
||||
}
|
||||
|
||||
fn embed_chunks(
|
||||
embedder: &Embedder,
|
||||
text_chunks: Vec<Vec<String>>,
|
||||
embedder_name: &str,
|
||||
possible_embedding_mistakes: &PossibleEmbeddingMistakes,
|
||||
unused_vectors_distribution: &UnusedVectorsDistribution,
|
||||
request_threads: &ThreadPoolNoAbort,
|
||||
) -> Result<Vec<Vec<Embeddings<f32>>>> {
|
||||
match embedder.embed_chunks(text_chunks, request_threads) {
|
||||
Ok(chunks) => Ok(chunks),
|
||||
Err(error) => {
|
||||
if let FaultSource::Bug = error.fault {
|
||||
Err(crate::Error::InternalError(crate::InternalError::VectorEmbeddingError(
|
||||
error.into(),
|
||||
)))
|
||||
} else {
|
||||
let mut msg =
|
||||
format!(r"While embedding documents for embedder `{embedder_name}`: {error}");
|
||||
|
||||
if let EmbedErrorKind::ManualEmbed(_) = &error.kind {
|
||||
msg += &format!("\n- Note: `{embedder_name}` has `source: userProvided`, so documents must provide embeddings as an array in `_vectors.{embedder_name}`.");
|
||||
}
|
||||
|
||||
let mut hint_count = 0;
|
||||
|
||||
for (vector_misspelling, count) in
|
||||
possible_embedding_mistakes.vector_mistakes().take(2)
|
||||
{
|
||||
msg += &format!("\n- Hint: try replacing `{vector_misspelling}` by `_vectors` in {count} document(s).");
|
||||
hint_count += 1;
|
||||
}
|
||||
|
||||
for (embedder_misspelling, count) in possible_embedding_mistakes
|
||||
.embedder_mistakes(embedder_name, unused_vectors_distribution)
|
||||
.take(2)
|
||||
{
|
||||
msg += &format!("\n- Hint: try replacing `_vectors.{embedder_misspelling}` by `_vectors.{embedder_name}` in {count} document(s).");
|
||||
hint_count += 1;
|
||||
}
|
||||
|
||||
if hint_count == 0 {
|
||||
if let EmbedErrorKind::ManualEmbed(_) = &error.kind {
|
||||
msg += &format!(
|
||||
"\n- Hint: opt-out for a document with `_vectors.{embedder_name}: null`"
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
Err(crate::Error::UserError(crate::UserError::DocumentEmbeddingError(msg)))
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
@ -32,6 +32,7 @@ use super::helpers::{as_cloneable_grenad, CursorClonableMmap, GrenadParameters};
|
||||
use super::{helpers, TypedChunk};
|
||||
use crate::index::IndexEmbeddingConfig;
|
||||
use crate::update::settings::InnerIndexSettingsDiff;
|
||||
use crate::vector::error::PossibleEmbeddingMistakes;
|
||||
use crate::{FieldId, Result, ThreadPoolNoAbort, ThreadPoolNoAbortBuilder};
|
||||
|
||||
/// Extract data for each databases from obkv documents in parallel.
|
||||
@ -47,6 +48,7 @@ pub(crate) fn data_from_obkv_documents(
|
||||
embedders_configs: Arc<Vec<IndexEmbeddingConfig>>,
|
||||
settings_diff: Arc<InnerIndexSettingsDiff>,
|
||||
max_positions_per_attributes: Option<u32>,
|
||||
possible_embedding_mistakes: Arc<PossibleEmbeddingMistakes>,
|
||||
) -> Result<()> {
|
||||
let (original_pipeline_result, flattened_pipeline_result): (Result<_>, Result<_>) = rayon::join(
|
||||
|| {
|
||||
@ -59,6 +61,7 @@ pub(crate) fn data_from_obkv_documents(
|
||||
lmdb_writer_sx.clone(),
|
||||
embedders_configs.clone(),
|
||||
settings_diff.clone(),
|
||||
possible_embedding_mistakes.clone(),
|
||||
)
|
||||
})
|
||||
.collect::<Result<()>>()
|
||||
@ -227,6 +230,7 @@ fn send_original_documents_data(
|
||||
lmdb_writer_sx: Sender<Result<TypedChunk>>,
|
||||
embedders_configs: Arc<Vec<IndexEmbeddingConfig>>,
|
||||
settings_diff: Arc<InnerIndexSettingsDiff>,
|
||||
possible_embedding_mistakes: Arc<PossibleEmbeddingMistakes>,
|
||||
) -> Result<()> {
|
||||
let original_documents_chunk =
|
||||
original_documents_chunk.and_then(|c| unsafe { as_cloneable_grenad(&c) })?;
|
||||
@ -248,7 +252,7 @@ fn send_original_documents_data(
|
||||
&embedders_configs,
|
||||
&settings_diff,
|
||||
) {
|
||||
Ok(extracted_vectors) => {
|
||||
Ok((extracted_vectors, unused_vectors_distribution)) => {
|
||||
for ExtractedVectorPoints {
|
||||
manual_vectors,
|
||||
remove_vectors,
|
||||
@ -263,6 +267,9 @@ fn send_original_documents_data(
|
||||
prompts,
|
||||
indexer,
|
||||
embedder.clone(),
|
||||
&embedder_name,
|
||||
&possible_embedding_mistakes,
|
||||
&unused_vectors_distribution,
|
||||
request_threads(),
|
||||
) {
|
||||
Ok(results) => Some(results),
|
||||
|
@ -427,6 +427,9 @@ where
|
||||
let settings_diff = Arc::new(settings_diff);
|
||||
let embedders_configs = Arc::new(self.index.embedding_configs(self.wtxn)?);
|
||||
|
||||
let possible_embedding_mistakes =
|
||||
crate::vector::error::PossibleEmbeddingMistakes::new(&field_distribution);
|
||||
|
||||
let backup_pool;
|
||||
let pool = match self.indexer_config.thread_pool {
|
||||
Some(ref pool) => pool,
|
||||
@ -542,6 +545,7 @@ where
|
||||
embedders_configs.clone(),
|
||||
settings_diff_cloned,
|
||||
max_positions_per_attributes,
|
||||
Arc::new(possible_embedding_mistakes)
|
||||
)
|
||||
});
|
||||
|
||||
|
@ -1574,7 +1574,6 @@ pub fn validate_embedding_settings(
|
||||
EmbedderSource::OpenAi => {
|
||||
check_unset(&revision, EmbeddingSettings::REVISION, inferred_source, name)?;
|
||||
|
||||
check_unset(&url, EmbeddingSettings::URL, inferred_source, name)?;
|
||||
check_unset(&query, EmbeddingSettings::QUERY, inferred_source, name)?;
|
||||
check_unset(&input_field, EmbeddingSettings::INPUT_FIELD, inferred_source, name)?;
|
||||
check_unset(
|
||||
|
@ -1,9 +1,11 @@
|
||||
use std::collections::BTreeMap;
|
||||
use std::path::PathBuf;
|
||||
|
||||
use hf_hub::api::sync::ApiError;
|
||||
|
||||
use super::parsed_vectors::ParsedVectorsDiff;
|
||||
use crate::error::FaultSource;
|
||||
use crate::PanicCatched;
|
||||
use crate::{FieldDistribution, PanicCatched};
|
||||
|
||||
#[derive(Debug, thiserror::Error)]
|
||||
#[error("Error while generating embeddings: {inner}")]
|
||||
@ -310,3 +312,68 @@ pub enum NewEmbedderErrorKind {
|
||||
#[error("loading model failed: {0}")]
|
||||
LoadModel(candle_core::Error),
|
||||
}
|
||||
|
||||
pub struct PossibleEmbeddingMistakes {
|
||||
vectors_mistakes: BTreeMap<String, u64>,
|
||||
}
|
||||
|
||||
impl PossibleEmbeddingMistakes {
|
||||
pub fn new(field_distribution: &FieldDistribution) -> Self {
|
||||
let mut vectors_mistakes = BTreeMap::new();
|
||||
let builder = levenshtein_automata::LevenshteinAutomatonBuilder::new(2, true);
|
||||
let automata = builder.build_dfa("_vectors");
|
||||
for (field, count) in field_distribution {
|
||||
if *count == 0 {
|
||||
continue;
|
||||
}
|
||||
if field.contains('.') {
|
||||
continue;
|
||||
}
|
||||
match automata.eval(field) {
|
||||
levenshtein_automata::Distance::Exact(0) => continue,
|
||||
levenshtein_automata::Distance::Exact(_) => {
|
||||
vectors_mistakes.insert(field.to_string(), *count);
|
||||
}
|
||||
levenshtein_automata::Distance::AtLeast(_) => continue,
|
||||
}
|
||||
}
|
||||
|
||||
Self { vectors_mistakes }
|
||||
}
|
||||
|
||||
pub fn vector_mistakes(&self) -> impl Iterator<Item = (&str, u64)> {
|
||||
self.vectors_mistakes.iter().map(|(misspelling, count)| (misspelling.as_str(), *count))
|
||||
}
|
||||
|
||||
pub fn embedder_mistakes<'a>(
|
||||
&'a self,
|
||||
embedder_name: &'a str,
|
||||
unused_vectors_distributions: &'a UnusedVectorsDistribution,
|
||||
) -> impl Iterator<Item = (&'a str, u64)> + 'a {
|
||||
let builder = levenshtein_automata::LevenshteinAutomatonBuilder::new(2, true);
|
||||
let automata = builder.build_dfa(embedder_name);
|
||||
|
||||
unused_vectors_distributions.0.iter().filter_map(move |(field, count)| {
|
||||
match automata.eval(field) {
|
||||
levenshtein_automata::Distance::Exact(0) => None,
|
||||
levenshtein_automata::Distance::Exact(_) => Some((field.as_str(), *count)),
|
||||
levenshtein_automata::Distance::AtLeast(_) => None,
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Default)]
|
||||
pub struct UnusedVectorsDistribution(BTreeMap<String, u64>);
|
||||
|
||||
impl UnusedVectorsDistribution {
|
||||
pub fn new() -> Self {
|
||||
Self::default()
|
||||
}
|
||||
|
||||
pub fn append(&mut self, parsed_vectors_diff: ParsedVectorsDiff) {
|
||||
for name in parsed_vectors_diff.into_new_vectors_keys_iter() {
|
||||
*self.0.entry(name).or_default() += 1;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
@ -20,7 +20,7 @@ impl Embedder {
|
||||
|
||||
pub fn embed(&self, mut texts: Vec<String>) -> Result<Vec<Embeddings<f32>>, EmbedError> {
|
||||
let Some(text) = texts.pop() else { return Ok(Default::default()) };
|
||||
Err(EmbedError::embed_on_manual_embedder(text))
|
||||
Err(EmbedError::embed_on_manual_embedder(text.chars().take(250).collect()))
|
||||
}
|
||||
|
||||
pub fn dimensions(&self) -> usize {
|
||||
|
@ -10,6 +10,7 @@ use crate::ThreadPoolNoAbort;
|
||||
|
||||
#[derive(Debug, Clone, Hash, PartialEq, Eq, serde::Deserialize, serde::Serialize)]
|
||||
pub struct EmbedderOptions {
|
||||
pub url: Option<String>,
|
||||
pub api_key: Option<String>,
|
||||
pub embedding_model: EmbeddingModel,
|
||||
pub dimensions: Option<usize>,
|
||||
@ -146,11 +147,13 @@ pub const OPENAI_EMBEDDINGS_URL: &str = "https://api.openai.com/v1/embeddings";
|
||||
|
||||
impl EmbedderOptions {
|
||||
pub fn with_default_model(api_key: Option<String>) -> Self {
|
||||
Self { api_key, embedding_model: Default::default(), dimensions: None, distribution: None }
|
||||
}
|
||||
|
||||
pub fn with_embedding_model(api_key: Option<String>, embedding_model: EmbeddingModel) -> Self {
|
||||
Self { api_key, embedding_model, dimensions: None, distribution: None }
|
||||
Self {
|
||||
api_key,
|
||||
embedding_model: Default::default(),
|
||||
dimensions: None,
|
||||
distribution: None,
|
||||
url: None,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@ -175,11 +178,13 @@ impl Embedder {
|
||||
&inferred_api_key
|
||||
});
|
||||
|
||||
let url = options.url.as_deref().unwrap_or(OPENAI_EMBEDDINGS_URL).to_owned();
|
||||
|
||||
let rest_embedder = RestEmbedder::new(RestEmbedderOptions {
|
||||
api_key: Some(api_key.clone()),
|
||||
distribution: None,
|
||||
dimensions: Some(options.dimensions()),
|
||||
url: OPENAI_EMBEDDINGS_URL.to_owned(),
|
||||
url,
|
||||
query: options.query(),
|
||||
input_field: vec!["input".to_owned()],
|
||||
input_type: crate::vector::rest::InputType::TextArray,
|
||||
@ -205,7 +210,6 @@ impl Embedder {
|
||||
}
|
||||
|
||||
fn try_embed_tokenized(&self, text: &[String]) -> Result<Vec<Embeddings<f32>>, EmbedError> {
|
||||
pub const OVERLAP_SIZE: usize = 200;
|
||||
let mut all_embeddings = Vec::with_capacity(text.len());
|
||||
for text in text {
|
||||
let max_token_count = self.options.embedding_model.max_token();
|
||||
@ -216,21 +220,10 @@ impl Embedder {
|
||||
continue;
|
||||
}
|
||||
|
||||
let mut tokens = encoded.as_slice();
|
||||
let tokens = &encoded.as_slice()[0..max_token_count];
|
||||
let mut embeddings_for_prompt = Embeddings::new(self.dimensions());
|
||||
while tokens.len() > max_token_count {
|
||||
let window = &tokens[..max_token_count];
|
||||
let embedding = self.rest_embedder.embed_tokens(window)?;
|
||||
embeddings_for_prompt.append(embedding.into_inner()).map_err(|got| {
|
||||
EmbedError::openai_unexpected_dimension(self.dimensions(), got.len())
|
||||
})?;
|
||||
|
||||
tokens = &tokens[max_token_count - OVERLAP_SIZE..];
|
||||
}
|
||||
|
||||
// end of text
|
||||
let embedding = self.rest_embedder.embed_tokens(tokens)?;
|
||||
|
||||
embeddings_for_prompt.append(embedding.into_inner()).map_err(|got| {
|
||||
EmbedError::openai_unexpected_dimension(self.dimensions(), got.len())
|
||||
})?;
|
||||
|
@ -179,6 +179,15 @@ impl ParsedVectorsDiff {
|
||||
|
||||
(old, new)
|
||||
}
|
||||
|
||||
pub fn into_new_vectors_keys_iter(self) -> impl Iterator<Item = String> {
|
||||
let maybe_it = match self.new {
|
||||
VectorsState::NoVectorsFid => None,
|
||||
VectorsState::NoVectorsFieldInDocument => None,
|
||||
VectorsState::Vectors(vectors) => Some(vectors.into_keys()),
|
||||
};
|
||||
maybe_it.into_iter().flatten()
|
||||
}
|
||||
}
|
||||
|
||||
pub struct ParsedVectors(pub BTreeMap<String, Vectors>);
|
||||
|
@ -1,4 +1,5 @@
|
||||
use deserr::Deserr;
|
||||
use rand::Rng;
|
||||
use rayon::iter::{IntoParallelIterator as _, ParallelIterator as _};
|
||||
use serde::{Deserialize, Serialize};
|
||||
|
||||
@ -264,7 +265,7 @@ where
|
||||
}
|
||||
};
|
||||
|
||||
for attempt in 0..7 {
|
||||
for attempt in 0..10 {
|
||||
let response = request.clone().send_json(&body);
|
||||
let result = check_response(response);
|
||||
|
||||
@ -277,6 +278,11 @@ where
|
||||
}?;
|
||||
|
||||
let retry_duration = retry_duration.min(std::time::Duration::from_secs(60)); // don't wait more than a minute
|
||||
|
||||
// randomly up to double the retry duration
|
||||
let retry_duration = retry_duration
|
||||
+ rand::thread_rng().gen_range(std::time::Duration::ZERO..retry_duration);
|
||||
|
||||
tracing::warn!("Attempt #{}, retrying after {}ms.", attempt, retry_duration.as_millis());
|
||||
std::thread::sleep(retry_duration);
|
||||
}
|
||||
|
@ -166,7 +166,16 @@ impl SettingsDiff {
|
||||
ReindexAction::push_action(&mut reindex_action, ReindexAction::FullReindex);
|
||||
}
|
||||
if url.apply(new_url) {
|
||||
ReindexAction::push_action(&mut reindex_action, ReindexAction::FullReindex);
|
||||
match source {
|
||||
// do not regenerate on an url change in OpenAI
|
||||
Setting::Set(EmbedderSource::OpenAi) | Setting::Reset => {}
|
||||
_ => {
|
||||
ReindexAction::push_action(
|
||||
&mut reindex_action,
|
||||
ReindexAction::FullReindex,
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
if query.apply(new_query) {
|
||||
ReindexAction::push_action(&mut reindex_action, ReindexAction::FullReindex);
|
||||
@ -271,7 +280,7 @@ fn apply_default_for_source(
|
||||
*model = Setting::Reset;
|
||||
*revision = Setting::NotSet;
|
||||
*dimensions = Setting::NotSet;
|
||||
*url = Setting::NotSet;
|
||||
*url = Setting::Reset;
|
||||
*query = Setting::NotSet;
|
||||
*input_field = Setting::NotSet;
|
||||
*path_to_embeddings = Setting::NotSet;
|
||||
@ -364,7 +373,7 @@ impl EmbeddingSettings {
|
||||
EmbedderSource::Ollama,
|
||||
EmbedderSource::Rest,
|
||||
],
|
||||
Self::URL => &[EmbedderSource::Ollama, EmbedderSource::Rest],
|
||||
Self::URL => &[EmbedderSource::Ollama, EmbedderSource::Rest, EmbedderSource::OpenAi],
|
||||
Self::QUERY => &[EmbedderSource::Rest],
|
||||
Self::INPUT_FIELD => &[EmbedderSource::Rest],
|
||||
Self::PATH_TO_EMBEDDINGS => &[EmbedderSource::Rest],
|
||||
@ -390,6 +399,7 @@ impl EmbeddingSettings {
|
||||
Self::DOCUMENT_TEMPLATE,
|
||||
Self::DIMENSIONS,
|
||||
Self::DISTRIBUTION,
|
||||
Self::URL,
|
||||
],
|
||||
EmbedderSource::HuggingFace => &[
|
||||
Self::SOURCE,
|
||||
@ -494,6 +504,7 @@ impl From<EmbeddingConfig> for EmbeddingSettings {
|
||||
distribution: distribution.map(Setting::Set).unwrap_or_default(),
|
||||
},
|
||||
super::EmbedderOptions::OpenAi(super::openai::EmbedderOptions {
|
||||
url,
|
||||
api_key,
|
||||
embedding_model,
|
||||
dimensions,
|
||||
@ -505,7 +516,7 @@ impl From<EmbeddingConfig> for EmbeddingSettings {
|
||||
api_key: api_key.map(Setting::Set).unwrap_or_default(),
|
||||
dimensions: dimensions.map(Setting::Set).unwrap_or_default(),
|
||||
document_template: Setting::Set(prompt.template),
|
||||
url: Setting::NotSet,
|
||||
url: url.map(Setting::Set).unwrap_or_default(),
|
||||
query: Setting::NotSet,
|
||||
input_field: Setting::NotSet,
|
||||
path_to_embeddings: Setting::NotSet,
|
||||
@ -608,6 +619,9 @@ impl From<EmbeddingSettings> for EmbeddingConfig {
|
||||
options.embedding_model = model;
|
||||
}
|
||||
}
|
||||
if let Some(url) = url.set() {
|
||||
options.url = Some(url);
|
||||
}
|
||||
if let Some(api_key) = api_key.set() {
|
||||
options.api_key = Some(api_key);
|
||||
}
|
||||
|
Loading…
Reference in New Issue
Block a user