Don't accept dimensions of 0 (ever) or dimensions greater than the default dimensions of the model

This commit is contained in:
Louis Dureuil 2024-02-07 11:48:47 +01:00
parent 32ee05ccef
commit 88d03c56ab
No known key found for this signature in database
3 changed files with 28 additions and 0 deletions

View File

@ -348,6 +348,8 @@ impl ErrorCode for milli::Error {
| UserError::MissingFieldForSource { .. } | UserError::MissingFieldForSource { .. }
| UserError::InvalidOpenAiModel { .. } | UserError::InvalidOpenAiModel { .. }
| UserError::InvalidOpenAiModelDimensions { .. } | UserError::InvalidOpenAiModelDimensions { .. }
| UserError::InvalidOpenAiModelDimensionsMax { .. }
| UserError::InvalidSettingsDimensions { .. }
| UserError::InvalidPrompt(_) => Code::InvalidSettingsEmbedders, | UserError::InvalidPrompt(_) => Code::InvalidSettingsEmbedders,
UserError::TooManyEmbedders(_) => Code::InvalidSettingsEmbedders, UserError::TooManyEmbedders(_) => Code::InvalidSettingsEmbedders,
UserError::InvalidPromptForEmbeddings(..) => Code::InvalidSettingsEmbedders, UserError::InvalidPromptForEmbeddings(..) => Code::InvalidSettingsEmbedders,

View File

@ -234,6 +234,15 @@ only composed of alphanumeric characters (a-z A-Z 0-9), hyphens (-) and undersco
dimensions: usize, dimensions: usize,
expected_dimensions: usize, expected_dimensions: usize,
}, },
#[error("`.embedders.{embedder_name}.dimensions`: Model `{model}` does not support overriding its dimensions to a value higher than {max_dimensions}. Found {dimensions}")]
InvalidOpenAiModelDimensionsMax {
embedder_name: String,
model: &'static str,
dimensions: usize,
max_dimensions: usize,
},
#[error("`.embedders.{embedder_name}.dimensions`: `dimensions` cannot be zero")]
InvalidSettingsDimensions { embedder_name: String },
} }
impl From<crate::vector::Error> for Error { impl From<crate::vector::Error> for Error {

View File

@ -1122,6 +1122,14 @@ pub fn validate_embedding_settings(
let Setting::Set(settings) = settings else { return Ok(settings) }; let Setting::Set(settings) = settings else { return Ok(settings) };
let EmbeddingSettings { source, model, revision, api_key, dimensions, document_template } = let EmbeddingSettings { source, model, revision, api_key, dimensions, document_template } =
settings; settings;
if let Some(0) = dimensions.set() {
return Err(crate::error::UserError::InvalidSettingsDimensions {
embedder_name: name.to_owned(),
}
.into());
}
let Some(inferred_source) = source.set() else { let Some(inferred_source) = source.set() else {
return Ok(Setting::Set(EmbeddingSettings { return Ok(Setting::Set(EmbeddingSettings {
source, source,
@ -1153,6 +1161,15 @@ pub fn validate_embedding_settings(
} }
.into()); .into());
} }
if dimensions > model.default_dimensions() {
return Err(crate::error::UserError::InvalidOpenAiModelDimensionsMax {
embedder_name: name.to_owned(),
model: model.name(),
dimensions,
max_dimensions: model.default_dimensions(),
}
.into());
}
} }
} }
} }