searchengines-web/MeiliSearch
searchengines-web/MeiliSearch
1
0
Fork 0
mirror of https://github.com/meilisearch/MeiliSearch synced 2025-01-09 13:04:30 +01:00
Code Issues Releases Wiki Activity

Replace hardcoded string with constants

Browse Source
This commit is contained in:
Gnosnay 2024-12-24 00:39:44 +08:00
parent fc23a0ee52
commit 44eb153619
26 changed files with 160 additions and 132 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

9
crates/index-scheduler/src/batch.rs
View File

@ -29,16 +29,13 @@ use bumpalo::Bump;
use dump::IndexMetadata; use dump::IndexMetadata;
use meilisearch_types::batches::BatchId; use meilisearch_types::batches::BatchId;
use meilisearch_types::heed::{RoTxn, RwTxn}; use meilisearch_types::heed::{RoTxn, RwTxn};
use meilisearch_types::milli::constants::RESERVED_VECTORS_FIELD_NAME;
use meilisearch_types::milli::documents::{obkv_to_object, DocumentsBatchReader, PrimaryKey}; use meilisearch_types::milli::documents::{obkv_to_object, DocumentsBatchReader, PrimaryKey};
use meilisearch_types::milli::heed::CompactionOption; use meilisearch_types::milli::heed::CompactionOption;
use meilisearch_types::milli::progress::Progress; use meilisearch_types::milli::progress::Progress;
use meilisearch_types::milli::update::new::indexer::{self, UpdateByFunction}; use meilisearch_types::milli::update::new::indexer::{self, UpdateByFunction};
use meilisearch_types::milli::update::{ use meilisearch_types::milli::update::{DocumentAdditionResult, IndexDocumentsMethod, Settings as MilliSettings};
DocumentAdditionResult, IndexDocumentsMethod, Settings as MilliSettings, use meilisearch_types::milli::vector::parsed_vectors::{ExplicitVectors, VectorOrArrayOfVectors};
};
use meilisearch_types::milli::vector::parsed_vectors::{
ExplicitVectors, VectorOrArrayOfVectors, RESERVED_VECTORS_FIELD_NAME,
};
use meilisearch_types::milli::{self, Filter, ThreadPoolNoAbortBuilder}; use meilisearch_types::milli::{self, Filter, ThreadPoolNoAbortBuilder};
use meilisearch_types::settings::{apply_settings_to_builder, Settings, Unchecked}; use meilisearch_types::settings::{apply_settings_to_builder, Settings, Unchecked};
use meilisearch_types::tasks::{Details, IndexSwap, Kind, KindWithContent, Status, Task}; use meilisearch_types::tasks::{Details, IndexSwap, Kind, KindWithContent, Status, Task};

6
crates/meilisearch/src/search/mod.rs
View File

@ -1209,8 +1209,7 @@ impl<'a> HitMaker<'a> {
.displayed_fields_ids(rtxn)? .displayed_fields_ids(rtxn)?
.map(|fields| fields.into_iter().collect::<BTreeSet<_>>()); .map(|fields| fields.into_iter().collect::<BTreeSet<_>>());
let vectors_fid = let vectors_fid = fields_ids_map.id(milli::constants::RESERVED_VECTORS_FIELD_NAME);
fields_ids_map.id(milli::vector::parsed_vectors::RESERVED_VECTORS_FIELD_NAME);
let vectors_is_hidden = match (&displayed_ids, vectors_fid) { let vectors_is_hidden = match (&displayed_ids, vectors_fid) {
// displayed_ids is a wildcard, so `_vectors` can be displayed regardless of its fid // displayed_ids is a wildcard, so `_vectors` can be displayed regardless of its fid
@ -1219,8 +1218,7 @@ impl<'a> HitMaker<'a> {
(Some(_), None) => { (Some(_), None) => {
// unwrap as otherwise we'd go to the first one // unwrap as otherwise we'd go to the first one
let displayed_names = index.displayed_fields(rtxn)?.unwrap(); let displayed_names = index.displayed_fields(rtxn)?.unwrap();
!displayed_names !displayed_names.contains(&milli::constants::RESERVED_VECTORS_FIELD_NAME)
.contains(&milli::vector::parsed_vectors::RESERVED_VECTORS_FIELD_NAME)
} }
// displayed_ids is a finit list, so hide if `_vectors` is not part of it // displayed_ids is a finit list, so hide if `_vectors` is not part of it
(Some(map), Some(vectors_fid)) => map.contains(&vectors_fid), (Some(map), Some(vectors_fid)) => map.contains(&vectors_fid),

3
crates/milli/src/asc_desc.rs
View File

@ -6,6 +6,7 @@ use std::str::FromStr;
use serde::{Deserialize, Serialize}; use serde::{Deserialize, Serialize};
use thiserror::Error; use thiserror::Error;
use crate::constants::RESERVED_GEO_FIELD_NAME;
use crate::error::is_reserved_keyword; use crate::error::is_reserved_keyword;
use crate::search::facet::BadGeoError; use crate::search::facet::BadGeoError;
use crate::{CriterionError, Error, UserError}; use crate::{CriterionError, Error, UserError};
@ -175,7 +176,7 @@ impl From<AscDescError> for SortError {
AscDescError::ReservedKeyword { name } if name.starts_with("_geoPoint") => { AscDescError::ReservedKeyword { name } if name.starts_with("_geoPoint") => {
SortError::BadGeoPointUsage { name } SortError::BadGeoPointUsage { name }
} }
AscDescError::ReservedKeyword { name } if &name == "_geo" => { AscDescError::ReservedKeyword { name } if &name == RESERVED_GEO_FIELD_NAME => {
SortError::ReservedNameForSettings { name } SortError::ReservedNameForSettings { name }
} }
AscDescError::ReservedKeyword { name } if name.starts_with("_geoRadius") => { AscDescError::ReservedKeyword { name } if name.starts_with("_geoRadius") => {

2
crates/milli/src/constants.rs Normal file
View File

@ -0,0 +1,2 @@
pub const RESERVED_VECTORS_FIELD_NAME: &str = "_vectors";
pub const RESERVED_GEO_FIELD_NAME: &str = "_geo";

3
crates/milli/src/criterion.rs
View File

@ -113,6 +113,7 @@ mod tests {
use CriterionError::*; use CriterionError::*;
use super::*; use super::*;
use crate::constants::RESERVED_GEO_FIELD_NAME;
#[test] #[test]
fn parse_criterion() { fn parse_criterion() {
@ -153,7 +154,7 @@ mod tests {
("price:aasc", InvalidName { name: S("price:aasc") }), ("price:aasc", InvalidName { name: S("price:aasc") }),
("price:asc and desc", InvalidName { name: S("price:asc and desc") }), ("price:asc and desc", InvalidName { name: S("price:asc and desc") }),
("price:asc:truc", InvalidName { name: S("price:asc:truc") }), ("price:asc:truc", InvalidName { name: S("price:asc:truc") }),
("_geo:asc", ReservedName { name: S("_geo") }), ("_geo:asc", ReservedName { name: S(RESERVED_GEO_FIELD_NAME) }),
("_geoDistance:asc", ReservedName { name: S("_geoDistance") }), ("_geoDistance:asc", ReservedName { name: S("_geoDistance") }),
("_geoPoint:asc", ReservedNameForSort { name: S("_geoPoint") }), ("_geoPoint:asc", ReservedNameForSort { name: S("_geoPoint") }),
("_geoPoint(42, 75):asc", ReservedNameForSort { name: S("_geoPoint") }), ("_geoPoint(42, 75):asc", ReservedNameForSort { name: S("_geoPoint") }),

4
crates/milli/src/error.rs
View File

@ -10,12 +10,14 @@ use rhai::EvalAltResult;
use serde_json::Value; use serde_json::Value;
use thiserror::Error; use thiserror::Error;
use crate::constants::RESERVED_GEO_FIELD_NAME;
use crate::documents::{self, DocumentsBatchCursorError}; use crate::documents::{self, DocumentsBatchCursorError};
use crate::thread_pool_no_abort::PanicCatched; use crate::thread_pool_no_abort::PanicCatched;
use crate::{CriterionError, DocumentId, FieldId, Object, SortError}; use crate::{CriterionError, DocumentId, FieldId, Object, SortError};
pub fn is_reserved_keyword(keyword: &str) -> bool { pub fn is_reserved_keyword(keyword: &str) -> bool {
["_geo", "_geoDistance", "_geoPoint", "_geoRadius", "_geoBoundingBox"].contains(&keyword) [RESERVED_GEO_FIELD_NAME, "_geoDistance", "_geoPoint", "_geoRadius", "_geoBoundingBox"]
.contains(&keyword)
} }
#[derive(Error, Debug)] #[derive(Error, Debug)]

3
crates/milli/src/fieldids_weights_map.rs
View File

@ -4,8 +4,7 @@ use std::collections::HashMap;
use serde::{Deserialize, Serialize}; use serde::{Deserialize, Serialize};
use crate::vector::parsed_vectors::RESERVED_VECTORS_FIELD_NAME; use crate::{constants::RESERVED_VECTORS_FIELD_NAME, FieldId, FieldsIdsMap, Weight};
use crate::{FieldId, FieldsIdsMap, Weight};
#[derive(Debug, Default, Serialize, Deserialize)] #[derive(Debug, Default, Serialize, Deserialize)]
pub struct FieldidsWeightsMap { pub struct FieldidsWeightsMap {

34
crates/milli/src/index.rs
View File

@ -10,6 +10,7 @@ use roaring::RoaringBitmap;
use rstar::RTree; use rstar::RTree;
use serde::{Deserialize, Serialize}; use serde::{Deserialize, Serialize};
use crate::constants::RESERVED_VECTORS_FIELD_NAME;
use crate::documents::PrimaryKey; use crate::documents::PrimaryKey;
use crate::error::{InternalError, UserError}; use crate::error::{InternalError, UserError};
use crate::fields_ids_map::FieldsIdsMap; use crate::fields_ids_map::FieldsIdsMap;
@ -20,7 +21,7 @@ use crate::heed_codec::facet::{
use crate::heed_codec::{BEU16StrCodec, FstSetCodec, StrBEU16Codec, StrRefCodec}; use crate::heed_codec::{BEU16StrCodec, FstSetCodec, StrBEU16Codec, StrRefCodec};
use crate::order_by_map::OrderByMap; use crate::order_by_map::OrderByMap;
use crate::proximity::ProximityPrecision; use crate::proximity::ProximityPrecision;
use crate::vector::parsed_vectors::RESERVED_VECTORS_FIELD_NAME;
use crate::vector::{ArroyWrapper, Embedding, EmbeddingConfig}; use crate::vector::{ArroyWrapper, Embedding, EmbeddingConfig};
use crate::{ use crate::{
default_criteria, CboRoaringBitmapCodec, Criterion, DocumentId, ExternalDocumentsIds, default_criteria, CboRoaringBitmapCodec, Criterion, DocumentId, ExternalDocumentsIds,
@ -1732,6 +1733,7 @@ pub(crate) mod tests {
use memmap2::Mmap; use memmap2::Mmap;
use tempfile::TempDir; use tempfile::TempDir;
use crate::constants::RESERVED_GEO_FIELD_NAME;
use crate::error::{Error, InternalError}; use crate::error::{Error, InternalError};
use crate::index::{DEFAULT_MIN_WORD_LEN_ONE_TYPO, DEFAULT_MIN_WORD_LEN_TWO_TYPOS}; use crate::index::{DEFAULT_MIN_WORD_LEN_ONE_TYPO, DEFAULT_MIN_WORD_LEN_TWO_TYPOS};
use crate::progress::Progress; use crate::progress::Progress;
@ -2173,16 +2175,16 @@ pub(crate) mod tests {
index index
.update_settings(|settings| { .update_settings(|settings| {
settings.set_filterable_fields(hashset! { S("_geo") }); settings.set_filterable_fields(hashset! { S(RESERVED_GEO_FIELD_NAME) });
}) })
.unwrap(); .unwrap();
index index
.add_documents(documents!([ .add_documents(documents!([
{ "id": 0, "_geo": { "lat": "0", "lng": "0" } }, { "id": 0, RESERVED_GEO_FIELD_NAME: { "lat": "0", "lng": "0" } },
{ "id": 1, "_geo": { "lat": 0, "lng": "-175" } }, { "id": 1, RESERVED_GEO_FIELD_NAME: { "lat": 0, "lng": "-175" } },
{ "id": 2, "_geo": { "lat": "0", "lng": 175 } }, { "id": 2, RESERVED_GEO_FIELD_NAME: { "lat": "0", "lng": 175 } },
{ "id": 3, "_geo": { "lat": 85, "lng": 0 } }, { "id": 3, RESERVED_GEO_FIELD_NAME: { "lat": 85, "lng": 0 } },
{ "id": 4, "_geo": { "lat": "-85", "lng": "0" } }, { "id": 4, RESERVED_GEO_FIELD_NAME: { "lat": "-85", "lng": "0" } },
])) ]))
.unwrap(); .unwrap();
@ -2859,19 +2861,24 @@ pub(crate) mod tests {
index index
.update_settings(|settings| { .update_settings(|settings| {
settings.set_primary_key("id".to_string()); settings.set_primary_key("id".to_string());
settings.set_filterable_fields(HashSet::from(["_geo".to_string()])); settings
.set_filterable_fields(HashSet::from([RESERVED_GEO_FIELD_NAME.to_string()]));
}) })
.unwrap(); .unwrap();
// happy path // happy path
index.add_documents(documents!({ "id" : 5, "_geo": {"lat": 12.0, "lng": 11.0}})).unwrap(); index
.add_documents(
documents!({ "id" : 5, RESERVED_GEO_FIELD_NAME: {"lat": 12.0, "lng": 11.0}}),
)
.unwrap();
db_snap!(index, geo_faceted_documents_ids); db_snap!(index, geo_faceted_documents_ids);
// both are unparseable, we expect GeoError::BadLatitudeAndLongitude // both are unparseable, we expect GeoError::BadLatitudeAndLongitude
let err1 = index let err1 = index
.add_documents( .add_documents(
documents!({ "id" : 6, "_geo": {"lat": "unparseable", "lng": "unparseable"}}), documents!({ "id" : 6, RESERVED_GEO_FIELD_NAME: {"lat": "unparseable", "lng": "unparseable"}}),
) )
.unwrap_err(); .unwrap_err();
assert!(matches!( assert!(matches!(
@ -2889,13 +2896,14 @@ pub(crate) mod tests {
index index
.update_settings(|settings| { .update_settings(|settings| {
settings.set_primary_key("id".to_string()); settings.set_primary_key("id".to_string());
settings.set_filterable_fields(HashSet::from(["_geo".to_string()])); settings
.set_filterable_fields(HashSet::from([RESERVED_GEO_FIELD_NAME.to_string()]));
}) })
.unwrap(); .unwrap();
let err = index let err = index
.add_documents( .add_documents(
documents!({ "id" : "doggo", "_geo": { "lat": 1, "lng": 2, "doggo": "are the best" }}), documents!({ "id" : "doggo", RESERVED_GEO_FIELD_NAME: { "lat": 1, "lng": 2, "doggo": "are the best" }}),
) )
.unwrap_err(); .unwrap_err();
insta::assert_snapshot!(err, @r###"The `_geo` field in the document with the id: `"doggo"` contains the following unexpected fields: `{"doggo":"are the best"}`."###); insta::assert_snapshot!(err, @r###"The `_geo` field in the document with the id: `"doggo"` contains the following unexpected fields: `{"doggo":"are the best"}`."###);
@ -2905,7 +2913,7 @@ pub(crate) mod tests {
// multiple fields and complex values // multiple fields and complex values
let err = index let err = index
.add_documents( .add_documents(
documents!({ "id" : "doggo", "_geo": { "lat": 1, "lng": 2, "doggo": "are the best", "and": { "all": ["cats", { "are": "beautiful" } ] } } }), documents!({ "id" : "doggo", RESERVED_GEO_FIELD_NAME: { "lat": 1, "lng": 2, "doggo": "are the best", "and": { "all": ["cats", { "are": "beautiful" } ] } } }),
) )
.unwrap_err(); .unwrap_err();
insta::assert_snapshot!(err, @r###"The `_geo` field in the document with the id: `"doggo"` contains the following unexpected fields: `{"and":{"all":["cats",{"are":"beautiful"}]},"doggo":"are the best"}`."###); insta::assert_snapshot!(err, @r###"The `_geo` field in the document with the id: `"doggo"` contains the following unexpected fields: `{"and":{"all":["cats",{"are":"beautiful"}]},"doggo":"are the best"}`."###);

1
crates/milli/src/lib.rs
View File

@ -30,6 +30,7 @@ pub mod vector;
#[cfg(test)] #[cfg(test)]
#[macro_use] #[macro_use]
pub mod snapshot_tests; pub mod snapshot_tests;
pub mod constants;
mod fieldids_weights_map; mod fieldids_weights_map;
pub mod progress; pub mod progress;

24
crates/milli/src/search/facet/filter.rs
View File

@ -10,6 +10,7 @@ use roaring::{MultiOps, RoaringBitmap};
use serde_json::Value; use serde_json::Value;
use super::facet_range_search; use super::facet_range_search;
use crate::constants::RESERVED_GEO_FIELD_NAME;
use crate::error::{Error, UserError}; use crate::error::{Error, UserError};
use crate::heed_codec::facet::{ use crate::heed_codec::facet::{
FacetGroupKey, FacetGroupKeyCodec, FacetGroupValue, FacetGroupValueCodec, OrderedF64Codec, FacetGroupKey, FacetGroupKeyCodec, FacetGroupValue, FacetGroupValueCodec, OrderedF64Codec,
@ -501,7 +502,7 @@ impl<'a> Filter<'a> {
} }
} }
FilterCondition::GeoLowerThan { point, radius } => { FilterCondition::GeoLowerThan { point, radius } => {
if filterable_fields.contains("_geo") { if filterable_fields.contains(RESERVED_GEO_FIELD_NAME) {
let base_point: [f64; 2] = let base_point: [f64; 2] =
[point[0].parse_finite_float()?, point[1].parse_finite_float()?]; [point[0].parse_finite_float()?, point[1].parse_finite_float()?];
if !(-90.0..=90.0).contains(&base_point[0]) { if !(-90.0..=90.0).contains(&base_point[0]) {
@ -530,13 +531,13 @@ impl<'a> Filter<'a> {
Ok(result) Ok(result)
} else { } else {
Err(point[0].as_external_error(FilterError::AttributeNotFilterable { Err(point[0].as_external_error(FilterError::AttributeNotFilterable {
attribute: "_geo", attribute: RESERVED_GEO_FIELD_NAME,
filterable_fields: filterable_fields.clone(), filterable_fields: filterable_fields.clone(),
}))? }))?
} }
} }
FilterCondition::GeoBoundingBox { top_right_point, bottom_left_point } => { FilterCondition::GeoBoundingBox { top_right_point, bottom_left_point } => {
if filterable_fields.contains("_geo") { if filterable_fields.contains(RESERVED_GEO_FIELD_NAME) {
let top_right: [f64; 2] = [ let top_right: [f64; 2] = [
top_right_point[0].parse_finite_float()?, top_right_point[0].parse_finite_float()?,
top_right_point[1].parse_finite_float()?, top_right_point[1].parse_finite_float()?,
@ -663,7 +664,7 @@ impl<'a> Filter<'a> {
} else { } else {
Err(top_right_point[0].as_external_error( Err(top_right_point[0].as_external_error(
FilterError::AttributeNotFilterable { FilterError::AttributeNotFilterable {
attribute: "_geo", attribute: RESERVED_GEO_FIELD_NAME,
filterable_fields: filterable_fields.clone(), filterable_fields: filterable_fields.clone(),
}, },
))? ))?
@ -689,6 +690,7 @@ mod tests {
use maplit::hashset; use maplit::hashset;
use roaring::RoaringBitmap; use roaring::RoaringBitmap;
use crate::constants::RESERVED_GEO_FIELD_NAME;
use crate::index::tests::TempIndex; use crate::index::tests::TempIndex;
use crate::Filter; use crate::Filter;
@ -899,7 +901,7 @@ mod tests {
index index
.update_settings(|settings| { .update_settings(|settings| {
settings.set_filterable_fields(hashset! { S("_geo") }); settings.set_filterable_fields(hashset! { S(RESERVED_GEO_FIELD_NAME) });
}) })
.unwrap(); .unwrap();
@ -911,7 +913,7 @@ mod tests {
"address": "Viale Vittorio Veneto, 30, 20124, Milan, Italy", "address": "Viale Vittorio Veneto, 30, 20124, Milan, Italy",
"type": "pizza", "type": "pizza",
"rating": 9, "rating": 9,
"_geo": { RESERVED_GEO_FIELD_NAME: {
"lat": 45.4777599, "lat": 45.4777599,
"lng": 9.1967508 "lng": 9.1967508
} }
@ -922,7 +924,7 @@ mod tests {
"address": "Via Dogana, 1, 20123 Milan, Italy", "address": "Via Dogana, 1, 20123 Milan, Italy",
"type": "ice cream", "type": "ice cream",
"rating": 10, "rating": 10,
"_geo": { RESERVED_GEO_FIELD_NAME: {
"lat": 45.4632046, "lat": 45.4632046,
"lng": 9.1719421 "lng": 9.1719421
} }
@ -945,8 +947,8 @@ mod tests {
index index
.update_settings(|settings| { .update_settings(|settings| {
settings.set_searchable_fields(vec![S("_geo"), S("price")]); // to keep the fields order settings.set_searchable_fields(vec![S(RESERVED_GEO_FIELD_NAME), S("price")]); // to keep the fields order
settings.set_filterable_fields(hashset! { S("_geo"), S("price") }); settings.set_filterable_fields(hashset! { S(RESERVED_GEO_FIELD_NAME), S("price") });
}) })
.unwrap(); .unwrap();
@ -995,8 +997,8 @@ mod tests {
index index
.update_settings(|settings| { .update_settings(|settings| {
settings.set_searchable_fields(vec![S("_geo"), S("price")]); // to keep the fields order settings.set_searchable_fields(vec![S(RESERVED_GEO_FIELD_NAME), S("price")]); // to keep the fields order
settings.set_filterable_fields(hashset! { S("_geo"), S("price") }); settings.set_filterable_fields(hashset! { S(RESERVED_GEO_FIELD_NAME), S("price") });
}) })
.unwrap(); .unwrap();

5
crates/milli/src/search/new/mod.rs
View File

@ -50,6 +50,7 @@ use self::graph_based_ranking_rule::Words;
use self::interner::Interned; use self::interner::Interned;
use self::vector_sort::VectorSort; use self::vector_sort::VectorSort;
use crate::index::PrefixSearch; use crate::index::PrefixSearch;
use crate::constants::RESERVED_GEO_FIELD_NAME;
use crate::localized_attributes_rules::LocalizedFieldIds; use crate::localized_attributes_rules::LocalizedFieldIds;
use crate::score_details::{ScoreDetails, ScoringStrategy}; use crate::score_details::{ScoreDetails, ScoringStrategy};
use crate::search::new::distinct::apply_distinct_rule; use crate::search::new::distinct::apply_distinct_rule;
@ -863,12 +864,12 @@ fn check_sort_criteria(
} }
.into()); .into());
} }
Member::Geo(_) if !sortable_fields.contains("_geo") => { Member::Geo(_) if !sortable_fields.contains(RESERVED_GEO_FIELD_NAME) => {
let (valid_fields, hidden_fields) = let (valid_fields, hidden_fields) =
ctx.index.remove_hidden_fields(ctx.txn, sortable_fields)?; ctx.index.remove_hidden_fields(ctx.txn, sortable_fields)?;
return Err(UserError::InvalidSortableAttribute { return Err(UserError::InvalidSortableAttribute {
field: "_geo".to_string(), field: RESERVED_GEO_FIELD_NAME.to_string(),
valid_fields, valid_fields,
hidden_fields, hidden_fields,
} }

35
crates/milli/src/search/new/tests/geo_sort.rs
View File

@ -6,6 +6,7 @@ use big_s::S;
use heed::RoTxn; use heed::RoTxn;
use maplit::hashset; use maplit::hashset;
use crate::constants::RESERVED_GEO_FIELD_NAME;
use crate::index::tests::TempIndex; use crate::index::tests::TempIndex;
use crate::score_details::ScoreDetails; use crate::score_details::ScoreDetails;
use crate::search::new::tests::collect_field_values; use crate::search::new::tests::collect_field_values;
@ -17,7 +18,7 @@ fn create_index() -> TempIndex {
index index
.update_settings(|s| { .update_settings(|s| {
s.set_primary_key("id".to_owned()); s.set_primary_key("id".to_owned());
s.set_sortable_fields(hashset! { S("_geo") }); s.set_sortable_fields(hashset! { S(RESERVED_GEO_FIELD_NAME) });
s.set_criteria(vec![Criterion::Words, Criterion::Sort]); s.set_criteria(vec![Criterion::Words, Criterion::Sort]);
}) })
.unwrap(); .unwrap();
@ -68,12 +69,12 @@ fn test_geo_sort() {
index index
.add_documents(documents!([ .add_documents(documents!([
{ "id": 2, "_geo": { "lat": 2, "lng": -1 } }, { "id": 2, RESERVED_GEO_FIELD_NAME: { "lat": 2, "lng": -1 } },
{ "id": 3, "_geo": { "lat": -2, "lng": -2 } }, { "id": 3, RESERVED_GEO_FIELD_NAME: { "lat": -2, "lng": -2 } },
{ "id": 5, "_geo": { "lat": 6, "lng": -5 } }, { "id": 5, RESERVED_GEO_FIELD_NAME: { "lat": 6, "lng": -5 } },
{ "id": 4, "_geo": { "lat": 3, "lng": 5 } }, { "id": 4, RESERVED_GEO_FIELD_NAME: { "lat": 3, "lng": 5 } },
{ "id": 0, "_geo": { "lat": 0, "lng": 0 } }, { "id": 0, RESERVED_GEO_FIELD_NAME: { "lat": 0, "lng": 0 } },
{ "id": 1, "_geo": { "lat": 1, "lng": 1 } }, { "id": 1, RESERVED_GEO_FIELD_NAME: { "lat": 1, "lng": 1 } },
{ "id": 6 }, { "id": 8 }, { "id": 7 }, { "id": 10 }, { "id": 9 }, { "id": 6 }, { "id": 8 }, { "id": 7 }, { "id": 10 }, { "id": 9 },
])) ]))
.unwrap(); .unwrap();
@ -100,12 +101,12 @@ fn test_geo_sort_around_the_edge_of_the_flat_earth() {
index index
.add_documents(documents!([ .add_documents(documents!([
{ "id": 0, "_geo": { "lat": 0, "lng": 0 } }, { "id": 0, RESERVED_GEO_FIELD_NAME: { "lat": 0, "lng": 0 } },
{ "id": 1, "_geo": { "lat": 88, "lng": 0 } }, { "id": 1, RESERVED_GEO_FIELD_NAME: { "lat": 88, "lng": 0 } },
{ "id": 2, "_geo": { "lat": -89, "lng": 0 } }, { "id": 2, RESERVED_GEO_FIELD_NAME: { "lat": -89, "lng": 0 } },
{ "id": 3, "_geo": { "lat": 0, "lng": 178 } }, { "id": 3, RESERVED_GEO_FIELD_NAME: { "lat": 0, "lng": 178 } },
{ "id": 4, "_geo": { "lat": 0, "lng": -179 } }, { "id": 4, RESERVED_GEO_FIELD_NAME: { "lat": 0, "lng": -179 } },
])) ]))
.unwrap(); .unwrap();
@ -177,11 +178,11 @@ fn geo_sort_mixed_with_words() {
index index
.add_documents(documents!([ .add_documents(documents!([
{ "id": 0, "doggo": "jean", "_geo": { "lat": 0, "lng": 0 } }, { "id": 0, "doggo": "jean", RESERVED_GEO_FIELD_NAME: { "lat": 0, "lng": 0 } },
{ "id": 1, "doggo": "intel", "_geo": { "lat": 88, "lng": 0 } }, { "id": 1, "doggo": "intel", RESERVED_GEO_FIELD_NAME: { "lat": 88, "lng": 0 } },
{ "id": 2, "doggo": "jean bob", "_geo": { "lat": -89, "lng": 0 } }, { "id": 2, "doggo": "jean bob", RESERVED_GEO_FIELD_NAME: { "lat": -89, "lng": 0 } },
{ "id": 3, "doggo": "jean michel", "_geo": { "lat": 0, "lng": 178 } }, { "id": 3, "doggo": "jean michel", RESERVED_GEO_FIELD_NAME: { "lat": 0, "lng": 178 } },
{ "id": 4, "doggo": "bob marley", "_geo": { "lat": 0, "lng": -179 } }, { "id": 4, "doggo": "bob marley", RESERVED_GEO_FIELD_NAME: { "lat": 0, "lng": -179 } },
])) ]))
.unwrap(); .unwrap();

3
crates/milli/src/search/new/tests/integration.rs
View File

@ -11,6 +11,7 @@ use crate::update::{IndexDocumentsMethod, IndexerConfig, Settings};
use crate::vector::EmbeddingConfigs; use crate::vector::EmbeddingConfigs;
use crate::{db_snap, Criterion, Index}; use crate::{db_snap, Criterion, Index};
pub const CONTENT: &str = include_str!("../../../../tests/assets/test_set.ndjson"); pub const CONTENT: &str = include_str!("../../../../tests/assets/test_set.ndjson");
use crate::constants::RESERVED_GEO_FIELD_NAME;
pub fn setup_search_index_with_criteria(criteria: &[Criterion]) -> Index { pub fn setup_search_index_with_criteria(criteria: &[Criterion]) -> Index {
let path = tempfile::tempdir().unwrap(); let path = tempfile::tempdir().unwrap();
@ -27,7 +28,7 @@ pub fn setup_search_index_with_criteria(criteria: &[Criterion]) -> Index {
builder.set_filterable_fields(hashset! { builder.set_filterable_fields(hashset! {
S("tag"), S("tag"),
S("asc_desc_rank"), S("asc_desc_rank"),
S("_geo"), S(RESERVED_GEO_FIELD_NAME),
S("opt1"), S("opt1"),
S("opt1.opt2"), S("opt1.opt2"),
S("tag_in") S("tag_in")

3
crates/milli/src/update/clear_documents.rs
View File

@ -103,6 +103,7 @@ impl<'t, 'i> ClearDocuments<'t, 'i> {
#[cfg(test)] #[cfg(test)]
mod tests { mod tests {
use super::*; use super::*;
use crate::constants::RESERVED_GEO_FIELD_NAME;
use crate::index::tests::TempIndex; use crate::index::tests::TempIndex;
#[test] #[test]
@ -114,7 +115,7 @@ mod tests {
.add_documents_using_wtxn(&mut wtxn, documents!([ .add_documents_using_wtxn(&mut wtxn, documents!([
{ "id": 0, "name": "kevin", "age": 20 }, { "id": 0, "name": "kevin", "age": 20 },
{ "id": 1, "name": "kevina" }, { "id": 1, "name": "kevina" },
{ "id": 2, "name": "benoit", "country": "France", "_geo": { "lng": 42, "lat": 35 } } { "id": 2, "name": "benoit", "country": "France", RESERVED_GEO_FIELD_NAME: { "lng": 42, "lat": 35 } }
])) ]))
.unwrap(); .unwrap();

7
crates/milli/src/update/index_documents/enrich.rs
View File

@ -5,6 +5,7 @@ use std::result::Result as StdResult;
use serde::{Deserialize, Serialize}; use serde::{Deserialize, Serialize};
use serde_json::Value; use serde_json::Value;
use crate::constants::RESERVED_GEO_FIELD_NAME;
use crate::documents::{ use crate::documents::{
DocumentIdExtractionError, DocumentsBatchIndex, DocumentsBatchReader, DocumentIdExtractionError, DocumentsBatchIndex, DocumentsBatchReader,
EnrichedDocumentsBatchReader, PrimaryKey, DEFAULT_PRIMARY_KEY, EnrichedDocumentsBatchReader, PrimaryKey, DEFAULT_PRIMARY_KEY,
@ -93,10 +94,10 @@ pub fn enrich_documents_batch<R: Read + Seek>(
// If the settings specifies that a _geo field must be used therefore we must check the // If the settings specifies that a _geo field must be used therefore we must check the
// validity of it in all the documents of this batch and this is when we return `Some`. // validity of it in all the documents of this batch and this is when we return `Some`.
let geo_field_id = match documents_batch_index.id("_geo") { let geo_field_id = match documents_batch_index.id(RESERVED_GEO_FIELD_NAME) {
Some(geo_field_id) Some(geo_field_id)
if index.sortable_fields(rtxn)?.contains("_geo") if index.sortable_fields(rtxn)?.contains(RESERVED_GEO_FIELD_NAME)
|| index.filterable_fields(rtxn)?.contains("_geo") => || index.filterable_fields(rtxn)?.contains(RESERVED_GEO_FIELD_NAME) =>
{ {
Some(geo_field_id) Some(geo_field_id)
} }

3
crates/milli/src/update/index_documents/extract/extract_vector_points.rs
View File

@ -13,13 +13,14 @@ use roaring::RoaringBitmap;
use serde_json::Value; use serde_json::Value;
use super::helpers::{create_writer, writer_into_reader, GrenadParameters}; use super::helpers::{create_writer, writer_into_reader, GrenadParameters};
use crate::constants::RESERVED_VECTORS_FIELD_NAME;
use crate::error::FaultSource; use crate::error::FaultSource;
use crate::index::IndexEmbeddingConfig; use crate::index::IndexEmbeddingConfig;
use crate::prompt::{FieldsIdsMapWithMetadata, Prompt}; use crate::prompt::{FieldsIdsMapWithMetadata, Prompt};
use crate::update::del_add::{DelAdd, KvReaderDelAdd, KvWriterDelAdd}; use crate::update::del_add::{DelAdd, KvReaderDelAdd, KvWriterDelAdd};
use crate::update::settings::InnerIndexSettingsDiff; use crate::update::settings::InnerIndexSettingsDiff;
use crate::vector::error::{EmbedErrorKind, PossibleEmbeddingMistakes, UnusedVectorsDistribution}; use crate::vector::error::{EmbedErrorKind, PossibleEmbeddingMistakes, UnusedVectorsDistribution};
use crate::vector::parsed_vectors::{ParsedVectorsDiff, VectorState, RESERVED_VECTORS_FIELD_NAME}; use crate::vector::parsed_vectors::{ParsedVectorsDiff, VectorState};
use crate::vector::settings::ReindexAction; use crate::vector::settings::ReindexAction;
use crate::vector::{Embedder, Embedding}; use crate::vector::{Embedder, Embedding};
use crate::{try_split_array_at, DocumentId, FieldId, Result, ThreadPoolNoAbort}; use crate::{try_split_array_at, DocumentId, FieldId, Result, ThreadPoolNoAbort};

74
crates/milli/src/update/index_documents/mod.rs
View File

@ -27,6 +27,7 @@ pub use self::enrich::{extract_finite_float_from_value, DocumentId};
pub use self::helpers::*; pub use self::helpers::*;
pub use self::transform::{Transform, TransformOutput}; pub use self::transform::{Transform, TransformOutput};
use super::new::StdResult; use super::new::StdResult;
use crate::documents::{obkv_to_object, DocumentsBatchReader}; use crate::documents::{obkv_to_object, DocumentsBatchReader};
use crate::error::{Error, InternalError}; use crate::error::{Error, InternalError};
use crate::index::{PrefixSearch, PrefixSettings}; use crate::index::{PrefixSearch, PrefixSettings};
@ -763,6 +764,7 @@ mod tests {
use maplit::hashset; use maplit::hashset;
use super::*; use super::*;
use crate::constants::RESERVED_GEO_FIELD_NAME;
use crate::documents::mmap_from_objects; use crate::documents::mmap_from_objects;
use crate::index::tests::TempIndex; use crate::index::tests::TempIndex;
use crate::index::IndexEmbeddingConfig; use crate::index::IndexEmbeddingConfig;
@ -944,12 +946,12 @@ mod tests {
index.index_documents_config.update_method = IndexDocumentsMethod::ReplaceDocuments; index.index_documents_config.update_method = IndexDocumentsMethod::ReplaceDocuments;
index.add_documents(documents!([ index.add_documents(documents!([
{ "id": 2, "title": "Pride and Prejudice", "author": "Jane Austin", "genre": "romance", "price": 3.5, "_geo": { "lat": 12, "lng": 42 } }, { "id": 2, "title": "Pride and Prejudice", "author": "Jane Austin", "genre": "romance", "price": 3.5, RESERVED_GEO_FIELD_NAME: { "lat": 12, "lng": 42 } },
{ "id": 456, "title": "Le Petit Prince", "author": "Antoine de Saint-Exupéry", "genre": "adventure" , "price": 10.0 }, { "id": 456, "title": "Le Petit Prince", "author": "Antoine de Saint-Exupéry", "genre": "adventure" , "price": 10.0 },
{ "id": 1, "title": "Alice In Wonderland", "author": "Lewis Carroll", "genre": "fantasy", "price": 25.99 }, { "id": 1, "title": "Alice In Wonderland", "author": "Lewis Carroll", "genre": "fantasy", "price": 25.99 },
{ "id": 1344, "title": "The Hobbit", "author": "J. R. R. Tolkien", "genre": "fantasy" }, { "id": 1344, "title": "The Hobbit", "author": "J. R. R. Tolkien", "genre": "fantasy" },
{ "id": 4, "title": "Harry Potter and the Half-Blood Prince", "author": "J. K. Rowling", "genre": "fantasy" }, { "id": 4, "title": "Harry Potter and the Half-Blood Prince", "author": "J. K. Rowling", "genre": "fantasy" },
{ "id": 42, "title": "The Hitchhiker's Guide to the Galaxy", "author": "Douglas Adams", "_geo": { "lat": 35, "lng": 23 } } { "id": 42, "title": "The Hitchhiker's Guide to the Galaxy", "author": "Douglas Adams", RESERVED_GEO_FIELD_NAME: { "lat": 35, "lng": 23 } }
])).unwrap(); ])).unwrap();
db_snap!(index, word_docids, "initial"); db_snap!(index, word_docids, "initial");
@ -989,18 +991,18 @@ mod tests {
// We send 6 documents and mix the ones that have _geo and those that don't have it. // We send 6 documents and mix the ones that have _geo and those that don't have it.
index index
.add_documents(documents!([ .add_documents(documents!([
{ "id": 2, "price": 3.5, "_geo": { "lat": 12, "lng": 42 } }, { "id": 2, "price": 3.5, RESERVED_GEO_FIELD_NAME: { "lat": 12, "lng": 42 } },
{ "id": 456 }, { "id": 456 },
{ "id": 1 }, { "id": 1 },
{ "id": 1344 }, { "id": 1344 },
{ "id": 4 }, { "id": 4 },
{ "id": 42, "_geo": { "lat": 35, "lng": 23 } } { "id": 42, RESERVED_GEO_FIELD_NAME: { "lat": 35, "lng": 23 } }
])) ]))
.unwrap(); .unwrap();
index index
.update_settings(|settings| { .update_settings(|settings| {
settings.set_filterable_fields(hashset!(S("_geo"))); settings.set_filterable_fields(hashset!(S(RESERVED_GEO_FIELD_NAME)));
}) })
.unwrap(); .unwrap();
} }
@ -1012,13 +1014,13 @@ mod tests {
index index
.update_settings(|settings| { .update_settings(|settings| {
settings.set_filterable_fields(hashset!(S("_geo"))); settings.set_filterable_fields(hashset!(S(RESERVED_GEO_FIELD_NAME)));
}) })
.unwrap(); .unwrap();
let error = index let error = index
.add_documents(documents!([ .add_documents(documents!([
{ "id": 0, "_geo": { "lng": 42 } } { "id": 0, RESERVED_GEO_FIELD_NAME: { "lng": 42 } }
])) ]))
.unwrap_err(); .unwrap_err();
assert_eq!( assert_eq!(
@ -1028,7 +1030,7 @@ mod tests {
let error = index let error = index
.add_documents(documents!([ .add_documents(documents!([
{ "id": 0, "_geo": { "lat": 42 } } { "id": 0, RESERVED_GEO_FIELD_NAME: { "lat": 42 } }
])) ]))
.unwrap_err(); .unwrap_err();
assert_eq!( assert_eq!(
@ -1038,7 +1040,7 @@ mod tests {
let error = index let error = index
.add_documents(documents!([ .add_documents(documents!([
{ "id": 0, "_geo": { "lat": "lol", "lng": 42 } } { "id": 0, RESERVED_GEO_FIELD_NAME: { "lat": "lol", "lng": 42 } }
])) ]))
.unwrap_err(); .unwrap_err();
assert_eq!( assert_eq!(
@ -1048,7 +1050,7 @@ mod tests {
let error = index let error = index
.add_documents(documents!([ .add_documents(documents!([
{ "id": 0, "_geo": { "lat": [12, 13], "lng": 42 } } { "id": 0, RESERVED_GEO_FIELD_NAME: { "lat": [12, 13], "lng": 42 } }
])) ]))
.unwrap_err(); .unwrap_err();
assert_eq!( assert_eq!(
@ -1058,7 +1060,7 @@ mod tests {
let error = index let error = index
.add_documents(documents!([ .add_documents(documents!([
{ "id": 0, "_geo": { "lat": 12, "lng": "hello" } } { "id": 0, RESERVED_GEO_FIELD_NAME: { "lat": 12, "lng": "hello" } }
])) ]))
.unwrap_err(); .unwrap_err();
assert_eq!( assert_eq!(
@ -1076,7 +1078,7 @@ mod tests {
{ "objectId": 123, "title": "Pride and Prejudice", "comment": "A great book" }, { "objectId": 123, "title": "Pride and Prejudice", "comment": "A great book" },
{ "objectId": 456, "title": "Le Petit Prince", "comment": "A french book" }, { "objectId": 456, "title": "Le Petit Prince", "comment": "A french book" },
{ "objectId": 1, "title": "Alice In Wonderland", "comment": "A weird book" }, { "objectId": 1, "title": "Alice In Wonderland", "comment": "A weird book" },
{ "objectId": 30, "title": "Hamlet", "_geo": { "lat": 12, "lng": 89 } } { "objectId": 30, "title": "Hamlet", RESERVED_GEO_FIELD_NAME: { "lat": 12, "lng": 89 } }
])) ]))
.unwrap(); .unwrap();
@ -1091,7 +1093,7 @@ mod tests {
index index
.add_documents(documents!([ .add_documents(documents!([
{ "objectId": 30, "title": "Hamlet", "_geo": { "lat": 12, "lng": 89 } } { "objectId": 30, "title": "Hamlet", RESERVED_GEO_FIELD_NAME: { "lat": 12, "lng": 89 } }
])) ]))
.unwrap(); .unwrap();
@ -1102,7 +1104,7 @@ mod tests {
index index
.add_documents(documents!([ .add_documents(documents!([
{ "objectId": 30, "title": "Hamlet", "_geo": { "lat": 12, "lng": 89 } } { "objectId": 30, "title": "Hamlet", RESERVED_GEO_FIELD_NAME: { "lat": 12, "lng": 89 } }
])) ]))
.unwrap(); .unwrap();
} }
@ -3146,34 +3148,34 @@ mod tests {
index index
.update_settings_using_wtxn(&mut wtxn, |settings| { .update_settings_using_wtxn(&mut wtxn, |settings| {
settings.set_primary_key(S("id")); settings.set_primary_key(S("id"));
settings.set_filterable_fields(hashset!(S("_geo"))); settings.set_filterable_fields(hashset!(S(RESERVED_GEO_FIELD_NAME)));
settings.set_sortable_fields(hashset!(S("_geo"))); settings.set_sortable_fields(hashset!(S(RESERVED_GEO_FIELD_NAME)));
}) })
.unwrap(); .unwrap();
wtxn.commit().unwrap(); wtxn.commit().unwrap();
let mut wtxn = index.write_txn().unwrap(); let mut wtxn = index.write_txn().unwrap();
index.add_documents_using_wtxn(&mut wtxn, documents!([ index.add_documents_using_wtxn(&mut wtxn, documents!([
{ "id": "1", "city": "Lille", "_geo": { "lat": 50.6299, "lng": 3.0569 } }, { "id": "1", "city": "Lille", RESERVED_GEO_FIELD_NAME: { "lat": 50.6299, "lng": 3.0569 } },
{ "id": "2", "city": "Mons-en-Barœul", "_geo": { "lat": 50.6415, "lng": 3.1106 } }, { "id": "2", "city": "Mons-en-Barœul", RESERVED_GEO_FIELD_NAME: { "lat": 50.6415, "lng": 3.1106 } },
{ "id": "3", "city": "Hellemmes", "_geo": { "lat": 50.6312, "lng": 3.1106 } }, { "id": "3", "city": "Hellemmes", RESERVED_GEO_FIELD_NAME: { "lat": 50.6312, "lng": 3.1106 } },
{ "id": "4", "city": "Villeneuve-d'Ascq", "_geo": { "lat": 50.6224, "lng": 3.1476 } }, { "id": "4", "city": "Villeneuve-d'Ascq", RESERVED_GEO_FIELD_NAME: { "lat": 50.6224, "lng": 3.1476 } },
{ "id": "5", "city": "Hem", "_geo": { "lat": 50.6552, "lng": 3.1897 } }, { "id": "5", "city": "Hem", RESERVED_GEO_FIELD_NAME: { "lat": 50.6552, "lng": 3.1897 } },
{ "id": "6", "city": "Roubaix", "_geo": { "lat": 50.6924, "lng": 3.1763 } }, { "id": "6", "city": "Roubaix", RESERVED_GEO_FIELD_NAME: { "lat": 50.6924, "lng": 3.1763 } },
{ "id": "7", "city": "Tourcoing", "_geo": { "lat": 50.7263, "lng": 3.1541 } }, { "id": "7", "city": "Tourcoing", RESERVED_GEO_FIELD_NAME: { "lat": 50.7263, "lng": 3.1541 } },
{ "id": "8", "city": "Mouscron", "_geo": { "lat": 50.7453, "lng": 3.2206 } }, { "id": "8", "city": "Mouscron", RESERVED_GEO_FIELD_NAME: { "lat": 50.7453, "lng": 3.2206 } },
{ "id": "9", "city": "Tournai", "_geo": { "lat": 50.6053, "lng": 3.3758 } }, { "id": "9", "city": "Tournai", RESERVED_GEO_FIELD_NAME: { "lat": 50.6053, "lng": 3.3758 } },
{ "id": "10", "city": "Ghent", "_geo": { "lat": 51.0537, "lng": 3.6957 } }, { "id": "10", "city": "Ghent", RESERVED_GEO_FIELD_NAME: { "lat": 51.0537, "lng": 3.6957 } },
{ "id": "11", "city": "Brussels", "_geo": { "lat": 50.8466, "lng": 4.3370 } }, { "id": "11", "city": "Brussels", RESERVED_GEO_FIELD_NAME: { "lat": 50.8466, "lng": 4.3370 } },
{ "id": "12", "city": "Charleroi", "_geo": { "lat": 50.4095, "lng": 4.4347 } }, { "id": "12", "city": "Charleroi", RESERVED_GEO_FIELD_NAME: { "lat": 50.4095, "lng": 4.4347 } },
{ "id": "13", "city": "Mons", "_geo": { "lat": 50.4502, "lng": 3.9623 } }, { "id": "13", "city": "Mons", RESERVED_GEO_FIELD_NAME: { "lat": 50.4502, "lng": 3.9623 } },
{ "id": "14", "city": "Valenciennes", "_geo": { "lat": 50.3518, "lng": 3.5326 } }, { "id": "14", "city": "Valenciennes", RESERVED_GEO_FIELD_NAME: { "lat": 50.3518, "lng": 3.5326 } },
{ "id": "15", "city": "Arras", "_geo": { "lat": 50.2844, "lng": 2.7637 } }, { "id": "15", "city": "Arras", RESERVED_GEO_FIELD_NAME: { "lat": 50.2844, "lng": 2.7637 } },
{ "id": "16", "city": "Cambrai", "_geo": { "lat": 50.1793, "lng": 3.2189 } }, { "id": "16", "city": "Cambrai", RESERVED_GEO_FIELD_NAME: { "lat": 50.1793, "lng": 3.2189 } },
{ "id": "17", "city": "Bapaume", "_geo": { "lat": 50.1112, "lng": 2.8547 } }, { "id": "17", "city": "Bapaume", RESERVED_GEO_FIELD_NAME: { "lat": 50.1112, "lng": 2.8547 } },
{ "id": "18", "city": "Amiens", "_geo": { "lat": 49.9314, "lng": 2.2710 } }, { "id": "18", "city": "Amiens", RESERVED_GEO_FIELD_NAME: { "lat": 49.9314, "lng": 2.2710 } },
{ "id": "19", "city": "Compiègne", "_geo": { "lat": 49.4449, "lng": 2.7913 } }, { "id": "19", "city": "Compiègne", RESERVED_GEO_FIELD_NAME: { "lat": 49.4449, "lng": 2.7913 } },
{ "id": "20", "city": "Paris", "_geo": { "lat": 48.9021, "lng": 2.3708 } } { "id": "20", "city": "Paris", RESERVED_GEO_FIELD_NAME: { "lat": 48.9021, "lng": 2.3708 } }
])).unwrap(); ])).unwrap();
wtxn.commit().unwrap(); wtxn.commit().unwrap();

6
crates/milli/src/update/index_documents/transform.rs
View File

@ -836,10 +836,8 @@ impl<'a, 'i> Transform<'a, 'i> {
}) })
.collect(); .collect();
let old_vectors_fid = settings_diff let old_vectors_fid =
.old settings_diff.old.fields_ids_map.id(crate::constants::RESERVED_VECTORS_FIELD_NAME);
.fields_ids_map
.id(crate::vector::parsed_vectors::RESERVED_VECTORS_FIELD_NAME);
// We initialize the sorter with the user indexing settings. // We initialize the sorter with the user indexing settings.
let mut flattened_sorter = let mut flattened_sorter =

3
crates/milli/src/update/index_documents/typed_chunk.rs
View File

@ -137,8 +137,7 @@ pub(crate) fn write_typed_chunk_into_index(
let _entered = span.enter(); let _entered = span.enter();
let fields_ids_map = index.fields_ids_map(wtxn)?; let fields_ids_map = index.fields_ids_map(wtxn)?;
let vectors_fid = let vectors_fid = fields_ids_map.id(crate::constants::RESERVED_VECTORS_FIELD_NAME);
fields_ids_map.id(crate::vector::parsed_vectors::RESERVED_VECTORS_FIELD_NAME);
let mut builder = MergerBuilder::new(KeepLatestObkv); let mut builder = MergerBuilder::new(KeepLatestObkv);
for typed_chunk in typed_chunks { for typed_chunk in typed_chunks {

20
crates/milli/src/update/new/document.rs
View File

@ -7,8 +7,8 @@ use serde_json::value::RawValue;
use super::vector_document::VectorDocument; use super::vector_document::VectorDocument;
use super::{KvReaderFieldId, KvWriterFieldId}; use super::{KvReaderFieldId, KvWriterFieldId};
use crate::constants::{RESERVED_GEO_FIELD_NAME, RESERVED_VECTORS_FIELD_NAME};
use crate::documents::FieldIdMapper; use crate::documents::FieldIdMapper;
use crate::vector::parsed_vectors::RESERVED_VECTORS_FIELD_NAME;
use crate::{DocumentId, GlobalFieldsIdsMap, Index, InternalError, Result, UserError}; use crate::{DocumentId, GlobalFieldsIdsMap, Index, InternalError, Result, UserError};
/// A view into a document that can represent either the current version from the DB, /// A view into a document that can represent either the current version from the DB,
@ -80,7 +80,7 @@ impl<'t, Mapper: FieldIdMapper> Document<'t> for DocumentFromDb<'t, Mapper> {
Err(error) => return Some(Err(error.into())), Err(error) => return Some(Err(error.into())),
}; };
if name == RESERVED_VECTORS_FIELD_NAME || name == "_geo" { if name == RESERVED_VECTORS_FIELD_NAME || name == RESERVED_GEO_FIELD_NAME {
continue; continue;
} }
@ -100,7 +100,7 @@ impl<'t, Mapper: FieldIdMapper> Document<'t> for DocumentFromDb<'t, Mapper> {
} }
fn geo_field(&self) -> Result<Option<&'t RawValue>> { fn geo_field(&self) -> Result<Option<&'t RawValue>> {
self.field("_geo") self.field(RESERVED_GEO_FIELD_NAME)
} }
fn top_level_fields_count(&self) -> usize { fn top_level_fields_count(&self) -> usize {
@ -115,7 +115,7 @@ impl<'t, Mapper: FieldIdMapper> Document<'t> for DocumentFromDb<'t, Mapper> {
} }
fn top_level_field(&self, k: &str) -> Result<Option<&'t RawValue>> { fn top_level_field(&self, k: &str) -> Result<Option<&'t RawValue>> {
if k == RESERVED_VECTORS_FIELD_NAME || k == "_geo" { if k == RESERVED_VECTORS_FIELD_NAME || k == RESERVED_GEO_FIELD_NAME {
return Ok(None); return Ok(None);
} }
self.field(k) self.field(k)
@ -367,7 +367,9 @@ where
} }
if let Some(geo_value) = document.geo_field()? { if let Some(geo_value) = document.geo_field()? {
let fid = fields_ids_map.id_or_insert("_geo").ok_or(UserError::AttributeLimitReached)?; let fid = fields_ids_map
.id_or_insert(RESERVED_GEO_FIELD_NAME)
.ok_or(UserError::AttributeLimitReached)?;
fields_ids_map.id_or_insert("_geo.lat").ok_or(UserError::AttributeLimitReached)?; fields_ids_map.id_or_insert("_geo.lat").ok_or(UserError::AttributeLimitReached)?;
fields_ids_map.id_or_insert("_geo.lng").ok_or(UserError::AttributeLimitReached)?; fields_ids_map.id_or_insert("_geo.lng").ok_or(UserError::AttributeLimitReached)?;
unordered_field_buffer.push((fid, geo_value)); unordered_field_buffer.push((fid, geo_value));
@ -409,7 +411,9 @@ impl<'doc> Versions<'doc> {
} }
pub fn iter_top_level_fields(&self) -> impl Iterator<Item = (&'doc str, &'doc RawValue)> + '_ { pub fn iter_top_level_fields(&self) -> impl Iterator<Item = (&'doc str, &'doc RawValue)> + '_ {
self.data.iter().filter(|(k, _)| *k != RESERVED_VECTORS_FIELD_NAME && *k != "_geo") self.data
.iter()
.filter(|(k, _)| *k != RESERVED_VECTORS_FIELD_NAME && *k != RESERVED_GEO_FIELD_NAME)
} }
pub fn vectors_field(&self) -> Option<&'doc RawValue> { pub fn vectors_field(&self) -> Option<&'doc RawValue> {
@ -417,7 +421,7 @@ impl<'doc> Versions<'doc> {
} }
pub fn geo_field(&self) -> Option<&'doc RawValue> { pub fn geo_field(&self) -> Option<&'doc RawValue> {
self.data.get("_geo") self.data.get(RESERVED_GEO_FIELD_NAME)
} }
pub fn len(&self) -> usize { pub fn len(&self) -> usize {
@ -429,7 +433,7 @@ impl<'doc> Versions<'doc> {
} }
pub fn top_level_field(&self, k: &str) -> Option<&'doc RawValue> { pub fn top_level_field(&self, k: &str) -> Option<&'doc RawValue> {
if k == RESERVED_VECTORS_FIELD_NAME || k == "_geo" { if k == RESERVED_VECTORS_FIELD_NAME || k == RESERVED_GEO_FIELD_NAME {
return None; return None;
} }
self.data.get(k) self.data.get(k)

25
crates/milli/src/update/new/extract/documents.rs
View File

@ -4,6 +4,7 @@ use bumpalo::Bump;
use hashbrown::HashMap; use hashbrown::HashMap;
use super::DelAddRoaringBitmap; use super::DelAddRoaringBitmap;
use crate::constants::RESERVED_GEO_FIELD_NAME;
use crate::update::new::channel::DocumentsSender; use crate::update::new::channel::DocumentsSender;
use crate::update::new::document::{write_to_obkv, Document as _}; use crate::update::new::document::{write_to_obkv, Document as _};
use crate::update::new::indexer::document_changes::{DocumentChangeContext, Extractor}; use crate::update::new::indexer::document_changes::{DocumentChangeContext, Extractor};
@ -62,8 +63,10 @@ impl<'a, 'b, 'extractor> Extractor<'extractor> for DocumentsExtractor<'a, 'b> {
context.index, context.index,
&context.db_fields_ids_map, &context.db_fields_ids_map,
)?; )?;
let geo_iter = let geo_iter = content
content.geo_field().transpose().map(|res| res.map(|rv| ("_geo", rv))); .geo_field()
.transpose()
.map(|res| res.map(|rv| (RESERVED_GEO_FIELD_NAME, rv)));
for res in content.iter_top_level_fields().chain(geo_iter) { for res in content.iter_top_level_fields().chain(geo_iter) {
let (f, _) = res?; let (f, _) = res?;
let entry = document_extractor_data let entry = document_extractor_data
@ -79,8 +82,10 @@ impl<'a, 'b, 'extractor> Extractor<'extractor> for DocumentsExtractor<'a, 'b> {
let docid = update.docid(); let docid = update.docid();
let content = let content =
update.current(&context.rtxn, context.index, &context.db_fields_ids_map)?; update.current(&context.rtxn, context.index, &context.db_fields_ids_map)?;
let geo_iter = let geo_iter = content
content.geo_field().transpose().map(|res| res.map(|rv| ("_geo", rv))); .geo_field()
.transpose()
.map(|res| res.map(|rv| (RESERVED_GEO_FIELD_NAME, rv)));
for res in content.iter_top_level_fields().chain(geo_iter) { for res in content.iter_top_level_fields().chain(geo_iter) {
let (f, _) = res?; let (f, _) = res?;
let entry = document_extractor_data let entry = document_extractor_data
@ -90,8 +95,10 @@ impl<'a, 'b, 'extractor> Extractor<'extractor> for DocumentsExtractor<'a, 'b> {
*entry -= 1; *entry -= 1;
} }
let content = update.updated(); let content = update.updated();
let geo_iter = let geo_iter = content
content.geo_field().transpose().map(|res| res.map(|rv| ("_geo", rv))); .geo_field()
.transpose()
.map(|res| res.map(|rv| (RESERVED_GEO_FIELD_NAME, rv)));
for res in content.iter_top_level_fields().chain(geo_iter) { for res in content.iter_top_level_fields().chain(geo_iter) {
let (f, _) = res?; let (f, _) = res?;
let entry = document_extractor_data let entry = document_extractor_data
@ -121,8 +128,10 @@ impl<'a, 'b, 'extractor> Extractor<'extractor> for DocumentsExtractor<'a, 'b> {
DocumentChange::Insertion(insertion) => { DocumentChange::Insertion(insertion) => {
let docid = insertion.docid(); let docid = insertion.docid();
let content = insertion.inserted(); let content = insertion.inserted();
let geo_iter = let geo_iter = content
content.geo_field().transpose().map(|res| res.map(|rv| ("_geo", rv))); .geo_field()
.transpose()
.map(|res| res.map(|rv| (RESERVED_GEO_FIELD_NAME, rv)));
for res in content.iter_top_level_fields().chain(geo_iter) { for res in content.iter_top_level_fields().chain(geo_iter) {
let (f, _) = res?; let (f, _) = res?;
let entry = document_extractor_data let entry = document_extractor_data

3
crates/milli/src/update/new/extract/faceted/facet_document.rs
View File

@ -1,5 +1,6 @@
use serde_json::Value; use serde_json::Value;
use crate::constants::RESERVED_GEO_FIELD_NAME;
use crate::update::new::document::Document; use crate::update::new::document::Document;
use crate::update::new::extract::geo::extract_geo_coordinates; use crate::update::new::extract::geo::extract_geo_coordinates;
use crate::update::new::extract::perm_json_p; use crate::update::new::extract::perm_json_p;
@ -69,7 +70,7 @@ pub fn extract_document_facets<'doc>(
} }
} }
if attributes_to_extract.contains(&"_geo") { if attributes_to_extract.contains(&RESERVED_GEO_FIELD_NAME) {
if let Some(geo_value) = document.geo_field()? { if let Some(geo_value) = document.geo_field()? {
if let Some([lat, lng]) = extract_geo_coordinates(external_document_id, geo_value)? { if let Some([lat, lng]) = extract_geo_coordinates(external_document_id, geo_value)? {
let (lat_fid, lng_fid) = field_id_map let (lat_fid, lng_fid) = field_id_map

5
crates/milli/src/update/new/extract/geo/mod.rs
View File

@ -9,6 +9,7 @@ use heed::RoTxn;
use serde_json::value::RawValue; use serde_json::value::RawValue;
use serde_json::Value; use serde_json::Value;
use crate::constants::RESERVED_GEO_FIELD_NAME;
use crate::error::GeoError; use crate::error::GeoError;
use crate::update::new::document::Document; use crate::update::new::document::Document;
use crate::update::new::indexer::document_changes::{DocumentChangeContext, Extractor}; use crate::update::new::indexer::document_changes::{DocumentChangeContext, Extractor};
@ -28,8 +29,8 @@ impl GeoExtractor {
index: &Index, index: &Index,
grenad_parameters: GrenadParameters, grenad_parameters: GrenadParameters,
) -> Result<Option<Self>> { ) -> Result<Option<Self>> {
let is_sortable = index.sortable_fields(rtxn)?.contains("_geo"); let is_sortable = index.sortable_fields(rtxn)?.contains(RESERVED_GEO_FIELD_NAME);
let is_filterable = index.filterable_fields(rtxn)?.contains("_geo"); let is_filterable = index.filterable_fields(rtxn)?.contains(RESERVED_GEO_FIELD_NAME);
if is_sortable || is_filterable { if is_sortable || is_filterable {
Ok(Some(GeoExtractor { grenad_parameters })) Ok(Some(GeoExtractor { grenad_parameters }))
} else { } else {

5
crates/milli/src/update/new/vector_document.rs
View File

@ -10,11 +10,10 @@ use serde_json::value::RawValue;
use super::document::{Document, DocumentFromDb, DocumentFromVersions, Versions}; use super::document::{Document, DocumentFromDb, DocumentFromVersions, Versions};
use super::indexer::de::DeserrRawValue; use super::indexer::de::DeserrRawValue;
use crate::constants::RESERVED_VECTORS_FIELD_NAME;
use crate::documents::FieldIdMapper; use crate::documents::FieldIdMapper;
use crate::index::IndexEmbeddingConfig; use crate::index::IndexEmbeddingConfig;
use crate::vector::parsed_vectors::{ use crate::vector::parsed_vectors::{RawVectors, RawVectorsError, VectorOrArrayOfVectors};
RawVectors, RawVectorsError, VectorOrArrayOfVectors, RESERVED_VECTORS_FIELD_NAME,
};
use crate::vector::{ArroyWrapper, Embedding, EmbeddingConfigs}; use crate::vector::{ArroyWrapper, Embedding, EmbeddingConfigs};
use crate::{DocumentId, Index, InternalError, Result, UserError}; use crate::{DocumentId, Index, InternalError, Result, UserError};

4
crates/milli/src/update/settings.rs
View File

@ -14,6 +14,7 @@ use time::OffsetDateTime;
use super::del_add::DelAddOperation; use super::del_add::DelAddOperation;
use super::index_documents::{IndexDocumentsConfig, Transform}; use super::index_documents::{IndexDocumentsConfig, Transform};
use super::IndexerConfig; use super::IndexerConfig;
use crate::constants::{RESERVED_GEO_FIELD_NAME, RESERVED_VECTORS_FIELD_NAME};
use crate::criterion::Criterion; use crate::criterion::Criterion;
use crate::error::UserError; use crate::error::UserError;
use crate::index::{ use crate::index::{
@ -25,7 +26,6 @@ use crate::prompt::default_max_bytes;
use crate::proximity::ProximityPrecision; use crate::proximity::ProximityPrecision;
use crate::update::index_documents::IndexDocumentsMethod; use crate::update::index_documents::IndexDocumentsMethod;
use crate::update::{IndexDocuments, UpdateIndexingStep}; use crate::update::{IndexDocuments, UpdateIndexingStep};
use crate::vector::parsed_vectors::RESERVED_VECTORS_FIELD_NAME;
use crate::vector::settings::{ use crate::vector::settings::{
check_set, check_unset, EmbedderAction, EmbedderSource, EmbeddingSettings, ReindexAction, check_set, check_unset, EmbedderAction, EmbedderSource, EmbeddingSettings, ReindexAction,
WriteBackToDocuments, WriteBackToDocuments,
@ -1535,7 +1535,7 @@ impl InnerIndexSettings {
.filter_map(|(field, count)| (count != 0).then_some(field)) .filter_map(|(field, count)| (count != 0).then_some(field))
.collect(); .collect();
// index.fields_ids_map($a)? ==>> fields_ids_map // index.fields_ids_map($a)? ==>> fields_ids_map
let geo_fields_ids = match fields_ids_map.id("_geo") { let geo_fields_ids = match fields_ids_map.id(RESERVED_GEO_FIELD_NAME) {
Some(gfid) => { Some(gfid) => {
let is_sortable = index.sortable_fields_ids(rtxn)?.contains(&gfid); let is_sortable = index.sortable_fields_ids(rtxn)?.contains(&gfid);
let is_filterable = index.filterable_fields_ids(rtxn)?.contains(&gfid); let is_filterable = index.filterable_fields_ids(rtxn)?.contains(&gfid);

2
crates/milli/src/vector/parsed_vectors.rs
View File

@ -10,8 +10,6 @@ use crate::index::IndexEmbeddingConfig;
use crate::update::del_add::{DelAdd, KvReaderDelAdd}; use crate::update::del_add::{DelAdd, KvReaderDelAdd};
use crate::{DocumentId, FieldId, InternalError, UserError}; use crate::{DocumentId, FieldId, InternalError, UserError};
pub const RESERVED_VECTORS_FIELD_NAME: &str = "_vectors";
#[derive(serde::Serialize, Debug)] #[derive(serde::Serialize, Debug)]
#[serde(untagged)] #[serde(untagged)]
pub enum RawVectors<'doc> { pub enum RawVectors<'doc> {