searchengines-web/MeiliSearch
searchengines-web/MeiliSearch
1
0
Fork 0
mirror of https://github.com/meilisearch/MeiliSearch synced 2025-01-24 20:27:32 +01:00
Code Issues Releases Wiki Activity

Create update files in new format

Browse Source
This commit is contained in:
Louis Dureuil 2025-01-16 16:54:05 +01:00
parent b9d92c481b
commit 50fca8fc70
No known key found for this signature in database
1 changed files with 7 additions and 12 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

19
crates/index-scheduler/src/scheduler/process_dump_creation.rs
View File

@ -4,7 +4,6 @@ use std::sync::atomic::Ordering;
use dump::IndexMetadata; use dump::IndexMetadata;
use meilisearch_types::milli::constants::RESERVED_VECTORS_FIELD_NAME; use meilisearch_types::milli::constants::RESERVED_VECTORS_FIELD_NAME;
use meilisearch_types::milli::documents::{obkv_to_object, DocumentsBatchReader};
use meilisearch_types::milli::progress::Progress; use meilisearch_types::milli::progress::Progress;
use meilisearch_types::milli::vector::parsed_vectors::{ExplicitVectors, VectorOrArrayOfVectors}; use meilisearch_types::milli::vector::parsed_vectors::{ExplicitVectors, VectorOrArrayOfVectors};
use meilisearch_types::milli::{self}; use meilisearch_types::milli::{self};
@ -89,19 +88,15 @@ impl IndexScheduler {
if status == Status::Enqueued { if status == Status::Enqueued {
let content_file = self.queue.file_store.get_update(content_file)?; let content_file = self.queue.file_store.get_update(content_file)?;
let reader = DocumentsBatchReader::from_reader(content_file) for document in
.map_err(|e| Error::from_milli(e.into(), None))?; serde_json::de::Deserializer::from_reader(content_file).into_iter()
let (mut cursor, documents_batch_index) = reader.into_cursor_and_fields_index();
while let Some(doc) =
cursor.next_document().map_err(|e| Error::from_milli(e.into(), None))?
{ {
dump_content_file.push_document( let document = document.map_err(|e| {
&obkv_to_object(doc, &documents_batch_index) Error::from_milli(milli::InternalError::SerdeJson(e).into(), None)
.map_err(|e| Error::from_milli(e, None))?, })?;
)?; dump_content_file.push_document(&document)?;
} }
dump_content_file.flush()?; dump_content_file.flush()?;
} }
} }