From 50fca8fc7063f053f7bc75d396e4e1852d08e929 Mon Sep 17 00:00:00 2001 From: Louis Dureuil Date: Thu, 16 Jan 2025 16:54:05 +0100 Subject: [PATCH] Create update files in new format --- .../src/scheduler/process_dump_creation.rs | 19 +++++++------------ 1 file changed, 7 insertions(+), 12 deletions(-) diff --git a/crates/index-scheduler/src/scheduler/process_dump_creation.rs b/crates/index-scheduler/src/scheduler/process_dump_creation.rs index 6b580abeb..9691bdbef 100644 --- a/crates/index-scheduler/src/scheduler/process_dump_creation.rs +++ b/crates/index-scheduler/src/scheduler/process_dump_creation.rs @@ -4,7 +4,6 @@ use std::sync::atomic::Ordering; use dump::IndexMetadata; use meilisearch_types::milli::constants::RESERVED_VECTORS_FIELD_NAME; -use meilisearch_types::milli::documents::{obkv_to_object, DocumentsBatchReader}; use meilisearch_types::milli::progress::Progress; use meilisearch_types::milli::vector::parsed_vectors::{ExplicitVectors, VectorOrArrayOfVectors}; use meilisearch_types::milli::{self}; @@ -89,19 +88,15 @@ impl IndexScheduler { if status == Status::Enqueued { let content_file = self.queue.file_store.get_update(content_file)?; - let reader = DocumentsBatchReader::from_reader(content_file) - .map_err(|e| Error::from_milli(e.into(), None))?; - - let (mut cursor, documents_batch_index) = reader.into_cursor_and_fields_index(); - - while let Some(doc) = - cursor.next_document().map_err(|e| Error::from_milli(e.into(), None))? + for document in + serde_json::de::Deserializer::from_reader(content_file).into_iter() { - dump_content_file.push_document( - &obkv_to_object(doc, &documents_batch_index) - .map_err(|e| Error::from_milli(e, None))?, - )?; + let document = document.map_err(|e| { + Error::from_milli(milli::InternalError::SerdeJson(e).into(), None) + })?; + dump_content_file.push_document(&document)?; } + dump_content_file.flush()?; } }