review edits

This commit is contained in:
Marin Postma 2021-05-31 16:40:59 +02:00 committed by mpostma
parent 1c4f0b2ccf
commit 6609f9e3be
13 changed files with 100 additions and 105 deletions

View file

@ -1,12 +1,9 @@
use std::{
fs::{create_dir_all, File},
io::{BufRead, BufReader},
path::Path,
sync::Arc,
};
use std::fs::{create_dir_all, File};
use std::io::{BufRead, BufReader, Write};
use std::path::Path;
use std::sync::Arc;
use anyhow::bail;
use anyhow::Context;
use anyhow::{bail, Context};
use heed::RoTxn;
use indexmap::IndexMap;
use milli::update::{IndexDocumentsMethod, UpdateFormat::JsonStream};
@ -55,7 +52,7 @@ impl Index {
}
serde_json::to_writer(&mut document_file, &json_map)?;
std::io::Write::write(&mut document_file, b"\n")?;
document_file.write_all(b"\n")?;
json_map.clear();
}
@ -82,7 +79,7 @@ impl Index {
pub fn load_dump(
src: impl AsRef<Path>,
dst: impl AsRef<Path>,
size: u64,
size: usize,
indexing_options: &IndexerOpts,
) -> anyhow::Result<()> {
let dir_name = src
@ -99,7 +96,7 @@ impl Index {
primary_key,
} = serde_json::from_reader(&mut meta_file)?;
let settings = settings.check();
let index = Self::open(&dst_dir_path, size as usize)?;
let index = Self::open(&dst_dir_path, size)?;
let mut txn = index.write_txn()?;
let handler = UpdateHandler::new(&indexing_options)?;

View file

@ -1,10 +1,9 @@
use std::collections::{BTreeSet, HashSet};
use std::fs::create_dir_all;
use std::marker::PhantomData;
use std::ops::Deref;
use std::path::Path;
use std::sync::Arc;
use std::{
collections::{BTreeSet, HashSet},
marker::PhantomData,
path::Path,
};
use anyhow::{bail, Context};
use heed::{EnvOpenOptions, RoTxn};
@ -44,7 +43,7 @@ where
impl Index {
pub fn open(path: impl AsRef<Path>, size: usize) -> anyhow::Result<Self> {
std::fs::create_dir_all(&path)?;
create_dir_all(&path)?;
let mut options = EnvOpenOptions::new();
options.map_size(size);
let index = milli::Index::new(options, &path)?;
@ -113,8 +112,6 @@ impl Index {
let mut documents = Vec::new();
println!("fields to display: {:?}", fields_to_display);
for entry in iter {
let (_id, obkv) = entry?;
let object = obkv_to_json(&fields_to_display, &fields_ids_map, obkv)?;

View file

@ -197,10 +197,8 @@ impl Index {
builder.update_format(format);
builder.index_documents_method(method);
//let indexing_callback =
//|indexing_step, update_id| info!("update {}: {:?}", update_id, indexing_step);
let indexing_callback = |_, _| ();
let indexing_callback =
|indexing_step, update_id| info!("update {}: {:?}", update_id, indexing_step);
let gzipped = false;
let addition = match content {