searchengines-web/MeiliSearch
searchengines-web/MeiliSearch
1
0
Fork 0
mirror of https://github.com/meilisearch/MeiliSearch synced 2024-12-04 18:45:46 +01:00
Code Issues Releases Wiki Activity

Keep the ZK flow when enqueuing tasks

Browse Source
This commit is contained in:
Clément Renault 2023-08-30 17:15:15 +02:00
parent c488a4a351
commit 2d1434da81
No known key found for this signature in database
GPG Key ID: F250A4C4E3AE5F5F
1 changed files with 262 additions and 241 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

37
index-scheduler/src/lib.rs
View File

@ -609,7 +609,8 @@ impl IndexScheduler {
#[cfg(test)]
run.breakpoint(Breakpoint::Init);
if let Some(zookeeper) = self.zookeeper.clone() {
let latch = match self.zookeeper.clone() {
Some(zookeeper) => {
let id = Uuid::new_v4().to_string();
let latch = LeaderLatch::new(zookeeper.clone(), id, "/election".to_string());
latch.start().unwrap();
@ -659,8 +660,12 @@ impl IndexScheduler {
let dst = snapshot_dir.join("indexes");
for result in std::fs::read_dir(&dst).unwrap() {
let entry = result.unwrap();
let uuid =
entry.file_name().as_os_str().to_str().unwrap().to_string();
let uuid = entry
.file_name()
.as_os_str()
.to_str()
.unwrap()
.to_string();
log::info!("\tDownloading the index {}", uuid.to_string());
std::fs::copy(
dst.join(&uuid),
@ -739,8 +744,12 @@ impl IndexScheduler {
let path = path.unwrap();
// Add raw task content in local DB
log::info!("Received a new task from the cluster at {}", path);
let (data, _stat) =
this.zookeeper.as_ref().unwrap().get_data(&path, false).unwrap();
let (data, _stat) = this
.zookeeper
.as_ref()
.unwrap()
.get_data(&path, false)
.unwrap();
let task = serde_json::from_slice(data.as_slice()).unwrap();
let mut wtxn = this.env.write_txn().unwrap();
this.register_raw_task(&mut wtxn, &task).unwrap();
@ -757,16 +766,21 @@ impl IndexScheduler {
this.wake_up.signal();
})
.unwrap();
Some(latch)
}
None => None,
};
let this = self.private_clone();
std::thread::spawn(move || {
loop {
// we're either a leader or not running in a cluster,
// either way we should wait until we receive a task.
let wake_up = this.wake_up.clone();
let _ = wake_up.wait();
this.wake_up.wait();
// TODO watch the /election node and send a signal once it changes (be careful about atomics ordering)
if latch.as_ref().map_or(true, |latch| latch.has_leadership()) {
match this.tick() {
Ok(TickOutcome::TickAgain(n)) => {
// We must tick again.
@ -849,7 +863,9 @@ impl IndexScheduler {
vec![],
None,
);
log::info!("Notified everyone about the new snapshot {snapshot_id}");
log::info!(
"Notified everyone about the new snapshot {snapshot_id}"
);
// We can now delete all the tasks that has been processed
let processed = this
@ -878,6 +894,7 @@ impl IndexScheduler {
}
}
}
}
});
}
@ -1264,12 +1281,14 @@ impl IndexScheduler {
// (that it does not contain duplicate indexes).
check_index_swap_validity(&task)?;
if self.zookeeper.is_none() {
this.register_raw_task(&mut wtxn, &task)?;
if let Err(e) = wtxn.commit() {
this.delete_persisted_task_data(&task)?;
return Err(e.into());
}
}
// If the registered task is a task cancelation
// we inform the processing tasks to stop (if necessary).
@ -1281,8 +1300,10 @@ impl IndexScheduler {
}
}
if self.zookeeper.is_none() {
// notify the scheduler loop to execute a new tick
this.wake_up.signal();
}
// TODO: send task to ZK in raw json.
if let Some(zookeeper) = &self.zookeeper {