bug(lib): drop env on last use

fixes the `too many open files` error when running tests by closing the
environment on last drop
This commit is contained in:
Marin Postma 2021-12-15 17:55:39 +01:00
parent ea0a5271f7
commit b28a465304
8 changed files with 51 additions and 24 deletions

View file

@ -114,7 +114,7 @@ impl Clone for TaskStore {
}
impl TaskStore {
pub fn new(env: heed::Env) -> Result<Self> {
pub fn new(env: Arc<heed::Env>) -> Result<Self> {
let mut store = Store::new(env)?;
let unfinished_tasks = store.reset_and_return_unfinished_tasks()?;
let store = Arc::new(store);
@ -293,7 +293,7 @@ impl TaskStore {
Ok(())
}
pub fn load_dump(src: impl AsRef<Path>, env: Env) -> anyhow::Result<()> {
pub fn load_dump(src: impl AsRef<Path>, env: Arc<Env>) -> anyhow::Result<()> {
// create a dummy update field store, since it is not needed right now.
let store = Self::new(env.clone())?;
@ -340,7 +340,7 @@ pub mod test {
}
impl MockTaskStore {
pub fn new(env: heed::Env) -> Result<Self> {
pub fn new(env: Arc<heed::Env>) -> Result<Self> {
Ok(Self::Real(TaskStore::new(env)?))
}
@ -432,7 +432,7 @@ pub mod test {
}
}
pub fn load_dump(path: impl AsRef<Path>, env: Env) -> anyhow::Result<()> {
pub fn load_dump(path: impl AsRef<Path>, env: Arc<Env>) -> anyhow::Result<()> {
TaskStore::load_dump(path, env)
}
}

View file

@ -10,6 +10,7 @@ use std::convert::TryInto;
use std::mem::size_of;
use std::ops::Range;
use std::result::Result as StdResult;
use std::sync::Arc;
use heed::types::{ByteSlice, OwnedType, SerdeJson, Unit};
use heed::{BytesDecode, BytesEncode, Database, Env, RoTxn, RwTxn};
@ -53,18 +54,26 @@ impl<'a> BytesDecode<'a> for IndexUidTaskIdCodec {
}
pub struct Store {
env: Env,
env: Arc<Env>,
uids_task_ids: Database<IndexUidTaskIdCodec, Unit>,
tasks: Database<OwnedType<BEU64>, SerdeJson<Task>>,
}
impl Drop for Store {
fn drop(&mut self) {
if Arc::strong_count(&self.env) == 1 {
self.env.as_ref().clone().prepare_for_closing();
}
}
}
impl Store {
/// Create a new store from the specified `Path`.
/// Be really cautious when calling this function, the returned `Store` may
/// be in an invalid state, with dangling processing tasks.
/// You want to patch all un-finished tasks and put them in your pending
/// queue with the `reset_and_return_unfinished_update` method.
pub fn new(env: heed::Env) -> Result<Self> {
pub fn new(env: Arc<heed::Env>) -> Result<Self> {
let uids_task_ids = env.create_database(Some(UID_TASK_IDS))?;
let tasks = env.create_database(Some(TASKS))?;
@ -257,10 +266,10 @@ pub mod test {
Fake(Mocker),
}
pub struct TmpEnv(TempDir, heed::Env);
pub struct TmpEnv(TempDir, Arc<heed::Env>);
impl TmpEnv {
pub fn env(&self) -> heed::Env {
pub fn env(&self) -> Arc<heed::Env> {
self.1.clone()
}
}
@ -271,13 +280,13 @@ pub mod test {
let mut options = EnvOpenOptions::new();
options.map_size(4096 * 100000);
options.max_dbs(1000);
let env = options.open(tmp.path()).unwrap();
let env = Arc::new(options.open(tmp.path()).unwrap());
TmpEnv(tmp, env)
}
impl MockStore {
pub fn new(env: heed::Env) -> Result<Self> {
pub fn new(env: Arc<heed::Env>) -> Result<Self> {
Ok(Self::Real(Store::new(env)?))
}