Add WIP bloom stage (rewrite git history)

src/bloom.c

@@ -1,47 +1,45 @@
 #include "bloom.h"
 #include "ens/storage.h"
+#include "exposure-notification.h"
 
-bloom_filter_t* bloom_create(size_t size) {
+bloom_filter_t* bloom_init(size_t size) {
     bloom_filter_t* bloom = k_calloc(1, sizeof(bloom_filter_t));
+    if (!bloom) {
+        return NULL;
+    }
     bloom->size = size;
-    bloom->data = k_malloc(size * sizeof(uint8_t));
+    bloom->data = k_calloc(size, sizeof(uint8_t));
+    if (!bloom->data) {
+        k_free(bloom);
+        return NULL;
+    }
     return bloom;
 }
 
-void bloom_delete(bloom_filter_t* bloom) {
+void bloom_destroy(bloom_filter_t* bloom) {
     if (bloom) {
-        while (bloom->func) {
-            bloom_hash_t* h;
-            h = bloom->func;
-            bloom->func = h->next;
-            k_free(h);
-        }
         k_free(bloom->data);
         k_free(bloom);
     }
 }
 
-void bloom_add_record(bloom_filter_t* bloom, record_t* record) {
+void bloom_add_record(bloom_filter_t* bloom, ENIntervalIdentifier* rpi) {
-    bloom_hash_t* h = bloom->func;
     uint8_t* data = bloom->data;
-    while (h) {
+    for (int i = 0; i < sizeof(*rpi); i += 2) {
-        uint32_t hash = h->func(record);
+        uint32_t hash = rpi->b[i] << 8 & rpi->b[i + 1];
         hash %= bloom->size * 8;
         data[hash / 8] |= 1 << (hash % 8);
-        h = h->next;
     }
 }
 
-bool bloom_probably_has_record(bloom_filter_t* bloom, record_t* record) {
+bool bloom_probably_has_record(bloom_filter_t* bloom, ENIntervalIdentifier* rpi) {
-    bloom_hash_t* h = bloom->func;
     uint8_t* data = bloom->data;
-    while (h) {
+    for (int i = 0; i < sizeof(*rpi); i += 2) {
-        uint32_t hash = h->func(record);
+        uint32_t hash = rpi->b[i] << 8 & rpi->b[i + 1];
         hash %= bloom->size * 8;
-        if ((data[hash / 8] & (1 << (hash % 8))) == 0) {
+        if (!(data[hash / 8] & (1 << (hash % 8)))) {
             return false;
         }
-        h = h->next;
     }
     return true;
-}
+}

src/bloom.h

@@ -2,16 +2,9 @@
 #define BLOOM_H
 
 #include "ens/storage.h"
-
+#include "exposure-notification.h"
-typedef uint32_t (*hash_function)(const void* data);
-
-typedef struct bloom_hash {
-    hash_function func;
-    struct bloom_hash* next;
-} bloom_hash_t;
 
 typedef struct bloom_filter {
-    bloom_hash_t* func;
     uint8_t* data;
     size_t size;
 } bloom_filter_t;
@@ -19,12 +12,14 @@ typedef struct bloom_filter {
 /**
  * Initialize the bloom filter on basis of the already registerred records.
  */
-bloom_filter_t* bloom_create(size_t size);
+bloom_filter_t* bloom_init(size_t size);
+
+void bloom_destroy(bloom_filter_t* bloom);
 
 // TODO lome: maybe only use RPI (should be sufficient)
-void bloom_add_record(bloom_filter_t* bloom, record_t* record);
+void bloom_add_record(bloom_filter_t* bloom, ENIntervalIdentifier* rpi);
 
 // TODO lome: maybe only use RPI (should be sufficient)
-bool bloom_probably_has_record(bloom_filter_t* bloom, record_t* record);
+bool bloom_probably_has_record(bloom_filter_t* bloom, ENIntervalIdentifier* rpi);
 
-#endif
+#endif

src/contacts.c

@@ -54,6 +54,9 @@ int register_record(record_t* record) {
     return rc;
 }
 
+/**
+ * Dumb implementation, where a single iterator is created for iterating over the entire flash.
+ */
 int get_number_of_infected_for_multiple_intervals_dumb(infected_for_period_key_ctx_t* ctx, int count) {
     record_iterator_t iterator;
     int rc = ens_records_iterator_init_timerange(&iterator, NULL, NULL);
@@ -73,8 +76,10 @@ int get_number_of_infected_for_multiple_intervals_dumb(infected_for_period_key_c
     return 0;
 }
 
+/**
+ * Simple implementation, where an iterator is created for every element in the passed arrray.
+ */
 int get_number_of_infected_for_multiple_intervals_simple(infected_for_period_key_ctx_t* ctx, int count) {
-    printk("start of simple\n");
     record_iterator_t iterator;
     for (int i = 0; i < count; i++) {
         int rc = ens_records_iterator_init_timerange(&iterator, &ctx[i].search_start, &ctx[i].search_end);
@@ -93,8 +98,10 @@ int get_number_of_infected_for_multiple_intervals_simple(infected_for_period_key
     return 0;
 }
 
+/**
+ * Optimized implementation, where overlapping search intervals for consecutive RPI are merged.
+ */
 int get_number_of_infected_for_multiple_intervals_optimized(infected_for_period_key_ctx_t* ctx, int count) {
-    printk("start of opt\n");
     record_iterator_t iterator;
     int i = 0;
     while (i < count) {
@@ -124,9 +131,163 @@ int get_number_of_infected_for_multiple_intervals_optimized(infected_for_period_
     return 0;
 }
 
+/**
+ * Fill the bloom filter with all stored records.
+ */
+void fill_bloom_with_stored_records(bloom_filter_t* bloom) {
+    // init iterator for filling bloom filter
+    record_iterator_t iterator;
+    int rc = ens_records_iterator_init_timerange(&iterator, NULL, NULL);
+    if (rc) {
+        printk("init iterator failed0 (err %d)\n", rc);
+        return;
+    }
+
+    // fill bloom filter with records
+    while (!iterator.finished) {
+        bloom_add_record(bloom, &iterator.current);
+        ens_records_iterator_next(&iterator);
+    }
+}
+
+/**
+ * Fill the bloom filter with flash records and test passed RPIs against it.
+ */
+int64_t test_bloom_performance(infected_for_period_key_ctx_t* ctx, int count) {
+    bloom_filter_t* bloom = bloom_init(get_num_records() * 2);
+    if (!bloom) {
+        printk("bloom init failed\n");
+        return -1;
+    }
+
+    // Measure bloom creation time
+    timing_t start_time, end_time;
+    uint64_t total_cycles;
+    uint64_t total_ns;
+    start_time = timing_counter_get();
+
+    fill_bloom_with_stored_records(bloom);
+
+    end_time = timing_counter_get();
+    total_cycles = timing_cycles_get(&start_time, &end_time);
+    total_ns = timing_cycles_to_ns(total_cycles);
+    printk("\nbloom init took %lld ms\n\n", total_ns / 1000000);
+
+    // test bloom performance
+    for (int i = 0; i < count; i++) {
+        if (bloom_probably_has_record(bloom, &ctx[i].interval_identifier)) {
+            ctx[i].infected++;
+        }
+    }
+
+    int amount = 0;
+    for (int i = 0; i < count; i++) {
+        amount += ctx[i].infected;
+    }
+    printk("amount of infected records: %d\n", amount);
+
+    int ret = get_number_of_infected_for_multiple_intervals_optimized(ctx, count);
+cleanup:
+    k_free(bloom);
+    return ret;
+}
+
+/**
+ * Fill bloom with passed RPIs and test flash records against it.
+ */
+int64_t test_bloom_reverse_performance(infected_for_period_key_ctx_t* ctx, int count) {
+    bloom_filter_t* bloom = bloom_init(count * 2);
+    if (!bloom) {
+        printk("bloom init failed\n");
+        return -1;
+    }
+
+    timing_t start_time, end_time;
+    uint64_t total_cycles;
+    uint64_t total_ns;
+
+    start_time = timing_counter_get();
+    for (int i = 0; i < count; i++) {
+        bloom_add_record(bloom, &ctx[i].interval_identifier);
+    }
+    end_time = timing_counter_get();
+
+    total_cycles = timing_cycles_get(&start_time, &end_time);
+    total_ns = timing_cycles_to_ns(total_cycles);
+
+    printk("\nbloom init took %lld ms\n\n", total_ns / 1000000);
+
+    int64_t amount = 0;
+
+    record_iterator_t iterator;
+    int rc = ens_records_iterator_init_timerange(&iterator, NULL, NULL);
+    if (rc) {
+        printk("init iterator failed (err %d)\n", rc);
+        amount = rc;
+        goto cleanup;
+    }
+
+    while (!iterator.finished) {
+        if (bloom_probably_has_record(bloom, &iterator.current.rolling_proximity_identifier)) {
+            for (int i = 0; i < count; i++) {
+                if (memcmp(&iterator.current.rolling_proximity_identifier, &ctx[i].interval_identifier,
+                           sizeof(iterator.current.rolling_proximity_identifier)) == 0) {
+                    amount++;
+                    break;
+                }
+                if (iterator.current.timestamp > ctx[i].search_end) {
+                    break;
+                }
+            }
+        }
+        ens_records_iterator_next(&iterator);
+    }
+
+cleanup:
+    k_free(bloom);
+    return amount;
+}
+
+////////////////////
+// FILL TEST DATA //
+////////////////////
+
+static ENPeriodKey infectedPeriodKey = {
+    .b = {0x75, 0xc7, 0x34, 0xc6, 0xdd, 0x1a, 0x78, 0x2d, 0xe7, 0xa9, 0x65, 0xda, 0x5e, 0xb9, 0x31, 0x25}};
+static ENPeriodKey dummyPeriodKey = {
+    .b = {0x89, 0xa7, 0x34, 0xc6, 0xdd, 0x1a, 0x14, 0xda, 0xe7, 0x00, 0x65, 0xda, 0x6a, 0x9b, 0x13, 0x52}};
+
+static ENPeriodIdentifierKey infectedPik;
+static ENPeriodIdentifierKey dummyPik;
+
+void fill_test_rki_data(infected_for_period_key_ctx_t* infectedIntervals, int count) {
+    int infectedCount = 50;
+    int spread = count / infectedCount;
+    for (int i = 0; i < infectedCount; i++) {
+        int intervalNumber = (i + 2) * 2;
+        float range = 1.5;
+        for (int j = 0; j < spread; j++) {
+            int offset = (EN_INTERVAL_LENGTH / spread) * j;
+            infectedIntervals[i * spread + j].infected = 0;
+            infectedIntervals[i * spread + j].search_start =
+                (intervalNumber - range) * EN_INTERVAL_LENGTH + offset;  // start one and a half intervals before
+            infectedIntervals[i * spread + j].search_end =
+                (intervalNumber + range) * EN_INTERVAL_LENGTH + offset;  // end one and a half intervals after
+            en_derive_interval_identifier(&infectedIntervals[i * spread + j].interval_identifier, &infectedPik,
+                                          intervalNumber);
+        }
+    }
+}
+
+////////////////////
+// MEASURING FUNC //
+////////////////////
+
 void measure_perf(test_func_t func, const char* label, infected_for_period_key_ctx_t* infectedIntervals, int count) {
     printk("---------------------------\n'%s': starting measurement\n", label);
 
+    fill_test_rki_data(infectedIntervals, count);
+
     timing_t start_time, end_time;
     uint64_t total_cycles;
     uint64_t total_ns;
@@ -144,34 +305,30 @@ void measure_perf(test_func_t func, const char* label, infected_for_period_key_c
 
     timing_stop();
 
-    printk("\n'%s' took %lld ns\n---------------------------\n", label, total_ns);
+    printk("\n'%s' took %lld ms\n---------------------------\n", label, total_ns / 1000000);
 }
 
 void setup_test_data() {
-    ENPeriodKey infectedPeriodKey = {
-        .b = {0x75, 0xc7, 0x34, 0xc6, 0xdd, 0x1a, 0x78, 0x2d, 0xe7, 0xa9, 0x65, 0xda, 0x5e, 0xb9, 0x31, 0x25}};
-    ENPeriodKey dummyPeriodKey = {
-        .b = {0x89, 0xa7, 0x34, 0xc6, 0xdd, 0x1a, 0x14, 0xda, 0xe7, 0x00, 0x65, 0xda, 0x6a, 0x9b, 0x13, 0x52}};
-
-    ENPeriodIdentifierKey infectedPik;
-    ENPeriodIdentifierKey dummyPik;
     en_derive_period_identifier_key(&infectedPik, &infectedPeriodKey);
     en_derive_period_identifier_key(&dummyPik, &dummyPeriodKey);
 
+// every 100th interval has an infected record
+#define INTERVAL_SPREAD 100
+
     for (int i = 0; i < EN_TEK_ROLLING_PERIOD; i++) {
         // create infected record
         record_t infectedRecord;
         infectedRecord.timestamp = i * EN_INTERVAL_LENGTH;
         en_derive_interval_identifier((ENIntervalIdentifier*)&infectedRecord.rolling_proximity_identifier, &infectedPik,
                                       i);
-        record_t dummyRecord;
-        en_derive_interval_identifier((ENIntervalIdentifier*)&dummyRecord.rolling_proximity_identifier, &dummyPik, i);
         int rc;
-        if ((rc = add_record(&infectedRecord))) {
+        if (i % INTERVAL_SPREAD == 0 && (rc = add_record(&infectedRecord))) {
             printk("err %d\n", rc);
         }
 
-        int spread = 4;
+        record_t dummyRecord;
+        en_derive_interval_identifier((ENIntervalIdentifier*)&dummyRecord.rolling_proximity_identifier, &dummyPik, i);
+        int spread = 1;
 
         for (int j = 0; j < EN_INTERVAL_LENGTH / spread; j++) {
             dummyRecord.timestamp = i * EN_INTERVAL_LENGTH + j * spread + 1;
@@ -182,29 +339,18 @@ void setup_test_data() {
         printk("period %d\n", i);
     }
 
-    int infectedCount = 36;
+#define INFECTED_INTERVALS_COUNT 500
-    int spread = 2;
-
     // setup our ordered array with infected RPIs
-    infected_for_period_key_ctx_t infectedIntervals[infectedCount * spread];
+    static infected_for_period_key_ctx_t infectedIntervals[INFECTED_INTERVALS_COUNT];
-    for (int i = 0; i < infectedCount; i++) {
-        int intervalNumber = (i + 2) * 2;
-        float range = 1.5;
-        for (int j = 0; j < spread; j++) {
-            int offset = (EN_INTERVAL_LENGTH / spread) * j;
-            infectedIntervals[i * spread + j].infected = 0;
-            infectedIntervals[i * spread + j].search_start =
-                (intervalNumber - range) * EN_INTERVAL_LENGTH + offset;  // start one and a half interval before
-            infectedIntervals[i * spread + j].search_end =
-                (intervalNumber + range) * EN_INTERVAL_LENGTH + offset;  // end one and a half interval after
-            en_derive_interval_identifier(&infectedIntervals[i * spread + j].interval_identifier, &infectedPik,
-                                          intervalNumber);
-        }
-    }
 
-    measure_perf(get_number_of_infected_for_multiple_intervals_dumb, "dumb", infectedIntervals, infectedCount * spread);
+    // measure_perf(get_number_of_infected_for_multiple_intervals_dumb, "dumb", infectedIntervals,
-    measure_perf(get_number_of_infected_for_multiple_intervals_simple, "simple", infectedIntervals,
+    //              INFECTED_INTERVALS_COUNT);
-                 infectedCount * spread);
+    // measure_perf(get_number_of_infected_for_multiple_intervals_simple, "simple", infectedIntervals,
-    measure_perf(get_number_of_infected_for_multiple_intervals_optimized, "optimized", infectedIntervals,
+    //              INFECTED_INTERVALS_COUNT);
-                 infectedCount * spread);
+    // measure_perf(get_number_of_infected_for_multiple_intervals_optimized, "optimized", infectedIntervals,
+    //              INFECTED_INTERVALS_COUNT);
+
+    measure_perf(test_bloom_performance, "bloom", infectedIntervals, INFECTED_INTERVALS_COUNT);
+
+    measure_perf(test_bloom_reverse_performance, "bloom reverse", infectedIntervals, INFECTED_INTERVALS_COUNT);
 }

src/main.c

@@ -50,12 +50,6 @@ void main(void) {
     setup_test_data();
 #endif
 
-    // err = bloom_create(1);
-    // if (err) {
-    //     printk("init bloom failed (err %d)\n", err);
-    //     return;
-    // }
-
     err = init_io();
     if (err) {
         printk("Button init failed (err %d)\n", err);

zephyr/prj.conf

@@ -33,7 +33,7 @@ CONFIG_DEBUG=y
 CONFIG_LOG=y
 
 CONFIG_NEWLIB_LIBC=y
-CONFIG_HEAP_MEM_POOL_SIZE=4096
+CONFIG_HEAP_MEM_POOL_SIZE=131072
 CONFIG_NORDIC_QSPI_NOR=y # configuration options for MX25R64 flash device
 CONFIG_NORDIC_QSPI_NOR_FLASH_LAYOUT_PAGE_SIZE=4096