covid19/CovidBracelet
covid19
/
CovidBracelet
mirror of https://github.com/CovidBraceletPrj/CovidBracelet.git
1
0
Fork 0
Code Releases Activity

Add WIP bloom stage (rewrite git history)

This commit is contained in:
H1ghBre4k3r 2021-09-21 21:41:40 +02:00 committed by Patrick Rathje
parent a720bb7d9e
commit 3a2dc985f4
5 changed files with 210 additions and 77 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

40
src/bloom.c
View File

@ -1,47 +1,45 @@
#include "bloom.h"
#include "ens/storage.h"
#include "exposure-notification.h"
bloom_filter_t* bloom_create(size_t size) {
bloom_filter_t* bloom_init(size_t size) {
bloom_filter_t* bloom = k_calloc(1, sizeof(bloom_filter_t));
if (!bloom) {
return NULL;
}
bloom->size = size;
bloom->data = k_malloc(size * sizeof(uint8_t));
bloom->data = k_calloc(size, sizeof(uint8_t));
if (!bloom->data) {
k_free(bloom);
return NULL;
}
return bloom;
}
void bloom_delete(bloom_filter_t* bloom) {
void bloom_destroy(bloom_filter_t* bloom) {
if (bloom) {
while (bloom->func) {
bloom_hash_t* h;
h = bloom->func;
bloom->func = h->next;
k_free(h);
}
k_free(bloom->data);
k_free(bloom);
}
}
void bloom_add_record(bloom_filter_t* bloom, record_t* record) {
bloom_hash_t* h = bloom->func;
void bloom_add_record(bloom_filter_t* bloom, ENIntervalIdentifier* rpi) {
uint8_t* data = bloom->data;
while (h) {
uint32_t hash = h->func(record);
for (int i = 0; i < sizeof(*rpi); i += 2) {
uint32_t hash = rpi->b[i] << 8 & rpi->b[i + 1];
hash %= bloom->size * 8;
data[hash / 8] |= 1 << (hash % 8);
h = h->next;
}
}
bool bloom_probably_has_record(bloom_filter_t* bloom, record_t* record) {
bloom_hash_t* h = bloom->func;
bool bloom_probably_has_record(bloom_filter_t* bloom, ENIntervalIdentifier* rpi) {
uint8_t* data = bloom->data;
while (h) {
uint32_t hash = h->func(record);
for (int i = 0; i < sizeof(*rpi); i += 2) {
uint32_t hash = rpi->b[i] << 8 & rpi->b[i + 1];
hash %= bloom->size * 8;
if ((data[hash / 8] & (1 << (hash % 8))) == 0) {
if (!(data[hash / 8] & (1 << (hash % 8)))) {
return false;
}
h = h->next;
}
return true;
}
}

19
src/bloom.h
View File

@ -2,16 +2,9 @@
#define BLOOM_H
#include "ens/storage.h"
typedef uint32_t (*hash_function)(const void* data);
typedef struct bloom_hash {
hash_function func;
struct bloom_hash* next;
} bloom_hash_t;
#include "exposure-notification.h"
typedef struct bloom_filter {
bloom_hash_t* func;
uint8_t* data;
size_t size;
} bloom_filter_t;
@ -19,12 +12,14 @@ typedef struct bloom_filter {
/**
* Initialize the bloom filter on basis of the already registerred records.
*/
bloom_filter_t* bloom_create(size_t size);
bloom_filter_t* bloom_init(size_t size);
void bloom_destroy(bloom_filter_t* bloom);
// TODO lome: maybe only use RPI (should be sufficient)
void bloom_add_record(bloom_filter_t* bloom, record_t* record);
void bloom_add_record(bloom_filter_t* bloom, ENIntervalIdentifier* rpi);
// TODO lome: maybe only use RPI (should be sufficient)
bool bloom_probably_has_record(bloom_filter_t* bloom, record_t* record);
bool bloom_probably_has_record(bloom_filter_t* bloom, ENIntervalIdentifier* rpi);
#endif
#endif

220
src/contacts.c
View File

@ -54,6 +54,9 @@ int register_record(record_t* record) {
return rc;
}
/**
* Dumb implementation, where a single iterator is created for iterating over the entire flash.
*/
int get_number_of_infected_for_multiple_intervals_dumb(infected_for_period_key_ctx_t* ctx, int count) {
record_iterator_t iterator;
int rc = ens_records_iterator_init_timerange(&iterator, NULL, NULL);
@ -73,8 +76,10 @@ int get_number_of_infected_for_multiple_intervals_dumb(infected_for_period_key_c
return 0;
}
/**
* Simple implementation, where an iterator is created for every element in the passed arrray.
*/
int get_number_of_infected_for_multiple_intervals_simple(infected_for_period_key_ctx_t* ctx, int count) {
printk("start of simple\n");
record_iterator_t iterator;
for (int i = 0; i < count; i++) {
int rc = ens_records_iterator_init_timerange(&iterator, &ctx[i].search_start, &ctx[i].search_end);
@ -93,8 +98,10 @@ int get_number_of_infected_for_multiple_intervals_simple(infected_for_period_key
return 0;
}
/**
* Optimized implementation, where overlapping search intervals for consecutive RPI are merged.
*/
int get_number_of_infected_for_multiple_intervals_optimized(infected_for_period_key_ctx_t* ctx, int count) {
printk("start of opt\n");
record_iterator_t iterator;
int i = 0;
while (i < count) {
@ -124,9 +131,163 @@ int get_number_of_infected_for_multiple_intervals_optimized(infected_for_period_
return 0;
}
/**
* Fill the bloom filter with all stored records.
*/
void fill_bloom_with_stored_records(bloom_filter_t* bloom) {
// init iterator for filling bloom filter
record_iterator_t iterator;
int rc = ens_records_iterator_init_timerange(&iterator, NULL, NULL);
if (rc) {
printk("init iterator failed0 (err %d)\n", rc);
return;
}
// fill bloom filter with records
while (!iterator.finished) {
bloom_add_record(bloom, &iterator.current);
ens_records_iterator_next(&iterator);
}
}
/**
* Fill the bloom filter with flash records and test passed RPIs against it.
*/
int64_t test_bloom_performance(infected_for_period_key_ctx_t* ctx, int count) {
bloom_filter_t* bloom = bloom_init(get_num_records() * 2);
if (!bloom) {
printk("bloom init failed\n");
return -1;
}
// Measure bloom creation time
timing_t start_time, end_time;
uint64_t total_cycles;
uint64_t total_ns;
start_time = timing_counter_get();
fill_bloom_with_stored_records(bloom);
end_time = timing_counter_get();
total_cycles = timing_cycles_get(&start_time, &end_time);
total_ns = timing_cycles_to_ns(total_cycles);
printk("\nbloom init took %lld ms\n\n", total_ns / 1000000);
// test bloom performance
for (int i = 0; i < count; i++) {
if (bloom_probably_has_record(bloom, &ctx[i].interval_identifier)) {
ctx[i].infected++;
}
}
int amount = 0;
for (int i = 0; i < count; i++) {
amount += ctx[i].infected;
}
printk("amount of infected records: %d\n", amount);
int ret = get_number_of_infected_for_multiple_intervals_optimized(ctx, count);
cleanup:
k_free(bloom);
return ret;
}
/**
* Fill bloom with passed RPIs and test flash records against it.
*/
int64_t test_bloom_reverse_performance(infected_for_period_key_ctx_t* ctx, int count) {
bloom_filter_t* bloom = bloom_init(count * 2);
if (!bloom) {
printk("bloom init failed\n");
return -1;
}
timing_t start_time, end_time;
uint64_t total_cycles;
uint64_t total_ns;
start_time = timing_counter_get();
for (int i = 0; i < count; i++) {
bloom_add_record(bloom, &ctx[i].interval_identifier);
}
end_time = timing_counter_get();
total_cycles = timing_cycles_get(&start_time, &end_time);
total_ns = timing_cycles_to_ns(total_cycles);
printk("\nbloom init took %lld ms\n\n", total_ns / 1000000);
int64_t amount = 0;
record_iterator_t iterator;
int rc = ens_records_iterator_init_timerange(&iterator, NULL, NULL);
if (rc) {
printk("init iterator failed (err %d)\n", rc);
amount = rc;
goto cleanup;
}
while (!iterator.finished) {
if (bloom_probably_has_record(bloom, &iterator.current.rolling_proximity_identifier)) {
for (int i = 0; i < count; i++) {
if (memcmp(&iterator.current.rolling_proximity_identifier, &ctx[i].interval_identifier,
sizeof(iterator.current.rolling_proximity_identifier)) == 0) {
amount++;
break;
}
if (iterator.current.timestamp > ctx[i].search_end) {
break;
}
}
}
ens_records_iterator_next(&iterator);
}
cleanup:
k_free(bloom);
return amount;
}
////////////////////
// FILL TEST DATA //
////////////////////
static ENPeriodKey infectedPeriodKey = {
.b = {0x75, 0xc7, 0x34, 0xc6, 0xdd, 0x1a, 0x78, 0x2d, 0xe7, 0xa9, 0x65, 0xda, 0x5e, 0xb9, 0x31, 0x25}};
static ENPeriodKey dummyPeriodKey = {
.b = {0x89, 0xa7, 0x34, 0xc6, 0xdd, 0x1a, 0x14, 0xda, 0xe7, 0x00, 0x65, 0xda, 0x6a, 0x9b, 0x13, 0x52}};
static ENPeriodIdentifierKey infectedPik;
static ENPeriodIdentifierKey dummyPik;
void fill_test_rki_data(infected_for_period_key_ctx_t* infectedIntervals, int count) {
int infectedCount = 50;
int spread = count / infectedCount;
for (int i = 0; i < infectedCount; i++) {
int intervalNumber = (i + 2) * 2;
float range = 1.5;
for (int j = 0; j < spread; j++) {
int offset = (EN_INTERVAL_LENGTH / spread) * j;
infectedIntervals[i * spread + j].infected = 0;
infectedIntervals[i * spread + j].search_start =
(intervalNumber - range) * EN_INTERVAL_LENGTH + offset; // start one and a half intervals before
infectedIntervals[i * spread + j].search_end =
(intervalNumber + range) * EN_INTERVAL_LENGTH + offset; // end one and a half intervals after
en_derive_interval_identifier(&infectedIntervals[i * spread + j].interval_identifier, &infectedPik,
intervalNumber);
}
}
}
////////////////////
// MEASURING FUNC //
////////////////////
void measure_perf(test_func_t func, const char* label, infected_for_period_key_ctx_t* infectedIntervals, int count) {
printk("---------------------------\n'%s': starting measurement\n", label);
fill_test_rki_data(infectedIntervals, count);
timing_t start_time, end_time;
uint64_t total_cycles;
uint64_t total_ns;
@ -144,34 +305,30 @@ void measure_perf(test_func_t func, const char* label, infected_for_period_key_c
timing_stop();
printk("\n'%s' took %lld ns\n---------------------------\n", label, total_ns);
printk("\n'%s' took %lld ms\n---------------------------\n", label, total_ns / 1000000);
}
void setup_test_data() {
ENPeriodKey infectedPeriodKey = {
.b = {0x75, 0xc7, 0x34, 0xc6, 0xdd, 0x1a, 0x78, 0x2d, 0xe7, 0xa9, 0x65, 0xda, 0x5e, 0xb9, 0x31, 0x25}};
ENPeriodKey dummyPeriodKey = {
.b = {0x89, 0xa7, 0x34, 0xc6, 0xdd, 0x1a, 0x14, 0xda, 0xe7, 0x00, 0x65, 0xda, 0x6a, 0x9b, 0x13, 0x52}};
ENPeriodIdentifierKey infectedPik;
ENPeriodIdentifierKey dummyPik;
en_derive_period_identifier_key(&infectedPik, &infectedPeriodKey);
en_derive_period_identifier_key(&dummyPik, &dummyPeriodKey);
// every 100th interval has an infected record
#define INTERVAL_SPREAD 100
for (int i = 0; i < EN_TEK_ROLLING_PERIOD; i++) {
// create infected record
record_t infectedRecord;
infectedRecord.timestamp = i * EN_INTERVAL_LENGTH;
en_derive_interval_identifier((ENIntervalIdentifier*)&infectedRecord.rolling_proximity_identifier, &infectedPik,
i);
record_t dummyRecord;
en_derive_interval_identifier((ENIntervalIdentifier*)&dummyRecord.rolling_proximity_identifier, &dummyPik, i);
int rc;
if ((rc = add_record(&infectedRecord))) {
if (i % INTERVAL_SPREAD == 0 && (rc = add_record(&infectedRecord))) {
printk("err %d\n", rc);
}
int spread = 4;
record_t dummyRecord;
en_derive_interval_identifier((ENIntervalIdentifier*)&dummyRecord.rolling_proximity_identifier, &dummyPik, i);
int spread = 1;
for (int j = 0; j < EN_INTERVAL_LENGTH / spread; j++) {
dummyRecord.timestamp = i * EN_INTERVAL_LENGTH + j * spread + 1;
@ -182,29 +339,18 @@ void setup_test_data() {
printk("period %d\n", i);
}
int infectedCount = 36;
int spread = 2;
#define INFECTED_INTERVALS_COUNT 500
// setup our ordered array with infected RPIs
infected_for_period_key_ctx_t infectedIntervals[infectedCount * spread];
for (int i = 0; i < infectedCount; i++) {
int intervalNumber = (i + 2) * 2;
float range = 1.5;
for (int j = 0; j < spread; j++) {
int offset = (EN_INTERVAL_LENGTH / spread) * j;
infectedIntervals[i * spread + j].infected = 0;
infectedIntervals[i * spread + j].search_start =
(intervalNumber - range) * EN_INTERVAL_LENGTH + offset; // start one and a half interval before
infectedIntervals[i * spread + j].search_end =
(intervalNumber + range) * EN_INTERVAL_LENGTH + offset; // end one and a half interval after
en_derive_interval_identifier(&infectedIntervals[i * spread + j].interval_identifier, &infectedPik,
intervalNumber);
}
}
static infected_for_period_key_ctx_t infectedIntervals[INFECTED_INTERVALS_COUNT];
measure_perf(get_number_of_infected_for_multiple_intervals_dumb, "dumb", infectedIntervals, infectedCount * spread);
measure_perf(get_number_of_infected_for_multiple_intervals_simple, "simple", infectedIntervals,
infectedCount * spread);
measure_perf(get_number_of_infected_for_multiple_intervals_optimized, "optimized", infectedIntervals,
infectedCount * spread);
// measure_perf(get_number_of_infected_for_multiple_intervals_dumb, "dumb", infectedIntervals,
// INFECTED_INTERVALS_COUNT);
// measure_perf(get_number_of_infected_for_multiple_intervals_simple, "simple", infectedIntervals,
// INFECTED_INTERVALS_COUNT);
// measure_perf(get_number_of_infected_for_multiple_intervals_optimized, "optimized", infectedIntervals,
// INFECTED_INTERVALS_COUNT);
measure_perf(test_bloom_performance, "bloom", infectedIntervals, INFECTED_INTERVALS_COUNT);
measure_perf(test_bloom_reverse_performance, "bloom reverse", infectedIntervals, INFECTED_INTERVALS_COUNT);
}

6
src/main.c
View File

@ -50,12 +50,6 @@ void main(void) {
setup_test_data();
#endif
// err = bloom_create(1);
// if (err) {
// printk("init bloom failed (err %d)\n", err);
// return;
// }
err = init_io();
if (err) {
printk("Button init failed (err %d)\n", err);

2
zephyr/prj.conf
View File

@ -33,7 +33,7 @@ CONFIG_DEBUG=y
CONFIG_LOG=y
CONFIG_NEWLIB_LIBC=y
CONFIG_HEAP_MEM_POOL_SIZE=4096
CONFIG_HEAP_MEM_POOL_SIZE=131072
CONFIG_NORDIC_QSPI_NOR=y # configuration options for MX25R64 flash device
CONFIG_NORDIC_QSPI_NOR_FLASH_LAYOUT_PAGE_SIZE=4096