diff --git a/CMakeLists.txt b/CMakeLists.txt index 9ccca08..dc1d4e9 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -14,3 +14,9 @@ target_include_directories(bloom_filter PUBLIC include) # https://stackoverflow.com/questions/70667513/cmake-cxx-standard-vs-target-compile-features # doesn't matter for this specific project though target_compile_features(bloom_filter PUBLIC cxx_std_20) + +add_executable(stats_basic_bloom_filter + example/stats_basic_bloom_filter.cpp +) + +target_link_libraries(stats_basic_bloom_filter PRIVATE bloom_filter) diff --git a/DEV.md b/DEV.md index d3db6be..3c26895 100644 --- a/DEV.md +++ b/DEV.md @@ -87,4 +87,43 @@ Now it's similar to xxhash where you pass in a void pointer and a length to it. ## Fixing clangd -Post the split, I needed to fix clangd as it did not +Post the split, I needed to fix clangd as it did not register variables across files as I did not +have a valid cmake. So added that and ran `cmake -S . -B build` and all worked fine. + +--- + +# The Math + +Better to quickly go over the math as well +n - no of keys added ( expected ), m -> filter size in bits, k -> no of hash functions + +Now there are m bits, and consider one insertion, and just one hash function +Proba that it's still 0 is (1-1/m) as 1/m is the chance that bit is occupied and hence 1. +Now there are n keys and k such hash functions. This takes you to + +(1-1/m) ^ nk + +and then P(some particular bit is set) is 1-(1-1/m)^nk + +For a test k, it's false positive if all of the k bits are set which gives you + +(1 - (1-1/m)^nk ) ^ k + +For large m, (1-1/x)^x ~= 1/e +so the inner part becomes 1 - e^(-kn/m) + +This last step is incorrect as the assuming some bitstates to be independent +P(bit A is 1 and bit B is 1) = P(bit A is 1) * P(bit B is 1) + +Now this looks correct except that it is not +https://www.math.umd.edu/~immortal/CMSC420/notes/bloomfilters.pdf + +But that's besides the point for now. + +## My error probab function? + +Based off of Guava, it uses the actual free bits at runtime to give a "live" estimate +bits filled fraction is P(any arbitrary bit is 1) +False positive = all of those bits end up being 1 for some hash = P^k + +This has the same independence issue in assumption though. diff --git a/example/stats_basic_bloom_filter.cpp b/example/stats_basic_bloom_filter.cpp new file mode 100644 index 0000000..1fe0c4d --- /dev/null +++ b/example/stats_basic_bloom_filter.cpp @@ -0,0 +1,68 @@ +#include "bloom_filter/bloom_filter.hpp" +#include +#include +#include +#include +#include +#include +#include + +int main(int argc, char *argv[]) { + if (argc != 4) { // REM: 0th ars it the program name + std::cerr << "need args n(number of keys inserted) m(filter size in " + "num bits) k(num hash funcs)"; + return 1; + } + + uint n = std::stoul(argv[1]); + uint m = std::stoul(argv[2]); + uint k = std::stoul(argv[3]); + + // std::random_device rd; you can uncomment and pass it in here to randomise + std::mt19937_64 rng{}; // no seed so this is deterministic + const uint TOTAL_TRIALS = 20; + + double sum_theorital_approx_fp_rate = 0; + double sum_filter_approx_fp_rate = 0; + double sum_actual_fp_rate = 0; + + for (int _t = 0; _t < TOTAL_TRIALS; _t++) { + uint64_t seed1 = rng(), seed2 = rng(); + + BloomFilter filter{m, k, seed1, seed2}; + + std::set actual_inserts; + for (int i = 0; i < n; i++) { + uint64_t num = rng(); + actual_inserts.insert(num); + filter.put(&num, sizeof(num)); + } + + const uint ELEMS_CHECK = 10000; + uint fp_count = 0; + for (int i = 0; i < ELEMS_CHECK; i++) { + uint64_t test = rng(); + bool actually_present = actual_inserts.count(test); + bool filter_contains = filter.may_contain(&test, sizeof(test)); + fp_count += !actually_present && filter_contains; + } + + double theoretical_approx_fp_rate = + std::pow(1 - std::exp(-static_cast(k) * n / m), k); + double filter_approx_fp_rate = filter.false_positive_probability(); + double actual_fp_rate = static_cast(fp_count) / ELEMS_CHECK; + + sum_theorital_approx_fp_rate += theoretical_approx_fp_rate; + sum_filter_approx_fp_rate += filter_approx_fp_rate; + sum_actual_fp_rate += actual_fp_rate; + } + + std::printf("simple bloom filter with n = %d, m = %d, k = %d\n", n, m, k); + std::printf("theoretical fp probs : %.8f\n", + sum_theorital_approx_fp_rate / TOTAL_TRIALS); + std::printf("filter reported fp probs : %.8f\n", + sum_filter_approx_fp_rate / TOTAL_TRIALS); + std::printf("actual fp rate : %.8f\n", sum_actual_fp_rate / TOTAL_TRIALS); + + return 0; +} diff --git a/main.cpp b/main.cpp deleted file mode 100644 index d7792d9..0000000 --- a/main.cpp +++ /dev/null @@ -1,69 +0,0 @@ -#include -#include -#include - -/* - * Bloom filters - * - n bit array - * - k hash functions - * - add_elem: - * - get the set of {h_i(elem) mod n} and mark them 1 - * - check elem: - * - x is elem - * - generate same positions set, if any are false this does not belong - * for sure - * - no guarantees on presence - */ - -const int N = 1024; -const int K = 5; -std::vector bf(N); - -int hash_i(int i, int x) { - switch (i) { - case 1: - return x; // h1(i) = i is perfectly ok, std::hash will do the same - case 2: - // here we need something better - // the | 1 makes it co-rime to N always - return std::hash{}(std::to_string(x)) | 1; - default: - return hash_i(1, x) + i * hash_i(2, x); - } -} - -void add(int elem){ - for(int i = 1;i<=K;i++){ - int hash = hash_i(i, elem) % N; - bf[hash] = 1; - } -} - -bool missing(int elem){ - for(int i = 1;i<=K;i++){ - int hash = hash_i(i, elem) % N; - if(!bf[hash]) return true; // actually missing - } - return false; // may be present -} - -int main() { - srand(time(nullptr)); - std::set actually_present{}; - const int LIM = 500; - for(int i = 1; i< LIM;i++){ - if(rand()&1){ - actually_present.insert(i); - add(i); - } - } - - int wrong = 0; - for(int i=1;i