diff --git a/.gitignore b/.gitignore index cba7efc..cfdd880 100644 --- a/.gitignore +++ b/.gitignore @@ -1 +1,14 @@ a.out +build/ +.cache/ + +# CMake in-source build artifacts +/CMakeCache.txt +/CMakeFiles/ +/Makefile +/cmake_install.cmake +/compile_commands.json + +# Compiler/linker outputs +*.o +*.a diff --git a/CMakeLists.txt b/CMakeLists.txt index e69de29..9ccca08 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -0,0 +1,16 @@ +cmake_minimum_required(VERSION 3.16) + +project(bloom_filter_cpp LANGUAGES CXX) + +set(CMAKE_EXPORT_COMPILE_COMMANDS ON) + +add_library(bloom_filter + src/bloom_filter.cpp +) + +target_include_directories(bloom_filter PUBLIC include) + +# I used to set this as set(CMAKE_CXX_STANDARD 20) but this is better +# https://stackoverflow.com/questions/70667513/cmake-cxx-standard-vs-target-compile-features +# doesn't matter for this specific project though +target_compile_features(bloom_filter PUBLIC cxx_std_20) diff --git a/DEV.md b/DEV.md index a15d9d8..d3db6be 100644 --- a/DEV.md +++ b/DEV.md @@ -22,3 +22,69 @@ This double hashed func almost behaves like a random number generator. ## std::hash you make a hasher object and them immediately call it + +## Designing the api + +Now that I have a basic version ready I need to design some sort +of an api going forward - for the header. For now I'm sticking +to the traditional impl. + +There was a google's impl +https://github.com/google/guava/blob/master/guava/src/com/google/common/hash/BloomFilter.java + +and also another cpp lib that's around 10y old +https://www.partow.net/programming/bloomfilter/index.html + +Ok going step by step, first the hash function + +### Hash func + +The guava one uses MurmurHash3 ( first I've heard of it ) +There are two sorts of hashes - cryptographic and non crypto + +The diff is explained here: https://security.stackexchange.com/questions/11839/what-is-the-difference-between-a-hash-function-and-a-cryptographic-hash-function + +The point being - cryptographic ones ensure security against some +adversary and guarantee some stuff like +1. preimage resistance - given hash find message +2. second preimage resitance, given just x and h(x), find another x' +with same hash +3. collision resistance - free to choose any pair but need to match +hashes + +When you loosen these constraints - faster hashes are possible. + +In any case - I googled and seems like xxhash is the fastest now +so I'll use that. + +From there I downloaded the simplest impl +https://create.stephan-brumme.com/xxhash/ the v2 version for 64. +It's a single header so that should be good enough. + +### Back to actual api? + +I liked Google's api for it - especially the naming and all. +I won't b supporting any set operations at all. + +mightContain(x) +expectedFalsePositiveProbab +put +approxElemCount() - estimates from fraction of set bits + +Constructors - there should be these atleast +(expectedPuts, numHashes, falsePositiveityRate) +(bitSize, numHashes) + +### The headers + +using vector as it's bit packed internally and allows for +runtime size declaration + +I ended up writing the entire impl in the header file and then split it. +Earlier I was going for a templated thing but that needed a lot of additional handling +due to how something with internal pointers would manage data ( say std::string ). +Now it's similar to xxhash where you pass in a void pointer and a length to it. + +## Fixing clangd + +Post the split, I needed to fix clangd as it did not diff --git a/include/bloom_filter/bloom_filter.hpp b/include/bloom_filter/bloom_filter.hpp index e69de29..7b50d3b 100644 --- a/include/bloom_filter/bloom_filter.hpp +++ b/include/bloom_filter/bloom_filter.hpp @@ -0,0 +1,33 @@ +#include +#include +#include + +class BloomFilter { + private: + size_t filter_size_; + std::vector filter_; + uint32_t num_hashes_; + uint64_t seed1_, seed2_; + + /// This uses double hased based probing for k points + /// similar to open addressing hash tables + std::vector probes_(const void *start, uint64_t length); + + public: + BloomFilter(size_t filter_size, uint32_t num_hashes, uint64_t seed1, + uint64_t seed2); + + /// Insert data as a chunk of bytes + /// @param start pointer to continuous block of data + /// @param length number of bytes + void put(const void *start, uint64_t length); + + /// Check if a chunk of bytes was added + /// false is guaranteed false + /// trues are with a probaility + bool may_contain(const void *start, uint64_t length); + + /// Probability of a may contain true result + /// being wrong + double false_positive_probability() const; +}; diff --git a/include/bloom_filter/xxhash64.h b/include/bloom_filter/xxhash64.h index 4d0bbc5..46408a6 100644 --- a/include/bloom_filter/xxhash64.h +++ b/include/bloom_filter/xxhash64.h @@ -7,12 +7,14 @@ #pragma once #include // for uint32_t and uint64_t -/// XXHash (64 bit), based on Yann Collet's descriptions, see http://cyan4973.github.io/xxHash/ +/// XXHash (64 bit), based on Yann Collet's descriptions, see +/// http://cyan4973.github.io/xxHash/ /** How to use: uint64_t myseed = 0; XXHash64 myhash(myseed); myhash.add(pointerToSomeBytes, numberOfBytes); - myhash.add(pointerToSomeMoreBytes, numberOfMoreBytes); // call add() as often as you like to ... + myhash.add(pointerToSomeMoreBytes, numberOfMoreBytes); // call add() as +often as you like to ... // and compute hash: uint64_t result = myhash.hash(); @@ -21,182 +23,177 @@ Note: my code is NOT endian-aware ! **/ -class XXHash64 -{ -public: - /// create new XXHash (64 bit) - /** @param seed your seed value, even zero is a valid seed **/ - explicit XXHash64(uint64_t seed) - { - state[0] = seed + Prime1 + Prime2; - state[1] = seed + Prime2; - state[2] = seed; - state[3] = seed - Prime1; - bufferSize = 0; - totalLength = 0; - } - - /// add a chunk of bytes - /** @param input pointer to a continuous block of data - @param length number of bytes - @return false if parameters are invalid / zero **/ - bool add(const void* input, uint64_t length) - { - // no data ? - if (!input || length == 0) - return false; - - totalLength += length; - // byte-wise access - const unsigned char* data = (const unsigned char*)input; - - // unprocessed old data plus new data still fit in temporary buffer ? - if (bufferSize + length < MaxBufferSize) - { - // just add new data - while (length-- > 0) - buffer[bufferSize++] = *data++; - return true; +class XXHash64 { + public: + /// create new XXHash (64 bit) + /** @param seed your seed value, even zero is a valid seed **/ + explicit XXHash64(uint64_t seed) { + state[0] = seed + Prime1 + Prime2; + state[1] = seed + Prime2; + state[2] = seed; + state[3] = seed - Prime1; + bufferSize = 0; + totalLength = 0; } - // point beyond last byte - const unsigned char* stop = data + length; - const unsigned char* stopBlock = stop - MaxBufferSize; + /// add a chunk of bytes + /** @param input pointer to a continuous block of data + @param length number of bytes + @return false if parameters are invalid / zero **/ + bool add(const void *input, uint64_t length) { + // no data ? + if (!input || length == 0) + return false; - // some data left from previous update ? - if (bufferSize > 0) - { - // make sure temporary buffer is full (16 bytes) - while (bufferSize < MaxBufferSize) - buffer[bufferSize++] = *data++; + totalLength += length; + // byte-wise access + const unsigned char *data = (const unsigned char *)input; - // process these 32 bytes (4x8) - process(buffer, state[0], state[1], state[2], state[3]); + // unprocessed old data plus new data still fit in temporary buffer ? + if (bufferSize + length < MaxBufferSize) { + // just add new data + while (length-- > 0) + buffer[bufferSize++] = *data++; + return true; + } + + // point beyond last byte + const unsigned char *stop = data + length; + const unsigned char *stopBlock = stop - MaxBufferSize; + + // some data left from previous update ? + if (bufferSize > 0) { + // make sure temporary buffer is full (16 bytes) + while (bufferSize < MaxBufferSize) + buffer[bufferSize++] = *data++; + + // process these 32 bytes (4x8) + process(buffer, state[0], state[1], state[2], state[3]); + } + + // copying state to local variables helps optimizer A LOT + uint64_t s0 = state[0], s1 = state[1], s2 = state[2], s3 = state[3]; + // 32 bytes at once + while (data <= stopBlock) { + // local variables s0..s3 instead of state[0]..state[3] are much + // faster + process(data, s0, s1, s2, s3); + data += 32; + } + // copy back + state[0] = s0; + state[1] = s1; + state[2] = s2; + state[3] = s3; + + // copy remainder to temporary buffer + bufferSize = stop - data; + for (uint64_t i = 0; i < bufferSize; i++) + buffer[i] = data[i]; + + // done + return true; } - // copying state to local variables helps optimizer A LOT - uint64_t s0 = state[0], s1 = state[1], s2 = state[2], s3 = state[3]; - // 32 bytes at once - while (data <= stopBlock) - { - // local variables s0..s3 instead of state[0]..state[3] are much faster - process(data, s0, s1, s2, s3); - data += 32; - } - // copy back - state[0] = s0; state[1] = s1; state[2] = s2; state[3] = s3; + /// get current hash + /** @return 64 bit XXHash **/ + uint64_t hash() const { + // fold 256 bit state into one single 64 bit value + uint64_t result; + if (totalLength >= MaxBufferSize) { + result = rotateLeft(state[0], 1) + rotateLeft(state[1], 7) + + rotateLeft(state[2], 12) + rotateLeft(state[3], 18); + result = (result ^ processSingle(0, state[0])) * Prime1 + Prime4; + result = (result ^ processSingle(0, state[1])) * Prime1 + Prime4; + result = (result ^ processSingle(0, state[2])) * Prime1 + Prime4; + result = (result ^ processSingle(0, state[3])) * Prime1 + Prime4; + } else { + // internal state wasn't set in add(), therefore original seed is + // still stored in state2 + result = state[2] + Prime5; + } - // copy remainder to temporary buffer - bufferSize = stop - data; - for (uint64_t i = 0; i < bufferSize; i++) - buffer[i] = data[i]; + result += totalLength; - // done - return true; - } + // process remaining bytes in temporary buffer + const unsigned char *data = buffer; + // point beyond last byte + const unsigned char *stop = data + bufferSize; - /// get current hash - /** @return 64 bit XXHash **/ - uint64_t hash() const - { - // fold 256 bit state into one single 64 bit value - uint64_t result; - if (totalLength >= MaxBufferSize) - { - result = rotateLeft(state[0], 1) + - rotateLeft(state[1], 7) + - rotateLeft(state[2], 12) + - rotateLeft(state[3], 18); - result = (result ^ processSingle(0, state[0])) * Prime1 + Prime4; - result = (result ^ processSingle(0, state[1])) * Prime1 + Prime4; - result = (result ^ processSingle(0, state[2])) * Prime1 + Prime4; - result = (result ^ processSingle(0, state[3])) * Prime1 + Prime4; - } - else - { - // internal state wasn't set in add(), therefore original seed is still stored in state2 - result = state[2] + Prime5; + // at least 8 bytes left ? => eat 8 bytes per step + for (; data + 8 <= stop; data += 8) + result = + rotateLeft(result ^ processSingle(0, *(uint64_t *)data), 27) * + Prime1 + + Prime4; + + // 4 bytes left ? => eat those + if (data + 4 <= stop) { + result = + rotateLeft(result ^ (*(uint32_t *)data) * Prime1, 23) * Prime2 + + Prime3; + data += 4; + } + + // take care of remaining 0..3 bytes, eat 1 byte per step + while (data != stop) + result = rotateLeft(result ^ (*data++) * Prime5, 11) * Prime1; + + // mix bits + result ^= result >> 33; + result *= Prime2; + result ^= result >> 29; + result *= Prime3; + result ^= result >> 32; + return result; } - result += totalLength; - - // process remaining bytes in temporary buffer - const unsigned char* data = buffer; - // point beyond last byte - const unsigned char* stop = data + bufferSize; - - // at least 8 bytes left ? => eat 8 bytes per step - for (; data + 8 <= stop; data += 8) - result = rotateLeft(result ^ processSingle(0, *(uint64_t*)data), 27) * Prime1 + Prime4; - - // 4 bytes left ? => eat those - if (data + 4 <= stop) - { - result = rotateLeft(result ^ (*(uint32_t*)data) * Prime1, 23) * Prime2 + Prime3; - data += 4; + /// combine constructor, add() and hash() in one static function (C style) + /** @param input pointer to a continuous block of data + @param length number of bytes + @param seed your seed value, e.g. zero is a valid seed + @return 64 bit XXHash **/ + static uint64_t hash(const void *input, uint64_t length, uint64_t seed) { + XXHash64 hasher(seed); + hasher.add(input, length); + return hasher.hash(); } - // take care of remaining 0..3 bytes, eat 1 byte per step - while (data != stop) - result = rotateLeft(result ^ (*data++) * Prime5, 11) * Prime1; + private: + /// magic constants :-) + static const uint64_t Prime1 = 11400714785074694791ULL; + static const uint64_t Prime2 = 14029467366897019727ULL; + static const uint64_t Prime3 = 1609587929392839161ULL; + static const uint64_t Prime4 = 9650029242287828579ULL; + static const uint64_t Prime5 = 2870177450012600261ULL; - // mix bits - result ^= result >> 33; - result *= Prime2; - result ^= result >> 29; - result *= Prime3; - result ^= result >> 32; - return result; - } + /// temporarily store up to 31 bytes between multiple add() calls + static const uint64_t MaxBufferSize = 31 + 1; + uint64_t state[4]; + unsigned char buffer[MaxBufferSize]; + uint64_t bufferSize; + uint64_t totalLength; - /// combine constructor, add() and hash() in one static function (C style) - /** @param input pointer to a continuous block of data - @param length number of bytes - @param seed your seed value, e.g. zero is a valid seed - @return 64 bit XXHash **/ - static uint64_t hash(const void* input, uint64_t length, uint64_t seed) - { - XXHash64 hasher(seed); - hasher.add(input, length); - return hasher.hash(); - } + /// rotate bits, should compile to a single CPU instruction (ROL) + static inline uint64_t rotateLeft(uint64_t x, unsigned char bits) { + return (x << bits) | (x >> (64 - bits)); + } -private: - /// magic constants :-) - static const uint64_t Prime1 = 11400714785074694791ULL; - static const uint64_t Prime2 = 14029467366897019727ULL; - static const uint64_t Prime3 = 1609587929392839161ULL; - static const uint64_t Prime4 = 9650029242287828579ULL; - static const uint64_t Prime5 = 2870177450012600261ULL; + /// process a single 64 bit value + static inline uint64_t processSingle(uint64_t previous, uint64_t input) { + return rotateLeft(previous + input * Prime2, 31) * Prime1; + } - /// temporarily store up to 31 bytes between multiple add() calls - static const uint64_t MaxBufferSize = 31+1; - - uint64_t state[4]; - unsigned char buffer[MaxBufferSize]; - uint64_t bufferSize; - uint64_t totalLength; - - /// rotate bits, should compile to a single CPU instruction (ROL) - static inline uint64_t rotateLeft(uint64_t x, unsigned char bits) - { - return (x << bits) | (x >> (64 - bits)); - } - - /// process a single 64 bit value - static inline uint64_t processSingle(uint64_t previous, uint64_t input) - { - return rotateLeft(previous + input * Prime2, 31) * Prime1; - } - - /// process a block of 4x4 bytes, this is the main part of the XXHash32 algorithm - static inline void process(const void* data, uint64_t& state0, uint64_t& state1, uint64_t& state2, uint64_t& state3) - { - const uint64_t* block = (const uint64_t*) data; - state0 = processSingle(state0, block[0]); - state1 = processSingle(state1, block[1]); - state2 = processSingle(state2, block[2]); - state3 = processSingle(state3, block[3]); - } + /// process a block of 4x4 bytes, this is the main part of the XXHash32 + /// algorithm + static inline void process(const void *data, uint64_t &state0, + uint64_t &state1, uint64_t &state2, + uint64_t &state3) { + const uint64_t *block = (const uint64_t *)data; + state0 = processSingle(state0, block[0]); + state1 = processSingle(state1, block[1]); + state2 = processSingle(state2, block[2]); + state3 = processSingle(state3, block[3]); + } }; diff --git a/src/bloom_filter.cpp b/src/bloom_filter.cpp index e69de29..827727f 100644 --- a/src/bloom_filter.cpp +++ b/src/bloom_filter.cpp @@ -0,0 +1,56 @@ +#include "bloom_filter/bloom_filter.hpp" +#include "bloom_filter/xxhash64.h" + +#include +#include +#include + +std::vector BloomFilter::probes_(const void *start, uint64_t length) { + std::vector hashes(num_hashes_); + + // this is fine as ( a mod m + i ( b mod m ) ) mod m + // is same as (a + i b ) mod m + // WARN: hash2 mod filter_size can be 0, then all the probes + // collapse to the same value + + hashes[0] = XXHash64::hash(start, length, seed1_) % filter_size_; + hashes[1] = XXHash64::hash(start, length, seed2_) % filter_size_; + + for (uint64_t i = 2; i < num_hashes_; i++) { + // these overflows will wrap around and are totally fine + hashes[i] = (hashes[0] + i * hashes[1]) % filter_size_; + } + + return hashes; +} + +BloomFilter::BloomFilter(size_t filter_size, uint32_t num_hashes, + uint64_t seed1, uint64_t seed2) + : filter_size_(filter_size), filter_(filter_size_, 0), + num_hashes_(num_hashes), seed1_(seed1), seed2_(seed2) { + // this can be disables in prod builds + assert(num_hashes >= 2); +} + +void BloomFilter::put(const void *start, uint64_t length) { + auto probes = probes_(start, length); + for (auto pos : probes) { + filter_[pos] = 1; + } +} + +bool BloomFilter::may_contain(const void *start, uint64_t length) { + auto probes = probes_(start, length); + for (auto pos : probes) { + if (!filter_[pos]) + return false; + } + return true; +} + +double BloomFilter::false_positive_probability() const { + // p = fraction of bits set ^ num_hashes + double set_bits = std::accumulate(filter_.begin(), filter_.end(), 0.); + const double fraction_set = set_bits / static_cast(filter_size_); + return std::pow(fraction_set, num_hashes_); +}