Algorithm Search⌘K

Bloomier filter in Python

Approximate function table that returns stored values for keys in a set.

File
bloomier_filter.py, 100 lines
Level
I0 not checked yet
Toolchain
Python 3.14, pytest 9
Test framework
pytest
Run the tests
python3 -m pytest
bloomier_filter.pyRawGitHub
# SPDX-License-Identifier: MIT
# Part of https://algorithm.tamnd.com/bloomier-filter/. The authors are listed in prov/8T9R6F.jsonl.

"""Bloomier filter (Chazelle, Kilian, Rubinfeld and Tal 2004) with three hash functions."""

import hashlib
from collections import deque
from collections.abc import Mapping

Key = str | bytes | int


class BloomierFilter:
    """A static function table: lookup(x) gives f(x) for x in S, and usually None otherwise.

    T has M cells of q bits that encode, for each key, which of its three cells
    holds its value; R has M cells of r bits that hold the values. A key not in
    S gets a value with probability 3 / 2^q. Values of keys in S can be changed
    with set(); the set S itself is fixed.
    """

    def __init__(self, f: Mapping[Key, int], r: int, q: int = 8, seed: int = 0, seg: int | None = None) -> None:
        if r < 1 or q < 2:
            raise ValueError("r must be positive and q at least 2")
        if any(not 0 <= v < 1 << r for v in f.values()):
            raise ValueError("values must fit in r bits")
        self.r, self.q = r, q
        keys = list(f)
        self.seg = seg if seg else (123 * len(keys) + 299) // 300 + 1  # cells per third
        self.seed, tries = seed, 0
        while True:  # @line L1: choose hash functions until peeling succeeds
            order = self._peel(keys)  # @line L2
            if order is not None:
                break
            self.seed, tries = self.seed + 1, tries + 1
            if tries % 10 == 0:
                self.seg += self.seg // 10 + 1
        size = 3 * self.seg
        self.T = [0] * size  # @line L3
        self.R = [0] * size
        for x, c in reversed(order):  # @line L4
            cells, mask = self._cells(x)
            tau = cells.index(c)  # @line L5
            acc = tau ^ mask
            for h in cells:
                if h != c:
                    acc ^= self.T[h]
            self.T[c] = acc  # @line L6
            self.R[c] = f[x]  # @line L7

    def _cells(self, key: Key) -> tuple[tuple[int, int, int], int]:
        """Three cells, one in each third of the table, and a q-bit mask."""
        d = hashlib.blake2b(repr(key).encode(), digest_size=32, key=self.seed.to_bytes(8, "little")).digest()
        a, b, c, m = (int.from_bytes(d[i : i + 8], "little") for i in range(0, 32, 8))
        s = self.seg
        return (a % s, s + b % s, 2 * s + c % s), m % (1 << self.q)

    def _peel(self, keys: list[Key]) -> list[tuple[Key, int]] | None:
        """A peeling order: pairs (x, c) where c is a cell of x and of no later key."""
        size = 3 * self.seg
        count, xor = [0] * size, [0] * size
        cells = [self._cells(x)[0] for x in keys]
        for i, cs in enumerate(cells):  # @line L12
            for c in cs:
                count[c] += 1
                xor[c] ^= i
        queue = deque(c for c in range(size) if count[c] == 1)
        order: list[tuple[Key, int]] = []
        while queue:  # @line L13
            c = queue.popleft()
            if count[c] != 1:
                continue
            i = xor[c]  # the only key left at cell c
            order.append((keys[i], c))
            for h in cells[i]:
                count[h] -= 1
                xor[h] ^= i
                if count[h] == 1:
                    queue.append(h)
        return order if len(order) == len(keys) else None  # @line L14

    def _tau(self, key: Key) -> tuple[tuple[int, int, int], int]:
        cells, mask = self._cells(key)
        return cells, mask ^ self.T[cells[0]] ^ self.T[cells[1]] ^ self.T[cells[2]]  # @line L8

    def lookup(self, key: Key) -> int | None:
        """f(key) for key in S; None, or with probability 3 / 2^q a wrong value, otherwise."""
        cells, tau = self._tau(key)
        if tau < 3:  # @line L9
            return self.R[cells[tau]]
        return None  # @line L10

    def set(self, key: Key, value: int) -> None:
        """Change f(key) for a key in S. Precondition: key is in S."""
        if not 0 <= value < 1 << self.r:
            raise ValueError("value must fit in r bits")
        cells, tau = self._tau(key)  # @line L11
        if tau >= 3:
            raise KeyError(key)
        self.R[cells[tau]] = value
test_bloomier_filter.py 64 lines
# SPDX-License-Identifier: MIT
# Part of https://algorithm.tamnd.com/bloomier-filter/. The authors are listed in prov/8T9R6F.jsonl.

import random

import pytest

from bloomier_filter import BloomierFilter, Key

CELLS: dict[Key, tuple[tuple[int, int, int], int]] = {
    "a": ((0, 2, 4), 9), "b": ((0, 3, 5), 4), "c": ((1, 3, 4), 12),
    "d": ((1, 2, 5), 3), "e": ((0, 3, 4), 7),
}


class FixedBloomier(BloomierFilter):
    def _cells(self, key: Key) -> tuple[tuple[int, int, int], int]:
        return CELLS[key]


def test_example() -> None:
    g = FixedBloomier({"a": 7, "b": 2, "c": 5}, r=3, q=4, seg=2)
    assert g._peel(["a", "b", "c"]) == [("c", 1), ("a", 2), ("b", 5)]
    assert g.T == [0, 12, 8, 0, 0, 6]
    assert g.R == [0, 5, 7, 0, 0, 2]
    assert (g.lookup("a"), g.lookup("b"), g.lookup("c")) == (7, 2, 5)
    assert g.lookup("d") == 7  # a false positive: tau = 1
    assert g.lookup("e") is None
    g.set("b", 6)
    assert g.lookup("b") == 6 and g.lookup("a") == 7


def test_edges() -> None:
    g = BloomierFilter({}, r=1)
    assert g.lookup("x") in (None, 0)
    g = BloomierFilter({"only": 1}, r=1)
    assert g.lookup("only") == 1
    with pytest.raises(ValueError):
        BloomierFilter({"a": 4}, r=2)


def test_random_against_dict() -> None:
    rng = random.Random(2004)
    for _ in range(500):
        n = rng.randint(0, 60)
        r = rng.randint(1, 8)
        f = {rng.randrange(10**6): rng.randrange(1 << r) for _ in range(n)}
        g = BloomierFilter(f, r=r, q=rng.randint(2, 10), seed=rng.randrange(1000))
        assert all(g.lookup(x) == v for x, v in f.items())
        if f:
            x = rng.choice(list(f))
            v = rng.randrange(1 << r)
            g.set(x, v)
            f[x] = v
            assert all(g.lookup(y) == w for y, w in f.items())


def test_space_and_rate() -> None:
    n = 20000
    f = {("k", i): i % 16 for i in range(n)}
    g = BloomierFilter(f, r=4, q=10, seed=3)
    assert len(g.T) <= 1.3 * n
    hits = sum(g.lookup(("absent", i)) is not None for i in range(30000))
    assert hits / 30000 < 3 / 1024 + 0.002

What the next level needs

  • the tests did not run in this build

Read the full page about Bloomier filter