Algorithm Search⌘K

CHD perfect hashing in Python

Practical minimal perfect hashing that uses displacement tables and about 2 bits per key.

File
chd_hashing.py, 99 lines
Level
I0 not checked yet
Toolchain
Python 3.14, pytest 9
Test framework
pytest
Run the tests
python3 -m pytest
chd_hashing.pyRawGitHub
# SPDX-License-Identifier: MIT
# Part of https://algorithm.tamnd.com/chd-hashing/. The authors are listed in prov/X3MZCW.jsonl.

import random

P61 = (1 << 61) - 1  # a Mersenne prime, larger than every key


def poly_hash(c: tuple[int, int, int], p: int, x: int) -> int:
    """(c0 + c1 x + c2 x^2) mod p, a 3-wise independent hash over Z_p."""
    return (c[0] + (c[1] + c[2] * x) * x) % p


class CHD:
    """A perfect hash function from a static set of integer keys into [m].

    With m = n (the default) the function is minimal: a bijection from the
    keys onto [0, n). The key x goes to bucket g(x) in [r], and its slot is
    (f1(x) + d0 f2(x) + d1) mod m, where (d0, d1) is decoded from the
    displacement index sigma[g(x)]. Keys outside the set get some slot in [m].
    """

    def __init__(
        self,
        keys: list[int],
        rng: random.Random | None = None,
        m: int | None = None,
        lam: int = 5,
        p: int = P61,
    ) -> None:
        if len(set(keys)) != len(keys):
            raise ValueError("keys must be distinct")
        if any(not 0 <= x < p for x in keys):
            raise ValueError("key out of range")
        n = len(keys)
        self.n, self.p = n, p
        self.m = max(n, 1) if m is None else m
        if self.m < max(n, 1):
            raise ValueError("m must be at least the number of keys")
        self.r = max(1, -(-n // lam))  # ceil(n / lam) buckets
        self.rng = rng if rng is not None else random.Random()
        self.tries = 0
        while True:  # @line L1
            self.tries += 1
            self.coef = [tuple(self.rng.randrange(p) for _ in range(3)) for _ in range(3)]  # @line L2
            if self._build(keys):  # @line L10
                return  # @line L11

    # ----------------------------------------------------------- hashing
    def g(self, x: int) -> int:
        return poly_hash(self.coef[0], self.p, x) % self.r

    def f1(self, x: int) -> int:
        return poly_hash(self.coef[1], self.p, x) % self.m

    def f2(self, x: int) -> int:
        return poly_hash(self.coef[2], self.p, x) % self.m

    def slot(self, x: int, ell: int) -> int:
        """The slot of x under the displacement index ell."""
        d0, d1 = divmod(ell, self.m)
        return (self.f1(x) + d0 * self.f2(x) + d1) % self.m

    # ----------------------------------------------------------- build
    def _build(self, keys: list[int]) -> bool:
        m = self.m
        buckets: list[list[int]] = [[] for _ in range(self.r)]
        for x in keys:  # @line L3
            buckets[self.g(x)].append(x)
        used = [False] * m  # @line L4
        self.sigma = [0] * self.r
        order = sorted(range(self.r), key=lambda i: (-len(buckets[i]), i))
        for i in order:  # @line L5
            b = buckets[i]
            pre = [(self.f1(x), self.f2(x)) for x in b]
            ell = 0  # @line L6
            while ell < m * m:  # @line L7
                d0, d1 = divmod(ell, m)
                s = [(a + d0 * c + d1) % m for a, c in pre]
                if len(set(s)) == len(s) and not any(used[t] for t in s):
                    break
                ell += 1
            if ell == m * m:  # @line L8
                return False
            self.sigma[i] = ell  # @line L9
            for t in s:
                used[t] = True
        return True

    # ----------------------------------------------------------- query
    def __call__(self, x: int) -> int:
        i = self.g(x)  # @line L12
        d0, d1 = divmod(self.sigma[i], self.m)
        return (self.f1(x) + d0 * self.f2(x) + d1) % self.m  # @line L13

    def gamma_bits(self) -> int:
        """Bits of the Elias gamma codes of sigma[i] + 1: a simple measure of
        how well the displacement indices compress (no O(1) access)."""
        return sum(2 * (s + 1).bit_length() - 1 for s in self.sigma)
test_chd_hashing.py 60 lines
# SPDX-License-Identifier: MIT
# Part of https://algorithm.tamnd.com/chd-hashing/. The authors are listed in prov/X3MZCW.jsonl.

import random

import pytest

from chd_hashing import CHD, P61

S = [10, 22, 37, 40, 52, 60, 70, 72, 75, 88]


def test_page_example() -> None:
    c = CHD(S, random.Random(1), lam=3, p=101)
    assert c.tries == 1 and c.r == 4 and c.m == 10
    assert [c.g(x) for x in S] == [2, 1, 1, 0, 3, 1, 1, 3, 3, 1]
    assert [(c.f1(x), c.f2(x)) for x in (52, 72, 75)] == [(4, 9), (0, 0), (4, 6)]
    assert c.sigma == [0, 0, 0, 21]
    assert {x: c(x) for x in S} == {10: 0, 22: 4, 37: 2, 40: 8, 52: 3, 60: 5, 70: 9, 72: 1, 75: 7, 88: 6}
    assert c.slot(52, 21) == 3 and c.slot(75, 1) == c.slot(52, 1)


def test_edges() -> None:
    c = CHD([], random.Random(0))
    assert c.m == 1 and c(5) == 0
    c = CHD([P61 - 1], random.Random(0))
    assert c(P61 - 1) == 0
    with pytest.raises(ValueError):
        CHD([1, 1])
    with pytest.raises(ValueError):
        CHD([1, 2, 3], m=2)
    with pytest.raises(ValueError):
        CHD([P61])


def test_non_minimal() -> None:
    rng = random.Random(7)
    keys = rng.sample(range(10**6), 300)
    c = CHD(keys, rng, m=330)
    vals = [c(x) for x in keys]
    assert len(set(vals)) == 300 and all(0 <= v < 330 for v in vals)


def test_random_against_dict() -> None:
    rng = random.Random(2009)
    for _ in range(600):
        n = rng.randint(0, 60)
        p = rng.choice([1009, 65521, P61])
        keys = rng.sample(range(p), n)
        lam = rng.randint(1, 6)
        c = CHD(keys, rng, lam=lam, p=p)
        slots = [c(x) for x in keys]
        assert sorted(slots) == list(range(n))  # minimal and perfect
        # use the function as the index of a static dictionary
        oracle = {x: rng.random() for x in keys}
        table: list[float | None] = [None] * c.m
        for x, v in oracle.items():
            table[c(x)] = v
        assert all(table[c(x)] == v for x, v in oracle.items())
        assert all(0 <= c(rng.randrange(p)) < c.m for _ in range(5))

What the next level needs

  • the tests did not run in this build

Read the full page about CHD perfect hashing