CHD perfect hashing in Python
Practical minimal perfect hashing that uses displacement tables and about 2 bits per key.
- File
chd_hashing.py, 99 lines- Level
- I0 not checked yet
- Toolchain
- Python 3.14, pytest 9
- Test framework
- pytest
- Run the tests
python3 -m pytest
# SPDX-License-Identifier: MIT
# Part of https://algorithm.tamnd.com/chd-hashing/. The authors are listed in prov/X3MZCW.jsonl.
import random
P61 = (1 << 61) - 1 # a Mersenne prime, larger than every key
def poly_hash(c: tuple[int, int, int], p: int, x: int) -> int:
"""(c0 + c1 x + c2 x^2) mod p, a 3-wise independent hash over Z_p."""
return (c[0] + (c[1] + c[2] * x) * x) % p
class CHD:
"""A perfect hash function from a static set of integer keys into [m].
With m = n (the default) the function is minimal: a bijection from the
keys onto [0, n). The key x goes to bucket g(x) in [r], and its slot is
(f1(x) + d0 f2(x) + d1) mod m, where (d0, d1) is decoded from the
displacement index sigma[g(x)]. Keys outside the set get some slot in [m].
"""
def __init__(
self,
keys: list[int],
rng: random.Random | None = None,
m: int | None = None,
lam: int = 5,
p: int = P61,
) -> None:
if len(set(keys)) != len(keys):
raise ValueError("keys must be distinct")
if any(not 0 <= x < p for x in keys):
raise ValueError("key out of range")
n = len(keys)
self.n, self.p = n, p
self.m = max(n, 1) if m is None else m
if self.m < max(n, 1):
raise ValueError("m must be at least the number of keys")
self.r = max(1, -(-n // lam)) # ceil(n / lam) buckets
self.rng = rng if rng is not None else random.Random()
self.tries = 0
while True: # @line L1
self.tries += 1
self.coef = [tuple(self.rng.randrange(p) for _ in range(3)) for _ in range(3)] # @line L2
if self._build(keys): # @line L10
return # @line L11
# ----------------------------------------------------------- hashing
def g(self, x: int) -> int:
return poly_hash(self.coef[0], self.p, x) % self.r
def f1(self, x: int) -> int:
return poly_hash(self.coef[1], self.p, x) % self.m
def f2(self, x: int) -> int:
return poly_hash(self.coef[2], self.p, x) % self.m
def slot(self, x: int, ell: int) -> int:
"""The slot of x under the displacement index ell."""
d0, d1 = divmod(ell, self.m)
return (self.f1(x) + d0 * self.f2(x) + d1) % self.m
# ----------------------------------------------------------- build
def _build(self, keys: list[int]) -> bool:
m = self.m
buckets: list[list[int]] = [[] for _ in range(self.r)]
for x in keys: # @line L3
buckets[self.g(x)].append(x)
used = [False] * m # @line L4
self.sigma = [0] * self.r
order = sorted(range(self.r), key=lambda i: (-len(buckets[i]), i))
for i in order: # @line L5
b = buckets[i]
pre = [(self.f1(x), self.f2(x)) for x in b]
ell = 0 # @line L6
while ell < m * m: # @line L7
d0, d1 = divmod(ell, m)
s = [(a + d0 * c + d1) % m for a, c in pre]
if len(set(s)) == len(s) and not any(used[t] for t in s):
break
ell += 1
if ell == m * m: # @line L8
return False
self.sigma[i] = ell # @line L9
for t in s:
used[t] = True
return True
# ----------------------------------------------------------- query
def __call__(self, x: int) -> int:
i = self.g(x) # @line L12
d0, d1 = divmod(self.sigma[i], self.m)
return (self.f1(x) + d0 * self.f2(x) + d1) % self.m # @line L13
def gamma_bits(self) -> int:
"""Bits of the Elias gamma codes of sigma[i] + 1: a simple measure of
how well the displacement indices compress (no O(1) access)."""
return sum(2 * (s + 1).bit_length() - 1 for s in self.sigma)
test_chd_hashing.py 60 lines
# SPDX-License-Identifier: MIT
# Part of https://algorithm.tamnd.com/chd-hashing/. The authors are listed in prov/X3MZCW.jsonl.
import random
import pytest
from chd_hashing import CHD, P61
S = [10, 22, 37, 40, 52, 60, 70, 72, 75, 88]
def test_page_example() -> None:
c = CHD(S, random.Random(1), lam=3, p=101)
assert c.tries == 1 and c.r == 4 and c.m == 10
assert [c.g(x) for x in S] == [2, 1, 1, 0, 3, 1, 1, 3, 3, 1]
assert [(c.f1(x), c.f2(x)) for x in (52, 72, 75)] == [(4, 9), (0, 0), (4, 6)]
assert c.sigma == [0, 0, 0, 21]
assert {x: c(x) for x in S} == {10: 0, 22: 4, 37: 2, 40: 8, 52: 3, 60: 5, 70: 9, 72: 1, 75: 7, 88: 6}
assert c.slot(52, 21) == 3 and c.slot(75, 1) == c.slot(52, 1)
def test_edges() -> None:
c = CHD([], random.Random(0))
assert c.m == 1 and c(5) == 0
c = CHD([P61 - 1], random.Random(0))
assert c(P61 - 1) == 0
with pytest.raises(ValueError):
CHD([1, 1])
with pytest.raises(ValueError):
CHD([1, 2, 3], m=2)
with pytest.raises(ValueError):
CHD([P61])
def test_non_minimal() -> None:
rng = random.Random(7)
keys = rng.sample(range(10**6), 300)
c = CHD(keys, rng, m=330)
vals = [c(x) for x in keys]
assert len(set(vals)) == 300 and all(0 <= v < 330 for v in vals)
def test_random_against_dict() -> None:
rng = random.Random(2009)
for _ in range(600):
n = rng.randint(0, 60)
p = rng.choice([1009, 65521, P61])
keys = rng.sample(range(p), n)
lam = rng.randint(1, 6)
c = CHD(keys, rng, lam=lam, p=p)
slots = [c(x) for x in keys]
assert sorted(slots) == list(range(n)) # minimal and perfect
# use the function as the index of a static dictionary
oracle = {x: rng.random() for x in keys}
table: list[float | None] = [None] * c.m
for x, v in oracle.items():
table[c(x)] = v
assert all(table[c(x)] == v for x, v in oracle.items())
assert all(0 <= c(rng.randrange(p)) < c.m for _ in range(5))
What the next level needs
- the tests did not run in this build