Locality-sensitive hashing in Python
Approximate nearest neighbor search by hashing points into buckets with an LSH family and checking only colliding points.
- File
locality_sensitive_hashing.py, 82 lines- Level
- I0 not checked yet
- Toolchain
- Python 3.14, pytest 9
- Test framework
- pytest
- Run the tests
python3 -m pytest
# SPDX-License-Identifier: MIT
# Part of https://algorithm.tamnd.com/locality-sensitive-hashing/. The authors are listed in prov/XTR0CK.jsonl.
import math
import random
from collections.abc import Callable, Hashable, Sequence
from typing import Generic, TypeVar
P = TypeVar("P")
HashFn = Callable[[P], Hashable]
Family = Callable[[random.Random], HashFn]
class LSHIndex(Generic[P]):
"""Locality-sensitive hashing for the (c, r)-near neighbor problem.
`family(rng)` draws one function of a locality-sensitive family. The index
builds L tables, each keyed by the tuple of k functions. Space O(nL) keys,
build time O(nLk) hash evaluations, query time O(Lk) evaluations plus at
most 3L distance computations.
"""
def __init__(
self,
points: Sequence[P],
family: Family,
k: int,
L: int,
dist: Callable[[P, P], float],
rng: random.Random | None = None,
) -> None:
if k < 1 or L < 1:
raise ValueError("k and L must be at least 1")
rng = rng if rng is not None else random.Random()
self.points = list(points)
self.dist = dist
self.L = L
self.g: list[list[HashFn]] = []
self.tables: list[dict[tuple, list[int]]] = []
for _ in range(L): # @line L1
fns = [family(rng) for _ in range(k)] # @line L2
self.g.append(fns)
table: dict[tuple, list[int]] = {}
for i, x in enumerate(self.points):
table.setdefault(self._key(fns, x), []).append(i) # @line L3 @line L4
self.tables.append(table)
@staticmethod
def _key(fns: list[HashFn], x: P) -> tuple:
return tuple(h(x) for h in fns)
def query(self, q: P, c: float, r: float) -> int | None:
"""Return the index of a point within c*r of q, or None.
A returned point is always within c*r. If some point is within r, a
point is returned with probability at least 2/3 - 1/e when k and L are
set as on the page.
"""
count = 0 # @line L5
for j in range(self.L): # @line L6
for i in self.tables[j].get(self._key(self.g[j], q), []): # @line L7
count += 1 # @line L8
if self.dist(q, self.points[i]) <= c * r: # @line L9
return i
if count >= 3 * self.L: # @line L10
return None
return None # @line L11
def candidates(self, q: P) -> set[int]:
"""Return the indices of all points that share a bucket with q in some table."""
out: set[int] = set()
for j in range(self.L):
out.update(self.tables[j].get(self._key(self.g[j], q), []))
return out
def parameters(n: int, p1: float, p2: float) -> tuple[int, int]:
"""Return (k, L) = (ceil(log_{1/p2} n), ceil(n^rho / p1)) for 0 < p2 < p1 <= 1 and n >= 2."""
rho = math.log(1 / p1) / math.log(1 / p2)
k = max(1, math.ceil(math.log(n) / math.log(1 / p2)))
L = math.ceil(n**rho / p1)
return k, L
test_locality_sensitive_hashing.py 91 lines
# SPDX-License-Identifier: MIT
# Part of https://algorithm.tamnd.com/locality-sensitive-hashing/. The authors are listed in prov/XTR0CK.jsonl.
import random
from locality_sensitive_hashing import LSHIndex, parameters
def hamming(a: str, b: str) -> int:
return sum(x != y for x, y in zip(a, b))
def bit_family(d: int):
def draw(rng: random.Random):
i = rng.randrange(d)
return lambda s: s[i]
return draw
class Fixed:
"""A family that hands out the given bit positions in order, for the worked example."""
def __init__(self, positions: list[int]) -> None:
self.positions = positions
self.t = 0
def __call__(self, rng: random.Random):
i = self.positions[self.t]
self.t += 1
return lambda s: s[i]
P = ["00010111", "11100000", "11010010", "01011011", "00111100", "10110100"]
Q = "10110001"
def test_example() -> None:
idx = LSHIndex(P, Fixed([0, 3, 1, 6, 2, 7]), 2, 3, hamming)
assert idx.tables[0][("1", "1")] == [2, 5]
assert idx.tables[1][("0", "0")] == [4, 5]
assert idx.query(Q, 2, 1) == 5
assert idx.candidates(Q) == {2, 4, 5}
assert idx.query(Q, 1, 1) is None
def test_parameters() -> None:
assert parameters(10**6, 0.9, 0.8) == (62, 757)
def test_never_returns_a_far_point() -> None:
rng = random.Random(7)
for _ in range(200):
d = 16
pts = ["".join(rng.choice("01") for _ in range(d)) for _ in range(rng.randint(1, 30))]
q = "".join(rng.choice("01") for _ in range(d))
idx = LSHIndex(pts, bit_family(d), rng.randint(1, 4), rng.randint(1, 5), hamming, rng)
c, r = 2, rng.randint(0, 4)
got = idx.query(q, c, r)
if got is not None:
assert hamming(q, pts[got]) <= c * r
if min(hamming(q, p) for p in pts) > c * r:
assert got is None
def test_success_probability() -> None:
rng = random.Random(2024)
d, n, r, c = 32, 60, 2, 2
k, L = parameters(n, 1 - r / d, 1 - c * r / d)
ok = trials = 0
for _ in range(300):
pts = ["".join(rng.choice("01") for _ in range(d)) for _ in range(n)]
base = pts[rng.randrange(n)]
flip = set(rng.sample(range(d), r))
q = "".join(("1" if ch == "0" else "0") if i in flip else ch for i, ch in enumerate(base))
idx = LSHIndex(pts, bit_family(d), k, L, hamming, rng)
got = idx.query(q, c, r)
trials += 1
ok += got is not None and hamming(q, pts[got]) <= c * r
assert ok / trials >= 2 / 3 - 1 / 2.718281828
def test_candidates_against_brute_force() -> None:
rng = random.Random(99)
for _ in range(500):
d = 10
pts = ["".join(rng.choice("01") for _ in range(d)) for _ in range(rng.randint(0, 12))]
q = "".join(rng.choice("01") for _ in range(d))
idx = LSHIndex(pts, bit_family(d), 2, 3, hamming, rng)
want = {i for i, p in enumerate(pts) if any(all(h(p) == h(q) for h in g) for g in idx.g)}
assert idx.candidates(q) == want
What the next level needs
- the tests did not run in this build