Algorithm Search⌘K

Locality-sensitive hashing in Python

Approximate nearest neighbor search by hashing points into buckets with an LSH family and checking only colliding points.

File
locality_sensitive_hashing.py, 82 lines
Level
I0 not checked yet
Toolchain
Python 3.14, pytest 9
Test framework
pytest
Run the tests
python3 -m pytest
locality_sensitive_hashing.pyRawGitHub
# SPDX-License-Identifier: MIT
# Part of https://algorithm.tamnd.com/locality-sensitive-hashing/. The authors are listed in prov/XTR0CK.jsonl.

import math
import random
from collections.abc import Callable, Hashable, Sequence
from typing import Generic, TypeVar

P = TypeVar("P")
HashFn = Callable[[P], Hashable]
Family = Callable[[random.Random], HashFn]


class LSHIndex(Generic[P]):
    """Locality-sensitive hashing for the (c, r)-near neighbor problem.

    `family(rng)` draws one function of a locality-sensitive family. The index
    builds L tables, each keyed by the tuple of k functions. Space O(nL) keys,
    build time O(nLk) hash evaluations, query time O(Lk) evaluations plus at
    most 3L distance computations.
    """

    def __init__(
        self,
        points: Sequence[P],
        family: Family,
        k: int,
        L: int,
        dist: Callable[[P, P], float],
        rng: random.Random | None = None,
    ) -> None:
        if k < 1 or L < 1:
            raise ValueError("k and L must be at least 1")
        rng = rng if rng is not None else random.Random()
        self.points = list(points)
        self.dist = dist
        self.L = L
        self.g: list[list[HashFn]] = []
        self.tables: list[dict[tuple, list[int]]] = []
        for _ in range(L):  # @line L1
            fns = [family(rng) for _ in range(k)]  # @line L2
            self.g.append(fns)
            table: dict[tuple, list[int]] = {}
            for i, x in enumerate(self.points):
                table.setdefault(self._key(fns, x), []).append(i)  # @line L3 @line L4
            self.tables.append(table)

    @staticmethod
    def _key(fns: list[HashFn], x: P) -> tuple:
        return tuple(h(x) for h in fns)

    def query(self, q: P, c: float, r: float) -> int | None:
        """Return the index of a point within c*r of q, or None.

        A returned point is always within c*r. If some point is within r, a
        point is returned with probability at least 2/3 - 1/e when k and L are
        set as on the page.
        """
        count = 0  # @line L5
        for j in range(self.L):  # @line L6
            for i in self.tables[j].get(self._key(self.g[j], q), []):  # @line L7
                count += 1  # @line L8
                if self.dist(q, self.points[i]) <= c * r:  # @line L9
                    return i
                if count >= 3 * self.L:  # @line L10
                    return None
        return None  # @line L11

    def candidates(self, q: P) -> set[int]:
        """Return the indices of all points that share a bucket with q in some table."""
        out: set[int] = set()
        for j in range(self.L):
            out.update(self.tables[j].get(self._key(self.g[j], q), []))
        return out


def parameters(n: int, p1: float, p2: float) -> tuple[int, int]:
    """Return (k, L) = (ceil(log_{1/p2} n), ceil(n^rho / p1)) for 0 < p2 < p1 <= 1 and n >= 2."""
    rho = math.log(1 / p1) / math.log(1 / p2)
    k = max(1, math.ceil(math.log(n) / math.log(1 / p2)))
    L = math.ceil(n**rho / p1)
    return k, L
test_locality_sensitive_hashing.py 91 lines
# SPDX-License-Identifier: MIT
# Part of https://algorithm.tamnd.com/locality-sensitive-hashing/. The authors are listed in prov/XTR0CK.jsonl.

import random

from locality_sensitive_hashing import LSHIndex, parameters


def hamming(a: str, b: str) -> int:
    return sum(x != y for x, y in zip(a, b))


def bit_family(d: int):
    def draw(rng: random.Random):
        i = rng.randrange(d)
        return lambda s: s[i]

    return draw


class Fixed:
    """A family that hands out the given bit positions in order, for the worked example."""

    def __init__(self, positions: list[int]) -> None:
        self.positions = positions
        self.t = 0

    def __call__(self, rng: random.Random):
        i = self.positions[self.t]
        self.t += 1
        return lambda s: s[i]


P = ["00010111", "11100000", "11010010", "01011011", "00111100", "10110100"]
Q = "10110001"


def test_example() -> None:
    idx = LSHIndex(P, Fixed([0, 3, 1, 6, 2, 7]), 2, 3, hamming)
    assert idx.tables[0][("1", "1")] == [2, 5]
    assert idx.tables[1][("0", "0")] == [4, 5]
    assert idx.query(Q, 2, 1) == 5
    assert idx.candidates(Q) == {2, 4, 5}
    assert idx.query(Q, 1, 1) is None


def test_parameters() -> None:
    assert parameters(10**6, 0.9, 0.8) == (62, 757)


def test_never_returns_a_far_point() -> None:
    rng = random.Random(7)
    for _ in range(200):
        d = 16
        pts = ["".join(rng.choice("01") for _ in range(d)) for _ in range(rng.randint(1, 30))]
        q = "".join(rng.choice("01") for _ in range(d))
        idx = LSHIndex(pts, bit_family(d), rng.randint(1, 4), rng.randint(1, 5), hamming, rng)
        c, r = 2, rng.randint(0, 4)
        got = idx.query(q, c, r)
        if got is not None:
            assert hamming(q, pts[got]) <= c * r
        if min(hamming(q, p) for p in pts) > c * r:
            assert got is None


def test_success_probability() -> None:
    rng = random.Random(2024)
    d, n, r, c = 32, 60, 2, 2
    k, L = parameters(n, 1 - r / d, 1 - c * r / d)
    ok = trials = 0
    for _ in range(300):
        pts = ["".join(rng.choice("01") for _ in range(d)) for _ in range(n)]
        base = pts[rng.randrange(n)]
        flip = set(rng.sample(range(d), r))
        q = "".join(("1" if ch == "0" else "0") if i in flip else ch for i, ch in enumerate(base))
        idx = LSHIndex(pts, bit_family(d), k, L, hamming, rng)
        got = idx.query(q, c, r)
        trials += 1
        ok += got is not None and hamming(q, pts[got]) <= c * r
    assert ok / trials >= 2 / 3 - 1 / 2.718281828


def test_candidates_against_brute_force() -> None:
    rng = random.Random(99)
    for _ in range(500):
        d = 10
        pts = ["".join(rng.choice("01") for _ in range(d)) for _ in range(rng.randint(0, 12))]
        q = "".join(rng.choice("01") for _ in range(d))
        idx = LSHIndex(pts, bit_family(d), 2, 3, hamming, rng)
        want = {i for i, p in enumerate(pts) if any(all(h(p) == h(q) for h in g) for g in idx.g)}
        assert idx.candidates(q) == want

What the next level needs

  • the tests did not run in this build

Read the full page about Locality-sensitive hashing