Algorithm Search⌘K

Levenshtein automaton in Python

Deterministic automaton that accepts all strings within edit distance k of a fixed word.

File
levenshtein_automaton.py, 136 lines
Level
I0 not checked yet
Toolchain
Python 3.14, pytest 9
Test framework
pytest
Run the tests
python3 -m pytest
levenshtein_automaton.pyRawGitHub
# SPDX-License-Identifier: MIT
# Part of https://algorithm.tamnd.com/levenshtein-automaton/. The authors are listed in prov/CK7G4C.jsonl.

"""Levenshtein automaton of a word, built lazily from capped rows.

Edit model: unit-cost insertion, deletion and substitution (Levenshtein),
no transpositions. Indices are 0-based. A state is the row
(min(ed(word[0:i], x), k + 1)) for i = 0..m, where x is the input read so far.
In search mode the first entry of each row stays 0, so the automaton tracks
the Sellers column instead, and an accepting state after reading text[0:j]
reports the end position j.
"""

from typing import Optional


class LevenshteinAutomaton:
    """Deterministic automaton for {x : ed(word, x) <= k}, built on demand."""

    def __init__(self, word: str, k: int, search: bool = False) -> None:
        """Make the start state. Precondition: k >= 0."""
        self.word = word
        self.k = k
        self.search = search
        self.cap = k + 1  # @line L1
        self.symbols = set(word)
        self.rows: list[tuple[int, ...]] = []
        self.index: dict[tuple[int, ...], int] = {}
        self.delta: dict[tuple[int, Optional[str]], int] = {}
        first = tuple(min(i, self.cap) for i in range(len(word) + 1))  # @line L2
        self.start = self._state(first)

    def _state(self, row: tuple[int, ...]) -> int:
        """Return the id of a row, and make a new state for a new row."""
        sid = self.index.get(row)
        if sid is None:
            sid = len(self.rows)
            self.index[row] = sid
            self.rows.append(row)
        return sid

    def step(self, state: int, ch: str) -> int:
        """Return the state after reading ch. A new row costs O(m) time."""
        key = (state, ch if ch in self.symbols else None)
        nxt = self.delta.get(key)
        if nxt is not None:
            return nxt
        row = self.rows[state]
        cap = self.cap
        new = [0] * len(row)
        new[0] = 0 if self.search else min(row[0] + 1, cap)  # @line L3
        for i in range(1, len(row)):  # @line L4
            sub = row[i - 1] + (self.word[i - 1] != ch)
            new[i] = min(sub, row[i] + 1, new[i - 1] + 1, cap)  # @line L5
        nxt = self._state(tuple(new))  # @line L6
        self.delta[key] = nxt
        return nxt

    def is_match(self, state: int) -> bool:
        """True when the input read so far is within distance k."""
        return self.rows[state][-1] <= self.k  # @line L7

    def can_match(self, state: int) -> bool:
        """False exactly for the dead state, from which no input is accepted."""
        return min(self.rows[state]) <= self.k  # @line L8

    def distance(self, state: int) -> int:
        """Return min(ed(word, x), k + 1) for the input x read so far."""
        return self.rows[state][-1]

    def accepts(self, s: str) -> bool:
        """Return True when ed(word, s) <= k (word mode)."""
        state = self.start
        for ch in s:
            state = self.step(state, ch)
            if not self.search and not self.can_match(state):
                return False
        return self.is_match(state)

    def find_ends(self, text: str) -> list[int]:
        """Search mode: return the end positions j in 0..n of the matches."""
        state = self.start
        out = [0] if self.is_match(state) else []
        for j, ch in enumerate(text, 1):
            state = self.step(state, ch)
            if self.is_match(state):
                out.append(j)
        return out

    def build_all(self) -> int:
        """Build every reachable state and return the number of states."""
        classes: list[str] = sorted(self.symbols)
        other = "\x00"
        while other in self.symbols:
            other = chr(ord(other) + 1)
        classes.append(other)
        todo = [self.start]
        seen = {self.start}
        while todo:
            s = todo.pop()
            for ch in classes:
                t = self.step(s, ch)
                if t not in seen:
                    seen.add(t)
                    todo.append(t)
        return len(seen)


def fuzzy_lookup(words: list[str], word: str, k: int) -> list[str]:
    """Return the sorted distinct words of the list within distance k of word,
    by a walk over a trie of the list in step with the automaton."""
    children: list[dict[str, int]] = [{}]
    final: list[Optional[str]] = [None]
    for w in words:
        u = 0
        for ch in w:
            v = children[u].get(ch)
            if v is None:
                v = len(children)
                children[u][ch] = v
                children.append({})
                final.append(None)
            u = v
        final[u] = w
    auto = LevenshteinAutomaton(word, k)
    out = []
    stack = [(0, auto.start)]  # @line L9
    while stack:  # @line L10
        u, s = stack.pop()
        if final[u] is not None and auto.is_match(s):  # @line L11
            out.append(final[u])
        for ch, v in children[u].items():  # @line L12
            t = auto.step(s, ch)
            if auto.can_match(t):
                stack.append((v, t))
    return sorted(out)
test_levenshtein_automaton.py 75 lines
# SPDX-License-Identifier: MIT
# Part of https://algorithm.tamnd.com/levenshtein-automaton/. The authors are listed in prov/CK7G4C.jsonl.

import itertools
import random

from levenshtein_automaton import LevenshteinAutomaton, fuzzy_lookup


def lev(a: str, b: str) -> int:
    d = [[i + j if i * j == 0 else 0 for j in range(len(b) + 1)] for i in range(len(a) + 1)]
    for i in range(1, len(a) + 1):
        for j in range(1, len(b) + 1):
            d[i][j] = min(d[i - 1][j] + 1, d[i][j - 1] + 1, d[i - 1][j - 1] + (a[i - 1] != b[j - 1]))
    return d[len(a)][len(b)]


def brute_ends(p: str, t: str, k: int) -> list[int]:
    return [j for j in range(len(t) + 1) if min(lev(p, t[h:j]) for h in range(j + 1)) <= k]


def bound(m: int, k: int) -> int:
    return (m + k + 1) * (k + 2) * 3 ** (2 * k) + 1


def test_example() -> None:
    a = LevenshteinAutomaton("cat", 1)
    assert a.accepts("cat") and a.accepts("cart") and a.accepts("at") and a.accepts("coat")
    assert not a.accepts("act") and not a.accepts("dog") and not a.accepts("")
    words = ["cat", "cart", "act", "at", "coat", "cut", "dog", "cats", "scat", "a", ""]
    assert fuzzy_lookup(words, "cat", 1) == ["at", "cart", "cat", "cats", "coat", "cut", "scat"]
    assert fuzzy_lookup(words, "cat", 0) == ["cat"]
    s = LevenshteinAutomaton("GATAA", 1, search=True)
    assert s.find_ends("CAGATAAGAGAA") == [6, 7, 8, 12]


def test_edges() -> None:
    a = LevenshteinAutomaton("", 0)
    assert a.accepts("") and not a.accepts("a")
    assert LevenshteinAutomaton("", 2).accepts("ab")
    assert not LevenshteinAutomaton("ab", 1).accepts("")
    assert LevenshteinAutomaton("", 0, search=True).find_ends("ab") == [0, 1, 2]
    assert LevenshteinAutomaton("ab", 0, search=True).find_ends("") == []
    assert fuzzy_lookup([], "ab", 1) == []
    assert fuzzy_lookup(["", "a"], "", 0) == [""]


def test_accepts_all_short_strings() -> None:
    rng = random.Random(1985)
    for alphabet in ("ab", "acgt"):
        for _ in range(60):
            w = "".join(rng.choice(alphabet) for _ in range(rng.randint(0, 5)))
            k = rng.randint(0, 2)
            a = LevenshteinAutomaton(w, k)
            for n in range(0, 6 if alphabet == "ab" else 4):
                for t in itertools.product(alphabet + "x", repeat=n):
                    x = "".join(t)
                    assert a.accepts(x) == (lev(w, x) <= k)
                    s = a.start
                    for ch in x:
                        s = a.step(s, ch)
                    assert a.distance(s) == min(lev(w, x), k + 1)
            assert a.build_all() <= bound(len(w), k)


def test_random_against_brute_force() -> None:
    rng = random.Random(2002)
    for alphabet in ("ab", "acgt"):
        for _ in range(300):
            w = "".join(rng.choice(alphabet) for _ in range(rng.randint(0, 6)))
            k = rng.randint(0, 3)
            words = ["".join(rng.choice(alphabet) for _ in range(rng.randint(0, 8))) for _ in range(rng.randint(0, 12))]
            assert fuzzy_lookup(words, w, k) == sorted({x for x in words if lev(w, x) <= k})
            t = "".join(rng.choice(alphabet) for _ in range(rng.randint(0, 12)))
            assert LevenshteinAutomaton(w, k, search=True).find_ends(t) == brute_ends(w, t, k)

What the next level needs

  • the tests did not run in this build

Read the full page about Levenshtein automaton