Levenshtein automaton in Python
Deterministic automaton that accepts all strings within edit distance k of a fixed word.
- File
levenshtein_automaton.py, 136 lines- Level
- I0 not checked yet
- Toolchain
- Python 3.14, pytest 9
- Test framework
- pytest
- Run the tests
python3 -m pytest
# SPDX-License-Identifier: MIT
# Part of https://algorithm.tamnd.com/levenshtein-automaton/. The authors are listed in prov/CK7G4C.jsonl.
"""Levenshtein automaton of a word, built lazily from capped rows.
Edit model: unit-cost insertion, deletion and substitution (Levenshtein),
no transpositions. Indices are 0-based. A state is the row
(min(ed(word[0:i], x), k + 1)) for i = 0..m, where x is the input read so far.
In search mode the first entry of each row stays 0, so the automaton tracks
the Sellers column instead, and an accepting state after reading text[0:j]
reports the end position j.
"""
from typing import Optional
class LevenshteinAutomaton:
"""Deterministic automaton for {x : ed(word, x) <= k}, built on demand."""
def __init__(self, word: str, k: int, search: bool = False) -> None:
"""Make the start state. Precondition: k >= 0."""
self.word = word
self.k = k
self.search = search
self.cap = k + 1 # @line L1
self.symbols = set(word)
self.rows: list[tuple[int, ...]] = []
self.index: dict[tuple[int, ...], int] = {}
self.delta: dict[tuple[int, Optional[str]], int] = {}
first = tuple(min(i, self.cap) for i in range(len(word) + 1)) # @line L2
self.start = self._state(first)
def _state(self, row: tuple[int, ...]) -> int:
"""Return the id of a row, and make a new state for a new row."""
sid = self.index.get(row)
if sid is None:
sid = len(self.rows)
self.index[row] = sid
self.rows.append(row)
return sid
def step(self, state: int, ch: str) -> int:
"""Return the state after reading ch. A new row costs O(m) time."""
key = (state, ch if ch in self.symbols else None)
nxt = self.delta.get(key)
if nxt is not None:
return nxt
row = self.rows[state]
cap = self.cap
new = [0] * len(row)
new[0] = 0 if self.search else min(row[0] + 1, cap) # @line L3
for i in range(1, len(row)): # @line L4
sub = row[i - 1] + (self.word[i - 1] != ch)
new[i] = min(sub, row[i] + 1, new[i - 1] + 1, cap) # @line L5
nxt = self._state(tuple(new)) # @line L6
self.delta[key] = nxt
return nxt
def is_match(self, state: int) -> bool:
"""True when the input read so far is within distance k."""
return self.rows[state][-1] <= self.k # @line L7
def can_match(self, state: int) -> bool:
"""False exactly for the dead state, from which no input is accepted."""
return min(self.rows[state]) <= self.k # @line L8
def distance(self, state: int) -> int:
"""Return min(ed(word, x), k + 1) for the input x read so far."""
return self.rows[state][-1]
def accepts(self, s: str) -> bool:
"""Return True when ed(word, s) <= k (word mode)."""
state = self.start
for ch in s:
state = self.step(state, ch)
if not self.search and not self.can_match(state):
return False
return self.is_match(state)
def find_ends(self, text: str) -> list[int]:
"""Search mode: return the end positions j in 0..n of the matches."""
state = self.start
out = [0] if self.is_match(state) else []
for j, ch in enumerate(text, 1):
state = self.step(state, ch)
if self.is_match(state):
out.append(j)
return out
def build_all(self) -> int:
"""Build every reachable state and return the number of states."""
classes: list[str] = sorted(self.symbols)
other = "\x00"
while other in self.symbols:
other = chr(ord(other) + 1)
classes.append(other)
todo = [self.start]
seen = {self.start}
while todo:
s = todo.pop()
for ch in classes:
t = self.step(s, ch)
if t not in seen:
seen.add(t)
todo.append(t)
return len(seen)
def fuzzy_lookup(words: list[str], word: str, k: int) -> list[str]:
"""Return the sorted distinct words of the list within distance k of word,
by a walk over a trie of the list in step with the automaton."""
children: list[dict[str, int]] = [{}]
final: list[Optional[str]] = [None]
for w in words:
u = 0
for ch in w:
v = children[u].get(ch)
if v is None:
v = len(children)
children[u][ch] = v
children.append({})
final.append(None)
u = v
final[u] = w
auto = LevenshteinAutomaton(word, k)
out = []
stack = [(0, auto.start)] # @line L9
while stack: # @line L10
u, s = stack.pop()
if final[u] is not None and auto.is_match(s): # @line L11
out.append(final[u])
for ch, v in children[u].items(): # @line L12
t = auto.step(s, ch)
if auto.can_match(t):
stack.append((v, t))
return sorted(out)
test_levenshtein_automaton.py 75 lines
# SPDX-License-Identifier: MIT
# Part of https://algorithm.tamnd.com/levenshtein-automaton/. The authors are listed in prov/CK7G4C.jsonl.
import itertools
import random
from levenshtein_automaton import LevenshteinAutomaton, fuzzy_lookup
def lev(a: str, b: str) -> int:
d = [[i + j if i * j == 0 else 0 for j in range(len(b) + 1)] for i in range(len(a) + 1)]
for i in range(1, len(a) + 1):
for j in range(1, len(b) + 1):
d[i][j] = min(d[i - 1][j] + 1, d[i][j - 1] + 1, d[i - 1][j - 1] + (a[i - 1] != b[j - 1]))
return d[len(a)][len(b)]
def brute_ends(p: str, t: str, k: int) -> list[int]:
return [j for j in range(len(t) + 1) if min(lev(p, t[h:j]) for h in range(j + 1)) <= k]
def bound(m: int, k: int) -> int:
return (m + k + 1) * (k + 2) * 3 ** (2 * k) + 1
def test_example() -> None:
a = LevenshteinAutomaton("cat", 1)
assert a.accepts("cat") and a.accepts("cart") and a.accepts("at") and a.accepts("coat")
assert not a.accepts("act") and not a.accepts("dog") and not a.accepts("")
words = ["cat", "cart", "act", "at", "coat", "cut", "dog", "cats", "scat", "a", ""]
assert fuzzy_lookup(words, "cat", 1) == ["at", "cart", "cat", "cats", "coat", "cut", "scat"]
assert fuzzy_lookup(words, "cat", 0) == ["cat"]
s = LevenshteinAutomaton("GATAA", 1, search=True)
assert s.find_ends("CAGATAAGAGAA") == [6, 7, 8, 12]
def test_edges() -> None:
a = LevenshteinAutomaton("", 0)
assert a.accepts("") and not a.accepts("a")
assert LevenshteinAutomaton("", 2).accepts("ab")
assert not LevenshteinAutomaton("ab", 1).accepts("")
assert LevenshteinAutomaton("", 0, search=True).find_ends("ab") == [0, 1, 2]
assert LevenshteinAutomaton("ab", 0, search=True).find_ends("") == []
assert fuzzy_lookup([], "ab", 1) == []
assert fuzzy_lookup(["", "a"], "", 0) == [""]
def test_accepts_all_short_strings() -> None:
rng = random.Random(1985)
for alphabet in ("ab", "acgt"):
for _ in range(60):
w = "".join(rng.choice(alphabet) for _ in range(rng.randint(0, 5)))
k = rng.randint(0, 2)
a = LevenshteinAutomaton(w, k)
for n in range(0, 6 if alphabet == "ab" else 4):
for t in itertools.product(alphabet + "x", repeat=n):
x = "".join(t)
assert a.accepts(x) == (lev(w, x) <= k)
s = a.start
for ch in x:
s = a.step(s, ch)
assert a.distance(s) == min(lev(w, x), k + 1)
assert a.build_all() <= bound(len(w), k)
def test_random_against_brute_force() -> None:
rng = random.Random(2002)
for alphabet in ("ab", "acgt"):
for _ in range(300):
w = "".join(rng.choice(alphabet) for _ in range(rng.randint(0, 6)))
k = rng.randint(0, 3)
words = ["".join(rng.choice(alphabet) for _ in range(rng.randint(0, 8))) for _ in range(rng.randint(0, 12))]
assert fuzzy_lookup(words, w, k) == sorted({x for x in words if lev(w, x) <= k})
t = "".join(rng.choice(alphabet) for _ in range(rng.randint(0, 12)))
assert LevenshteinAutomaton(w, k, search=True).find_ends(t) == brute_ends(w, t, k)
What the next level needs
- the tests did not run in this build