Blumer et al. suffix automaton algorithm in Python
Online linear-time construction of the suffix automaton by extending the automaton one character at a time.
- File
blumer_algorithm.py, 114 lines- Level
- I0 not checked yet
- Toolchain
- Python 3.14, pytest 9
- Test framework
- pytest
- Run the tests
python3 -m pytest
# SPDX-License-Identifier: MIT
# Part of https://algorithm.tamnd.com/blumer-algorithm/. The authors are listed in prov/EMPS9K.jsonl.
class SuffixAutomaton:
"""The suffix automaton (DAWG) of a string, built online by the algorithm of Blumer et al.
State 0 is the initial state. For each state v, length[v] is the length of the
longest string that reaches v, link[v] is its suffix link (-1 for the initial state)
and nxt[v] maps a symbol to the target state.
"""
def __init__(self, s: str = "") -> None:
self.length: list[int] = [0]
self.link: list[int] = [-1]
self.nxt: list[dict[str, int]] = [{}]
self.last = 0
self.text: list[str] = []
for c in s:
self.extend(c)
def _new_state(self, length: int, link: int, nxt: dict[str, int]) -> int:
self.length.append(length)
self.link.append(link)
self.nxt.append(nxt)
return len(self.length) - 1
def extend(self, c: str) -> None:
"""Append the symbol c to the text and update the automaton."""
self.text.append(c)
cur = self._new_state(self.length[self.last] + 1, -1, {}) # @line L1
p = self.last
while p != -1 and c not in self.nxt[p]: # @line L2
self.nxt[p][c] = cur
p = self.link[p]
if p == -1: # @line L3
self.link[cur] = 0
else:
q = self.nxt[p][c]
if self.length[q] == self.length[p] + 1: # @line L4
self.link[cur] = q
else:
clone = self._new_state(self.length[p] + 1, self.link[q], dict(self.nxt[q])) # @line L5
while p != -1 and self.nxt[p].get(c) == q: # @line L6
self.nxt[p][c] = clone
p = self.link[p]
self.link[q] = clone # @line L7
self.link[cur] = clone
self.last = cur # @line L8
def num_states(self) -> int:
return len(self.length)
def num_transitions(self) -> int:
return sum(len(d) for d in self.nxt)
def contains(self, w: str) -> bool:
"""Is w a substring of the text?"""
v = 0
for c in w:
if c not in self.nxt[v]:
return False
v = self.nxt[v][c]
return True
def is_suffix(self, w: str) -> bool:
"""Is w a suffix of the text? The terminal states are those on the link path from last."""
v = 0
for c in w:
if c not in self.nxt[v]:
return False
v = self.nxt[v][c]
u = self.last
while u != -1:
if u == v:
return True
u = self.link[u]
return False
def count_distinct_substrings(self) -> int:
"""The number of distinct nonempty substrings: the sum of len(v) - len(link(v))."""
return sum(self.length[v] - self.length[self.link[v]] for v in range(1, len(self.length)))
def check_invariant(self) -> None:
n = len(self.text)
k = len(self.length)
assert self.length[0] == 0 and self.link[0] == -1
assert self.length[self.last] == n
for v in range(1, k):
u = self.link[v]
assert 0 <= u < k and self.length[u] < self.length[v], "suffix link must shorten"
for c, w in self.nxt[v].items():
assert self.length[w] >= self.length[v] + 1
for c, w in self.nxt[0].items():
assert self.length[w] >= 1
if n >= 2:
assert k <= 2 * n - 1
if n >= 3:
assert self.num_transitions() <= 3 * n - 4
# every state is reached by a string of each length in (len(link), len], here checked via
# the longest string: following the transitions of a longest string reaches the state.
# Compute one longest string per state by a walk in topological (length) order.
word: list[str | None] = [None] * k
word[0] = ""
for v in sorted(range(k), key=lambda x: self.length[x]):
wv = word[v]
assert wv is not None, "state not reachable"
for c, w in self.nxt[v].items():
if self.length[w] == self.length[v] + 1 and word[w] is None:
word[w] = wv + c
text = "".join(self.text)
for v in range(k):
wv = word[v]
assert wv is not None and len(wv) == self.length[v] and wv in text
test_blumer_algorithm.py 61 lines
# SPDX-License-Identifier: MIT
# Part of https://algorithm.tamnd.com/blumer-algorithm/. The authors are listed in prov/EMPS9K.jsonl.
import random
from blumer_algorithm import SuffixAutomaton
def endpos_classes(s: str) -> int:
"""The number of states of the minimal suffix automaton: distinct end-position sets, plus the initial state."""
sets = set()
for i in range(len(s)):
for j in range(i + 1, len(s) + 1):
w = s[i:j]
sets.add(frozenset(k + len(w) for k in range(len(s) - len(w) + 1) if s.startswith(w, k)))
return len(sets) + 1
def test_example() -> None:
a = SuffixAutomaton()
sizes = []
for c in "abbb":
a.extend(c)
a.check_invariant()
sizes.append(a.num_states())
assert sizes == [2, 3, 5, 7]
assert a.length == [0, 1, 2, 3, 1, 4, 2]
assert a.link == [-1, 0, 4, 6, 0, 6, 4]
assert a.nxt == [{"a": 1, "b": 4}, {"b": 2}, {"b": 3}, {"b": 5}, {"b": 6}, {}, {"b": 5}]
assert a.count_distinct_substrings() == 7
assert a.contains("abb") and not a.contains("ba")
assert a.is_suffix("bb") and not a.is_suffix("ab")
def test_edges() -> None:
a = SuffixAutomaton("")
a.check_invariant()
assert a.num_states() == 1 and a.contains("") and a.is_suffix("")
a = SuffixAutomaton("aaaa")
a.check_invariant()
assert a.num_states() == 5 and a.count_distinct_substrings() == 4
# the worst case for states: a b^(n-1) has 2n - 1 states
n = 9
assert SuffixAutomaton("a" + "b" * (n - 1)).num_states() == 2 * n - 1
# the worst case for transitions: a b^(n-2) c has 3n - 4 transitions
assert SuffixAutomaton("a" + "b" * (n - 2) + "c").num_transitions() == 3 * n - 4
def test_random_against_naive() -> None:
rng = random.Random(1985)
for case in range(600):
s = "".join(rng.choice("ab" if case % 2 else "abc") for _ in range(rng.randint(0, 14)))
a = SuffixAutomaton(s)
a.check_invariant()
subs = {s[i:j] for i in range(len(s)) for j in range(i + 1, len(s) + 1)}
assert a.count_distinct_substrings() == len(subs)
assert a.num_states() == endpos_classes(s)
for _ in range(5):
w = "".join(rng.choice("abc") for _ in range(rng.randint(0, 4)))
assert a.contains(w) == (w in subs or w == "")
assert a.is_suffix(w) == s.endswith(w)
What the next level needs
- the tests did not run in this build
Read the full page about Blumer et al. suffix automaton algorithm