Algorithm Search⌘K

Blumer et al. suffix automaton algorithm in Python

Online linear-time construction of the suffix automaton by extending the automaton one character at a time.

File
blumer_algorithm.py, 114 lines
Level
I0 not checked yet
Toolchain
Python 3.14, pytest 9
Test framework
pytest
Run the tests
python3 -m pytest
blumer_algorithm.pyRawGitHub
# SPDX-License-Identifier: MIT
# Part of https://algorithm.tamnd.com/blumer-algorithm/. The authors are listed in prov/EMPS9K.jsonl.


class SuffixAutomaton:
    """The suffix automaton (DAWG) of a string, built online by the algorithm of Blumer et al.

    State 0 is the initial state. For each state v, length[v] is the length of the
    longest string that reaches v, link[v] is its suffix link (-1 for the initial state)
    and nxt[v] maps a symbol to the target state.
    """

    def __init__(self, s: str = "") -> None:
        self.length: list[int] = [0]
        self.link: list[int] = [-1]
        self.nxt: list[dict[str, int]] = [{}]
        self.last = 0
        self.text: list[str] = []
        for c in s:
            self.extend(c)

    def _new_state(self, length: int, link: int, nxt: dict[str, int]) -> int:
        self.length.append(length)
        self.link.append(link)
        self.nxt.append(nxt)
        return len(self.length) - 1

    def extend(self, c: str) -> None:
        """Append the symbol c to the text and update the automaton."""
        self.text.append(c)
        cur = self._new_state(self.length[self.last] + 1, -1, {})  # @line L1
        p = self.last
        while p != -1 and c not in self.nxt[p]:  # @line L2
            self.nxt[p][c] = cur
            p = self.link[p]
        if p == -1:  # @line L3
            self.link[cur] = 0
        else:
            q = self.nxt[p][c]
            if self.length[q] == self.length[p] + 1:  # @line L4
                self.link[cur] = q
            else:
                clone = self._new_state(self.length[p] + 1, self.link[q], dict(self.nxt[q]))  # @line L5
                while p != -1 and self.nxt[p].get(c) == q:  # @line L6
                    self.nxt[p][c] = clone
                    p = self.link[p]
                self.link[q] = clone  # @line L7
                self.link[cur] = clone
        self.last = cur  # @line L8

    def num_states(self) -> int:
        return len(self.length)

    def num_transitions(self) -> int:
        return sum(len(d) for d in self.nxt)

    def contains(self, w: str) -> bool:
        """Is w a substring of the text?"""
        v = 0
        for c in w:
            if c not in self.nxt[v]:
                return False
            v = self.nxt[v][c]
        return True

    def is_suffix(self, w: str) -> bool:
        """Is w a suffix of the text? The terminal states are those on the link path from last."""
        v = 0
        for c in w:
            if c not in self.nxt[v]:
                return False
            v = self.nxt[v][c]
        u = self.last
        while u != -1:
            if u == v:
                return True
            u = self.link[u]
        return False

    def count_distinct_substrings(self) -> int:
        """The number of distinct nonempty substrings: the sum of len(v) - len(link(v))."""
        return sum(self.length[v] - self.length[self.link[v]] for v in range(1, len(self.length)))

    def check_invariant(self) -> None:
        n = len(self.text)
        k = len(self.length)
        assert self.length[0] == 0 and self.link[0] == -1
        assert self.length[self.last] == n
        for v in range(1, k):
            u = self.link[v]
            assert 0 <= u < k and self.length[u] < self.length[v], "suffix link must shorten"
            for c, w in self.nxt[v].items():
                assert self.length[w] >= self.length[v] + 1
        for c, w in self.nxt[0].items():
            assert self.length[w] >= 1
        if n >= 2:
            assert k <= 2 * n - 1
        if n >= 3:
            assert self.num_transitions() <= 3 * n - 4
        # every state is reached by a string of each length in (len(link), len], here checked via
        # the longest string: following the transitions of a longest string reaches the state.
        # Compute one longest string per state by a walk in topological (length) order.
        word: list[str | None] = [None] * k
        word[0] = ""
        for v in sorted(range(k), key=lambda x: self.length[x]):
            wv = word[v]
            assert wv is not None, "state not reachable"
            for c, w in self.nxt[v].items():
                if self.length[w] == self.length[v] + 1 and word[w] is None:
                    word[w] = wv + c
        text = "".join(self.text)
        for v in range(k):
            wv = word[v]
            assert wv is not None and len(wv) == self.length[v] and wv in text
test_blumer_algorithm.py 61 lines
# SPDX-License-Identifier: MIT
# Part of https://algorithm.tamnd.com/blumer-algorithm/. The authors are listed in prov/EMPS9K.jsonl.

import random

from blumer_algorithm import SuffixAutomaton


def endpos_classes(s: str) -> int:
    """The number of states of the minimal suffix automaton: distinct end-position sets, plus the initial state."""
    sets = set()
    for i in range(len(s)):
        for j in range(i + 1, len(s) + 1):
            w = s[i:j]
            sets.add(frozenset(k + len(w) for k in range(len(s) - len(w) + 1) if s.startswith(w, k)))
    return len(sets) + 1


def test_example() -> None:
    a = SuffixAutomaton()
    sizes = []
    for c in "abbb":
        a.extend(c)
        a.check_invariant()
        sizes.append(a.num_states())
    assert sizes == [2, 3, 5, 7]
    assert a.length == [0, 1, 2, 3, 1, 4, 2]
    assert a.link == [-1, 0, 4, 6, 0, 6, 4]
    assert a.nxt == [{"a": 1, "b": 4}, {"b": 2}, {"b": 3}, {"b": 5}, {"b": 6}, {}, {"b": 5}]
    assert a.count_distinct_substrings() == 7
    assert a.contains("abb") and not a.contains("ba")
    assert a.is_suffix("bb") and not a.is_suffix("ab")


def test_edges() -> None:
    a = SuffixAutomaton("")
    a.check_invariant()
    assert a.num_states() == 1 and a.contains("") and a.is_suffix("")
    a = SuffixAutomaton("aaaa")
    a.check_invariant()
    assert a.num_states() == 5 and a.count_distinct_substrings() == 4
    # the worst case for states: a b^(n-1) has 2n - 1 states
    n = 9
    assert SuffixAutomaton("a" + "b" * (n - 1)).num_states() == 2 * n - 1
    # the worst case for transitions: a b^(n-2) c has 3n - 4 transitions
    assert SuffixAutomaton("a" + "b" * (n - 2) + "c").num_transitions() == 3 * n - 4


def test_random_against_naive() -> None:
    rng = random.Random(1985)
    for case in range(600):
        s = "".join(rng.choice("ab" if case % 2 else "abc") for _ in range(rng.randint(0, 14)))
        a = SuffixAutomaton(s)
        a.check_invariant()
        subs = {s[i:j] for i in range(len(s)) for j in range(i + 1, len(s) + 1)}
        assert a.count_distinct_substrings() == len(subs)
        assert a.num_states() == endpos_classes(s)
        for _ in range(5):
            w = "".join(rng.choice("abc") for _ in range(rng.randint(0, 4)))
            assert a.contains(w) == (w in subs or w == "")
            assert a.is_suffix(w) == s.endswith(w)

What the next level needs

  • the tests did not run in this build

Read the full page about Blumer et al. suffix automaton algorithm