Longest common substring via suffix tree in Python
Find the longest common substring by marking leaves of a generalized suffix tree and taking the deepest node with both marks.
- File
lcs_via_suffix_tree.py, 125 lines- Level
- I0 not checked yet
- Toolchain
- Python 3.14, pytest 9
- Test framework
- pytest
- Run the tests
python3 -m pytest
# SPDX-License-Identifier: MIT
# Part of https://algorithm.tamnd.com/lcs-via-suffix-tree/. The authors are listed in prov/2AY22B.jsonl.
"""Longest common substring of two strings with a generalized suffix tree.
The tree is built with Ukkonen's online algorithm on S = A + [sep1] + B + [sep2],
where the separators are symbols that occur in neither string. A postorder walk
marks each node with the set of strings whose suffixes end below it; the deepest
node marked by both strings spells the answer.
"""
SEP1 = -1
SEP2 = -2
class SuffixTree:
"""Suffix tree of a sequence of ints whose last symbol is unique (Ukkonen)."""
def __init__(self, s: list[int]) -> None:
self.s = s
self.start: list[int] = [-1]
self.end: list[int | None] = [-1] # None marks a leaf: its edge ends at len(s)
self.link: list[int] = [0]
self.children: list[dict[int, int]] = [{}]
self._build()
def _new_node(self, start: int, end: int | None) -> int:
self.start.append(start)
self.end.append(end)
self.link.append(0)
self.children.append({})
return len(self.start) - 1
def edge_len(self, v: int, i: int) -> int:
"""Length of the edge into v while position i is being added (i + 1 for leaves)."""
e = self.end[v]
return (i + 1 if e is None else e) - self.start[v]
def _build(self) -> None:
s = self.s
node, edge, length, remainder = 0, 0, 0, 0
for i, c in enumerate(s):
remainder += 1
last_new = -1
while remainder > 0:
if length == 0:
edge = i
a = s[edge]
ch = self.children[node]
if a not in ch:
ch[a] = self._new_node(i, None)
if last_new != -1:
self.link[last_new] = node
last_new = -1
else:
nxt = ch[a]
el = self.edge_len(nxt, i)
if length >= el:
edge += el
length -= el
node = nxt
continue
if s[self.start[nxt] + length] == c:
if last_new != -1:
self.link[last_new] = node
length += 1
break
split = self._new_node(self.start[nxt], self.start[nxt] + length)
ch[a] = split
self.children[split][c] = self._new_node(i, None)
self.start[nxt] += length
self.children[split][s[self.start[nxt]]] = nxt
if last_new != -1:
self.link[last_new] = split
last_new = split
remainder -= 1
if node == 0 and length > 0:
length -= 1
edge = i - remainder + 1
elif node != 0:
node = self.link[node]
def longest_common_substring(a: str, b: str) -> tuple[int, int, int]:
"""Return (length, i, j) with a[i:i+length] == b[j:j+length] of maximum length.
For length 0 the result is (0, 0, 0). Among the longest answers, the node
found first in the walk is returned.
"""
s = [ord(c) for c in a] + [SEP1] + [ord(c) for c in b] + [SEP2] # @line L1
tree = SuffixTree(s) # @line L2
n, na = len(s), len(a)
nnodes = len(tree.start)
depth = [0] * nnodes
mark = [0] * nnodes
pos_a = [-1] * nnodes # start in S of some suffix below the node that starts in a
pos_b = [-1] * nnodes
best, best_node = 0, -1
stack: list[tuple[int, bool]] = [(0, False)]
while stack: # @line L3
v, done = stack.pop()
if not done:
stack.append((v, True))
for w in tree.children[v].values():
depth[w] = depth[v] + tree.edge_len(w, n - 1)
stack.append((w, False))
continue
if not tree.children[v]: # @line L4
st = n - depth[v]
if st <= na:
mark[v], pos_a[v] = 1, st
else:
mark[v], pos_b[v] = 2, st
else: # @line L5
for w in tree.children[v].values():
mark[v] |= mark[w]
if pos_a[v] < 0:
pos_a[v] = pos_a[w]
if pos_b[v] < 0:
pos_b[v] = pos_b[w]
if mark[v] == 3 and depth[v] > best: # @line L6
best, best_node = depth[v], v
if best_node < 0: # @line L7
return (0, 0, 0)
return (best, pos_a[best_node], pos_b[best_node] - na - 1)
test_lcs_via_suffix_tree.py 72 lines
# SPDX-License-Identifier: MIT
# Part of https://algorithm.tamnd.com/lcs-via-suffix-tree/. The authors are listed in prov/2AY22B.jsonl.
import random
from lcs_via_suffix_tree import SuffixTree, longest_common_substring
def lcs_length_dp(a: str, b: str) -> int:
best = 0
prev = [0] * (len(b) + 1)
for i in range(1, len(a) + 1):
cur = [0] * (len(b) + 1)
for j in range(1, len(b) + 1):
if a[i - 1] == b[j - 1]:
cur[j] = prev[j - 1] + 1
best = max(best, cur[j])
prev = cur
return best
def check(a: str, b: str) -> None:
k, i, j = longest_common_substring(a, b)
assert k == lcs_length_dp(a, b)
assert a[i : i + k] == b[j : j + k]
def leaf_suffixes(s: list[int]) -> list[int]:
t = SuffixTree(s)
n = len(s)
out = []
stack = [(0, 0)]
while stack:
v, d = stack.pop()
if not t.children[v]:
out.append(n - d)
for w in t.children[v].values():
stack.append((w, d + t.edge_len(w, n - 1)))
return sorted(out)
def test_example() -> None:
k, i, j = longest_common_substring("xabxac", "abcabxabcd")
assert (k, "xabxac"[i : i + k]) == (4, "abxa")
k, i, j = longest_common_substring("banana", "ananas")
assert "banana"[i : i + k] == "anana"
def test_edges() -> None:
assert longest_common_substring("", "") == (0, 0, 0)
assert longest_common_substring("abc", "")[0] == 0
assert longest_common_substring("abc", "xyz")[0] == 0
assert longest_common_substring("aaaa", "aa")[0] == 2
k, i, j = longest_common_substring("same", "same")
assert (k, i, j) == (4, 0, 0)
check("abab", "baba")
def test_tree_has_all_suffixes() -> None:
rng = random.Random(77)
for _ in range(200):
s = [rng.randint(0, 2) for _ in range(rng.randint(0, 30))] + [-2]
assert leaf_suffixes(s) == list(range(len(s)))
def test_random_against_dp() -> None:
rng = random.Random(2024)
for _ in range(600):
alpha = "ab" if rng.random() < 0.5 else "abcd"
a = "".join(rng.choice(alpha) for _ in range(rng.randint(0, 25)))
b = "".join(rng.choice(alpha) for _ in range(rng.randint(0, 25)))
check(a, b)
What the next level needs
- the tests did not run in this build
Read the full page about Longest common substring via suffix tree