Algorithm Search⌘K

Chang-Lawler algorithm in Python

Sublinear expected-time k-differences matching for patterns that are long relative to k.

File
chang_lawler_algorithm.py, 122 lines
Level
I0 not checked yet
Toolchain
Python 3.14, pytest 9
Test framework
pytest
Run the tests
python3 -m pytest
chang_lawler_algorithm.pyRawGitHub
# SPDX-License-Identifier: MIT
# Part of https://algorithm.tamnd.com/chang-lawler-algorithm/. The authors are listed in prov/K5VMHF.jsonl.

"""The sublinear expected time (SET) algorithm of Chang and Lawler.

Edit model: unit-cost insertion, deletion and substitution (Levenshtein),
no transpositions. Indices are 0-based. The result is the sorted list of end
positions j in 0..len(text) such that some substring text[h:j] has edit
distance at most k to the pattern.

The text is cut into blocks of length b = (m - k) // 2. From the start of a
block the algorithm makes k + 1 jumps, each over a maximal substring of the
pattern plus one symbol, with the suffix automaton of the pattern. If the
jumps stay inside the block, no match contains the block. Sellers' dynamic
program runs only on the windows around the other blocks. All loops are
iterative.
"""


class SuffixAutomaton:
    """The suffix automaton of a string: it accepts exactly its substrings."""

    def __init__(self, s: str) -> None:
        self.next: list[dict[str, int]] = [{}]
        link = [-1]
        length = [0]
        last = 0
        for c in s:
            cur = len(self.next)
            self.next.append({})
            length.append(length[last] + 1)
            link.append(0)
            p = last
            while p != -1 and c not in self.next[p]:
                self.next[p][c] = cur
                p = link[p]
            if p != -1:
                q = self.next[p][c]
                if length[p] + 1 == length[q]:
                    link[cur] = q
                else:
                    clone = len(self.next)
                    self.next.append(dict(self.next[q]))
                    length.append(length[p] + 1)
                    link.append(link[q])
                    while p != -1 and self.next[p].get(c) == q:
                        self.next[p][c] = clone
                        p = link[p]
                    link[q] = clone
                    link[cur] = clone
            last = cur

    def longest_prefix(self, text: str, p: int) -> int:
        """Return ms(p): the length of the longest prefix of text[p:] that is
        a substring of the pattern. Cost: O(ms(p) + 1)."""
        state, j = 0, p
        while j < len(text) and text[j] in self.next[state]:
            state = self.next[state][text[j]]
            j += 1
        return j - p


def sellers(pattern: str, text: str, k: int) -> list[int]:
    """End positions j in 0..len(text) of substrings within distance k."""
    m = len(pattern)
    col = list(range(m + 1))
    ends = [0] if col[m] <= k else []
    for j in range(1, len(text) + 1):
        t = text[j - 1]
        prev, col[0] = col[0], 0
        for i in range(1, m + 1):
            new = min(col[i] + 1, col[i - 1] + 1, prev + (pattern[i - 1] != t))
            prev, col[i] = col[i], new
        if col[m] <= k:
            ends.append(j)
    return ends


def live_blocks(pattern: str, text: str, k: int) -> list[int]:
    """Return the numbers t of the blocks that the jumps do not rule out."""
    m, n = len(pattern), len(text)
    b = (m - k) // 2
    sa = SuffixAutomaton(pattern)
    live = []
    for t in range(n // b):
        end = (t + 1) * b
        p = t * b  # p_0 = tb, then p_(r+1) = p_r + ms(p_r) + 1
        for _ in range(k + 1):  # @line L4
            if p >= end:  # then p_(k+1) > end
                p = end + 1
                break
            p += sa.longest_prefix(text, p) + 1
        if p > end:  # @line L5
            live.append(t)
    return live


def chang_lawler(pattern: str, text: str, k: int) -> list[int]:
    """Return the end positions of the approximate matches of pattern in text.

    Precondition: k >= 0. Cost: O(m + n) time for the jumps plus Sellers'
    O(m) time per text symbol of the windows around live blocks; O(m)
    expected work per block on random text when k is small against m.
    """
    if k < 0:
        raise ValueError("k must be at least 0")
    m, n = len(pattern), len(text)
    if m - k < 2:  # @line L1
        return sellers(pattern, text, k)
    b = (m - k) // 2  # @line L2
    windows: list[list[int]] = []
    for t in live_blocks(pattern, text, k):  # @line L3
        lo = max(0, (t + 1) * b - m - k)  # @line L6
        hi = min(n, t * b + m + k)
        if windows and lo <= windows[-1][1]:  # @line L7
            windows[-1][1] = hi
        else:
            windows.append([lo, hi])
    ends: list[int] = []
    for lo, hi in windows:  # @line L8
        ends.extend(lo + j for j in sellers(pattern, text[lo:hi], k))
    return sorted(set(ends))  # @line L9
test_chang_lawler_algorithm.py 86 lines
# SPDX-License-Identifier: MIT
# Part of https://algorithm.tamnd.com/chang-lawler-algorithm/. The authors are listed in prov/K5VMHF.jsonl.

import random

import pytest
from chang_lawler_algorithm import SuffixAutomaton, chang_lawler, live_blocks


def edit_distance(a: str, b: str) -> int:
    prev = list(range(len(b) + 1))
    for i, x in enumerate(a, 1):
        cur = [i]
        for j, y in enumerate(b, 1):
            cur.append(min(prev[j] + 1, cur[j - 1] + 1, prev[j - 1] + (x != y)))
        prev = cur
    return prev[-1]


def brute(pattern: str, text: str, k: int) -> list[int]:
    return [
        j for j in range(len(text) + 1)
        if min(edit_distance(pattern, text[h:j]) for h in range(j + 1)) <= k
    ]


def test_example() -> None:
    assert chang_lawler("GATAA", "CAGATAAGAGAA", 1) == [6, 7, 8, 12]
    assert chang_lawler("survey", "surgery", 2) == [5, 6, 7]
    assert chang_lawler("abc", "xxabcxx", 0) == [5]


def test_edges() -> None:
    assert chang_lawler("", "", 0) == [0]
    assert chang_lawler("", "ab", 0) == [0, 1, 2]
    assert chang_lawler("ab", "", 1) == []
    assert chang_lawler("ab", "", 2) == [0]
    assert chang_lawler("abcd", "", 0) == []
    with pytest.raises(ValueError):
        chang_lawler("a", "a", -1)


def test_suffix_automaton() -> None:
    rng = random.Random(3)
    for _ in range(300):
        p = "".join(rng.choice("acgt") for _ in range(rng.randint(0, 12)))
        t = "".join(rng.choice("acgt") for _ in range(rng.randint(0, 20)))
        sa = SuffixAutomaton(p)
        for i in range(len(t) + 1):
            best = max(L for L in range(len(t) - i + 1) if t[i:i + L] in p)
            assert sa.longest_prefix(t, i) == best


@pytest.mark.parametrize("alphabet", ["ab", "acgt"])
def test_random_against_brute_force(alphabet: str) -> None:
    rng = random.Random(1106 + len(alphabet))
    pruned = 0
    for _ in range(800):
        m = rng.randint(0, 12)
        n = rng.randint(0, 28)
        pattern = "".join(rng.choice(alphabet) for _ in range(m))
        text = "".join(rng.choice(alphabet) for _ in range(n))
        if m and rng.random() < 0.3 and n >= m:
            i = rng.randint(0, n - m)
            text = text[:i] + pattern + text[i + m:]
        k = rng.randint(0, max(0, m // 2))
        if m - k >= 2 and n // ((m - k) // 2) > len(live_blocks(pattern, text, k)):
            pruned += 1
        assert chang_lawler(pattern, text, k) == brute(pattern, text, k)
    assert pruned > 50


def test_long_random_text() -> None:
    rng = random.Random(11)
    for _ in range(40):
        pattern = "".join(rng.choice("acgt") for _ in range(30))
        text = "".join(rng.choice("acgt") for _ in range(300))
        for _ in range(3):
            i = rng.randint(0, 270)
            piece = list(pattern)
            for _ in range(rng.randint(0, 3)):
                piece[rng.randrange(30)] = rng.choice("acgt")
            text = text[:i] + "".join(piece) + text[i + 30:]
        for k in (0, 2, 4):
            from chang_lawler_algorithm import sellers
            assert chang_lawler(pattern, text, k) == sellers(pattern, text, k)

What the next level needs

  • the tests did not run in this build

Read the full page about Chang-Lawler algorithm