Algorithm Search⌘K

LCP array in Python

For each adjacent pair in the suffix array, the length of their longest common prefix.

File
lcp_array.py, 95 lines
Level
I0 not checked yet
Toolchain
Python 3.14, pytest 9
Test framework
pytest
Run the tests
python3 -m pytest
lcp_array.pyRawGitHub
# SPDX-License-Identifier: MIT
# Part of https://algorithm.tamnd.com/lcp-array/. The authors are listed in prov/6RWY4Y.jsonl.

from collections.abc import Sequence


def suffix_array(text: Sequence) -> list[int]:
    """Return the suffix array of text by prefix doubling, in O(n log^2 n) time.

    Positions are 0-based. No sentinel is added: a proper prefix sorts before
    the longer string. Symbols only need a total order.
    """
    n = len(text)
    order = sorted(set(text))
    code = {c: i for i, c in enumerate(order)}
    rank = [code[c] for c in text]
    sa = list(range(n))
    k = 1
    while n > 1:
        key = [(rank[i], rank[i + k] if i + k < n else -1) for i in range(n)]
        sa.sort(key=lambda i: key[i])
        new = [0] * n
        for t in range(1, n):
            new[sa[t]] = new[sa[t - 1]] + (key[sa[t]] != key[sa[t - 1]])
        rank = new
        if rank[sa[-1]] == n - 1:
            break
        k *= 2
    return sa


class LcpArray:
    """The LCP array of a text, with a sparse table for O(1) LCP queries.

    lcp[0] = 0 and lcp[i] is the length of the longest common prefix of the
    suffixes sa[i - 1] and sa[i]. Positions and ranks are 0-based, and no
    sentinel is added. Building takes O(n log n) time and space (the sparse
    table); the LCP array alone takes O(n) time from the suffix array.
    """

    def __init__(self, text: Sequence, sa: list[int] | None = None) -> None:
        self.text = text
        n = self.n = len(text)
        self.sa = suffix_array(text) if sa is None else list(sa)
        self.rank = [0] * n
        for i in range(n):  # @line L1
            self.rank[self.sa[i]] = i
        self.lcp = [0] * n
        h = 0  # @line L2
        for p in range(n):  # @line L3
            if self.rank[p] == 0:  # @line L4
                h = 0
                continue
            q = self.sa[self.rank[p] - 1]  # @line L5
            while p + h < n and q + h < n and text[p + h] == text[q + h]:  # @line L6
                h += 1
            self.lcp[self.rank[p]] = h  # @line L7
            if h > 0:  # @line L8
                h -= 1
        # Sparse table: table[k][i] = min(lcp[i .. i + 2^k - 1]).
        self.table = [self.lcp[:]]
        k = 1
        while (1 << k) <= n:
            prev, half = self.table[-1], 1 << (k - 1)
            self.table.append([min(prev[i], prev[i + half]) for i in range(n - (1 << k) + 1)])
            k += 1

    def get(self, i: int) -> int:
        """Return lcp[i]."""
        return self.lcp[i]

    def lcp_ranks(self, i: int, j: int) -> int:
        """Return the LCP of the suffixes of ranks i and j, for 0 <= i, j < n."""
        if i == j:
            return self.n - self.sa[i]
        if i > j:
            i, j = j, i
        k = (j - i).bit_length() - 1  # @line L9
        return min(self.table[k][i + 1], self.table[k][j - (1 << k) + 1])  # @line L10

    def lce(self, p: int, q: int) -> int:
        """Return the LCP of the suffixes that start at text positions p and q (0 <= p, q <= n)."""
        if p == self.n or q == self.n:
            return 0
        if p == q:  # @line L11
            return self.n - p
        i, j = sorted((self.rank[p], self.rank[q]))  # @line L12
        return self.lcp_ranks(i, j)  # @line L13

    def check_invariant(self) -> None:
        for i in range(1, self.n):
            a, b = self.sa[i - 1], self.sa[i]
            h = self.lcp[i]
            assert self.text[a : a + h] == self.text[b : b + h]
            assert a + h == self.n or (b + h < self.n and self.text[a + h] < self.text[b + h])
test_lcp_array.py 68 lines
# SPDX-License-Identifier: MIT
# Part of https://algorithm.tamnd.com/lcp-array/. The authors are listed in prov/6RWY4Y.jsonl.

import random

from lcp_array import LcpArray, suffix_array


def naive_sa(t):
    return sorted(range(len(t)), key=lambda i: t[i:])


def naive_lcp(a, b):
    k = 0
    while k < len(a) and k < len(b) and a[k] == b[k]:
        k += 1
    return k


def texts(seed: int, count: int = 600, maxn: int = 14) -> list[str]:
    rng = random.Random(seed)
    out = ["", "a", "aa", "ab", "ba"]
    for n in range(1, 13):
        out += ["a" * n, "ab" * n, "aab" * n]
    for _ in range(count):
        sigma = rng.choice([1, 2, 4])
        out.append("".join(rng.choice("abcd"[:sigma]) for _ in range(rng.randrange(maxn + 1))))
    return out


def test_examples() -> None:
    x = LcpArray("banana")
    assert x.sa == [5, 3, 1, 0, 4, 2]
    assert x.lcp == [0, 1, 3, 0, 0, 2]
    x = LcpArray("mississippi")
    assert x.lcp == [0, 1, 1, 4, 0, 0, 1, 0, 2, 1, 3]
    assert x.lce(1, 4) == 4
    assert x.lcp_ranks(0, 3) == 1


def test_edges() -> None:
    x = LcpArray("")
    assert x.sa == [] and x.lcp == []
    assert x.lce(0, 0) == 0
    x = LcpArray("a")
    assert x.lcp == [0] and x.lce(0, 0) == 1
    assert LcpArray("aaaa").lcp == [0, 1, 2, 3]
    assert LcpArray([3, 1, 3, 1]).lcp == [0, 1, 0, 2]


def test_against_naive() -> None:
    for t in texts(7):
        x = LcpArray(t)
        x.check_invariant()
        assert x.sa == naive_sa(t)
        assert x.lcp == [0 if i == 0 else naive_lcp(t[x.sa[i - 1] :], t[x.sa[i] :]) for i in range(len(t))]
        for p in range(len(t) + 1):
            for q in range(len(t) + 1):
                assert x.lce(p, q) == naive_lcp(t[p:], t[q:])


def test_given_suffix_array() -> None:
    rng = random.Random(3)
    for _ in range(200):
        t = [rng.randrange(3) for _ in range(rng.randrange(20))]
        sa = naive_sa(t)
        assert suffix_array(t) == sa
        assert LcpArray(t, sa).lcp == LcpArray(t).lcp

What the next level needs

  • the tests did not run in this build

Read the full page about LCP array