Integrations

Framework adapters. Each module needs its extra (see Install).

hf

Hugging Face datasets adapter (pip install squeakycleantext[datasets]).

SCTMap

SCTMap(
    columns: Sequence[str] = ("text",),
    output: str = "overwrite",
    config: TextCleanerConfig | str | None = None,
    score: bool = False,
    *,
    schema: FieldSchema | None = None,
    context: DetectionContext | None = None,
    registry_factory: Callable[
        [TextCleanerConfig], DetectorRegistry
    ]
    | None = None,
)

Picklable batched map function; pickles only its config, never the cleaner.

columns instance-attribute

columns = as_columns(columns)

output instance-attribute

output = output

score instance-attribute

score = score

handle instance-attribute

handle = CleanerHandle(
    config, registry_factory=registry_factory
)

clean_dataset

clean_dataset(
    ds: Any,
    *,
    columns: Sequence[str] = ("text",),
    output: str = "overwrite",
    config: TextCleanerConfig | str | None = None,
    batch_size: int = 64,
    num_proc: int | None = None,
    score: bool = False,
    schema: FieldSchema | None = None,
    context: DetectionContext | None = None,
    registry_factory: Callable[
        [TextCleanerConfig], DetectorRegistry
    ]
    | None = None,
    **map_kwargs: Any,
) -> Any

Return ds with columns cleaned (Dataset or IterableDataset).

Run it under if __name__ == "__main__": when num_proc > 1. Custom registries bypass cache reuse unless new_fingerprint supplies a caller-owned identity for the complete input, configuration, provider and model behavior.

datatrove

DataTrove pipeline step (pip install squeakycleantext[datatrove]).

SCTCleaner

SCTCleaner(
    columns: Sequence[str] = ("text",),
    output: str = "overwrite",
    config: TextCleanerConfig | str | None = None,
    batch_size: int = 64,
    *,
    schema: FieldSchema | None = None,
    context: DetectionContext | None = None,
    registry_factory: Callable[
        [TextCleanerConfig], DetectorRegistry
    ]
    | None = None,
)

Bases: PipelineStep

Clean doc.text (and listed doc.metadata keys); doc.metadata["sct"] gets the signals.

Place it after language and quality filters. The cleaner is built inside each worker on first use.

name class-attribute instance-attribute

name = 'SCT'

type class-attribute instance-attribute

type = '🧹 - CLEAN'

columns instance-attribute

columns = as_columns(columns)

output instance-attribute

output = output

batch_size instance-attribute

batch_size = batch_size

handle instance-attribute

handle = CleanerHandle(
    config, registry_factory=registry_factory
)

run

run(
    data: DocumentsPipeline,
    rank: int = 0,
    world_size: int = 1,
) -> DocumentsPipeline

langchain

LangChain document transformer (pip install squeakycleantext[langchain]).

SCTDocumentTransformer

SCTDocumentTransformer(
    columns: Sequence[str] = ("page_content",),
    output: str = "overwrite",
    config: TextCleanerConfig | str | None = None,
    *,
    schema: FieldSchema | None = None,
    context: DetectionContext | None = None,
    registry_factory: Callable[
        [TextCleanerConfig], DetectorRegistry
    ]
    | None = None,
)

Bases: BaseDocumentTransformer

Clean page_content (and listed metadata keys); metadata["sct"] gets the signals.

Returns new documents; the inputs are not modified.

columns instance-attribute

columns = as_columns(columns)

output instance-attribute

output = output

handle instance-attribute

handle = CleanerHandle(
    config, registry_factory=registry_factory
)

transform_documents

transform_documents(
    documents: Sequence[Document], **kwargs: Any
) -> Sequence[Document]

haystack

Haystack component (pip install squeakycleantext[haystack]).

SCTDocumentCleaner

SCTDocumentCleaner(
    columns: Sequence[str] = ("content",),
    output: str = "overwrite",
    config: TextCleanerConfig | str | None = None,
    *,
    schema: FieldSchema | None = None,
    context: DetectionContext | None = None,
    registry_factory: Callable[
        [TextCleanerConfig], DetectorRegistry
    ]
    | None = None,
)

Clean content (and listed meta keys); meta["sct"] gets the signals.

Returns new documents with ids recomputed from the cleaned content.

columns instance-attribute

columns = as_columns(columns)

output instance-attribute

output = output

handle instance-attribute

handle = CleanerHandle(
    config, registry_factory=registry_factory
)

to_dict

to_dict() -> dict[str, Any]

Preset names stay names; a config keeps only its non-default fields.

Haystack raises SerializationError for values it cannot store (custom steps).

from_dict classmethod

from_dict(data: dict[str, Any]) -> SCTDocumentCleaner

run

run(documents: list[Document]) -> dict[str, list[Document]]