TextCleanerConfig

TextCleanerConfig dataclass

TextCleanerConfig(
    check_detect_language: bool = True,
    check_fix_bad_unicode: bool = True,
    check_to_ascii_unicode: bool = False,
    check_replace_html: bool = True,
    check_replace_urls: bool = True,
    check_replace_emails: bool = True,
    check_replace_years: bool = True,
    check_replace_dates: bool = True,
    check_fuzzy_replace_dates: bool = False,
    check_replace_phone_numbers: bool = True,
    check_replace_numbers: bool = True,
    check_replace_currency_symbols: bool = True,
    check_ner_process: bool = True,
    check_remove_isolated_letters: bool = True,
    check_remove_isolated_special_symbols: bool = True,
    check_remove_bracket_content: bool = True,
    check_remove_brace_content: bool = True,
    check_normalize_whitespace: bool = True,
    check_statistical_model_processing: bool = True,
    check_casefold: bool = True,
    check_smart_casefold: bool = False,
    check_remove_stopwords: bool = True,
    check_remove_punctuation: bool = True,
    check_remove_stext_custom_stop_words: bool = True,
    check_remove_emoji: bool = False,
    fuzzy_date_score_cutoff: int = 85,
    replace_with_url: str = "<URL>",
    replace_with_html: str = "<HTML>",
    replace_with_email: str = "<EMAIL>",
    replace_with_years: str = "<YEAR>",
    replace_with_dates: str = "<DATE>",
    replace_with_phone_numbers: str = "<PHONE>",
    replace_with_numbers: str = "<NUMBER>",
    replace_with_currency_symbols: str | None = None,
    positional_tags: tuple[str, ...] = (
        "PER",
        "LOC",
        "ORG",
        "MISC",
    ),
    ner_confidence_threshold: float | None = None,
    ner_batch_size: int = 8,
    language: str | tuple[str, ...] | None = None,
    ner_backend: NerBackend = "onnx",
    ner_mode: NerMode = "standard",
    replacement_mode: ReplacementMode = "placeholder",
    ner_models: Mapping[str, str] | None = None,
    ner_model_revisions: Mapping[str, str] | None = None,
    ner_ensemble: Mapping[str, tuple[str, ...]]
    | None = None,
    ner_ensemble_default_keys: tuple[str, ...]
    | None = None,
    torch_ner_models: Mapping[str, str] | None = None,
    torch_ner_model_revisions: Mapping[str, str]
    | None = None,
    gliner_model: str | None = None,
    gliner_revision: str | None = None,
    gliner_variant: str = "gliner",
    gliner_labels: tuple[str, ...] = (
        "person",
        "organization",
        "location",
    ),
    gliner_label_map: Mapping[str, str] | None = None,
    gliner_threshold: float | None = None,
    gliner_label_descriptions: Mapping[str, str]
    | None = None,
    gliner_onnx: bool = False,
    check_classify_document: bool = False,
    gliclass_model: str | None = None,
    gliclass_revision: str | None = None,
    gliclass_labels: tuple[str, ...] = (),
    gliclass_threshold: float = 0.5,
    gliclass_classification_type: str = "single-label",
    gliclass_onnx: bool = False,
    custom_pipeline_steps: tuple = (),
    extra_languages: tuple[str, ...] = (),
    custom_stopwords: Mapping[str, frozenset] | None = None,
    custom_month_names: Mapping[str, tuple[str, ...]]
    | None = None,
    supported_languages: frozenset = frozenset(),
    check_replace_ips: bool = True,
    replace_ips_public_only: bool = False,
    replace_with_ips: str = "<IP>",
    replace_numbers_in_words: bool = False,
    onnx_intra_op_threads: int | None = None,
    onnx_inter_op_threads: int | None = None,
    ner_chunk_overlap: int = 0,
    detect_all_languages: bool = True,
    language_confidence_warning: float = 0.0,
    check_decode_unicode_escapes: bool = True,
    ner_device: str | None = None,
    cuda_device_id: int = 0,
    onnx_cuda_provider_options: Mapping[str, Any]
    | None = None,
    json_key_cleaning: JsonKeyCleaning = "pii",
    synthetic_seed: int = 42,
    synthetic_per_document: bool = True,
    language_margin_warning: float = 0.15,
    fix_character_width: bool = False,
    legacy_type_names: bool = False,
    ner_subword_widen: float | None = 1.0,
    fold_styled_characters: bool = True,
    ner_subword_widen_same_script: bool = True,
    synthetic_locale: str | None = None,
    ner_sees_tags: bool = True,
    synthetic_low_confidence_locale: str | None = "en_US",
    synthetic_collapse_period: bool = True,
    synthetic_reference_date: str | None = "2026-01-01",
    ner_ensemble_vote: NerEnsembleVote = "max",
    ner_propagate_masked: bool = True,
    ner_propagate_min_score: float | None = None,
    check_replace_iban_cards: bool = True,
    replace_with_iban: str = "<IBAN>",
    replace_with_cards: str = "<CARD>",
    check_replace_identifiers: bool = False,
    replace_with_identifiers: str = "<ID>",
    bracket_content_max_chars: int = 32,
    bracket_content_max_words: int = 3,
    keep_json_braces: bool = True,
    date_month_name_slash_year: bool = True,
    date_builtin_month_languages: tuple[str, ...]
    | None = None,
    html_tag_strict: bool = True,
    ner_merge_location_gap: int = 0,
    ner_snap_word_edges: bool = True,
    url_trailing_punct_outside: bool = True,
    replace_numbers_mode: ReplaceNumbersMode = "identifiers",
    replace_numbers_min_digits: int = 4,
    phone_reject_embedded: bool = True,
    custom_deny_terms: Mapping[str, tuple[str, ...]]
    | None = None,
    custom_allow_terms: tuple[str, ...] = (),
    custom_patterns: Mapping[str, str] | None = None,
    custom_terms_case_sensitive: bool = False,
    numbers_letter_prefix_ids: bool = True,
    numbers_two_part_ids: bool = True,
    numbers_spaced_pair_ids: bool = True,
    custom_patterns_timeout: float | None = 2.0,
    custom_patterns_on_timeout: OnPatternTimeout = "raise",
    numbers_keep_currency_code_amounts: bool = True,
    detector_specs: tuple[DetectorSpec, ...] = (),
    allow_network_detectors: bool = False,
    pii_processing: Literal["legacy", "unified"] = "legacy",
    source_mapping: Literal[
        "best_effort", "require_source"
    ] = "best_effort",
    language_policy: Literal[
        "legacy", "abstain"
    ] = "legacy",
    language_min_confidence: float | None = None,
    language_min_margin: float | None = None,
    unknown_language_action: Literal[
        "multilingual", "error", "skip_ner"
    ] = "multilingual",
    routing_policy: Literal[
        "always_run", "calibrated_cascade"
    ] = "always_run",
    routing_fast_specs: tuple[DetectorSpec, ...] = (),
    routing_full_specs: tuple[DetectorSpec, ...] = (),
    routing_entity_types: tuple[str, ...] = (),
    routing_calibration: Mapping[str, Any] | None = None,
)

Thread-safe, immutable configuration for the text cleaning pipeline.

Each TextCleaner instance stores its own config copy. Use dataclasses.replace() to create modified copies::

import dataclasses
new_cfg = dataclasses.replace(cfg, check_ner_process=False)

check_detect_language class-attribute instance-attribute

check_detect_language: bool = True

check_fix_bad_unicode class-attribute instance-attribute

check_fix_bad_unicode: bool = True

check_to_ascii_unicode class-attribute instance-attribute

check_to_ascii_unicode: bool = False

check_replace_html class-attribute instance-attribute

check_replace_html: bool = True

check_replace_urls class-attribute instance-attribute

check_replace_urls: bool = True

check_replace_emails class-attribute instance-attribute

check_replace_emails: bool = True

check_replace_years class-attribute instance-attribute

check_replace_years: bool = True

check_replace_dates class-attribute instance-attribute

check_replace_dates: bool = True

check_fuzzy_replace_dates class-attribute instance-attribute

check_fuzzy_replace_dates: bool = False

check_replace_phone_numbers class-attribute instance-attribute

check_replace_phone_numbers: bool = True

check_replace_numbers class-attribute instance-attribute

check_replace_numbers: bool = True

check_replace_currency_symbols class-attribute instance-attribute

check_replace_currency_symbols: bool = True

check_ner_process class-attribute instance-attribute

check_ner_process: bool = True

check_remove_isolated_letters class-attribute instance-attribute

check_remove_isolated_letters: bool = True

check_remove_isolated_special_symbols class-attribute instance-attribute

check_remove_isolated_special_symbols: bool = True

check_remove_bracket_content class-attribute instance-attribute

check_remove_bracket_content: bool = True

check_remove_brace_content class-attribute instance-attribute

check_remove_brace_content: bool = True

check_normalize_whitespace class-attribute instance-attribute

check_normalize_whitespace: bool = True

check_statistical_model_processing class-attribute instance-attribute

check_statistical_model_processing: bool = True

check_casefold class-attribute instance-attribute

check_casefold: bool = True

check_smart_casefold class-attribute instance-attribute

check_smart_casefold: bool = False

check_remove_stopwords class-attribute instance-attribute

check_remove_stopwords: bool = True

check_remove_punctuation class-attribute instance-attribute

check_remove_punctuation: bool = True

check_remove_stext_custom_stop_words class-attribute instance-attribute

check_remove_stext_custom_stop_words: bool = True

check_remove_emoji class-attribute instance-attribute

check_remove_emoji: bool = False

fuzzy_date_score_cutoff class-attribute instance-attribute

fuzzy_date_score_cutoff: int = 85

replace_with_url class-attribute instance-attribute

replace_with_url: str = '<URL>'

replace_with_html class-attribute instance-attribute

replace_with_html: str = '<HTML>'

replace_with_email class-attribute instance-attribute

replace_with_email: str = '<EMAIL>'

replace_with_years class-attribute instance-attribute

replace_with_years: str = '<YEAR>'

replace_with_dates class-attribute instance-attribute

replace_with_dates: str = '<DATE>'

replace_with_phone_numbers class-attribute instance-attribute

replace_with_phone_numbers: str = '<PHONE>'

replace_with_numbers class-attribute instance-attribute

replace_with_numbers: str = '<NUMBER>'

replace_with_currency_symbols class-attribute instance-attribute

replace_with_currency_symbols: str | None = None

positional_tags class-attribute instance-attribute

positional_tags: tuple[str, ...] = (
    "PER",
    "LOC",
    "ORG",
    "MISC",
)

ner_confidence_threshold class-attribute instance-attribute

ner_confidence_threshold: float | None = None

ner_batch_size class-attribute instance-attribute

ner_batch_size: int = 8

language class-attribute instance-attribute

language: str | tuple[str, ...] | None = None

ner_backend class-attribute instance-attribute

ner_backend: NerBackend = 'onnx'

ner_mode class-attribute instance-attribute

ner_mode: NerMode = 'standard'

replacement_mode class-attribute instance-attribute

replacement_mode: ReplacementMode = 'placeholder'

ner_models class-attribute instance-attribute

ner_models: Mapping[str, str] | None = None

ner_model_revisions class-attribute instance-attribute

ner_model_revisions: Mapping[str, str] | None = None

ner_ensemble class-attribute instance-attribute

ner_ensemble: Mapping[str, tuple[str, ...]] | None = None

ner_ensemble_default_keys class-attribute instance-attribute

ner_ensemble_default_keys: tuple[str, ...] | None = None

torch_ner_models class-attribute instance-attribute

torch_ner_models: Mapping[str, str] | None = None

torch_ner_model_revisions class-attribute instance-attribute

torch_ner_model_revisions: Mapping[str, str] | None = None

gliner_model class-attribute instance-attribute

gliner_model: str | None = None

gliner_revision class-attribute instance-attribute

gliner_revision: str | None = None

gliner_variant class-attribute instance-attribute

gliner_variant: str = 'gliner'

gliner_labels class-attribute instance-attribute

gliner_labels: tuple[str, ...] = (
    "person",
    "organization",
    "location",
)

gliner_label_map class-attribute instance-attribute

gliner_label_map: Mapping[str, str] | None = None

gliner_threshold class-attribute instance-attribute

gliner_threshold: float | None = None

gliner_label_descriptions class-attribute instance-attribute

gliner_label_descriptions: Mapping[str, str] | None = None

gliner_onnx class-attribute instance-attribute

gliner_onnx: bool = False

check_classify_document class-attribute instance-attribute

check_classify_document: bool = False

gliclass_model class-attribute instance-attribute

gliclass_model: str | None = None

gliclass_revision class-attribute instance-attribute

gliclass_revision: str | None = None

gliclass_labels class-attribute instance-attribute

gliclass_labels: tuple[str, ...] = ()

gliclass_threshold class-attribute instance-attribute

gliclass_threshold: float = 0.5

gliclass_classification_type class-attribute instance-attribute

gliclass_classification_type: str = 'single-label'

gliclass_onnx class-attribute instance-attribute

gliclass_onnx: bool = False

custom_pipeline_steps class-attribute instance-attribute

custom_pipeline_steps: tuple = ()

extra_languages class-attribute instance-attribute

extra_languages: tuple[str, ...] = ()

custom_stopwords class-attribute instance-attribute

custom_stopwords: Mapping[str, frozenset] | None = None

custom_month_names class-attribute instance-attribute

custom_month_names: Mapping[str, tuple[str, ...]] | None = (
    None
)

supported_languages class-attribute instance-attribute

supported_languages: frozenset = frozenset()

check_replace_ips class-attribute instance-attribute

check_replace_ips: bool = True

replace_ips_public_only class-attribute instance-attribute

replace_ips_public_only: bool = False

replace_with_ips class-attribute instance-attribute

replace_with_ips: str = '<IP>'

replace_numbers_in_words class-attribute instance-attribute

replace_numbers_in_words: bool = False

onnx_intra_op_threads class-attribute instance-attribute

onnx_intra_op_threads: int | None = None

onnx_inter_op_threads class-attribute instance-attribute

onnx_inter_op_threads: int | None = None

ner_chunk_overlap class-attribute instance-attribute

ner_chunk_overlap: int = 0

detect_all_languages class-attribute instance-attribute

detect_all_languages: bool = True

language_confidence_warning class-attribute instance-attribute

language_confidence_warning: float = 0.0

check_decode_unicode_escapes class-attribute instance-attribute

check_decode_unicode_escapes: bool = True

ner_device class-attribute instance-attribute

ner_device: str | None = None

cuda_device_id class-attribute instance-attribute

cuda_device_id: int = 0

onnx_cuda_provider_options class-attribute instance-attribute

onnx_cuda_provider_options: Mapping[str, Any] | None = None

json_key_cleaning class-attribute instance-attribute

json_key_cleaning: JsonKeyCleaning = 'pii'

synthetic_seed class-attribute instance-attribute

synthetic_seed: int = 42

synthetic_per_document class-attribute instance-attribute

synthetic_per_document: bool = True

language_margin_warning class-attribute instance-attribute

language_margin_warning: float = 0.15

fix_character_width class-attribute instance-attribute

fix_character_width: bool = False

legacy_type_names class-attribute instance-attribute

legacy_type_names: bool = False

ner_subword_widen class-attribute instance-attribute

ner_subword_widen: float | None = 1.0

fold_styled_characters class-attribute instance-attribute

fold_styled_characters: bool = True

ner_subword_widen_same_script class-attribute instance-attribute

ner_subword_widen_same_script: bool = True

synthetic_locale class-attribute instance-attribute

synthetic_locale: str | None = None

ner_sees_tags class-attribute instance-attribute

ner_sees_tags: bool = True

synthetic_low_confidence_locale class-attribute instance-attribute

synthetic_low_confidence_locale: str | None = 'en_US'

synthetic_collapse_period class-attribute instance-attribute

synthetic_collapse_period: bool = True

synthetic_reference_date class-attribute instance-attribute

synthetic_reference_date: str | None = '2026-01-01'

ner_ensemble_vote class-attribute instance-attribute

ner_ensemble_vote: NerEnsembleVote = 'max'

ner_propagate_masked class-attribute instance-attribute

ner_propagate_masked: bool = True

ner_propagate_min_score class-attribute instance-attribute

ner_propagate_min_score: float | None = None

check_replace_iban_cards class-attribute instance-attribute

check_replace_iban_cards: bool = True

replace_with_iban class-attribute instance-attribute

replace_with_iban: str = '<IBAN>'

replace_with_cards class-attribute instance-attribute

replace_with_cards: str = '<CARD>'

check_replace_identifiers class-attribute instance-attribute

check_replace_identifiers: bool = False

replace_with_identifiers class-attribute instance-attribute

replace_with_identifiers: str = '<ID>'

bracket_content_max_chars class-attribute instance-attribute

bracket_content_max_chars: int = 32

bracket_content_max_words class-attribute instance-attribute

bracket_content_max_words: int = 3

keep_json_braces class-attribute instance-attribute

keep_json_braces: bool = True

date_month_name_slash_year class-attribute instance-attribute

date_month_name_slash_year: bool = True

date_builtin_month_languages class-attribute instance-attribute

date_builtin_month_languages: tuple[str, ...] | None = None

html_tag_strict class-attribute instance-attribute

html_tag_strict: bool = True

ner_merge_location_gap class-attribute instance-attribute

ner_merge_location_gap: int = 0

ner_snap_word_edges class-attribute instance-attribute

ner_snap_word_edges: bool = True

url_trailing_punct_outside class-attribute instance-attribute

url_trailing_punct_outside: bool = True

replace_numbers_mode class-attribute instance-attribute

replace_numbers_mode: ReplaceNumbersMode = 'identifiers'

replace_numbers_min_digits class-attribute instance-attribute

replace_numbers_min_digits: int = 4

phone_reject_embedded class-attribute instance-attribute

phone_reject_embedded: bool = True

custom_deny_terms class-attribute instance-attribute

custom_deny_terms: Mapping[str, tuple[str, ...]] | None = (
    None
)

custom_allow_terms class-attribute instance-attribute

custom_allow_terms: tuple[str, ...] = ()

custom_patterns class-attribute instance-attribute

custom_patterns: Mapping[str, str] | None = None

custom_terms_case_sensitive class-attribute instance-attribute

custom_terms_case_sensitive: bool = False

numbers_letter_prefix_ids class-attribute instance-attribute

numbers_letter_prefix_ids: bool = True

numbers_two_part_ids class-attribute instance-attribute

numbers_two_part_ids: bool = True

numbers_spaced_pair_ids class-attribute instance-attribute

numbers_spaced_pair_ids: bool = True

custom_patterns_timeout class-attribute instance-attribute

custom_patterns_timeout: float | None = 2.0

custom_patterns_on_timeout class-attribute instance-attribute

custom_patterns_on_timeout: OnPatternTimeout = 'raise'

numbers_keep_currency_code_amounts class-attribute instance-attribute

numbers_keep_currency_code_amounts: bool = True

detector_specs class-attribute instance-attribute

detector_specs: tuple[DetectorSpec, ...] = ()

allow_network_detectors class-attribute instance-attribute

allow_network_detectors: bool = False

pii_processing class-attribute instance-attribute

pii_processing: Literal['legacy', 'unified'] = 'legacy'

source_mapping class-attribute instance-attribute

source_mapping: Literal["best_effort", "require_source"] = (
    "best_effort"
)

language_policy class-attribute instance-attribute

language_policy: Literal['legacy', 'abstain'] = 'legacy'

language_min_confidence class-attribute instance-attribute

language_min_confidence: float | None = None

language_min_margin class-attribute instance-attribute

language_min_margin: float | None = None

unknown_language_action class-attribute instance-attribute

unknown_language_action: Literal[
    "multilingual", "error", "skip_ner"
] = "multilingual"

routing_policy class-attribute instance-attribute

routing_policy: Literal[
    "always_run", "calibrated_cascade"
] = "always_run"

routing_fast_specs class-attribute instance-attribute

routing_fast_specs: tuple[DetectorSpec, ...] = ()

routing_full_specs class-attribute instance-attribute

routing_full_specs: tuple[DetectorSpec, ...] = ()

routing_entity_types class-attribute instance-attribute

routing_entity_types: tuple[str, ...] = ()

routing_calibration class-attribute instance-attribute

routing_calibration: Mapping[str, Any] | None = None

effective_gliner_threshold property

effective_gliner_threshold: float

GLiNER score cut: explicit value, else 0.3 in PII mode, else 0.4.

effective_ner_confidence_threshold property

effective_ner_confidence_threshold: float

Final NER filter: explicit value, else the GLiNER cut in PII mode, else 0.85.

from_preset classmethod

from_preset(
    name: str = "training", **overrides: Any
) -> TextCleanerConfig

Build a config from a named preset (see sct.presets.PRESETS).