INNER CODE UNIT · Python

deduper

allenai/dolma · python/dolma/__init__.py:26

def deduper(config: dict):
    """
    Run the deduper with the given configuration.

    Args:
        config (dict): The configuration for the deduper.

    Raises:
        DolmaRustPipelineError: If there is an error running the deduper.

    """
    try:
        _dolma.deduper_entrypoint(json.dumps(config))
    except RuntimeError as e:
        raise DolmaRustPipelineError(f"Error running deduper: {e}") from e


def mixer(config: dict):

View source record →