INNER CODE UNIT · Python

main

allenai/dolma · scripts/validate_mixer/main.py:10

def main(config_path, num_samples, verbose):
    # Register the keyboard interrupt handler
    signal.signal(signal.SIGINT, keyboard_interrupt_handler)

    # Set verbose mode
    set_verbose(args.verbose)

    load_env_variables()
    config = load_and_validate_config(config_path)
    if config is None:
        print("Configuration loading or validation FAILED")

    if not validate_s3_paths_and_permissions(config):
        print("S3 path validation FAILED")
        # return

    if not validate_stream_filters(config):
        print("Filter validation FAILED.\n")

View source record →