INNER CODE UNIT · Python

main

fluid-cloudnative/fluid · tools/file-prefetcher/main.py:16

def main():
    glob_patterns = os.getenv("FILE_PREFETCHER_FILE_LIST", None)
    assert glob_patterns is not None, "env variable FILE_PREFETCHER_FILE_LIST is not set"

     # Get all files to prefetch
    files_to_prefetch = []
    glob_patterns = glob_patterns.split(";")
    for glob_pattern in glob_patterns:
        if os.path.isdir(glob_pattern):
            glob_pattern = os.path.join(glob_pattern, "**")
        files = glob.glob(glob_pattern, recursive=True)
        files_to_prefetch.extend([file for file in files if os.path.isfile(file)])

    print(f"Found {len(files_to_prefetch)} files to prefetch")
    start_time = time.time()
    for file in files_to_prefetch:
        file_start_time = time.time()
        file_read(file)

View source record →