INNER CODE UNIT · Python
_read_and_preprocess
yurijmikhalevich/rclip · rclip/main.py:51
def _read_and_preprocess(path: str) -> Tuple[str, str, npt.NDArray[np.float32]]:
"""Reads an image, computes its hash, and runs the CLIP preprocessing on it. Runs on the loader
threads so that the decoding and resizing happen in parallel and only the
model forward pass is left for the consumer."""
file_hash = helpers.compute_file_hash(path)
preprocessed = preprocess(helpers.read_image(path))
return path, file_hash, preprocessed
class RClip:
EXCLUDE_DIRS_DEFAULT = ["@eaDir", "node_modules", ".git", "System Volume Information"]
DB_IMAGES_BEFORE_COMMIT = 50_000
# how many indexing batches to keep loading ahead of the model; preprocessed
# images are small (a few hundred KB each), so a few batches of look-ahead
# cost little memory.
LOOKAHEAD_BATCHES = 3
MAX_IMAGE_LOADING_WORKERS = 16
SEARCH_BATCH_SIZE = 4096