INNER CODE UNIT · Python

max_in_flight

yurijmikhalevich/rclip · rclip/main.py:135

    max_in_flight = max(self.LOOKAHEAD_BATCHES * self._indexing_batch_size, self._image_loading_workers)
    items_iter = iter(items)
    in_flight: Deque[Tuple[str, ImageMeta, Future[Tuple[str, str, npt.NDArray[np.float32]]]]] = deque()

    def submit_next() -> None:
      item = next(items_iter, None)
      if item is not None:
        path, meta = item
        in_flight.append((path, meta, executor.submit(_read_and_preprocess, path)))

    for _ in range(max_in_flight):
      submit_next()

    while in_flight:
      path, meta, future = in_flight.popleft()
      submit_next()  # refill so the window stays full while the consumer is busy
      try:
        _, file_hash, preprocessed = future.result()

View source record →