concurrent

Concurrency concepts that support efficient multi-threading.

ExecutorFactory

Source code in pyiceberg/utils/concurrent.py

class ExecutorFactory:
    _instance: Optional[Executor] = None

    @staticmethod
    def get_or_create() -> Executor:
        """Return the same executor in each call."""
        if ExecutorFactory._instance is None:
            max_workers = ExecutorFactory.max_workers()
            ExecutorFactory._instance = ThreadPoolExecutor(max_workers=max_workers)

        return ExecutorFactory._instance

    @staticmethod
    def max_workers() -> Optional[int]:
        """Return the max number of workers configured."""
        config = Config()
        val = config.config.get("max-workers")

        if val is None:
            return None

        try:
            return int(val)  # type: ignore
        except ValueError as err:
            raise ValueError(f"Max workers should be an integer or left unset. Current value: {val}") from err

get_or_create() staticmethod

Return the same executor in each call.

Source code in pyiceberg/utils/concurrent.py

@staticmethod
def get_or_create() -> Executor:
    """Return the same executor in each call."""
    if ExecutorFactory._instance is None:
        max_workers = ExecutorFactory.max_workers()
        ExecutorFactory._instance = ThreadPoolExecutor(max_workers=max_workers)

    return ExecutorFactory._instance

max_workers() staticmethod

Return the max number of workers configured.

Source code in pyiceberg/utils/concurrent.py

@staticmethod
def max_workers() -> Optional[int]:
    """Return the max number of workers configured."""
    config = Config()
    val = config.config.get("max-workers")

    if val is None:
        return None

    try:
        return int(val)  # type: ignore
    except ValueError as err:
        raise ValueError(f"Max workers should be an integer or left unset. Current value: {val}") from err