fileformat

File Format API for writing Iceberg data files.

FileFormatFactory

Registry of FileFormatModel implementations.

Source code in pyiceberg/io/fileformat.py

class FileFormatFactory:
    """Registry of FileFormatModel implementations."""

    _registry: dict[FileFormat, FileFormatModel] = {}

    @classmethod
    def register(cls, model: FileFormatModel) -> None:
        if model.format in cls._registry:
            existing = cls._registry[model.format]
            raise ValueError(
                f"Cannot register {type(model).__name__}: {type(existing).__name__} is already registered for {model.format}"
            )
        cls._registry[model.format] = model

    @classmethod
    def get(cls, file_format: FileFormat) -> FileFormatModel:
        if file_format not in cls._registry:
            raise ValueError(f"No writer registered for {file_format}. Available: {list(cls._registry.keys())}")
        return cls._registry[file_format]

    @classmethod
    def available_formats(cls) -> list[FileFormat]:
        return list(cls._registry.keys())

FileFormatModel

Bases: Protocol

Represents a file format's capabilities. Creates writers.

Source code in pyiceberg/io/fileformat.py

@runtime_checkable
class FileFormatModel(Protocol):
    """Represents a file format's capabilities. Creates writers."""

    @property
    def format(self) -> FileFormat: ...

    def file_extension(self) -> str:
        """Return file extension without dot, e.g. 'parquet', 'orc'."""
        ...

    def create_writer(
        self,
        output_file: OutputFile,
        file_schema: Schema,
        properties: Properties,
    ) -> FileFormatWriter: ...

    def add_field_metadata(self, field: NestedField, metadata: dict[bytes, bytes], include_field_ids: bool) -> None:
        """Add format-specific Arrow field metadata."""
        ...

add_field_metadata(field, metadata, include_field_ids)

Add format-specific Arrow field metadata.

Source code in pyiceberg/io/fileformat.py

def add_field_metadata(self, field: NestedField, metadata: dict[bytes, bytes], include_field_ids: bool) -> None:
    """Add format-specific Arrow field metadata."""
    ...

file_extension()

Return file extension without dot, e.g. 'parquet', 'orc'.

Source code in pyiceberg/io/fileformat.py

def file_extension(self) -> str:
    """Return file extension without dot, e.g. 'parquet', 'orc'."""
    ...

FileFormatWriter

Bases: ABC

Writes data to a single file in a specific format.

Source code in pyiceberg/io/fileformat.py

class FileFormatWriter(ABC):
    """Writes data to a single file in a specific format."""

    _result: DataFileStatistics | None = None

    @abstractmethod
    def write(self, table: pa.Table) -> None:
        """Write a batch of data. May be called multiple times."""

    @abstractmethod
    def close(self) -> DataFileStatistics:
        """Finalize the file and return statistics."""

    def result(self) -> DataFileStatistics:
        """Return statistics from a previous close() call."""
        if self._result is None:
            raise RuntimeError("Writer has not been closed yet")
        return self._result

    def __enter__(self) -> FileFormatWriter:
        """Enter the context manager."""
        return self

    def __exit__(self, exc_type: Any, exc_val: Any, exc_tb: Any) -> None:
        """Exit the context manager, closing the writer and caching statistics."""
        if exc_type is not None:
            try:
                self.close()
            except Exception:
                pass
            return
        self._result = self.close()

__enter__()

Enter the context manager.

Source code in pyiceberg/io/fileformat.py

def __enter__(self) -> FileFormatWriter:
    """Enter the context manager."""
    return self

__exit__(exc_type, exc_val, exc_tb)

Exit the context manager, closing the writer and caching statistics.

Source code in pyiceberg/io/fileformat.py

def __exit__(self, exc_type: Any, exc_val: Any, exc_tb: Any) -> None:
    """Exit the context manager, closing the writer and caching statistics."""
    if exc_type is not None:
        try:
            self.close()
        except Exception:
            pass
        return
    self._result = self.close()

close() abstractmethod

Finalize the file and return statistics.

Source code in pyiceberg/io/fileformat.py

@abstractmethod
def close(self) -> DataFileStatistics:
    """Finalize the file and return statistics."""

result()

Return statistics from a previous close() call.

Source code in pyiceberg/io/fileformat.py

def result(self) -> DataFileStatistics:
    """Return statistics from a previous close() call."""
    if self._result is None:
        raise RuntimeError("Writer has not been closed yet")
    return self._result

write(table) abstractmethod

Write a batch of data. May be called multiple times.

Source code in pyiceberg/io/fileformat.py

@abstractmethod
def write(self, table: pa.Table) -> None:
    """Write a batch of data. May be called multiple times."""