tfaip.data

DataBase

Module that defines DataBase

tfaip.data.data.dict_to_input_layers(d: Dict[str, tensorflow.python.framework.tensor_spec.TensorSpec]) → Dict[str, tensorflow.python.keras.engine.input_layer.Input]

tfaip.data.data.validate_specs(func)

class tfaip.data.data.DataBase(params: TDP)

Bases: Generic[tfaip.data.data.TDP], abc.ABC

DataBase class to provide training and validation data.

Override _input_layer_specs, and _output_layer_specs in a custom implementation

classmethod params_cls() → Type[TDP]

classmethod default_params() → TDP

classmethod data_pipeline_cls() → Type[tfaip.data.pipeline.datapipeline.DataPipeline]

__init__(params: TDP)

Initialize self. See help(type(self)) for accurate signature.

preload(progress_bar=True)

print_params()

property params

padding_values() → Dict[str, Union[numpy.ndarray, int, numpy.int8, numpy.int16, numpy.int32, numpy.int64, float, numpy.float16, numpy.float32, numpy.float64, bool]]

element_length_fn() → Callable[[Dict[str, Union[tensorflow.python.framework.ops.Tensor, tensorflow.python.keras.engine.keras_tensor.KerasTensor]]], Union[tensorflow.python.framework.ops.Tensor, tensorflow.python.keras.engine.keras_tensor.KerasTensor]]

Element length for bucked_by_sequence_length

create_pipeline(pipeline_params: tfaip.data.databaseparams.DataPipelineParams, params: tfaip.data.databaseparams.DataGeneratorParams) → tfaip.data.pipeline.datapipeline.DataPipeline

get_or_create_pipeline(pipeline_params: tfaip.data.databaseparams.DataPipelineParams, params: Optional[tfaip.data.databaseparams.DataGeneratorParams]) → tfaip.data.pipeline.datapipeline.DataPipeline

pipeline_by_mode(mode: tfaip.data.pipeline.definitions.PipelineMode) → tfaip.data.pipeline.datapipeline.DataPipeline

create_input_layers() → Dict[str, tensorflow.python.keras.engine.input_layer.Input]

create_target_as_input_layers() → Dict[str, tensorflow.python.keras.engine.input_layer.Input]

create_meta_as_input_layers() → Dict[str, tensorflow.python.keras.engine.input_layer.Input]

input_layer_specs(**kwargs)

target_layer_specs(**kwargs)

meta_layer_specs(**kwargs)

register_resource_from_parameter(param_name: str) → tfaip.resource.resource.Resource

dump_resources(root_path: str, data_params_dict: dict)

DataBaseParams

Definition of DataBaseParams, DataPipelineParams, and DataGeneratorParams

class tfaip.data.databaseparams.DataGeneratorParams

Bases: abc.ABC

Parameter class that defines how to construct a DataGenerator.

static cls() → Type[DataGenerator]

create(mode: tfaip.data.pipeline.definitions.PipelineMode) → DataGenerator

__init__() → None

Initialize self. See help(type(self)) for accurate signature.

default_factory

alias of tfaip.data.databaseparams.DataGeneratorParams

classmethod from_dict(kvs: Optional[Union[dict, list, str, int, float, bool]], *, infer_missing=False) → A

classmethod from_json(s: Union[str, bytes, bytearray], *, parse_float=None, parse_int=None, parse_constant=None, infer_missing=False, **kw) → A

classmethod schema(*, infer_missing: bool = False, only=None, exclude=(), many: bool = False, context=None, load_only=(), dump_only=(), partial: bool = False, unknown=None) → dataclasses_json.mm.SchemaF[A]

to_dict(encode_json=False) → Dict[str, Optional[Union[dict, list, str, int, float, bool]]]

to_json(*, skipkeys: bool = False, ensure_ascii: bool = True, check_circular: bool = True, allow_nan: bool = True, indent: Optional[Union[int, str]] = None, separators: Optional[Tuple[str, str]] = None, default: Optional[Callable] = None, sort_keys: bool = False, **kw) → str

class tfaip.data.databaseparams.DataPipelineParams(batch_size: int = 16, limit: int = -1, prefetch: int = -1, num_processes: int = 4, batch_drop_remainder: bool = False, shuffle_buffer_size: int = -1, mode: tfaip.data.pipeline.definitions.PipelineMode = <PipelineMode.TRAINING: 'training'>, bucket_boundaries: List[int] = <factory>, bucket_batch_sizes: Optional[List[int]] = None)

Bases: object

Parameter class that defines the general parameters, e.g. batch size, prefetching, number of processes, … of a certain (e.g. train or val) pipeline.

batch_size: int = 16

limit: int = -1

prefetch: int = -1

num_processes: int = 4

batch_drop_remainder: bool = False

shuffle_buffer_size: int = -1

mode: tfaip.data.pipeline.definitions.PipelineMode = 'training'

bucket_boundaries: List[int]

bucket_batch_sizes: Optional[List[int]] = None

__init__(batch_size: int = 16, limit: int = -1, prefetch: int = -1, num_processes: int = 4, batch_drop_remainder: bool = False, shuffle_buffer_size: int = -1, mode: tfaip.data.pipeline.definitions.PipelineMode = <PipelineMode.TRAINING: 'training'>, bucket_boundaries: List[int] = <factory>, bucket_batch_sizes: Optional[List[int]] = None) → None

Initialize self. See help(type(self)) for accurate signature.

classmethod from_dict(kvs: Optional[Union[dict, list, str, int, float, bool]], *, infer_missing=False) → A

classmethod from_json(s: Union[str, bytes, bytearray], *, parse_float=None, parse_int=None, parse_constant=None, infer_missing=False, **kw) → A

classmethod schema(*, infer_missing: bool = False, only=None, exclude=(), many: bool = False, context=None, load_only=(), dump_only=(), partial: bool = False, unknown=None) → dataclasses_json.mm.SchemaF[A]

to_dict(encode_json=False) → Dict[str, Optional[Union[dict, list, str, int, float, bool]]]

to_json(*, skipkeys: bool = False, ensure_ascii: bool = True, check_circular: bool = True, allow_nan: bool = True, indent: Optional[Union[int, str]] = None, separators: Optional[Tuple[str, str]] = None, default: Optional[Callable] = None, sort_keys: bool = False, **kw) → str

class tfaip.data.databaseparams.DataBaseParams(pre_proc: tfaip.data.pipeline.processor.params.DataProcessorPipelineParams = <factory>, post_proc: tfaip.data.pipeline.processor.params.DataProcessorPipelineParams = <factory>, resource_base_path: str = '/home/docs/checkouts/readthedocs.org/user_builds/tfaip/checkouts/develop/docs/source')

Bases: object

Parameters that define the overall setup of the data pipelines (pre_proc and post_proc)

Parameters of this class will be shared among all DataProcessors.

pre_proc: tfaip.data.pipeline.processor.params.DataProcessorPipelineParams

post_proc: tfaip.data.pipeline.processor.params.DataProcessorPipelineParams

resource_base_path: str = '/home/docs/checkouts/readthedocs.org/user_builds/tfaip/checkouts/develop/docs/source'

__init__(pre_proc: tfaip.data.pipeline.processor.params.DataProcessorPipelineParams = <factory>, post_proc: tfaip.data.pipeline.processor.params.DataProcessorPipelineParams = <factory>, resource_base_path: str = '/home/docs/checkouts/readthedocs.org/user_builds/tfaip/checkouts/develop/docs/source') → None

Initialize self. See help(type(self)) for accurate signature.

classmethod from_dict(kvs: Optional[Union[dict, list, str, int, float, bool]], *, infer_missing=False) → A

classmethod from_json(s: Union[str, bytes, bytearray], *, parse_float=None, parse_int=None, parse_constant=None, infer_missing=False, **kw) → A

classmethod schema(*, infer_missing: bool = False, only=None, exclude=(), many: bool = False, context=None, load_only=(), dump_only=(), partial: bool = False, unknown=None) → dataclasses_json.mm.SchemaF[A]

to_dict(encode_json=False) → Dict[str, Optional[Union[dict, list, str, int, float, bool]]]

to_json(*, skipkeys: bool = False, ensure_ascii: bool = True, check_circular: bool = True, allow_nan: bool = True, indent: Optional[Union[int, str]] = None, separators: Optional[Tuple[str, str]] = None, default: Optional[Callable] = None, sort_keys: bool = False, **kw) → str