Index

A

AdapterError API (class in ketl.db.models) api_config (ketl.db.models.Creds attribute) (ketl.db.models.Source attribute)

api_config_id (ketl.db.models.Creds attribute) (ketl.db.models.Source attribute) api_hash (ketl.db.models.API property) archive_path (ketl.db.models.ExpectedFile attribute) auto (ketl.db.models.ExpectedMode attribute)

B

base_url (ketl.db.models.Source attribute) BaseExtractor (class in ketl.extractor.Extractor) BaseLoader (class in ketl.loader.Loader) BaseTransformer (class in ketl.transformer.Transformer)

BATCH_SIZE (ketl.db.models.API attribute) BLOCK_SIZE (ketl.db.models.CachedFile attribute) (ketl.db.models.ExpectedFile attribute) (ketl.extractor.Extractor.DefaultExtractor attribute)

C

cache_type (ketl.db.models.CachedFile attribute) cached_file (ketl.db.models.ExpectedFile attribute) cached_file_id (ketl.db.models.ExpectedFile attribute) cached_files (ketl.db.models.API property)

cached_files_on_disk() (ketl.db.models.API method) CachedFile (class in ketl.db.models) Creds (class in ketl.db.models) creds (ketl.db.models.API attribute) creds_details (ketl.db.models.Creds attribute)

D

data_dir (ketl.db.models.Source attribute) DatabaseLoader (class in ketl.loader.Loader) DefaultExtractor (class in ketl.extractor.Extractor)

DelimitedFileLoader (class in ketl.loader.Loader) DelimitedTableTransformer (class in ketl.transformer.Transformer) description (ketl.db.models.API attribute)

E

ETLPipeline (class in ketl.etl.Pipeline) execute() (ketl.etl.Pipeline.ETLPipeline method) expected_files (ketl.db.models.API property) (ketl.db.models.CachedFile attribute) (ketl.db.models.Source property) expected_mode (ketl.db.models.CachedFile attribute)

ExpectedFile (class in ketl.db.models) ExpectedMode (class in ketl.db.models) explicit (ketl.db.models.ExpectedMode attribute) extract() (ketl.extractor.Extractor.BaseExtractor method) (ketl.extractor.Extractor.DefaultExtractor method) extract_to (ketl.db.models.CachedFile attribute)

F

file_hash (ketl.db.models.CachedFile property) (ketl.db.models.ExpectedFile property) file_hash() (in module ketl.utils.file_utils) file_type (ketl.db.models.ExpectedFile attribute) finalize() (ketl.loader.Loader.BaseLoader method) (ketl.loader.Loader.DatabaseLoader method) (ketl.loader.Loader.HashLoader method) (ketl.loader.Loader.LocalFileLoader method) (ketl.loader.Loader.ParquetLoader method) (ketl.loader.Loader.PickleLoader method)

full_path (ketl.db.models.CachedFile property) full_path() (ketl.loader.Loader.LocalFileLoader method) full_url (ketl.db.models.CachedFile property)

G

get() (ketl.extractor.Rest.RestMixin method) get_engine() (in module ketl.db.settings) get_file() (ketl.extractor.Extractor.DefaultExtractor method)

get_instance() (ketl.db.models.API static method) get_or_create() (in module ketl.utils.db_utils) get_session() (in module ketl.db.settings)

H

hash (ketl.db.models.API attribute) (ketl.db.models.CachedFile attribute) (ketl.db.models.ExpectedFile attribute)

HashLoader (class in ketl.loader.Loader)

I

id (ketl.db.models.API attribute) (ketl.db.models.CachedFile attribute) (ketl.db.models.Creds attribute) (ketl.db.models.ExpectedFile attribute) (ketl.db.models.Source attribute)

InvalidConfigurationError InvalidLoaderConfiguration InvalidPipelineError is_archive (ketl.db.models.CachedFile attribute)

J

JsonTableTransformer (class in ketl.transformer.Transformer)

K

ketl module ketl.cli module ketl.db module ketl.db.models module ketl.db.settings module ketl.etl module ketl.etl.Pipeline module ketl.extractor module ketl.extractor.Extractor module

ketl.extractor.Rest module ketl.loader module ketl.loader.Loader module ketl.transformer module ketl.transformer.Transformer module ketl.utils module ketl.utils.db_utils module ketl.utils.file_utils module

L

last_download (ketl.db.models.CachedFile attribute) last_processed (ketl.db.models.ExpectedFile attribute) last_update (ketl.db.models.CachedFile attribute) load() (ketl.loader.Loader.BaseLoader method) (ketl.loader.Loader.DatabaseLoader method) (ketl.loader.Loader.DelimitedFileLoader method) (ketl.loader.Loader.HashLoader method) (ketl.loader.Loader.ParquetLoader method) (ketl.loader.Loader.PickleLoader method)

LocalFileLoader (class in ketl.loader.Loader)

M

merge_meta() (in module ketl.utils.db_utils) meta (ketl.db.models.CachedFile attribute) (ketl.db.models.ExpectedFile attribute) (ketl.db.models.Source attribute) module ketl ketl.cli ketl.db ketl.db.models ketl.db.settings ketl.etl ketl.etl.Pipeline ketl.extractor ketl.extractor.Extractor ketl.extractor.Rest ketl.loader ketl.loader.Loader ketl.transformer ketl.transformer.Transformer ketl.utils ketl.utils.db_utils ketl.utils.file_utils

N

name (ketl.db.models.API attribute)

NoValidSourcesError

P

ParquetLoader (class in ketl.loader.Loader) path (ketl.db.models.CachedFile attribute) (ketl.db.models.ExpectedFile attribute) PickleLoader (class in ketl.loader.Loader)

post() (ketl.extractor.Rest.RestMixin method) preprocess() (ketl.db.models.CachedFile method) processed (ketl.db.models.ExpectedFile attribute) put() (ketl.extractor.Rest.RestMixin method)

R

refresh_interval (ketl.db.models.CachedFile attribute)

RestMixin (class in ketl.extractor.Rest)

S

self (ketl.db.models.ExpectedMode attribute) setup() (ketl.db.models.API method) size (ketl.db.models.CachedFile attribute) (ketl.db.models.ExpectedFile attribute) Source (class in ketl.db.models)

source (ketl.db.models.CachedFile attribute) source_files (ketl.db.models.Source attribute) source_hash (ketl.db.models.Source property) source_id (ketl.db.models.CachedFile attribute) source_type (ketl.db.models.Source attribute) sources (ketl.db.models.API attribute)

T

transform() (ketl.transformer.Transformer.BaseTransformer method) (ketl.transformer.Transformer.DelimitedTableTransformer method) (ketl.transformer.Transformer.JsonTableTransformer method)

U

url (ketl.db.models.CachedFile attribute)

url_params (ketl.db.models.CachedFile attribute)