From 7dd7d5020d70fe9d7c067920f3a0f91ba9d694fd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jo=C3=A3o=20Ara=C3=BAjo?= Date: Fri, 10 Jun 2022 03:22:28 -0300 Subject: [PATCH 1/2] fix: inclui exemplos testados e python client com docs --- .../builder/builder_horovod.py | 142 +++++++++ learning_orchestra_client/train/horovod.py | 97 +++++++ pipeline/builder_mnist_horovod.py | 92 ++++++ pipeline/mnist_distributed_training.py | 271 ++++++++++++++++++ 4 files changed, 602 insertions(+) create mode 100644 learning_orchestra_client/builder/builder_horovod.py create mode 100644 learning_orchestra_client/train/horovod.py create mode 100644 pipeline/builder_mnist_horovod.py create mode 100644 pipeline/mnist_distributed_training.py diff --git a/learning_orchestra_client/builder/builder_horovod.py b/learning_orchestra_client/builder/builder_horovod.py new file mode 100644 index 0000000..573c24c --- /dev/null +++ b/learning_orchestra_client/builder/builder_horovod.py @@ -0,0 +1,142 @@ +from learning_orchestra_client.observe.observe import Observer +from learning_orchestra_client._util._response_treat import ResponseTreat +from learning_orchestra_client._util._entity_reader import EntityReader +import requests +from typing import Union + + +class BuilderHorovod: + __BUILDER_NAME = 'name', + __DESCRIPTION_NAME = 'description' + __MODELING_CODE_NAME = 'code', + __PARAMETERS = 'parameters' + + def __init__(self, cluster_ip: str): + self.__api_path = "/api/learningOrchestra/v1/builder/horovod" + self.__service_url = f'{cluster_ip}{self.__api_path}' + self.__response_treat = ResponseTreat() + self.__cluster_ip = cluster_ip + self.__entity_reader = EntityReader(self.__service_url) + self.__observer = Observer(self.__cluster_ip) + + def run_horovod_sync(self, builder_name: str, modeling_code: str, parameters: dict = dict({}), + description: str = '', + pretty_response: bool = False) -> \ + Union[dict, str]: + """ + description: This method call runs several steps of a machine + learning pipeline (transform, tune, train and evaluate, for instance) + using a model code. It represents a way to run + an entire pipeline. The caller waits until the method execution ends, + since it is a synchronous method. + + modeling_code: Represent python3 code used to build the pipeline. + builder_name: Represent the builder name that is accessible later via API. + parameters: Represent parameters in the cluster that is needed on the pipeline + + pretty_response: if True it represents a result useful for visualization + + return: The response of such python3 code. + """ + + request_body_content = { + self.__DESCRIPTION_NAME: description, + self.__BUILDER_NAME: builder_name, + self.__MODELING_CODE_NAME: modeling_code, + self.__PARAMETERS: parameters, + } + + response = requests.post(url=self.__service_url, + json=request_body_content) + + self.__observer.wait(builder_name) + + return self.__response_treat.treatment(response, pretty_response) + + def run_horovod_async(self, builder_name: str, modeling_code: str, parameters: dict = dict({}), + description: str = '', + pretty_response: bool = False) -> Union[dict, str]: + """ + description: This method call runs several steps of a machine + learning pipeline (transform, tune, train and evaluate, for instance) + using a model code. It represents a way to run + an entire pipeline. The caller waits until the method execution ends, + since it is a synchronous method. + + modeling_code: Represent python3 code used to build the pipeline. + builder_name: Represent the builder name that is accessible later via API. + parameters: Represent parameters in the cluster that is needed on the pipeline + + pretty_response: if True it represents a result useful for visualization + + return: The response of such python3 code. + """ + + request_body_content = { + self.__DESCRIPTION_NAME: description, + self.__BUILDER_NAME: builder_name, + self.__MODELING_CODE_NAME: modeling_code, + self.__PARAMETERS: parameters, + } + + response = requests.post(url=self.__service_url, + json=request_body_content) + + return self.__response_treat.treatment(response, pretty_response) + + def get_builder(self, builder_name: str, query: dict = {}, limit: int = 10, skip: int = 0, + pretty_response: bool = False) \ + -> Union[dict, str]: + """ + description: This method is responsible for retrieving the model + predictions content. + + pretty_response: If true it returns a string, otherwise a dictionary. + builder_name: Represents the model predictions name. + query: Query to make in MongoDB(default: empty query) + limit: Number of rows to return in pagination(default: 10) (maximum is + set at 20 rows per request) + skip: Number of rows to skip in pagination(default: 0) + + return: A page with some tuples or registers inside or an error if the + pipeline runs incorrectly. The current page is also returned to be used + in future content requests. + """ + response = self.__entity_reader.read_entity_content( + builder_name, query, limit, skip) + + return self.__response_treat.treatment(response, pretty_response) + + def delete_builder(self, builder_name: str, pretty_response: bool = False) \ + -> Union[dict, str]: + """ + description: This method is responsible for deleting a model prediction. + The delete operation is always asynchronous, + since the deletion is performed in background. + + pretty_response: If true it returns a string, otherwise a dictionary. + builder_name: Represents the pipeline name. + + return: JSON object with an error message, a warning message or a + correct delete message + """ + + cluster_url_dataset = f'{self.__service_url}/{builder_name}' + + response = requests.delete(cluster_url_dataset) + + return self.__response_treat.treatment(response, pretty_response) + + def wait(self, builder_name: str, timeout: int = None) -> dict: + """ + description: This method is responsible to create a synchronization + barrier for the run_horovod_async method. + + dataset_name: Represents the pipeline name. + timeout: Represents the time in seconds to wait for a builder to + finish its run. + + return: JSON object with an error message, a warning message or a + correct execution of a pipeline + """ + return self.__observer.wait(builder_name, timeout) diff --git a/learning_orchestra_client/train/horovod.py b/learning_orchestra_client/train/horovod.py new file mode 100644 index 0000000..93edd44 --- /dev/null +++ b/learning_orchestra_client/train/horovod.py @@ -0,0 +1,97 @@ +from ._train import Train +import requests +from typing import Union + + +class TrainHorovod(Train): + __PARENT_NAME_FIELD = "parentName" + __METHOD_NAME_FIELD = "method" + __ClASS_PARAMETERS_FIELD = "methodParameters" + __NAME_FIELD = "name" + __DESCRIPTION_FIELD = "description" + __COMPILE_CODE = "compileCode" + __MONITORING_PATH = "monitoringPath" + + def __init__(self, cluster_ip: str): + self.__api_path = "/api/learningOrchestra/v1/train/horovod" + self.__cluster_ip = cluster_ip + super().__init__(cluster_ip, self.__api_path) + + def create_training_async(self, + name: str, + model_name: str, + parent_name: str, + parameters: dict, + description: str = "", + compiling_code: str = "", + monitoring_path: str = None, + pretty_response: bool = False) -> \ + Union[dict, str]: + """ + description: This method is responsible to train models in async mode. + A wait method call is mandatory due to the asynchronous aspect. + + pretty_response: If true it returns a string, otherwise a dictionary. + name: Is the name of the train output object that will be created. + parent_name: Is the name of the previous ML step of the pipeline + method_name: is the name of the method to be executed (the ML tool way + to train models) + parameters: Is the set of parameters used by the method + + return: A JSON object with an error or warning message or a URL + indicating the correct operation. + """ + request_body = { + self.__NAME_FIELD: name, + self.__MODEL_NAME_FIELD: model_name, + self.__PARENT_NAME_FIELD: parent_name, + self.__ClASS_PARAMETERS_FIELD: parameters, + self.__DESCRIPTION_FIELD: description, + self.__COMPILE_CODE: compiling_code, + self.__MONITORING_PATH: monitoring_path, + } + + request_url = self.__service_url + + response = requests.post(url=request_url, json=request_body) + return self.__response_treat.treatment(response, pretty_response) + + def create_training_sync(self, + name: str, + model_name: str, + parent_name: str, + parameters: dict, + description: str = "", + compiling_code: str = "", + monitoring_path: str = None, + pretty_response: bool = False) -> \ + Union[dict, str]: + """ + description: This method is responsible to train models in sync mode + + pretty_response: If true it returns a string, otherwise a dictionary. + name: Is the name of the train output object that will be created. + parent_name: Is the name of the previous ML step of the pipeline + method_name: is the name of the method to be executed (the ML tool way + to train models) + parameters: Is the set of parameters used by the method + + return: A JSON object with an error or warning message or a URL + indicating the correct operation. + """ + request_body = { + self.__NAME_FIELD: name, + self.__MODEL_NAME_FIELD: model_name, + self.__PARENT_NAME_FIELD: parent_name, + self.__ClASS_PARAMETERS_FIELD: parameters, + self.__DESCRIPTION_FIELD: description, + self.__COMPILE_CODE: compiling_code, + self.__MONITORING_PATH: monitoring_path, + } + + request_url = self.__service_url + + response = requests.post(url=request_url, json=request_body) + self.__observer.wait(name) + + return self.__response_treat.treatment(response, pretty_response) diff --git a/pipeline/builder_mnist_horovod.py b/pipeline/builder_mnist_horovod.py new file mode 100644 index 0000000..2184c66 --- /dev/null +++ b/pipeline/builder_mnist_horovod.py @@ -0,0 +1,92 @@ +from learning_orchestra_client.builder.builder_horovod import BuilderHorovod + +code = """ +def train(num_epochs): + import tensorflow as tf + import horovod.tensorflow.keras as hvd + hvd.init() + + (x_train, y_train), (x_test, y_test) = tf.keras.datasets.cifar10.load_data() + y_train = tf.keras.utils.to_categorical(y_train, 10) + y_test = tf.keras.utils.to_categorical(y_test, 10) + + train_dat = tf.keras.preprocessing.image.ImageDataGenerator( + width_shift_range=0.33, height_shift_range=0.33, zoom_range=0.5, horizontal_flip=True, + preprocessing_function=tf.keras.applications.resnet50.preprocess_input + ) + + test_dat = tf.keras.preprocessing.image.ImageDataGenerator( + zoom_range=(0.875, 0.875), + preprocessing_function=tf.keras.applications.resnet50.preprocess_input + ) + + model = tf.keras.applications.ResNet50( + include_top=False, + weights=None, + input_shape=[32, 32, 3], + classes=10, + ) + + scaled_lr = 0.0125 * hvd.size() + opt = tf.optimizers.SGD(lr=scaled_lr, momentum=0.9) + + opt = hvd.DistributedOptimizer( + opt + ) + + model_config = model.get_config() + for layer, layer_config in zip(model.layers, model_config['layers']): + if hasattr(layer, 'kernel_regularizer'): + regularizer = tf.keras.regularizers.l2(0.00005) + layer_config['config']['kernel_regularizer'] = \ + {'class_name': regularizer.__class__.__name__, + 'config': regularizer.get_config()} + if type(layer) == tf.keras.layers.BatchNormalization: + layer_config['config']['momentum'] = 0.9 + layer_config['config']['epsilon'] = 1e-5 + + model = tf.keras.models.Model.from_config(model_config) + + model.compile( + loss=tf.losses.categorical_crossentropy, + optimizer=opt, + metrics=['accuracy', 'top_k_categorical_accuracy'], + ) + + callbacks = [ + hvd.callbacks.BroadcastGlobalVariablesCallback(0), + hvd.callbacks.MetricAverageCallback(), + hvd.callbacks.LearningRateWarmupCallback( + initial_lr=scaled_lr, warmup_epochs=5, verbose=1 + ) + ] + + if hvd.rank() == 0: + callbacks.append(tf.keras.callbacks.ModelCheckpoint("./checkpoint-{epoch}.h5")) + callbacks.append(tf.keras.callbacks.TensorBoard('./logs')) + + verbose = 1 if hvd.rank() == 0 else 0 + + model.fit_generator(train_dat.flow(x_train, y_train, batch_size=32), + steps_per_epoch=len(x_train) // hvd.size(), + callbacks=callbacks, + epochs=num_epochs, + verbose=verbose, + validation_data=test_dat.flow(x_test, y_test, batch_size=32), + validation_steps=3 * len(x_test) // hvd.size(), + ) + + score = hvd.allreduce(model.evaluate(test_dat.flow(x_test, y_test))) + + if verbose: + print('Test loss:', score[0]) + print('Test accuracy:', score[1]) +""" + +if __name__ == '__main__': + CLUSTER_IP = "http://34.125.36.237" + builder = BuilderHorovod(CLUSTER_IP) + builder.run_horovod_sync( + 'my builder', + code, + ) diff --git a/pipeline/mnist_distributed_training.py b/pipeline/mnist_distributed_training.py new file mode 100644 index 0000000..0e2f296 --- /dev/null +++ b/pipeline/mnist_distributed_training.py @@ -0,0 +1,271 @@ +from learning_orchestra_client.dataset.generic import DatasetGeneric +from learning_orchestra_client.function.python import FunctionPython +from learning_orchestra_client.model.tensorflow import ModelTensorflow +from learning_orchestra_client.train.horovod import TrainHorovod +from learning_orchestra_client.predict.tensorflow import PredictTensorflow +from learning_orchestra_client.evaluate.tensorflow import EvaluateTensorflow + +CLUSTER_IP = "http://34.125.36.237" + +dataset_generic = DatasetGeneric(CLUSTER_IP) +dataset_generic.insert_dataset_async( + dataset_name="mnist_train_images", + url="https://drive.google.com/u/0/uc?" + "id=1ec6hVwvq4UPyQ7DmJVxzofE2TcKUTHNY&export=download", +) + +dataset_generic.insert_dataset_async( + dataset_name="mnist_train_labels", + url="https://drive.google.com/u/0/uc?" + "id=187ID_LfQCTJOYieC-yo94jxnWiFJZ7uX&export=download", +) + +dataset_generic.insert_dataset_async( + dataset_name="mnist_test_images", + url="https://drive.google.com/u/0/uc?" + "id=1ZNuiRJLKSFzmegRgIHXUl4t-UdjEPYVf&export=download", +) + +dataset_generic.insert_dataset_async( + dataset_name="mnist_test_labels", + url="https://drive.google.com/u/0/uc?" + "id=1v0PRmUL8nOg3mHakjfTxOjNA9bi6XkkA&export=download", +) + +dataset_generic.wait("mnist_train_images") +dataset_generic.wait("mnist_train_labels") +dataset_generic.wait("mnist_test_images") +dataset_generic.wait("mnist_test_labels") + +function_python = FunctionPython(CLUSTER_IP) +mnist_datasets_treatment = ''' +import numpy as np +import struct as st +import math +training_filenames = {'images': mnist_train_images, + 'labels': mnist_train_labels} +test_filenames = {'images': mnist_test_images, + 'labels': mnist_test_labels} +data_types = { + 0x08: ('ubyte', 'B', 1), + 0x09: ('byte', 'b', 1), + 0x0B: ('>i2', 'h', 2), + 0x0C: ('>i4', 'i', 4), + 0x0D: ('>f4', 'f', 4), + 0x0E: ('>f8', 'd', 8)} +def treat_dataset(dataset: dict) -> tuple: + global np, st, math, data_types + for name in dataset.keys(): + if name == 'images': + images_file = dataset[name] + if name == 'labels': + labels_file = dataset[name] + images_file.seek(0) + magic = st.unpack('>4B', images_file.read(4)) + data_format = data_types[magic[2]][1] + data_size = data_types[magic[2]][2] + images_file.seek(4) + content_amount = st.unpack('>I', images_file.read(4))[0] + rows_amount = st.unpack('>I', images_file.read(4))[0] + columns_amount = st.unpack('>I', images_file.read(4))[0] + labels_file.seek(8) + labels_array = np.asarray( + st.unpack( + '>' + data_format * content_amount, + labels_file.read(content_amount * data_size))).reshape( + (content_amount, 1)) + n_batch = 10000 + n_iter = int(math.ceil(content_amount / n_batch)) + n_bytes = n_batch * rows_amount * columns_amount * data_size + images_array = np.array([]) + for i in range(0, n_iter): + temp_images_array = np.asarray( + st.unpack('>' + data_format * n_bytes, + images_file.read(n_bytes))).reshape( + (n_batch, rows_amount, columns_amount)) + if images_array.size == 0: + images_array = temp_images_array + else: + images_array = np.vstack((images_array, temp_images_array)) + temp_images_array = np.array([]) + return images_array, labels_array +train_images, train_labels = treat_dataset(training_filenames) +test_images, test_labels = treat_dataset(test_filenames) +response = { + "train_images": train_images, + "train_labels": train_labels, + "test_images": test_images, + "test_labels": test_labels, +} +''' + +function_python.run_function_async( + name="mnist_datasets_treated", + parameters={ + "mnist_train_images": "$mnist_train_images", + "mnist_train_labels": "$mnist_train_labels", + "mnist_test_images": "$mnist_test_images", + "mnist_test_labels": "$mnist_test_labels" + }, + code=mnist_datasets_treatment) +function_python.wait("mnist_datasets_treated") + +mnist_datasets_normalization = ''' +test_images = test_images / 255 +train_images = train_images / 255 +print(train_images.shape) +print(train_labels.shape) +print(test_images.shape) +print(test_labels.shape) +response = { + "test_images": test_images, + "test_labels": test_labels, + "train_images": train_images, + "train_labels": train_labels +} +''' + +function_python.run_function_async( + name="mnist_datasets_normalized", + parameters={ + "train_images": "$mnist_datasets_treated.train_images", + "train_labels": "$mnist_datasets_treated.train_labels", + "test_images": "$mnist_datasets_treated.test_images", + "test_labels": "$mnist_datasets_treated.test_labels" + }, + code=mnist_datasets_normalization) +function_python.wait("mnist_datasets_normalized") + +model_tensorflow = ModelTensorflow(CLUSTER_IP) +model_tensorflow.create_model_async( + name="mnist_model", + module_path="tensorflow.keras.models", + class_name="Sequential", + class_parameters={ + "layers": + [ + "#tensorflow.keras.layers.Flatten(input_shape=(28, 28))", + "#tensorflow.keras.layers.Dense(128, activation='relu')", + "#tensorflow.keras.layers.Dense(10, activation='softmax')", + ]} +) +model_tensorflow.wait("mnist_model") + +model_compilation = ''' +import tensorflow as tf +model.compile( + optimizer=tf.keras.optimizers.Adam(0.001), + loss=tf.keras.losses.SparseCategoricalCrossentropy(), + metrics=[tf.keras.metrics.SparseCategoricalAccuracy()], +) +response = model +''' + +function_python.run_function_async( + name="mnist_model_compiled", + parameters={ + "model": "$mnist_model" + }, + code=model_compilation) +function_python.wait("mnist_model_compiled") + +distributed_model_compilation = ''' +import tensorflow as tf +import horovod.tensorflow.keras as hvd +hvd.init() +mnist_model.compile( + optimizer=hvd.DistributedOptimizer(tf.optimizers.Adam(0.001* hvd.size()), + loss=tf.keras.losses.SparseCategoricalCrossentropy(), + metrics=[tf.keras.metrics.SparseCategoricalAccuracy()], +) +response = mnist_model +''' + +train_horovod = TrainHorovod(CLUSTER_IP) +train_horovod.create_training_async( + name="mnist_model_trained", + model_name="mnist_model", + parent_name="mnist_model_compiled", + compiling_code=distributed_model_compilation, + parameters={ + "x": "$mnist_datasets_normalized.train_images", + "y": "$mnist_datasets_normalized.train_labels", + "validation_split": 0.1, + "epochs": 5, + "callbacks": [ + '#hvd.callbacks.BroadcastGlobalVariablesCallback(0)', + '#hvd.callbacks.MetricAverageCallback()', + '#hvd.callbacks.LearningRateWarmupCallback(initial_lr=0.0125 * hvd.size(), warmup_epochs=3, verbose=1)', + ], + "rank0callbacks": [ + '#tensorflow.keras.callbacks.TensorBoard(histogram_freq=1)' + ] + }, + monitoring_path='logs/' +) +train_horovod.wait("mnist_model_trained") + +predict_tensorflow = PredictTensorflow(CLUSTER_IP) +predict_tensorflow.create_prediction_async( + name="mnist_model_predicted", + model_name="mnist_model", + parent_name="mnist_model_trained", + method_name="predict", + parameters={ + "x": "$mnist_datasets_normalized.test_images" + } +) + +evaluate_tensorflow = EvaluateTensorflow(CLUSTER_IP) +evaluate_tensorflow.create_evaluate_async( + name="mnist_model_evaluated", + model_name="mnist_model", + parent_name="mnist_model_trained", + method_name="evaluate", + parameters={ + "x": "$mnist_datasets_normalized.test_images", + "y": "$mnist_datasets_normalized.test_labels" + } +) + +predict_tensorflow.wait("mnist_model_predicted") +evaluate_tensorflow.wait("mnist_model_evaluated") + +show_mnist_predict = ''' +print(mnist_predicted) +response = None +''' +function_python.run_function_async( + name="mnist_model_predicted_print", + parameters={ + "mnist_predicted": "$mnist_model_predicted" + }, + code=show_mnist_predict +) + +show_mnist_evaluate = ''' +print(mnist_evaluated) +response = None +''' +function_python.run_function_async( + name="mnist_model_evaluated_print", + parameters={ + "mnist_evaluated": "$mnist_model_evaluated" + }, + code=show_mnist_evaluate +) + +function_python.wait("mnist_model_evaluated_print") +function_python.wait("mnist_model_predicted_print") + +print(function_python.search_execution_content( + name="mnist_model_predicted_print", + limit=1, + skip=1, + pretty_response=True)) + +print(function_python.search_execution_content( + name="mnist_model_evaluated_print", + limit=1, + skip=1, + pretty_response=True)) From 1987bae51dae99b9a992c68a234799efb7be44d9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jo=C3=A3o=20Ara=C3=BAjo?= Date: Fri, 10 Jun 2022 03:22:53 -0300 Subject: [PATCH 2/2] inclui html para docs --- .../builder/builder.html | 754 ++++++++++++++++++ .../builder/builder_horovod.html | 579 ++++++++++++++ .../builder/index.html | 70 ++ .../dataset/csv.html | 97 +++ .../dataset/generic.html | 97 +++ .../dataset/index.html | 70 ++ .../evaluate/index.html | 70 ++ .../evaluate/scikitlearn.html | 97 +++ .../evaluate/tensorflow.html | 97 +++ .../explore/histogram.html | 645 +++++++++++++++ .../explore/index.html | 75 ++ .../explore/scikitlearn.html | 97 +++ .../explore/tensorflow.html | 97 +++ .../function/index.html | 65 ++ .../function/python.html | 667 ++++++++++++++++ html/learning_orchestra_client/index.html | 105 +++ .../model/index.html | 70 ++ .../model/scikitlearn.html | 97 +++ .../model/tensorflow.html | 97 +++ .../observe/index.html | 65 ++ .../observe/observe.html | 374 +++++++++ .../predict/index.html | 70 ++ .../predict/scikitlearn.html | 97 +++ .../predict/tensorflow.html | 97 +++ .../train/horovod.html | 396 +++++++++ .../train/index.html | 75 ++ .../train/scikitlearn.html | 109 +++ .../train/tensorflow.html | 109 +++ .../transform/data_type.html | 606 ++++++++++++++ .../transform/index.html | 80 ++ .../transform/projection.html | 657 +++++++++++++++ .../transform/scikitlearn.html | 97 +++ .../transform/tensorflow.html | 97 +++ 33 files changed, 6875 insertions(+) create mode 100644 html/learning_orchestra_client/builder/builder.html create mode 100644 html/learning_orchestra_client/builder/builder_horovod.html create mode 100644 html/learning_orchestra_client/builder/index.html create mode 100644 html/learning_orchestra_client/dataset/csv.html create mode 100644 html/learning_orchestra_client/dataset/generic.html create mode 100644 html/learning_orchestra_client/dataset/index.html create mode 100644 html/learning_orchestra_client/evaluate/index.html create mode 100644 html/learning_orchestra_client/evaluate/scikitlearn.html create mode 100644 html/learning_orchestra_client/evaluate/tensorflow.html create mode 100644 html/learning_orchestra_client/explore/histogram.html create mode 100644 html/learning_orchestra_client/explore/index.html create mode 100644 html/learning_orchestra_client/explore/scikitlearn.html create mode 100644 html/learning_orchestra_client/explore/tensorflow.html create mode 100644 html/learning_orchestra_client/function/index.html create mode 100644 html/learning_orchestra_client/function/python.html create mode 100644 html/learning_orchestra_client/index.html create mode 100644 html/learning_orchestra_client/model/index.html create mode 100644 html/learning_orchestra_client/model/scikitlearn.html create mode 100644 html/learning_orchestra_client/model/tensorflow.html create mode 100644 html/learning_orchestra_client/observe/index.html create mode 100644 html/learning_orchestra_client/observe/observe.html create mode 100644 html/learning_orchestra_client/predict/index.html create mode 100644 html/learning_orchestra_client/predict/scikitlearn.html create mode 100644 html/learning_orchestra_client/predict/tensorflow.html create mode 100644 html/learning_orchestra_client/train/horovod.html create mode 100644 html/learning_orchestra_client/train/index.html create mode 100644 html/learning_orchestra_client/train/scikitlearn.html create mode 100644 html/learning_orchestra_client/train/tensorflow.html create mode 100644 html/learning_orchestra_client/transform/data_type.html create mode 100644 html/learning_orchestra_client/transform/index.html create mode 100644 html/learning_orchestra_client/transform/projection.html create mode 100644 html/learning_orchestra_client/transform/scikitlearn.html create mode 100644 html/learning_orchestra_client/transform/tensorflow.html diff --git a/html/learning_orchestra_client/builder/builder.html b/html/learning_orchestra_client/builder/builder.html new file mode 100644 index 0000000..c16eb9e --- /dev/null +++ b/html/learning_orchestra_client/builder/builder.html @@ -0,0 +1,754 @@ + + + + + + +learning_orchestra_client.builder.builder API documentation + + + + + + + + + + + +
+
+
+

Module learning_orchestra_client.builder.builder

+
+
+
+ +Expand source code + +
from learning_orchestra_client.observe.observe import Observer
+from learning_orchestra_client._util._response_treat import ResponseTreat
+from learning_orchestra_client._util._entity_reader import EntityReader
+import requests
+from typing import Union
+
+
+class BuilderSparkMl:
+    __TRAIN_FIELD = "trainDatasetName"
+    __TEST_FIELD = "testDatasetName"
+    __CODE_FIELD = "modelingCode"
+    __CLASSIFIERS_LIST_FIELD = "classifiersList"
+
+    def __init__(self, cluster_ip: str):
+        self.__api_path = "/api/learningOrchestra/v1/builder/sparkml"
+        self.__service_url = f'{cluster_ip}{self.__api_path}'
+        self.__response_treat = ResponseTreat()
+        self.__cluster_ip = cluster_ip
+        self.__entity_reader = EntityReader(self.__service_url)
+        self.__observer = Observer(self.__cluster_ip)
+
+    def run_spark_ml_sync(self,
+                          train_dataset_name: str,
+                          test_dataset_name: str,
+                          modeling_code: str,
+                          model_classifiers: list,
+                          pretty_response: bool = False) -> Union[dict, str]:
+        """
+        description: This method call runs several steps of a machine
+        learning pipeline (transform, tune, train and evaluate, for instance)
+        using a model code and several classifiers. It represents a way to run
+        an entire pipeline. The caller waits until the method execution ends,
+        since it is a synchronous method.
+
+        train_dataset_name: Represent final train dataset.
+        test_dataset_name: Represent final test dataset.
+        modeling_code: Represent Python3 code for pyspark pre-processing model
+        model_classifiers: list of initial classifiers to be used in the model
+        pretty_response: if True it represents a result useful for visualization
+
+        return: The set of predictions (URIs of them).
+        """
+
+        request_body_content = {
+            self.__TRAIN_FIELD: train_dataset_name,
+            self.__TEST_FIELD: test_dataset_name,
+            self.__CODE_FIELD: modeling_code,
+            self.__CLASSIFIERS_LIST_FIELD: model_classifiers,
+        }
+        response = requests.post(url=self.__service_url,
+                                 json=request_body_content)
+
+        for classifier in model_classifiers:
+            self.__observer.wait(f'{test_dataset_name}{classifier}')
+
+        return self.__response_treat.treatment(response, pretty_response)
+
+    def run_spark_ml_async(self,
+                           train_dataset_name: str,
+                           test_dataset_name: str,
+                           modeling_code: str,
+                           model_classifiers: list,
+                           pretty_response: bool = False) -> Union[dict, str]:
+        """
+        description: This method call runs several steps of a machine
+        learning pipeline (transform, tune, train and evaluate, for instance)
+        using a model code and several classifiers. It represents a way to run
+        an entire pipeline. The caller does not wait until the method execution
+        ends, since it is an asynchronous method.
+
+        train_dataset_name: Represent final train dataset.
+        test_dataset_name: Represent final test dataset.
+        modeling_code: Represent Python3 code for pyspark pre-processing model
+        model_classifiers: list of initial classifiers to be used in the model
+        pretty_response: if True it represents a result useful for visualization
+
+        return: the URL to retrieve the Spark pipeline result
+        """
+
+        request_body_content = {
+            self.__TRAIN_FIELD: train_dataset_name,
+            self.__TEST_FIELD: test_dataset_name,
+            self.__CODE_FIELD: modeling_code,
+            self.__CLASSIFIERS_LIST_FIELD: model_classifiers,
+        }
+        response = requests.post(url=self.__service_url,
+                                 json=request_body_content)
+
+        return self.__response_treat.treatment(response, pretty_response)
+
+    def search_all_builders(self, pretty_response: bool = False) \
+            -> Union[dict, str]:
+        """
+        description: This method retrieves all model predictions metadata. It
+        does not retrieve the model predictions content.
+
+        pretty_response: If true it returns a string, otherwise a dictionary.
+
+        return: A list with all model predictions metadata stored in Learning
+        Orchestra or an empty result.
+        """
+
+        response = self.__entity_reader.read_all_instances_from_entity()
+
+        return self.__response_treat.treatment(response, pretty_response)
+
+    def search_builder_register_predictions(self,
+                                            builder_name: str,
+                                            query: dict = {},
+                                            limit: int = 10,
+                                            skip: int = 0,
+                                            pretty_response: bool = False) \
+            -> Union[dict, str]:
+        """
+        description: This method is responsible for retrieving the model
+        predictions content.
+
+        pretty_response: If true it returns a string, otherwise a dictionary.
+        builder_name: Represents the model predictions name.
+        query: Query to make in MongoDB(default: empty query)
+        limit: Number of rows to return in pagination(default: 10) (maximum is
+        set at 20 rows per request)
+        skip: Number of rows to skip in pagination(default: 0)
+
+        return: A page with some tuples or registers inside or an error if the
+        pipeline runs incorrectly. The current page is also returned to be used
+        in future content requests.
+        """
+
+        response = self.__entity_reader.read_entity_content(
+            builder_name, query, limit, skip)
+
+        return self.__response_treat.treatment(response, pretty_response)
+
+    def search_builder(self, builder_name: str, pretty_response: bool = False) \
+            -> Union[dict, str]:
+        """
+        description:  This method is responsible for retrieving a specific
+        model metadata.
+
+        pretty_response: If true return indented string, else return dict.
+        builder_name: Represents the model predictions name.
+        limit: Number of rows to return in pagination(default: 10) (maximum is
+        set at 20 rows per request)
+        skip: Number of rows to skip in pagination(default: 0)
+
+        return: Specific model prediction metadata stored in Learning Orchestra
+        or an error if there is no such projections.
+        """
+        response = self.search_builder_register_predictions(
+            builder_name, limit=1,
+            pretty_response=pretty_response)
+
+        return response
+
+    def delete_builder(self, builder_name: str, pretty_response: bool = False) \
+            -> Union[dict, str]:
+        """
+        description: This method is responsible for deleting a model prediction.
+        The delete operation is always asynchronous,
+        since the deletion is performed in background.
+
+        pretty_response: If true it returns a string, otherwise a dictionary.
+        builder_name: Represents the pipeline name.
+
+        return: JSON object with an error message, a warning message or a
+        correct delete message
+        """
+
+        cluster_url_dataset = f'{self.__service_url}/{builder_name}'
+
+        response = requests.delete(cluster_url_dataset)
+
+        return self.__response_treat.treatment(response, pretty_response)
+
+    def wait(self, dataset_name: str, timeout: int = None) -> dict:
+        """
+           description: This method is responsible to create a synchronization
+           barrier for the run_spark_ml_async method.
+
+           dataset_name: Represents the pipeline name.
+           timeout: Represents the time in seconds to wait for a builder to
+           finish its run.
+
+           return: JSON object with an error message, a warning message or a
+           correct execution of a pipeline
+        """
+        return self.__observer.wait(dataset_name, timeout)
+
+
+
+
+
+
+
+
+
+

Classes

+
+
+class BuilderSparkMl +(cluster_ip: str) +
+
+
+
+ +Expand source code + +
class BuilderSparkMl:
+    __TRAIN_FIELD = "trainDatasetName"
+    __TEST_FIELD = "testDatasetName"
+    __CODE_FIELD = "modelingCode"
+    __CLASSIFIERS_LIST_FIELD = "classifiersList"
+
+    def __init__(self, cluster_ip: str):
+        self.__api_path = "/api/learningOrchestra/v1/builder/sparkml"
+        self.__service_url = f'{cluster_ip}{self.__api_path}'
+        self.__response_treat = ResponseTreat()
+        self.__cluster_ip = cluster_ip
+        self.__entity_reader = EntityReader(self.__service_url)
+        self.__observer = Observer(self.__cluster_ip)
+
+    def run_spark_ml_sync(self,
+                          train_dataset_name: str,
+                          test_dataset_name: str,
+                          modeling_code: str,
+                          model_classifiers: list,
+                          pretty_response: bool = False) -> Union[dict, str]:
+        """
+        description: This method call runs several steps of a machine
+        learning pipeline (transform, tune, train and evaluate, for instance)
+        using a model code and several classifiers. It represents a way to run
+        an entire pipeline. The caller waits until the method execution ends,
+        since it is a synchronous method.
+
+        train_dataset_name: Represent final train dataset.
+        test_dataset_name: Represent final test dataset.
+        modeling_code: Represent Python3 code for pyspark pre-processing model
+        model_classifiers: list of initial classifiers to be used in the model
+        pretty_response: if True it represents a result useful for visualization
+
+        return: The set of predictions (URIs of them).
+        """
+
+        request_body_content = {
+            self.__TRAIN_FIELD: train_dataset_name,
+            self.__TEST_FIELD: test_dataset_name,
+            self.__CODE_FIELD: modeling_code,
+            self.__CLASSIFIERS_LIST_FIELD: model_classifiers,
+        }
+        response = requests.post(url=self.__service_url,
+                                 json=request_body_content)
+
+        for classifier in model_classifiers:
+            self.__observer.wait(f'{test_dataset_name}{classifier}')
+
+        return self.__response_treat.treatment(response, pretty_response)
+
+    def run_spark_ml_async(self,
+                           train_dataset_name: str,
+                           test_dataset_name: str,
+                           modeling_code: str,
+                           model_classifiers: list,
+                           pretty_response: bool = False) -> Union[dict, str]:
+        """
+        description: This method call runs several steps of a machine
+        learning pipeline (transform, tune, train and evaluate, for instance)
+        using a model code and several classifiers. It represents a way to run
+        an entire pipeline. The caller does not wait until the method execution
+        ends, since it is an asynchronous method.
+
+        train_dataset_name: Represent final train dataset.
+        test_dataset_name: Represent final test dataset.
+        modeling_code: Represent Python3 code for pyspark pre-processing model
+        model_classifiers: list of initial classifiers to be used in the model
+        pretty_response: if True it represents a result useful for visualization
+
+        return: the URL to retrieve the Spark pipeline result
+        """
+
+        request_body_content = {
+            self.__TRAIN_FIELD: train_dataset_name,
+            self.__TEST_FIELD: test_dataset_name,
+            self.__CODE_FIELD: modeling_code,
+            self.__CLASSIFIERS_LIST_FIELD: model_classifiers,
+        }
+        response = requests.post(url=self.__service_url,
+                                 json=request_body_content)
+
+        return self.__response_treat.treatment(response, pretty_response)
+
+    def search_all_builders(self, pretty_response: bool = False) \
+            -> Union[dict, str]:
+        """
+        description: This method retrieves all model predictions metadata. It
+        does not retrieve the model predictions content.
+
+        pretty_response: If true it returns a string, otherwise a dictionary.
+
+        return: A list with all model predictions metadata stored in Learning
+        Orchestra or an empty result.
+        """
+
+        response = self.__entity_reader.read_all_instances_from_entity()
+
+        return self.__response_treat.treatment(response, pretty_response)
+
+    def search_builder_register_predictions(self,
+                                            builder_name: str,
+                                            query: dict = {},
+                                            limit: int = 10,
+                                            skip: int = 0,
+                                            pretty_response: bool = False) \
+            -> Union[dict, str]:
+        """
+        description: This method is responsible for retrieving the model
+        predictions content.
+
+        pretty_response: If true it returns a string, otherwise a dictionary.
+        builder_name: Represents the model predictions name.
+        query: Query to make in MongoDB(default: empty query)
+        limit: Number of rows to return in pagination(default: 10) (maximum is
+        set at 20 rows per request)
+        skip: Number of rows to skip in pagination(default: 0)
+
+        return: A page with some tuples or registers inside or an error if the
+        pipeline runs incorrectly. The current page is also returned to be used
+        in future content requests.
+        """
+
+        response = self.__entity_reader.read_entity_content(
+            builder_name, query, limit, skip)
+
+        return self.__response_treat.treatment(response, pretty_response)
+
+    def search_builder(self, builder_name: str, pretty_response: bool = False) \
+            -> Union[dict, str]:
+        """
+        description:  This method is responsible for retrieving a specific
+        model metadata.
+
+        pretty_response: If true return indented string, else return dict.
+        builder_name: Represents the model predictions name.
+        limit: Number of rows to return in pagination(default: 10) (maximum is
+        set at 20 rows per request)
+        skip: Number of rows to skip in pagination(default: 0)
+
+        return: Specific model prediction metadata stored in Learning Orchestra
+        or an error if there is no such projections.
+        """
+        response = self.search_builder_register_predictions(
+            builder_name, limit=1,
+            pretty_response=pretty_response)
+
+        return response
+
+    def delete_builder(self, builder_name: str, pretty_response: bool = False) \
+            -> Union[dict, str]:
+        """
+        description: This method is responsible for deleting a model prediction.
+        The delete operation is always asynchronous,
+        since the deletion is performed in background.
+
+        pretty_response: If true it returns a string, otherwise a dictionary.
+        builder_name: Represents the pipeline name.
+
+        return: JSON object with an error message, a warning message or a
+        correct delete message
+        """
+
+        cluster_url_dataset = f'{self.__service_url}/{builder_name}'
+
+        response = requests.delete(cluster_url_dataset)
+
+        return self.__response_treat.treatment(response, pretty_response)
+
+    def wait(self, dataset_name: str, timeout: int = None) -> dict:
+        """
+           description: This method is responsible to create a synchronization
+           barrier for the run_spark_ml_async method.
+
+           dataset_name: Represents the pipeline name.
+           timeout: Represents the time in seconds to wait for a builder to
+           finish its run.
+
+           return: JSON object with an error message, a warning message or a
+           correct execution of a pipeline
+        """
+        return self.__observer.wait(dataset_name, timeout)
+
+

Methods

+
+
+def delete_builder(self, builder_name: str, pretty_response: bool = False) ‑> Union[dict, str] +
+
+

description: This method is responsible for deleting a model prediction. +The delete operation is always asynchronous, +since the deletion is performed in background.

+

pretty_response: If true it returns a string, otherwise a dictionary. +builder_name: Represents the pipeline name.

+

return: JSON object with an error message, a warning message or a +correct delete message

+
+ +Expand source code + +
def delete_builder(self, builder_name: str, pretty_response: bool = False) \
+        -> Union[dict, str]:
+    """
+    description: This method is responsible for deleting a model prediction.
+    The delete operation is always asynchronous,
+    since the deletion is performed in background.
+
+    pretty_response: If true it returns a string, otherwise a dictionary.
+    builder_name: Represents the pipeline name.
+
+    return: JSON object with an error message, a warning message or a
+    correct delete message
+    """
+
+    cluster_url_dataset = f'{self.__service_url}/{builder_name}'
+
+    response = requests.delete(cluster_url_dataset)
+
+    return self.__response_treat.treatment(response, pretty_response)
+
+
+
+def run_spark_ml_async(self, train_dataset_name: str, test_dataset_name: str, modeling_code: str, model_classifiers: list, pretty_response: bool = False) ‑> Union[dict, str] +
+
+

description: This method call runs several steps of a machine +learning pipeline (transform, tune, train and evaluate, for instance) +using a model code and several classifiers. It represents a way to run +an entire pipeline. The caller does not wait until the method execution +ends, since it is an asynchronous method.

+

train_dataset_name: Represent final train dataset. +test_dataset_name: Represent final test dataset. +modeling_code: Represent Python3 code for pyspark pre-processing model +model_classifiers: list of initial classifiers to be used in the model +pretty_response: if True it represents a result useful for visualization

+

return: the URL to retrieve the Spark pipeline result

+
+ +Expand source code + +
def run_spark_ml_async(self,
+                       train_dataset_name: str,
+                       test_dataset_name: str,
+                       modeling_code: str,
+                       model_classifiers: list,
+                       pretty_response: bool = False) -> Union[dict, str]:
+    """
+    description: This method call runs several steps of a machine
+    learning pipeline (transform, tune, train and evaluate, for instance)
+    using a model code and several classifiers. It represents a way to run
+    an entire pipeline. The caller does not wait until the method execution
+    ends, since it is an asynchronous method.
+
+    train_dataset_name: Represent final train dataset.
+    test_dataset_name: Represent final test dataset.
+    modeling_code: Represent Python3 code for pyspark pre-processing model
+    model_classifiers: list of initial classifiers to be used in the model
+    pretty_response: if True it represents a result useful for visualization
+
+    return: the URL to retrieve the Spark pipeline result
+    """
+
+    request_body_content = {
+        self.__TRAIN_FIELD: train_dataset_name,
+        self.__TEST_FIELD: test_dataset_name,
+        self.__CODE_FIELD: modeling_code,
+        self.__CLASSIFIERS_LIST_FIELD: model_classifiers,
+    }
+    response = requests.post(url=self.__service_url,
+                             json=request_body_content)
+
+    return self.__response_treat.treatment(response, pretty_response)
+
+
+
+def run_spark_ml_sync(self, train_dataset_name: str, test_dataset_name: str, modeling_code: str, model_classifiers: list, pretty_response: bool = False) ‑> Union[dict, str] +
+
+

description: This method call runs several steps of a machine +learning pipeline (transform, tune, train and evaluate, for instance) +using a model code and several classifiers. It represents a way to run +an entire pipeline. The caller waits until the method execution ends, +since it is a synchronous method.

+

train_dataset_name: Represent final train dataset. +test_dataset_name: Represent final test dataset. +modeling_code: Represent Python3 code for pyspark pre-processing model +model_classifiers: list of initial classifiers to be used in the model +pretty_response: if True it represents a result useful for visualization

+

return: The set of predictions (URIs of them).

+
+ +Expand source code + +
def run_spark_ml_sync(self,
+                      train_dataset_name: str,
+                      test_dataset_name: str,
+                      modeling_code: str,
+                      model_classifiers: list,
+                      pretty_response: bool = False) -> Union[dict, str]:
+    """
+    description: This method call runs several steps of a machine
+    learning pipeline (transform, tune, train and evaluate, for instance)
+    using a model code and several classifiers. It represents a way to run
+    an entire pipeline. The caller waits until the method execution ends,
+    since it is a synchronous method.
+
+    train_dataset_name: Represent final train dataset.
+    test_dataset_name: Represent final test dataset.
+    modeling_code: Represent Python3 code for pyspark pre-processing model
+    model_classifiers: list of initial classifiers to be used in the model
+    pretty_response: if True it represents a result useful for visualization
+
+    return: The set of predictions (URIs of them).
+    """
+
+    request_body_content = {
+        self.__TRAIN_FIELD: train_dataset_name,
+        self.__TEST_FIELD: test_dataset_name,
+        self.__CODE_FIELD: modeling_code,
+        self.__CLASSIFIERS_LIST_FIELD: model_classifiers,
+    }
+    response = requests.post(url=self.__service_url,
+                             json=request_body_content)
+
+    for classifier in model_classifiers:
+        self.__observer.wait(f'{test_dataset_name}{classifier}')
+
+    return self.__response_treat.treatment(response, pretty_response)
+
+
+
+def search_all_builders(self, pretty_response: bool = False) ‑> Union[dict, str] +
+
+

description: This method retrieves all model predictions metadata. It +does not retrieve the model predictions content.

+

pretty_response: If true it returns a string, otherwise a dictionary.

+

return: A list with all model predictions metadata stored in Learning +Orchestra or an empty result.

+
+ +Expand source code + +
def search_all_builders(self, pretty_response: bool = False) \
+        -> Union[dict, str]:
+    """
+    description: This method retrieves all model predictions metadata. It
+    does not retrieve the model predictions content.
+
+    pretty_response: If true it returns a string, otherwise a dictionary.
+
+    return: A list with all model predictions metadata stored in Learning
+    Orchestra or an empty result.
+    """
+
+    response = self.__entity_reader.read_all_instances_from_entity()
+
+    return self.__response_treat.treatment(response, pretty_response)
+
+
+
+def search_builder(self, builder_name: str, pretty_response: bool = False) ‑> Union[dict, str] +
+
+

description: +This method is responsible for retrieving a specific +model metadata.

+

pretty_response: If true return indented string, else return dict. +builder_name: Represents the model predictions name. +limit: Number of rows to return in pagination(default: 10) (maximum is +set at 20 rows per request) +skip: Number of rows to skip in pagination(default: 0)

+

return: Specific model prediction metadata stored in Learning Orchestra +or an error if there is no such projections.

+
+ +Expand source code + +
def search_builder(self, builder_name: str, pretty_response: bool = False) \
+        -> Union[dict, str]:
+    """
+    description:  This method is responsible for retrieving a specific
+    model metadata.
+
+    pretty_response: If true return indented string, else return dict.
+    builder_name: Represents the model predictions name.
+    limit: Number of rows to return in pagination(default: 10) (maximum is
+    set at 20 rows per request)
+    skip: Number of rows to skip in pagination(default: 0)
+
+    return: Specific model prediction metadata stored in Learning Orchestra
+    or an error if there is no such projections.
+    """
+    response = self.search_builder_register_predictions(
+        builder_name, limit=1,
+        pretty_response=pretty_response)
+
+    return response
+
+
+
+def search_builder_register_predictions(self, builder_name: str, query: dict = {}, limit: int = 10, skip: int = 0, pretty_response: bool = False) ‑> Union[dict, str] +
+
+

description: This method is responsible for retrieving the model +predictions content.

+

pretty_response: If true it returns a string, otherwise a dictionary. +builder_name: Represents the model predictions name. +query: Query to make in MongoDB(default: empty query) +limit: Number of rows to return in pagination(default: 10) (maximum is +set at 20 rows per request) +skip: Number of rows to skip in pagination(default: 0)

+

return: A page with some tuples or registers inside or an error if the +pipeline runs incorrectly. The current page is also returned to be used +in future content requests.

+
+ +Expand source code + +
def search_builder_register_predictions(self,
+                                        builder_name: str,
+                                        query: dict = {},
+                                        limit: int = 10,
+                                        skip: int = 0,
+                                        pretty_response: bool = False) \
+        -> Union[dict, str]:
+    """
+    description: This method is responsible for retrieving the model
+    predictions content.
+
+    pretty_response: If true it returns a string, otherwise a dictionary.
+    builder_name: Represents the model predictions name.
+    query: Query to make in MongoDB(default: empty query)
+    limit: Number of rows to return in pagination(default: 10) (maximum is
+    set at 20 rows per request)
+    skip: Number of rows to skip in pagination(default: 0)
+
+    return: A page with some tuples or registers inside or an error if the
+    pipeline runs incorrectly. The current page is also returned to be used
+    in future content requests.
+    """
+
+    response = self.__entity_reader.read_entity_content(
+        builder_name, query, limit, skip)
+
+    return self.__response_treat.treatment(response, pretty_response)
+
+
+
+def wait(self, dataset_name: str, timeout: int = None) ‑> dict +
+
+

description: This method is responsible to create a synchronization +barrier for the run_spark_ml_async method.

+

dataset_name: Represents the pipeline name. +timeout: Represents the time in seconds to wait for a builder to +finish its run.

+

return: JSON object with an error message, a warning message or a +correct execution of a pipeline

+
+ +Expand source code + +
def wait(self, dataset_name: str, timeout: int = None) -> dict:
+    """
+       description: This method is responsible to create a synchronization
+       barrier for the run_spark_ml_async method.
+
+       dataset_name: Represents the pipeline name.
+       timeout: Represents the time in seconds to wait for a builder to
+       finish its run.
+
+       return: JSON object with an error message, a warning message or a
+       correct execution of a pipeline
+    """
+    return self.__observer.wait(dataset_name, timeout)
+
+
+
+
+
+
+
+ +
+ + + \ No newline at end of file diff --git a/html/learning_orchestra_client/builder/builder_horovod.html b/html/learning_orchestra_client/builder/builder_horovod.html new file mode 100644 index 0000000..e272643 --- /dev/null +++ b/html/learning_orchestra_client/builder/builder_horovod.html @@ -0,0 +1,579 @@ + + + + + + +learning_orchestra_client.builder.builder_horovod API documentation + + + + + + + + + + + +
+
+
+

Module learning_orchestra_client.builder.builder_horovod

+
+
+
+ +Expand source code + +
from learning_orchestra_client.observe.observe import Observer
+from learning_orchestra_client._util._response_treat import ResponseTreat
+from learning_orchestra_client._util._entity_reader import EntityReader
+import requests
+from typing import Union
+
+
+class BuilderHorovod:
+    __BUILDER_NAME = 'name',
+    __DESCRIPTION_NAME = 'description'
+    __MODELING_CODE_NAME = 'code',
+    __PARAMETERS = 'parameters'
+
+    def __init__(self, cluster_ip: str):
+        self.__api_path = "/api/learningOrchestra/v1/builder/horovod"
+        self.__service_url = f'{cluster_ip}{self.__api_path}'
+        self.__response_treat = ResponseTreat()
+        self.__cluster_ip = cluster_ip
+        self.__entity_reader = EntityReader(self.__service_url)
+        self.__observer = Observer(self.__cluster_ip)
+
+    def run_horovod_sync(self, builder_name: str, modeling_code: str, parameters: dict = dict({}),
+                         description: str = '',
+                         pretty_response: bool = False) -> \
+            Union[dict, str]:
+        """
+                description: This method call runs several steps of a machine
+                learning pipeline (transform, tune, train and evaluate, for instance)
+                using a model code. It represents a way to run
+                an entire pipeline. The caller waits until the method execution ends,
+                since it is a synchronous method.
+
+                modeling_code: Represent python3 code used to build the pipeline.
+                builder_name: Represent the builder name that is accessible later via API.
+                parameters: Represent parameters in the cluster that is needed on the pipeline
+
+                pretty_response: if True it represents a result useful for visualization
+
+                return: The response of such python3 code.
+                """
+
+        request_body_content = {
+            self.__DESCRIPTION_NAME: description,
+            self.__BUILDER_NAME: builder_name,
+            self.__MODELING_CODE_NAME: modeling_code,
+            self.__PARAMETERS: parameters,
+        }
+
+        response = requests.post(url=self.__service_url,
+                                 json=request_body_content)
+
+        self.__observer.wait(builder_name)
+
+        return self.__response_treat.treatment(response, pretty_response)
+
+    def run_horovod_async(self, builder_name: str, modeling_code: str, parameters: dict = dict({}),
+                          description: str = '',
+                          pretty_response: bool = False) -> Union[dict, str]:
+        """
+                description: This method call runs several steps of a machine
+                learning pipeline (transform, tune, train and evaluate, for instance)
+                using a model code. It represents a way to run
+                an entire pipeline. The caller waits until the method execution ends,
+                since it is a synchronous method.
+
+                modeling_code: Represent python3 code used to build the pipeline.
+                builder_name: Represent the builder name that is accessible later via API.
+                parameters: Represent parameters in the cluster that is needed on the pipeline
+
+                pretty_response: if True it represents a result useful for visualization
+
+                return: The response of such python3 code.
+                """
+
+        request_body_content = {
+            self.__DESCRIPTION_NAME: description,
+            self.__BUILDER_NAME: builder_name,
+            self.__MODELING_CODE_NAME: modeling_code,
+            self.__PARAMETERS: parameters,
+        }
+
+        response = requests.post(url=self.__service_url,
+                                 json=request_body_content)
+
+        return self.__response_treat.treatment(response, pretty_response)
+
+    def get_builder(self, builder_name: str, query: dict = {}, limit: int = 10, skip: int = 0,
+                    pretty_response: bool = False) \
+            -> Union[dict, str]:
+        """
+                description: This method is responsible for retrieving the model
+                predictions content.
+
+                pretty_response: If true it returns a string, otherwise a dictionary.
+                builder_name: Represents the model predictions name.
+                query: Query to make in MongoDB(default: empty query)
+                limit: Number of rows to return in pagination(default: 10) (maximum is
+                set at 20 rows per request)
+                skip: Number of rows to skip in pagination(default: 0)
+
+                return: A page with some tuples or registers inside or an error if the
+                pipeline runs incorrectly. The current page is also returned to be used
+                in future content requests.
+                """
+        response = self.__entity_reader.read_entity_content(
+            builder_name, query, limit, skip)
+
+        return self.__response_treat.treatment(response, pretty_response)
+
+    def delete_builder(self, builder_name: str, pretty_response: bool = False) \
+            -> Union[dict, str]:
+        """
+        description: This method is responsible for deleting a model prediction.
+        The delete operation is always asynchronous,
+        since the deletion is performed in background.
+
+        pretty_response: If true it returns a string, otherwise a dictionary.
+        builder_name: Represents the pipeline name.
+
+        return: JSON object with an error message, a warning message or a
+        correct delete message
+        """
+
+        cluster_url_dataset = f'{self.__service_url}/{builder_name}'
+
+        response = requests.delete(cluster_url_dataset)
+
+        return self.__response_treat.treatment(response, pretty_response)
+
+    def wait(self, builder_name: str, timeout: int = None) -> dict:
+        """
+           description: This method is responsible to create a synchronization
+           barrier for the run_horovod_async method.
+
+           dataset_name: Represents the pipeline name.
+           timeout: Represents the time in seconds to wait for a builder to
+           finish its run.
+
+           return: JSON object with an error message, a warning message or a
+           correct execution of a pipeline
+        """
+        return self.__observer.wait(builder_name, timeout)
+
+
+
+
+
+
+
+
+
+

Classes

+
+
+class BuilderHorovod +(cluster_ip: str) +
+
+
+
+ +Expand source code + +
class BuilderHorovod:
+    __BUILDER_NAME = 'name',
+    __DESCRIPTION_NAME = 'description'
+    __MODELING_CODE_NAME = 'code',
+    __PARAMETERS = 'parameters'
+
+    def __init__(self, cluster_ip: str):
+        self.__api_path = "/api/learningOrchestra/v1/builder/horovod"
+        self.__service_url = f'{cluster_ip}{self.__api_path}'
+        self.__response_treat = ResponseTreat()
+        self.__cluster_ip = cluster_ip
+        self.__entity_reader = EntityReader(self.__service_url)
+        self.__observer = Observer(self.__cluster_ip)
+
+    def run_horovod_sync(self, builder_name: str, modeling_code: str, parameters: dict = dict({}),
+                         description: str = '',
+                         pretty_response: bool = False) -> \
+            Union[dict, str]:
+        """
+                description: This method call runs several steps of a machine
+                learning pipeline (transform, tune, train and evaluate, for instance)
+                using a model code. It represents a way to run
+                an entire pipeline. The caller waits until the method execution ends,
+                since it is a synchronous method.
+
+                modeling_code: Represent python3 code used to build the pipeline.
+                builder_name: Represent the builder name that is accessible later via API.
+                parameters: Represent parameters in the cluster that is needed on the pipeline
+
+                pretty_response: if True it represents a result useful for visualization
+
+                return: The response of such python3 code.
+                """
+
+        request_body_content = {
+            self.__DESCRIPTION_NAME: description,
+            self.__BUILDER_NAME: builder_name,
+            self.__MODELING_CODE_NAME: modeling_code,
+            self.__PARAMETERS: parameters,
+        }
+
+        response = requests.post(url=self.__service_url,
+                                 json=request_body_content)
+
+        self.__observer.wait(builder_name)
+
+        return self.__response_treat.treatment(response, pretty_response)
+
+    def run_horovod_async(self, builder_name: str, modeling_code: str, parameters: dict = dict({}),
+                          description: str = '',
+                          pretty_response: bool = False) -> Union[dict, str]:
+        """
+                description: This method call runs several steps of a machine
+                learning pipeline (transform, tune, train and evaluate, for instance)
+                using a model code. It represents a way to run
+                an entire pipeline. The caller waits until the method execution ends,
+                since it is a synchronous method.
+
+                modeling_code: Represent python3 code used to build the pipeline.
+                builder_name: Represent the builder name that is accessible later via API.
+                parameters: Represent parameters in the cluster that is needed on the pipeline
+
+                pretty_response: if True it represents a result useful for visualization
+
+                return: The response of such python3 code.
+                """
+
+        request_body_content = {
+            self.__DESCRIPTION_NAME: description,
+            self.__BUILDER_NAME: builder_name,
+            self.__MODELING_CODE_NAME: modeling_code,
+            self.__PARAMETERS: parameters,
+        }
+
+        response = requests.post(url=self.__service_url,
+                                 json=request_body_content)
+
+        return self.__response_treat.treatment(response, pretty_response)
+
+    def get_builder(self, builder_name: str, query: dict = {}, limit: int = 10, skip: int = 0,
+                    pretty_response: bool = False) \
+            -> Union[dict, str]:
+        """
+                description: This method is responsible for retrieving the model
+                predictions content.
+
+                pretty_response: If true it returns a string, otherwise a dictionary.
+                builder_name: Represents the model predictions name.
+                query: Query to make in MongoDB(default: empty query)
+                limit: Number of rows to return in pagination(default: 10) (maximum is
+                set at 20 rows per request)
+                skip: Number of rows to skip in pagination(default: 0)
+
+                return: A page with some tuples or registers inside or an error if the
+                pipeline runs incorrectly. The current page is also returned to be used
+                in future content requests.
+                """
+        response = self.__entity_reader.read_entity_content(
+            builder_name, query, limit, skip)
+
+        return self.__response_treat.treatment(response, pretty_response)
+
+    def delete_builder(self, builder_name: str, pretty_response: bool = False) \
+            -> Union[dict, str]:
+        """
+        description: This method is responsible for deleting a model prediction.
+        The delete operation is always asynchronous,
+        since the deletion is performed in background.
+
+        pretty_response: If true it returns a string, otherwise a dictionary.
+        builder_name: Represents the pipeline name.
+
+        return: JSON object with an error message, a warning message or a
+        correct delete message
+        """
+
+        cluster_url_dataset = f'{self.__service_url}/{builder_name}'
+
+        response = requests.delete(cluster_url_dataset)
+
+        return self.__response_treat.treatment(response, pretty_response)
+
+    def wait(self, builder_name: str, timeout: int = None) -> dict:
+        """
+           description: This method is responsible to create a synchronization
+           barrier for the run_horovod_async method.
+
+           dataset_name: Represents the pipeline name.
+           timeout: Represents the time in seconds to wait for a builder to
+           finish its run.
+
+           return: JSON object with an error message, a warning message or a
+           correct execution of a pipeline
+        """
+        return self.__observer.wait(builder_name, timeout)
+
+

Methods

+
+
+def delete_builder(self, builder_name: str, pretty_response: bool = False) ‑> Union[dict, str] +
+
+

description: This method is responsible for deleting a model prediction. +The delete operation is always asynchronous, +since the deletion is performed in background.

+

pretty_response: If true it returns a string, otherwise a dictionary. +builder_name: Represents the pipeline name.

+

return: JSON object with an error message, a warning message or a +correct delete message

+
+ +Expand source code + +
def delete_builder(self, builder_name: str, pretty_response: bool = False) \
+        -> Union[dict, str]:
+    """
+    description: This method is responsible for deleting a model prediction.
+    The delete operation is always asynchronous,
+    since the deletion is performed in background.
+
+    pretty_response: If true it returns a string, otherwise a dictionary.
+    builder_name: Represents the pipeline name.
+
+    return: JSON object with an error message, a warning message or a
+    correct delete message
+    """
+
+    cluster_url_dataset = f'{self.__service_url}/{builder_name}'
+
+    response = requests.delete(cluster_url_dataset)
+
+    return self.__response_treat.treatment(response, pretty_response)
+
+
+
+def get_builder(self, builder_name: str, query: dict = {}, limit: int = 10, skip: int = 0, pretty_response: bool = False) ‑> Union[dict, str] +
+
+

description: This method is responsible for retrieving the model +predictions content.

+

pretty_response: If true it returns a string, otherwise a dictionary. +builder_name: Represents the model predictions name. +query: Query to make in MongoDB(default: empty query) +limit: Number of rows to return in pagination(default: 10) (maximum is +set at 20 rows per request) +skip: Number of rows to skip in pagination(default: 0)

+

return: A page with some tuples or registers inside or an error if the +pipeline runs incorrectly. The current page is also returned to be used +in future content requests.

+
+ +Expand source code + +
def get_builder(self, builder_name: str, query: dict = {}, limit: int = 10, skip: int = 0,
+                pretty_response: bool = False) \
+        -> Union[dict, str]:
+    """
+            description: This method is responsible for retrieving the model
+            predictions content.
+
+            pretty_response: If true it returns a string, otherwise a dictionary.
+            builder_name: Represents the model predictions name.
+            query: Query to make in MongoDB(default: empty query)
+            limit: Number of rows to return in pagination(default: 10) (maximum is
+            set at 20 rows per request)
+            skip: Number of rows to skip in pagination(default: 0)
+
+            return: A page with some tuples or registers inside or an error if the
+            pipeline runs incorrectly. The current page is also returned to be used
+            in future content requests.
+            """
+    response = self.__entity_reader.read_entity_content(
+        builder_name, query, limit, skip)
+
+    return self.__response_treat.treatment(response, pretty_response)
+
+
+
+def run_horovod_async(self, builder_name: str, modeling_code: str, parameters: dict = {}, description: str = '', pretty_response: bool = False) ‑> Union[dict, str] +
+
+

description: This method call runs several steps of a machine +learning pipeline (transform, tune, train and evaluate, for instance) +using a model code. It represents a way to run +an entire pipeline. The caller waits until the method execution ends, +since it is a synchronous method.

+

modeling_code: Represent python3 code used to build the pipeline. +builder_name: Represent the builder name that is accessible later via API. +parameters: Represent parameters in the cluster that is needed on the pipeline

+

pretty_response: if True it represents a result useful for visualization

+

return: The response of such python3 code.

+
+ +Expand source code + +
def run_horovod_async(self, builder_name: str, modeling_code: str, parameters: dict = dict({}),
+                      description: str = '',
+                      pretty_response: bool = False) -> Union[dict, str]:
+    """
+            description: This method call runs several steps of a machine
+            learning pipeline (transform, tune, train and evaluate, for instance)
+            using a model code. It represents a way to run
+            an entire pipeline. The caller waits until the method execution ends,
+            since it is a synchronous method.
+
+            modeling_code: Represent python3 code used to build the pipeline.
+            builder_name: Represent the builder name that is accessible later via API.
+            parameters: Represent parameters in the cluster that is needed on the pipeline
+
+            pretty_response: if True it represents a result useful for visualization
+
+            return: The response of such python3 code.
+            """
+
+    request_body_content = {
+        self.__DESCRIPTION_NAME: description,
+        self.__BUILDER_NAME: builder_name,
+        self.__MODELING_CODE_NAME: modeling_code,
+        self.__PARAMETERS: parameters,
+    }
+
+    response = requests.post(url=self.__service_url,
+                             json=request_body_content)
+
+    return self.__response_treat.treatment(response, pretty_response)
+
+
+
+def run_horovod_sync(self, builder_name: str, modeling_code: str, parameters: dict = {}, description: str = '', pretty_response: bool = False) ‑> Union[dict, str] +
+
+

description: This method call runs several steps of a machine +learning pipeline (transform, tune, train and evaluate, for instance) +using a model code. It represents a way to run +an entire pipeline. The caller waits until the method execution ends, +since it is a synchronous method.

+

modeling_code: Represent python3 code used to build the pipeline. +builder_name: Represent the builder name that is accessible later via API. +parameters: Represent parameters in the cluster that is needed on the pipeline

+

pretty_response: if True it represents a result useful for visualization

+

return: The response of such python3 code.

+
+ +Expand source code + +
def run_horovod_sync(self, builder_name: str, modeling_code: str, parameters: dict = dict({}),
+                     description: str = '',
+                     pretty_response: bool = False) -> \
+        Union[dict, str]:
+    """
+            description: This method call runs several steps of a machine
+            learning pipeline (transform, tune, train and evaluate, for instance)
+            using a model code. It represents a way to run
+            an entire pipeline. The caller waits until the method execution ends,
+            since it is a synchronous method.
+
+            modeling_code: Represent python3 code used to build the pipeline.
+            builder_name: Represent the builder name that is accessible later via API.
+            parameters: Represent parameters in the cluster that is needed on the pipeline
+
+            pretty_response: if True it represents a result useful for visualization
+
+            return: The response of such python3 code.
+            """
+
+    request_body_content = {
+        self.__DESCRIPTION_NAME: description,
+        self.__BUILDER_NAME: builder_name,
+        self.__MODELING_CODE_NAME: modeling_code,
+        self.__PARAMETERS: parameters,
+    }
+
+    response = requests.post(url=self.__service_url,
+                             json=request_body_content)
+
+    self.__observer.wait(builder_name)
+
+    return self.__response_treat.treatment(response, pretty_response)
+
+
+
+def wait(self, builder_name: str, timeout: int = None) ‑> dict +
+
+

description: This method is responsible to create a synchronization +barrier for the run_horovod_async method.

+

dataset_name: Represents the pipeline name. +timeout: Represents the time in seconds to wait for a builder to +finish its run.

+

return: JSON object with an error message, a warning message or a +correct execution of a pipeline

+
+ +Expand source code + +
def wait(self, builder_name: str, timeout: int = None) -> dict:
+    """
+       description: This method is responsible to create a synchronization
+       barrier for the run_horovod_async method.
+
+       dataset_name: Represents the pipeline name.
+       timeout: Represents the time in seconds to wait for a builder to
+       finish its run.
+
+       return: JSON object with an error message, a warning message or a
+       correct execution of a pipeline
+    """
+    return self.__observer.wait(builder_name, timeout)
+
+
+
+
+
+
+
+ +
+ + + \ No newline at end of file diff --git a/html/learning_orchestra_client/builder/index.html b/html/learning_orchestra_client/builder/index.html new file mode 100644 index 0000000..6dbc5cb --- /dev/null +++ b/html/learning_orchestra_client/builder/index.html @@ -0,0 +1,70 @@ + + + + + + +learning_orchestra_client.builder API documentation + + + + + + + + + + + +
+ + +
+ + + \ No newline at end of file diff --git a/html/learning_orchestra_client/dataset/csv.html b/html/learning_orchestra_client/dataset/csv.html new file mode 100644 index 0000000..6f160ed --- /dev/null +++ b/html/learning_orchestra_client/dataset/csv.html @@ -0,0 +1,97 @@ + + + + + + +learning_orchestra_client.dataset.csv API documentation + + + + + + + + + + + +
+
+
+

Module learning_orchestra_client.dataset.csv

+
+
+
+ +Expand source code + +
from ._dataset import Dataset
+
+
+class DatasetCsv(Dataset):
+    def __init__(self, cluster_ip: str):
+        self.__api_path = "/api/learningOrchestra/v1/dataset/csv"
+        self.__cluster_ip = cluster_ip
+        super().__init__(cluster_ip, self.__api_path)
+
+
+
+
+
+
+
+
+
+

Classes

+
+
+class DatasetCsv +(cluster_ip: str) +
+
+
+
+ +Expand source code + +
class DatasetCsv(Dataset):
+    def __init__(self, cluster_ip: str):
+        self.__api_path = "/api/learningOrchestra/v1/dataset/csv"
+        self.__cluster_ip = cluster_ip
+        super().__init__(cluster_ip, self.__api_path)
+
+

Ancestors

+
    +
  • learning_orchestra_client.dataset._dataset.Dataset
    • +
+
+
+
+
+ +
+ + + \ No newline at end of file diff --git a/html/learning_orchestra_client/dataset/generic.html b/html/learning_orchestra_client/dataset/generic.html new file mode 100644 index 0000000..978bf10 --- /dev/null +++ b/html/learning_orchestra_client/dataset/generic.html @@ -0,0 +1,97 @@ + + + + + + +learning_orchestra_client.dataset.generic API documentation + + + + + + + + + + + +
+
+
+

Module learning_orchestra_client.dataset.generic

+
+
+
+ +Expand source code + +
from ._dataset import Dataset
+
+
+class DatasetGeneric(Dataset):
+    def __init__(self, cluster_ip: str):
+        self.__api_path = "/api/learningOrchestra/v1/dataset/generic"
+        self.__cluster_ip = cluster_ip
+        super().__init__(cluster_ip, self.__api_path)
+
+
+
+
+
+
+
+
+
+

Classes

+
+
+class DatasetGeneric +(cluster_ip: str) +
+
+
+
+ +Expand source code + +
class DatasetGeneric(Dataset):
+    def __init__(self, cluster_ip: str):
+        self.__api_path = "/api/learningOrchestra/v1/dataset/generic"
+        self.__cluster_ip = cluster_ip
+        super().__init__(cluster_ip, self.__api_path)
+
+

Ancestors

+
    +
  • learning_orchestra_client.dataset._dataset.Dataset
    • +
+
+
+
+
+ +
+ + + \ No newline at end of file diff --git a/html/learning_orchestra_client/dataset/index.html b/html/learning_orchestra_client/dataset/index.html new file mode 100644 index 0000000..bfd2569 --- /dev/null +++ b/html/learning_orchestra_client/dataset/index.html @@ -0,0 +1,70 @@ + + + + + + +learning_orchestra_client.dataset API documentation + + + + + + + + + + + +
+ + +
+ + + \ No newline at end of file diff --git a/html/learning_orchestra_client/evaluate/index.html b/html/learning_orchestra_client/evaluate/index.html new file mode 100644 index 0000000..05bdd32 --- /dev/null +++ b/html/learning_orchestra_client/evaluate/index.html @@ -0,0 +1,70 @@ + + + + + + +learning_orchestra_client.evaluate API documentation + + + + + + + + + + + +
+ + +
+ + + \ No newline at end of file diff --git a/html/learning_orchestra_client/evaluate/scikitlearn.html b/html/learning_orchestra_client/evaluate/scikitlearn.html new file mode 100644 index 0000000..93a1201 --- /dev/null +++ b/html/learning_orchestra_client/evaluate/scikitlearn.html @@ -0,0 +1,97 @@ + + + + + + +learning_orchestra_client.evaluate.scikitlearn API documentation + + + + + + + + + + + +
+
+
+

Module learning_orchestra_client.evaluate.scikitlearn

+
+
+
+ +Expand source code + +
from ._evaluate import Evaluate
+
+
+class EvaluateScikitLearn(Evaluate):
+    def __init__(self, cluster_ip: str):
+        self.__api_path = "/api/learningOrchestra/v1/evaluate/scikitlearn"
+        self.__cluster_ip = cluster_ip
+        super().__init__(cluster_ip, self.__api_path)
+
+
+
+
+
+
+
+
+
+

Classes

+
+
+class EvaluateScikitLearn +(cluster_ip: str) +
+
+
+
+ +Expand source code + +
class EvaluateScikitLearn(Evaluate):
+    def __init__(self, cluster_ip: str):
+        self.__api_path = "/api/learningOrchestra/v1/evaluate/scikitlearn"
+        self.__cluster_ip = cluster_ip
+        super().__init__(cluster_ip, self.__api_path)
+
+

Ancestors

+
    +
  • learning_orchestra_client.evaluate._evaluate.Evaluate
    • +
+
+
+
+
+ +
+ + + \ No newline at end of file diff --git a/html/learning_orchestra_client/evaluate/tensorflow.html b/html/learning_orchestra_client/evaluate/tensorflow.html new file mode 100644 index 0000000..826de3a --- /dev/null +++ b/html/learning_orchestra_client/evaluate/tensorflow.html @@ -0,0 +1,97 @@ + + + + + + +learning_orchestra_client.evaluate.tensorflow API documentation + + + + + + + + + + + +
+
+
+

Module learning_orchestra_client.evaluate.tensorflow

+
+
+
+ +Expand source code + +
from ._evaluate import Evaluate
+
+
+class EvaluateTensorflow(Evaluate):
+    def __init__(self, cluster_ip: str):
+        self.__api_path = "/api/learningOrchestra/v1/evaluate/tensorflow"
+        self.__cluster_ip = cluster_ip
+        super().__init__(cluster_ip, self.__api_path)
+
+
+
+
+
+
+
+
+
+

Classes

+
+
+class EvaluateTensorflow +(cluster_ip: str) +
+
+
+
+ +Expand source code + +
class EvaluateTensorflow(Evaluate):
+    def __init__(self, cluster_ip: str):
+        self.__api_path = "/api/learningOrchestra/v1/evaluate/tensorflow"
+        self.__cluster_ip = cluster_ip
+        super().__init__(cluster_ip, self.__api_path)
+
+

Ancestors

+
    +
  • learning_orchestra_client.evaluate._evaluate.Evaluate
    • +
+
+
+
+
+ +
+ + + \ No newline at end of file diff --git a/html/learning_orchestra_client/explore/histogram.html b/html/learning_orchestra_client/explore/histogram.html new file mode 100644 index 0000000..421cf38 --- /dev/null +++ b/html/learning_orchestra_client/explore/histogram.html @@ -0,0 +1,645 @@ + + + + + + +learning_orchestra_client.explore.histogram API documentation + + + + + + + + + + + +
+
+
+

Module learning_orchestra_client.explore.histogram

+
+
+
+ +Expand source code + +
from learning_orchestra_client.observe.observe import Observer
+from learning_orchestra_client._util._response_treat import ResponseTreat
+import requests
+from typing import Union
+from learning_orchestra_client._util._entity_reader import EntityReader
+
+
+class ExploreHistogram:
+    __INPUT_NAME = "inputDatasetName"
+    __OUTPUT_NAME = "outputDatasetName"
+    __FIELDS = "names"
+
+    def __init__(self, cluster_ip: str):
+        self.__cluster_ip = cluster_ip
+        self.__api_path = "/api/learningOrchestra/v1/explore/histogram"
+        self.__service_url = f'{cluster_ip}{self.__api_path}'
+        self.__response_treat = ResponseTreat()
+        self.__observer = Observer(self.__cluster_ip)
+        self.__entity_reader = EntityReader(self.__service_url)
+
+    def run_histogram_sync(self,
+                           dataset_name: str,
+                           histogram_name: str,
+                           fields: list,
+                           pretty_response: bool = False) \
+            -> Union[dict, str]:
+        """
+        description: This method creates a histogram
+        synchronously, so the caller waits until the histogram is inserted into
+        the Learning Orchestra storage mechanism.
+
+        dataset_name: Represents the name of dataset.
+        histogram_name: Represents the name of histogram.
+        fields: Represents a list of attributes.
+        pretty_response: If true it returns a string, otherwise a dictionary.
+
+        return: A JSON object with error or warning messages. In case of
+        success, it returns a histogram.
+        """
+
+        request_body = {
+            self.__INPUT_NAME: dataset_name,
+            self.__OUTPUT_NAME: histogram_name,
+            self.__FIELDS: fields,
+        }
+        request_url = self.__service_url
+
+        response = requests.post(url=request_url, json=request_body)
+        self.__observer.wait(dataset_name)
+
+        return self.__response_treat.treatment(response, pretty_response)
+
+    def run_histogram_async(self,
+                            dataset_name: str,
+                            histogram_name: str,
+                            fields: list,
+                            pretty_response: bool = False) \
+            -> Union[dict, str]:
+        """
+        description: This method creates a histogram
+        asynchronously, so the caller does not wait until the histogram is
+        inserted into the Learning Orchestra storage mechanism.
+
+        dataset_name: Represents the name of dataset.
+        histogram_name: Represents the name of histogram.
+        fields: Represents a list of attributes.
+        pretty_response: If true it returns a string, otherwise a dictionary.
+
+        return: A JSON object with error or warning messages. In case of
+        success, it returns a histogram.
+        """
+
+        request_body = {
+            self.__INPUT_NAME: dataset_name,
+            self.__OUTPUT_NAME: histogram_name,
+            self.__FIELDS: fields,
+        }
+        request_url = self.__service_url
+
+        response = requests.post(url=request_url, json=request_body)
+
+        return self.__response_treat.treatment(response, pretty_response)
+
+    def search_all_histograms(self, pretty_response: bool = False) \
+            -> Union[dict, str]:
+        """
+        description: This method retrieves all histogram metadata, it does not
+        retrieve the histogram content.
+
+        pretty_response: If true it returns a string, otherwise a dictionary.
+
+        return: A list with all histogram metadata stored in Learning Orchestra
+        or an empty result.
+        """
+
+        response = self.__entity_reader.read_all_instances_from_entity()
+        return self.__response_treat.treatment(response, pretty_response)
+
+    def search_histogram_content(self,
+                                 histogram_name: str,
+                                 query: dict = {},
+                                 limit: int = 10,
+                                 skip: int = 0,
+                                 pretty_response: bool = False) \
+            -> Union[dict, str]:
+        """
+        description: This method is responsible for retrieving the histogram
+        content.
+
+        pretty_response: If true it returns a string, otherwise a dictionary.
+        histogram_name: Represents the histogram name.
+        query: Query to make in MongoDB(default: empty query)
+        limit: Number of rows to return in pagination(default: 10) (maximum is
+        set at 20 rows per request)
+        skip: Number of rows to skip in pagination(default: 0)
+
+        return: A page with some tuples or registers inside or an error if there
+        is no such projection. The current page is also returned to be used in
+        future content requests.
+        """
+
+        response = self.__entity_reader.read_entity_content(
+            histogram_name, query, limit, skip)
+
+        return self.__response_treat.treatment(response, pretty_response)
+
+    def delete_histogram(self, histogram_name: str,
+                         pretty_response: bool = False) -> Union[dict, str]:
+        """
+        description: This method is responsible for deleting a histogram.
+        The delete operation is always asynchronous,
+        since the deletion is performed in background.
+
+        pretty_response: If true it returns a string, otherwise a dictionary.
+        histogram_name: Represents the histogram name.
+
+        return: JSON object with an error message, a warning message or a
+        correct delete message
+        """
+
+        cluster_url_histogram = f'{self.__service_url}/{histogram_name}'
+        response = requests.delete(cluster_url_histogram)
+
+        return self.__response_treat.treatment(response, pretty_response)
+
+    def wait(self, name: str, timeout: int = None) -> dict:
+        """
+           description: This method is responsible to create a synchronization
+           barrier for the run_histogram_async method or delete_histogram
+           method.
+
+           name: Represents the histogram name.
+           timeout: Represents the time in seconds to wait for a histogram to
+           finish its run.
+
+           return: JSON object with an error message, a warning message or a
+           correct histogram result
+        """
+        return self.__observer.wait(name, timeout)
+
+
+
+
+
+
+
+
+
+

Classes

+
+
+class ExploreHistogram +(cluster_ip: str) +
+
+
+
+ +Expand source code + +
class ExploreHistogram:
+    __INPUT_NAME = "inputDatasetName"
+    __OUTPUT_NAME = "outputDatasetName"
+    __FIELDS = "names"
+
+    def __init__(self, cluster_ip: str):
+        self.__cluster_ip = cluster_ip
+        self.__api_path = "/api/learningOrchestra/v1/explore/histogram"
+        self.__service_url = f'{cluster_ip}{self.__api_path}'
+        self.__response_treat = ResponseTreat()
+        self.__observer = Observer(self.__cluster_ip)
+        self.__entity_reader = EntityReader(self.__service_url)
+
+    def run_histogram_sync(self,
+                           dataset_name: str,
+                           histogram_name: str,
+                           fields: list,
+                           pretty_response: bool = False) \
+            -> Union[dict, str]:
+        """
+        description: This method creates a histogram
+        synchronously, so the caller waits until the histogram is inserted into
+        the Learning Orchestra storage mechanism.
+
+        dataset_name: Represents the name of dataset.
+        histogram_name: Represents the name of histogram.
+        fields: Represents a list of attributes.
+        pretty_response: If true it returns a string, otherwise a dictionary.
+
+        return: A JSON object with error or warning messages. In case of
+        success, it returns a histogram.
+        """
+
+        request_body = {
+            self.__INPUT_NAME: dataset_name,
+            self.__OUTPUT_NAME: histogram_name,
+            self.__FIELDS: fields,
+        }
+        request_url = self.__service_url
+
+        response = requests.post(url=request_url, json=request_body)
+        self.__observer.wait(dataset_name)
+
+        return self.__response_treat.treatment(response, pretty_response)
+
+    def run_histogram_async(self,
+                            dataset_name: str,
+                            histogram_name: str,
+                            fields: list,
+                            pretty_response: bool = False) \
+            -> Union[dict, str]:
+        """
+        description: This method creates a histogram
+        asynchronously, so the caller does not wait until the histogram is
+        inserted into the Learning Orchestra storage mechanism.
+
+        dataset_name: Represents the name of dataset.
+        histogram_name: Represents the name of histogram.
+        fields: Represents a list of attributes.
+        pretty_response: If true it returns a string, otherwise a dictionary.
+
+        return: A JSON object with error or warning messages. In case of
+        success, it returns a histogram.
+        """
+
+        request_body = {
+            self.__INPUT_NAME: dataset_name,
+            self.__OUTPUT_NAME: histogram_name,
+            self.__FIELDS: fields,
+        }
+        request_url = self.__service_url
+
+        response = requests.post(url=request_url, json=request_body)
+
+        return self.__response_treat.treatment(response, pretty_response)
+
+    def search_all_histograms(self, pretty_response: bool = False) \
+            -> Union[dict, str]:
+        """
+        description: This method retrieves all histogram metadata, it does not
+        retrieve the histogram content.
+
+        pretty_response: If true it returns a string, otherwise a dictionary.
+
+        return: A list with all histogram metadata stored in Learning Orchestra
+        or an empty result.
+        """
+
+        response = self.__entity_reader.read_all_instances_from_entity()
+        return self.__response_treat.treatment(response, pretty_response)
+
+    def search_histogram_content(self,
+                                 histogram_name: str,
+                                 query: dict = {},
+                                 limit: int = 10,
+                                 skip: int = 0,
+                                 pretty_response: bool = False) \
+            -> Union[dict, str]:
+        """
+        description: This method is responsible for retrieving the histogram
+        content.
+
+        pretty_response: If true it returns a string, otherwise a dictionary.
+        histogram_name: Represents the histogram name.
+        query: Query to make in MongoDB(default: empty query)
+        limit: Number of rows to return in pagination(default: 10) (maximum is
+        set at 20 rows per request)
+        skip: Number of rows to skip in pagination(default: 0)
+
+        return: A page with some tuples or registers inside or an error if there
+        is no such projection. The current page is also returned to be used in
+        future content requests.
+        """
+
+        response = self.__entity_reader.read_entity_content(
+            histogram_name, query, limit, skip)
+
+        return self.__response_treat.treatment(response, pretty_response)
+
+    def delete_histogram(self, histogram_name: str,
+                         pretty_response: bool = False) -> Union[dict, str]:
+        """
+        description: This method is responsible for deleting a histogram.
+        The delete operation is always asynchronous,
+        since the deletion is performed in background.
+
+        pretty_response: If true it returns a string, otherwise a dictionary.
+        histogram_name: Represents the histogram name.
+
+        return: JSON object with an error message, a warning message or a
+        correct delete message
+        """
+
+        cluster_url_histogram = f'{self.__service_url}/{histogram_name}'
+        response = requests.delete(cluster_url_histogram)
+
+        return self.__response_treat.treatment(response, pretty_response)
+
+    def wait(self, name: str, timeout: int = None) -> dict:
+        """
+           description: This method is responsible to create a synchronization
+           barrier for the run_histogram_async method or delete_histogram
+           method.
+
+           name: Represents the histogram name.
+           timeout: Represents the time in seconds to wait for a histogram to
+           finish its run.
+
+           return: JSON object with an error message, a warning message or a
+           correct histogram result
+        """
+        return self.__observer.wait(name, timeout)
+
+

Methods

+
+
+def delete_histogram(self, histogram_name: str, pretty_response: bool = False) ‑> Union[dict, str] +
+
+

description: This method is responsible for deleting a histogram. +The delete operation is always asynchronous, +since the deletion is performed in background.

+

pretty_response: If true it returns a string, otherwise a dictionary. +histogram_name: Represents the histogram name.

+

return: JSON object with an error message, a warning message or a +correct delete message

+
+ +Expand source code + +
def delete_histogram(self, histogram_name: str,
+                     pretty_response: bool = False) -> Union[dict, str]:
+    """
+    description: This method is responsible for deleting a histogram.
+    The delete operation is always asynchronous,
+    since the deletion is performed in background.
+
+    pretty_response: If true it returns a string, otherwise a dictionary.
+    histogram_name: Represents the histogram name.
+
+    return: JSON object with an error message, a warning message or a
+    correct delete message
+    """
+
+    cluster_url_histogram = f'{self.__service_url}/{histogram_name}'
+    response = requests.delete(cluster_url_histogram)
+
+    return self.__response_treat.treatment(response, pretty_response)
+
+
+
+def run_histogram_async(self, dataset_name: str, histogram_name: str, fields: list, pretty_response: bool = False) ‑> Union[dict, str] +
+
+

description: This method creates a histogram +asynchronously, so the caller does not wait until the histogram is +inserted into the Learning Orchestra storage mechanism.

+

dataset_name: Represents the name of dataset. +histogram_name: Represents the name of histogram. +fields: Represents a list of attributes. +pretty_response: If true it returns a string, otherwise a dictionary.

+

return: A JSON object with error or warning messages. In case of +success, it returns a histogram.

+
+ +Expand source code + +
def run_histogram_async(self,
+                        dataset_name: str,
+                        histogram_name: str,
+                        fields: list,
+                        pretty_response: bool = False) \
+        -> Union[dict, str]:
+    """
+    description: This method creates a histogram
+    asynchronously, so the caller does not wait until the histogram is
+    inserted into the Learning Orchestra storage mechanism.
+
+    dataset_name: Represents the name of dataset.
+    histogram_name: Represents the name of histogram.
+    fields: Represents a list of attributes.
+    pretty_response: If true it returns a string, otherwise a dictionary.
+
+    return: A JSON object with error or warning messages. In case of
+    success, it returns a histogram.
+    """
+
+    request_body = {
+        self.__INPUT_NAME: dataset_name,
+        self.__OUTPUT_NAME: histogram_name,
+        self.__FIELDS: fields,
+    }
+    request_url = self.__service_url
+
+    response = requests.post(url=request_url, json=request_body)
+
+    return self.__response_treat.treatment(response, pretty_response)
+
+
+
+def run_histogram_sync(self, dataset_name: str, histogram_name: str, fields: list, pretty_response: bool = False) ‑> Union[dict, str] +
+
+

description: This method creates a histogram +synchronously, so the caller waits until the histogram is inserted into +the Learning Orchestra storage mechanism.

+

dataset_name: Represents the name of dataset. +histogram_name: Represents the name of histogram. +fields: Represents a list of attributes. +pretty_response: If true it returns a string, otherwise a dictionary.

+

return: A JSON object with error or warning messages. In case of +success, it returns a histogram.

+
+ +Expand source code + +
def run_histogram_sync(self,
+                       dataset_name: str,
+                       histogram_name: str,
+                       fields: list,
+                       pretty_response: bool = False) \
+        -> Union[dict, str]:
+    """
+    description: This method creates a histogram
+    synchronously, so the caller waits until the histogram is inserted into
+    the Learning Orchestra storage mechanism.
+
+    dataset_name: Represents the name of dataset.
+    histogram_name: Represents the name of histogram.
+    fields: Represents a list of attributes.
+    pretty_response: If true it returns a string, otherwise a dictionary.
+
+    return: A JSON object with error or warning messages. In case of
+    success, it returns a histogram.
+    """
+
+    request_body = {
+        self.__INPUT_NAME: dataset_name,
+        self.__OUTPUT_NAME: histogram_name,
+        self.__FIELDS: fields,
+    }
+    request_url = self.__service_url
+
+    response = requests.post(url=request_url, json=request_body)
+    self.__observer.wait(dataset_name)
+
+    return self.__response_treat.treatment(response, pretty_response)
+
+
+
+def search_all_histograms(self, pretty_response: bool = False) ‑> Union[dict, str] +
+
+

description: This method retrieves all histogram metadata, it does not +retrieve the histogram content.

+

pretty_response: If true it returns a string, otherwise a dictionary.

+

return: A list with all histogram metadata stored in Learning Orchestra +or an empty result.

+
+ +Expand source code + +
def search_all_histograms(self, pretty_response: bool = False) \
+        -> Union[dict, str]:
+    """
+    description: This method retrieves all histogram metadata, it does not
+    retrieve the histogram content.
+
+    pretty_response: If true it returns a string, otherwise a dictionary.
+
+    return: A list with all histogram metadata stored in Learning Orchestra
+    or an empty result.
+    """
+
+    response = self.__entity_reader.read_all_instances_from_entity()
+    return self.__response_treat.treatment(response, pretty_response)
+
+
+
+def search_histogram_content(self, histogram_name: str, query: dict = {}, limit: int = 10, skip: int = 0, pretty_response: bool = False) ‑> Union[dict, str] +
+
+

description: This method is responsible for retrieving the histogram +content.

+

pretty_response: If true it returns a string, otherwise a dictionary. +histogram_name: Represents the histogram name. +query: Query to make in MongoDB(default: empty query) +limit: Number of rows to return in pagination(default: 10) (maximum is +set at 20 rows per request) +skip: Number of rows to skip in pagination(default: 0)

+

return: A page with some tuples or registers inside or an error if there +is no such projection. The current page is also returned to be used in +future content requests.

+
+ +Expand source code + +
def search_histogram_content(self,
+                             histogram_name: str,
+                             query: dict = {},
+                             limit: int = 10,
+                             skip: int = 0,
+                             pretty_response: bool = False) \
+        -> Union[dict, str]:
+    """
+    description: This method is responsible for retrieving the histogram
+    content.
+
+    pretty_response: If true it returns a string, otherwise a dictionary.
+    histogram_name: Represents the histogram name.
+    query: Query to make in MongoDB(default: empty query)
+    limit: Number of rows to return in pagination(default: 10) (maximum is
+    set at 20 rows per request)
+    skip: Number of rows to skip in pagination(default: 0)
+
+    return: A page with some tuples or registers inside or an error if there
+    is no such projection. The current page is also returned to be used in
+    future content requests.
+    """
+
+    response = self.__entity_reader.read_entity_content(
+        histogram_name, query, limit, skip)
+
+    return self.__response_treat.treatment(response, pretty_response)
+
+
+
+def wait(self, name: str, timeout: int = None) ‑> dict +
+
+

description: This method is responsible to create a synchronization +barrier for the run_histogram_async method or delete_histogram +method.

+

name: Represents the histogram name. +timeout: Represents the time in seconds to wait for a histogram to +finish its run.

+

return: JSON object with an error message, a warning message or a +correct histogram result

+
+ +Expand source code + +
def wait(self, name: str, timeout: int = None) -> dict:
+    """
+       description: This method is responsible to create a synchronization
+       barrier for the run_histogram_async method or delete_histogram
+       method.
+
+       name: Represents the histogram name.
+       timeout: Represents the time in seconds to wait for a histogram to
+       finish its run.
+
+       return: JSON object with an error message, a warning message or a
+       correct histogram result
+    """
+    return self.__observer.wait(name, timeout)
+
+
+
+
+
+
+
+ +
+ + + \ No newline at end of file diff --git a/html/learning_orchestra_client/explore/index.html b/html/learning_orchestra_client/explore/index.html new file mode 100644 index 0000000..7c4bfa6 --- /dev/null +++ b/html/learning_orchestra_client/explore/index.html @@ -0,0 +1,75 @@ + + + + + + +learning_orchestra_client.explore API documentation + + + + + + + + + + + +
+ + +
+ + + \ No newline at end of file diff --git a/html/learning_orchestra_client/explore/scikitlearn.html b/html/learning_orchestra_client/explore/scikitlearn.html new file mode 100644 index 0000000..4c92864 --- /dev/null +++ b/html/learning_orchestra_client/explore/scikitlearn.html @@ -0,0 +1,97 @@ + + + + + + +learning_orchestra_client.explore.scikitlearn API documentation + + + + + + + + + + + +
+
+
+

Module learning_orchestra_client.explore.scikitlearn

+
+
+
+ +Expand source code + +
from ._explore import Explore
+
+
+class ExploreScikitLearn(Explore):
+    def __init__(self, cluster_ip: str):
+        self.__api_path = "/api/learningOrchestra/v1/explore/scikitlearn"
+        self.__cluster_ip = cluster_ip
+        super().__init__(cluster_ip, self.__api_path)
+
+
+
+
+
+
+
+
+
+

Classes

+
+
+class ExploreScikitLearn +(cluster_ip: str) +
+
+
+
+ +Expand source code + +
class ExploreScikitLearn(Explore):
+    def __init__(self, cluster_ip: str):
+        self.__api_path = "/api/learningOrchestra/v1/explore/scikitlearn"
+        self.__cluster_ip = cluster_ip
+        super().__init__(cluster_ip, self.__api_path)
+
+

Ancestors

+
    +
  • learning_orchestra_client.explore._explore.Explore
    • +
+
+
+
+
+ +
+ + + \ No newline at end of file diff --git a/html/learning_orchestra_client/explore/tensorflow.html b/html/learning_orchestra_client/explore/tensorflow.html new file mode 100644 index 0000000..870d184 --- /dev/null +++ b/html/learning_orchestra_client/explore/tensorflow.html @@ -0,0 +1,97 @@ + + + + + + +learning_orchestra_client.explore.tensorflow API documentation + + + + + + + + + + + +
+
+
+

Module learning_orchestra_client.explore.tensorflow

+
+
+
+ +Expand source code + +
from ._explore import Explore
+
+
+class ExploreTensorflow(Explore):
+    def __init__(self, cluster_ip: str):
+        self.__api_path = "/api/learningOrchestra/v1/explore/tensorflow"
+        self.__cluster_ip = cluster_ip
+        super().__init__(cluster_ip, self.__api_path)
+
+
+
+
+
+
+
+
+
+

Classes

+
+
+class ExploreTensorflow +(cluster_ip: str) +
+
+
+
+ +Expand source code + +
class ExploreTensorflow(Explore):
+    def __init__(self, cluster_ip: str):
+        self.__api_path = "/api/learningOrchestra/v1/explore/tensorflow"
+        self.__cluster_ip = cluster_ip
+        super().__init__(cluster_ip, self.__api_path)
+
+

Ancestors

+
    +
  • learning_orchestra_client.explore._explore.Explore
    • +
+
+
+
+
+ +
+ + + \ No newline at end of file diff --git a/html/learning_orchestra_client/function/index.html b/html/learning_orchestra_client/function/index.html new file mode 100644 index 0000000..af278ff --- /dev/null +++ b/html/learning_orchestra_client/function/index.html @@ -0,0 +1,65 @@ + + + + + + +learning_orchestra_client.function API documentation + + + + + + + + + + + +
+ + +
+ + + \ No newline at end of file diff --git a/html/learning_orchestra_client/function/python.html b/html/learning_orchestra_client/function/python.html new file mode 100644 index 0000000..d0c9776 --- /dev/null +++ b/html/learning_orchestra_client/function/python.html @@ -0,0 +1,667 @@ + + + + + + +learning_orchestra_client.function.python API documentation + + + + + + + + + + + +
+
+
+

Module learning_orchestra_client.function.python

+
+
+
+ +Expand source code + +
from learning_orchestra_client.observe.observe import Observer
+from learning_orchestra_client._util._response_treat import ResponseTreat
+from learning_orchestra_client._util._entity_reader import EntityReader
+import requests
+from typing import Union
+
+
+class FunctionPython:
+    __CODE_FIELD = "function"
+    __PARAMETERS_FIELD = "functionParameters"
+    __NAME_FIELD = "name"
+    __DESCRIPTION_FIELD = "description"
+
+    def __init__(self, cluster_ip: str):
+        self.__api_path = "/api/learningOrchestra/v1/function/python"
+        self.__service_url = f'{cluster_ip}{self.__api_path}'
+        self.__response_treat = ResponseTreat()
+        self.__cluster_ip = cluster_ip
+        self.__entity_reader = EntityReader(self.__service_url)
+        self.__observer = Observer(self.__cluster_ip)
+
+    def run_function_sync(self,
+                          name: str,
+                          parameters: dict,
+                          code: str,
+                          description: str = "",
+                          pretty_response: bool = False) -> Union[dict, str]:
+        """
+        description: This method runs a python 3 code in sync mode, so it
+        represents a wildcard for the data scientist. It can be used when
+        train, predict, tune, explore or any other pipe must be customized. The
+        function is also useful for new pipes. pretty_response: If true it
+        returns a string, otherwise a dictionary.
+
+        name: Is the name of the object stored in Learning Orchestra storage
+        system (volume or mongoDB).
+        url: Url to CSV file.
+
+        return: A JSON object with an error or warning message or the correct
+        operation result.
+        """
+        request_body = {
+            self.__NAME_FIELD: name,
+            self.__PARAMETERS_FIELD: parameters,
+            self.__CODE_FIELD: code,
+            self.__DESCRIPTION_FIELD: description}
+
+        request_url = self.__service_url
+        response = requests.post(url=request_url, json=request_body)
+        self.__observer.wait(name)
+
+        return self.__response_treat.treatment(response, pretty_response)
+
+    def run_function_async(self,
+                           name: str,
+                           parameters: dict,
+                           code: str,
+                           description: str = "",
+                           pretty_response: bool = False) -> Union[dict, str]:
+        """
+        description: This method runs a python 3 code in async mode, so it
+        represents a wildcard for the data scientist. It does not lock the
+        caller, so a wait method must be used. It can be used when train,
+        predict, tune, explore or any other pipe must be customized. The
+        function is also useful for new pipes.
+
+        pretty_response: If true it returns a string, otherwise a dictionary.
+        name: Is the name of the function to be called
+        code: the Python code
+        parameters: the parameters of the function being called
+
+        return: A JSON object with an error or warning message or the correct
+        operation result.
+        """
+        request_body = {
+            self.__NAME_FIELD: name,
+            self.__PARAMETERS_FIELD: parameters,
+            self.__CODE_FIELD: code,
+            self.__DESCRIPTION_FIELD: description}
+
+        request_url = self.__service_url
+
+        response = requests.post(url=request_url, json=request_body)
+        return self.__response_treat.treatment(response, pretty_response)
+
+    def search_all_executions(self, pretty_response: bool = False) \
+            -> Union[dict, str]:
+        """
+        description: This method retrieves all created functions metadata,
+        i.e., it does not retrieve the function result content.
+
+        pretty_response: If true it returns a string, otherwise a dictionary.
+
+        return: All function executions metadata stored in Learning Orchestra
+        or an empty result.
+        """
+        response = self.__entity_reader.read_all_instances_from_entity()
+        return self.__response_treat.treatment(response, pretty_response)
+
+    def delete_execution(self, name: str, pretty_response=False) \
+            -> Union[dict, str]:
+        """
+        description: This method is responsible for deleting the function.
+        This delete operation is asynchronous, so it does not lock the caller
+         until the deletion finished. Instead, it returns a JSON object with a
+         URL for a future use. The caller uses the URL for delete checks.
+
+        pretty_response: If true it returns a string, otherwise a dictionary.
+        name: Represents the function name.
+
+        return: JSON object with an error message, a warning message or a
+        correct delete message
+        """
+
+        request_url = f'{self.__service_url}/{name}'
+
+        response = requests.delete(request_url)
+        return self.__response_treat.treatment(response, pretty_response)
+
+    def search_execution_content(self,
+                                 name: str,
+                                 query: dict = {},
+                                 limit: int = 10,
+                                 skip: int = 0,
+                                 pretty_response: bool = False) \
+            -> Union[dict, str]:
+        """
+        description:  This method is responsible for retrieving the function
+        results, including metadata. A function is executed many times, using
+        different parameters,
+        thus many results are stored
+        in Learning Orchestra.
+
+        pretty_response: If true it returns a string, otherwise a dictionary.
+        name: Is the name of the function.
+        query: Query to make in MongoDB(default: empty query)
+        limit: Number of rows to return in pagination(default: 10) (maximum is
+        set at 20 rows per request)
+        skip: Number of rows to skip in pagination(default: 0)
+
+        return:
+         A page with some function results inside or an error if there
+        is no such function. The current page is also returned to be used in
+        future content requests.
+        """
+
+        response = self.__entity_reader.read_entity_content(
+            name, query, limit, skip)
+
+        return self.__response_treat.treatment(response, pretty_response)
+
+    def wait(self, dataset_name: str, timeout: int = None) -> dict:
+        """
+           description: This method is responsible to create a synchronization
+           barrier for the run_function_async method or delete_function method.
+
+           name: Represents the function name.
+           timeout: Represents the time in seconds to wait for a function to
+           finish its run.
+
+           return: JSON object with an error message, a warning message or a
+           correct function result
+        """
+        return self.__observer.wait(dataset_name, timeout)
+
+
+
+
+
+
+
+
+
+

Classes

+
+
+class FunctionPython +(cluster_ip: str) +
+
+
+
+ +Expand source code + +
class FunctionPython:
+    __CODE_FIELD = "function"
+    __PARAMETERS_FIELD = "functionParameters"
+    __NAME_FIELD = "name"
+    __DESCRIPTION_FIELD = "description"
+
+    def __init__(self, cluster_ip: str):
+        self.__api_path = "/api/learningOrchestra/v1/function/python"
+        self.__service_url = f'{cluster_ip}{self.__api_path}'
+        self.__response_treat = ResponseTreat()
+        self.__cluster_ip = cluster_ip
+        self.__entity_reader = EntityReader(self.__service_url)
+        self.__observer = Observer(self.__cluster_ip)
+
+    def run_function_sync(self,
+                          name: str,
+                          parameters: dict,
+                          code: str,
+                          description: str = "",
+                          pretty_response: bool = False) -> Union[dict, str]:
+        """
+        description: This method runs a python 3 code in sync mode, so it
+        represents a wildcard for the data scientist. It can be used when
+        train, predict, tune, explore or any other pipe must be customized. The
+        function is also useful for new pipes. pretty_response: If true it
+        returns a string, otherwise a dictionary.
+
+        name: Is the name of the object stored in Learning Orchestra storage
+        system (volume or mongoDB).
+        url: Url to CSV file.
+
+        return: A JSON object with an error or warning message or the correct
+        operation result.
+        """
+        request_body = {
+            self.__NAME_FIELD: name,
+            self.__PARAMETERS_FIELD: parameters,
+            self.__CODE_FIELD: code,
+            self.__DESCRIPTION_FIELD: description}
+
+        request_url = self.__service_url
+        response = requests.post(url=request_url, json=request_body)
+        self.__observer.wait(name)
+
+        return self.__response_treat.treatment(response, pretty_response)
+
+    def run_function_async(self,
+                           name: str,
+                           parameters: dict,
+                           code: str,
+                           description: str = "",
+                           pretty_response: bool = False) -> Union[dict, str]:
+        """
+        description: This method runs a python 3 code in async mode, so it
+        represents a wildcard for the data scientist. It does not lock the
+        caller, so a wait method must be used. It can be used when train,
+        predict, tune, explore or any other pipe must be customized. The
+        function is also useful for new pipes.
+
+        pretty_response: If true it returns a string, otherwise a dictionary.
+        name: Is the name of the function to be called
+        code: the Python code
+        parameters: the parameters of the function being called
+
+        return: A JSON object with an error or warning message or the correct
+        operation result.
+        """
+        request_body = {
+            self.__NAME_FIELD: name,
+            self.__PARAMETERS_FIELD: parameters,
+            self.__CODE_FIELD: code,
+            self.__DESCRIPTION_FIELD: description}
+
+        request_url = self.__service_url
+
+        response = requests.post(url=request_url, json=request_body)
+        return self.__response_treat.treatment(response, pretty_response)
+
+    def search_all_executions(self, pretty_response: bool = False) \
+            -> Union[dict, str]:
+        """
+        description: This method retrieves all created functions metadata,
+        i.e., it does not retrieve the function result content.
+
+        pretty_response: If true it returns a string, otherwise a dictionary.
+
+        return: All function executions metadata stored in Learning Orchestra
+        or an empty result.
+        """
+        response = self.__entity_reader.read_all_instances_from_entity()
+        return self.__response_treat.treatment(response, pretty_response)
+
+    def delete_execution(self, name: str, pretty_response=False) \
+            -> Union[dict, str]:
+        """
+        description: This method is responsible for deleting the function.
+        This delete operation is asynchronous, so it does not lock the caller
+         until the deletion finished. Instead, it returns a JSON object with a
+         URL for a future use. The caller uses the URL for delete checks.
+
+        pretty_response: If true it returns a string, otherwise a dictionary.
+        name: Represents the function name.
+
+        return: JSON object with an error message, a warning message or a
+        correct delete message
+        """
+
+        request_url = f'{self.__service_url}/{name}'
+
+        response = requests.delete(request_url)
+        return self.__response_treat.treatment(response, pretty_response)
+
+    def search_execution_content(self,
+                                 name: str,
+                                 query: dict = {},
+                                 limit: int = 10,
+                                 skip: int = 0,
+                                 pretty_response: bool = False) \
+            -> Union[dict, str]:
+        """
+        description:  This method is responsible for retrieving the function
+        results, including metadata. A function is executed many times, using
+        different parameters,
+        thus many results are stored
+        in Learning Orchestra.
+
+        pretty_response: If true it returns a string, otherwise a dictionary.
+        name: Is the name of the function.
+        query: Query to make in MongoDB(default: empty query)
+        limit: Number of rows to return in pagination(default: 10) (maximum is
+        set at 20 rows per request)
+        skip: Number of rows to skip in pagination(default: 0)
+
+        return:
+         A page with some function results inside or an error if there
+        is no such function. The current page is also returned to be used in
+        future content requests.
+        """
+
+        response = self.__entity_reader.read_entity_content(
+            name, query, limit, skip)
+
+        return self.__response_treat.treatment(response, pretty_response)
+
+    def wait(self, dataset_name: str, timeout: int = None) -> dict:
+        """
+           description: This method is responsible to create a synchronization
+           barrier for the run_function_async method or delete_function method.
+
+           name: Represents the function name.
+           timeout: Represents the time in seconds to wait for a function to
+           finish its run.
+
+           return: JSON object with an error message, a warning message or a
+           correct function result
+        """
+        return self.__observer.wait(dataset_name, timeout)
+
+

Methods

+
+
+def delete_execution(self, name: str, pretty_response=False) ‑> Union[dict, str] +
+
+

description: This method is responsible for deleting the function. +This delete operation is asynchronous, so it does not lock the caller +until the deletion finished. Instead, it returns a JSON object with a +URL for a future use. The caller uses the URL for delete checks.

+

pretty_response: If true it returns a string, otherwise a dictionary. +name: Represents the function name.

+

return: JSON object with an error message, a warning message or a +correct delete message

+
+ +Expand source code + +
def delete_execution(self, name: str, pretty_response=False) \
+        -> Union[dict, str]:
+    """
+    description: This method is responsible for deleting the function.
+    This delete operation is asynchronous, so it does not lock the caller
+     until the deletion finished. Instead, it returns a JSON object with a
+     URL for a future use. The caller uses the URL for delete checks.
+
+    pretty_response: If true it returns a string, otherwise a dictionary.
+    name: Represents the function name.
+
+    return: JSON object with an error message, a warning message or a
+    correct delete message
+    """
+
+    request_url = f'{self.__service_url}/{name}'
+
+    response = requests.delete(request_url)
+    return self.__response_treat.treatment(response, pretty_response)
+
+
+
+def run_function_async(self, name: str, parameters: dict, code: str, description: str = '', pretty_response: bool = False) ‑> Union[dict, str] +
+
+

description: This method runs a python 3 code in async mode, so it +represents a wildcard for the data scientist. It does not lock the +caller, so a wait method must be used. It can be used when train, +predict, tune, explore or any other pipe must be customized. The +function is also useful for new pipes.

+

pretty_response: If true it returns a string, otherwise a dictionary. +name: Is the name of the function to be called +code: the Python code +parameters: the parameters of the function being called

+

return: A JSON object with an error or warning message or the correct +operation result.

+
+ +Expand source code + +
def run_function_async(self,
+                       name: str,
+                       parameters: dict,
+                       code: str,
+                       description: str = "",
+                       pretty_response: bool = False) -> Union[dict, str]:
+    """
+    description: This method runs a python 3 code in async mode, so it
+    represents a wildcard for the data scientist. It does not lock the
+    caller, so a wait method must be used. It can be used when train,
+    predict, tune, explore or any other pipe must be customized. The
+    function is also useful for new pipes.
+
+    pretty_response: If true it returns a string, otherwise a dictionary.
+    name: Is the name of the function to be called
+    code: the Python code
+    parameters: the parameters of the function being called
+
+    return: A JSON object with an error or warning message or the correct
+    operation result.
+    """
+    request_body = {
+        self.__NAME_FIELD: name,
+        self.__PARAMETERS_FIELD: parameters,
+        self.__CODE_FIELD: code,
+        self.__DESCRIPTION_FIELD: description}
+
+    request_url = self.__service_url
+
+    response = requests.post(url=request_url, json=request_body)
+    return self.__response_treat.treatment(response, pretty_response)
+
+
+
+def run_function_sync(self, name: str, parameters: dict, code: str, description: str = '', pretty_response: bool = False) ‑> Union[dict, str] +
+
+

description: This method runs a python 3 code in sync mode, so it +represents a wildcard for the data scientist. It can be used when +train, predict, tune, explore or any other pipe must be customized. The +function is also useful for new pipes. pretty_response: If true it +returns a string, otherwise a dictionary.

+

name: Is the name of the object stored in Learning Orchestra storage +system (volume or mongoDB). +url: Url to CSV file.

+

return: A JSON object with an error or warning message or the correct +operation result.

+
+ +Expand source code + +
def run_function_sync(self,
+                      name: str,
+                      parameters: dict,
+                      code: str,
+                      description: str = "",
+                      pretty_response: bool = False) -> Union[dict, str]:
+    """
+    description: This method runs a python 3 code in sync mode, so it
+    represents a wildcard for the data scientist. It can be used when
+    train, predict, tune, explore or any other pipe must be customized. The
+    function is also useful for new pipes. pretty_response: If true it
+    returns a string, otherwise a dictionary.
+
+    name: Is the name of the object stored in Learning Orchestra storage
+    system (volume or mongoDB).
+    url: Url to CSV file.
+
+    return: A JSON object with an error or warning message or the correct
+    operation result.
+    """
+    request_body = {
+        self.__NAME_FIELD: name,
+        self.__PARAMETERS_FIELD: parameters,
+        self.__CODE_FIELD: code,
+        self.__DESCRIPTION_FIELD: description}
+
+    request_url = self.__service_url
+    response = requests.post(url=request_url, json=request_body)
+    self.__observer.wait(name)
+
+    return self.__response_treat.treatment(response, pretty_response)
+
+
+
+def search_all_executions(self, pretty_response: bool = False) ‑> Union[dict, str] +
+
+

description: This method retrieves all created functions metadata, +i.e., it does not retrieve the function result content.

+

pretty_response: If true it returns a string, otherwise a dictionary.

+

return: All function executions metadata stored in Learning Orchestra +or an empty result.

+
+ +Expand source code + +
def search_all_executions(self, pretty_response: bool = False) \
+        -> Union[dict, str]:
+    """
+    description: This method retrieves all created functions metadata,
+    i.e., it does not retrieve the function result content.
+
+    pretty_response: If true it returns a string, otherwise a dictionary.
+
+    return: All function executions metadata stored in Learning Orchestra
+    or an empty result.
+    """
+    response = self.__entity_reader.read_all_instances_from_entity()
+    return self.__response_treat.treatment(response, pretty_response)
+
+
+
+def search_execution_content(self, name: str, query: dict = {}, limit: int = 10, skip: int = 0, pretty_response: bool = False) ‑> Union[dict, str] +
+
+

description: +This method is responsible for retrieving the function +results, including metadata. A function is executed many times, using +different parameters, +thus many results are stored +in Learning Orchestra.

+

pretty_response: If true it returns a string, otherwise a dictionary. +name: Is the name of the function. +query: Query to make in MongoDB(default: empty query) +limit: Number of rows to return in pagination(default: 10) (maximum is +set at 20 rows per request) +skip: Number of rows to skip in pagination(default: 0)

+

return: +A page with some function results inside or an error if there +is no such function. The current page is also returned to be used in +future content requests.

+
+ +Expand source code + +
def search_execution_content(self,
+                             name: str,
+                             query: dict = {},
+                             limit: int = 10,
+                             skip: int = 0,
+                             pretty_response: bool = False) \
+        -> Union[dict, str]:
+    """
+    description:  This method is responsible for retrieving the function
+    results, including metadata. A function is executed many times, using
+    different parameters,
+    thus many results are stored
+    in Learning Orchestra.
+
+    pretty_response: If true it returns a string, otherwise a dictionary.
+    name: Is the name of the function.
+    query: Query to make in MongoDB(default: empty query)
+    limit: Number of rows to return in pagination(default: 10) (maximum is
+    set at 20 rows per request)
+    skip: Number of rows to skip in pagination(default: 0)
+
+    return:
+     A page with some function results inside or an error if there
+    is no such function. The current page is also returned to be used in
+    future content requests.
+    """
+
+    response = self.__entity_reader.read_entity_content(
+        name, query, limit, skip)
+
+    return self.__response_treat.treatment(response, pretty_response)
+
+
+
+def wait(self, dataset_name: str, timeout: int = None) ‑> dict +
+
+

description: This method is responsible to create a synchronization +barrier for the run_function_async method or delete_function method.

+

name: Represents the function name. +timeout: Represents the time in seconds to wait for a function to +finish its run.

+

return: JSON object with an error message, a warning message or a +correct function result

+
+ +Expand source code + +
def wait(self, dataset_name: str, timeout: int = None) -> dict:
+    """
+       description: This method is responsible to create a synchronization
+       barrier for the run_function_async method or delete_function method.
+
+       name: Represents the function name.
+       timeout: Represents the time in seconds to wait for a function to
+       finish its run.
+
+       return: JSON object with an error message, a warning message or a
+       correct function result
+    """
+    return self.__observer.wait(dataset_name, timeout)
+
+
+
+
+
+
+
+ +
+ + + \ No newline at end of file diff --git a/html/learning_orchestra_client/index.html b/html/learning_orchestra_client/index.html new file mode 100644 index 0000000..9de3671 --- /dev/null +++ b/html/learning_orchestra_client/index.html @@ -0,0 +1,105 @@ + + + + + + +learning_orchestra_client API documentation + + + + + + + + + + + +
+ + +
+ + + \ No newline at end of file diff --git a/html/learning_orchestra_client/model/index.html b/html/learning_orchestra_client/model/index.html new file mode 100644 index 0000000..7a75edf --- /dev/null +++ b/html/learning_orchestra_client/model/index.html @@ -0,0 +1,70 @@ + + + + + + +learning_orchestra_client.model API documentation + + + + + + + + + + + +
+ + +
+ + + \ No newline at end of file diff --git a/html/learning_orchestra_client/model/scikitlearn.html b/html/learning_orchestra_client/model/scikitlearn.html new file mode 100644 index 0000000..3eacdee --- /dev/null +++ b/html/learning_orchestra_client/model/scikitlearn.html @@ -0,0 +1,97 @@ + + + + + + +learning_orchestra_client.model.scikitlearn API documentation + + + + + + + + + + + +
+
+
+

Module learning_orchestra_client.model.scikitlearn

+
+
+
+ +Expand source code + +
from ._model import Model
+
+
+class ModelScikitLearn(Model):
+    def __init__(self, cluster_ip: str):
+        self.__api_path = "/api/learningOrchestra/v1/model/scikitlearn"
+        self.__cluster_ip = cluster_ip
+        super().__init__(cluster_ip, self.__api_path)
+
+
+
+
+
+
+
+
+
+

Classes

+
+
+class ModelScikitLearn +(cluster_ip: str) +
+
+
+
+ +Expand source code + +
class ModelScikitLearn(Model):
+    def __init__(self, cluster_ip: str):
+        self.__api_path = "/api/learningOrchestra/v1/model/scikitlearn"
+        self.__cluster_ip = cluster_ip
+        super().__init__(cluster_ip, self.__api_path)
+
+

Ancestors

+
    +
  • learning_orchestra_client.model._model.Model
    • +
+
+
+
+
+ +
+ + + \ No newline at end of file diff --git a/html/learning_orchestra_client/model/tensorflow.html b/html/learning_orchestra_client/model/tensorflow.html new file mode 100644 index 0000000..1ed0030 --- /dev/null +++ b/html/learning_orchestra_client/model/tensorflow.html @@ -0,0 +1,97 @@ + + + + + + +learning_orchestra_client.model.tensorflow API documentation + + + + + + + + + + + +
+
+
+

Module learning_orchestra_client.model.tensorflow

+
+
+
+ +Expand source code + +
from ._model import Model
+
+
+class ModelTensorflow(Model):
+    def __init__(self, cluster_ip: str):
+        self.__api_path = "/api/learningOrchestra/v1/model/tensorflow"
+        self.__cluster_ip = cluster_ip
+        super().__init__(cluster_ip, self.__api_path)
+
+
+
+
+
+
+
+
+
+

Classes

+
+
+class ModelTensorflow +(cluster_ip: str) +
+
+
+
+ +Expand source code + +
class ModelTensorflow(Model):
+    def __init__(self, cluster_ip: str):
+        self.__api_path = "/api/learningOrchestra/v1/model/tensorflow"
+        self.__cluster_ip = cluster_ip
+        super().__init__(cluster_ip, self.__api_path)
+
+

Ancestors

+
    +
  • learning_orchestra_client.model._model.Model
    • +
+
+
+
+
+ +
+ + + \ No newline at end of file diff --git a/html/learning_orchestra_client/observe/index.html b/html/learning_orchestra_client/observe/index.html new file mode 100644 index 0000000..c21c5e9 --- /dev/null +++ b/html/learning_orchestra_client/observe/index.html @@ -0,0 +1,65 @@ + + + + + + +learning_orchestra_client.observe API documentation + + + + + + + + + + + +
+ + +
+ + + \ No newline at end of file diff --git a/html/learning_orchestra_client/observe/observe.html b/html/learning_orchestra_client/observe/observe.html new file mode 100644 index 0000000..a0b1ffa --- /dev/null +++ b/html/learning_orchestra_client/observe/observe.html @@ -0,0 +1,374 @@ + + + + + + +learning_orchestra_client.observe.observe API documentation + + + + + + + + + + + +
+
+
+

Module learning_orchestra_client.observe.observe

+
+
+
+ +Expand source code + +
from pymongo import MongoClient, change_stream
+
+
+class Observer:
+    __TIMEOUT_TIME_MULTIPLICATION = 1000
+    __MAX_WAIT_TIME = 240
+
+    def __init__(self, cluster_ip: str):
+        cluster_ip = cluster_ip.replace("http://", "")
+        mongo_url = f'mongodb://root:owl45%2321@{cluster_ip}'
+        mongo_client = MongoClient(
+            mongo_url
+        )
+
+        self.__database = mongo_client.database
+
+    def wait(self, name: str, timeout: int = None) -> dict:
+        """
+        :description: Observe the end of a pipe for a timeout seconds or
+        until the pipe finishes its execution.
+
+        name: Represents the pipe name. Any tune, train, predict service can
+        wait its finish with a
+        wait method call.
+        timeout: the maximum time to wait the observed step, in seconds.
+        Default time is 4 minutes.
+
+        :return: If True it returns a String. Otherwise, it returns
+        a dictionary with the content of a mongo collection, representing
+        any pipe result
+        """
+
+        dataset_collection = self.__database[name]
+        metadata_query = {"_id": 0}
+        dataset_metadata = dataset_collection.find_one(metadata_query)
+
+        if dataset_metadata["finished"]:
+            return dataset_metadata
+
+        observer_query = [
+            {'$match': {
+                '$and':
+                    [
+                        {'operationType': 'update'},
+                        {'fullDocument.finished': {'$eq': True}}
+                    ]
+            }}
+        ]
+
+        timeout = self.__MAX_WAIT_TIME if timeout is None else timeout 
+
+        return dataset_collection.watch(
+            observer_query,
+            full_document='updateLookup',
+            max_await_time_ms=timeout * self.__TIMEOUT_TIME_MULTIPLICATION
+        ).next()['fullDocument']
+
+    def observe_pipe(self, name: str, timeout: int = None) -> \
+            change_stream.CollectionChangeStream:
+        """
+        :description: It waits until a pipe change its content
+        (replace, insert, update and delete mongoDB collection operation
+        types), so it is a bit different
+        from wait method with a timeout and a finish explicit condition.
+
+        :name: the name of the pipe to be observed. A train, predict, explore,
+        transform or any
+        other pipe can be observed.
+        timeout: the maximum time to wait the observed step, in milliseconds.
+
+        :return: A pymongo CollectionChangeStream object. You must use the
+        builtin next() method to iterate over changes.
+        """
+
+        observer_query = [
+            {'$match': {
+                '$or': [
+                    {'operationType': 'replace'},
+                    {'operationType': 'insert'},
+                    {'operationType': 'update'},
+                    {'operationType': 'delete'}
+
+                ]
+            }}
+        ]
+        return self.__database[name].watch(
+            observer_query,
+            max_await_time_ms=timeout * self.__TIMEOUT_TIME_MULTIPLICATION,
+            full_document='updateLookup')
+
+
+
+
+
+
+
+
+
+

Classes

+
+
+class Observer +(cluster_ip: str) +
+
+
+
+ +Expand source code + +
class Observer:
+    __TIMEOUT_TIME_MULTIPLICATION = 1000
+    __MAX_WAIT_TIME = 240
+
+    def __init__(self, cluster_ip: str):
+        cluster_ip = cluster_ip.replace("http://", "")
+        mongo_url = f'mongodb://root:owl45%2321@{cluster_ip}'
+        mongo_client = MongoClient(
+            mongo_url
+        )
+
+        self.__database = mongo_client.database
+
+    def wait(self, name: str, timeout: int = None) -> dict:
+        """
+        :description: Observe the end of a pipe for a timeout seconds or
+        until the pipe finishes its execution.
+
+        name: Represents the pipe name. Any tune, train, predict service can
+        wait its finish with a
+        wait method call.
+        timeout: the maximum time to wait the observed step, in seconds.
+        Default time is 4 minutes.
+
+        :return: If True it returns a String. Otherwise, it returns
+        a dictionary with the content of a mongo collection, representing
+        any pipe result
+        """
+
+        dataset_collection = self.__database[name]
+        metadata_query = {"_id": 0}
+        dataset_metadata = dataset_collection.find_one(metadata_query)
+
+        if dataset_metadata["finished"]:
+            return dataset_metadata
+
+        observer_query = [
+            {'$match': {
+                '$and':
+                    [
+                        {'operationType': 'update'},
+                        {'fullDocument.finished': {'$eq': True}}
+                    ]
+            }}
+        ]
+
+        timeout = self.__MAX_WAIT_TIME if timeout is None else timeout 
+
+        return dataset_collection.watch(
+            observer_query,
+            full_document='updateLookup',
+            max_await_time_ms=timeout * self.__TIMEOUT_TIME_MULTIPLICATION
+        ).next()['fullDocument']
+
+    def observe_pipe(self, name: str, timeout: int = None) -> \
+            change_stream.CollectionChangeStream:
+        """
+        :description: It waits until a pipe change its content
+        (replace, insert, update and delete mongoDB collection operation
+        types), so it is a bit different
+        from wait method with a timeout and a finish explicit condition.
+
+        :name: the name of the pipe to be observed. A train, predict, explore,
+        transform or any
+        other pipe can be observed.
+        timeout: the maximum time to wait the observed step, in milliseconds.
+
+        :return: A pymongo CollectionChangeStream object. You must use the
+        builtin next() method to iterate over changes.
+        """
+
+        observer_query = [
+            {'$match': {
+                '$or': [
+                    {'operationType': 'replace'},
+                    {'operationType': 'insert'},
+                    {'operationType': 'update'},
+                    {'operationType': 'delete'}
+
+                ]
+            }}
+        ]
+        return self.__database[name].watch(
+            observer_query,
+            max_await_time_ms=timeout * self.__TIMEOUT_TIME_MULTIPLICATION,
+            full_document='updateLookup')
+
+

Methods

+
+
+def observe_pipe(self, name: str, timeout: int = None) ‑> pymongo.change_stream.CollectionChangeStream +
+
+

:description: It waits until a pipe change its content +(replace, insert, update and delete mongoDB collection operation +types), so it is a bit different +from wait method with a timeout and a finish explicit condition.

+

:name: the name of the pipe to be observed. A train, predict, explore, +transform or any +other pipe can be observed. +timeout: the maximum time to wait the observed step, in milliseconds.

+

:return: A pymongo CollectionChangeStream object. You must use the +builtin next() method to iterate over changes.

+
+ +Expand source code + +
def observe_pipe(self, name: str, timeout: int = None) -> \
+        change_stream.CollectionChangeStream:
+    """
+    :description: It waits until a pipe change its content
+    (replace, insert, update and delete mongoDB collection operation
+    types), so it is a bit different
+    from wait method with a timeout and a finish explicit condition.
+
+    :name: the name of the pipe to be observed. A train, predict, explore,
+    transform or any
+    other pipe can be observed.
+    timeout: the maximum time to wait the observed step, in milliseconds.
+
+    :return: A pymongo CollectionChangeStream object. You must use the
+    builtin next() method to iterate over changes.
+    """
+
+    observer_query = [
+        {'$match': {
+            '$or': [
+                {'operationType': 'replace'},
+                {'operationType': 'insert'},
+                {'operationType': 'update'},
+                {'operationType': 'delete'}
+
+            ]
+        }}
+    ]
+    return self.__database[name].watch(
+        observer_query,
+        max_await_time_ms=timeout * self.__TIMEOUT_TIME_MULTIPLICATION,
+        full_document='updateLookup')
+
+
+
+def wait(self, name: str, timeout: int = None) ‑> dict +
+
+

:description: Observe the end of a pipe for a timeout seconds or +until the pipe finishes its execution.

+

name: Represents the pipe name. Any tune, train, predict service can +wait its finish with a +wait method call. +timeout: the maximum time to wait the observed step, in seconds. +Default time is 4 minutes.

+

:return: If True it returns a String. Otherwise, it returns +a dictionary with the content of a mongo collection, representing +any pipe result

+
+ +Expand source code + +
def wait(self, name: str, timeout: int = None) -> dict:
+    """
+    :description: Observe the end of a pipe for a timeout seconds or
+    until the pipe finishes its execution.
+
+    name: Represents the pipe name. Any tune, train, predict service can
+    wait its finish with a
+    wait method call.
+    timeout: the maximum time to wait the observed step, in seconds.
+    Default time is 4 minutes.
+
+    :return: If True it returns a String. Otherwise, it returns
+    a dictionary with the content of a mongo collection, representing
+    any pipe result
+    """
+
+    dataset_collection = self.__database[name]
+    metadata_query = {"_id": 0}
+    dataset_metadata = dataset_collection.find_one(metadata_query)
+
+    if dataset_metadata["finished"]:
+        return dataset_metadata
+
+    observer_query = [
+        {'$match': {
+            '$and':
+                [
+                    {'operationType': 'update'},
+                    {'fullDocument.finished': {'$eq': True}}
+                ]
+        }}
+    ]
+
+    timeout = self.__MAX_WAIT_TIME if timeout is None else timeout 
+
+    return dataset_collection.watch(
+        observer_query,
+        full_document='updateLookup',
+        max_await_time_ms=timeout * self.__TIMEOUT_TIME_MULTIPLICATION
+    ).next()['fullDocument']
+
+
+
+
+
+
+
+ +
+ + + \ No newline at end of file diff --git a/html/learning_orchestra_client/predict/index.html b/html/learning_orchestra_client/predict/index.html new file mode 100644 index 0000000..f152cef --- /dev/null +++ b/html/learning_orchestra_client/predict/index.html @@ -0,0 +1,70 @@ + + + + + + +learning_orchestra_client.predict API documentation + + + + + + + + + + + +
+ + +
+ + + \ No newline at end of file diff --git a/html/learning_orchestra_client/predict/scikitlearn.html b/html/learning_orchestra_client/predict/scikitlearn.html new file mode 100644 index 0000000..19a199d --- /dev/null +++ b/html/learning_orchestra_client/predict/scikitlearn.html @@ -0,0 +1,97 @@ + + + + + + +learning_orchestra_client.predict.scikitlearn API documentation + + + + + + + + + + + +
+
+
+

Module learning_orchestra_client.predict.scikitlearn

+
+
+
+ +Expand source code + +
from ._predict import Predict
+
+
+class PredictScikitLearn(Predict):
+    def __init__(self, cluster_ip: str):
+        self.__api_path = "/api/learningOrchestra/v1/predict/scikitlearn"
+        self.__cluster_ip = cluster_ip
+        super().__init__(cluster_ip, self.__api_path)
+
+
+
+
+
+
+
+
+
+

Classes

+
+
+class PredictScikitLearn +(cluster_ip: str) +
+
+
+
+ +Expand source code + +
class PredictScikitLearn(Predict):
+    def __init__(self, cluster_ip: str):
+        self.__api_path = "/api/learningOrchestra/v1/predict/scikitlearn"
+        self.__cluster_ip = cluster_ip
+        super().__init__(cluster_ip, self.__api_path)
+
+

Ancestors

+
    +
  • learning_orchestra_client.predict._predict.Predict
    • +
+
+
+
+
+ +
+ + + \ No newline at end of file diff --git a/html/learning_orchestra_client/predict/tensorflow.html b/html/learning_orchestra_client/predict/tensorflow.html new file mode 100644 index 0000000..bb610fa --- /dev/null +++ b/html/learning_orchestra_client/predict/tensorflow.html @@ -0,0 +1,97 @@ + + + + + + +learning_orchestra_client.predict.tensorflow API documentation + + + + + + + + + + + +
+
+
+

Module learning_orchestra_client.predict.tensorflow

+
+
+
+ +Expand source code + +
from ._predict import Predict
+
+
+class PredictTensorflow(Predict):
+    def __init__(self, cluster_ip: str):
+        self.__api_path = "/api/learningOrchestra/v1/predict/tensorflow"
+        self.__cluster_ip = cluster_ip
+        super().__init__(cluster_ip, self.__api_path)
+
+
+
+
+
+
+
+
+
+

Classes

+
+
+class PredictTensorflow +(cluster_ip: str) +
+
+
+
+ +Expand source code + +
class PredictTensorflow(Predict):
+    def __init__(self, cluster_ip: str):
+        self.__api_path = "/api/learningOrchestra/v1/predict/tensorflow"
+        self.__cluster_ip = cluster_ip
+        super().__init__(cluster_ip, self.__api_path)
+
+

Ancestors

+
    +
  • learning_orchestra_client.predict._predict.Predict
    • +
+
+
+
+
+ +
+ + + \ No newline at end of file diff --git a/html/learning_orchestra_client/train/horovod.html b/html/learning_orchestra_client/train/horovod.html new file mode 100644 index 0000000..0ff658f --- /dev/null +++ b/html/learning_orchestra_client/train/horovod.html @@ -0,0 +1,396 @@ + + + + + + +learning_orchestra_client.train.horovod API documentation + + + + + + + + + + + +
+
+
+

Module learning_orchestra_client.train.horovod

+
+
+
+ +Expand source code + +
from ._train import Train
+import requests
+from typing import Union
+
+
+class TrainHorovod(Train):
+    __PARENT_NAME_FIELD = "parentName"
+    __METHOD_NAME_FIELD = "method"
+    __ClASS_PARAMETERS_FIELD = "methodParameters"
+    __NAME_FIELD = "name"
+    __DESCRIPTION_FIELD = "description"
+    __COMPILE_CODE = "compileCode"
+    __MONITORING_PATH = "monitoringPath"
+
+    def __init__(self, cluster_ip: str):
+        self.__api_path = "/api/learningOrchestra/v1/train/horovod"
+        self.__cluster_ip = cluster_ip
+        super().__init__(cluster_ip, self.__api_path)
+
+    def create_training_async(self,
+                              name: str,
+                              model_name: str,
+                              parent_name: str,
+                              parameters: dict,
+                              description: str = "",
+                              compiling_code: str = "",
+                              monitoring_path: str = None,
+                              pretty_response: bool = False) -> \
+            Union[dict, str]:
+        """
+                description: This method is responsible to train models in async mode.
+                A wait method call is mandatory due to the asynchronous aspect.
+
+                pretty_response: If true it returns a string, otherwise a dictionary.
+                name: Is the name of the train output object that will be created.
+                parent_name: Is the name of the previous ML step of the pipeline
+                method_name: is the name of the method to be executed (the ML tool way
+                to train models)
+                parameters: Is the set of parameters used by the method
+
+                return: A JSON object with an error or warning message or a URL
+                indicating the correct operation.
+                """
+        request_body = {
+            self.__NAME_FIELD: name,
+            self.__MODEL_NAME_FIELD: model_name,
+            self.__PARENT_NAME_FIELD: parent_name,
+            self.__ClASS_PARAMETERS_FIELD: parameters,
+            self.__DESCRIPTION_FIELD: description,
+            self.__COMPILE_CODE: compiling_code,
+            self.__MONITORING_PATH: monitoring_path,
+        }
+
+        request_url = self.__service_url
+
+        response = requests.post(url=request_url, json=request_body)
+        return self.__response_treat.treatment(response, pretty_response)
+
+    def create_training_sync(self,
+                             name: str,
+                             model_name: str,
+                             parent_name: str,
+                             parameters: dict,
+                             description: str = "",
+                             compiling_code: str = "",
+                             monitoring_path: str = None,
+                             pretty_response: bool = False) -> \
+            Union[dict, str]:
+        """
+        description: This method is responsible to train models in sync mode
+
+        pretty_response: If true it returns a string, otherwise a dictionary.
+        name: Is the name of the train output object that will be created.
+        parent_name: Is the name of the previous ML step of the pipeline
+        method_name: is the name of the method to be executed (the ML tool way
+        to train models)
+        parameters: Is the set of parameters used by the method
+
+        return: A JSON object with an error or warning message or a URL
+        indicating the correct operation.
+        """
+        request_body = {
+            self.__NAME_FIELD: name,
+            self.__MODEL_NAME_FIELD: model_name,
+            self.__PARENT_NAME_FIELD: parent_name,
+            self.__ClASS_PARAMETERS_FIELD: parameters,
+            self.__DESCRIPTION_FIELD: description,
+            self.__COMPILE_CODE: compiling_code,
+            self.__MONITORING_PATH: monitoring_path,
+        }
+
+        request_url = self.__service_url
+
+        response = requests.post(url=request_url, json=request_body)
+        self.__observer.wait(name)
+
+        return self.__response_treat.treatment(response, pretty_response)
+
+
+
+
+
+
+
+
+
+

Classes

+
+
+class TrainHorovod +(cluster_ip: str) +
+
+
+
+ +Expand source code + +
class TrainHorovod(Train):
+    __PARENT_NAME_FIELD = "parentName"
+    __METHOD_NAME_FIELD = "method"
+    __ClASS_PARAMETERS_FIELD = "methodParameters"
+    __NAME_FIELD = "name"
+    __DESCRIPTION_FIELD = "description"
+    __COMPILE_CODE = "compileCode"
+    __MONITORING_PATH = "monitoringPath"
+
+    def __init__(self, cluster_ip: str):
+        self.__api_path = "/api/learningOrchestra/v1/train/horovod"
+        self.__cluster_ip = cluster_ip
+        super().__init__(cluster_ip, self.__api_path)
+
+    def create_training_async(self,
+                              name: str,
+                              model_name: str,
+                              parent_name: str,
+                              parameters: dict,
+                              description: str = "",
+                              compiling_code: str = "",
+                              monitoring_path: str = None,
+                              pretty_response: bool = False) -> \
+            Union[dict, str]:
+        """
+                description: This method is responsible to train models in async mode.
+                A wait method call is mandatory due to the asynchronous aspect.
+
+                pretty_response: If true it returns a string, otherwise a dictionary.
+                name: Is the name of the train output object that will be created.
+                parent_name: Is the name of the previous ML step of the pipeline
+                method_name: is the name of the method to be executed (the ML tool way
+                to train models)
+                parameters: Is the set of parameters used by the method
+
+                return: A JSON object with an error or warning message or a URL
+                indicating the correct operation.
+                """
+        request_body = {
+            self.__NAME_FIELD: name,
+            self.__MODEL_NAME_FIELD: model_name,
+            self.__PARENT_NAME_FIELD: parent_name,
+            self.__ClASS_PARAMETERS_FIELD: parameters,
+            self.__DESCRIPTION_FIELD: description,
+            self.__COMPILE_CODE: compiling_code,
+            self.__MONITORING_PATH: monitoring_path,
+        }
+
+        request_url = self.__service_url
+
+        response = requests.post(url=request_url, json=request_body)
+        return self.__response_treat.treatment(response, pretty_response)
+
+    def create_training_sync(self,
+                             name: str,
+                             model_name: str,
+                             parent_name: str,
+                             parameters: dict,
+                             description: str = "",
+                             compiling_code: str = "",
+                             monitoring_path: str = None,
+                             pretty_response: bool = False) -> \
+            Union[dict, str]:
+        """
+        description: This method is responsible to train models in sync mode
+
+        pretty_response: If true it returns a string, otherwise a dictionary.
+        name: Is the name of the train output object that will be created.
+        parent_name: Is the name of the previous ML step of the pipeline
+        method_name: is the name of the method to be executed (the ML tool way
+        to train models)
+        parameters: Is the set of parameters used by the method
+
+        return: A JSON object with an error or warning message or a URL
+        indicating the correct operation.
+        """
+        request_body = {
+            self.__NAME_FIELD: name,
+            self.__MODEL_NAME_FIELD: model_name,
+            self.__PARENT_NAME_FIELD: parent_name,
+            self.__ClASS_PARAMETERS_FIELD: parameters,
+            self.__DESCRIPTION_FIELD: description,
+            self.__COMPILE_CODE: compiling_code,
+            self.__MONITORING_PATH: monitoring_path,
+        }
+
+        request_url = self.__service_url
+
+        response = requests.post(url=request_url, json=request_body)
+        self.__observer.wait(name)
+
+        return self.__response_treat.treatment(response, pretty_response)
+
+

Ancestors

+
    +
  • learning_orchestra_client.train._train.Train
    • +
+

Methods

+
+
+def create_training_async(self, name: str, model_name: str, parent_name: str, parameters: dict, description: str = '', compiling_code: str = '', monitoring_path: str = None, pretty_response: bool = False) ‑> Union[dict, str] +
+
+

description: This method is responsible to train models in async mode. +A wait method call is mandatory due to the asynchronous aspect.

+

pretty_response: If true it returns a string, otherwise a dictionary. +name: Is the name of the train output object that will be created. +parent_name: Is the name of the previous ML step of the pipeline +method_name: is the name of the method to be executed (the ML tool way +to train models) +parameters: Is the set of parameters used by the method

+

return: A JSON object with an error or warning message or a URL +indicating the correct operation.

+
+ +Expand source code + +
def create_training_async(self,
+                          name: str,
+                          model_name: str,
+                          parent_name: str,
+                          parameters: dict,
+                          description: str = "",
+                          compiling_code: str = "",
+                          monitoring_path: str = None,
+                          pretty_response: bool = False) -> \
+        Union[dict, str]:
+    """
+            description: This method is responsible to train models in async mode.
+            A wait method call is mandatory due to the asynchronous aspect.
+
+            pretty_response: If true it returns a string, otherwise a dictionary.
+            name: Is the name of the train output object that will be created.
+            parent_name: Is the name of the previous ML step of the pipeline
+            method_name: is the name of the method to be executed (the ML tool way
+            to train models)
+            parameters: Is the set of parameters used by the method
+
+            return: A JSON object with an error or warning message or a URL
+            indicating the correct operation.
+            """
+    request_body = {
+        self.__NAME_FIELD: name,
+        self.__MODEL_NAME_FIELD: model_name,
+        self.__PARENT_NAME_FIELD: parent_name,
+        self.__ClASS_PARAMETERS_FIELD: parameters,
+        self.__DESCRIPTION_FIELD: description,
+        self.__COMPILE_CODE: compiling_code,
+        self.__MONITORING_PATH: monitoring_path,
+    }
+
+    request_url = self.__service_url
+
+    response = requests.post(url=request_url, json=request_body)
+    return self.__response_treat.treatment(response, pretty_response)
+
+
+
+def create_training_sync(self, name: str, model_name: str, parent_name: str, parameters: dict, description: str = '', compiling_code: str = '', monitoring_path: str = None, pretty_response: bool = False) ‑> Union[dict, str] +
+
+

description: This method is responsible to train models in sync mode

+

pretty_response: If true it returns a string, otherwise a dictionary. +name: Is the name of the train output object that will be created. +parent_name: Is the name of the previous ML step of the pipeline +method_name: is the name of the method to be executed (the ML tool way +to train models) +parameters: Is the set of parameters used by the method

+

return: A JSON object with an error or warning message or a URL +indicating the correct operation.

+
+ +Expand source code + +
def create_training_sync(self,
+                         name: str,
+                         model_name: str,
+                         parent_name: str,
+                         parameters: dict,
+                         description: str = "",
+                         compiling_code: str = "",
+                         monitoring_path: str = None,
+                         pretty_response: bool = False) -> \
+        Union[dict, str]:
+    """
+    description: This method is responsible to train models in sync mode
+
+    pretty_response: If true it returns a string, otherwise a dictionary.
+    name: Is the name of the train output object that will be created.
+    parent_name: Is the name of the previous ML step of the pipeline
+    method_name: is the name of the method to be executed (the ML tool way
+    to train models)
+    parameters: Is the set of parameters used by the method
+
+    return: A JSON object with an error or warning message or a URL
+    indicating the correct operation.
+    """
+    request_body = {
+        self.__NAME_FIELD: name,
+        self.__MODEL_NAME_FIELD: model_name,
+        self.__PARENT_NAME_FIELD: parent_name,
+        self.__ClASS_PARAMETERS_FIELD: parameters,
+        self.__DESCRIPTION_FIELD: description,
+        self.__COMPILE_CODE: compiling_code,
+        self.__MONITORING_PATH: monitoring_path,
+    }
+
+    request_url = self.__service_url
+
+    response = requests.post(url=request_url, json=request_body)
+    self.__observer.wait(name)
+
+    return self.__response_treat.treatment(response, pretty_response)
+
+
+
+
+
+
+
+ +
+ + + \ No newline at end of file diff --git a/html/learning_orchestra_client/train/index.html b/html/learning_orchestra_client/train/index.html new file mode 100644 index 0000000..b009587 --- /dev/null +++ b/html/learning_orchestra_client/train/index.html @@ -0,0 +1,75 @@ + + + + + + +learning_orchestra_client.train API documentation + + + + + + + + + + + +
+ + +
+ + + \ No newline at end of file diff --git a/html/learning_orchestra_client/train/scikitlearn.html b/html/learning_orchestra_client/train/scikitlearn.html new file mode 100644 index 0000000..f98998d --- /dev/null +++ b/html/learning_orchestra_client/train/scikitlearn.html @@ -0,0 +1,109 @@ + + + + + + +learning_orchestra_client.train.scikitlearn API documentation + + + + + + + + + + + +
+
+
+

Module learning_orchestra_client.train.scikitlearn

+
+
+
+ +Expand source code + +
from ._train import Train
+
+
+class TrainScikitLearn(Train):
+    __PARENT_NAME_FIELD = "parentName"
+    __METHOD_NAME_FIELD = "method"
+    __ClASS_PARAMETERS_FIELD = "methodParameters"
+    __NAME_FIELD = "name"
+    __DESCRIPTION_FIELD = "description"
+
+    def __init__(self, cluster_ip: str):
+        self.__api_path = "/api/learningOrchestra/v1/train/scikitlearn"
+        self.__cluster_ip = cluster_ip
+        super().__init__(cluster_ip, self.__api_path)
+
+
+
+
+
+
+
+
+
+

Classes

+
+
+class TrainScikitLearn +(cluster_ip: str) +
+
+
+
+ +Expand source code + +
class TrainScikitLearn(Train):
+    __PARENT_NAME_FIELD = "parentName"
+    __METHOD_NAME_FIELD = "method"
+    __ClASS_PARAMETERS_FIELD = "methodParameters"
+    __NAME_FIELD = "name"
+    __DESCRIPTION_FIELD = "description"
+
+    def __init__(self, cluster_ip: str):
+        self.__api_path = "/api/learningOrchestra/v1/train/scikitlearn"
+        self.__cluster_ip = cluster_ip
+        super().__init__(cluster_ip, self.__api_path)
+
+

Ancestors

+
    +
  • learning_orchestra_client.train._train.Train
    • +
+
+
+
+
+ +
+ + + \ No newline at end of file diff --git a/html/learning_orchestra_client/train/tensorflow.html b/html/learning_orchestra_client/train/tensorflow.html new file mode 100644 index 0000000..68caf10 --- /dev/null +++ b/html/learning_orchestra_client/train/tensorflow.html @@ -0,0 +1,109 @@ + + + + + + +learning_orchestra_client.train.tensorflow API documentation + + + + + + + + + + + +
+
+
+

Module learning_orchestra_client.train.tensorflow

+
+
+
+ +Expand source code + +
from ._train import Train
+
+
+class TrainTensorflow(Train):
+    __PARENT_NAME_FIELD = "parentName"
+    __METHOD_NAME_FIELD = "method"
+    __ClASS_PARAMETERS_FIELD = "methodParameters"
+    __NAME_FIELD = "name"
+    __DESCRIPTION_FIELD = "description"
+
+    def __init__(self, cluster_ip: str):
+        self.__api_path = "/api/learningOrchestra/v1/train/tensorflow"
+        self.__cluster_ip = cluster_ip
+        super().__init__(cluster_ip, self.__api_path)
+
+
+
+
+
+
+
+
+
+

Classes

+
+
+class TrainTensorflow +(cluster_ip: str) +
+
+
+
+ +Expand source code + +
class TrainTensorflow(Train):
+    __PARENT_NAME_FIELD = "parentName"
+    __METHOD_NAME_FIELD = "method"
+    __ClASS_PARAMETERS_FIELD = "methodParameters"
+    __NAME_FIELD = "name"
+    __DESCRIPTION_FIELD = "description"
+
+    def __init__(self, cluster_ip: str):
+        self.__api_path = "/api/learningOrchestra/v1/train/tensorflow"
+        self.__cluster_ip = cluster_ip
+        super().__init__(cluster_ip, self.__api_path)
+
+

Ancestors

+
    +
  • learning_orchestra_client.train._train.Train
    • +
+
+
+
+
+ +
+ + + \ No newline at end of file diff --git a/html/learning_orchestra_client/transform/data_type.html b/html/learning_orchestra_client/transform/data_type.html new file mode 100644 index 0000000..5618ff9 --- /dev/null +++ b/html/learning_orchestra_client/transform/data_type.html @@ -0,0 +1,606 @@ + + + + + + +learning_orchestra_client.transform.data_type API documentation + + + + + + + + + + + +
+
+
+

Module learning_orchestra_client.transform.data_type

+
+
+
+ +Expand source code + +
from learning_orchestra_client._util._response_treat import ResponseTreat
+from learning_orchestra_client._util._entity_reader import EntityReader
+from learning_orchestra_client.observe.observe import Observer
+import requests
+from typing import Union
+
+
+class TransformDataType:
+    __INPUT_NAME = "inputDatasetName"
+    __TYPES = "types"
+
+    def __init__(self, cluster_ip: str):
+        self.__api_path = "/api/learningOrchestra/v1/transform/dataType"
+        self.__service_url = f'{cluster_ip}{self.__api_path}'
+        self.__response_treat = ResponseTreat()
+        self.__cluster_ip = cluster_ip
+        self.__entity_reader = EntityReader(self.__service_url)
+        self.__observer = Observer(self.__cluster_ip)
+
+    def update_dataset_type_sync(self,
+                                 dataset_name: str,
+                                 types: dict,
+                                 pretty_response: bool = False) \
+            -> Union[dict, str]:
+        """
+        description: Change dataset field types (from number to string and
+        vice-versa). Many type modifications can be performed in one method
+        call.
+
+        dataset_name: Represents the dataset name.
+        types: Represents a map, where the pair key:value is a field:type
+
+        return: A JSON object with error or warning messages or a correct
+        datatype result.
+        """
+        url_request = self.__service_url
+        body_request = {
+            self.__INPUT_NAME: dataset_name,
+            self.__TYPES: types
+        }
+
+        response = requests.patch(url=url_request, json=body_request)
+        self.__observer.wait(dataset_name)
+
+        return self.__response_treat.treatment(response, pretty_response)
+
+    def update_dataset_type_async(self,
+                                  dataset_name: str,
+                                  types: dict,
+                                  pretty_response: bool = False) \
+            -> Union[dict, str]:
+        """
+        description: Change dataset field types (from number to string and
+        vice-versa). Many type modifications can be performed in one method
+        call. Is is an asynchronous call, thus a wait method must be also
+        called to guarantee a synchronization barrier.
+
+        dataset_name: Represents the dataset name.
+        types: Represents a map, where the pair key:value is a field:type
+
+        return: A JSON object with error or warning messages or a correct
+        datatype result.
+        """
+        url_request = self.__service_url
+        body_request = {
+            self.__INPUT_NAME: dataset_name,
+            self.__TYPES: types
+        }
+
+        response = requests.patch(url=url_request, json=body_request)
+        return self.__response_treat.treatment(response, pretty_response)
+
+    def search_all_datatype(self, pretty_response: bool = False) \
+            -> Union[dict, str]:
+        """
+        description: This method retrieves all datatype metadata, i.e., it does
+        not retrieve the datatype content.
+
+        pretty_response: If true it returns a string, otherwise a dictionary.
+
+        return: All predict metadata stored in Learning Orchestra or an empty
+        result.
+        """
+        response = self.__entity_reader.read_all_instances_from_entity()
+        return self.__response_treat.treatment(response, pretty_response)
+
+    def delete_datatype(self, name: str, pretty_response=False) \
+            -> Union[dict, str]:
+        """
+        description: This method is responsible for deleting the datatype step.
+        This delete operation is asynchronous, so it does not lock the caller
+         until the deletion finished. Instead, it returns a JSON object with a
+         URL for a future use. The caller uses the URL for delete checks.
+
+        pretty_response: If true it returns a string, otherwise a dictionary.
+        name: Represents the datatype name.
+
+        return: JSON object with an error message, a warning message or a
+        correct delete message
+        """
+        request_url = f'{self.__service_url}/{name}'
+
+        response = requests.delete(request_url)
+        return self.__response_treat.treatment(response, pretty_response)
+
+    def search_datatype_content(self,
+                                name: str,
+                                query: dict = {},
+                                limit: int = 10,
+                                skip: int = 0,
+                                pretty_response: bool = False) \
+            -> Union[dict, str]:
+        """
+        description:  This method is responsible for retrieving all the datatype
+        tuples or registers, as well as the metadata content
+
+        pretty_response: If true it returns a string, otherwise a dictionary.
+        name: Is the name of the datatype object
+        query: Query to make in MongoDB(default: empty query)
+        limit: Number of rows to return in pagination(default: 10) (maximum is
+        set at 20 rows per request)
+        skip: Number of rows to skip in pagination(default: 0)
+
+        return: A page with some registers or tuples inside or an error if there
+        is no such datatype object. The current page is also returned to be
+        used in future content requests.
+        """
+        response = self.__entity_reader.read_entity_content(
+            name, query, limit, skip)
+
+        return self.__response_treat.treatment(response, pretty_response)
+
+    def wait(self, dataset_name: str, timeout: int = None) -> dict:
+        """
+           description: This method is responsible to create a synchronization
+           barrier for the update_dataset_type_async method, delete_datatype
+           method.
+
+           name: Represents the datatype name.
+           timeout: Represents the time in seconds to wait for a datatype to
+           finish its run.
+
+           return: JSON object with an error message, a warning message or a
+           correct datatype result
+        """
+        return self.__observer.wait(dataset_name, timeout)
+
+
+
+
+
+
+
+
+
+

Classes

+
+
+class TransformDataType +(cluster_ip: str) +
+
+
+
+ +Expand source code + +
class TransformDataType:
+    __INPUT_NAME = "inputDatasetName"
+    __TYPES = "types"
+
+    def __init__(self, cluster_ip: str):
+        self.__api_path = "/api/learningOrchestra/v1/transform/dataType"
+        self.__service_url = f'{cluster_ip}{self.__api_path}'
+        self.__response_treat = ResponseTreat()
+        self.__cluster_ip = cluster_ip
+        self.__entity_reader = EntityReader(self.__service_url)
+        self.__observer = Observer(self.__cluster_ip)
+
+    def update_dataset_type_sync(self,
+                                 dataset_name: str,
+                                 types: dict,
+                                 pretty_response: bool = False) \
+            -> Union[dict, str]:
+        """
+        description: Change dataset field types (from number to string and
+        vice-versa). Many type modifications can be performed in one method
+        call.
+
+        dataset_name: Represents the dataset name.
+        types: Represents a map, where the pair key:value is a field:type
+
+        return: A JSON object with error or warning messages or a correct
+        datatype result.
+        """
+        url_request = self.__service_url
+        body_request = {
+            self.__INPUT_NAME: dataset_name,
+            self.__TYPES: types
+        }
+
+        response = requests.patch(url=url_request, json=body_request)
+        self.__observer.wait(dataset_name)
+
+        return self.__response_treat.treatment(response, pretty_response)
+
+    def update_dataset_type_async(self,
+                                  dataset_name: str,
+                                  types: dict,
+                                  pretty_response: bool = False) \
+            -> Union[dict, str]:
+        """
+        description: Change dataset field types (from number to string and
+        vice-versa). Many type modifications can be performed in one method
+        call. Is is an asynchronous call, thus a wait method must be also
+        called to guarantee a synchronization barrier.
+
+        dataset_name: Represents the dataset name.
+        types: Represents a map, where the pair key:value is a field:type
+
+        return: A JSON object with error or warning messages or a correct
+        datatype result.
+        """
+        url_request = self.__service_url
+        body_request = {
+            self.__INPUT_NAME: dataset_name,
+            self.__TYPES: types
+        }
+
+        response = requests.patch(url=url_request, json=body_request)
+        return self.__response_treat.treatment(response, pretty_response)
+
+    def search_all_datatype(self, pretty_response: bool = False) \
+            -> Union[dict, str]:
+        """
+        description: This method retrieves all datatype metadata, i.e., it does
+        not retrieve the datatype content.
+
+        pretty_response: If true it returns a string, otherwise a dictionary.
+
+        return: All predict metadata stored in Learning Orchestra or an empty
+        result.
+        """
+        response = self.__entity_reader.read_all_instances_from_entity()
+        return self.__response_treat.treatment(response, pretty_response)
+
+    def delete_datatype(self, name: str, pretty_response=False) \
+            -> Union[dict, str]:
+        """
+        description: This method is responsible for deleting the datatype step.
+        This delete operation is asynchronous, so it does not lock the caller
+         until the deletion finished. Instead, it returns a JSON object with a
+         URL for a future use. The caller uses the URL for delete checks.
+
+        pretty_response: If true it returns a string, otherwise a dictionary.
+        name: Represents the datatype name.
+
+        return: JSON object with an error message, a warning message or a
+        correct delete message
+        """
+        request_url = f'{self.__service_url}/{name}'
+
+        response = requests.delete(request_url)
+        return self.__response_treat.treatment(response, pretty_response)
+
+    def search_datatype_content(self,
+                                name: str,
+                                query: dict = {},
+                                limit: int = 10,
+                                skip: int = 0,
+                                pretty_response: bool = False) \
+            -> Union[dict, str]:
+        """
+        description:  This method is responsible for retrieving all the datatype
+        tuples or registers, as well as the metadata content
+
+        pretty_response: If true it returns a string, otherwise a dictionary.
+        name: Is the name of the datatype object
+        query: Query to make in MongoDB(default: empty query)
+        limit: Number of rows to return in pagination(default: 10) (maximum is
+        set at 20 rows per request)
+        skip: Number of rows to skip in pagination(default: 0)
+
+        return: A page with some registers or tuples inside or an error if there
+        is no such datatype object. The current page is also returned to be
+        used in future content requests.
+        """
+        response = self.__entity_reader.read_entity_content(
+            name, query, limit, skip)
+
+        return self.__response_treat.treatment(response, pretty_response)
+
+    def wait(self, dataset_name: str, timeout: int = None) -> dict:
+        """
+           description: This method is responsible to create a synchronization
+           barrier for the update_dataset_type_async method, delete_datatype
+           method.
+
+           name: Represents the datatype name.
+           timeout: Represents the time in seconds to wait for a datatype to
+           finish its run.
+
+           return: JSON object with an error message, a warning message or a
+           correct datatype result
+        """
+        return self.__observer.wait(dataset_name, timeout)
+
+

Methods

+
+
+def delete_datatype(self, name: str, pretty_response=False) ‑> Union[dict, str] +
+
+

description: This method is responsible for deleting the datatype step. +This delete operation is asynchronous, so it does not lock the caller +until the deletion finished. Instead, it returns a JSON object with a +URL for a future use. The caller uses the URL for delete checks.

+

pretty_response: If true it returns a string, otherwise a dictionary. +name: Represents the datatype name.

+

return: JSON object with an error message, a warning message or a +correct delete message

+
+ +Expand source code + +
def delete_datatype(self, name: str, pretty_response=False) \
+        -> Union[dict, str]:
+    """
+    description: This method is responsible for deleting the datatype step.
+    This delete operation is asynchronous, so it does not lock the caller
+     until the deletion finished. Instead, it returns a JSON object with a
+     URL for a future use. The caller uses the URL for delete checks.
+
+    pretty_response: If true it returns a string, otherwise a dictionary.
+    name: Represents the datatype name.
+
+    return: JSON object with an error message, a warning message or a
+    correct delete message
+    """
+    request_url = f'{self.__service_url}/{name}'
+
+    response = requests.delete(request_url)
+    return self.__response_treat.treatment(response, pretty_response)
+
+
+
+def search_all_datatype(self, pretty_response: bool = False) ‑> Union[dict, str] +
+
+

description: This method retrieves all datatype metadata, i.e., it does +not retrieve the datatype content.

+

pretty_response: If true it returns a string, otherwise a dictionary.

+

return: All predict metadata stored in Learning Orchestra or an empty +result.

+
+ +Expand source code + +
def search_all_datatype(self, pretty_response: bool = False) \
+        -> Union[dict, str]:
+    """
+    description: This method retrieves all datatype metadata, i.e., it does
+    not retrieve the datatype content.
+
+    pretty_response: If true it returns a string, otherwise a dictionary.
+
+    return: All predict metadata stored in Learning Orchestra or an empty
+    result.
+    """
+    response = self.__entity_reader.read_all_instances_from_entity()
+    return self.__response_treat.treatment(response, pretty_response)
+
+
+
+def search_datatype_content(self, name: str, query: dict = {}, limit: int = 10, skip: int = 0, pretty_response: bool = False) ‑> Union[dict, str] +
+
+

description: +This method is responsible for retrieving all the datatype +tuples or registers, as well as the metadata content

+

pretty_response: If true it returns a string, otherwise a dictionary. +name: Is the name of the datatype object +query: Query to make in MongoDB(default: empty query) +limit: Number of rows to return in pagination(default: 10) (maximum is +set at 20 rows per request) +skip: Number of rows to skip in pagination(default: 0)

+

return: A page with some registers or tuples inside or an error if there +is no such datatype object. The current page is also returned to be +used in future content requests.

+
+ +Expand source code + +
def search_datatype_content(self,
+                            name: str,
+                            query: dict = {},
+                            limit: int = 10,
+                            skip: int = 0,
+                            pretty_response: bool = False) \
+        -> Union[dict, str]:
+    """
+    description:  This method is responsible for retrieving all the datatype
+    tuples or registers, as well as the metadata content
+
+    pretty_response: If true it returns a string, otherwise a dictionary.
+    name: Is the name of the datatype object
+    query: Query to make in MongoDB(default: empty query)
+    limit: Number of rows to return in pagination(default: 10) (maximum is
+    set at 20 rows per request)
+    skip: Number of rows to skip in pagination(default: 0)
+
+    return: A page with some registers or tuples inside or an error if there
+    is no such datatype object. The current page is also returned to be
+    used in future content requests.
+    """
+    response = self.__entity_reader.read_entity_content(
+        name, query, limit, skip)
+
+    return self.__response_treat.treatment(response, pretty_response)
+
+
+
+def update_dataset_type_async(self, dataset_name: str, types: dict, pretty_response: bool = False) ‑> Union[dict, str] +
+
+

description: Change dataset field types (from number to string and +vice-versa). Many type modifications can be performed in one method +call. Is is an asynchronous call, thus a wait method must be also +called to guarantee a synchronization barrier.

+

dataset_name: Represents the dataset name. +types: Represents a map, where the pair key:value is a field:type

+

return: A JSON object with error or warning messages or a correct +datatype result.

+
+ +Expand source code + +
def update_dataset_type_async(self,
+                              dataset_name: str,
+                              types: dict,
+                              pretty_response: bool = False) \
+        -> Union[dict, str]:
+    """
+    description: Change dataset field types (from number to string and
+    vice-versa). Many type modifications can be performed in one method
+    call. Is is an asynchronous call, thus a wait method must be also
+    called to guarantee a synchronization barrier.
+
+    dataset_name: Represents the dataset name.
+    types: Represents a map, where the pair key:value is a field:type
+
+    return: A JSON object with error or warning messages or a correct
+    datatype result.
+    """
+    url_request = self.__service_url
+    body_request = {
+        self.__INPUT_NAME: dataset_name,
+        self.__TYPES: types
+    }
+
+    response = requests.patch(url=url_request, json=body_request)
+    return self.__response_treat.treatment(response, pretty_response)
+
+
+
+def update_dataset_type_sync(self, dataset_name: str, types: dict, pretty_response: bool = False) ‑> Union[dict, str] +
+
+

description: Change dataset field types (from number to string and +vice-versa). Many type modifications can be performed in one method +call.

+

dataset_name: Represents the dataset name. +types: Represents a map, where the pair key:value is a field:type

+

return: A JSON object with error or warning messages or a correct +datatype result.

+
+ +Expand source code + +
def update_dataset_type_sync(self,
+                             dataset_name: str,
+                             types: dict,
+                             pretty_response: bool = False) \
+        -> Union[dict, str]:
+    """
+    description: Change dataset field types (from number to string and
+    vice-versa). Many type modifications can be performed in one method
+    call.
+
+    dataset_name: Represents the dataset name.
+    types: Represents a map, where the pair key:value is a field:type
+
+    return: A JSON object with error or warning messages or a correct
+    datatype result.
+    """
+    url_request = self.__service_url
+    body_request = {
+        self.__INPUT_NAME: dataset_name,
+        self.__TYPES: types
+    }
+
+    response = requests.patch(url=url_request, json=body_request)
+    self.__observer.wait(dataset_name)
+
+    return self.__response_treat.treatment(response, pretty_response)
+
+
+
+def wait(self, dataset_name: str, timeout: int = None) ‑> dict +
+
+

description: This method is responsible to create a synchronization +barrier for the update_dataset_type_async method, delete_datatype +method.

+

name: Represents the datatype name. +timeout: Represents the time in seconds to wait for a datatype to +finish its run.

+

return: JSON object with an error message, a warning message or a +correct datatype result

+
+ +Expand source code + +
def wait(self, dataset_name: str, timeout: int = None) -> dict:
+    """
+       description: This method is responsible to create a synchronization
+       barrier for the update_dataset_type_async method, delete_datatype
+       method.
+
+       name: Represents the datatype name.
+       timeout: Represents the time in seconds to wait for a datatype to
+       finish its run.
+
+       return: JSON object with an error message, a warning message or a
+       correct datatype result
+    """
+    return self.__observer.wait(dataset_name, timeout)
+
+
+
+
+
+
+
+ +
+ + + \ No newline at end of file diff --git a/html/learning_orchestra_client/transform/index.html b/html/learning_orchestra_client/transform/index.html new file mode 100644 index 0000000..7c1ef8d --- /dev/null +++ b/html/learning_orchestra_client/transform/index.html @@ -0,0 +1,80 @@ + + + + + + +learning_orchestra_client.transform API documentation + + + + + + + + + + + +
+ + +
+ + + \ No newline at end of file diff --git a/html/learning_orchestra_client/transform/projection.html b/html/learning_orchestra_client/transform/projection.html new file mode 100644 index 0000000..7db5588 --- /dev/null +++ b/html/learning_orchestra_client/transform/projection.html @@ -0,0 +1,657 @@ + + + + + + +learning_orchestra_client.transform.projection API documentation + + + + + + + + + + + +
+
+
+

Module learning_orchestra_client.transform.projection

+
+
+
+ +Expand source code + +
from learning_orchestra_client._util._response_treat import ResponseTreat
+from learning_orchestra_client.observe.observe import Observer
+from learning_orchestra_client._util._entity_reader import EntityReader
+import requests
+from typing import Union
+
+
+class TransformProjection:
+    __INPUT_NAME = "inputDatasetName"
+    __OUTPUT_NAME = "outputDatasetName"
+    __FIELDS = "names"
+
+    def __init__(self, cluster_ip: str):
+        self.__api_path = "/api/learningOrchestra/v1/transform/projection"
+        self.__service_url = f'{cluster_ip}{self.__api_path}'
+        self.__response_treat = ResponseTreat()
+        self.__cluster_ip = cluster_ip
+        self.__entity_reader = EntityReader(self.__service_url)
+        self.__observer = Observer(self.__cluster_ip)
+
+    def remove_dataset_attributes_sync(self,
+                                       dataset_name: str,
+                                       projection_name: str,
+                                       fields: list,
+                                       pretty_response: bool = False) \
+            -> Union[dict, str]:
+        """
+        description: This method removes a set of attributes of a dataset
+        synchronously, the caller waits until the projection is inserted into
+        the Learning Orchestra storage mechanism.
+
+        pretty_response: If returns true a string, otherwise a dictionary.
+        projection_name: Represents the projection name.
+        dataset_name: Represents the dataset name.
+        fields: Represents the set of attributes to be removed. This is list
+        with some attributes.
+
+        return: A JSON object with error or warning messages. In case of
+        success, it returns the projection metadata.
+        """
+
+        request_body = {
+            self.__INPUT_NAME: dataset_name,
+            self.__OUTPUT_NAME: projection_name,
+            self.__FIELDS: fields,
+        }
+        request_url = self.__service_url
+        response = requests.post(url=request_url, json=request_body)
+        self.__observer.wait(dataset_name)
+
+        return self.__response_treat.treatment(response, pretty_response)
+
+    def remove_dataset_attributes_async(self,
+                                        dataset_name: str,
+                                        projection_name: str,
+                                        fields: list,
+                                        pretty_response: bool = False) \
+            -> Union[dict, str]:
+        """
+        description: This method removes a set of attributes of a dataset
+        asynchronously; this way, the caller does not wait until the projection
+        is inserted into the Learning Orchestra storage mechanism. A wait
+        method call must occur to guarantee a synchronization barrier.
+
+        pretty_response: If returns true a string, otherwise a dictionary.
+        projection_name: Represents the projection name.
+        dataset_name: Represents the dataset name.
+        fields: Represents the set of attributes to be removed. This is list
+        with some attributes.
+
+        return: A JSON object with error or warning messages. In case of
+        success, it returns the projection URL to be obtained latter with a
+        wait method call.
+        """
+
+        request_body = {
+            self.__INPUT_NAME: dataset_name,
+            self.__OUTPUT_NAME: projection_name,
+            self.__FIELDS: fields,
+        }
+        request_url = self.__service_url
+        response = requests.post(url=request_url, json=request_body)
+
+        return self.__response_treat.treatment(response, pretty_response)
+
+    def search_all_projections(self, pretty_response: bool = False) \
+            -> Union[dict, str]:
+        """
+        description: This method retrieves all projection metadata, i.e., it
+        does not retrieve the projection content.
+
+        pretty_response: If true it returns a string, otherwise a dictionary.
+
+        return: A list with all projections metadata stored in Learning
+        Orchestra or an empty result.
+        """
+        response = self.__entity_reader.read_all_instances_from_entity()
+        return self.__response_treat.treatment(response, pretty_response)
+
+    def search_projection_content(self,
+                                  projection_name: str,
+                                  query: dict = {},
+                                  limit: int = 10,
+                                  skip: int = 0,
+                                  pretty_response: bool = False) \
+            -> Union[dict, str]:
+        """
+        description: This method is responsible for retrieving the projection
+        content.
+
+        pretty_response: If true it returns a string, otherwise a dictionary.
+        projection_name: Represents the projection name.
+        query: Query to make in MongoDB(default: empty query)
+        limit: Number of rows to return in pagination(default: 10) (maximum is
+        set at 20 rows per request)
+        skip: Number of rows to skip in pagination(default: 0)
+
+        return: A page with some tuples or registers inside or an error if there
+        is no such projection. The current page is also returned to be used in
+        future content requests.
+        """
+
+        response = self.__entity_reader.read_entity_content(
+            projection_name, query, limit, skip)
+
+        return self.__response_treat.treatment(response, pretty_response)
+
+    def delete_projection(self, projection_name: str,
+                          pretty_response: bool = False) \
+            -> Union[dict, str]:
+        """
+        description: This method is responsible for deleting a projection.
+        The delete operation is always asynchronous and performed in background.
+
+
+        pretty_response: If true it returns a string, otherwise a dictionary.
+        projection_name: Represents the projection name.
+
+        return: JSON object with an error message, a warning message or a
+        correct delete message
+        """
+        cluster_url_projection = f'{self.__service_url}/{projection_name}'
+
+        response = requests.delete(cluster_url_projection)
+        response.raise_for_status()
+
+        return self.__response_treat.treatment(response, pretty_response)
+
+    def wait(self, projection_name: str, timeout: int = None) -> dict:
+        """
+           description: This method is responsible to create a synchronization
+           barrier for the remove_dataset_attributes_async method,
+           delete_projection method.
+
+           name: Represents the projection name.
+           timeout: Represents the time in seconds to wait for a projection to
+           finish its run.
+
+           return: JSON object with an error message, a warning message or a
+           correct projection result
+        """
+        return self.__observer.wait(projection_name, timeout)
+
+
+
+
+
+
+
+
+
+

Classes

+
+
+class TransformProjection +(cluster_ip: str) +
+
+
+
+ +Expand source code + +
class TransformProjection:
+    __INPUT_NAME = "inputDatasetName"
+    __OUTPUT_NAME = "outputDatasetName"
+    __FIELDS = "names"
+
+    def __init__(self, cluster_ip: str):
+        self.__api_path = "/api/learningOrchestra/v1/transform/projection"
+        self.__service_url = f'{cluster_ip}{self.__api_path}'
+        self.__response_treat = ResponseTreat()
+        self.__cluster_ip = cluster_ip
+        self.__entity_reader = EntityReader(self.__service_url)
+        self.__observer = Observer(self.__cluster_ip)
+
+    def remove_dataset_attributes_sync(self,
+                                       dataset_name: str,
+                                       projection_name: str,
+                                       fields: list,
+                                       pretty_response: bool = False) \
+            -> Union[dict, str]:
+        """
+        description: This method removes a set of attributes of a dataset
+        synchronously, the caller waits until the projection is inserted into
+        the Learning Orchestra storage mechanism.
+
+        pretty_response: If returns true a string, otherwise a dictionary.
+        projection_name: Represents the projection name.
+        dataset_name: Represents the dataset name.
+        fields: Represents the set of attributes to be removed. This is list
+        with some attributes.
+
+        return: A JSON object with error or warning messages. In case of
+        success, it returns the projection metadata.
+        """
+
+        request_body = {
+            self.__INPUT_NAME: dataset_name,
+            self.__OUTPUT_NAME: projection_name,
+            self.__FIELDS: fields,
+        }
+        request_url = self.__service_url
+        response = requests.post(url=request_url, json=request_body)
+        self.__observer.wait(dataset_name)
+
+        return self.__response_treat.treatment(response, pretty_response)
+
+    def remove_dataset_attributes_async(self,
+                                        dataset_name: str,
+                                        projection_name: str,
+                                        fields: list,
+                                        pretty_response: bool = False) \
+            -> Union[dict, str]:
+        """
+        description: This method removes a set of attributes of a dataset
+        asynchronously; this way, the caller does not wait until the projection
+        is inserted into the Learning Orchestra storage mechanism. A wait
+        method call must occur to guarantee a synchronization barrier.
+
+        pretty_response: If returns true a string, otherwise a dictionary.
+        projection_name: Represents the projection name.
+        dataset_name: Represents the dataset name.
+        fields: Represents the set of attributes to be removed. This is list
+        with some attributes.
+
+        return: A JSON object with error or warning messages. In case of
+        success, it returns the projection URL to be obtained latter with a
+        wait method call.
+        """
+
+        request_body = {
+            self.__INPUT_NAME: dataset_name,
+            self.__OUTPUT_NAME: projection_name,
+            self.__FIELDS: fields,
+        }
+        request_url = self.__service_url
+        response = requests.post(url=request_url, json=request_body)
+
+        return self.__response_treat.treatment(response, pretty_response)
+
+    def search_all_projections(self, pretty_response: bool = False) \
+            -> Union[dict, str]:
+        """
+        description: This method retrieves all projection metadata, i.e., it
+        does not retrieve the projection content.
+
+        pretty_response: If true it returns a string, otherwise a dictionary.
+
+        return: A list with all projections metadata stored in Learning
+        Orchestra or an empty result.
+        """
+        response = self.__entity_reader.read_all_instances_from_entity()
+        return self.__response_treat.treatment(response, pretty_response)
+
+    def search_projection_content(self,
+                                  projection_name: str,
+                                  query: dict = {},
+                                  limit: int = 10,
+                                  skip: int = 0,
+                                  pretty_response: bool = False) \
+            -> Union[dict, str]:
+        """
+        description: This method is responsible for retrieving the projection
+        content.
+
+        pretty_response: If true it returns a string, otherwise a dictionary.
+        projection_name: Represents the projection name.
+        query: Query to make in MongoDB(default: empty query)
+        limit: Number of rows to return in pagination(default: 10) (maximum is
+        set at 20 rows per request)
+        skip: Number of rows to skip in pagination(default: 0)
+
+        return: A page with some tuples or registers inside or an error if there
+        is no such projection. The current page is also returned to be used in
+        future content requests.
+        """
+
+        response = self.__entity_reader.read_entity_content(
+            projection_name, query, limit, skip)
+
+        return self.__response_treat.treatment(response, pretty_response)
+
+    def delete_projection(self, projection_name: str,
+                          pretty_response: bool = False) \
+            -> Union[dict, str]:
+        """
+        description: This method is responsible for deleting a projection.
+        The delete operation is always asynchronous and performed in background.
+
+
+        pretty_response: If true it returns a string, otherwise a dictionary.
+        projection_name: Represents the projection name.
+
+        return: JSON object with an error message, a warning message or a
+        correct delete message
+        """
+        cluster_url_projection = f'{self.__service_url}/{projection_name}'
+
+        response = requests.delete(cluster_url_projection)
+        response.raise_for_status()
+
+        return self.__response_treat.treatment(response, pretty_response)
+
+    def wait(self, projection_name: str, timeout: int = None) -> dict:
+        """
+           description: This method is responsible to create a synchronization
+           barrier for the remove_dataset_attributes_async method,
+           delete_projection method.
+
+           name: Represents the projection name.
+           timeout: Represents the time in seconds to wait for a projection to
+           finish its run.
+
+           return: JSON object with an error message, a warning message or a
+           correct projection result
+        """
+        return self.__observer.wait(projection_name, timeout)
+
+

Methods

+
+
+def delete_projection(self, projection_name: str, pretty_response: bool = False) ‑> Union[dict, str] +
+
+

description: This method is responsible for deleting a projection. +The delete operation is always asynchronous and performed in background.

+

pretty_response: If true it returns a string, otherwise a dictionary. +projection_name: Represents the projection name.

+

return: JSON object with an error message, a warning message or a +correct delete message

+
+ +Expand source code + +
def delete_projection(self, projection_name: str,
+                      pretty_response: bool = False) \
+        -> Union[dict, str]:
+    """
+    description: This method is responsible for deleting a projection.
+    The delete operation is always asynchronous and performed in background.
+
+
+    pretty_response: If true it returns a string, otherwise a dictionary.
+    projection_name: Represents the projection name.
+
+    return: JSON object with an error message, a warning message or a
+    correct delete message
+    """
+    cluster_url_projection = f'{self.__service_url}/{projection_name}'
+
+    response = requests.delete(cluster_url_projection)
+    response.raise_for_status()
+
+    return self.__response_treat.treatment(response, pretty_response)
+
+
+
+def remove_dataset_attributes_async(self, dataset_name: str, projection_name: str, fields: list, pretty_response: bool = False) ‑> Union[dict, str] +
+
+

description: This method removes a set of attributes of a dataset +asynchronously; this way, the caller does not wait until the projection +is inserted into the Learning Orchestra storage mechanism. A wait +method call must occur to guarantee a synchronization barrier.

+

pretty_response: If returns true a string, otherwise a dictionary. +projection_name: Represents the projection name. +dataset_name: Represents the dataset name. +fields: Represents the set of attributes to be removed. This is list +with some attributes.

+

return: A JSON object with error or warning messages. In case of +success, it returns the projection URL to be obtained latter with a +wait method call.

+
+ +Expand source code + +
def remove_dataset_attributes_async(self,
+                                    dataset_name: str,
+                                    projection_name: str,
+                                    fields: list,
+                                    pretty_response: bool = False) \
+        -> Union[dict, str]:
+    """
+    description: This method removes a set of attributes of a dataset
+    asynchronously; this way, the caller does not wait until the projection
+    is inserted into the Learning Orchestra storage mechanism. A wait
+    method call must occur to guarantee a synchronization barrier.
+
+    pretty_response: If returns true a string, otherwise a dictionary.
+    projection_name: Represents the projection name.
+    dataset_name: Represents the dataset name.
+    fields: Represents the set of attributes to be removed. This is list
+    with some attributes.
+
+    return: A JSON object with error or warning messages. In case of
+    success, it returns the projection URL to be obtained latter with a
+    wait method call.
+    """
+
+    request_body = {
+        self.__INPUT_NAME: dataset_name,
+        self.__OUTPUT_NAME: projection_name,
+        self.__FIELDS: fields,
+    }
+    request_url = self.__service_url
+    response = requests.post(url=request_url, json=request_body)
+
+    return self.__response_treat.treatment(response, pretty_response)
+
+
+
+def remove_dataset_attributes_sync(self, dataset_name: str, projection_name: str, fields: list, pretty_response: bool = False) ‑> Union[dict, str] +
+
+

description: This method removes a set of attributes of a dataset +synchronously, the caller waits until the projection is inserted into +the Learning Orchestra storage mechanism.

+

pretty_response: If returns true a string, otherwise a dictionary. +projection_name: Represents the projection name. +dataset_name: Represents the dataset name. +fields: Represents the set of attributes to be removed. This is list +with some attributes.

+

return: A JSON object with error or warning messages. In case of +success, it returns the projection metadata.

+
+ +Expand source code + +
def remove_dataset_attributes_sync(self,
+                                   dataset_name: str,
+                                   projection_name: str,
+                                   fields: list,
+                                   pretty_response: bool = False) \
+        -> Union[dict, str]:
+    """
+    description: This method removes a set of attributes of a dataset
+    synchronously, the caller waits until the projection is inserted into
+    the Learning Orchestra storage mechanism.
+
+    pretty_response: If returns true a string, otherwise a dictionary.
+    projection_name: Represents the projection name.
+    dataset_name: Represents the dataset name.
+    fields: Represents the set of attributes to be removed. This is list
+    with some attributes.
+
+    return: A JSON object with error or warning messages. In case of
+    success, it returns the projection metadata.
+    """
+
+    request_body = {
+        self.__INPUT_NAME: dataset_name,
+        self.__OUTPUT_NAME: projection_name,
+        self.__FIELDS: fields,
+    }
+    request_url = self.__service_url
+    response = requests.post(url=request_url, json=request_body)
+    self.__observer.wait(dataset_name)
+
+    return self.__response_treat.treatment(response, pretty_response)
+
+
+
+def search_all_projections(self, pretty_response: bool = False) ‑> Union[dict, str] +
+
+

description: This method retrieves all projection metadata, i.e., it +does not retrieve the projection content.

+

pretty_response: If true it returns a string, otherwise a dictionary.

+

return: A list with all projections metadata stored in Learning +Orchestra or an empty result.

+
+ +Expand source code + +
def search_all_projections(self, pretty_response: bool = False) \
+        -> Union[dict, str]:
+    """
+    description: This method retrieves all projection metadata, i.e., it
+    does not retrieve the projection content.
+
+    pretty_response: If true it returns a string, otherwise a dictionary.
+
+    return: A list with all projections metadata stored in Learning
+    Orchestra or an empty result.
+    """
+    response = self.__entity_reader.read_all_instances_from_entity()
+    return self.__response_treat.treatment(response, pretty_response)
+
+
+
+def search_projection_content(self, projection_name: str, query: dict = {}, limit: int = 10, skip: int = 0, pretty_response: bool = False) ‑> Union[dict, str] +
+
+

description: This method is responsible for retrieving the projection +content.

+

pretty_response: If true it returns a string, otherwise a dictionary. +projection_name: Represents the projection name. +query: Query to make in MongoDB(default: empty query) +limit: Number of rows to return in pagination(default: 10) (maximum is +set at 20 rows per request) +skip: Number of rows to skip in pagination(default: 0)

+

return: A page with some tuples or registers inside or an error if there +is no such projection. The current page is also returned to be used in +future content requests.

+
+ +Expand source code + +
def search_projection_content(self,
+                              projection_name: str,
+                              query: dict = {},
+                              limit: int = 10,
+                              skip: int = 0,
+                              pretty_response: bool = False) \
+        -> Union[dict, str]:
+    """
+    description: This method is responsible for retrieving the projection
+    content.
+
+    pretty_response: If true it returns a string, otherwise a dictionary.
+    projection_name: Represents the projection name.
+    query: Query to make in MongoDB(default: empty query)
+    limit: Number of rows to return in pagination(default: 10) (maximum is
+    set at 20 rows per request)
+    skip: Number of rows to skip in pagination(default: 0)
+
+    return: A page with some tuples or registers inside or an error if there
+    is no such projection. The current page is also returned to be used in
+    future content requests.
+    """
+
+    response = self.__entity_reader.read_entity_content(
+        projection_name, query, limit, skip)
+
+    return self.__response_treat.treatment(response, pretty_response)
+
+
+
+def wait(self, projection_name: str, timeout: int = None) ‑> dict +
+
+

description: This method is responsible to create a synchronization +barrier for the remove_dataset_attributes_async method, +delete_projection method.

+

name: Represents the projection name. +timeout: Represents the time in seconds to wait for a projection to +finish its run.

+

return: JSON object with an error message, a warning message or a +correct projection result

+
+ +Expand source code + +
def wait(self, projection_name: str, timeout: int = None) -> dict:
+    """
+       description: This method is responsible to create a synchronization
+       barrier for the remove_dataset_attributes_async method,
+       delete_projection method.
+
+       name: Represents the projection name.
+       timeout: Represents the time in seconds to wait for a projection to
+       finish its run.
+
+       return: JSON object with an error message, a warning message or a
+       correct projection result
+    """
+    return self.__observer.wait(projection_name, timeout)
+
+
+
+
+
+
+
+ +
+ + + \ No newline at end of file diff --git a/html/learning_orchestra_client/transform/scikitlearn.html b/html/learning_orchestra_client/transform/scikitlearn.html new file mode 100644 index 0000000..5905620 --- /dev/null +++ b/html/learning_orchestra_client/transform/scikitlearn.html @@ -0,0 +1,97 @@ + + + + + + +learning_orchestra_client.transform.scikitlearn API documentation + + + + + + + + + + + +
+
+
+

Module learning_orchestra_client.transform.scikitlearn

+
+
+
+ +Expand source code + +
from ._transform import Transform
+
+
+class TransformScikitLearn(Transform):
+    def __init__(self, cluster_ip: str):
+        self.__api_path = "/api/learningOrchestra/v1/transform/scikitlearn"
+        self.__cluster_ip = cluster_ip
+        super().__init__(cluster_ip, self.__api_path)
+
+
+
+
+
+
+
+
+
+

Classes

+
+
+class TransformScikitLearn +(cluster_ip: str) +
+
+
+
+ +Expand source code + +
class TransformScikitLearn(Transform):
+    def __init__(self, cluster_ip: str):
+        self.__api_path = "/api/learningOrchestra/v1/transform/scikitlearn"
+        self.__cluster_ip = cluster_ip
+        super().__init__(cluster_ip, self.__api_path)
+
+

Ancestors

+
    +
  • learning_orchestra_client.transform._transform.Transform
    • +
+
+
+
+
+ +
+ + + \ No newline at end of file diff --git a/html/learning_orchestra_client/transform/tensorflow.html b/html/learning_orchestra_client/transform/tensorflow.html new file mode 100644 index 0000000..a825eec --- /dev/null +++ b/html/learning_orchestra_client/transform/tensorflow.html @@ -0,0 +1,97 @@ + + + + + + +learning_orchestra_client.transform.tensorflow API documentation + + + + + + + + + + + +
+
+
+

Module learning_orchestra_client.transform.tensorflow

+
+
+
+ +Expand source code + +
from ._transform import Transform
+
+
+class TransformTensorflow(Transform):
+    def __init__(self, cluster_ip: str):
+        self.__api_path = "/api/learningOrchestra/v1/transform/tensorflow"
+        self.__cluster_ip = cluster_ip
+        super().__init__(cluster_ip, self.__api_path)
+
+
+
+
+
+
+
+
+
+

Classes

+
+
+class TransformTensorflow +(cluster_ip: str) +
+
+
+
+ +Expand source code + +
class TransformTensorflow(Transform):
+    def __init__(self, cluster_ip: str):
+        self.__api_path = "/api/learningOrchestra/v1/transform/tensorflow"
+        self.__cluster_ip = cluster_ip
+        super().__init__(cluster_ip, self.__api_path)
+
+

Ancestors

+
    +
  • learning_orchestra_client.transform._transform.Transform
    • +
+
+
+
+
+ +
+ + + \ No newline at end of file