diff --git a/aztk/models/plugins/internal/plugin_manager.py b/aztk/models/plugins/internal/plugin_manager.py index b7c0797d..814c2e44 100644 --- a/aztk/models/plugins/internal/plugin_manager.py +++ b/aztk/models/plugins/internal/plugin_manager.py @@ -26,6 +26,9 @@ class PluginManager: tensorflow_on_spark=plugins.TensorflowOnSparkPlugin, openblas=plugins.OpenBLASPlugin, nvblas=plugins.NvBLASPlugin, + apt_get=plugins.AptGetPlugin, + pip_install=plugins.PipPlugin, + conda_install=plugins.CondaPlugin ) def __init__(self): diff --git a/aztk/models/plugins/internal/plugin_reference.py b/aztk/models/plugins/internal/plugin_reference.py index 7451052d..2c8c1fd8 100644 --- a/aztk/models/plugins/internal/plugin_reference.py +++ b/aztk/models/plugins/internal/plugin_reference.py @@ -25,7 +25,7 @@ class PluginReference(Model): script = fields.String(default=None) target = fields.Enum(PluginTarget, default=None) target_role = fields.Enum(PluginTargetRole, default=None) - args = fields.String(default=None) + args = fields.Field(default=None) def get_plugin(self) -> PluginConfiguration: self.validate() diff --git a/aztk/models/plugins/plugin_configuration.py b/aztk/models/plugins/plugin_configuration.py index d46ab29b..54561f90 100644 --- a/aztk/models/plugins/plugin_configuration.py +++ b/aztk/models/plugins/plugin_configuration.py @@ -51,7 +51,7 @@ class PluginConfiguration(Model): runOn: Where the plugin should run execute: Path to the file to execute(This must match the target of one of the files) files: List of files to upload - args: List of argumenets to pass to the executing script + args: List of arguments to pass to the executing script env: Dict of environment variables to pass to the script """ name = fields.String() diff --git a/aztk/spark/models/plugins/__init__.py b/aztk/spark/models/plugins/__init__.py index 9bf81f09..82539ad2 100644 --- a/aztk/spark/models/plugins/__init__.py +++ b/aztk/spark/models/plugins/__init__.py @@ -8,3 +8,6 @@ from .tensorflow_on_spark import TensorflowOnSparkPlugin from .openblas import OpenBLASPlugin from .nvblas import NvBLASPlugin +from .install.conda import CondaPlugin +from .install.apt_get import AptGetPlugin +from .install.pip import PipPlugin diff --git a/aztk/spark/models/plugins/install/__init__.py b/aztk/spark/models/plugins/install/__init__.py new file mode 100644 index 00000000..2ec26f31 --- /dev/null +++ b/aztk/spark/models/plugins/install/__init__.py @@ -0,0 +1 @@ +from .configuration import * diff --git a/aztk/spark/models/plugins/install/apt_get/__init__.py b/aztk/spark/models/plugins/install/apt_get/__init__.py new file mode 100644 index 00000000..2ec26f31 --- /dev/null +++ b/aztk/spark/models/plugins/install/apt_get/__init__.py @@ -0,0 +1 @@ +from .configuration import * diff --git a/aztk/spark/models/plugins/install/apt_get/configuration.py b/aztk/spark/models/plugins/install/apt_get/configuration.py new file mode 100644 index 00000000..fc1c16f3 --- /dev/null +++ b/aztk/spark/models/plugins/install/apt_get/configuration.py @@ -0,0 +1,14 @@ +import os +from aztk.models.plugins.plugin_configuration import PluginConfiguration, PluginPort, PluginTargetRole +from aztk.models.plugins.plugin_file import PluginFile +from aztk.spark.models.plugins.install import InstallPlugin +from aztk.utils import constants + +dir_path = os.path.dirname(os.path.realpath(__file__)) + +def AptGetPlugin(packages=None): + return InstallPlugin( + name="apt-get", + command="apt-get update && apt-get install -y", + packages=packages + ) diff --git a/aztk/spark/models/plugins/install/conda/__init__.py b/aztk/spark/models/plugins/install/conda/__init__.py new file mode 100644 index 00000000..2ec26f31 --- /dev/null +++ b/aztk/spark/models/plugins/install/conda/__init__.py @@ -0,0 +1 @@ +from .configuration import * diff --git a/aztk/spark/models/plugins/install/conda/configuration.py b/aztk/spark/models/plugins/install/conda/configuration.py new file mode 100644 index 00000000..b10bdb31 --- /dev/null +++ b/aztk/spark/models/plugins/install/conda/configuration.py @@ -0,0 +1,14 @@ +import os +from aztk.models.plugins.plugin_configuration import PluginConfiguration, PluginPort, PluginTargetRole +from aztk.models.plugins.plugin_file import PluginFile +from aztk.spark.models.plugins.install import InstallPlugin +from aztk.utils import constants + +dir_path = os.path.dirname(os.path.realpath(__file__)) + +def CondaPlugin(packages=None): + return InstallPlugin( + name="conda", + command="conda install -y", + packages=packages + ) diff --git a/aztk/spark/models/plugins/install/configuration.py b/aztk/spark/models/plugins/install/configuration.py new file mode 100644 index 00000000..a57b63e8 --- /dev/null +++ b/aztk/spark/models/plugins/install/configuration.py @@ -0,0 +1,18 @@ +import os +from aztk.models.plugins.plugin_configuration import PluginConfiguration, PluginPort, PluginTargetRole +from aztk.models.plugins.plugin_file import PluginFile +from aztk.utils import constants + +dir_path = os.path.dirname(os.path.realpath(__file__)) + +def InstallPlugin(name, command, packages=None): + return PluginConfiguration( + name=name, + target_role=PluginTargetRole.All, + execute="install.sh", + files=[ + PluginFile("install.sh", os.path.join(dir_path, "install.sh")) + ], + args=packages, + env=dict(COMMAND=command) + ) diff --git a/aztk/spark/models/plugins/install/install.sh b/aztk/spark/models/plugins/install/install.sh new file mode 100644 index 00000000..85765ddb --- /dev/null +++ b/aztk/spark/models/plugins/install/install.sh @@ -0,0 +1,3 @@ +#!/bin/bash + +eval $COMMAND $@ diff --git a/aztk/spark/models/plugins/install/pip/__init__.py b/aztk/spark/models/plugins/install/pip/__init__.py new file mode 100644 index 00000000..2ec26f31 --- /dev/null +++ b/aztk/spark/models/plugins/install/pip/__init__.py @@ -0,0 +1 @@ +from .configuration import * diff --git a/aztk/spark/models/plugins/install/pip/configuration.py b/aztk/spark/models/plugins/install/pip/configuration.py new file mode 100644 index 00000000..ac0531ee --- /dev/null +++ b/aztk/spark/models/plugins/install/pip/configuration.py @@ -0,0 +1,14 @@ +import os +from aztk.models.plugins.plugin_configuration import PluginConfiguration, PluginPort, PluginTargetRole +from aztk.models.plugins.plugin_file import PluginFile +from aztk.spark.models.plugins.install import InstallPlugin +from aztk.utils import constants + +dir_path = os.path.dirname(os.path.realpath(__file__)) + +def PipPlugin(packages=None): + return InstallPlugin( + name="pip", + command="pip install", + packages=packages + ) diff --git a/aztk_cli/config/cluster.yaml b/aztk_cli/config/cluster.yaml index 5ad3d1b9..ac17a698 100644 --- a/aztk_cli/config/cluster.yaml +++ b/aztk_cli/config/cluster.yaml @@ -47,6 +47,21 @@ plugins: # - name: tensorflow_on_spark # - name: openblas # - name: nvblas + # - name: apt_get + # args: + # packages: + # - 'vim' + # - 'htop' + # - name: pip_install + # args: + # packages: + # - 'numpy==1.14.2' + # - 'requests' + # - name: conda_install + # args: + # packages: + # - 'numpy=1.14.2' + # - 'requests' # Allow master node to also be a worker (Default: true) # worker_on_master: true