Conda, Apt-Get and Pip Install Plugins (#594)

* Added install plugins

* Moved packages to directory

* Removed channel from conda install

* changed default to none

* Added line

* fixed template

* Fixed naming of apt get

aztk/models/plugins/internal/plugin_manager.py

@@ -26,6 +26,9 @@ class PluginManager:
         tensorflow_on_spark=plugins.TensorflowOnSparkPlugin,
         openblas=plugins.OpenBLASPlugin,
         nvblas=plugins.NvBLASPlugin,
+        apt_get=plugins.AptGetPlugin,
+        pip_install=plugins.PipPlugin,
+        conda_install=plugins.CondaPlugin
     )
 
     def __init__(self):

aztk/models/plugins/internal/plugin_reference.py

@@ -25,7 +25,7 @@ class PluginReference(Model):
     script = fields.String(default=None)
     target = fields.Enum(PluginTarget, default=None)
     target_role = fields.Enum(PluginTargetRole, default=None)
-    args = fields.String(default=None)
+    args = fields.Field(default=None)
 
     def get_plugin(self) -> PluginConfiguration:
         self.validate()

aztk/models/plugins/plugin_configuration.py

@@ -51,7 +51,7 @@ class PluginConfiguration(Model):
         runOn: Where the plugin should run
         execute: Path to the file to execute(This must match the target of one of the files)
         files: List of files to upload
-        args: List of argumenets to pass to the executing script
+        args: List of arguments to pass to the executing script
         env: Dict of environment variables to pass to the script
     """
     name = fields.String()

aztk/spark/models/plugins/__init__.py

@@ -8,3 +8,6 @@
 from .tensorflow_on_spark import TensorflowOnSparkPlugin
 from .openblas import OpenBLASPlugin
 from .nvblas import NvBLASPlugin
+from .install.conda import CondaPlugin
+from .install.apt_get import AptGetPlugin
+from .install.pip import PipPlugin

aztk/spark/models/plugins/install/__init__.py

@@ -0,0 +1 @@
+from .configuration import *

aztk/spark/models/plugins/install/apt_get/__init__.py

@@ -0,0 +1 @@
+from .configuration import *

aztk/spark/models/plugins/install/apt_get/configuration.py

@@ -0,0 +1,14 @@
+import os
+from aztk.models.plugins.plugin_configuration import PluginConfiguration, PluginPort, PluginTargetRole
+from aztk.models.plugins.plugin_file import PluginFile
+from aztk.spark.models.plugins.install import InstallPlugin
+from aztk.utils import constants
+
+dir_path = os.path.dirname(os.path.realpath(__file__))
+
+def AptGetPlugin(packages=None):
+    return InstallPlugin(
+        name="apt-get",
+        command="apt-get update && apt-get install -y",
+        packages=packages
+    )

aztk/spark/models/plugins/install/conda/__init__.py

@@ -0,0 +1 @@
+from .configuration import *

aztk/spark/models/plugins/install/conda/configuration.py

@@ -0,0 +1,14 @@
+import os
+from aztk.models.plugins.plugin_configuration import PluginConfiguration, PluginPort, PluginTargetRole
+from aztk.models.plugins.plugin_file import PluginFile
+from aztk.spark.models.plugins.install import InstallPlugin
+from aztk.utils import constants
+
+dir_path = os.path.dirname(os.path.realpath(__file__))
+
+def CondaPlugin(packages=None):
+    return InstallPlugin(
+        name="conda",
+        command="conda install -y",
+        packages=packages
+    )

aztk/spark/models/plugins/install/configuration.py

@@ -0,0 +1,18 @@
+import os
+from aztk.models.plugins.plugin_configuration import PluginConfiguration, PluginPort, PluginTargetRole
+from aztk.models.plugins.plugin_file import PluginFile
+from aztk.utils import constants
+
+dir_path = os.path.dirname(os.path.realpath(__file__))
+
+def InstallPlugin(name, command, packages=None):
+    return PluginConfiguration(
+        name=name,
+        target_role=PluginTargetRole.All,
+        execute="install.sh",
+        files=[
+            PluginFile("install.sh", os.path.join(dir_path, "install.sh"))
+        ],
+        args=packages,
+        env=dict(COMMAND=command)
+    )

aztk/spark/models/plugins/install/install.sh

@@ -0,0 +1,3 @@
+#!/bin/bash
+
+eval $COMMAND $@

aztk/spark/models/plugins/install/pip/__init__.py

@@ -0,0 +1 @@
+from .configuration import *

aztk/spark/models/plugins/install/pip/configuration.py

@@ -0,0 +1,14 @@
+import os
+from aztk.models.plugins.plugin_configuration import PluginConfiguration, PluginPort, PluginTargetRole
+from aztk.models.plugins.plugin_file import PluginFile
+from aztk.spark.models.plugins.install import InstallPlugin
+from aztk.utils import constants
+
+dir_path = os.path.dirname(os.path.realpath(__file__))
+
+def PipPlugin(packages=None):
+    return InstallPlugin(
+        name="pip",
+        command="pip install",
+        packages=packages
+    )

aztk_cli/config/cluster.yaml

@@ -47,6 +47,21 @@ plugins:
   # - name: tensorflow_on_spark
   # - name: openblas
   # - name: nvblas
+  # - name: apt_get
+  #   args:
+  #     packages:
+  #       - 'vim'
+  #       - 'htop'
+  # - name: pip_install
+  #   args:
+  #     packages:
+  #       - 'numpy==1.14.2'
+  #       - 'requests'
+  # - name: conda_install
+  #   args:
+  #     packages:
+  #       - 'numpy=1.14.2'
+  #       - 'requests'
 
 # Allow master node to also be a worker <true/false> (Default: true)
 # worker_on_master: true