Internal: Move node scripts under aztk and upload all aztk to cluster (…

…#433)

aztk/internal/cluster_data/node_data.py

@@ -13,7 +13,7 @@
 ROOT_PATH = constants.ROOT_PATH
 
 # Constants for node data
-NODE_SCRIPT_FOLDER = "node_scripts"
+NODE_SCRIPT_FOLDER = "aztk"
 CUSTOM_SCRIPT_FOLDER = "custom-scripts"
 CUSTOM_SCRIPT_METADATA_FILE = "custom-scripts.yaml"
 PLUGIN_FOLDER = "plugins"
@@ -36,7 +36,6 @@ def add_core(self):
         self._add_plugins()
         self._add_spark_configuration()
         self._add_user_conf()
-        self.add_file(os.path.join(constants.ROOT_PATH, 'aztk', 'utils', 'command_builder.py'), '', binary=False)
         return self
 
     def done(self):
@@ -63,7 +62,7 @@ def add_files(self, file_paths: List[str], zip_dir, binary: bool = True):
         for file in file_paths:
             self.add_file(file, zip_dir, binary)
 
-    def add_dir(self, path: str, exclude: List[str] = []):
+    def add_dir(self, path: str, dest: str = None, exclude: List[str] = []):
         """
             Zip all the files in the given directory into the zip file handler
         """
@@ -72,7 +71,7 @@ def add_dir(self, path: str, exclude: List[str] = []):
             for file in files:
                 if self._includeFile(file, exclude):
                     with io.open(os.path.join(base, file), 'r') as f:
-                        self.zipf.writestr(os.path.join(relative_folder, file), f.read().replace('\r\n', '\n'))
+                        self.zipf.writestr(os.path.join(dest, relative_folder, file), f.read().replace('\r\n', '\n'))
 
     def _add_custom_scripts(self):
         data = []
@@ -155,7 +154,7 @@ def _add_plugins(self):
         return zipf
 
     def _add_node_scripts(self):
-        self.add_dir(os.path.join(ROOT_PATH, NODE_SCRIPT_FOLDER), exclude=['*.pyc'])
+        self.add_dir(os.path.join(ROOT_PATH, NODE_SCRIPT_FOLDER), NODE_SCRIPT_FOLDER, exclude=['*.pyc*'])
 
     def _includeFile(self, filename: str, exclude: List[str] = []) -> bool:
         for pattern in exclude:

node_scripts/docker_main.sh → aztk/node_scripts/docker_main.sh

@@ -8,6 +8,7 @@ set -e
 # Setup custom scripts
 # --------------------
 custom_script_dir=$DOCKER_WORKING_DIR/custom-scripts
+aztk_dir=$DOCKER_WORKING_DIR/aztk
 
 # -----------------------
 # Preload jupyter samples
@@ -25,7 +26,9 @@ done
 # ----------------------------
 # use python v3.5.4 to run aztk software
 echo "Starting setup using Docker"
+
 $(pyenv root)/versions/$AZTK_PYTHON_VERSION/bin/pip install -r $(dirname $0)/requirements.txt
+export PYTHONPATH=$PYTHONPATH:$DOCKER_WORKING_DIR
 
 echo "Running main.py script"
 $(pyenv root)/versions/$AZTK_PYTHON_VERSION/bin/python $(dirname $0)/main.py install

node_scripts/job_submission.py → aztk/node_scripts/job_submission.py

@@ -6,7 +6,7 @@
 import azure.batch.models as batch_models
 import azure.storage.blob as blob
 import yaml
-from command_builder import CommandBuilder
+from aztk.utils.command_builder import CommandBuilder
 from core import config
 from install.pick_master import get_master_node_id
 

node_scripts/setup_node.sh → aztk/node_scripts/setup_node.sh

@@ -44,7 +44,7 @@ echo "Pulling $repo_name"
 
 # Unzip resource files and set permissions
 apt-get -y install unzip
-chmod 777 $AZ_BATCH_TASK_WORKING_DIR/docker_main.sh
+chmod 777 $AZ_BATCH_TASK_WORKING_DIR/aztk/node_scripts/docker_main.sh
 
 # Check docker is running
 docker info > /dev/null 2>&1
@@ -68,7 +68,7 @@ else
     done;
 
     # wait until container setup is complete
-    docker exec spark /bin/bash -c 'python $DOCKER_WORKING_DIR/wait_until_setup_complete.py'
+    docker exec spark /bin/bash -c 'python $DOCKER_WORKING_DIR/aztk/node_scripts/wait_until_setup_complete.py'
 
     # Setup symbolic link for the docker logs
     docker_log=$(docker inspect --format='{{.LogPath}}' $container_name)

aztk/spark/helpers/create_cluster.py

@@ -74,7 +74,7 @@ def __docker_run_cmd(docker_repo: str = None,
                 cmd.add_option('-p', '{0}:{1}'.format(port.internal, port.internal))       # Jupyter UI
 
     cmd.add_option('-d', docker_repo)
-    cmd.add_argument('/bin/bash /mnt/batch/tasks/startup/wd/docker_main.sh')
+    cmd.add_argument('/bin/bash /mnt/batch/tasks/startup/wd/aztk/node_scripts/docker_main.sh')
 
     return cmd.to_str()
 
@@ -162,8 +162,8 @@ def __cluster_install_cmd(zip_resource_file: batch_models.ResourceFile,
         'apt-get -y install unzip',
         'unzip $AZ_BATCH_TASK_WORKING_DIR/{0}'.format(
             zip_resource_file.file_path),
-        'chmod 777 $AZ_BATCH_TASK_WORKING_DIR/setup_node.sh',
-        '/bin/bash $AZ_BATCH_TASK_WORKING_DIR/setup_node.sh {0} {1} {2} "{3}"'.format(
+        'chmod 777 $AZ_BATCH_TASK_WORKING_DIR/aztk/node_scripts/setup_node.sh',
+        '/bin/bash $AZ_BATCH_TASK_WORKING_DIR/aztk/node_scripts/setup_node.sh {0} {1} {2} "{3}"'.format(
             constants.DOCKER_SPARK_CONTAINER_NAME,
             gpu_enabled,
             docker_repo,