Merge pull request aws#70 in VFS/bamboos from feature/VFS-1457-rename…

…-to-cluster-manager to develop

# By Michał Żmuda
# Via Michał Żmuda
* commit 'b605f933fc0c4c4d76a250e63b8afc6c4b423760':
  VFS-1457 facilitating acceptance testing
  VFS-1457 PR issues: refactoring starting cluster workers
  VFS-1457 fix ccm app name
  VFS-1457 initial cluster-manager renaming

docker/provider_ccm_up.py → docker/cluster_manager_up.py

@@ -4,18 +4,18 @@
 Copyright (C) 2015 ACK CYFRONET AGH
 This software is released under the MIT license cited in 'LICENSE.txt'
 
-A script to bring up a set of oneprovider ccm nodes. They can create separate
+A script to bring up a set of oneprovider cm nodes. They can create separate
 clusters.
 Run the script with -h flag to learn about script's running options.
 """
 
 from __future__ import print_function
 import json
 
-from environment import common, provider_ccm
+from environment import common, cluster_manager
 
 
-parser = common.standard_arg_parser('Bring up op_ccm nodes.')
+parser = common.standard_arg_parser('Bring up cluster_manager nodes.')
 parser.add_argument(
     '-l', '--logdir',
     action='store',
@@ -24,7 +24,7 @@
     dest='logdir')
 
 args = parser.parse_args()
-output = provider_ccm.up(args.image, args.bin, args.dns, args.uid,
+output = cluster_manager.up(args.image, args.bin, args.dns, args.uid,
                      args.config_path, args.logdir)
 
 print(json.dumps(output))

docker/cluster_up.py

@@ -12,10 +12,10 @@
 from __future__ import print_function
 import json
 import os
-from environment import common, cluster_worker, provider_ccm, dns
+from environment import common, cluster_worker, cluster_manager, dns
 
 parser = common.standard_arg_parser(
-    'Bring up bare cluster nodes (workers and ccms).')
+    'Bring up bare cluster nodes (workers and cms).')
 parser.add_argument(
     '-l', '--logdir',
     action='store',
@@ -29,17 +29,17 @@
     help='the path to cluster-worker repository (precompiled)',
     dest='bin_op_worker')
 parser.add_argument(
-    '-bccm', '--bin-ccm',
+    '-bcm', '--bin-cm',
     action='store',
-    default=os.getcwd() + '/op_ccm',
-    help='the path to op_ccm repository (precompiled)',
-    dest='bin_op_ccm')
+    default=os.getcwd() + '/cluster_manager',
+    help='the path to cluster_manager repository (precompiled)',
+    dest='bin_cluster_manager')
 
 # Prepare config
 args = parser.parse_args()
 config = common.parse_json_file(args.config_path)
 output = {
-    'op_ccm_nodes': [],
+    'cluster_manager_nodes': [],
     'cluster_worker_nodes': [],
 }
 uid = common.generate_uid()
@@ -48,9 +48,9 @@
 [dns_server], dns_output = dns.maybe_start('auto', uid)
 common.merge(output, dns_output)
 
-# Start ccms
-ccm_output = provider_ccm.up(args.image, args.bin_op_ccm, dns_server, uid, args.config_path, args.logdir)
-common.merge(output, ccm_output)
+# Start cms
+cm_output = cluster_manager.up(args.image, args.bin_cluster_manager, dns_server, uid, args.config_path, args.logdir)
+common.merge(output, cm_output)
 
 # Start workers
 worker_output = cluster_worker.up(args.image, args.bin_op_worker, dns_server, uid, args.config_path, args.logdir)

docker/ct_run.py

@@ -173,10 +173,10 @@
                         configs_to_change.extend(
                             data['provider_domains'][provider][
                                 'op_worker'].values())
-                    if 'op_ccm' in data['provider_domains'][provider]:
+                    if 'cluster_manager' in data['provider_domains'][provider]:
                         configs_to_change.extend(
                             data['provider_domains'][provider][
-                                'op_ccm'].values())
+                                'cluster_manager'].values())
             if 'globalregistry_domains' in data:
                 for globalregistry in data['globalregistry_domains']:
                     configs_to_change.extend(

docker/env_up.py

@@ -43,11 +43,11 @@
     dest='bin_cluster_worker')
 
 parser.add_argument(
-    '-bccm', '--bin-ccm',
+    '-bcm', '--bin-cm',
     action='store',
-    default=env.default('bin_op_ccm'),
-    help='the path to op_ccm repository (precompiled)',
-    dest='bin_op_ccm')
+    default=env.default('bin_cluster_manager'),
+    help='the path to cluster_manager repository (precompiled)',
+    dest='bin_cluster_manager')
 
 parser.add_argument(
     '-bg', '--bin-gr',
@@ -85,7 +85,7 @@
 args = parser.parse_args()
 
 output = env.up(args.config_path, image=args.image, bin_am=args.bin_am,
-       bin_gr=args.bin_gr, bin_op_ccm=args.bin_op_ccm,
+       bin_gr=args.bin_gr, bin_cluster_manager=args.bin_cluster_manager,
        bin_op_worker=args.bin_op_worker, bin_cluster_worker=args.bin_cluster_worker,
        bin_oc=args.bin_oc, logdir=args.logdir)
 

docker/environment/appmock.py

@@ -12,7 +12,7 @@
 import random
 import string
 
-from . import common, docker, dns, provider_ccm, worker, globalregistry
+from . import common, docker, dns, cluster_manager, worker, globalregistry
 
 APPMOCK_WAIT_FOR_NAGIOS_SECONDS = 60 * 2
 
@@ -48,7 +48,7 @@ def _tweak_config(config, appmock_node, appmock_instance, uid):
     # Node name depends on mocked app, if none is specified,
     # default appmock_erl_node_name will be used.
     node_name = {
-        'op_ccm': provider_ccm.ccm_erl_node_name(appmock_node,
+        'cluster_manager': cluster_manager.cm_erl_node_name(appmock_node,
                                                  appmock_instance, uid),
         'op_worker': worker.worker_erl_node_name(appmock_node,
                                                  appmock_instance,

docker/environment/provider_ccm.py → docker/environment/cluster_manager.py

@@ -2,7 +2,7 @@
 Copyright (C) 2015 ACK CYFRONET AGH
 This software is released under the MIT license cited in 'LICENSE.txt'
 
-Brings up a set of oneprovider ccm nodes. They can create separate clusters.
+Brings up a set of oneprovider cm nodes. They can create separate clusters.
 """
 
 from __future__ import print_function
@@ -14,32 +14,32 @@
 from . import common, docker, dns
 
 
-def ccm_hostname(node_name, op_instance, uid):
-    """Formats hostname for a docker hosting op_ccm.
+def cm_hostname(node_name, op_instance, uid):
+    """Formats hostname for a docker hosting cluster_manager.
     NOTE: Hostnames are also used as docker names!
     """
     return common.format_hostname([node_name, op_instance], uid)
 
 
-def ccm_erl_node_name(node_name, op_instance, uid):
-    """Formats erlang node name for a vm on op_ccm docker.
+def cm_erl_node_name(node_name, op_instance, uid):
+    """Formats erlang node name for a vm on cluster_manager docker.
     """
-    hostname = ccm_hostname(node_name, op_instance, uid)
-    return common.format_erl_node_name('ccm', hostname)
+    hostname = cm_hostname(node_name, op_instance, uid)
+    return common.format_erl_node_name('cm', hostname)
 
 
-def _tweak_config(config, ccm_node, op_instance, uid):
+def _tweak_config(config, cm_node, op_instance, uid):
     cfg = copy.deepcopy(config)
-    cfg['nodes'] = {'node': cfg['nodes'][ccm_node]}
+    cfg['nodes'] = {'node': cfg['nodes'][cm_node]}
 
     sys_config = cfg['nodes']['node']['sys.config']
-    sys_config['ccm_nodes'] = [ccm_erl_node_name(n, op_instance, uid)
-                               for n in sys_config['ccm_nodes']]
+    sys_config['cm_nodes'] = [cm_erl_node_name(n, op_instance, uid)
+                               for n in sys_config['cm_nodes']]
 
     if 'vm.args' not in cfg['nodes']['node']:
         cfg['nodes']['node']['vm.args'] = {}
     vm_args = cfg['nodes']['node']['vm.args']
-    vm_args['name'] = ccm_erl_node_name(ccm_node, op_instance, uid)
+    vm_args['name'] = cm_erl_node_name(cm_node, op_instance, uid)
 
     return cfg
 
@@ -58,9 +58,9 @@ def _node_up(image, bindir, config, dns_servers, logdir):
 EOF
 set -e
 escript bamboos/gen_dev/gen_dev.escript /tmp/gen_dev_args.json
-/root/bin/node/bin/op_ccm console'''
+/root/bin/node/bin/cluster_manager console'''
     command = command.format(
-        gen_dev_args=json.dumps({'op_ccm': config}),
+        gen_dev_args=json.dumps({'cluster_manager': config}),
         uid=os.geteuid(),
         gid=os.getegid())
 
@@ -84,7 +84,7 @@ def _node_up(image, bindir, config, dns_servers, logdir):
 
     return {
         'docker_ids': [container],
-        'op_ccm_nodes': [node_name]
+        'cluster_manager_nodes': [node_name]
     }
 
 
@@ -94,29 +94,29 @@ def _ready(container):
 
 def up(image, bindir, dns_server, uid, config_path, logdir=None, domains_name='provider_domains'):
     config = common.parse_json_file(config_path)
-    input_dir = config['dirs_config']['op_ccm']['input_dir']
+    input_dir = config['dirs_config']['cluster_manager']['input_dir']
     dns_servers, output = dns.maybe_start(dns_server, uid)
 
-    # CCMs of every provider are started together
+    # CMs of every provider are started together
     for op_instance in config[domains_name]:
         gen_dev_cfg = {
             'config': {
                 'input_dir': input_dir,
                 'target_dir': '/root/bin'
             },
-            'nodes': config[domains_name][op_instance]['op_ccm']
+            'nodes': config[domains_name][op_instance]['cluster_manager']
         }
 
         tweaked_configs = [
-            _tweak_config(gen_dev_cfg, ccm_node, op_instance, uid)
-            for ccm_node in gen_dev_cfg['nodes']]
+            _tweak_config(gen_dev_cfg, cm_node, op_instance, uid)
+            for cm_node in gen_dev_cfg['nodes']]
 
-        ccms = []
+        cms = []
         for cfg in tweaked_configs:
             node_out = _node_up(image, bindir, cfg, dns_servers, logdir)
-            ccms.extend(node_out['docker_ids'])
+            cms.extend(node_out['docker_ids'])
             common.merge(output, node_out)
 
-        common.wait_until(_ready, ccms, 0)
+        common.wait_until(_ready, cms, 0)
 
     return output

docker/environment/cluster_worker.py

@@ -20,7 +20,7 @@ class ClusterWorkerConfigurator:
     def tweak_config(self, cfg, uid):
         return cfg
 
-    def configure_started_instance(self, bindir, instance, config, os_config, output):
+    def configure_started_instance(self, bindir, instance, config, output):
         pass
 
     def extra_volumes(self, config):

docker/environment/common.py

@@ -147,7 +147,7 @@ def format_hostname(domain_parts, uid):
     """Formats hostname for a docker based on domain parts and uid.
     NOTE: Hostnames are also used as docker names!
     domain_parts - a single or a list of consecutive domain parts that constitute a unique name
-    within environment e.g.: ['worker1', 'prov1'], ['ccm1', 'prov1'], 'client1'
+    within environment e.g.: ['worker1', 'prov1'], ['cm1', 'prov1'], 'client1'
     uid - timestamp
     """
     if isinstance(domain_parts, (str, unicode)):
@@ -161,7 +161,7 @@ def format_hostname(domain_parts, uid):
 def format_erl_node_name(app_name, hostname):
     """Formats full node name for an erlang VM hosted on docker based on app_name and hostname.
     NOTE: Hostnames are also used as docker names!
-    app_name - application name, e.g.: 'op_ccm', 'globalregistry'
+    app_name - application name, e.g.: 'cluster_manager', 'globalregistry'
     hostname - hostname aquired by format_*_hostname
     """
     return '{0}@{1}'.format(app_name, hostname)

docker/environment/env.py

@@ -10,7 +10,7 @@
 import copy
 import subprocess
 import json
-from . import appmock, client, common, globalregistry, provider_ccm, \
+from . import appmock, client, common, globalregistry, cluster_manager, \
     provider_worker, cluster_worker, docker, dns
 
 
@@ -20,13 +20,13 @@ def default(key):
             'bin_gr': '{0}/globalregistry'.format(os.getcwd()),
             'bin_op_worker': '{0}/op_worker'.format(os.getcwd()),
             'bin_cluster_worker': '{0}/cluster_worker'.format(os.getcwd()),
-            'bin_op_ccm': '{0}/op_ccm'.format(os.getcwd()),
+            'bin_cluster_manager': '{0}/cluster_manager'.format(os.getcwd()),
             'bin_oc': '{0}/oneclient'.format(os.getcwd()),
             'logdir': None}[key]
 
 
 def up(config_path, image=default('image'), bin_am=default('bin_am'),
-       bin_gr=default('bin_gr'), bin_op_ccm=default('bin_op_ccm'),
+       bin_gr=default('bin_gr'), bin_cluster_manager=default('bin_cluster_manager'),
        bin_op_worker=default('bin_op_worker'), bin_cluster_worker=default('bin_cluster_worker'),
        bin_oc=default('bin_oc'), logdir=default('logdir')):
     config = common.parse_json_file(config_path)
@@ -36,7 +36,7 @@ def up(config_path, image=default('image'), bin_am=default('bin_am'),
         'docker_ids': [],
         'gr_nodes': [],
         'gr_db_nodes': [],
-        'op_ccm_nodes': [],
+        'cluster_manager_nodes': [],
         'op_worker_nodes': [],
         'cluster_worker_nodes': [],
         'appmock_nodes': [],
@@ -68,34 +68,12 @@ def up(config_path, image=default('image'), bin_am=default('bin_am'),
         dns.maybe_restart_with_configuration('auto', uid, output)
 
     # Start provider cluster instances
-    if 'provider_domains' in config:
-        # Start op_ccm instances
-        op_ccm_output = provider_ccm.up(image, bin_op_ccm, dns_server,
-                                        uid, config_path, logdir)
-        common.merge(output, op_ccm_output)
+    setup_worker(provider_worker, bin_op_worker, 'provider_domains',
+                 bin_cluster_manager, config, config_path, dns_server, image, logdir, output, uid)
 
-        # Start op_worker instances
-        op_worker_output = provider_worker.up(image, bin_op_worker, dns_server, uid, config_path, logdir)
-        common.merge(output, op_worker_output)
-        # Make sure OP domains are added to the dns server.
-        # Setting first arg to 'auto' will force the restart and this is needed
-        # so that dockers that start after can immediately see the domains.
-        dns.maybe_restart_with_configuration('auto', uid, output)
-
-    # Start provider cluster instances
-    if 'cluster_domains' in config:
-        # Start op_ccm instances
-        op_ccm_output = provider_ccm.up(image, bin_op_ccm, dns_server,
-                                        uid, config_path, logdir, domains_name='cluster_domains')
-        common.merge(output, op_ccm_output)
-
-        # Start op_worker instances
-        cluster_worker_output = cluster_worker.up(image, bin_cluster_worker, dns_server, uid, config_path, logdir)
-        common.merge(output, cluster_worker_output)
-        # Make sure OP domains are added to the dns server.
-        # Setting first arg to 'auto' will force the restart and this is needed
-        # so that dockers that start after can immediately see the domains.
-        dns.maybe_restart_with_configuration('auto', uid, output)
+    # Start stock cluster worker instances
+    setup_worker(cluster_worker, bin_cluster_worker, 'cluster_domains',
+                 bin_cluster_manager, config, config_path, dns_server, image, logdir, output, uid)
 
     # Start oneclient instances
     if 'oneclient' in config:
@@ -153,3 +131,19 @@ def up(config_path, image=default('image'), bin_am=default('bin_am'),
         )
 
     return output
+
+
+def setup_worker(worker, bin_worker, domains_name, bin_cm, config, config_path, dns_server, image, logdir, output, uid):
+    if domains_name in config:
+        # Start cluster_manager instances
+        cluster_manager_output = cluster_manager.up(image, bin_cm, dns_server, uid, config_path, logdir,
+                                                    domains_name=domains_name)
+        common.merge(output, cluster_manager_output)
+
+        # Start op_worker instances
+        cluster_worker_output = worker.up(image, bin_worker, dns_server, uid, config_path, logdir)
+        common.merge(output, cluster_worker_output)
+        # Make sure OP domains are added to the dns server.
+        # Setting first arg to 'auto' will force the restart and this is needed
+        # so that dockers that start after can immediately see the domains.
+        dns.maybe_restart_with_configuration('auto', uid, output)

docker/environment/provider_worker.py

@@ -25,13 +25,15 @@ def tweak_config(self, cfg, uid):
             sys_config['global_registry_domain'] = gr_hostname
         return cfg
 
-    def configure_started_instance(self, bindir, instance, config, os_config, output):
-        create_storages(config['os_configs'][os_config]['storages'],
-                        output[self.nodes_list_attribute()],
-                        config[self.domains_attribute()][instance][self.app_name()], bindir)
+    def configure_started_instance(self, bindir, instance, config, output):
+        if 'os_config' in config[self.domains_attribute()][instance]:
+            os_config = config[self.domains_attribute()][instance]['os_config']
+            create_storages(config['os_configs'][os_config]['storages'],
+                            output[self.nodes_list_attribute()],
+                            config[self.domains_attribute()][instance][self.app_name()], bindir)
 
     def extra_volumes(self, config):
-        return [common.volume_for_storage(s) for s in config['os_config']['storages']]
+        return [common.volume_for_storage(s) for s in config['os_config']['storages']] if 'os_config' in config else []
 
     def app_name(self):
         return "op_worker"