Merge pull request aws#59 in VFS/bamboos from VFS-1269-docker-environ…

…ment-refactor to develop

# By Piotr Ociepka
# Via Piotr Ociepka (6) and Michał Wrzeszcz (1)
* commit 'df968334e4ea1b1c5724fac67f4b30d3372464ea': (31 commits)
  Switch to CouchBase in example environments.
  Remove obsolete files in example_env.
  Even better error handling.
  Safe calls in escript, logs from escript.
  Update example_env file.
  Make acceptance tests work.
  Use Erlang cookie defined in env.json file while creating provider spaces.
  Refactor
  Directory structure for new client description.
  Refactor.
  Update example environemnt descriptions.
  Handle key- and cert-files.
  Refator.
  Remove obsolete escript.
  Move storage creation to dedicated function.
  Move storage creation to dedicated function.
  Refactor.
  Move hardcoded docker path to global variable.
  Refactor.
  get_token.escript
  ...

docker/environment/client.py

@@ -22,38 +22,37 @@ def client_hostname(node_name, uid):
     return common.format_hostname(node_name, uid)
 
 
-def _tweak_config(config, name, uid):
+def _tweak_config(config, os_config, name, uid):
     cfg = copy.deepcopy(config)
     cfg = {'node': cfg[name]}
     node = cfg['node']
     node['name'] = client_hostname(name, uid)
-    node['op_domain'] = provider_worker.provider_domain(node['op_domain'],
-                                                        uid)
-    node['gr_domain'] = globalregistry.gr_domain(node['gr_domain'], uid)
+    os_config_name = cfg['node']['os_config']
+    cfg['os_config'] = os_config[os_config_name]
+    node['clients'] = []
+    clients = config[name]['clients']
+    for cl in clients:
+        client = clients[cl]
+        client_config = {'name': client['name'],
+                         'op_domain': provider_worker.provider_domain(client['op_domain'], uid),
+                         'gr_domain': globalregistry.gr_domain(client['gr_domain'], uid),
+                         'user_key': client['user_key'],
+                         'user_cert': client['user_cert'],
+                         'mounting_path': client['mounting_path'],
+                         'token_for': client['token_for']}
+
+        node['clients'].append(client_config)
 
     return cfg
 
 
 def _node_up(image, bindir, config, config_path, dns_servers):
     node = config['node']
     hostname = node['name']
+    shortname = hostname.split(".")[0]
+    os_config = config['os_config']
 
-    cert_file_path = node['user_cert']
-    key_file_path = node['user_key']
-    # cert_file_path and key_file_path can both be an absolute path
-    # or relative to gen_dev_args.json
-    cert_file_path = os.path.join(common.get_file_dir(config_path),
-                                  cert_file_path)
-    key_file_path = os.path.join(common.get_file_dir(config_path),
-                                 key_file_path)
-
-    node['user_cert'] = '/tmp/cert'
-    node['user_key'] = '/tmp/key'
-
-    envs = {'X509_USER_CERT': node['user_cert'],
-            'X509_USER_KEY': node['user_key'],
-            'PROVIDER_HOSTNAME': node['op_domain'],
-            'GLOBAL_REGISTRY_URL': node['gr_domain']}
+    client_data = {}
 
     # We want the binary from debug more than relwithdebinfo, and any of these
     # more than from release (ifs are in reverse order so it works when
@@ -62,29 +61,52 @@ def _node_up(image, bindir, config, config_path, dns_servers):
 [ -d /root/build/release ] && cp /root/build/release/oneclient /root/bin/oneclient
 [ -d /root/build/relwithdebinfo ] && cp /root/build/relwithdebinfo/oneclient /root/bin/oneclient
 [ -d /root/build/debug ] && cp /root/build/debug/oneclient /root/bin/oneclient
-cat <<"EOF" > /tmp/cert
+mkdir /tmp/certs
+mkdir /tmp/keys
+'''
+
+    for client in node['clients']:
+        # for each client instance we want to have separated certs and keys
+        client_name = client["name"]
+        client_data[client_name] = {'client_name': client_name,
+                                    'op_domain': client['op_domain'],
+                                    'gr_domain': client['gr_domain'],
+                                    'mounting_path': client['mounting_path'],
+                                    'token_for': client['token_for']}
+        # cert_file_path and key_file_path can both be an absolute path
+        # or relative to gen_dev_args.json
+        cert_file_path = os.path.join(common.get_file_dir(config_path),
+                                      client['user_cert'])
+        key_file_path = os.path.join(common.get_file_dir(config_path),
+                                     client['user_key'])
+        command += '''mkdir /tmp/certs/{client_name}
+mkdir /tmp/keys/{client_name}
+cat <<"EOF" > /tmp/certs/{client_name}/cert
 {cert_file}
 EOF
-cat <<"EOF" > /tmp/key
+cat <<"EOF" > /tmp/keys/{client_name}/key
 {key_file}
 EOF
-bash'''
-    command = command.format(
-        cert_file=open(cert_file_path, 'r').read(),
-        key_file=open(key_file_path, 'r').read())
+'''
+
+        command = command.format(
+            client_name=client_name,
+            cert_file=open(cert_file_path, 'r').read(),
+            key_file=open(key_file_path, 'r').read())
+
+        client_data[client_name]['user_cert'] = os.path.join('/tmp', 'certs', client_name, 'cert')
+        client_data[client_name]['user_key'] = os.path.join('/tmp', 'keys', client_name, 'key')
+
+    command += '''bash'''
 
     volumes = [(bindir, '/root/build', 'ro')]
-    storages = node['storage']
-    for name in storages:
-        s = storages[name]
-        volumes.append((s['host_path'], s['volume_path'], 'rw'))
+    volumes += [common.volume_for_storage(s) for s in os_config['storages']]
 
     container = docker.run(
         image=image,
         name=hostname,
         hostname=hostname,
         detach=True,
-        envs=envs,
         interactive=True,
         tty=True,
         workdir='/root/bin',
@@ -93,17 +115,18 @@ def _node_up(image, bindir, config, config_path, dns_servers):
         run_params=["--privileged"],
         command=command)
 
-    for user in node['docker_users']:
-        command = "docker exec %s adduser --disabled-password --gecos '' %s" % (container, user)
-        # print command
-        subprocess.check_call(command, stdout=sys.stdout, shell=True)
+    # create system users and groups
+    common.create_users(container, os_config['users'])
+    common.create_groups(container, os_config['groups'])
 
-    return {'docker_ids': [container], 'client_nodes': [hostname]}
+    return {'docker_ids': [container], 'client_nodes': [hostname], 'client_data': {shortname: client_data}}
 
 
 def up(image, bindir, dns_server, uid, config_path):
-    config = common.parse_json_file(config_path)['oneclient']
-    configs = [_tweak_config(config, node, uid) for node in config]
+    json_config = common.parse_json_file(config_path)
+    config = json_config['oneclient']
+    os_config = json_config['os_configs']
+    configs = [_tweak_config(config, os_config, node, uid) for node in config]
 
     dns_servers, output = dns.maybe_start(dns_server, uid)
 

docker/environment/common.py

@@ -172,3 +172,33 @@ def generate_uid():
     that can be used to group dockers in DNS
     """
     return str(int(time.time()))
+
+
+def create_users(container, users):
+    """Creates system users on docker specified by 'container'.
+    """
+    for user in users:
+        uid = str(hash(user) % 50000 + 10000)
+        command = ["adduser", "--disabled-password", "--gecos", "''",
+                   "--uid", uid, user]
+        assert 0 is docker.exec_(container, command, interactive=True)
+
+
+def create_groups(container, groups):
+    """Creates system groups on docker specified by 'container'.
+    """
+    for group in groups:
+        gid = str(hash(group) % 50000 + 10000)
+        command = ["groupadd", "-g", gid, group]
+        assert 0 is docker.exec_(container, command, interactive=True)
+        for user in groups[group]:
+            command = ["usermod", "-a", "-G", group, user]
+            assert 0 is docker.exec_(container, command, interactive=True)
+
+
+def volume_for_storage(storage):
+    """Returns tuple (path_on_host, path_on_docker, read_wtire_mode)
+    for a given storage
+    """
+    return os.path.join('/tmp/onedata/storage/', storage), storage, 'rw'
+

docker/environment/create_storage.escript

@@ -0,0 +1,23 @@
+#!/usr/bin/env escript
+%%! -name create_storage@test_env
+
+main([Cookie, Node, Name, Path]) ->
+  erlang:set_cookie(node(), list_to_atom(Cookie)),
+  NodeAtom = list_to_atom(Node),
+  Helper = safe_call(NodeAtom, fslogic_storage, new_helper_init, [<<"DirectIO">>, #{<<"root_path">> => list_to_binary(Path)}]),
+  Storage = safe_call(NodeAtom, fslogic_storage, new_storage, [list_to_binary(Name), [Helper]]),
+  safe_call(NodeAtom, storage, create, [Storage]).
+
+safe_call(Node, Module, Function, Args) ->
+  case rpc:call(Node, Module, Function, Args) of
+    {badrpc, X} ->
+      io:format(standard_error, "ERROR: in module ~p:~n {badrpc, ~p} in rpc:call(~p, ~p, ~p, ~p).~n",
+        [?MODULE, X, Node, Module, Function, Args]),
+      halt(42);
+    {error, X} ->
+      io:format(standard_error, "ERROR: in module ~p:~n {error, ~p} in rpc:call(~p, ~p, ~p, ~p).~n",
+        [?MODULE, X, Node, Module, Function, Args]),
+      halt(42);
+    X ->
+      X
+  end.

docker/environment/provider_worker.py

@@ -8,11 +8,14 @@
 import copy
 import json
 import os
+import subprocess
+import sys
 
 from . import common, docker, riak, couchbase, dns, globalregistry, provider_ccm
 
 PROVIDER_WAIT_FOR_NAGIOS_SECONDS = 60 * 2
-
+# mounting point for op-worker-node docker
+DOCKER_BINDIR_PATH = '/root/build'
 
 def provider_domain(op_instance, uid):
     """Formats domain for a provider."""
@@ -80,7 +83,8 @@ def _node_up(image, bindir, config, dns_servers, db_node_mappings, logdir):
         uid=os.geteuid(),
         gid=os.getegid())
 
-    volumes = [(bindir, '/root/build', 'ro')]
+    volumes = [(bindir, DOCKER_BINDIR_PATH, 'ro')]
+    volumes += [common.volume_for_storage(s) for s in config['os_config']['storages']]
 
     if logdir:
         logdir = os.path.join(os.path.abspath(logdir), hostname)
@@ -93,11 +97,15 @@ def _node_up(image, bindir, config, dns_servers, db_node_mappings, logdir):
         detach=True,
         interactive=True,
         tty=True,
-        workdir='/root/build',
+        workdir=DOCKER_BINDIR_PATH,
         volumes=volumes,
         dns_list=dns_servers,
         command=command)
 
+    # create system users and grous
+    common.create_users(container, config['os_config']['users'])
+    common.create_groups(container, config['os_config']['groups'])
+
     return container, {
         'docker_ids': [container],
         'op_worker_nodes': [node_name]
@@ -161,13 +169,15 @@ def up(image, bindir, dns_server, uid, config_path, logdir=None):
 
     # Workers of every provider are started together
     for op_instance in config['provider_domains']:
+        os_config = config['provider_domains'][op_instance]['os_config']
         gen_dev_cfg = {
             'config': {
                 'input_dir': input_dir,
                 'target_dir': '/root/bin'
             },
             'nodes': config['provider_domains'][op_instance]['op_worker'],
-            'db_driver': _db_driver(config['provider_domains'][op_instance])
+            'db_driver': _db_driver(config['provider_domains'][op_instance]),
+            'os_config': config['os_configs'][os_config]
         }
 
         # Tweak configs, retrieve lis of riak nodes to start
@@ -217,6 +227,32 @@ def up(image, bindir, dns_server, uid, config_path, logdir=None):
         }
         common.merge(output, domains)
 
+        # create storages
+        create_storages(config['os_configs'][os_config]['storages'],
+                        output['op_worker_nodes'],
+                        config['provider_domains'][op_instance]['op_worker'],
+                        bindir)
+
     # Make sure domains are added to the dns server.
     dns.maybe_restart_with_configuration(dns_server, uid, output)
     return output
+
+def create_storages(storages, op_nodes, op_config, bindir):
+    # copy escript to docker host
+    script_name = 'create_storage.escript'
+    pwd = common.get_script_dir()
+    command = ['cp', os.path.join(pwd, script_name), os.path.join(bindir, script_name)]
+    subprocess.check_call(command)
+    # execute escript
+    for node in op_nodes:
+        container = node.split("@")[1]
+        worker_name = container.split(".")[0]
+        cookie = op_config[worker_name]['vm.args']['setcookie']
+        script_path = os.path.join(DOCKER_BINDIR_PATH, script_name)
+        for st_path in storages:
+            st_name = st_path
+            command = ['escript', script_path, cookie, node, st_name, st_path]
+            assert 0 is docker.exec_(container, command, tty=True, stdout=sys.stdout, stderr=sys.stderr)
+    # clean-up
+    command = ['rm', os.path.join(bindir, script_name)]
+    subprocess.check_call(command)

example_env/appmock_gr.json

@@ -21,8 +21,25 @@
             "target_dir": "rel/test_cluster"
         }
     },
+    "os_configs": {
+        "cfg1": {
+            "storages": [
+                "/mnt/st1"
+            ],
+            "users": [
+                "user1"
+            ],
+            "groups": {
+                "group1": [
+                    "user1"
+                ]
+            }
+        }
+    },
     "provider_domains": {
         "p1": {
+            "db_driver": "couchbase",
+            "os_config": "cfg1",
             "op_ccm": {
                 "ccm1": {
                     "vm.args": {

example_env/appmock_worker.json

@@ -21,8 +21,25 @@
             "target_dir": "rel/test_cluster"
         }
     },
+    "os_configs": {
+        "cfg1": {
+            "storages": [
+                "/mnt/st1"
+            ],
+            "users": [
+                "user1"
+            ],
+            "groups": {
+                "group1": [
+                    "user1"
+                ]
+            }
+        }
+    },
     "provider_domains": {
         "p1": {
+            "db_driver": "couchbase",
+            "os_config": "cfg1",
             "op_ccm": {
                 "ccm1": {
                     "vm.args": {