Squashed 'appmock/' changes from fab87fc..d14fcb8

d14fcb8 Merge commit '8bcc4366b044703bafe72c43a990929ada798021' into feature/VFS-1652-define-space-storage-per-provider
8bcc436 Squashed 'bamboos/' changes from 2f522cc..434dc53
688932b Merge pull request aws#32 in VFS/appmock from feature/VFS-1665-finish-native-erlang-implementation to develop

git-subtree-dir: appmock
git-subtree-split: d14fcb8b280d5a9c1210e7f5d8125ba929fa6612

CHANGELOG.md

@@ -4,6 +4,18 @@
 CHANGELOG
 ---------
 
+### 1.0.1
+
+* VFS-1571 Mount compilation caches by default in make.py .
+* VFS-1571 Add rebar 2.6.1 built with rebar_beam_cache.
+* VFS-1447 sys.config per app
+* VFS-1508 direct output from ember live build to log file in dev environment
+ to toggle gui livereload
+* VFS-1508 change owner of GUI tmp dir periodically rather than run ember build as user
+* VFS-1508 run livereload as user
+* VFS-1508 change build process of GUI to use ember-cli, add livereload
+
+
 ### 1.0.0
 
 
@@ -30,7 +42,7 @@ CHANGELOG
 * VFS-1010 Don't shed root privileges on non-Linux platforms.
 * VFS-1010 Run scripts in make.py as a running user.
 * VFS-1010 Cleanup scripts associated with running distributed tests.
-* VFS-1000 Add a first version of globalregistry_up.
+* VFS-1000 Add a first version of zone_up.
 * VFS-1000 Add a makefile, fetch required images automatically.
 * VFS-1000 Configure the cluster through gen_dev_args.json
 * VFS-1000 Implement a make.py script and improve cluster_up.

docker/client_up.py

@@ -18,9 +18,15 @@
 
 parser = common.standard_arg_parser(
     'Set up dockers with oneclient preconfigured.')
+parser.add_argument(
+    '-l', '--logdir',
+    action='store',
+    default=None,
+    help='path to a directory where the logs will be stored',
+    dest='logdir')
 
 args = parser.parse_args()
 output = client.up(args.image, args.bin, args.dns, args.uid,
-                   args.config_path)
+                   args.config_path, args.logdir)
 
 print(json.dumps(output))

docker/cluster_up.py

@@ -34,30 +34,43 @@
     default=os.getcwd() + '/cluster_manager',
     help='the path to cluster_manager repository (precompiled)',
     dest='bin_cluster_manager')
+parser.add_argument(
+    '-do', '--domain',
+    action='store',
+    default='provider_domains',
+    help='worker domain name, can be: provider_domains or cluster_domains',
+    dest='domains_name')
 
 # Prepare config
 args = parser.parse_args()
-config = common.parse_json_file(args.config_path)
+config = common.parse_json_config_file(args.config_path)
 output = {
     'cluster_manager_nodes': [],
     'cluster_worker_nodes': [],
 }
-uid = common.generate_uid()
+uid = args.uid
 
 # Start DNS
-[dns_server], dns_output = dns.maybe_start('auto', uid)
-common.merge(output, dns_output)
+if args.dns == 'auto':
+    [dns_server], dns_output = dns.maybe_start('auto', uid)
+    common.merge(output, dns_output)
+else:
+    dns_server = args.dns
 
 # Start cms
-cm_output = cluster_manager.up(args.image, args.bin_cluster_manager, dns_server, uid, args.config_path, args.logdir)
+cm_output = cluster_manager.up(args.image, args.bin_cluster_manager, dns_server,
+                               uid, args.config_path, args.logdir,
+                               args.domains_name)
 common.merge(output, cm_output)
 
 # Start workers
-worker_output = cluster_worker.up(args.image, args.bin_op_worker, dns_server, uid, args.config_path, args.logdir)
+worker_output = cluster_worker.up(args.image, args.bin_op_worker, dns_server,
+                                  uid, args.config_path, args.logdir)
 common.merge(output, worker_output)
 
-# Make sure domain are added to the dns server
-dns.maybe_restart_with_configuration('auto', uid, output)
+if dns_server != 'none':
+    # Make sure domain are added to the dns server
+    dns.maybe_restart_with_configuration('auto', uid, output)
 
 # Print results
 print(json.dumps(output))

docker/couchbase_up.py

@@ -49,7 +49,14 @@
     help='number of couchbase nodes to bring up',
     dest='nodes')
 
+parser.add_argument(
+        '-cn', '--cluster_name',
+        action='store',
+        default="cluster",
+        help='name of couchbase cluster',
+        dest='cluster_name')
+
 args = parser.parse_args()
 
-output = couchbase.up(args.image, args.dns, args.uid, args.nodes)
+output = couchbase.up(args.image, args.dns, args.uid, args.cluster_name, args.nodes)
 print(json.dumps(output))

docker/ct_run.py

@@ -14,19 +14,35 @@
 """
 
 from __future__ import print_function
+
 import argparse
 import glob
+import json
 import os
 import platform
-import sys
-import time
 import re
 import shutil
-import json
+import sys
+import time
+import glob
+import xml.etree.ElementTree as ElementTree
 
 sys.path.insert(0, 'bamboos/docker')
 from environment import docker
 
+
+def skipped_test_exists(junit_report_path):
+    reports = glob.glob(junit_report_path)
+    # if there are many reports, check only the last one
+    reports.sort()
+    tree = ElementTree.parse(reports[-1])
+    testsuites = tree.getroot()
+    for testsuite in testsuites:
+        if testsuite.attrib['skipped'] != '0':
+            return True
+    return False
+
+
 parser = argparse.ArgumentParser(
     formatter_class=argparse.ArgumentDefaultsHelpFormatter,
     description='Run Common Tests.')
@@ -188,10 +204,10 @@
                             ('cluster_manager', data['cluster_domains'][cluster]['cluster_manager'].values())
                         )
 
-            if 'globalregistry_domains' in data:
-                for globalregistry in data['globalregistry_domains']:
+            if 'zone_domains' in data:
+                for zone in data['zone_domains']:
                     configs_to_change.extend(
-                        ('globalregistry', data['globalregistry_domains'][globalregistry]['globalregistry'].values())
+                        ('oz_worker', data['zone_domains'][zone]['oz_worker'].values())
                     )
 
             for (app_name, config) in configs_to_change:
@@ -250,4 +266,7 @@
         os.remove(file)
         shutil.move(file + '.bak', file)
 
+if ret != 0 and not skipped_test_exists("test_distributed/logs/*/surefire.xml"):
+    ret = 0
+
 sys.exit(ret)

docker/dns_up.py

@@ -52,5 +52,5 @@
 
 if args.root_servers:
     args.root_servers = args.root_servers.split(',')
-output = dns.up(args.uid, args.root_servers, args.restart)
+output = dns.up(args.uid, args.root_servers, [], args.restart)
 print(json.dumps(output))

docker/env_up.py

@@ -10,13 +10,12 @@
 """
 
 from __future__ import print_function
+
 import argparse
 import json
-import os
 
 from environment import env
 
-
 parser = argparse.ArgumentParser(
     formatter_class=argparse.ArgumentDefaultsHelpFormatter,
     description='Bring up onedata environment.')
@@ -50,11 +49,11 @@
     dest='bin_cluster_manager')
 
 parser.add_argument(
-    '-bg', '--bin-gr',
+    '-boz', '--bin-oz',
     action='store',
-    default=env.default('bin_gr'),
-    help='the path to globalregistry repository (precompiled)',
-    dest='bin_gr')
+    default=env.default('bin_oz'),
+    help='the path to zone repository (precompiled)',
+    dest='bin_oz')
 
 parser.add_argument(
     '-ba', '--bin-appmock',
@@ -85,8 +84,10 @@
 args = parser.parse_args()
 
 output = env.up(args.config_path, image=args.image, bin_am=args.bin_am,
-       bin_gr=args.bin_gr, bin_cluster_manager=args.bin_cluster_manager,
-       bin_op_worker=args.bin_op_worker, bin_cluster_worker=args.bin_cluster_worker,
-       bin_oc=args.bin_oc, logdir=args.logdir)
+                bin_oz=args.bin_oz,
+                bin_cluster_manager=args.bin_cluster_manager,
+                bin_op_worker=args.bin_op_worker,
+                bin_cluster_worker=args.bin_cluster_worker,
+                bin_oc=args.bin_oc, logdir=args.logdir)
 
 print(json.dumps(output))

docker/environment/appmock.py

@@ -12,14 +12,14 @@
 import random
 import string
 
-from . import common, docker, dns, cluster_manager, worker, globalregistry
+from . import common, docker, dns, cluster_manager, worker
 
 APPMOCK_WAIT_FOR_NAGIOS_SECONDS = 60 * 2
 
 
 def domain(appmock_instance, uid):
     """Formats domain for an appmock instance.
-    It is intended to fake OP or GR domain.
+    It is intended to fake OP or OZ domain.
     """
     return common.format_hostname(appmock_instance, uid)
 
@@ -49,12 +49,12 @@ def _tweak_config(config, appmock_node, appmock_instance, uid):
     # default appmock_erl_node_name will be used.
     node_name = {
         'cluster_manager': cluster_manager.cm_erl_node_name(appmock_node,
-                                                 appmock_instance, uid),
+                                                            appmock_instance,
+                                                            uid),
         'op_worker': worker.worker_erl_node_name(appmock_node,
                                                  appmock_instance,
                                                  uid),
-        'globalregistry': globalregistry.gr_erl_node_name(appmock_node,
-                                                          appmock_instance, uid)
+        'oz_worker': worker.worker_erl_node_name(appmock_node, appmock_instance, uid)
     }.get(mocked_app, appmock_erl_node_name(appmock_node, uid))
 
     if 'vm.args' not in cfg['nodes']['node']:
@@ -84,7 +84,10 @@ def _node_up(image, bindir, config, config_path, dns_servers, logdir):
     # file_name must be preserved as it must match the Erlang module name
     sys_config['app_description_file'] = '/tmp/' + app_desc_file_name
 
-    command = '''set -e
+    command = '''mkdir -p /root/bin/node/log/
+echo 'while ((1)); do chown -R {uid}:{gid} /root/bin/node/log; sleep 1; done' > /root/bin/chown_logs.sh
+bash /root/bin/chown_logs.sh &
+set -e
 cat <<"EOF" > /tmp/{app_desc_file_name}
 {app_desc_file}
 EOF
@@ -94,6 +97,8 @@ def _node_up(image, bindir, config, config_path, dns_servers, logdir):
 escript bamboos/gen_dev/gen_dev.escript /tmp/gen_dev_args.json
 /root/bin/node/bin/appmock console'''
     command = command.format(
+        uid=os.geteuid(),
+        gid=os.getegid(),
         app_desc_file_name=app_desc_file_name,
         app_desc_file=open(app_desc_file_path, 'r').read(),
         gen_dev_args=json.dumps({'appmock': config}))
@@ -154,7 +159,7 @@ def up(image, bindir, dns_server, uid, config_path, logdir=None):
             appmocks.append(appmock_id)
             if 'mocked_app' in cfg['nodes']['node']:
                 mocked_app = cfg['nodes']['node']['mocked_app']
-                if mocked_app == 'op_worker' or mocked_app == 'globalregistry':
+                if mocked_app == 'op_worker' or mocked_app == 'oz_worker':
                     include_domain = True
                     appmock_ips.append(common.get_docker_ip(appmock_id))
             common.merge(output, node_out)