Adding log generator script and container file (#254)

Adding log generator script and container file. Updated setup.cfg to set openshift and kubernetes version

README.md

@@ -20,7 +20,10 @@ environments, develop in SNAFU and then write benchmark-operator benchmark to in
 | smallfile                      | metadata-intensive ops | Working            |
 | fs-drift                       | metadata-intensive mix | Working            |
 | cyclictest                     | Real-Time Latency      | Working            |
-| oslat                          | Real-Time Latency      | Working            |
+| oslat                          | Real-Time Latency              | Working            |
+| OpenShift Upgrade              | Time to upgrade                | Working            |
+| OpenShift Scaling              | Time to scale                  | Working            |
+| Log Generator                  | Log throughput to backend      | Working            |
 
 ## What backend storage do we support?
 

setup.cfg

@@ -21,7 +21,7 @@ zip_safe = False
 packages = find:
 include_package_data = True
 # Add here dependencies of your project (semicolon/line-separated), e.g.
-install_requires = configparser; elasticsearch>=6.0.0,<=7.0.2; statistics; numpy; pyyaml; requests; redis; python-dateutil>=2.7.3; prometheus_api_client; scipy; openshift; setuptools>=40.3.0; flent
+install_requires = configparser; elasticsearch>=6.0.0,<=7.0.2; statistics; numpy; pyyaml; requests; redis; python-dateutil>=2.7.3; prometheus_api_client; scipy; openshift==0.11; kubernetes==11; setuptools>=40.3.0; boto3; flent; importlib_metadata
 # tests_require = pytest; pytest-cov
 # Require a specific Python version, e.g. Python 2.7 or >= 3.4
 python_requires = >=3.6

snafu/log_generator_wrapper/Dockerfile

@@ -0,0 +1,10 @@
+FROM registry.access.redhat.com/ubi8:latest
+
+COPY snafu/image_resources/centos8-appstream.repo /etc/yum.repos.d/centos8-appstream.repo
+RUN dnf install -y --nodocs python3 python3-pip && dnf clean all
+RUN dnf install -y --nodocs redis --enablerepo=centos8-appstream && dnf clean all
+RUN curl -L https://mirror.openshift.com/pub/openshift-v4/clients/ocp/latest/openshift-client-linux.tar.gz | tar xz -C /usr/bin/ oc
+RUN ln -s /usr/bin/python3 /usr/bin/python
+RUN mkdir -p /opt/snafu/
+COPY . /opt/snafu/
+RUN pip3 install -e /opt/snafu/

snafu/log_generator_wrapper/ci_test.sh

@@ -0,0 +1,11 @@
+#!/bin/bash
+
+set -x
+
+source ci/common.sh
+
+# Build image for ci
+image_spec=$SNAFU_WRAPPER_IMAGE_PREFIX/log_generator:$SNAFU_IMAGE_TAG
+build_and_push snafu/log_generator_wrapper/Dockerfile $image_spec
+pushd ripsaw
+source tests/test_log_generator.sh

snafu/log_generator_wrapper/log_generator_wrapper.py

@@ -0,0 +1,103 @@
+#!/usr/bin/env python
+#   Licensed under the Apache License, Version 2.0 (the "License");
+#   you may not use this file except in compliance with the License.
+#   You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+#   Unless required by applicable law or agreed to in writing, software
+#   distributed under the License is distributed on an "AS IS" BASIS,
+#   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#   See the License for the specific language governing permissions and
+#   limitations under the License.
+
+import os
+import argparse
+from .trigger_log_generator import Trigger_log_generator
+
+
+class log_generator_wrapper():
+
+    def __init__(self, parent_parser):
+        parser_object = argparse.ArgumentParser(description="Log Generator Wrapper script",
+                                                parents=[parent_parser],
+                                                formatter_class=argparse.ArgumentDefaultsHelpFormatter)
+        parser = parser_object.add_argument_group("Log Generator")
+        parser.add_argument(
+            '-u', '--uuid',
+            required=True,
+            help='Provide the uuid')
+        parser.add_argument(
+            '--size',
+            type=int,
+            required=True,
+            help='Size in bytes of a message')
+        parser.add_argument(
+            '--messages-per-minute',
+            type=int,
+            help='How many messages to send in one minute')
+        parser.add_argument(
+            '--messages-per-second',
+            type=int,
+            help='How many messages to send in one second')
+        parser.add_argument(
+            '--duration',
+            type=int,
+            required=True,
+            help='How long to run the test for in minutes')
+        parser.add_argument(
+            '--user',
+            default="snafu",
+            help='Enter the user')
+        parser.add_argument(
+            '--pod-count',
+            type=int,
+            default=1,
+            help='Total number of log generator pods to run')
+        parser.add_argument(
+            '--pod-name',
+            default=None,
+            help='Pod Name of log generator')
+        parser.add_argument(
+            '--namespace',
+            default=None,
+            help='Namespace log generator lives in')
+        parser.add_argument(
+            '--timeout',
+            type=int,
+            default=600,
+            help='Max amount of time (in seconds) to wait for the backend service\
+                  to obtain all the logs once the test is complete')
+        parser.add_argument(
+            '--cloudwatch-log-group',
+            help='The cloudwatch log group to check for messages')
+        parser.add_argument(
+            '--aws-access-key',
+            default=None,
+            help='AWS access key id used for verification with cloudwatch')
+        parser.add_argument(
+            '--aws-secret-key',
+            default=None,
+            help='AWS secret key used for verification with cloudwatch')
+        parser.add_argument(
+            '--aws-region',
+            default=None,
+            help='AWS region that CloudWatch is in')
+        parser.add_argument(
+            '--es-url',
+            help='Provide elastic server url')
+        parser.add_argument(
+            '--es-token',
+            help='Bearer token to access ES server')
+        parser.add_argument(
+            '--es-index',
+            type=str,
+            default="app*",
+            help='The ES index to search for the messages')
+
+        self.args = parser_object.parse_args()
+
+        self.args.cluster_name = os.getenv("clustername", "mycluster")
+
+    def run(self):
+        yield Trigger_log_generator(self.args)