Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Enhancement for HammerDB doc generation/indexing #120

Merged
merged 14 commits into from
Jan 28, 2020
Merged

Enhancement for HammerDB doc generation/indexing #120

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
14 commits
Select commit Hold shift + click to select a range
1ded583
stuff added. and things
Jan 15, 2020
9422641
cloud-bulldozer now
Jan 15, 2020
3c4dd5c
added paths to the COPY directives
Jan 15, 2020
b31d518
added the patch Ben proposed to prevent from silently failing when th…
Jan 15, 2020
abe282c
reverting to the old way snafu gets copied in
Jan 16, 2020
06cb2d0
cosmetics for the Dockerfile
Jan 17, 2020
ebe12d7
added missing env parameters
Jan 17, 2020
0f6c20e
merging
Jan 17, 2020
147c1e3
fixed a loop variable
Jan 28, 2020
a8da271
changes to make elasticsearch entries work. hopefully.
Jan 28, 2020
34d9013
Merge branch 'master' into master
mkarg75 Jan 28, 2020
b963d87
removed superfluous files
Jan 28, 2020
c6b0033
Merge branch 'master' of github.com:mkarg75/snafu
Jan 28, 2020
555cdd1
another one bites the dust
Jan 28, 2020
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 2 additions & 1 deletion hammerdb/Dockerfile
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,14 +1,15 @@
FROM registry.access.redhat.com/ubi8:latest

# install requirements
RUN dnf install --nodocs -y tcl unixODBC python3-pip python3-requests
RUN dnf install --nodocs -y tcl unixODBC python3-pip python3-requests

RUN curl https://packages.microsoft.com/config/rhel/8/prod.repo -o /etc/yum.repos.d/mssql-release.repo
COPY image_resources/centos8.repo /etc/yum.repos.d/centos8.repo
RUN ACCEPT_EULA=Y dnf -y install --nodocs msodbcsql17 --enablerepo=centos8
RUN dnf clean all
RUN pip3 install --upgrade-strategy=only-if-needed "elasticsearch>=6.0.0,<=7.0.2" pyyaml
COPY . /opt/snafu
RUN ln -s /usr/bin/python3 /usr/bin/python

# Download and install the hammer suite
RUN curl -L 'https://downloads.sourceforge.net/project/hammerdb/HammerDB/HammerDB-3.2/HammerDB-3.2-Linux-x86-64-Install?r=https%3A%2F%2Fsourceforge.net%2Fprojects%2Fhammerdb%2Ffiles%2FHammerDB%2FHammerDB-3.2%2FHammerDB-3.2-Linux-x86-64-Install%2Fdownload&ts=1564587940&use_mirror=autoselect' -o hammer_installer
Expand Down
117 changes: 66 additions & 51 deletions hammerdb/hammerd_wrapper.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -34,59 +34,69 @@ def _parse_stdout(stdout):

def _json_payload(data, uuid, db_server, db_port, db_warehouses, db_num_workers, db_tcp, db_user, transactions, test_type, runtime, rampup, samples, timed_test, timestamp):
processed = []
for i in range(0,len(data)):
processed.append({
"workload" : "hammerdb",
"uuid" : uuid,
"db_server" : db_server,
"db_port" : db_port,
"db_warehouses" : db_warehouses,
"db_num_workers" : db_num_workers,
"db_tcp": db_tcp,
"db_user": db_user,
"transactions": transactions,
"test_type": test_type,
"runtime": runtime,
"rampup": rampup,
"samples": samples,
"timed_test": timed_test,
"worker": data[i][0],
"tpm": data[i][1],
"nopm": data[i][2],
"timestamp": timestamp
})
for current_worker in range(1, int(db_num_workers)):
for current_sample in range(1, int(samples)):
for i in range(0,len(data)):
processed.append({
"workload" : "hammerdb",
"uuid" : uuid,
"db_server" : db_server,
"db_port" : db_port,
"db_warehouses" : db_warehouses,
"db_num_workers" : db_num_workers,
"db_tcp": db_tcp,
"db_user": db_user,
"transactions": transactions,
"test_type": test_type,
"runtime": runtime,
"rampup": rampup,
"samples": samples,
"current_sample": current_sample,
"current_worker": current_worker,
"timed_test": timed_test,
"worker": data[i][0],
"tpm": data[i][1],
"nopm": data[i][2],
"timestamp": timestamp
})
return processed

def _summarize_data(data):
for i in range(0,len(data)):
entry = data[i]

print("+{} HammerDB Results {}+".format("-"*(50), "-"*(50)))
print("HammerDB setup")
print("")
print("HammerDB results for:")
print("UUID: {}".format(entry['uuid']))
print("Database server: {}".format(entry['db_server']))
print("Database port: {}".format(entry['db_port']))
print("Number of database warehouses: {}".format(entry['db_warehouses']))
print("Number of workers: {}".format(entry['db_num_workers']))
print("TCP connection to the DB: {}".format(entry['db_tcp']))
print("Database user: {}".format(entry['db_user']))
print("Transactions: {}".format(entry['transactions']))
print("Test type: {}".format(entry['test_type']))
print("Runtime: {}".format(entry['runtime']))
print("Rampup time: {}".format(entry['rampup']))
print("Worker: {}".format(entry['worker']))
print("Samples: {}".format(entry['samples']))
print("Timed test: {}".format(entry['timed_test']))
print("HammerDB results (TPM):")
print("""
TPM: {}""".format(entry['tpm']))
print("HammerDB results (NOPM):")
print("""
NOPM: {}""".format(entry['nopm']))
print("Timestamp: {}".format(entry['timestamp']))
print("+{}+".format("-"*(115)))
max_workers = int(data[0]['db_num_workers'])
mkarg75 marked this conversation as resolved.
Show resolved Hide resolved
max_samples = int(data[0]['samples'])
for current_worker in range(1,max_workers):
for current_sample in range(1,max_samples):
for i in range(0,len(data)):
entry = data[i]

print("+{} HammerDB Results {}+".format("-"*(50), "-"*(50)))
print("HammerDB setup")
print("")
print("HammerDB results for:")
print("UUID: {}".format(entry['uuid']))
print("Database server: {}".format(entry['db_server']))
print("Database port: {}".format(entry['db_port']))
print("Number of database warehouses: {}".format(entry['db_warehouses']))
print("Number of workers: {}".format(entry['db_num_workers']))
print("TCP connection to the DB: {}".format(entry['db_tcp']))
print("Database user: {}".format(entry['db_user']))
print("Transactions: {}".format(entry['transactions']))
print("Test type: {}".format(entry['test_type']))
print("Runtime: {}".format(entry['runtime']))
print("Rampup time: {}".format(entry['rampup']))
print("Worker: {}".format(current_worker))
print("Samples: {}".format(entry['samples']))
print("Current sample {}".format(current_sample))
print("Timed test: {}".format(entry['timed_test']))
print("HammerDB results (TPM):")
print("""
TPM: {}""".format(entry['tpm']))
print("HammerDB results (NOPM):")
print("""
NOPM: {}""".format(entry['nopm']))
print("Timestamp: {}".format(entry['timestamp']))
print("+{}+".format("-"*(115)))

def _index_result(index,es_server,es_port,payload):
_es_connection_string = str(es_server) + ':' + str(es_port)
Expand Down Expand Up @@ -123,6 +133,9 @@ def main():
iteration = ""
test_type = ""
timestamp = ""
db_tcp = ""
timed_test = ""


if "es_server" in os.environ:
es_server = os.environ["es_server"]
Expand Down Expand Up @@ -167,11 +180,13 @@ def main():
exit(1)
data = _parse_stdout(stdout[0])
documents = _json_payload(data, uuid, db_server, db_port, db_warehouses, db_num_workers, db_tcp, db_user, transactions, test_type, runtime, rampup, samples, timed_test, timestamp)
if len(documents) > 0 :
_summarize_data(documents)
if es_server != "" :
if len(documents) > 0 :
_index_result("ripsaw-hammerdb-results", es_server, es_port, documents)
if len(documents) > 0 :
_summarize_data(documents)
else:
raise Exception('Failed to produce hammerdb results document')


if __name__ == '__main__':
Expand Down