Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Update example code for refactor #80

Merged
merged 6 commits into from
Dec 9, 2021
Merged
Show file tree
Hide file tree
Changes from 1 commit
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
12 changes: 7 additions & 5 deletions CryptoCurrency/script.py
Original file line number Diff line number Diff line change
@@ -1,9 +1,11 @@
import os
os.system("pip install pycoingecko")
try:
from pycoingecko import CoinGeckoAPI
except ImportError:
os.system("pip install pycoingecko")
from pycoingecko import CoinGeckoAPI

from pycoingecko import CoinGeckoAPI

from deephaven.DBTimeUtils import secondsToTime, millisToTime
from deephaven.DateTimeUtils import secondsToTime, millisToTime
from deephaven.TableTools import merge
from deephaven import DynamicTableWriter
import deephaven.Types as dht
Expand Down Expand Up @@ -63,4 +65,4 @@ def thread_func():
sleep(secondsToSleep)

thread = threading.Thread(target = thread_func)
thread.start()
thread.start()
71 changes: 37 additions & 34 deletions Fit/accessFit.py
Original file line number Diff line number Diff line change
@@ -1,36 +1,39 @@
from deephaven import DynamicTableWriter, Types as dht
from deephaven.DBTimeUtils import convertDateTime
from deephaven.DateTimeUtils import convertDateTime
# Ensure fitparse is installed.
import os
os.system("pip install fitparse")
from fitparse import FitFile
try:
from fitparse import FitFile
except ImportError:
os.system("pip install fitparse")
from fitparse import FitFile

# Change to the name of the downloaded file (including any intermediate directory added to docker)
fitfile = FitFile('/data/ThursMorn.fit')
fitfile = FitFile('/data/examples/Fit/ThursMorn.fit')

# See the size of your file
records = list(fitfile.get_messages('record'))
print("Number of data points:" + str(len(records)))
print("Number of data points: {}".format(len(records)))

# Setup deephaven tables to hold results
# Heart rate
columnNames = ["Timestamp", "Heart Rate"]
columnTypes = [dht.datetime, dht.int_]
hrTableWriter = DynamicTableWriter(columnNames, columnTypes)
heartRateData = hrTableWriter.getTable()
column_names = ["Timestamp", "Heart Rate"]
hythloda marked this conversation as resolved.
Show resolved Hide resolved
column_types = [dht.datetime, dht.int_]
hr_table_writer = DynamicTableWriter(column_names, column_types)
heart_rate_data = hrTableWriter.getTable()
# Gps data
columnNames = ["Timestamp", "Enhanced Altitude m", "Enhanced Speed m/s", "GPPS Accuracy m", "Position lat semicircles", "Position long semicircles", "Speed m/s"]
columnTypes = [dht.datetime, dht.double, dht.double, dht.int_, dht.int_, dht.int_, dht.double]
gpsTableWriter = DynamicTableWriter(columnNames, columnTypes)
gpsData = gpsTableWriter.getTable()
column_names = ["Timestamp", "Enhanced Altitude m", "Enhanced Speed m/s", "GPPS Accuracy m", "Position lat semicircles", "Position long semicircles", "Speed m/s"]
hythloda marked this conversation as resolved.
Show resolved Hide resolved
column_types = [dht.datetime, dht.double, dht.double, dht.int_, dht.int_, dht.int_, dht.double]
gps_table_writer = DynamicTableWriter(column_names, column_types)
gps_data = gps_table_writer.getTable()

# Set timezone based on preferences. Fit data may not include timezone
timezone="MT"
timezone=" " + timezone # Ensure there is a blank space before timezone for later parsing.
timezone = "MT"
timezone = " " + timezone # Ensure there is a blank space before timezone for later parsing.

# Process in smaller batches first, until you are happy with the results you are working with
counter = 20
counter = len(records)+1
counter = len(records) + 1

## Debug your files by looking at individual records at a time.
# record = records[0]
Expand Down Expand Up @@ -60,25 +63,25 @@

# Go through all the data entries in this record
items=list(record)
if (mode == "hr" ):
rawHeartRate=str(items[0]).split(" ")[1]
finalHeartRate=int(rawHeartRate)
rawTime=str(items[1]).split(" ")[1]
finalTime=convertDateTime(rawTime.replace(" ", "T") + timezone)
hrTableWriter.logRow(finalTime, finalHeartRate)
if (mode == "hr"):
raw_heart_rate = str(items[0]).split()[1]
final_heart_rate = int(raw_heart_rate)
raw_time = str(items[1]).split()[1]
final_time = convertDateTime(raw_time.replace(" ", "T") + timezone)
hr_table_writer.logRow(final_time, final_heart_rate)

if (mode == "gps" ):
rawTime=str(items[6])[11:30]
finalTime=convertDateTime(rawTime.replace(" ", "T") + timezone)
if (mode == "gps"):
raw_time = str(items[6])[11:30]
final_time = convertDateTime(raw_time.replace(" ", "T") + timezone)

finalAltitude=float(str(items[0]).split(" ")[1])
finalEnhSpeed=float(str(items[1]).split(" ")[1])
finalPosLat=int(str(items[3]).split(" ")[1])
finalPosLong=int(str(items[4]).split(" ")[1])
finalSpeed=float(str(items[5]).split(" ")[1])
final_altitude = float(str(items[0]).split()[1])
final_enh_speed = float(str(items[1]).split()[1])
final_pos_lat = int(str(items[3]).split()[1])
final_pos_long = int(str(items[4]).split()[1])
final_speed = float(str(items[5]).split()[1])

rawGPSAcc=str(items[2]).split(" ")[1]
raw_gps_acc = str(items[2]).split()[1]
# If preferred, the col type for GPS could be set as String, then further processing done even when value is None
if rawGPSAcc != "None":
finalGPSAcc=int(str(items[2]).split(" ")[1])
gpsTableWriter.logRow(finalTime, finalAltitude,finalEnhSpeed,finalGPSAcc,finalPosLat,finalPosLong,finalSpeed)
if raw_gps_acc != "None":
final_gps_acc = int(str(items[2]).split()[1])
gps_table_writer.logRow(final_time, final_altitude, final_enh_speed, final_gps_acc, final_pos_lat, final_pos_long, final_speed)
2 changes: 1 addition & 1 deletion Prometheus/app.d/prometheus.py
Original file line number Diff line number Diff line change
Expand Up @@ -14,7 +14,7 @@
"""
from deephaven.TableTools import newTable, stringCol, dateTimeCol, doubleCol
from deephaven import DynamicTableWriter
from deephaven.DBTimeUtils import millisToTime
from deephaven.DateTimeUtils import millisToTime
import deephaven.Types as dht
from typing import Callable

Expand Down
35 changes: 18 additions & 17 deletions TickingHeartRate/runTickingHeartRateReplay.py
Original file line number Diff line number Diff line change
@@ -1,34 +1,35 @@
from deephaven.TableTools import readHeaderlessCsv
from deephaven import csv
hythloda marked this conversation as resolved.
Show resolved Hide resolved
from deephaven import DynamicTableWriter, Types as dht
from deephaven import DBTimeUtils
from deephaven.DBTimeUtils import autoEpochToTime
from deephaven import DateTimeUtils
import pathlib
import time
import threading

import sys

hythloda marked this conversation as resolved.
Show resolved Hide resolved
# Max number of csv files to pull in
csvFiles=500
csv_files=500

# Setup deephaven tables to hold Heart Rate results
columnNames = ["Timestamp", "Heart Rate"]
columnTypes = [dht.datetime, dht.int_]
hrTableWriter = DynamicTableWriter(columnNames, columnTypes)
heartRateData = hrTableWriter.getTable()
column_names = ["Timestamp", "Heart Rate"]
hythloda marked this conversation as resolved.
Show resolved Hide resolved
column_types = [dht.datetime, dht.int_]
hr_table_writer = DynamicTableWriter(column_names, column_types)
heart_rate_data = hr_table_writer.getTable()

# Function to log data to the dynamic table
def thread_func():
for x in range(1,csvFiles):
nextFile=("/data/csv/%d.csv" % x)
print(nextFile)
path = pathlib.Path(nextFile)
for x in range(1, csv_files):
next_file = ("/data/examples/TickingHeartRate/csv/%d.csv" % x)
print(next_file)
path = pathlib.Path(next_file)
if path.exists() and path.is_file():
nextHR=readHeaderlessCsv(nextFile).update("Timestamp=Column1", "Heart_rate=Column2").select("Timestamp", "Heart_rate")
nextRecord=nextHR.getRecord(0, "Timestamp", "Heart_rate")
timestamp=DBTimeUtils.autoEpochToTime(nextRecord[0])
hrTableWriter.logRow(timestamp, int(nextRecord[1]))
next_hr = csv.read(next_file, headless = True).update("Timestamp=Column1", "Heart_rate=Column2").select("Timestamp", "Heart_rate")
hythloda marked this conversation as resolved.
Show resolved Hide resolved
next_record = next_hr.getRecord(0, "Timestamp", "Heart_rate")
timestamp = next_record[0]
hr_table_writer.logRow(timestamp, int(next_record[1]))
time.sleep(1)
else:
print("File does not exist: " + nextFile)
print("File does not exist: " + next_file)

# Thread to log data to the dynamic table
thread = threading.Thread(target = thread_func)
Expand Down