Post fd 2024 (#65)

* add QSO classes chart, other refactoring

changed colors to non-primary TABLEAU colors
changed index on contacts data to qso_id uuid from n1mm+
etc.

* update config data

* uppercase more key values.

* factor out numpy requirement. (eliminate dependency.)

collector.py

@@ -17,7 +17,7 @@
 import dataaccess
 
 __author__ = 'Jeffrey B. Otterson, N1KDO'
-__copyright__ = 'Copyright 2016, 2017, 2019 Jeffrey B. Otterson'
+__copyright__ = 'Copyright 2016, 2017, 2019, 2024 Jeffrey B. Otterson'
 __license__ = 'Simplified BSD'
 
 BROADCAST_BUF_SIZE = 2048
@@ -82,7 +82,7 @@ class N1mmMessageParser:
     """
     this is a cheap and dirty class to parse N1MM+ broadcast messages.
     It accepts the message and returns a dict, keyed by the element name.
-    This is unsuitable for any  for any other purpose, since it throws away the
+    This is unsuitable for any other purpose, since it throws away the
     outer _contactinfo_ (or whatever) element -- instead it returns the name of
     the outer element as the value of the __messagetype__ key.
     OTOH, hopefully it is faster than using the DOM-based minidom.parse
@@ -162,14 +162,12 @@ def process_message(parser, db, cursor, operators, stations, message, seen):
     """
     Process a N1MM+ contactinfo message
     """
-    bInsert = False
     message = compress_message(message)
-    #logging.debug(message)
     data = parser.parse(message)
-    message_type = data.get('__messagetype__') or ''
-    logging.debug('Received UDP message %s' % (message_type))
-    if message_type == 'contactinfo' or message_type == 'contactreplace':
-        qso_id = data.get('ID') or '';
+    message_type = data.get('__messagetype__', '')
+    logging.debug(f'Received UDP message {message_type}')
+    if message_type in ['contactinfo', 'contactreplace']:
+        qso_id = data.get('ID', '')
 
         # If no ID tag from N1MM, generate a hash for uniqueness
         if len(qso_id) == 0:
@@ -178,24 +176,23 @@ def process_message(parser, db, cursor, operators, stations, message, seen):
            qso_id = qso_id.replace('-','')
 
         qso_timestamp = data.get('timestamp')
-        mycall = data.get('mycall')
+        mycall = data.get('mycall', '').upper()
         band = data.get('band')
-        mode = data.get('mode')
-        operator = data.get('operator')
-        station_name = data.get('StationName')
+        mode = data.get('mode', '').upper()
+        operator = data.get('operator', '').upper()
+        station_name = data.get('StationName', '').upper()
         if station_name is None or station_name == '':
-            station_name = data.get('NetBiosName')
-        station = station_name
+            station_name = data.get('NetBiosName', '')
+        station = station_name.upper()
         rx_freq = int(data.get('rxfreq')) * 10  # convert to Hz
         tx_freq = int(data.get('txfreq')) * 10
-        callsign = data.get('call')
+        callsign = data.get('call', '').upper()
         rst_sent = data.get('snt')
         rst_recv = data.get('rcv')
-        exchange = data.get('exchange1')
-        section = data.get('section')
-        comment = data.get('comment') or ''
+        exchange = data.get('exchange1', '').upper()
+        section = data.get('section', '').upper()
+        comment = data.get('comment', '')
 
-
         # convert qso_timestamp to datetime object
         timestamp = convert_timestamp(qso_timestamp)
 
@@ -204,29 +201,31 @@ def process_message(parser, db, cursor, operators, stations, message, seen):
                                            rx_freq, tx_freq, callsign, rst_sent, rst_recv,
                                            exchange, section, comment, qso_id)
     elif message_type == 'RadioInfo':
-        logging.debug('Received radioInfo message')
+        logging.debug('Received RadioInfo message')
     elif message_type == 'contactdelete':
-        qso_id = data.get('ID') or '';
+        qso_id = data.get('ID') or ''
 
         # If no ID tag from N1MM, generate a hash for uniqueness
         if len(qso_id) == 0:
-           qso_id = checksum(data)
+            qso_id = checksum(data)
         else:
-           qso_id = qso_id.replace('-','')
+            qso_id = qso_id.replace('-', '')
 
-        logging.info('Delete QSO Request with ID %s' % (qso_id))
+        logging.info(f'Delete QSO Request with ID {qso_id}')
         dataaccess.delete_contact_by_qso_id(db, cursor, qso_id)
 
     elif message_type == 'dynamicresults':
         logging.debug('Received Score message')
     else:
-        logging.warning('unknown message type {} received, ignoring.'.format(message_type))
+        logging.warning(f'unknown message type "{message_type}" received, ignoring.')
         logging.debug(message)
 
 
 def message_processor(q, event):
     global run
     logging.info('collector message_processor starting.')
+    message_count = 0
+    seen = set()
     db = sqlite3.connect(config.DATABASE_FILENAME)
     try:
         cursor = db.cursor()
@@ -235,8 +234,6 @@ def message_processor(q, event):
         operators = Operators(db, cursor)
         stations = Stations(db, cursor)
         parser = N1mmMessageParser()
-        message_count = 0
-        seen = set()
 
         thread_run = True
         while not event.is_set() and thread_run:
@@ -251,7 +248,7 @@ def message_processor(q, event):
         db.close()
         logging.info('db closed')
         run = False
-    logging.info('collector message_processor exited, {} messages collected.'.format(message_count))
+        logging.info(f'collector message_processor exited, {message_count} messages collected.')
 
 
 def main():

config.py

@@ -8,16 +8,11 @@
 """ name of database file """
 DATABASE_FILENAME = 'n1mm_view.db'
 """ Name of the event/contest """
-EVENT_NAME = 'Field Day'
+EVENT_NAME = 'N4N Field Day'
 """ start time of the event/contest in YYYY-MM-DD hh:mm:ss format """
-# EVENT_START_TIME = datetime.datetime.strptime('2019-06-25 18:00:00', '%Y-%m-%d %H:%M:%S')
-# EVENT_START_TIME = datetime.datetime.strptime('2021-06-26 18:00:00', '%Y-%m-%d %H:%M:%S')
-# EVENT_START_TIME = datetime.datetime.strptime('2022-06-25 18:00:00', '%Y-%m-%d %H:%M:%S')
+
 EVENT_START_TIME = datetime.datetime.strptime('2024-06-22 18:00:00', '%Y-%m-%d %H:%M:%S')
 """ end time of the event/contest """
-# EVENT_END_TIME = datetime.datetime.strptime('2019-06-26 17:59:59', '%Y-%m-%d %H:%M:%S')
-# EVENT_END_TIME = datetime.datetime.strptime('2021-06-27 17:59:59', '%Y-%m-%d %H:%M:%S')
-# EVENT_END_TIME = datetime.datetime.strptime('2022-06-26 17:59:59', '%Y-%m-%d %H:%M:%S')
 EVENT_END_TIME = datetime.datetime.strptime('2024-06-23 17:59:59', '%Y-%m-%d %H:%M:%S')
 """ port number used by N1MM+ for UDP broadcasts This matches the port you set in N1MM Configurator UDP logging """
 N1MM_BROADCAST_PORT = 12060
@@ -27,8 +22,7 @@
 """
 N1MM_BROADCAST_ADDRESS = '192.168.1.255'
 """ n1mm+ log file name used by replayer """
-# N1MM_LOG_FILE_NAME = 'MyClubCall-2019.s3db'
-N1MM_LOG_FILE_NAME = 'fd2024.s3db'
+N1MM_LOG_FILE_NAME = 'FD2024-N4N.s3db'
 """ QTH here is the location of your event. We mark this location with a red dot when we generate the map views."""
 """ QTH Latitude """
 QTH_LATITUDE = 34.0109629
@@ -42,12 +36,11 @@
 """
 DATA_DWELL_TIME = 60
 """ log level for apps -- one of logging.WARN, logging.INFO, logging.DEBUG """
-LOG_LEVEL = logging.INFO
+LOG_LEVEL = logging.DEBUG
 #
 """images directory, or None if not writing image files"""
-IMAGE_DIR = '/mnt/ramdisk/n1mm_view/html'
-IMAGE_WIDTH = 1920
-IMAGE_HEIGHT = 1080
+IMAGE_DIR = None  #  '/mnt/ramdisk/n1mm_view/html'
+
 """ set HEADLESS True to not open graphics window. This is for using only the Apache option."""
 HEADLESS = False
 

dashboard.py

@@ -30,9 +30,10 @@
 QSO_STATIONS_PIE_INDEX = 5
 QSO_BANDS_PIE_INDEX = 6
 QSO_MODES_PIE_INDEX = 7
-QSO_RATE_CHART_IMAGE_INDEX = 8
-SECTIONS_WORKED_MAP_INDEX = 9
-IMAGE_COUNT = 10
+QSO_CLASSES_PIE_INDEX = 8
+QSO_RATE_CHART_IMAGE_INDEX = 9
+SECTIONS_WORKED_MAP_INDEX = 10
+IMAGE_COUNT = 11
 
 IMAGE_MESSAGE = 1
 CRAWL_MESSAGE = 2
@@ -57,6 +58,7 @@ def load_data(size, q, last_qso_timestamp):
     operator_qso_rates = []
     qsos_per_hour = []
     qsos_by_section = {}
+    qso_classes = []
 
     db = None
     data_updated = False
@@ -85,6 +87,9 @@ def load_data(size, q, last_qso_timestamp):
             # get something else.
             qso_band_modes = dataaccess.get_qso_band_modes(cursor)
 
+            # load qso exchange data: what class are the other stations?
+            qso_classes = dataaccess.get_qso_classes(cursor)
+
             # load QSOs per Hour by Operator
             operator_qso_rates = dataaccess.get_qsos_per_hour_per_operator(cursor, last_qso_time)
 
@@ -150,7 +155,12 @@ def load_data(size, q, last_qso_timestamp):
         except Exception as e:
             logging.exception(e)
         try:
-            image_data, image_size = graphics.qso_rates_chart(size, qsos_per_hour)
+            image_data, image_size = graphics.qso_classes_graph(size, qso_classes)
+            enqueue_image(q, QSO_CLASSES_PIE_INDEX, image_data, image_size)
+        except Exception as e:
+            logging.exception(e)
+        try:
+            image_data, image_size = graphics.qso_rates_graph(size, qsos_per_hour)
             enqueue_image(q, QSO_RATE_CHART_IMAGE_INDEX, image_data, image_size)
         except Exception as e:
             logging.exception(e)

dataaccess.py

@@ -47,7 +47,7 @@ def create_tables(db, cursor):
                    '     exchange char(4),\n'
                    '     section char(4),\n'
                    '     comment TEXT,\n'
-                   '     qso_id  char(32) UNIQUE NOT NULL);')
+                   '     qso_id  char(32) PRIMARY KEY NOT NULL);')  # this is primary key to speed up Update & Delete
     cursor.execute('CREATE INDEX IF NOT EXISTS qso_log_band_id ON qso_log(band_id);')
     cursor.execute('CREATE INDEX IF NOT EXISTS qso_log_mode_id ON qso_log(mode_id);')
     cursor.execute('CREATE INDEX IF NOT EXISTS qso_log_operator_id ON qso_log(operator_id);')
@@ -56,6 +56,7 @@ def create_tables(db, cursor):
     cursor.execute('CREATE INDEX IF NOT EXISTS qso_log_qso_id ON qso_log(qso_id);')
     db.commit()
 
+
 def record_contact_combined(db, cursor, operators, stations,
                    timestamp, mycall, band, mode, operator, station,
                    rx_freq, tx_freq, callsign, rst_sent, rst_recv,
@@ -92,7 +93,6 @@ def record_contact_combined(db, cursor, operators, stations,
         logging.warning('[dataaccess] Insert Failed: %s\nError: %s' % (qso_id, str(err)))
 
 
-
 def record_contact(db, cursor, operators, stations,
                    timestamp, mycall, band, mode, operator, station,
                    rx_freq, tx_freq, callsign, rst_sent, rst_recv,
@@ -128,6 +128,7 @@ def record_contact(db, cursor, operators, stations,
     except Exception as err:
         logging.warning('[dataaccess] Insert Failed: %s\nError: %s' % (qso_id, str(err)))
 
+
 def update_contact(db, cursor, operators, stations,
                    timestamp, mycall, band, mode, operator, station,
                    rx_freq, tx_freq, callsign, rst_sent, rst_recv,
@@ -163,6 +164,7 @@ def update_contact(db, cursor, operators, stations,
     except Exception as err:
         logging.warning('[dataaccess] Update Failed: %s\nError: %s' % (qso_id, str(err)))
 
+
 def delete_contact(db, cursor, timestamp, station, callsign):
     """
     Delete the results of a delete in N1MM
@@ -180,6 +182,7 @@ def delete_contact(db, cursor, timestamp, station, callsign):
         logging.exception('[dataaccess] Exception deleting contact from db.')
         return ''
 
+
 def delete_contact_by_qso_id(db, cursor, qso_id):
     """
     Delete the results of a delete in N1MM
@@ -262,6 +265,13 @@ def get_qso_band_modes(cursor):
     return qso_band_modes
 
 
+def get_qso_classes(cursor):
+    cursor.execute('SELECT COUNT(*), exchange FROM qso_log group by exchange;')
+    exchanges = []
+    for row in cursor:
+        exchanges.append((row[0], row[1]))
+    return exchanges
+
 def get_qsos_per_hour_per_band(cursor):
     qsos_per_hour = []
     qsos_by_band = [0] * constants.Bands.count()