diff --git a/.vscode/settings.json b/.vscode/settings.json index c60ab30..9ea6d62 100644 --- a/.vscode/settings.json +++ b/.vscode/settings.json @@ -5,5 +5,5 @@ "python3.8InterpreterPath": "/Library/Frameworks/Python.framework/Versions/3.8/bin/python3.8", "modulename": "${workspaceFolderBasename}", "distname": "${workspaceFolderBasename}", - "moduleversion": "1.0.31" + "moduleversion": "1.0.32" } \ No newline at end of file diff --git a/README.md b/README.md index 95d66fb..49935e7 100644 --- a/README.md +++ b/README.md @@ -318,15 +318,24 @@ For help and full list of optional arguments, type: ### EXPERIMENTAL -Provides a simple simulation of a GNSS serial stream by generating synthetic UBX or NMEA messages based on parameters defined in a json configuration file. Can simulate a motion vector based on a specified course over ground and speed. +Provides a simple simulation of a GNSS serial stream by generating synthetic UBX or NMEA messages based on parameters defined in a json configuration file. Can simulate a motion vector based on a specified course over ground and speed. Location of configuration file can be set via environment variable `UBXSIMULATOR`. -Example usage:: +Example usage: + +```shell +ubxsimulator --simconfigfile "/home/myuser/ubxsimulator.json" --interval 1000 --timeout 3 --verbosity 3 +``` ```python -from pygnssutils import UBXSimulator +from os import getenv +from pygnssutils import UBXSimulator, UBXSIMULATOR from pyubx2 import UBXReader -with UBXSimulator(configfile="/home/myuser/ubxsimulator.json", interval=1, timeout=3) as stream: +with UBXSimulator( + configfile=getenv(UBXSIMULATOR, "/home/myuser/ubxsimulator.json"), + interval=1000, + timeout=3, +) as stream: ubr = UBXReader(stream) for raw, parsed in ubr: print(parsed) diff --git a/RELEASE_NOTES.md b/RELEASE_NOTES.md index df0c495..5130e91 100644 --- a/RELEASE_NOTES.md +++ b/RELEASE_NOTES.md @@ -1,5 +1,34 @@ # pygnssutils Release Notes +### RELEASE 1.0.32 + +ENHANCEMENTS: + +1. Add configuration file option to all CLI utilities via `-C` or `--config` argument. Default location of configuration file can be specified in environment variable `{utility}_CONF` e.g. `GNSSDUMP_CONF`, `GNSSNTRIPCLIENT_CONF`, etc. Config files are text files containing key-value pairs which mirror the existing CLI arguments, e.g. +```shell +gnssdump -C gnssdump.conf +``` +where gnssdump.conf contains... + + filename=pygpsdata-MIXED3.log + verbosity=3 + format=2 + clioutput=1 + output=testfile.bin + +is equivalent to: +```shell +gnssdump --filename pygpsdata-MIXED3.log --verbosity 3 --format 2 --clioutput 1 --output testfile.bin +``` +2. Streamline logging. CLI usage unchanged; to use pygnssutils logging within calling application, invoke `logging.getLogger("pygnssutils")` in calling module. +3. Internal enhancements to experimental UBXSimulator to add close() and in_waiting() methods; recognise incoming RTCM data. +4. GGA message sent to NTRIP Caster in GGALIVE mode will include additional live attributes (siv, hdop, quality, diffage, diffstation). Thanks to @yydgis for contribution. + +FIXES: + +1. gnssntripclient - update HTTP GET request for better NTRIP 2.0 compliance +1. issue with delay on gnssntripclient retry limit + ### RELEASE 1.0.31 ENHANCEMENTS: diff --git a/examples/gnssapp.py b/examples/gnssapp.py index 16de03a..40a7772 100644 --- a/examples/gnssapp.py +++ b/examples/gnssapp.py @@ -7,7 +7,8 @@ data from a receiver until the stop Event is set or stop() method invoked. Assumes receiver is connected via serial USB or UART1 port. -The app also implements basic methods needed by certain pygnssutils classes. +The app also implements public methods used by certain pygnssutils classes: +- get_coordinates() Optional keyword arguments: @@ -29,15 +30,14 @@ :license: BSD 3-Clause """ -# pylint: disable=invalid-name, too-many-instance-attributes - from argparse import ArgumentDefaultsHelpFormatter, ArgumentParser +from logging import getLogger from queue import Empty, Queue from threading import Event, Thread from time import sleep from pynmeagps import NMEAMessageError, NMEAParseError -from pyrtcm import RTCMMessage, RTCMMessageError, RTCMParseError +from pyrtcm import RTCMMessageError, RTCMParseError from pyubx2 import ( CARRSOLN, FIXTYPE, @@ -51,8 +51,32 @@ ) from serial import Serial +from pygnssutils import UBXSIMULATOR, VERBOSITY_MEDIUM, UBXSimulator, set_common_args + DISCONNECTED = 0 CONNECTED = 1 +FIXTYPE_GGA = { + 0: "NO FIX", + 1: "3D", + 2: "3D", + 4: "RTK FIXED", + 5: "RTK FLOAT", + 6: "DR", +} +DIFFAGE_PVT = { + 0: 0, + 1: 1, + 2: 2, + 3: 5, + 4: 10, + 5: 15, + 6: 20, + 7: 30, + 8: 45, + 9: 60, + 10: 90, + 11: 120, +} class GNSSSkeletonApp: @@ -72,13 +96,15 @@ def __init__( :param Event stopevent: stop event """ + self.verbosity = kwargs.get("verbosity", VERBOSITY_MEDIUM) + # configure logger with name "pygnssutils" in calling module + self.logger = getLogger("pygnssutils.gnssapp") self.port = port self.baudrate = baudrate self.timeout = timeout self.stopevent = stopevent self.recvqueue = kwargs.get("recvqueue", None) self.sendqueue = kwargs.get("sendqueue", None) - self.verbosity = kwargs.get("verbosity", 1) self.enableubx = kwargs.get("enableubx", True) self.showstatus = kwargs.get("showstatus", True) self.stream = None @@ -90,6 +116,11 @@ def __init__( self.alt = 0 self.sep = 0 self.hacc = 0 + self.sip = 0 + self.fix = "NO FIX" + self.hdop = 0 + self.diffage = 0 + self.diffstation = 0 def __enter__(self): """ @@ -112,9 +143,14 @@ def run(self): Run GNSS reader/writer. """ + self.logger.info("Starting GNSS reader/writer...") self.enable_ubx(self.enableubx) - self.stream = Serial(self.port, self.baudrate, timeout=self.timeout) + if self.port.upper() == UBXSIMULATOR: + self.stream = UBXSimulator() + self.stream.start() + else: + self.stream = Serial(self.port, self.baudrate, timeout=self.timeout) self.connected = CONNECTED self.stopevent.clear() @@ -139,6 +175,7 @@ def stop(self): self.connected = DISCONNECTED if self.stream is not None: self.stream.close() + self.logger.info("GNSS reader/writer stopped") def _read_loop( self, stream: Serial, stopevent: Event, recvqueue: Queue, sendqueue: Queue @@ -163,10 +200,8 @@ def _read_loop( raw_data, parsed_data = ubr.read() if parsed_data: self._extract_data(parsed_data) - if self.verbosity == 1: - print(f"GNSS>> {parsed_data.identity}") - elif self.verbosity == 2: - print(parsed_data) + self.logger.info(f"GNSS>> {parsed_data.identity}") + self.logger.debug(parsed_data) if recvqueue is not None: # place data on receive queue recvqueue.put((raw_data, parsed_data)) @@ -182,7 +217,7 @@ def _read_loop( RTCMMessageError, RTCMParseError, ) as err: - print(f"Error parsing data stream {err}") + self.logger.critical(f"Error parsing data stream {err}") continue def _extract_data(self, parsed_data: object): @@ -193,17 +228,30 @@ def _extract_data(self, parsed_data: object): """ if hasattr(parsed_data, "fixType"): - self.fix = FIXTYPE[parsed_data.fixType] + self.fix = FIXTYPE.get(parsed_data.fixType, "NO FIX") if hasattr(parsed_data, "carrSoln"): - self.fix = f"{self.fix} {CARRSOLN[parsed_data.carrSoln]}" + if parsed_data.carrSoln != 0: # NO RTK + self.fix = f"{CARRSOLN.get(parsed_data.carrSoln, self.fix)}" + if hasattr(parsed_data, "quality"): + self.fix = FIXTYPE_GGA.get(parsed_data.quality, "NO FIX") if hasattr(parsed_data, "numSV"): - self.siv = parsed_data.numSV + self.sip = parsed_data.numSV if hasattr(parsed_data, "lat"): self.lat = parsed_data.lat if hasattr(parsed_data, "lon"): self.lon = parsed_data.lon if hasattr(parsed_data, "alt"): self.alt = parsed_data.alt + if hasattr(parsed_data, "HDOP"): + self.hdop = parsed_data.HDOP + if hasattr(parsed_data, "hDOP"): + self.hdop = parsed_data.hDOP + if hasattr(parsed_data, "diffAge"): + self.diffage = parsed_data.diffAge + if hasattr(parsed_data, "lastCorrectionAge"): + self.diffage = DIFFAGE_PVT.get(parsed_data.lastCorrectionAge, 0) + if hasattr(parsed_data, "diffStation"): + self.diffstation = parsed_data.diffStation if hasattr(parsed_data, "hMSL"): # UBX hMSL is in mm self.alt = parsed_data.hMSL / 1000 if hasattr(parsed_data, "sep"): @@ -214,9 +262,9 @@ def _extract_data(self, parsed_data: object): unit = 1 if parsed_data.identity == "PUBX00" else 1000 self.hacc = parsed_data.hAcc / unit if self.showstatus: - print( - f"fix {self.fix}, siv {self.siv}, lat {self.lat},", - f"lon {self.lon}, alt {self.alt:.3f} m, hAcc {self.hacc:.3f} m", + self.logger.info( + f"fix {self.fix}, sip {self.sip}, lat {self.lat}, " + f"lon {self.lon}, alt {self.alt:.3f} m, hAcc {self.hacc:.3f} m" ) def _send_data(self, stream: Serial, sendqueue: Queue): @@ -233,10 +281,8 @@ def _send_data(self, stream: Serial, sendqueue: Queue): while not sendqueue.empty(): data = sendqueue.get(False) raw_data, parsed_data = data - if self.verbosity == 1: - print(f"GNSS<< {parsed_data.identity}") - elif self.verbosity == 2: - print(parsed_data) + self.logger.info(f"GNSS<< {parsed_data.identity}") + self.logger.debug(f"{parsed_data}") stream.write(raw_data) sendqueue.task_done() except Empty: @@ -246,6 +292,8 @@ def enable_ubx(self, enable: bool): """ Enable UBX output and suppress NMEA. + NB: only works for Gen 9+ receivers e.g. F9P. + :param bool enable: enable UBX and suppress NMEA output """ @@ -263,63 +311,50 @@ def enable_ubx(self, enable: bool): msg = UBXMessage.config_set(layers, transaction, cfg_data) self.sendqueue.put((msg.serialize(), msg)) - def get_coordinates(self) -> tuple: + def get_coordinates(self) -> dict: """ Return current receiver navigation solution. (method needed by certain pygnssutils classes) - :return: tuple of (connection status, lat, lon, alt and sep) - :rtype: tuple + :return: dict + :rtype: dict """ - return (self.connected, self.lat, self.lon, self.alt, self.sep) - - -if __name__ == "__main__": - arp = ArgumentParser( - formatter_class=ArgumentDefaultsHelpFormatter, - ) - arp.add_argument( - "-P", "--port", required=False, help="Serial port", default="/dev/ttyACM1" - ) - arp.add_argument( - "-B", "--baudrate", required=False, help="Baud rate", default=38400, type=int - ) - arp.add_argument( - "-T", "--timeout", required=False, help="Timeout in secs", default=3, type=float - ) - arp.add_argument( - "--verbosity", - required=False, - help="Verbosity", - default=1, - choices=[0, 1, 2], - type=int, - ) - arp.add_argument( - "--enableubx", required=False, help="Enable UBX output", default=1, type=int - ) - arp.add_argument( - "--showstatus", required=False, help="Show GNSS status", default=1, type=int - ) + return { + "connection": self.connected, + "lat": self.lat, + "lon": self.lon, + "alt": self.alt, + "sep": self.sep, + "sip": self.sip, + "fix": self.fix, + "hdop": self.hdop, + "diffage": self.diffage, + "diffstation": self.diffstation, + } + + +def main(**kwargs): + """ + Main routine - CLI entry point. + """ - args = arp.parse_args() recv_queue = Queue() # set to None to print data to stdout send_queue = Queue() stop_event = Event() try: - print("Starting GNSS reader/writer...\n") + with GNSSSkeletonApp( - args.port, - int(args.baudrate), - float(args.timeout), + kwargs.get("port", "/dev/ttyACM0"), + int(kwargs.get("baudrate", 38400)), + float(kwargs.get("timeout", 3)), stop_event, recvqueue=recv_queue, sendqueue=send_queue, - verbosity=int(args.verbosity), - enableubx=int(args.enableubx), - showstatus=int(args.showstatus), + verbosity=int(kwargs.get("verbosity", VERBOSITY_MEDIUM)), + enableubx=int(kwargs.get("enableubx", 1)), + showstatus=int(kwargs.get("showstatus", 1)), ) as gna: gna.run() while True: @@ -327,4 +362,28 @@ def get_coordinates(self) -> tuple: except KeyboardInterrupt: stop_event.set() - print("Terminated by user") + + +if __name__ == "__main__": + + ap = ArgumentParser( + formatter_class=ArgumentDefaultsHelpFormatter, + ) + ap.add_argument( + "-P", "--port", required=False, help="Serial port", default="/dev/ttyACM1" + ) + ap.add_argument( + "-B", "--baudrate", required=False, help="Baud rate", default=38400, type=int + ) + ap.add_argument( + "-T", "--timeout", required=False, help="Timeout in secs", default=3, type=float + ) + ap.add_argument( + "--enableubx", required=False, help="Enable UBX output", default=1, type=int + ) + ap.add_argument( + "--showstatus", required=False, help="Show GNSS status", default=1, type=int + ) + args = set_common_args(ap) + + main(**args) diff --git a/examples/gnssdump.conf b/examples/gnssdump.conf new file mode 100644 index 0000000..9ea9277 --- /dev/null +++ b/examples/gnssdump.conf @@ -0,0 +1,5 @@ +filename=pygpsdata-MIXED3.log +verbosity=3 +format=2 +clioutput=1 +output=testfile.bin \ No newline at end of file diff --git a/examples/rtcm_ntrip_client.py b/examples/rtcm_ntrip_client.py index 6092af8..4fd29c5 100644 --- a/examples/rtcm_ntrip_client.py +++ b/examples/rtcm_ntrip_client.py @@ -22,11 +22,12 @@ class from pygnssutils library. Can be used with any :license: BSD 3-Clause """ +from logging import getLogger from os import getenv from sys import argv from time import sleep -from pygnssutils import GNSSNTRIPClient +from pygnssutils import VERBOSITY_HIGH, GNSSNTRIPClient, set_logging DEFAULT_PORT = 2101 @@ -36,6 +37,8 @@ def main(**kwargs): Main routine. """ + logger = getLogger("pygnssutils.gnssntripclient") + set_logging(logger, VERBOSITY_HIGH) server = kwargs.get("server", "rtk2go.com") port = int(kwargs.get("port", DEFAULT_PORT)) mountpoint = kwargs.get("mountpoint", "") @@ -48,7 +51,7 @@ def main(**kwargs): https = 1 if port == 443 else 0 - print( + logger.info( f"RTCM NTRIP Client started, writing output to {outfile}... Press CTRL-C to terminate." ) gnc.run( @@ -67,7 +70,7 @@ def main(**kwargs): while True: sleep(3) except KeyboardInterrupt: - print("RTCM NTRIP Client terminated by User") + logger.info("RTCM NTRIP Client terminated by User") if __name__ == "__main__": diff --git a/examples/rtk_example.py b/examples/rtk_example.py index 087079f..d5ccd9f 100644 --- a/examples/rtk_example.py +++ b/examples/rtk_example.py @@ -18,7 +18,7 @@ connected to a local serial port (USB or UART1). GNSSNTRIPClient receives RTCM3 or SPARTN data from the NTRIP -caster and outputs it to a message queue. An example +caster and outputs it to a message queue. A basic GNSSSkeletonApp class reads data from this queue and sends it to the receiver, while reading and parsing data from the receiver and printing it to the terminal. @@ -26,6 +26,8 @@ GNSSNtripClient optionally sends NMEA GGA position sentences to the caster at a prescribed interval, using either fixed reference coordinates or live coordinates from the receiver. +For NTRIP 2.0 protocol, the first GGA sentence is embedded +in the HTTP GET request header. NB: Some NTRIP casters may stop sending RTK data after a while if they're not receiving legitimate NMEA GGA position updates @@ -40,16 +42,26 @@ # pylint: disable=invalid-name +from logging import getLogger +from queue import Empty, Queue from sys import argv -from queue import Queue, Empty from threading import Event from time import sleep -from pygnssutils import VERBOSITY_LOW, GNSSNTRIPClient from gnssapp import GNSSSkeletonApp +from pygnssutils import ( + VERBOSITY_DEBUG, + VERBOSITY_HIGH, + VERBOSITY_MEDIUM, + GNSSNTRIPClient, + set_logging, +) + CONNECTED = 1 +logger = getLogger("pygnssutils") + def main(**kwargs): """ @@ -57,7 +69,7 @@ def main(**kwargs): """ # GNSS receiver serial port parameters - AMEND AS REQUIRED: - SERIAL_PORT = "/dev/ttyACM0" + SERIAL_PORT = "/dev/ttyACM0" # use "UBXSIMULATOR" to use dummy UBX serial stream BAUDRATE = 38400 TIMEOUT = 10 @@ -86,10 +98,12 @@ def main(**kwargs): recv_queue = Queue() # data from receiver placed on this queue send_queue = Queue() # data to receiver placed on this queue stop_event = Event() - verbosity = 0 # 0 - no output, 1 - print identities, 2 - print full message + + set_logging(logger, VERBOSITY_HIGH) + mylogger = getLogger("pygnssutils.rtk_example") try: - print(f"Starting GNSS reader/writer on {SERIAL_PORT} @ {BAUDRATE}...\n") + mylogger.info(f"Starting GNSS reader/writer on {SERIAL_PORT} @ {BAUDRATE}...\n") with GNSSSkeletonApp( SERIAL_PORT, BAUDRATE, @@ -97,15 +111,14 @@ def main(**kwargs): stopevent=stop_event, recvqueue=recv_queue, sendqueue=send_queue, - verbosity=verbosity, enableubx=True, showstatus=True, ) as gna: gna.run() sleep(2) # wait for receiver to output at least 1 navigation solution - print(f"Starting NTRIP client on {NTRIP_SERVER}:{NTRIP_PORT}...\n") - with GNSSNTRIPClient(gna, verbosity=VERBOSITY_LOW) as gnc: + mylogger.info(f"Starting NTRIP client on {NTRIP_SERVER}:{NTRIP_PORT}...\n") + with GNSSNTRIPClient(gna) as gnc: streaming = gnc.run( ipprot=IPPROT, server=NTRIP_SERVER, @@ -134,10 +147,6 @@ def main(**kwargs): try: while not recv_queue.empty(): (_, parsed_data) = recv_queue.get(False) - if verbosity == 1: - print(f"GNSS>> {parsed_data.identity}") - elif verbosity == 2: - print(parsed_data) recv_queue.task_done() except Empty: pass @@ -146,7 +155,7 @@ def main(**kwargs): except KeyboardInterrupt: stop_event.set() - print("Terminated by user") + mylogger.info("Terminated by user") if __name__ == "__main__": diff --git a/examples/spartn_decrypt.py b/examples/spartn_decrypt.py index fe0b525..82389c9 100644 --- a/examples/spartn_decrypt.py +++ b/examples/spartn_decrypt.py @@ -35,7 +35,7 @@ from os import getenv from sys import argv -from pyspartn import SPARTNReader, ERRIGNORE +from pyspartn import ERRIGNORE, SPARTNReader def main(**kwargs): @@ -43,6 +43,8 @@ def main(**kwargs): Read, decrypt and decode SPARTN log file. """ + # pylint: disable=protected-access + infile = kwargs.get("infile", "d9s_spartn_data.bin") key = kwargs.get( "key", getenv("MQTTKEY", default="bc75cdd919406d61c3df9e26c2f7e77a") diff --git a/examples/spartn_mqtt_client.py b/examples/spartn_mqtt_client.py index 41d985f..a07dcd2 100644 --- a/examples/spartn_mqtt_client.py +++ b/examples/spartn_mqtt_client.py @@ -28,13 +28,13 @@ class from pygnssutils library. Can be used with the """ from datetime import datetime, timezone -from os import path, getenv +from logging import getLogger +from os import getenv, path from pathlib import Path from sys import argv from time import sleep -from pygnssutils import GNSSMQTTClient - +from pygnssutils import VERBOSITY_HIGH, GNSSMQTTClient, set_logging SERVER = "pp.services.u-blox.com" PORT = 8883 @@ -45,6 +45,8 @@ def main(**kwargs): Main routine. """ + logger = getLogger("pygnssutils.gnssmqttclient") + set_logging(logger, VERBOSITY_HIGH) clientid = kwargs.get("clientid", getenv("MQTTCLIENTID", "")) region = kwargs.get("region", "eu") decode = int(kwargs.get("decode", 0)) @@ -54,7 +56,7 @@ def main(**kwargs): with open(outfile, "wb") as out: gmc = GNSSMQTTClient() - print(f"SPARTN MQTT Client started, writing output to {outfile}...") + logger.info(f"SPARTN MQTT Client started, writing output to {outfile}...") gmc.start( server=SERVER, port=PORT, @@ -76,10 +78,11 @@ def main(**kwargs): while True: sleep(3) except KeyboardInterrupt: - print("SPARTN MQTT Client terminated by User") - print( - f"To decrypt the contents of the output file {outfile} using pyspartn,", - f"use kwargs: decode=True, key=key_supplied_by_service_provider, basedate={repr(datetime.now(timezone.utc))}", + logger.info("SPARTN MQTT Client terminated by User") + logger.info( + f"To decrypt the contents of the output file {outfile} using pyspartn, " + f"use kwargs: decode=True, key=key_supplied_by_service_provider, " + "basedate={repr(datetime.now(timezone.utc))}", ) diff --git a/examples/spartn_ntrip_client.py b/examples/spartn_ntrip_client.py index 5d5660e..ca9c05a 100644 --- a/examples/spartn_ntrip_client.py +++ b/examples/spartn_ntrip_client.py @@ -28,12 +28,13 @@ class from pygnssutils library. Can be used with the :license: BSD 3-Clause """ -from os import path, getenv +from datetime import datetime, timezone +from logging import getLogger +from os import getenv from sys import argv from time import sleep -from datetime import datetime, timezone -from pygnssutils import GNSSNTRIPClient +from pygnssutils import VERBOSITY_HIGH, GNSSNTRIPClient, set_logging SERVER = "ppntrip.services.u-blox.com" PORT = 2102 @@ -45,6 +46,8 @@ def main(**kwargs): Main routine. """ + logger = getLogger("pygnssutils.gnssntripclient") + set_logging(logger, VERBOSITY_HIGH) user = kwargs.get("user", getenv("PYGPSCLIENT_USER", "user")) password = kwargs.get("password", getenv("PYGPSCLIENT_PASSWORD", "password")) decode = int(kwargs.get("decode", 0)) @@ -55,8 +58,9 @@ def main(**kwargs): with open(outfile, "wb") as out: gnc = GNSSNTRIPClient() - print( - f"SPARTN NTRIP Client started, writing output to {outfile}... Press CTRL-C to terminate." + logger.info( + "SPARTN NTRIP Client started, writing output " + f"to {outfile}... Press CTRL-C to terminate." ) gnc.run( server=SERVER, @@ -77,7 +81,7 @@ def main(**kwargs): while True: sleep(3) except KeyboardInterrupt: - print("SPARTN NTRIP Client terminated by User") + logger.info("SPARTN NTRIP Client terminated by User") if __name__ == "__main__": diff --git a/examples/ubxsimulator.json b/examples/ubxsimulator.json index f078107..1456130 100644 --- a/examples/ubxsimulator.json +++ b/examples/ubxsimulator.json @@ -1,21 +1,21 @@ { - "interval": 1, + "interval": 1000, "timeout": 3, - "logfile": "/home/myuser/ubxsimulator.log", + "logfile": "/Users/steve/ubxsimulator.log", "simVector": 1, "global": { - "lat": 52.474715, - "lon": 13.403288, + "lat": 52.477311, + "lon": 13.391426, "alt": 50.034, "height": 47494, "sep": 2.54, "hMSL": 50034, "NS": "N", "EW": "E", - "spd": 26.0693, - "cog": 125, - "gSpeed": 13411, - "headMot": 125 + "gSpeed": 100000, + "headMot": 126, + "spd": 194.3844, + "cog": 126 }, "ubxmessages": [ { @@ -31,7 +31,10 @@ "gnssFixOk": 1, "pDOP": 0.843, "hAcc": 585, - "vAcc": 543 + "vAcc": 543, + "diffSoln": 1, + "carrSoln": 1, + "lastCorrectionAge": 5 } }, { @@ -241,9 +244,11 @@ "navStatus": "V", "sepUnit": "M", "altUnit": "M", - "quality": 2, + "quality": 5, "numSV": 12, - "HDOP": 0.9873476 + "HDOP": 0.9873476, + "diffAge": 10, + "diffStation": 2746 } }, { diff --git a/pyproject.toml b/pyproject.toml index 983abb7..b8434a8 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -7,7 +7,7 @@ name = "pygnssutils" authors = [{ name = "semuadmin", email = "semuadmin@semuconsulting.com" }] maintainers = [{ name = "semuadmin", email = "semuadmin@semuconsulting.com" }] description = "GNSS Command Line Utilities" -version = "1.0.31" +version = "1.0.32" license = { file = "LICENSE" } readme = "README.md" requires-python = ">=3.8" diff --git a/src/pygnssutils/_version.py b/src/pygnssutils/_version.py index 40552ef..a2f6afe 100644 --- a/src/pygnssutils/_version.py +++ b/src/pygnssutils/_version.py @@ -8,4 +8,4 @@ :license: BSD 3-Clause """ -__version__ = "1.0.31" +__version__ = "1.0.32" diff --git a/src/pygnssutils/globals.py b/src/pygnssutils/globals.py index 477f843..d9f89a1 100644 --- a/src/pygnssutils/globals.py +++ b/src/pygnssutils/globals.py @@ -11,7 +11,6 @@ CLIAPP = "CLI" OUTPORT = 50010 OUTPORT_NTRIP = 2101 -DEFAULT_TLS_PORTS = (443, 2102) MIN_NMEA_PAYLOAD = 3 # minimum viable length of NMEA message payload EARTH_RADIUS = 6371 # km DEFAULT_BUFSIZE = 4096 # buffer size for NTRIP client @@ -26,11 +25,13 @@ OUTPUT_FILE = 1 OUTPUT_SERIAL = 2 OUTPUT_SOCKET = 3 +OUTPUT_HANDLER = 4 VERBOSITY_CRITICAL = -1 VERBOSITY_LOW = 0 VERBOSITY_MEDIUM = 1 VERBOSITY_HIGH = 2 VERBOSITY_DEBUG = 3 +UBXSIMULATOR = "UBXSIMULATOR" LOGGING_LEVELS = { VERBOSITY_CRITICAL: "CRITICAL", VERBOSITY_LOW: "ERROR", @@ -59,13 +60,16 @@ } FIXES = { + "NO FIX": 0, + "TIME ONLY": 0, + "2D": 1, "3D": 1, - "2D": 2, - "RTK FIXED": 4, - "RTK FLOAT": 5, + "GPS + DR": 1, + "GNSS+DR": 1, "RTK": 5, + "RTK FLOAT": 5, + "RTK FIXED": 4, "DR": 6, - "NO FIX": 0, } HTTPCODES = { @@ -77,9 +81,14 @@ 405: "Method Not Allowed", 406: "Not Acceptable", 408: "Request Timeout", + 409: "Conflict", + 429: "Too Many Requests", + 500: "Internal Server Error", + 501: "Not Implemented", + 503: "Service Unavailable", } -HTTPERR = [f"{i[0]} {i[1]}" for i in HTTPCODES.items() if 400 <= i[0] <= 499] +HTTPERR = [f"{i[0]} {i[1]}" for i in HTTPCODES.items() if 400 <= i[0] <= 599] # ranges for ubxsetrate CLI ALLNMEA = "allnmea" diff --git a/src/pygnssutils/gnssdump_cli.py b/src/pygnssutils/gnssdump_cli.py index 4a0e93e..76bacc7 100644 --- a/src/pygnssutils/gnssdump_cli.py +++ b/src/pygnssutils/gnssdump_cli.py @@ -11,18 +11,39 @@ """ from argparse import ArgumentDefaultsHelpFormatter, ArgumentParser +from queue import Queue +from threading import Thread + +from serial import Serial from pygnssutils._version import __version__ as VERSION from pygnssutils.globals import ( CLIAPP, EPILOG, - VERBOSITY_CRITICAL, - VERBOSITY_DEBUG, - VERBOSITY_HIGH, - VERBOSITY_LOW, - VERBOSITY_MEDIUM, + FORMAT_BINARY, + FORMAT_HEX, + FORMAT_HEXTABLE, + FORMAT_JSON, + FORMAT_PARSED, + FORMAT_PARSEDSTRING, + OUTPUT_FILE, + OUTPUT_HANDLER, + OUTPUT_NONE, + OUTPUT_SERIAL, + OUTPUT_SOCKET, ) from pygnssutils.gnssstreamer import GNSSStreamer +from pygnssutils.helpers import set_common_args +from pygnssutils.socket_server import runserver + + +def runclient(**kwargs): + """ + Start GNSSStreamer with CLI parameters. + """ + + with GNSSStreamer(CLIAPP, **kwargs) as gns: + gns.run() def main(): @@ -74,11 +95,16 @@ def main(): "--format", required=False, help=( - "Output format 1 = parsed, 2 = binary, 4 = hex, 8 = tabulated hex, " - "16 = parsed as string, 32 = JSON (can be OR'd)" + f"{FORMAT_PARSED} - parsed as object; " + f"{FORMAT_BINARY} - binary (raw); " + f"{FORMAT_HEX} - hexadecimal; " + f"{FORMAT_HEXTABLE} - tabular hexadecimal; " + f"{FORMAT_PARSEDSTRING} - parsed as string; " + f"{FORMAT_JSON} - JSON. " + f"Options can be OR'd e.g. {FORMAT_PARSED} | {FORMAT_HEXTABLE}." ), type=int, - default=1, + default=FORMAT_PARSED, ) ap.add_argument( "--validate", @@ -138,55 +164,70 @@ def main(): default=0, ) ap.add_argument( - "--verbosity", + "--clioutput", required=False, help=( - f"Log message verbosity " - f"{VERBOSITY_CRITICAL} = critical, " - f"{VERBOSITY_LOW} = low (error), " - f"{VERBOSITY_MEDIUM} = medium (warning), " - f"{VERBOSITY_HIGH} = high (info), {VERBOSITY_DEBUG} = debug" + f"CLI output type {OUTPUT_NONE} = none, " + f"{OUTPUT_FILE} = binary file, " + f"{OUTPUT_SERIAL} = serial port, " + f"{OUTPUT_SOCKET} = TCP socket server, " + f"{OUTPUT_HANDLER} = evaluable Python expression" ), type=int, choices=[ - VERBOSITY_CRITICAL, - VERBOSITY_LOW, - VERBOSITY_MEDIUM, - VERBOSITY_HIGH, - VERBOSITY_DEBUG, + OUTPUT_NONE, + OUTPUT_FILE, + OUTPUT_SERIAL, + OUTPUT_SOCKET, + OUTPUT_HANDLER, ], - default=VERBOSITY_HIGH, + default=OUTPUT_NONE, ) ap.add_argument( - "--outfile", + "--output", required=False, - help="Fully qualified path to output file", + help=( + f"Output medium as formatted string. " + f"If clioutput = {OUTPUT_FILE}, format = file name (e.g. '/home/myuser/ubxdata.ubx'); " + f"If clioutput = {OUTPUT_SERIAL}, format = port@baudrate (e.g. '/dev/tty.ACM0@38400'); " + f"If clioutput = {OUTPUT_SOCKET}, format = hostip:port (e.g. '0.0.0.0:50010'); " + f"If clioutput = {OUTPUT_HANDLER}, format = evaluable Python expression. " + "NB: gnssdump will have exclusive use of any serial or server port." + ), default=None, ) - ap.add_argument( - "--logtofile", - required=False, - help="fully qualified log file name, or '' for no log file", - type=str, - default="", - ) - ap.add_argument( - "--outputhandler", - required=False, - help="Either writeable output medium or evaluable expression", - ) - ap.add_argument( - "--errorhandler", - required=False, - help="Either writeable output medium or evaluable expression", - ) - - kwargs = vars(ap.parse_args()) + kwargs = set_common_args("gnssdump", ap) + cliout = int(kwargs.pop("clioutput", OUTPUT_NONE)) try: - with GNSSStreamer(CLIAPP, **kwargs) as gns: - gns.run() - + if cliout == OUTPUT_FILE: + filename = kwargs["output"] + ftyp = "wb" if int(kwargs["format"]) == FORMAT_BINARY else "w" + with open(filename, ftyp) as output: + kwargs["output"] = output + runclient(**kwargs) + elif cliout == OUTPUT_SERIAL: + port, baud = kwargs["output"].split("@") + with Serial(port, int(baud), timeout=3) as output: + kwargs["output"] = output + runclient(**kwargs) + elif cliout == OUTPUT_SOCKET: + host, port = kwargs["output"].split(":") + kwargs["output"] = Queue() + # socket server runs as background thread, piping + # output from ntrip client via a message queue + Thread( + target=runserver, + args=(host, int(port), kwargs["output"]), + daemon=True, + ).start() + runclient(**kwargs) + elif cliout == OUTPUT_HANDLER: + kwargs["output"] = eval(kwargs["output"]) # pylint: disable=eval-used + runclient(**kwargs) + else: + kwargs["output"] = None + runclient(**kwargs) except KeyboardInterrupt: pass diff --git a/src/pygnssutils/gnssmqttclient.py b/src/pygnssutils/gnssmqttclient.py index 54899fe..3b1b786 100644 --- a/src/pygnssutils/gnssmqttclient.py +++ b/src/pygnssutils/gnssmqttclient.py @@ -21,10 +21,10 @@ # pylint: disable=invalid-name -import logging import socket from datetime import datetime, timezone from io import BufferedWriter, BytesIO, TextIOWrapper +from logging import getLogger from os import getenv, path from pathlib import Path from queue import Queue @@ -52,16 +52,12 @@ TOPIC_DATA, TOPIC_FREQ, TOPIC_KEY, - VERBOSITY_MEDIUM, ) -from pygnssutils.helpers import set_logging from pygnssutils.mqttmessage import MQTTMessage TIMEOUT = 8 DLGTSPARTN = "SPARTN Configuration" -logger = logging.getLogger(__name__) - class GNSSMQTTClient: """ @@ -73,16 +69,11 @@ def __init__(self, app=None, **kwargs): Constructor. :param object app: application from which this class is invoked (None) - :param int verbosity: (kwarg) log verbosity (1 = medium) - :param str logtofile: (kwarg) fully qualifed log file name ('') """ self.__app = app # Reference to calling application class (if applicable) - set_logging( - logger, - kwargs.pop("verbosity", VERBOSITY_MEDIUM), - kwargs.pop("logtofile", ""), - ) + # configure logger with name "pygnssutils" in calling module + self.logger = getLogger(__name__) self._validargs = True clientid = getenv("MQTTCLIENTID", default="enter-client-id") @@ -105,8 +96,6 @@ def __init__(self, app=None, **kwargs): self._timeout = kwargs.get("timeout", TIMEOUT) self.errevent = kwargs.get("errevent", Event()) - self._verbosity = int(kwargs.get("verbosity", VERBOSITY_MEDIUM)) - self._logtofile = int(kwargs.get("logtofile", 0)) self._logpath = kwargs.get("logpath", ".") self._loglines = 0 self._socket = None @@ -166,36 +155,43 @@ def start(self, **kwargs) -> int: """ try: - for kwarg in [ - "server", - "port", - "clientid", - "region", - "mode", - "topic_ip", - "topic_mga", - "topic_key", - "tlscrt", - "tlskey", - "spartndecode", - "spartnkey", - "spartnbasedate", - "output", - ]: - if kwarg in kwargs: - self._settings[kwarg] = kwargs.get(kwarg) - self._verbosity = int(kwargs.get("verbosity", self._verbosity)) - self._logtofile = int(kwargs.get("logtofile", self._logtofile)) - self._logpath = kwargs.get("logpath", self._logpath) + self._settings["server"] = kwargs.get("server", self._settings["server"]) + self._settings["port"] = int(kwargs.get("port", self._settings["port"])) + self._settings["clientid"] = kwargs.get( + "clientid", self._settings["clientid"] + ) + self._settings["region"] = kwargs.get("region", self._settings["region"]) + self._settings["mode"] = int(kwargs.get("mode", self._settings["mode"])) + self._settings["topic_ip"] = int( + kwargs.get("topic_ip", self._settings["topic_ip"]) + ) + self._settings["topic_mga"] = int( + kwargs.get("topic_mga", self._settings["topic_mga"]) + ) + self._settings["topic_key"] = int( + kwargs.get("topic_key", self._settings["topic_key"]) + ) + self._settings["tlscrt"] = kwargs.get("tlscrt", self._settings["tlscrt"]) + self._settings["tlskey"] = kwargs.get("tlskey", self._settings["tlskey"]) + self._settings["spartndecode"] = int( + kwargs.get("spartndecode", self._settings["spartndecode"]) + ) + self._settings["spartnkey"] = kwargs.get( + "spartnkey", self._settings["spartnkey"] + ) + self._settings["spartnbasedate"] = kwargs.get( + "spartnbasedate", self._settings["spartnbasedate"] + ) + self._settings["output"] = kwargs.get("output", self._settings["output"]) except (ParameterError, ValueError, TypeError) as err: - logger.critical( + self.logger.critical( f"Invalid input arguments {kwargs}\n{err}\nType gnssntripclient -h for help." ) self._validargs = False return 0 - logger.info(f"Starting MQTT client with arguments {self._settings}.") + self.logger.info(f"Starting MQTT client with arguments {self._settings}.") self._stopevent.clear() self._mqtt_thread = Thread( target=self._run, @@ -217,7 +213,7 @@ def stop(self): self._stopevent.set() self._mqtt_thread = None - logger.info("MQTT Client Stopped.") + self.logger.info("MQTT Client Stopped.") def _run( self, @@ -256,7 +252,7 @@ def _run( "decode": settings["spartndecode"], "key": settings["spartnkey"], "basedate": settings["spartnbasedate"], - "verbosity": self._verbosity, + "logger": self.logger, } try: @@ -286,7 +282,7 @@ def _run( f"Unable to connect to {settings['server']}" + f":{settings['port']} in {timeout} seconds. {err}" ) from err - logger.info(f"Trying to connect {i} ...") + self.logger.info(f"Trying to connect {i} ...") sleep(timeout / 4) i += 1 @@ -296,7 +292,7 @@ def _run( # client.loop(timeout=0.1) sleep(0.1) except (FileNotFoundError, TimeoutError) as err: - logger.critical(f"ERROR! {err}") + self.logger.critical(f"ERROR! {err}") GNSSMQTTClient.on_error(userdata, err) self.stop() self.errevent.set() @@ -358,6 +354,7 @@ def on_message(client, userdata, msg): # pylint: disable=unused-argument output = userdata["output"] app = userdata["app"] + msglogger = userdata["logger"] def do_write(raw: bytes, parsed: object): """ @@ -371,8 +368,8 @@ def do_write(raw: bytes, parsed: object): """ if hasattr(parsed, "identity"): - logger.info(parsed.identity) - logger.debug(parsed) + msglogger.info(parsed.identity) + msglogger.debug(parsed) if output is not None: if isinstance(output, (Serial, BufferedWriter)): @@ -422,11 +419,13 @@ def on_error(userdata: dict, err: object): :param object rcd: return code (int or str) """ + errlogger = userdata["logger"] + if isinstance(err, int): err = mqtt.error_string(err) app = userdata["app"] if app is None: - logger.error(err) + errlogger.error(err) else: if hasattr(app, "dialog"): dlg = app.dialog(DLGTSPARTN) diff --git a/src/pygnssutils/gnssmqttclient_cli.py b/src/pygnssutils/gnssmqttclient_cli.py index 6b23cf5..355c222 100644 --- a/src/pygnssutils/gnssmqttclient_cli.py +++ b/src/pygnssutils/gnssmqttclient_cli.py @@ -30,13 +30,9 @@ OUTPUT_SERIAL, OUTPUT_SOCKET, SPARTN_PPSERVER, - VERBOSITY_CRITICAL, - VERBOSITY_DEBUG, - VERBOSITY_HIGH, - VERBOSITY_LOW, - VERBOSITY_MEDIUM, ) from pygnssutils.gnssmqttclient import TIMEOUT, GNSSMQTTClient +from pygnssutils.helpers import set_common_args from pygnssutils.socket_server import runserver TIMEOUT = 8 @@ -48,13 +44,12 @@ def runclient(**kwargs): Start MQTT client with CLI parameters. """ + waittime = float(kwargs["waittime"]) with GNSSMQTTClient(CLIAPP, **kwargs) as gsc: streaming = gsc.start(**kwargs) - while ( - streaming and not kwargs["errevent"].is_set() - ): # run until error or user presses CTRL-C - sleep(kwargs["waittime"]) - sleep(kwargs["waittime"]) + while streaming and not kwargs["errevent"].is_set(): + sleep(waittime) + sleep(waittime) def main(): @@ -75,7 +70,7 @@ def main(): ) ap.add_argument("-V", "--version", action="version", version="%(prog)s " + VERSION) ap.add_argument( - "-C", + "-I", "--clientid", required=False, help="Client ID", @@ -169,33 +164,6 @@ def main(): help="Decryption basedate for encrypted payloads", default=datetime.now(timezone.utc), ) - ap.add_argument( - "--verbosity", - required=False, - help=( - f"Log message verbosity " - f"{VERBOSITY_CRITICAL} = critical, " - f"{VERBOSITY_LOW} = low (error), " - f"{VERBOSITY_MEDIUM} = medium (warning), " - f"{VERBOSITY_HIGH} = high (info), {VERBOSITY_DEBUG} = debug" - ), - type=int, - choices=[ - VERBOSITY_CRITICAL, - VERBOSITY_LOW, - VERBOSITY_MEDIUM, - VERBOSITY_HIGH, - VERBOSITY_DEBUG, - ], - default=VERBOSITY_MEDIUM, - ) - ap.add_argument( - "--logtofile", - required=False, - help="fully qualified log file name, or '' for no log file", - type=str, - default="", - ) ap.add_argument( "--waittime", required=False, @@ -210,12 +178,6 @@ def main(): type=int, default=TIMEOUT, ) - ap.add_argument( - "--errevent", - required=False, - help="Error event", - default=Event(), - ) ap.add_argument( "--clioutput", required=False, @@ -241,11 +203,10 @@ def main(): ), default=None, ) + kwargs = set_common_args("gnssmqttclient", ap) - args = ap.parse_args() - kwargs = vars(args) - - cliout = kwargs.pop("clioutput", OUTPUT_NONE) + kwargs["errevent"] = Event() + cliout = int(kwargs.pop("clioutput", OUTPUT_NONE)) try: if cliout == OUTPUT_FILE: filename = kwargs["output"] diff --git a/src/pygnssutils/gnssntripclient.py b/src/pygnssutils/gnssntripclient.py index 7b4929b..408a1f6 100644 --- a/src/pygnssutils/gnssntripclient.py +++ b/src/pygnssutils/gnssntripclient.py @@ -14,6 +14,10 @@ - dialog() - return reference to NTRIP config client dialog - get_coordinates() - return coordinates from receiver +NB: This utility is used by PyGPSClient - do not change footprint of +any public methods without first checking impact on PyGPSClient - +https://github.com/semuconsulting/PyGPSClient. + Created on 03 Jun 2022 :author: semuadmin @@ -23,12 +27,12 @@ # pylint: disable=invalid-name -import logging import socket import ssl from base64 import b64encode from datetime import datetime, timedelta, timezone from io import BufferedWriter, TextIOWrapper +from logging import getLogger from os import getenv from queue import Queue from threading import Event, Thread @@ -41,19 +45,19 @@ from pyubx2 import ERR_IGNORE, RTCM3_PROTOCOL, UBXReader from serial import Serial +from pygnssutils._version import __version__ as VERSION from pygnssutils.exceptions import ParameterError from pygnssutils.globals import ( CLIAPP, DEFAULT_BUFSIZE, - DEFAULT_TLS_PORTS, + FIXES, HTTPERR, MAXPORT, NOGGA, NTRIP_EVENT, OUTPORT_NTRIP, - VERBOSITY_MEDIUM, ) -from pygnssutils.helpers import find_mp_distance, format_conn, ipprot2int, set_logging +from pygnssutils.helpers import find_mp_distance, format_conn, ipprot2int TIMEOUT = 10 GGALIVE = 0 @@ -66,8 +70,6 @@ INACTIVITY_TIMEOUT = 10 WAITTIME = 3 -logger = logging.getLogger(__name__) - class GNSSNTRIPClient: """ @@ -79,8 +81,6 @@ def __init__(self, app=None, **kwargs): Constructor. :param object app: application from which this class is invoked (None) - :param int verbosity: (kwarg) log verbosity (1 = medium) - :param str logtofile: (kwarg) fully qualifed log file name ('') :param int retries: (kwarg) maximum failed connection retries (5) :param int retryinterval: (kwarg) retry interval in seconds (10) :param int timeout: (kwarg) inactivity timeout in seconds (10) @@ -89,11 +89,8 @@ def __init__(self, app=None, **kwargs): # pylint: disable=consider-using-with self.__app = app # Reference to calling application class (if applicable) - set_logging( - logger, - kwargs.pop("verbosity", VERBOSITY_MEDIUM), - kwargs.pop("logtofile", ""), - ) + # configure logger with name "pygnssutils" in calling module + self.logger = getLogger(__name__) self._validargs = True self._ntripqueue = Queue() # persist settings to allow any calling app to retrieve them @@ -127,7 +124,7 @@ def __init__(self, app=None, **kwargs): self._retryinterval = int(kwargs.pop("retryinterval", RETRY_INTERVAL)) self._timeout = int(kwargs.pop("timeout", INACTIVITY_TIMEOUT)) except (ParameterError, ValueError, TypeError) as err: - logger.critical( + self.logger.critical( f"Invalid input arguments {kwargs=}\n{err=}\nType gnssntripclient -h for help.", ) self._validargs = False @@ -225,9 +222,7 @@ def run(self, **kwargs) -> bool: self._settings["ipprot"] = ipprot2int(ipprot) self._settings["server"] = server = kwargs.get("server", "") self._settings["port"] = port = int(kwargs.get("port", OUTPORT_NTRIP)) - self._settings["https"] = int( - kwargs.get("https", 1 if port in DEFAULT_TLS_PORTS else 0) - ) + self._settings["https"] = int(kwargs.get("https", 0)) self._settings["flowinfo"] = int(kwargs.get("flowinfo", 0)) self._settings["scopeid"] = int(kwargs.get("scopeid", 0)) self._settings["mountpoint"] = mountpoint = kwargs.get("mountpoint", "") @@ -260,7 +255,7 @@ def run(self, **kwargs) -> bool: raise ParameterError(f"Invalid port {port}") except (ParameterError, ValueError, TypeError) as err: - logger.critical( + self.logger.critical( f"Invalid input arguments {kwargs}\n{err}\nType gnssntripclient -h for help." ) self._validargs = False @@ -306,25 +301,40 @@ def _app_get_coordinates(self) -> tuple: Get live coordinates from receiver, or use fixed reference position, depending on ggamode setting. - :returns: tuple of (lat, lon, alt, sep) + NB: 'fix' is a string e.g. "3D" or "RTK FLOAT" + + :returns: tuple of coordinate and fix data :rtype: tuple """ lat = lon = alt = sep = 0.0 - if self._settings["ggamode"] == GGAFIXED: # Fixed reference position + fix, sip, hdop, diffage, diffstation = ("3D", 15, 0.98, 0, 0) + if self._settings["ggamode"] == GGAFIXED: # fixed reference position lat = self._settings["reflat"] lon = self._settings["reflon"] alt = self._settings["refalt"] sep = self._settings["refsep"] elif self.__app is not None: if hasattr(self.__app, "get_coordinates"): # live position from receiver - _, lat, lon, alt, sep = self.__app.get_coordinates() + coords = self.__app.get_coordinates() + if isinstance(coords, tuple): # old version (PyGPSClient <=1.4.19) + _, lat, lon, alt, sep = coords + else: # new version uses dict (PyGPSClient >=1.4.20) + lat = coords.get("lat", lat) + lon = coords.get("lon", lon) + alt = coords.get("alt", alt) + sep = coords.get("sep", sep) + sip = coords.get("sip", sip) + fix = coords.get("fix", fix) + hdop = coords.get("hdop", hdop) + diffage = coords.get("diffage", diffage) + diffstation = coords.get("diffstation", diffstation) lat, lon, alt, sep = [ 0.0 if c == "" else float(c) for c in (lat, lon, alt, sep) ] - return lat, lon, alt, sep + return lat, lon, alt, sep, fix, sip, hdop, diffage, diffstation def _formatGET(self, settings: dict) -> str: """ @@ -336,16 +346,31 @@ def _formatGET(self, settings: dict) -> str: :rtype: str """ + ggahdr = "" + if settings["version"] == "2.0": + hver = "1.1" + nver = "Ntrip-Version: Ntrip/2.0\r\n" + if settings["ggainterval"] != NOGGA: + gga, _ = self._formatGGA() + ggahdr = f"Ntrip-GGA: {gga.decode('utf-8')}" # includes \r\n + else: + hver = "1.0" + nver = "" + mountpoint = "/" + settings["mountpoint"] user = settings["ntripuser"] + ":" + settings["ntrippassword"] user = b64encode(user.encode(encoding="utf-8")) req = ( - f"GET {mountpoint} HTTP/1.0\r\n" - + "User-Agent: NTRIP pygnssutils\r\n" - + "Accept: */*\r\n" - + f"Authorization: Basic {user.decode(encoding='utf-8')}\r\n" - + "Connection: close\r\n\r\n" # NECESSARY!!! + f"GET {mountpoint} HTTP/{hver}\r\n" + f"Host: {settings['server']}:{settings['port']}\r\n" + f"{nver}" + f"User-Agent: NTRIP pygnssutils/{VERSION}\r\n" + "Accept: */*\r\n" + f"Authorization: Basic {user.decode(encoding='utf-8')}\r\n" + f"{ggahdr}" + "Connection: close\r\n\r\n" # NECESSARY!!! ) + self.logger.debug(f"HTTP Header\n{req}") return req.encode(encoding="utf-8") def _formatGGA(self) -> tuple: @@ -353,37 +378,40 @@ def _formatGGA(self) -> tuple: THREADED Format NMEA GGA sentence using pynmeagps. The raw string output is suitable for sending to an NTRIP socket. + GGA timestamp will default to current UTC. GGA quality is + derived from fix string. :return: tuple of (raw NMEA message as bytes, NMEAMessage) :rtype: tuple + :rtype: tuple """ - # time will default to current UTC try: - lat, lon, alt, sep = self._app_get_coordinates() + lat, lon, alt, sep, fixs, sip, hdop, diffage, diffstation = ( + self._app_get_coordinates() + ) lat = float(lat) lon = float(lon) - + fixi = FIXES.get(fixs, 1) parsed_data = NMEAMessage( "GP", "GGA", GET, lat=lat, lon=lon, - quality=1, - numSV=15, - HDOP=0, + quality=fixi, + numSV=sip, + HDOP=hdop, alt=alt, altUnit="M", sep=sep, sepUnit="M", - diffAge="", - diffStation=0, + diffAge=diffage, + diffStation=diffstation, ) raw_data = parsed_data.serialize() return raw_data, parsed_data - except ValueError: return None, None @@ -415,13 +443,13 @@ def _get_closest_mountpoint(self) -> tuple: """ try: - lat, lon, _, _ = self._app_get_coordinates() + lat, lon, _, _, _, _, _, _, _ = self._app_get_coordinates() closest_mp, dist = find_mp_distance( float(lat), float(lon), self._settings["sourcetable"] ) if self._settings["mountpoint"] == "": self._settings["mountpoint"] = closest_mp - logger.info( + self.logger.info( "Closest mountpoint to reference location" f"({lat}, {lon}) = {closest_mp}, {dist} km." ) @@ -462,7 +490,7 @@ def _stop_read_thread(self): self._stopevent.set() self._ntrip_thread = None - logger.info("Streaming terminated.") + self.logger.info("Streaming terminated.") def _read_thread( self, @@ -500,7 +528,7 @@ def _read_thread( else "" ) ) - logger.error(f"SSL Certificate Verification Error{tip}\n{err}") + self.logger.error(f"SSL Certificate Verification Error{tip}\n{err}") self._retrycount = self._retries stopevent.set() self._connected = False @@ -522,16 +550,16 @@ def _read_thread( if self._retrycount == self._retries: stopevent.set() self._connected = False - logger.critical(errl) - else: - self._retrycount += 1 - errr = ( - f". Retrying in {self._retryinterval * self._retrycount} secs " - f"({self._retrycount}/{self._retries}) ..." - ) - erra += errr - errl += errr - logger.warning(errl) + self.logger.critical(errl) + break + self._retrycount += 1 + errr = ( + f". Retrying in {self._retryinterval * self._retrycount} secs " + f"({self._retrycount}/{self._retries}) ..." + ) + erra += errr + errl += errr + self.logger.warning(errl) self._app_update_status(False, (erra, "red")) sleep(self._retryinterval * self._retrycount) @@ -571,14 +599,14 @@ def _do_connection( self._socket.connect(conn) self._socket.sendall(self._formatGET(settings)) # send GGA sentence with request - if mountpoint != "": - self._send_GGA(ggainterval, output) + # if mountpoint != "": + # self._send_GGA(ggainterval, output) while not stopevent.is_set(): rc = self._do_header(self._socket, stopevent, output) if rc == "0": # streaming RTCM3/SPARTN data from mountpoint self._retrycount = 0 msg = f"Streaming {datatype} data from {server}:{port}/{mountpoint} ..." - logger.info(msg) + self.logger.info(msg) self._app_update_status(True, (msg, "blue")) self._do_data( self._socket, @@ -592,7 +620,7 @@ def _do_connection( self._connected = False self._app_update_status(False, ("Sourcetable retrieved", "blue")) else: # error message - logger.critical( + self.logger.critical( f"Error connecting to {server}:{port}/{mountpoint=}: {rc}" ) stopevent.set() @@ -619,7 +647,7 @@ def _do_header(self, sock: socket, stopevent: Event, output: object) -> str: header_lines = data.decode(encoding="utf-8").split("\r\n") for line in header_lines: # if sourcetable request, populate list - if True in [line.find(cd) > 0 for cd in HTTPERR]: # HTTP 40x + if True in [line.find(cd) > 0 for cd in HTTPERR]: # HTTP 4nn, 50n return line if line.find("STR;") >= 0: # sourcetable entry strbits = line.split(";") @@ -630,7 +658,7 @@ def _do_header(self, sock: socket, stopevent: Event, output: object) -> str: self._settings["sourcetable"] = stable mp, dist = self._get_closest_mountpoint() self._do_output(output, stable, (mp, dist)) - logger.info(f"Complete sourcetable follows...\n{stable}") + self.logger.info(f"Complete sourcetable follows...\n{stable}") return "1" except UnicodeDecodeError: @@ -722,8 +750,8 @@ def _do_output(self, output: object, raw: bytes, parsed: object): """ if hasattr(parsed, "identity"): - logger.info(f"{type(parsed).__name__} received: {parsed.identity}") - logger.debug(parsed) + self.logger.info(f"{type(parsed).__name__} received: {parsed.identity}") + self.logger.debug(parsed) if output is not None: # serialize sourcetable if outputting to stream if isinstance(raw, list) and not isinstance(output, Queue): diff --git a/src/pygnssutils/gnssntripclient_cli.py b/src/pygnssutils/gnssntripclient_cli.py index 0f903f9..e047b9d 100644 --- a/src/pygnssutils/gnssntripclient_cli.py +++ b/src/pygnssutils/gnssntripclient_cli.py @@ -22,21 +22,14 @@ from pygnssutils._version import __version__ as VERSION from pygnssutils.globals import ( CLIAPP, - DEFAULT_TLS_PORTS, EPILOG, OUTPUT_FILE, OUTPUT_NONE, OUTPUT_SERIAL, OUTPUT_SOCKET, - VERBOSITY_CRITICAL, - VERBOSITY_DEBUG, - VERBOSITY_HIGH, - VERBOSITY_LOW, - VERBOSITY_MEDIUM, ) from pygnssutils.gnssntripclient import ( GGAFIXED, - GGALIVE, INACTIVITY_TIMEOUT, MAX_RETRY, RETRY_INTERVAL, @@ -45,6 +38,7 @@ WAITTIME, GNSSNTRIPClient, ) +from pygnssutils.helpers import set_common_args from pygnssutils.socket_server import runserver @@ -76,7 +70,7 @@ def main(): ) ap.add_argument("-V", "--version", action="version", version="%(prog)s " + VERSION) ap.add_argument( - "-S", "--server", required=True, help="NTRIP server (caster) URL", default="" + "-S", "--server", required=False, help="NTRIP server (caster) URL", default="" ) ap.add_argument( "-P", "--port", required=False, help="NTRIP port", type=int, default=2101 @@ -92,10 +86,7 @@ def main(): "-H", "--https", required=False, - help=( - f"HTTPS (TLS) connection? 0 = HTTP, " - f"1 = HTTPS (defaults to 1 if port in {DEFAULT_TLS_PORTS})" - ), + help=("HTTPS (TLS) connection? 0 = HTTP, 1 = HTTPS"), type=int, choices=[0, 1], default=0, @@ -168,14 +159,6 @@ def main(): type=int, default=-1, ) - ap.add_argument( - "--ggamode", - required=False, - help=f"GGA pos source; {GGALIVE} = live from receiver, {GGAFIXED} = fixed reference", - type=int, - choices=[GGALIVE, GGAFIXED], - default=GGAFIXED, - ) ap.add_argument( "--reflat", required=False, help="reference latitude", type=float, default=0.0 ) @@ -208,33 +191,6 @@ def main(): help="Decryption basedate for encrypted SPARTN payloads", default=datetime.now(timezone.utc), ) - ap.add_argument( - "--verbosity", - required=False, - help=( - f"Log message verbosity " - f"{VERBOSITY_CRITICAL} = critical, " - f"{VERBOSITY_LOW} = low (error), " - f"{VERBOSITY_MEDIUM} = medium (warning), " - f"{VERBOSITY_HIGH} = high (info), {VERBOSITY_DEBUG} = debug" - ), - type=int, - choices=[ - VERBOSITY_CRITICAL, - VERBOSITY_LOW, - VERBOSITY_MEDIUM, - VERBOSITY_HIGH, - VERBOSITY_DEBUG, - ], - default=VERBOSITY_MEDIUM, - ) - ap.add_argument( - "--logtofile", - required=False, - help="fully qualified log file name, or '' for no log file", - type=str, - default="", - ) ap.add_argument( "--clioutput", required=False, @@ -260,14 +216,10 @@ def main(): ), default=None, ) + kwargs = set_common_args("gnssntripclient", ap) - args = ap.parse_args() - kwargs = vars(args) - - # assume HTTPS if port is 443 or 2102 (PointPerfect NTRIP TLS port) - kwargs["https"] = 1 if kwargs["port"] in DEFAULT_TLS_PORTS else kwargs["https"] - - cliout = kwargs.pop("clioutput", OUTPUT_NONE) + kwargs["ggamode"] = GGAFIXED # only fixed reference mode is available via CLI + cliout = int(kwargs.pop("clioutput", OUTPUT_NONE)) try: if cliout == OUTPUT_FILE: filename = kwargs["output"] diff --git a/src/pygnssutils/gnssserver.py b/src/pygnssutils/gnssserver.py index 029039d..632270f 100644 --- a/src/pygnssutils/gnssserver.py +++ b/src/pygnssutils/gnssserver.py @@ -16,24 +16,16 @@ # pylint: disable=too-many-arguments -import logging +from logging import getLogger from queue import Queue from threading import Thread from time import sleep -from pygnssutils.globals import ( - CONNECTED, - FORMAT_BINARY, - OUTPORT, - OUTPORT_NTRIP, - VERBOSITY_MEDIUM, -) +from pygnssutils.globals import CONNECTED, FORMAT_BINARY, OUTPORT, OUTPORT_NTRIP from pygnssutils.gnssstreamer import GNSSStreamer -from pygnssutils.helpers import format_conn, ipprot2int, set_logging +from pygnssutils.helpers import format_conn, ipprot2int from pygnssutils.socket_server import ClientHandler, SocketServer -logger = logging.getLogger(__name__) - class GNSSSocketServer: """ @@ -69,17 +61,13 @@ def __init__(self, app=None, **kwargs): :param int protfilter: (kwarg) 1 = NMEA, 2 = UBX, 4 = RTCM3 (7 - ALL) :param str msgfilter: (kwarg) comma-separated string of message identities e.g. 'NAV-PVT,GNGSA' (None) :param int limit: (kwarg) maximum number of messages to read (0 = unlimited) - :param int verbosity: (kwarg) log message verbosity 0 = low, 1 = medium, 3 = high (1) - :param str logtofile: (kwarg) fully qualifed log file name ('') """ # Reference to calling application class (if applicable) self.__app = app # pylint: disable=unused-private-member - set_logging( - logger, - kwargs.pop("verbosity", VERBOSITY_MEDIUM), - kwargs.pop("logtofile", ""), - ) + # configure logger with name "pygnssutils" in calling module + self.logger = getLogger(__name__) + self.logger.debug(kwargs) try: self._kwargs = kwargs # overrideable command line arguments.. @@ -105,8 +93,9 @@ def __init__(self, app=None, **kwargs): self._kwargs["maxclients"] = int(kwargs.get("maxclients", 5)) self._kwargs["format"] = int(kwargs.get("format", FORMAT_BINARY)) # required fixed arguments... - msgqueue = Queue() - self._kwargs["outputhandler"] = msgqueue + # msgqueue = Queue() + # self._kwargs["outputhandler"] = msgqueue + self._kwargs["output"] = Queue() self._socket_server = None self._streamer = None self._in_thread = None @@ -115,7 +104,7 @@ def __init__(self, app=None, **kwargs): self._validargs = True except ValueError as err: - logger.critical(f"Invalid input arguments {kwargs}\n{err}") + self.logger.critical(f"Invalid input arguments {kwargs}\n{err}") self._validargs = False def __enter__(self): @@ -143,7 +132,7 @@ def run(self) -> int: """ if self._validargs: - logger.info("Starting server (type CTRL-C to stop)...") + self.logger.info("Starting server (type CTRL-C to stop)...") self._in_thread = self._start_input_thread(**self._kwargs) sleep(0.5) if self._in_thread.is_alive(): @@ -158,12 +147,12 @@ def stop(self): Shutdown server. """ - logger.info("Stopping server...") + self.logger.info("Stopping server...") if self._streamer is not None: self._streamer.stop() if self._socket_server is not None: self._socket_server.shutdown() - logger.info("Server shutdown.") + self.logger.info("Server shutdown.") def _start_input_thread(self, **kwargs) -> Thread: """ @@ -174,7 +163,7 @@ def _start_input_thread(self, **kwargs) -> Thread: :rtype: Thread """ - logger.info(f"Starting input thread, reading from {kwargs['port']}...") + self.logger.info(f"Starting input thread, reading from {kwargs['port']}...") thread = Thread( target=self._input_thread, args=(kwargs,), @@ -192,7 +181,7 @@ def _start_output_thread(self, **kwargs) -> Thread: :rtype: Thread """ - logger.info( + self.logger.info( f"Starting output thread, broadcasting on {kwargs['hostip']}:{kwargs['outport']}..." ) thread = Thread( @@ -231,7 +220,7 @@ def _output_thread(self, app: object, kwargs): app, kwargs["ntripmode"], kwargs["maxclients"], - kwargs["outputhandler"], + kwargs["output"], conn, ClientHandler, ntripuser=kwargs["ntripuser"], @@ -240,7 +229,7 @@ def _output_thread(self, app: object, kwargs): ) as self._socket_server: self._socket_server.serve_forever() except OSError as err: - logger.critical(f"Error starting socket server {err}") + self.logger.critical(f"Error starting socket server {err}") def notify_client(self, address: tuple, status: int): """ @@ -257,6 +246,6 @@ def notify_client(self, address: tuple, status: int): else: pre = "dis" self._clients -= 1 - logger.info( + self.logger.info( f"Client {address} has {pre}connected. Total clients: {self._clients}" ) diff --git a/src/pygnssutils/gnssserver_cli.py b/src/pygnssutils/gnssserver_cli.py index bd7efff..394a84b 100644 --- a/src/pygnssutils/gnssserver_cli.py +++ b/src/pygnssutils/gnssserver_cli.py @@ -15,16 +15,9 @@ from time import sleep from pygnssutils._version import __version__ as VERSION -from pygnssutils.globals import ( - CLIAPP, - EPILOG, - VERBOSITY_CRITICAL, - VERBOSITY_DEBUG, - VERBOSITY_HIGH, - VERBOSITY_LOW, - VERBOSITY_MEDIUM, -) +from pygnssutils.globals import CLIAPP, EPILOG from pygnssutils.gnssserver import GNSSSocketServer +from pygnssutils.helpers import set_common_args def main(): @@ -171,26 +164,6 @@ def main(): type=int, default=0, ) - ap.add_argument( - "--verbosity", - required=False, - help=( - f"Log message verbosity " - f"{VERBOSITY_CRITICAL} = critical, " - f"{VERBOSITY_LOW} = low (error), " - f"{VERBOSITY_MEDIUM} = medium (warning), " - f"{VERBOSITY_HIGH} = high (info), {VERBOSITY_DEBUG} = debug" - ), - type=int, - choices=[ - VERBOSITY_CRITICAL, - VERBOSITY_LOW, - VERBOSITY_MEDIUM, - VERBOSITY_HIGH, - VERBOSITY_DEBUG, - ], - default=VERBOSITY_MEDIUM, - ) ap.add_argument( "--outfile", required=False, @@ -204,16 +177,8 @@ def main(): type=int, default=1, ) - ap.add_argument( - "--logtofile", - required=False, - help="fully qualified log file name, or '' for no log file", - type=str, - default="", - ) + kwargs = set_common_args("gnssserver", ap) - args = ap.parse_args() - kwargs = vars(args) if kwargs["hostip"] == "0.0.0.0" and kwargs["ipprot"] == "IPv6": kwargs["hostip"] = "::" @@ -222,8 +187,8 @@ def main(): goodtogo = server.run() while goodtogo: # run until user presses CTRL-C - sleep(args.waittime) - sleep(args.waittime) + sleep(kwargs["waittime"]) + sleep(kwargs["waittime"]) except KeyboardInterrupt: pass diff --git a/src/pygnssutils/gnssstreamer.py b/src/pygnssutils/gnssstreamer.py index f33a002..258c65f 100644 --- a/src/pygnssutils/gnssstreamer.py +++ b/src/pygnssutils/gnssstreamer.py @@ -5,6 +5,10 @@ to stream the parsed UBX, NMEA or RTCM3 output of a GNSS device to stdout or a designated output handler. +NB: This utility is used by PyGPSClient - do not change footprint of +any public methods without first checking impact on PyGPSClient - +https://github.com/semuconsulting/PyGPSClient. + Created on 26 May 2022 :author: semuadmin @@ -14,9 +18,9 @@ # pylint: disable=line-too-long eval-used -import logging from collections import defaultdict from io import BufferedWriter, TextIOWrapper +from logging import getLogger from queue import Queue from socket import AF_INET6, SOCK_STREAM, socket from time import time @@ -48,11 +52,10 @@ FORMAT_JSON, FORMAT_PARSED, FORMAT_PARSEDSTRING, - VERBOSITY_HIGH, + UBXSIMULATOR, ) -from pygnssutils.helpers import format_conn, format_json, ipprot2int, set_logging - -logger = logging.getLogger(__name__) +from pygnssutils.helpers import format_conn, format_json, ipprot2int +from pygnssutils.ubxsimulator import UBXSimulator class GNSSStreamer: @@ -95,27 +98,22 @@ def __init__(self, app=None, **kwargs): :param int protfilter: (kwarg) 1 = NMEA, 2 = UBX, 4 = RTCM3 (7 - ALL) :param str msgfilter: (kwarg) comma-separated string of message identities e.g. 'NAV-PVT,GNGSA' (None) :param int limit: (kwarg) maximum number of messages to read (0 = unlimited) - :param int verbosity: (kwarg) log message verbosity 0 = low, 1 = medium, 3 = high (1) - :param str outfile: (kwarg) fully qualified path to output file (None) - :param str logtofile: (kwarg) fully qualifed log file name ('') - :param object outputhandler: (kwarg) either writeable output medium or evaluable expression (None) - :param object errorhandler: (kwarg) either writeable output medium or evaluable expression (None) + :param object output: (kwarg) either writeable output medium or callback function (None) :raises: ParameterError """ # pylint: disable=raise-missing-from # Reference to calling application class (if applicable) self.__app = app # pylint: disable=unused-private-member - set_logging( - logger, kwargs.pop("verbosity", VERBOSITY_HIGH), kwargs.pop("logtofile", "") - ) + # configure logger with name "pygnssutils" in calling module + self.logger = getLogger(__name__) self._reader = None self.ctx_mgr = False self._datastream = kwargs.get("datastream", None) self._port = kwargs.get("port", None) self._socket = kwargs.get("socket", None) - self._outfile = kwargs.get("outfile", None) self._ipprot = ipprot2int(kwargs.get("ipprot", "IPv4")) + self._output = kwargs.get("output", None) if self._socket is not None: if self._ipprot == AF_INET6: # IPv6 host ip must be enclosed in [] @@ -177,50 +175,16 @@ def __init__(self, app=None, **kwargs): self._outcount = defaultdict(int) self._errcount = 0 self._validargs = True - self._output = None self._stopevent = False - self._outputhandler = None - self._errorhandler = None # flag to signify beginning of JSON array self._jsontop = True - self._setup_output_handlers(**kwargs) - except (ParameterError, ValueError, TypeError) as err: raise ParameterError( f"Invalid input arguments {kwargs}\n{err}\nType gnssdump -h for help." ) - def _setup_output_handlers(self, **kwargs): - """ - Set up output handlers. - - Output handlers can either be writeable output media - (Serial, File, socket or Queue) or an evaluable expression. - - (Note: ast.literal_eval can't replace eval here) - - 'allhandler' applies to all protocols and overrides - individual output handlers. - """ - - htypes = (Serial, TextIOWrapper, BufferedWriter, Queue, socket) - - erh = kwargs.get("errorhandler", None) - if erh is not None: - if isinstance(erh, htypes): - self._errorhandler = erh - else: - self._errorhandler = eval(erh) - - oph = kwargs.get("outputhandler", None) - if oph is not None: - if isinstance(oph, htypes): - self._outputhandler = oph - else: - self._outputhandler = eval(oph) - def __enter__(self): """ Context manager enter routine. @@ -248,10 +212,6 @@ def run(self, **kwargs) -> int: """ # pylint: disable=consider-using-with - if self._outfile is not None: - ftyp = "wb" if self._format == FORMAT_BINARY else "w" - self._output = open(self._outfile, ftyp) - self._limit = int(kwargs.get("limit", self._limit)) # open the specified input stream @@ -259,10 +219,14 @@ def run(self, **kwargs) -> int: with self._datastream as self._stream: self._start_reader() elif self._port is not None: # serial - with Serial( - self._port, self._baudrate, timeout=self._timeout - ) as self._stream: - self._start_reader() + if self._port.upper() == UBXSIMULATOR: + with UBXSimulator() as self._stream: + self._start_reader() + else: + with Serial( + self._port, self._baudrate, timeout=self._timeout + ) as self._stream: + self._start_reader() elif self._socket is not None: # socket with socket(self._ipprot, SOCK_STREAM) as self._stream: self._stream.connect( @@ -294,16 +258,13 @@ def stop(self): f"Messages output: {dict(sorted(self._outcount.items()))}", ] for msg in msgs: - logger.info(msg) + self.logger.info(msg) msg = ( f"Streaming terminated, {self._msgcount:,} message{mss} " f"processed with {self._errcount:,} error{ers}." ) - logger.info(msg) - - if self._output is not None: - self._output.close() + self.logger.info(msg) def _start_reader(self): """Create UBXReader instance.""" @@ -316,7 +277,7 @@ def _start_reader(self): msgmode=self._msgmode, parsebitfield=self._parsebitfield, ) - logger.info(f"Parsing GNSS data stream from: {self._stream}...") + self.logger.info(f"Parsing GNSS data stream from: {self._stream}...") # if outputting json, add opening tag if self._format == FORMAT_JSON: @@ -361,7 +322,7 @@ def _do_parse(self): raise EOFError # get the message protocol (NMEA or UBX) - handler = self._outputhandler + handler = self._output msgprot = 0 # establish the appropriate handler and identity for this protocol if isinstance(parsed_data, UBXMessage): @@ -416,7 +377,9 @@ def _filtered(self, protocol: int, identity: str) -> bool: return False toc = time() elapsed = toc - tic - logger.debug(f"Time since last {identity} message was sent: {elapsed}") + self.logger.debug( + f"Time since last {identity} message was sent: {elapsed}" + ) # check if at least 95% of filter period has elapsed if elapsed >= 0.95 * per: self._msgfilter[identity] = (per, toc) @@ -426,8 +389,8 @@ def _filtered(self, protocol: int, identity: str) -> bool: def _do_output(self, raw: bytes, parsed: object, handler: object): """ - Output message to terminal in specified format(s) OR pass - to external output handler if one is specified. + Output message to stdout in specified format(s) OR pass + to writeable output media / callback function if specified. :param bytes raw: raw (binary) message :param object parsed: parsed message @@ -471,51 +434,31 @@ def _do_output(self, raw: bytes, parsed: object, handler: object): handler.put(output) elif isinstance(handler, socket): handler.sendall(output) - # treated as evaluable expression + # callback function else: handler(output) def _do_print(self, data: object): """ - Print data to outfile or stdout. + Print data to stdout. :param object data: data to print """ - if self._outfile is None: - print(data) - else: - if (self._format == FORMAT_BINARY and not isinstance(data, bytes)) or ( - self._format != FORMAT_BINARY and not isinstance(data, str) - ): - data = f"{data}\n" - self._output.write(data) + print(data) def _do_error(self, err: Exception): """ Handle error according to quitonerror flag; - either ignore, log, (re)raise or pass to - external error handler if one is specified. + either ignore, log, or (re)raise. :param err Exception: error """ - if self._errorhandler is None: - if self._quitonerror == ERR_RAISE: - raise err - if self._quitonerror == ERR_LOG: - logger.critical(err) - elif isinstance(self._errorhandler, (Serial, BufferedWriter)): - self._errorhandler.write(err) - elif isinstance(self._errorhandler, TextIOWrapper): - self._errorhandler.write(str(err)) - elif isinstance(self._errorhandler, Queue): - self._errorhandler.put(err) - elif isinstance(self._errorhandler, socket): - self._errorhandler.sendall(err) - else: - self._errorhandler(err) - self._errcount += 1 + if self._quitonerror == ERR_RAISE: + raise err + if self._quitonerror == ERR_LOG: + self.logger.critical(err) def _do_json(self, parsed: object) -> str: """ @@ -547,7 +490,7 @@ def _cap_json(self, start: int): else: cap = "]}" - oph = self._outputhandler + oph = self._output if oph is None: print(cap) elif isinstance(oph, (Serial, TextIOWrapper, BufferedWriter)): diff --git a/src/pygnssutils/helpers.py b/src/pygnssutils/helpers.py index fca9ef3..bb6f1a1 100644 --- a/src/pygnssutils/helpers.py +++ b/src/pygnssutils/helpers.py @@ -12,13 +12,113 @@ import logging import logging.handlers +from argparse import ArgumentParser from math import cos, radians, sin +from os import getenv from socket import AF_INET, AF_INET6, gaierror, getaddrinfo from pynmeagps import haversine from pyubx2 import itow2utc -from pygnssutils.globals import LOGFORMAT, LOGGING_LEVELS, LOGLIMIT, VERBOSITY_MEDIUM +from pygnssutils.globals import ( + LOGFORMAT, + LOGGING_LEVELS, + LOGLIMIT, + VERBOSITY_CRITICAL, + VERBOSITY_DEBUG, + VERBOSITY_HIGH, + VERBOSITY_LOW, + VERBOSITY_MEDIUM, +) + + +def parse_config(configfile: str) -> dict: + """ + Parse config file. + + :param str configfile: fully qualified path to config file + :return: config as kwargs, or None if file not found + :rtype: dict + """ + + try: + config = {} + with open(configfile, "r", encoding="utf-8") as infile: + for cf in infile: + key, val = cf.split("=", 1) + config[key.strip()] = val.strip() + return config + except (FileNotFoundError, ValueError): + return None + + +def set_common_args( + name: str, + ap: ArgumentParser, + logname: str = "pygnssutils", + logdefault: int = VERBOSITY_MEDIUM, +) -> dict: + """ + Set common argument parser and logging args. + + :param str name: name of CLI utility e.g. "gnssdump" + :param ArgumentParserap: argument parser instance + :param str logname: logger name + :param int logdefault: default logger verbosity level + :return: parsed arguments as kwargs + :rtype: dict + """ + + ap.add_argument( + "-C", + "--config", + required=False, + help=( + "Fully qualified path to CLI configuration file " + f"(will use environment variable {name.upper()}_CONF where set)" + ), + default=getenv(f"{name.upper()}_CONF", None), + ) + ap.add_argument( + "--verbosity", + required=False, + help=( + f"Log message verbosity " + f"{VERBOSITY_CRITICAL} = critical, " + f"{VERBOSITY_LOW} = low (error), " + f"{VERBOSITY_MEDIUM} = medium (warning), " + f"{VERBOSITY_HIGH} = high (info), {VERBOSITY_DEBUG} = debug" + ), + type=int, + choices=[ + VERBOSITY_CRITICAL, + VERBOSITY_LOW, + VERBOSITY_MEDIUM, + VERBOSITY_HIGH, + VERBOSITY_DEBUG, + ], + default=logdefault, + ) + ap.add_argument( + "--logtofile", + required=False, + help="fully qualified log file name, or '' for no log file", + type=str, + default="", + ) + + kwargs = vars(ap.parse_args()) + # config file settings will supplement CLI and default args + cfg = kwargs.pop("config", None) + if cfg is not None: + kwargs = {**kwargs, **parse_config(cfg)} + + logger = logging.getLogger(logname) + set_logging( + logger, kwargs.pop("verbosity", logdefault), kwargs.pop("logtofile", "") + ) + + return kwargs def set_logging( @@ -60,6 +160,20 @@ def set_logging( logger.addHandler(loghandler) +def progbar(i: int, lim: int, inc: int = 50): + """ + Display progress bar on console. + """ + + i = min(i, lim) + pct = int(i * inc / lim) + if not i % int(lim / inc): + print( + f"{int(pct*100/inc):02}% " + "\u2593" * pct + "\u2591" * (inc - pct), + end="\r", + ) + + def get_mp_distance(lat: float, lon: float, mp: list) -> float: """ Get distance to mountpoint from current location (if known). diff --git a/src/pygnssutils/socket_server.py b/src/pygnssutils/socket_server.py index b8d3e9d..1b3fc49 100644 --- a/src/pygnssutils/socket_server.py +++ b/src/pygnssutils/socket_server.py @@ -21,6 +21,10 @@ export PYGPSCLIENT_USER="user" export PYGPSCLIENT_PASSWORD="password" +NB: This utility is used by PyGPSClient - do not change footprint of +any public methods without first checking impact on PyGPSClient - +https://github.com/semuconsulting/PyGPSClient. + Created on 16 May 2022 :author: semuadmin diff --git a/src/pygnssutils/ubxsave.py b/src/pygnssutils/ubxsave.py index fe31a54..1254b07 100644 --- a/src/pygnssutils/ubxsave.py +++ b/src/pygnssutils/ubxsave.py @@ -51,26 +51,13 @@ from pygnssutils._version import __version__ as VERSION from pygnssutils.globals import EPILOG +from pygnssutils.helpers import progbar # try increasing these values if device response is too slow: DELAY = 0.02 # delay between polls WRAPUP = 5 # delay for final responses -def progbar(i: int, lim: int, inc: int = 50): - """ - Display progress bar on console. - """ - - i = min(i, lim) - pct = int(i * inc / lim) - if not i % int(lim / inc): - print( - f"{int(pct*100/inc):02}% " + "\u2593" * pct + "\u2591" * (inc - pct), - end="\r", - ) - - class UBXSaver: """UBX Configuration Saver Class.""" diff --git a/src/pygnssutils/ubxsimulator.py b/src/pygnssutils/ubxsimulator.py index 0b84c08..2cbe36b 100644 --- a/src/pygnssutils/ubxsimulator.py +++ b/src/pygnssutils/ubxsimulator.py @@ -9,7 +9,7 @@ based on parameters defined in a json configuration file. Can simulate a motion vector based on a specified course over ground and speed. -Example usage:: +Example usage: from pygnssutils import UBXSimulator from pyubx2 import UBXReader @@ -39,37 +39,38 @@ # pylint: disable=too-many-locals, too-many-instance-attributes -import logging from datetime import datetime, timedelta from json import JSONDecodeError, load +from logging import getLogger from math import cos, pi, sin -from os import path +from os import getenv, path from pathlib import Path from queue import Queue from threading import Event, Thread from time import sleep from pynmeagps import NMEAMessage +from pyrtcm import RTCMMessage, RTCMMessageError, RTCMParseError, RTCMReader from pyubx2 import ( GET, + RTCM3_PROTOCOL, + UBX_PROTOCOL, UBXMessage, UBXMessageError, UBXParseError, UBXReader, escapeall, getinputmode, + protocol, utc2itow, ) -from pygnssutils.globals import EARTH_RADIUS, VERBOSITY_MEDIUM -from pygnssutils.helpers import set_logging +from pygnssutils.globals import EARTH_RADIUS, UBXSIMULATOR DEFAULT_INTERVAL = 1000 # milliseconds DEFAULT_TIMEOUT = 3 # seconds DEFAULT_PATH = path.join(Path.home(), "ubxsimulator") -logger = logging.getLogger(__name__) - class UBXSimulator: """ @@ -87,16 +88,15 @@ def __init__(self, app=None, **kwargs): # Reference to calling application class (if applicable) self.__app = app # pylint: disable=unused-private-member - set_logging( - logger, - kwargs.pop("verbosity", VERBOSITY_MEDIUM), - kwargs.pop("logtofile", ""), - ) + # configure logger with name "pygnssutils" in calling module + self.logger = getLogger(__name__) self._config = self._readconfig( - kwargs.get("configfile", DEFAULT_PATH + ".json") + kwargs.get( + "configfile", + getenv(f"{UBXSIMULATOR.upper()}_JSON", DEFAULT_PATH + ".json"), + ) ) - self._logfile = self._config.get("logfile", DEFAULT_PATH + ".log") - logger.info(f"Configuration loaded:\n{self._config}") + self.logger.debug(f"Configuration loaded:\n{self._config}") self._interval = kwargs.get( "interval", (self._config.get("interval", DEFAULT_INTERVAL)) ) # milliseconds @@ -125,7 +125,7 @@ def _readconfig(self, cfile: str) -> dict: with open(cfile, "r", encoding="utf-8") as jsonfile: config = load(jsonfile) except (OSError, JSONDecodeError) as err: - logger.error(f"Unable to read configuration file:\n{err}") + self.logger.error(f"Unable to read configuration file:\n{err}") return { "interval": DEFAULT_INTERVAL, "timeout": DEFAULT_TIMEOUT, @@ -154,7 +154,7 @@ def start(self): Start streaming. """ - logger.info("UBX Simulator started") + self.logger.info("UBX Simulator started") self._stopevent.clear() self._msgfactory_thread = Thread( target=self._msgfactory, @@ -187,7 +187,7 @@ def stop(self): self._mainloop_thread.join() if self._msgfactory_thread is not None: self._msgfactory_thread.join() - logger.info("UBX Simulator stopped") + self.logger.info("UBX Simulator stopped") def _mainloop(self, stop: Event, outq: Queue, inq: Queue): """ @@ -204,7 +204,7 @@ def _mainloop(self, stop: Event, outq: Queue, inq: Queue): outq.task_done() while not inq.empty(): data = inq.get() - self._ubxhandler(data, outq) + self._datahandler(data, outq) inq.task_done() sleep(self._interval / 10000) @@ -282,26 +282,27 @@ def _msgfactory( sleep(self._interval / 1000) loops = (loops + 1) % 1024 - def _ubxhandler(self, data: UBXMessage, outq: Queue): + def _datahandler(self, data: bytes, outq: Queue): """ THREADED - Process incoming UBX data. + Process incoming UBX or RTCM3 data. TODO enhance to mimic wider range of command or poll responses. - :param bytes data: UBXMessage + :param bytes data: UBXMessage or RTCMMessage :param Queue outq: output queue """ if data is None: return - self._do_ackack(data, outq) + if isinstance(data, UBXMessage): + self._do_ackack(data, outq) - if data.identity == "MON-VER": - self._do_monver(outq) - if data.identity == "CFG-RATE": - self._do_cfgrate(data, outq) + if data.identity == "MON-VER": + self._do_monver(outq) + if data.identity == "CFG-RATE": + self._do_cfgrate(data, outq) def _do_send(self, msg: UBXMessage, outq: Queue): """ @@ -313,7 +314,7 @@ def _do_send(self, msg: UBXMessage, outq: Queue): raw = msg.serialize() outq.put(raw) - logger.info(f"Response Sent by Simulator:\n{raw}\n{msg}") + self.logger.info(f"Response Sent by Simulator:\n{raw}\n{msg}") def _do_ackack(self, data: UBXMessage, outq: Queue): """ @@ -436,21 +437,55 @@ def write(self, data: bytes): :param bytes data: UBX data """ + prot = protocol(data) try: - msgmode = getinputmode(data) # returns SET or POLL - ubx = UBXReader.parse(data, msgmode=msgmode) - self._inqueue.put(ubx) - val = ("Valid UBX", ubx) - except (UBXParseError, UBXMessageError) as err: + if prot == RTCM3_PROTOCOL: + rtm = RTCMReader.parse(data) + self._inqueue.put(rtm) + val = ("RTCM", rtm) + elif prot == UBX_PROTOCOL: + msgmode = getinputmode(data) # returns SET or POLL + ubx = UBXReader.parse(data, msgmode=msgmode) + self._inqueue.put(ubx) + val = ("UBX", ubx) + else: + val = (f"Other Protocol {prot}", None) + except ( + UBXParseError, + UBXMessageError, + RTCMParseError, + RTCMMessageError, + ) as err: val = ("Invalid/Unknown Data:", f"{err}") - logger.info( + self.logger.debug( f"{val[0]} Data Received by Simulator:\n{escapeall(data)}\n{val[1]}" ) + def close(self): + """ + Close dummy serial stream. + """ + + self.stop() + @property - def is_open(self): + def is_open(self) -> bool: """ Return status. + + :return: true or false + :rtype: bool """ return self._mainloop_thread is not None + + @property + def in_waiting(self) -> int: + """ + Return number of bytes in buffer. + + :return: buffer length + :rtype: int + """ + + return len(self._buffer) diff --git a/src/pygnssutils/ubxsimulator_cli.py b/src/pygnssutils/ubxsimulator_cli.py index 1510afa..85a0d42 100644 --- a/src/pygnssutils/ubxsimulator_cli.py +++ b/src/pygnssutils/ubxsimulator_cli.py @@ -11,21 +11,18 @@ """ from argparse import ArgumentDefaultsHelpFormatter, ArgumentParser +from logging import getLogger +from os import getenv from pyubx2 import UBXReader from pygnssutils._version import __version__ as VERSION -from pygnssutils.globals import ( - CLIAPP, - EPILOG, - VERBOSITY_CRITICAL, - VERBOSITY_DEBUG, - VERBOSITY_HIGH, - VERBOSITY_LOW, - VERBOSITY_MEDIUM, -) +from pygnssutils.globals import CLIAPP, EPILOG, UBXSIMULATOR +from pygnssutils.helpers import set_common_args from pygnssutils.ubxsimulator import DEFAULT_PATH, UBXSimulator +SIMCONFIG = f"{UBXSIMULATOR.upper()}_JSON" + def main(): """ @@ -43,8 +40,8 @@ def main(): "--interval", required=False, type=float, - help="Simulated navigation interval in seconds (Hz = 1/interval)", - default=1, + help="Simulated navigation interval in milliseconds (Hz = 1000/interval)", + default=1000, ) ap.add_argument( "-T", @@ -55,53 +52,32 @@ def main(): default=3, ) ap.add_argument( - "-C", - "--configfile", + "--simconfigfile", required=False, type=str, - help="Fully qualified path to json configuration file", - default=DEFAULT_PATH + ".json", - ) - ap.add_argument( - "--verbosity", - required=False, help=( - f"Log message verbosity " - f"{VERBOSITY_CRITICAL} = critical, " - f"{VERBOSITY_LOW} = low (error), " - f"{VERBOSITY_MEDIUM} = medium (warning), " - f"{VERBOSITY_HIGH} = high (info), {VERBOSITY_DEBUG} = debug" + "Fully qualified path to simulator json configuration file " + f"(will use environment variable {SIMCONFIG} if set)" ), - type=int, - choices=[ - VERBOSITY_CRITICAL, - VERBOSITY_LOW, - VERBOSITY_MEDIUM, - VERBOSITY_HIGH, - VERBOSITY_DEBUG, - ], - default=VERBOSITY_MEDIUM, - ) - ap.add_argument( - "--logtofile", - required=False, - help="fully qualified log file name, or '' for no log file", - type=str, - default="", + default=getenv(SIMCONFIG, DEFAULT_PATH + ".json"), ) + kwargs = set_common_args("ubxsimulator", ap) - kwargs = vars(ap.parse_args()) + logger = getLogger("pygnssutils.ubxsimulator") + kwargs["configfile"] = kwargs.pop( + "simconfigfile", getenv(SIMCONFIG, DEFAULT_PATH + ".json") + ) with UBXSimulator(CLIAPP, **kwargs) as stream: try: ubr = UBXReader(stream) i = 0 for _, parsed in ubr: - print(parsed) + logger.debug(str(parsed)) i += 1 except KeyboardInterrupt: - print(f"Terminated by user, {i} messages read") + logger.info(f"Terminated by user, {i} messages processed") if __name__ == "__main__": diff --git a/tests/gnssdump.conf b/tests/gnssdump.conf new file mode 100644 index 0000000..9ea9277 --- /dev/null +++ b/tests/gnssdump.conf @@ -0,0 +1,5 @@ +filename=pygpsdata-MIXED3.log +verbosity=3 +format=2 +clioutput=1 +output=testfile.bin \ No newline at end of file diff --git a/tests/test_gnssdump.py b/tests/test_gnssdump.py index d062825..3324e06 100644 --- a/tests/test_gnssdump.py +++ b/tests/test_gnssdump.py @@ -161,7 +161,7 @@ def testgnssdump_outputhandler(self): protfilter=2, msgfilter="NAV-PVT", limit=2, - outputhandler="lambda msg: print(f'lat: {msg.lat}, lon: {msg.lon}')", + output=eval("lambda msg: print(f'lat: {msg.lat}, lon: {msg.lon}')"), ) gns.run() sys.stdout = saved_stdout @@ -181,18 +181,18 @@ def testgnssdump_errorhandler(self): sys.stdout = saved_stdout print(f"output = {out.getvalue().strip()}") - def testgnssdump_outfile(self): - saved_stdout = sys.stdout - out = StringIO() - sys.stdout = out - gns = GNSSStreamer( - filename=self.mixedfile, - format=FORMAT_PARSED, - outfile=self.outfilename, - ) - gns.run() - sys.stdout = saved_stdout - print(f"output = {out.getvalue().strip()}") + # def testgnssdump_outfile(self): + # saved_stdout = sys.stdout + # out = StringIO() + # sys.stdout = out + # gns = GNSSStreamer( + # filename=self.mixedfile, + # format=FORMAT_PARSED, + # outfile=self.outfilename, + # ) + # gns.run() + # sys.stdout = saved_stdout + # print(f"output = {out.getvalue().strip()}") # def testgnssdump_outputhandler_file1(self): @@ -217,7 +217,7 @@ def testgnssdump_outputhandler_file2(self): gns = GNSSStreamer( filename=self.mixedfile, format=FORMAT_PARSEDSTRING, - outputhandler=ofile, + output=ofile, ) gns.run() sys.stdout = saved_stdout @@ -231,7 +231,7 @@ def testgnssdump_outputhandler_file3(self): gns = GNSSStreamer( filename=self.mixedfile, format=FORMAT_BINARY, - outputhandler=ofile, + output=ofile, ) gns.run() sys.stdout = saved_stdout @@ -245,7 +245,7 @@ def testgnssdump_outputhandler_file4(self): gns = GNSSStreamer( filename=self.mixedfile, format=FORMAT_HEX, - outputhandler=ofile, + output=ofile, ) gns.run() sys.stdout = saved_stdout @@ -259,7 +259,7 @@ def testgnssdump_outputhandler_file5(self): gns = GNSSStreamer( filename=self.mixedfile, format=FORMAT_HEXTABLE, - outputhandler=ofile, + output=ofile, ) gns.run() sys.stdout = saved_stdout @@ -273,7 +273,7 @@ def testgnssdump_outputhandler_file6(self): gns = GNSSStreamer( filename=self.mixedfile, format=FORMAT_JSON, - outputhandler=ofile, + output=ofile, ) gns.run() sys.stdout = saved_stdout diff --git a/tests/test_static.py b/tests/test_static.py index 1825f14..7cbf130 100644 --- a/tests/test_static.py +++ b/tests/test_static.py @@ -10,6 +10,8 @@ # pylint: disable=line-too-long, invalid-name, missing-docstring, no-member +from os import path +from pathlib import Path import unittest from socket import AF_INET, AF_INET6 from pyubx2 import UBXReader, itow2utc @@ -22,6 +24,7 @@ ipprot2str, format_json, get_mp_distance, + parse_config, ) from pygnssutils.mqttmessage import MQTTMessage from tests.test_sourcetable import TESTSRT @@ -181,6 +184,17 @@ def testparsemqttfreq(self): # test MQTTMessage constructor with self.assertRaises(ValueError): MQTTMessage(topic, payload=b"arsebiscuits") + def testparseconfig(self): + EXPECTED_RESULT = { + "filename": "pygpsdata-MIXED3.log", + "verbosity": "3", + "format": "2", + "clioutput": "1", + "output": "testfile.bin", + } + cfg = parse_config(path.join(path.dirname(__file__), "gnssdump.conf")) + self.assertEqual(cfg, EXPECTED_RESULT) + if __name__ == "__main__": # import sys;sys.argv = ['', 'Test.testName']