From cfe0781663050dff1a3718b800bcf3cf49728f33 Mon Sep 17 00:00:00 2001 From: semuadmin <28569967+semuadmin@users.noreply.github.com> Date: Thu, 11 Apr 2024 18:00:34 +0100 Subject: [PATCH 01/11] refine nested group parsing --- RELEASE_NOTES.md | 6 ++++++ src/pyrtcm/rtcmmessage.py | 10 ++++++---- src/pyrtcm/rtcmtypes_get.py | 12 ++++++------ 3 files changed, 18 insertions(+), 10 deletions(-) diff --git a/RELEASE_NOTES.md b/RELEASE_NOTES.md index dcb44ef..1e5751a 100644 --- a/RELEASE_NOTES.md +++ b/RELEASE_NOTES.md @@ -1,5 +1,11 @@ # pyrtcm Release Notes +### RELEASE 1.0.19 + +ENHANCEMENTS + +1. Internal streamlining of nested group parsing - no functional changes. + ### RELEASE 1.0.18 FIXES: diff --git a/src/pyrtcm/rtcmmessage.py b/src/pyrtcm/rtcmmessage.py index d080014..8f4e59f 100644 --- a/src/pyrtcm/rtcmmessage.py +++ b/src/pyrtcm/rtcmmessage.py @@ -130,10 +130,12 @@ def _set_attribute_group(self, att: tuple, offset: int, index: list) -> tuple: if isinstance(numr, int): # fixed number of repeats rng = numr else: # number of repeats is defined in named attribute - # if attribute is within a group - # append group index to name e.g. "DF379_01", "IDF023_03" - if numr in ("DF379", "IDF023"): - numr += f"_{index[-1]:02d}" + # "+n" suffix signifies that one or more nested group indices + # must be appended to name e.g. "DF379_01", "IDF023_03" + if "+" in numr: + numr, nestlevel = numr.split("+") + for i in range(int(nestlevel)): + numr += f"_{index[i]:02d}" rng = getattr(self, numr) if numr == "IDF035": # 4076_201 range is n-1 rng += 1 diff --git a/src/pyrtcm/rtcmtypes_get.py b/src/pyrtcm/rtcmtypes_get.py index eb91934..ba089c5 100644 --- a/src/pyrtcm/rtcmtypes_get.py +++ b/src/pyrtcm/rtcmtypes_get.py @@ -492,8 +492,8 @@ { "IDF011": "GNSS Satellite ID", "IDF023": "No. of Biases Processed", - "groupbias": ( - "IDF023", + "groupbias": ( # nested group + "IDF023+1", # +1 signifies 1 nested group index must be added { "IDF024": "GNSS Signal and Tracking Mode Identifier", "IDF025": "Code Bias", @@ -522,8 +522,8 @@ "IDF023": "No. of Biases Processed", "IDF026": "Yaw Angle", "IDF027": "Yaw Rate", - "groupbias": ( - "IDF023", + "groupbias": ( # nested group + "IDF023+1", # +1 signifies 1 nested group index must be added { "IDF024": "GNSS Signal and Tracking Mode Identifier", "IDF029": "Signal Integer Indicator", @@ -1423,7 +1423,7 @@ "DF068": "GPS Satellite ID", "DF379": "No. of Code Biases Processed", "groupbias": ( # nested group - "DF379", + "DF379+1", # +1 signifies 1 nested group index must be added { "DF380": "GPS Signal and Tracking Mode Indicator", "DF383": "Code Bias", @@ -1551,7 +1551,7 @@ "DF384": "GLONASS Satellite ID", "DF379": "No. of Code Biases Processed", "groupbias": ( # nested group - "DF379", + "DF379+1", # +1 signifies 1 nested group index must be added { "DF381": "GLONASS Signal and Tracking Mode Indicator", "DF383": "Code Bias", From 4fecc449b04b8deac62c60ed6af4e617bd67d95e Mon Sep 17 00:00:00 2001 From: semuadmin <28569967+semuadmin@users.noreply.github.com> Date: Tue, 16 Apr 2024 08:57:32 +0100 Subject: [PATCH 02/11] streamline boolean group handling --- src/pyrtcm/rtcmmessage.py | 12 +++++++++--- src/pyrtcm/rtcmtypes_core.py | 27 +++++++++++++++++++++++---- src/pyrtcm/rtcmtypes_get.py | 16 ++++++++-------- 3 files changed, 40 insertions(+), 15 deletions(-) diff --git a/src/pyrtcm/rtcmmessage.py b/src/pyrtcm/rtcmmessage.py index 8f4e59f..afedda8 100644 --- a/src/pyrtcm/rtcmmessage.py +++ b/src/pyrtcm/rtcmmessage.py @@ -24,9 +24,9 @@ sat2prn, ) from pyrtcm.rtcmtypes_core import ( - ATT_BOOL, ATT_NCELL, ATT_NSAT, + BOOL, NCELL, NHARMCOEFFC, NHARMCOEFFS, @@ -37,6 +37,8 @@ RTCM_MSGIDS, ) +BOOL = "B" + class RTCMMessage: """RTCM Message Class.""" @@ -173,10 +175,14 @@ def _set_attribute_single( # pylint: disable=no-member # if attribute is part of a (nested) repeating group, suffix name with index - # one index for each nested level (unless it's a 'boolean' group) + # one index for each nested level (unless it's a "BOOL" group) + if "+" in key: + key, marker = key.split("+") + else: + marker = "" keyr = key for i in index: - if i > 0 and keyr not in ATT_BOOL: + if i > 0 and marker != BOOL: # BOOL signifies occurrence = 0 or 1 keyr += f"_{i:02d}" # get value of required number of bits at current payload offset diff --git a/src/pyrtcm/rtcmtypes_core.py b/src/pyrtcm/rtcmtypes_core.py index 6b597c4..818e0af 100644 --- a/src/pyrtcm/rtcmtypes_core.py +++ b/src/pyrtcm/rtcmtypes_core.py @@ -10,7 +10,27 @@ # pylint: disable=line-too-long -NMEA_HDR = [b"\x24\x47", b"\x24\x50"] +NMEA_HDR = [ + b"$V", + b"$M", + b"$P", + b"$B", + b"$D", + b"$I", + b"$L", + b"$G", + b"$F", + b"$S", + b"$H", + b"$R", + b"$E", + b"$Y", + b"$A", + b"$C", + b"$Z", + b"$T", + b"$W", +] UBX_HDR = b"\xb5\x62" RTCM_HDR = b"\xd3" NMEA_PROTOCOL = 1 @@ -31,6 +51,7 @@ NRES = 16 # number of Residuals groups in MT1023 and MT1024 NHARMCOEFFC = "_NHarmCoeffC" # number of cosine harmonic coefficients in 4076 NHARMCOEFFS = "_NHarmCoeffS" # number of sine harmonic coefficients in 4076 +BOOL = "B" # Power of 2 scaling constants P2_P4 = 16 # 2**4 @@ -71,7 +92,7 @@ BIT12 = "BIT012" # 12 bit bit field BIT32 = "BIT032" # 32 bit bit field BIT64 = "BIT064" # 64 bit bit field -BITX = "BIT999" # variable bit field TODO check against usage +BITX = "BIT999" # variable bit field CHAR8 = "CHA008" # 8 bit characters, ISO 8859-1 (not limited to ASCII) INT6 = "INT006" # 6 bit 2’s complement integer INT8 = "INT008" # 8 bit 2’s complement integer @@ -903,8 +924,6 @@ "4095": "Ashtech", } -# list of 'group' attributes which have an occurrence of 0 or 1 -ATT_BOOL = ("DF423", "DF424", "DF425", "DF426") # list of MSM attributes to label if `labelmsm` is True ATT_NSAT = ["DF397", "DF398", "DF399", "DF419", "ExtSatInfo"] ATT_NCELL = [ diff --git a/src/pyrtcm/rtcmtypes_get.py b/src/pyrtcm/rtcmtypes_get.py index ba089c5..0f773bd 100644 --- a/src/pyrtcm/rtcmtypes_get.py +++ b/src/pyrtcm/rtcmtypes_get.py @@ -1991,27 +1991,27 @@ "DF422_3": "GLONASS FDMA signals mask L2 C/A", "DF422_4": "GLONASS FDMA signals mask L2 P", "groupL1CA": ( - "DF422_1", + "DF422_1", # occurrence either 0 or 1 { - "DF423": "GLONASS L1 C/A Code-Phase Bias", + "DF423+B": "GLONASS L1 C/A Code-Phase Bias", }, ), "groupL1P": ( - "DF422_2", + "DF422_2", # occurrence either 0 or 1 { - "DF424": "GLONASS L1 P Code-Phase Bias", + "DF424+B": "GLONASS L1 P Code-Phase Bias", }, ), "groupL2CA": ( - "DF422_3", + "DF422_3", # occurrence either 0 or 1 { - "DF425": "GLONASS L2 C/A Code-Phase Bias", + "DF425+B": "GLONASS L2 C/A Code-Phase Bias", }, ), "groupL2P": ( - "DF422_4", + "DF422_4", # occurrence either 0 or 1 { - "DF426": "GLONASS L2 P Code-Phase Bias", + "DF426+B": "GLONASS L2 P Code-Phase Bias", }, ), }, From be318b691d4e97093f10ee43ddbefddb97b85c88 Mon Sep 17 00:00:00 2001 From: semuadmin <28569967+semuadmin@users.noreply.github.com> Date: Tue, 16 Apr 2024 09:15:56 +0100 Subject: [PATCH 03/11] update to v1.0.19 --- .vscode/settings.json | 2 +- RELEASE_NOTES.md | 2 +- pyproject.toml | 2 +- src/pyrtcm/_version.py | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/.vscode/settings.json b/.vscode/settings.json index 0a30638..320115c 100644 --- a/.vscode/settings.json +++ b/.vscode/settings.json @@ -4,5 +4,5 @@ "editor.formatOnSave": true, "modulename": "${workspaceFolderBasename}", "distname": "${workspaceFolderBasename}", - "moduleversion": "1.0.18" + "moduleversion": "1.0.19" } \ No newline at end of file diff --git a/RELEASE_NOTES.md b/RELEASE_NOTES.md index 1e5751a..b5dfb08 100644 --- a/RELEASE_NOTES.md +++ b/RELEASE_NOTES.md @@ -4,7 +4,7 @@ ENHANCEMENTS -1. Internal streamlining of nested group parsing - no functional changes. +1. Minor internal streamlining of nested group parsing - no functional changes. ### RELEASE 1.0.18 diff --git a/pyproject.toml b/pyproject.toml index bfab38c..477ece0 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -7,7 +7,7 @@ name = "pyrtcm" authors = [{ name = "semuadmin", email = "semuadmin@semuconsulting.com" }] maintainers = [{ name = "semuadmin", email = "semuadmin@semuconsulting.com" }] description = "RTCM3 protocol parser" -version = "1.0.18" +version = "1.0.19" license = { file = "LICENSE" } readme = "README.md" requires-python = ">=3.8" diff --git a/src/pyrtcm/_version.py b/src/pyrtcm/_version.py index 2c88417..16e2d65 100644 --- a/src/pyrtcm/_version.py +++ b/src/pyrtcm/_version.py @@ -8,4 +8,4 @@ :license: BSD 3-Clause """ -__version__ = "1.0.18" +__version__ = "1.0.19" From 70f406e7f5b7997f54498f41b7a2aa4c67a394da Mon Sep 17 00:00:00 2001 From: semuadmin <28569967+semuadmin@users.noreply.github.com> Date: Thu, 18 Apr 2024 11:12:40 +0100 Subject: [PATCH 04/11] add ismsm property and msmparser example --- README.md | 5 +- examples/msmparser.py | 108 +++++++++++++++++++++++++++++++++++ src/pyrtcm/rtcmmessage.py | 25 +++++--- src/pyrtcm/rtcmtypes_core.py | 47 +++++++-------- tests/test_static.py | 15 ++++- 5 files changed, 167 insertions(+), 33 deletions(-) create mode 100644 examples/msmparser.py diff --git a/README.md b/README.md index 6c2e0f4..cc49699 100644 --- a/README.md +++ b/README.md @@ -35,7 +35,7 @@ This is an independent project and we have no affiliation whatsoever with the Ra ![Contributors](https://img.shields.io/github/contributors/semuconsulting/pyrtcm.svg) ![Open Issues](https://img.shields.io/github/issues-raw/semuconsulting/pyrtcm) -Parses RTCM3 messages into their constituent data fields - `DF002`, `DF003`, etc. Refer to the `RTCM_MSGIDS` dictionary in [`rtcmtypes_core.py`](https://github.com/semuconsulting/pyrtcm/blob/main/src/pyrtcm/rtcmtypes_core.py) for a list of message types currently implemented. Additional message types can be readily added - see [Extensibility](#extensibility). +Parses RTCM3 messages into their constituent data fields - `DF002`, `DF003`, etc. Refer to the `RTCM_MSGIDS` dictionary in [`rtcmtypes_core.py`](https://github.com/semuconsulting/pyrtcm/blob/main/src/pyrtcm/rtcmtypes_core.py#L695) for a list of message types currently implemented. Additional message types can be readily added - see [Extensibility](#extensibility). Sphinx API Documentation in HTML format is available at [https://www.semuconsulting.com/pyrtcm](https://www.semuconsulting.com/pyrtcm). @@ -236,7 +236,8 @@ The following examples are available in the /examples folder: 1. `rtcmpoller.py` - illustrates how to read and display RTCM messages 'concurrently' with other tasks using threads and queues. This represents a useful generic pattern for many end user applications. 1. `rtcmfile.py` - illustrates how to stream RTCM data from binary log file. -1. `rtcmsocket.py` illustrates how to implement a TCP Socket reader for RTCM messages using RTCMReader iterator functionality. +1. `rtcmsocket.py` - illustrates how to implement a TCP Socket reader for RTCM messages using RTCMReader iterator functionality. +1. `msmparser.py` - illustrates how to parse RTCM3 MSM (multiple signal messages) into a series of iterable data arrays keyed on satellite PRN and signal ID. 1. `ntripclient.py` - illustrates a simple [NTRIP](https://en.wikipedia.org/wiki/Networked_Transport_of_RTCM_via_Internet_Protocol) client using pyrtcm to parse the RTCM3 output. --- diff --git a/examples/msmparser.py b/examples/msmparser.py new file mode 100644 index 0000000..2418b6d --- /dev/null +++ b/examples/msmparser.py @@ -0,0 +1,108 @@ +""" +msmparser.py + +Each RTCM3 MSM message contains data for multiple satellites and cells +(combination of satellite and signal). The mapping between each +data item and its corresponding satellite PRN or signal ID can be performed +by pyrtcm helper functions `sat2prn` and `cell2prn`. + +pyrtcm parses MSM messages into a flat data structure, with repeating +element names suffixed by a 2-digit index (e.g. `DF405_02`) and +(optionally) labelled with their corresponding satellite PRN and signal ID +e.g. `DF405_02(005,2L)` + +It is sometimes more convenient to parse the data into one or more +iterable arrays, with each array element corresponding to a particular +satellite or cell. + +This example illustrates how to parse MSM messages from a binary data log +into a series of iterable data arrays keyed on satellite PRN and signal ID. + +The output data structure is a tuple containing: + +(metadata, [satellite data array], [cell data array]). + +Arrays could just as easily be numpy arrays if preferred. + +Created on 18 Apr 2024 + +:author: semuadmin +:copyright: SEMU Consulting © 2024 +:license: BSD 3-Clause +""" + +from pyrtcm import ( + ATT_NCELL, + ATT_NSAT, + RTCMMessage, + RTCMReader, + cell2prn, + sat2prn, +) + +# binary log of RTCM3 messages e.g. from NTRIP caster or ZED-F9P receiver +INFILE = "./tests/pygpsdata-RTCM3.log" + +# map of GNSS name, epoch attribute name +GNSS = { + "107": ("GPS", "DF004"), + "108": ("GLONASS", "DF034"), + "109": ("GALILEO", "DF248"), + "110": ("SBAS", "DF004"), + "111": ("QZSS", "DF428"), + "112": ("BEIDOU", "DF427"), + "113": ("NAVIC", "DF546"), +} + + +def process_msm(msg: RTCMMessage) -> tuple: + """ + Process individual MSM message. + + :return: tuple of (metadata, sat data array, cell data array) + :rtype: tuple + """ + + satmap = sat2prn(msg) # array of satellite PRN + cellmap = cell2prn(msg) # array of cells (satellite PRN, signal ID) + meta = {} + gmap = GNSS[msg.identity[0:3]] + meta["identity"] = msg.identity + meta["gnss"] = gmap[0] + meta["station"] = msg.DF003 + meta["epoch"] = getattr(msg, gmap[1]) + meta["sats"] = msg.NSat + meta["cells"] = msg.NCell + msmsats = [] + for i in range(msg.NSat): # iterate through satellites + sats = {} + sats["PRN"] = satmap[i + 1] + for attr in ATT_NSAT: + if hasattr(msg, f"{attr}_{i+1:02d}"): + sats[attr] = getattr(msg, f"{attr}_{i+1:02d}") + msmsats.append(sats) + msmcells = [] + for i in range(msg.NCell): # iterate through satellite/signal cells + cells = {} + cells["PRN"], cells["SIGNAL"] = cellmap[i + 1] + for attr in ATT_NCELL: + if hasattr(msg, f"{attr}_{i+1:02d}"): + cells[attr] = getattr(msg, f"{attr}_{i+1:02d}") + msmcells.append(cells) + + return (meta, msmsats, msmcells) + + +with open(INFILE, "rb") as stream: + rtr = RTCMReader(stream) + for raw, parsed in rtr: + if parsed is not None: + if parsed.ismsm: + # print(parsed) + msmarray = process_msm(parsed) + print(msmarray) + + # to then iterate through a specific data item, + # e.g. the satellite DF398 (rough range) value: + for sat in msmarray[1]: # satellite data array + print(f"PRN {sat["PRN"]}: {sat["DF398"]}") diff --git a/src/pyrtcm/rtcmmessage.py b/src/pyrtcm/rtcmmessage.py index afedda8..762da17 100644 --- a/src/pyrtcm/rtcmmessage.py +++ b/src/pyrtcm/rtcmmessage.py @@ -275,14 +275,11 @@ def __str__(self) -> str: # if MSM message and labelmsm flag is set, # label NSAT and NCELL group attributes with - # corresponding satellite PRN and signal ID - is_msm = False + # corresponding satellite PRN and signal ID (RINEX code or freq band) if not self._unknown: - if self._labelmsm and "MSM" in RTCM_MSGIDS[self.identity]: + if self._labelmsm and self.ismsm: sats = sat2prn(self) - sigcode = 0 if self._labelmsm == 2 else 1 # freq band or RINEX code - cells = cell2prn(self, sigcode) - is_msm = True + cells = cell2prn(self, 0 if self._labelmsm == 2 else 1) stg = f" str: val = escapeall(val) # label MSM NSAT and NCELL group attributes lbl = "" - if is_msm: + if self._labelmsm and self.ismsm: aname = att2name(att) if aname in ATT_NSAT: prn = sats[att2idx(att)] @@ -380,3 +377,17 @@ def payload(self) -> bytes: """ return self._payload + + @property + def ismsm(self) -> bool: + """ + Check if message is Multiple Signal Message (MSM) type. + + :return: True/False + :rtype: bool + """ + + try: + return "MSM" in RTCM_MSGIDS[self.identity] + except KeyError: + return False diff --git a/src/pyrtcm/rtcmtypes_core.py b/src/pyrtcm/rtcmtypes_core.py index 818e0af..879cd49 100644 --- a/src/pyrtcm/rtcmtypes_core.py +++ b/src/pyrtcm/rtcmtypes_core.py @@ -845,12 +845,13 @@ "1230": "GLONASS L1 and L2 Code-Phase Biases", # "1240-1263": "SSR Messages" # - # Proprietary messages have not yet been implemented + # NB: Only those proprietary messages with public + # domain definitions have been implemented. # # "4001-4095":"Proprietary Messages", - "4072": "Mitsubishi Electric Corp", - "4073": "Unicore Communications Inc", - "4075": "Alberding GmbH", + # "4072": "Mitsubishi Electric Corp", + # "4073": "Unicore Communications Inc", + # "4075": "Alberding GmbH", # "4076": "International GNSS Service (IGS)", "4076_021": "GPS SSR Orbit Correction", "4076_022": "GPS SSR Clock Correction", @@ -903,25 +904,25 @@ # "4076_141-160": "Reserved for NavIC/IRNSS", # "4076_161-200": "Reserved", "4076_201": "GNSS SSR Ionosphere VTEC Spherical Harmonics", - "4077": "Hemisphere GNSS Inc.", - "4078": "ComNav Technology Ltd.", - "4079": "SubCarrier Systems Corp. (SCSC) The makers of SNIP", - "4080": "NavCom Technology, Inc.", - "4081": "Seoul National University GNSS Lab", - "4082": "Cooperative Research Centre for Spatial Information", - "4083": "German Aerospace Center, Institute of Communications and Navigation (DLR)", - "4084": "Geodetics, Inc.", - "4085": "European GNSS Supervisory Authority", - "4086": "inPosition GmbH", - "4087": "Fugro", - "4088": "IfEN GmbH", - "4089": "Septentrio Satellite Navigation", - "4090": "Geo++", - "4091": "Topcon Positioning Systems", - "4092": "Leica Geosystems", - "4093": "NovAtel Inc.", - "4094": "Trimble Navigation Ltd.", - "4095": "Ashtech", + # "4077": "Hemisphere GNSS Inc.", + # "4078": "ComNav Technology Ltd.", + # "4079": "SubCarrier Systems Corp. (SCSC) The makers of SNIP", + # "4080": "NavCom Technology, Inc.", + # "4081": "Seoul National University GNSS Lab", + # "4082": "Cooperative Research Centre for Spatial Information", + # "4083": "German Aerospace Center, Institute of Communications and Navigation (DLR)", + # "4084": "Geodetics, Inc.", + # "4085": "European GNSS Supervisory Authority", + # "4086": "inPosition GmbH", + # "4087": "Fugro", + # "4088": "IfEN GmbH", + # "4089": "Septentrio Satellite Navigation", + # "4090": "Geo++", + # "4091": "Topcon Positioning Systems", + # "4092": "Leica Geosystems", + # "4093": "NovAtel Inc.", + # "4094": "Trimble Navigation Ltd.", + # "4095": "Ashtech", } # list of MSM attributes to label if `labelmsm` is True diff --git a/tests/test_static.py b/tests/test_static.py index 1113137..4045788 100644 --- a/tests/test_static.py +++ b/tests/test_static.py @@ -7,12 +7,13 @@ @author: semuadmin """ + # pylint: disable=line-too-long, invalid-name, missing-docstring, no-member import os import unittest -from pyrtcm import RTCM_DATA_FIELDS +from pyrtcm import RTCM_DATA_FIELDS, RTCMMessage import pyrtcm.rtcmtypes_core as rtt from pyrtcm.rtcmhelpers import ( hextable, @@ -184,6 +185,18 @@ def testescapeall(self): print(res) self.assertEqual(res, EXPECTED_RESULT) + def testismsm(self): + msg1077 = RTCMMessage( + payload=b"CP\x000\xab\x88\xa6\x00\x00\x05GX\x02\x00\x00\x00\x00 \x00\x80\x00\x7f\x7fZZZ\x8aB\x1a\x82Z\x92Z8\x00\x00\x00\x00\x00\r\x11\xe1\xa4tf:f\xe3L,\xb1~\x9d\xf6\x87\xaf\xa0\xee\xff\x98\x14(B!A\xfc\xa9\xfaX\x96\n\x89K\x91\x971\x19c\xb6\x04\xa9\xe1F9l\xc3\x8ee\xd8\xe1\xaas\xa5\x1f?\xe9yc\x97\x98\xc6\x1f`)\xc9\xdck\xa5\x8e\xbcZ\x02SP\x82Yu\x06ex\x06Y\x00x\x10N\xf8T\x00\x05\xb0\xfa\x83\x90\xa2\x83\x89\xdc\xfc\xf1l|\xfeW~\\\xdb~h\x1c\x06\xc3\x82\x07#\x07\xfa\xe6pz\xf0\x03\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xa9:\xaa\xaa\xaa\xa0\x00\x0bB`\xac'\t\xc2P\xb4.\x0b\x82p\x88-\t\x81\xf0\xb4.\nB\xdf\x8d\xc1k\xef\xf7\xde\xb7\xfa\xf0\x18\x13'\xf5/\xea\xa2J\xe4\x99\"T\x04\xb8\x19\xec\xb5Y\xdes\xbc\x10\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00" + ) + msg1007 = RTCMMessage(payload=b">\xf4\xd2\x03ABC\xea") + msg4072 = RTCMMessage( + payload=b"\xfe\x80\x01\x00\x00\x00\x13\n\xb8\x8a@\x00\x00\x08\x00\x00\x00\x00\x00\x00\x00\x01\xff\x9f\x00\x16\x02\x00\xfe\\\x00\x19\x02\x01\xfe\xdd\x00\x1d\x03\x00\x02\x86\x00\x13\x05\x00\x00\x00\x01\x90\x06\x00\x03\xf7\x00\x1a\x06\x01\x04%\x00\x1e" + ) + self.assertTrue(msg1077.ismsm) + self.assertFalse(msg1007.ismsm) + self.assertFalse(msg4072.ismsm) + if __name__ == "__main__": # import sys;sys.argv = ['', 'Test.testName'] From 2fe21692f7f93c5ea50623905ab90ca0135689b6 Mon Sep 17 00:00:00 2001 From: semuadmin <28569967+semuadmin@users.noreply.github.com> Date: Thu, 18 Apr 2024 11:27:44 +0100 Subject: [PATCH 05/11] update msmparser example --- examples/msmparser.py | 44 ++++++++++++++++++++++++++++--------------- 1 file changed, 29 insertions(+), 15 deletions(-) diff --git a/examples/msmparser.py b/examples/msmparser.py index 2418b6d..ad2d5e5 100644 --- a/examples/msmparser.py +++ b/examples/msmparser.py @@ -1,6 +1,10 @@ """ msmparser.py +Usage: + +python3 msmparser.py "../tests/pygpsdata-RTCM3.log" + Each RTCM3 MSM message contains data for multiple satellites and cells (combination of satellite and signal). The mapping between each data item and its corresponding satellite PRN or signal ID can be performed @@ -31,6 +35,8 @@ :license: BSD 3-Clause """ +from sys import argv + from pyrtcm import ( ATT_NCELL, ATT_NSAT, @@ -40,9 +46,6 @@ sat2prn, ) -# binary log of RTCM3 messages e.g. from NTRIP caster or ZED-F9P receiver -INFILE = "./tests/pygpsdata-RTCM3.log" - # map of GNSS name, epoch attribute name GNSS = { "107": ("GPS", "DF004"), @@ -93,16 +96,27 @@ def process_msm(msg: RTCMMessage) -> tuple: return (meta, msmsats, msmcells) -with open(INFILE, "rb") as stream: - rtr = RTCMReader(stream) - for raw, parsed in rtr: - if parsed is not None: - if parsed.ismsm: - # print(parsed) - msmarray = process_msm(parsed) - print(msmarray) +def main(fname: str): + """ + Main routine. + + :param str fname: fully qualified path to input file + """ + + with open(fname, "rb") as stream: + rtr = RTCMReader(stream) + for _, parsed in rtr: + if parsed is not None: + if parsed.ismsm: + # print(parsed) + msmarray = process_msm(parsed) + print(msmarray) + + # to then iterate through a specific data item, + # e.g. the satellite DF398 (rough range) value: + for sat in msmarray[1]: # satellite data array + print(f"PRN {sat["PRN"]}: {sat["DF398"]}") + +if __name__ == "__main__": - # to then iterate through a specific data item, - # e.g. the satellite DF398 (rough range) value: - for sat in msmarray[1]: # satellite data array - print(f"PRN {sat["PRN"]}: {sat["DF398"]}") + main(argv[1]) From 1a266743ff9b5a47e1698a9cb5f3b5b837fe0cbc Mon Sep 17 00:00:00 2001 From: semuadmin <28569967+semuadmin@users.noreply.github.com> Date: Thu, 18 Apr 2024 13:05:53 +0100 Subject: [PATCH 06/11] update msmparser example --- examples/msmparser.py | 49 +++++++++++++++++++------------------------ 1 file changed, 22 insertions(+), 27 deletions(-) diff --git a/examples/msmparser.py b/examples/msmparser.py index ad2d5e5..0c157eb 100644 --- a/examples/msmparser.py +++ b/examples/msmparser.py @@ -5,16 +5,16 @@ python3 msmparser.py "../tests/pygpsdata-RTCM3.log" +pyrtcm parses MSM messages into a flat data structure, with repeating +element names suffixed by a 2-digit index (e.g. `DF405_02`) and +(optionally) labelled with their corresponding satellite PRN and signal ID +e.g. `DF405_02(005,2L)`. Note that indices start at 1, not 0. + Each RTCM3 MSM message contains data for multiple satellites and cells (combination of satellite and signal). The mapping between each data item and its corresponding satellite PRN or signal ID can be performed by pyrtcm helper functions `sat2prn` and `cell2prn`. -pyrtcm parses MSM messages into a flat data structure, with repeating -element names suffixed by a 2-digit index (e.g. `DF405_02`) and -(optionally) labelled with their corresponding satellite PRN and signal ID -e.g. `DF405_02(005,2L)` - It is sometimes more convenient to parse the data into one or more iterable arrays, with each array element corresponding to a particular satellite or cell. @@ -37,17 +37,12 @@ from sys import argv -from pyrtcm import ( - ATT_NCELL, - ATT_NSAT, - RTCMMessage, - RTCMReader, - cell2prn, - sat2prn, -) - -# map of GNSS name, epoch attribute name -GNSS = { +from pyrtcm import ATT_NCELL # list of all sat attribute names +from pyrtcm import ATT_NSAT # list of all cell attribute names +from pyrtcm import RTCMMessage, RTCMReader, cell2prn, sat2prn + +# map of msg identity to GNSS name, epoch attribute name +GNSSMAP = { "107": ("GPS", "DF004"), "108": ("GLONASS", "DF034"), "109": ("GALILEO", "DF248"), @@ -66,10 +61,10 @@ def process_msm(msg: RTCMMessage) -> tuple: :rtype: tuple """ - satmap = sat2prn(msg) # array of satellite PRN - cellmap = cell2prn(msg) # array of cells (satellite PRN, signal ID) + satmap = sat2prn(msg) # maps indices to satellite PRNs + cellmap = cell2prn(msg) # maps indices to cells (satellite PRN, signal ID) meta = {} - gmap = GNSS[msg.identity[0:3]] + gmap = GNSSMAP[msg.identity[0:3]] meta["identity"] = msg.identity meta["gnss"] = gmap[0] meta["station"] = msg.DF003 @@ -77,20 +72,20 @@ def process_msm(msg: RTCMMessage) -> tuple: meta["sats"] = msg.NSat meta["cells"] = msg.NCell msmsats = [] - for i in range(msg.NSat): # iterate through satellites + for i in range(1, msg.NSat + 1): # iterate through satellites sats = {} - sats["PRN"] = satmap[i + 1] + sats["PRN"] = satmap[i] for attr in ATT_NSAT: - if hasattr(msg, f"{attr}_{i+1:02d}"): - sats[attr] = getattr(msg, f"{attr}_{i+1:02d}") + if hasattr(msg, f"{attr}_{i:02d}"): + sats[attr] = getattr(msg, f"{attr}_{i:02d}") msmsats.append(sats) msmcells = [] - for i in range(msg.NCell): # iterate through satellite/signal cells + for i in range(1, msg.NCell + 1): # iterate through cells (satellite/signal) cells = {} - cells["PRN"], cells["SIGNAL"] = cellmap[i + 1] + cells["PRN"], cells["SIGNAL"] = cellmap[i] for attr in ATT_NCELL: - if hasattr(msg, f"{attr}_{i+1:02d}"): - cells[attr] = getattr(msg, f"{attr}_{i+1:02d}") + if hasattr(msg, f"{attr}_{i:02d}"): + cells[attr] = getattr(msg, f"{attr}_{i:02d}") msmcells.append(cells) return (meta, msmsats, msmcells) From f715049ec40aad93c7338a0fd6278e76ba779d2f Mon Sep 17 00:00:00 2001 From: semuadmin <28569967+semuadmin@users.noreply.github.com> Date: Thu, 18 Apr 2024 13:26:37 +0100 Subject: [PATCH 07/11] update msmparser example --- examples/msmparser.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/examples/msmparser.py b/examples/msmparser.py index 0c157eb..a635e90 100644 --- a/examples/msmparser.py +++ b/examples/msmparser.py @@ -110,7 +110,8 @@ def main(fname: str): # to then iterate through a specific data item, # e.g. the satellite DF398 (rough range) value: for sat in msmarray[1]: # satellite data array - print(f"PRN {sat["PRN"]}: {sat["DF398"]}") + print(f'PRN {sat["PRN"]}: {sat["DF398"]}') + if __name__ == "__main__": From d4298003d26071a924a5edd0ca315fbc5a447813 Mon Sep 17 00:00:00 2001 From: semuadmin <28569967+semuadmin@users.noreply.github.com> Date: Fri, 3 May 2024 13:52:40 +0100 Subject: [PATCH 08/11] update readme --- README.md | 95 +++++++++++++++++++++++++++++++------------------------ 1 file changed, 54 insertions(+), 41 deletions(-) diff --git a/README.md b/README.md index cc49699..cab0a3b 100644 --- a/README.md +++ b/README.md @@ -97,32 +97,35 @@ Individual RTCM messages can then be read using the `RTCMReader.read()` function Example - Serial input: ```python ->>> from serial import Serial ->>> from pyrtcm import RTCMReader ->>> stream = Serial('/dev/tty.usbmodem14101', 9600, timeout=3) ->>> rtr = RTCMReader(stream) ->>> (raw_data, parsed_data) = rtr.read() ->>> print(parsed_data) +from serial import Serial +from pyrtcm import RTCMReader +stream = Serial('/dev/tty.usbmodem14101', 9600, timeout=3) +rtr = RTCMReader(stream) +(raw_data, parsed_data) = rtr.read() +print(parsed_data) +``` +``` , ``` Example - File input (using iterator). ```python ->>> from pyrtcm import RTCMReader ->>> stream = open('rtcmdata.log', 'rb') ->>> rtr = RTCMReader(stream) ->>> for (raw_data, parsed_data) in rtr: print(parsed_data) -... +from pyrtcm import RTCMReader +stream = open('rtcmdata.log', 'rb') +rtr = RTCMReader(stream) +for (raw_data, parsed_data) in rtr: + print(parsed_data) ``` Example - Socket input (using iterator): ```python ->>> import socket ->>> from pyrtcm import RTCMReader ->>> stream = socket.socket(socket.AF_INET, socket.SOCK_STREAM): ->>> stream.connect(("localhost", 50007)) ->>> rtr = RTCMReader(stream) ->>> for (raw_data, parsed_data) in rtr: print(parsed_data) +import socket +from pyrtcm import RTCMReader +stream = socket.socket(socket.AF_INET, socket.SOCK_STREAM): +stream.connect(("localhost", 50007)) +rtr = RTCMReader(stream) +for (raw_data, parsed_data) in rtr: + print(parsed_data) ``` --- @@ -134,22 +137,26 @@ You can parse individual RTCM messages using the static `RTCMReader.parse(data)` Example: ```python ->>> from pyrtcm import RTCMReader ->>> msg = RTCMReader.parse(b"\xd3\x00\x13>\xd0\x00\x03\x8aX\xd9I<\x87/4\x10\x9d\x07\xd6\xafH Z\xd7\xf7") ->>> print(msg) +from pyrtcm import RTCMReader +msg = RTCMReader.parse(b"\xd3\x00\x13>\xd0\x00\x03\x8aX\xd9I<\x87/4\x10\x9d\x07\xd6\xafH Z\xd7\xf7") +print(msg) +``` +``` ``` The `RTCMMessage` object exposes different public attributes depending on its message type or 'identity'. Attributes are defined as data fields (`DF002`, `DF003`, etc.) e.g. the `1087` multiple signal message (MSM) contains the following data fields: ```python ->>> print(msg) +print(msg) +print(msg.identity) +print(msg.DF034) +print(msg.DF419_03) +``` +``` ->>> msg.identity '1087' ->>> msg.DF034 42119001 ->>> msg.DF419_03 8 ``` @@ -158,15 +165,17 @@ Attributes within repeating groups are parsed with a two-digit suffix (`DF419_01 Helper methods are available to interpret the individual datafields: ```python ->>> from pyrtcm import RTCM_DATA_FIELDS, datasiz, datascale, datadesc ->>> dfname = "DF012" ->>> RTCM_DATA_FIELDS[dfname] +from pyrtcm import RTCM_DATA_FIELDS, datasiz, datascale, datadesc +dfname = "DF012" +print(RTCM_DATA_FIELDS[dfname]) +print(datasiz(dfname)) +print(datascale(dfname)) +print(datadesc(dfname)) +``` +``` (INT20, 0.0001, "GPS L1 PhaseRange - L1 Pseudorange") ->>> datasiz(dfname) # size in bits 20 ->>> datascale(dfname) # scaling factor 0.0001 ->>> datadesc(dfname) # description 'GPS L1 PhaseRange - L1 Pseudorange' ``` @@ -203,9 +212,11 @@ You can create an `RTCMMessage` object by calling the constructor with the follo Example: ```python ->>> from pyrtcm import RTCMMessage ->>> msg = RTCMMessage(payload=b">\xd0\x00\x03\x8aX\xd9I<\x87/4\x10\x9d\x07\xd6\xafH ") ->>> print(msg) +from pyrtcm import RTCMMessage +msg = RTCMMessage(payload=b">\xd0\x00\x03\x8aX\xd9I<\x87/4\x10\x9d\x07\xd6\xafH ") +print(msg) +``` +``` ``` @@ -217,16 +228,18 @@ The `RTCMMessage` class implements a `serialize()` method to convert a `RTCMMess e.g. to create and send a `1005` message type: ```python ->>> from serial import Serial ->>> serialOut = Serial('COM7', 38400, timeout=5) ->>> from pyrtcm import RTCMMessage ->>> msg = RTCMMessage(payload=b">\xd0\x00\x03\x8aX\xd9I<\x87/4\x10\x9d\x07\xd6\xafH ") ->>> print(msg) +from serial import Serial +from pyrtcm import RTCMMessage +serialOut = Serial('COM7', 38400, timeout=5) +msg = RTCMMessage(payload=b">\xd0\x00\x03\x8aX\xd9I<\x87/4\x10\x9d\x07\xd6\xafH ") +print(msg) +output = msg.serialize() +print(output) +serialOut.write(output) +``` +``` ->>> output = msg.serialize() ->>> output b'\xd3\x00\x13>\xd0\x00\x03\x8aX\xd9I<\x87/4\x10\x9d\x07\xd6\xafH Z\xd7\xf7' ->>> serialOut.write(output) ``` --- From c985b82459ff31441fa8ad524630b16861263637 Mon Sep 17 00:00:00 2001 From: semuadmin <28569967+semuadmin@users.noreply.github.com> Date: Fri, 3 May 2024 13:56:19 +0100 Subject: [PATCH 09/11] update example --- examples/msmparser.py | 33 ++++++++++++++++++--------------- 1 file changed, 18 insertions(+), 15 deletions(-) diff --git a/examples/msmparser.py b/examples/msmparser.py index a635e90..913fe11 100644 --- a/examples/msmparser.py +++ b/examples/msmparser.py @@ -5,16 +5,16 @@ python3 msmparser.py "../tests/pygpsdata-RTCM3.log" -pyrtcm parses MSM messages into a flat data structure, with repeating -element names suffixed by a 2-digit index (e.g. `DF405_02`) and -(optionally) labelled with their corresponding satellite PRN and signal ID -e.g. `DF405_02(005,2L)`. Note that indices start at 1, not 0. - Each RTCM3 MSM message contains data for multiple satellites and cells (combination of satellite and signal). The mapping between each data item and its corresponding satellite PRN or signal ID can be performed by pyrtcm helper functions `sat2prn` and `cell2prn`. +pyrtcm parses MSM messages into a flat data structure, with repeating +element names suffixed by a 2-digit index (e.g. `DF405_02`) and +(optionally) labelled with their corresponding satellite PRN and signal ID +e.g. `DF405_02(005,2L)`. Note that indices start at 1, not 0. + It is sometimes more convenient to parse the data into one or more iterable arrays, with each array element corresponding to a particular satellite or cell. @@ -39,7 +39,7 @@ from pyrtcm import ATT_NCELL # list of all sat attribute names from pyrtcm import ATT_NSAT # list of all cell attribute names -from pyrtcm import RTCMMessage, RTCMReader, cell2prn, sat2prn +from pyrtcm import RTCM_MSGIDS, RTCMMessage, RTCMReader, cell2prn, sat2prn # map of msg identity to GNSS name, epoch attribute name GNSSMAP = { @@ -102,15 +102,18 @@ def main(fname: str): rtr = RTCMReader(stream) for _, parsed in rtr: if parsed is not None: - if parsed.ismsm: - # print(parsed) - msmarray = process_msm(parsed) - print(msmarray) - - # to then iterate through a specific data item, - # e.g. the satellite DF398 (rough range) value: - for sat in msmarray[1]: # satellite data array - print(f'PRN {sat["PRN"]}: {sat["DF398"]}') + try: + if "MSM" in RTCM_MSGIDS[parsed.identity]: + # print(parsed) + msmarray = process_msm(parsed) + print(msmarray) + + # to then iterate through a specific data item, + # e.g. the satellite DF398 (rough range) value: + for sat in msmarray[1]: # satellite data array + print(f'PRN {sat["PRN"]}: {sat["DF398"]}') + except KeyError: + pass # unimplemented message type if __name__ == "__main__": From de45c6898d0a0adc757d5ffd2cae01968f05d606 Mon Sep 17 00:00:00 2001 From: semuadmin <28569967+semuadmin@users.noreply.github.com> Date: Fri, 3 May 2024 14:05:22 +0100 Subject: [PATCH 10/11] update example --- examples/hmc_4076_201.py | 10 ++++++---- examples/msmparser.py | 9 +++++---- 2 files changed, 11 insertions(+), 8 deletions(-) diff --git a/examples/hmc_4076_201.py b/examples/hmc_4076_201.py index b062bd7..53eeb61 100644 --- a/examples/hmc_4076_201.py +++ b/examples/hmc_4076_201.py @@ -8,7 +8,7 @@ Usage: -python3 hmc_4076_201.py "inputfile" +python3 hmc_4076_201.py infile="../tests/pygpsdata-NTRIP-4076.log" """ @@ -61,14 +61,16 @@ def process_message(parsed: RTCMMessage): print(hmc) -def main(fname: str): +def main(**kwargs): """ Main routine. :param str fname: fully qualified path to input file """ - with open(fname, "rb") as infile: + infile = kwargs.get("infile", "../tests/pygpsdata-NTRIP-4076.log") + + with open(infile, "rb") as infile: rtr = RTCMReader(infile) for _, parsed in rtr: if parsed.identity == "4076_201": @@ -77,4 +79,4 @@ def main(fname: str): if __name__ == "__main__": - main(argv[1]) + main(**dict(arg.split("=") for arg in argv[1:])) diff --git a/examples/msmparser.py b/examples/msmparser.py index 913fe11..5e07f33 100644 --- a/examples/msmparser.py +++ b/examples/msmparser.py @@ -3,7 +3,7 @@ Usage: -python3 msmparser.py "../tests/pygpsdata-RTCM3.log" +python3 msmparser.py infile="../tests/pygpsdata-RTCM3.log" Each RTCM3 MSM message contains data for multiple satellites and cells (combination of satellite and signal). The mapping between each @@ -91,14 +91,15 @@ def process_msm(msg: RTCMMessage) -> tuple: return (meta, msmsats, msmcells) -def main(fname: str): +def main(**kwargs): """ Main routine. :param str fname: fully qualified path to input file """ - with open(fname, "rb") as stream: + infile = kwargs.get("infile", "../tests/pygpsdata-RTCM3.log") + with open(infile, "rb") as stream: rtr = RTCMReader(stream) for _, parsed in rtr: if parsed is not None: @@ -118,4 +119,4 @@ def main(fname: str): if __name__ == "__main__": - main(argv[1]) + main(**dict(arg.split("=") for arg in argv[1:])) From 726acce2d587bb276c684090dd16702be377ee4a Mon Sep 17 00:00:00 2001 From: semuadmin <28569967+semuadmin@users.noreply.github.com> Date: Fri, 3 May 2024 14:14:43 +0100 Subject: [PATCH 11/11] update ntrip client example --- README.md | 2 +- examples/ntripclient.py | 7 ---- examples/rtcm_ntrip_client.py | 75 +++++++++++++++++++++++++++++++++++ 3 files changed, 76 insertions(+), 8 deletions(-) delete mode 100644 examples/ntripclient.py create mode 100644 examples/rtcm_ntrip_client.py diff --git a/README.md b/README.md index cab0a3b..54286ab 100644 --- a/README.md +++ b/README.md @@ -251,7 +251,7 @@ The following examples are available in the /examples folder: 1. `rtcmfile.py` - illustrates how to stream RTCM data from binary log file. 1. `rtcmsocket.py` - illustrates how to implement a TCP Socket reader for RTCM messages using RTCMReader iterator functionality. 1. `msmparser.py` - illustrates how to parse RTCM3 MSM (multiple signal messages) into a series of iterable data arrays keyed on satellite PRN and signal ID. -1. `ntripclient.py` - illustrates a simple [NTRIP](https://en.wikipedia.org/wiki/Networked_Transport_of_RTCM_via_Internet_Protocol) client using pyrtcm to parse the RTCM3 output. +1. `rtcm_ntrip_client.py` - illustrates a simple [NTRIP](https://en.wikipedia.org/wiki/Networked_Transport_of_RTCM_via_Internet_Protocol) client using pyrtcm to parse the RTCM3 output. --- ## Extensibility diff --git a/examples/ntripclient.py b/examples/ntripclient.py deleted file mode 100644 index 88c67ed..0000000 --- a/examples/ntripclient.py +++ /dev/null @@ -1,7 +0,0 @@ -#!/usr/bin/python -u -""" -See GNSSNtripClient in pygnssutils library for an illustration of how to implement an NTRIP client: - -https://github.com/semuconsulting/pygnssutils#gnssntripclient - -""" diff --git a/examples/rtcm_ntrip_client.py b/examples/rtcm_ntrip_client.py new file mode 100644 index 0000000..0451dcb --- /dev/null +++ b/examples/rtcm_ntrip_client.py @@ -0,0 +1,75 @@ +""" +rtcm_ntrip_client.py + +Illustration of RTCM3 NTRIP Client using GNSSNTRIPClient +class from pygnssutils library. Can be used with any +NTRIP caster. + +NB: requires a valid userid and password. These can be set as +environment variables PYGPSCLIENT_USER and PYGPSCLIENT_PASSWORD, +or passed as keyword arguments user and password. + +Usage: + +python3 rtcm_ntrip_client.py server="yourcaster" mountpoint="yourmountpoint" user="youruser" password="yourpassword" outfile="rtcmntrip.log" + +Run from /examples folder. + +Created on 12 Feb 2023 + +:author: semuadmin +:copyright: SEMU Consulting © 2023 +:license: BSD 3-Clause +""" + +from os import getenv +from sys import argv +from time import sleep + +from pygnssutils import GNSSNTRIPClient + +PORT = 2101 +HTTPS = 0 +if PORT == 443: + HTTPS = 1 + + +def main(**kwargs): + """ + Main routine. + """ + + server = kwargs.get("server", "rtk2go.com") + mountpoint = kwargs.get("mountpoint", "") + user = kwargs.get("user", getenv("PYGPSCLIENT_USER", "user")) + password = kwargs.get("password", getenv("PYGPSCLIENT_PASSWORD", "password")) + outfile = kwargs.get("outfile", "rtcmntrip.log") + + with open(outfile, "wb") as out: + gnc = GNSSNTRIPClient() + + print( + f"RTCM NTRIP Client started, writing output to {outfile}... Press CTRL-C to terminate." + ) + gnc.run( + server=server, + port=PORT, + https=HTTPS, + mountpoint=mountpoint, + datatype="RTCM", + ntripuser=user, + ntrippassword=password, + ggainterval=-1, + output=out, + ) + + try: + while True: + sleep(3) + except KeyboardInterrupt: + print("RTCM NTRIP Client terminated by User") + + +if __name__ == "__main__": + + main(**dict(arg.split("=") for arg in argv[1:]))