From c5b0378a803ef2d90dd171901b83f00e200e45df Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 17 Jan 2024 12:22:15 +0000 Subject: [PATCH 001/131] Bump actions/setup-python from 4 to 5 Bumps [actions/setup-python](https://github.com/actions/setup-python) from 4 to 5. - [Release notes](https://github.com/actions/setup-python/releases) - [Commits](https://github.com/actions/setup-python/compare/v4...v5) --- updated-dependencies: - dependency-name: actions/setup-python dependency-type: direct:production update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] --- .github/workflows/build_server.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build_server.yml b/.github/workflows/build_server.yml index bb692f3d..80890af9 100644 --- a/.github/workflows/build_server.yml +++ b/.github/workflows/build_server.yml @@ -32,7 +32,7 @@ jobs: repository: DJ2LS/FreeDATA - name: Set up Python 3.11 - uses: actions/setup-python@v4 + uses: actions/setup-python@v5 with: python-version: "3.11" From 49dc83553edb2eafd56c4cf3a498421dabe46226 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 17 Jan 2024 12:22:55 +0000 Subject: [PATCH 002/131] Bump emoji-picker-element from 1.20.1 to 1.21.0 in /gui Bumps [emoji-picker-element](https://github.com/nolanlawson/emoji-picker-element) from 1.20.1 to 1.21.0. - [Changelog](https://github.com/nolanlawson/emoji-picker-element/blob/master/CHANGELOG.md) - [Commits](https://github.com/nolanlawson/emoji-picker-element/compare/v1.20.1...v1.21.0) --- updated-dependencies: - dependency-name: emoji-picker-element dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- gui/package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/gui/package.json b/gui/package.json index 85f605b7..b6d4c984 100644 --- a/gui/package.json +++ b/gui/package.json @@ -46,7 +46,7 @@ "chartjs-plugin-annotation": "3.0.1", "electron-log": "5.0.3", "electron-updater": "6.1.7", - "emoji-picker-element": "1.20.1", + "emoji-picker-element": "1.21.0", "emoji-picker-element-data": "1.6.0", "file-saver": "2.0.5", "gridstack": "10.0.1", From 9d9f54a00695c93340da3715f5af09e4b124fea8 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 17 Jan 2024 12:23:10 +0000 Subject: [PATCH 003/131] Bump @vitejs/plugin-vue from 4.5.2 to 5.0.3 in /gui Bumps [@vitejs/plugin-vue](https://github.com/vitejs/vite-plugin-vue/tree/HEAD/packages/plugin-vue) from 4.5.2 to 5.0.3. - [Release notes](https://github.com/vitejs/vite-plugin-vue/releases) - [Changelog](https://github.com/vitejs/vite-plugin-vue/blob/main/packages/plugin-vue/CHANGELOG.md) - [Commits](https://github.com/vitejs/vite-plugin-vue/commits/plugin-vue@5.0.3/packages/plugin-vue) --- updated-dependencies: - dependency-name: "@vitejs/plugin-vue" dependency-type: direct:development update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] --- gui/package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/gui/package.json b/gui/package.json index 85f605b7..8ce54341 100644 --- a/gui/package.json +++ b/gui/package.json @@ -68,7 +68,7 @@ "devDependencies": { "@types/nconf": "^0.10.6", "@typescript-eslint/eslint-plugin": "6.17.0", - "@vitejs/plugin-vue": "4.5.2", + "@vitejs/plugin-vue": "5.0.3", "electron": "28.1.3", "electron-builder": "24.9.1", "eslint": "8.56.0", From 334bfa52810472ca570baac79774b6fb0248d934 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 17 Jan 2024 12:23:26 +0000 Subject: [PATCH 004/131] Bump bootstrap-icons from 1.11.2 to 1.11.3 in /gui Bumps [bootstrap-icons](https://github.com/twbs/icons) from 1.11.2 to 1.11.3. - [Release notes](https://github.com/twbs/icons/releases) - [Commits](https://github.com/twbs/icons/compare/v1.11.2...v1.11.3) --- updated-dependencies: - dependency-name: bootstrap-icons dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- gui/package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/gui/package.json b/gui/package.json index 85f605b7..cbb24998 100644 --- a/gui/package.json +++ b/gui/package.json @@ -39,7 +39,7 @@ "@vueuse/electron": "10.7.1", "blob-util": "2.0.2", "bootstrap": "5.3.2", - "bootstrap-icons": "1.11.2", + "bootstrap-icons": "1.11.3", "bootswatch": "5.3.2", "browser-image-compression": "2.0.2", "chart.js": "4.4.1", From a31fce3301d0ebe47b00b221cab8bd6ada4e0720 Mon Sep 17 00:00:00 2001 From: DJ2LS Date: Sat, 20 Jan 2024 13:52:35 +0100 Subject: [PATCH 005/131] work on data dispatcher --- modem/arq_session_irs.py | 4 ++- modem/data_dispatcher.py | 52 +++++++++++++++++++++++++++++++++++ tests/test_data_dispatcher.py | 33 ++++++++++++++++++++++ 3 files changed, 88 insertions(+), 1 deletion(-) create mode 100644 modem/data_dispatcher.py create mode 100644 tests/test_data_dispatcher.py diff --git a/modem/arq_session_irs.py b/modem/arq_session_irs.py index 4b52a959..cbdb02e1 100644 --- a/modem/arq_session_irs.py +++ b/modem/arq_session_irs.py @@ -5,6 +5,8 @@ from modem_frametypes import FRAME_TYPE from codec2 import FREEDV_MODE from enum import Enum import time +from data_dispatcher import DataDispatcher + class IRS_State(Enum): NEW = 0 OPEN_ACK_SENT = 1 @@ -191,7 +193,7 @@ class ARQSessionIRS(arq_session.ARQSession): self.set_state(IRS_State.ENDED) self.event_manager.send_arq_session_finished( False, self.id, self.dxcall, True, self.state.name, data=self.received_data, statistics=self.calculate_session_statistics()) - + DataDispatcher().dispatch(self.received_data) else: ack = self.frame_factory.build_arq_burst_ack(self.id, diff --git a/modem/data_dispatcher.py b/modem/data_dispatcher.py new file mode 100644 index 00000000..43b87a4c --- /dev/null +++ b/modem/data_dispatcher.py @@ -0,0 +1,52 @@ +import json +import structlog +class DataDispatcher: + def __init__(self): + self.logger = structlog.get_logger(type(self).__name__) + + # Hardcoded endpoints + self.endpoints = { + "p2pmsg": self.handle_p2pmsg, + "test": self.handle_test, + } + self.default_handler = self.handle_raw # Default handler for unrecognized types + + def log(self, message, isWarning = False): + msg = f"[{type(self).__name__}]: {message}" + logger = self.logger.warn if isWarning else self.logger.info + logger(msg) + + def encapsulate(self, data, type_key="p2pmsg"): + """Encapsulate data into the specified format with the given type key.""" + formatted_data = {type_key: data} + return json.dumps(formatted_data) + + def decapsulate(self, byte_data): + """Decapsulate data from the specified format, returning both the data and the type.""" + try: + json_data = byte_data.decode('utf-8') # Decode byte array to string + parsed_data = json.loads(json_data) + if parsed_data and isinstance(parsed_data, dict): + for key, value in parsed_data.items(): + return key, value # Return type and data + return "raw", byte_data # Treat as raw data if no matching type is found + except (json.JSONDecodeError, UnicodeDecodeError): + return "raw", byte_data # Return original data as raw if there's an error + + def dispatch(self, byte_data): + """Decapsulate and dispatch data to the appropriate endpoint based on its type.""" + type_key, data = self.decapsulate(byte_data) + if type_key in self.endpoints: + self.endpoints[type_key](data) + else: + # Use the default handler for unrecognized types + self.default_handler(data) + + def handle_p2pmsg(self, data): + self.log(f"Handling p2pmsg: {data}") + + def handle_raw(self, data): + self.log(f"Handling raw data: {data}") + + def handle_test(self, data): + self.log(f"Handling test data: {data}") \ No newline at end of file diff --git a/tests/test_data_dispatcher.py b/tests/test_data_dispatcher.py new file mode 100644 index 00000000..6b30c8e8 --- /dev/null +++ b/tests/test_data_dispatcher.py @@ -0,0 +1,33 @@ +import sys +sys.path.append('modem') + +import unittest +from data_dispatcher import DataDispatcher + +class TestDispatcher(unittest.TestCase): + + @classmethod + def setUpClass(cls): + cls.data_dispatcher = DataDispatcher() + + + def testEncapsulator(self): + message_type = "p2pmsg" + message_data = {"message": "Hello, P2P World!"} + + encapsulated = self.data_dispatcher.encapsulate(message_data, message_type) + type, decapsulated = self.data_dispatcher.decapsulate(encapsulated.encode('utf-8')) + self.assertEqual(type, message_type) + self.assertEqual(decapsulated, message_data) + + def testDispatcher(self): + message_type = "test" + message_data = {"message": "Hello, P2P World!"} + + encapsulated = self.data_dispatcher.encapsulate(message_data, message_type) + self.data_dispatcher.dispatch(encapsulated.encode('utf-8')) + + + +if __name__ == '__main__': + unittest.main() From 47363b2521e90494d507d503c256ea9dc1375fd9 Mon Sep 17 00:00:00 2001 From: DJ2LS Date: Sat, 20 Jan 2024 14:35:04 +0100 Subject: [PATCH 006/131] small adjustments --- modem/data_dispatcher.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/modem/data_dispatcher.py b/modem/data_dispatcher.py index 43b87a4c..f3593f05 100644 --- a/modem/data_dispatcher.py +++ b/modem/data_dispatcher.py @@ -4,12 +4,11 @@ class DataDispatcher: def __init__(self): self.logger = structlog.get_logger(type(self).__name__) - # Hardcoded endpoints + # endpoints self.endpoints = { "p2pmsg": self.handle_p2pmsg, "test": self.handle_test, } - self.default_handler = self.handle_raw # Default handler for unrecognized types def log(self, message, isWarning = False): msg = f"[{type(self).__name__}]: {message}" @@ -40,7 +39,7 @@ class DataDispatcher: self.endpoints[type_key](data) else: # Use the default handler for unrecognized types - self.default_handler(data) + self.handle_raw(data) def handle_p2pmsg(self, data): self.log(f"Handling p2pmsg: {data}") From 26478ef0a4d46acf00c2f182e79d7c0af9e8cec6 Mon Sep 17 00:00:00 2001 From: DJ2LS Date: Sat, 20 Jan 2024 21:47:21 +0100 Subject: [PATCH 007/131] adjusted and splitted dispatcher --- modem/arq_data_formatter.py | 24 +++++++++++++ modem/arq_received_data_dispatcher.py | 35 ++++++++++++++++++ modem/arq_session.py | 10 ++++-- modem/arq_session_irs.py | 5 ++- modem/data_dispatcher.py | 51 --------------------------- tests/test_data_dispatcher.py | 12 ++++--- 6 files changed, 76 insertions(+), 61 deletions(-) create mode 100644 modem/arq_data_formatter.py create mode 100644 modem/arq_received_data_dispatcher.py delete mode 100644 modem/data_dispatcher.py diff --git a/modem/arq_data_formatter.py b/modem/arq_data_formatter.py new file mode 100644 index 00000000..d81b1fd6 --- /dev/null +++ b/modem/arq_data_formatter.py @@ -0,0 +1,24 @@ +# File: arq_data_formatter.py + +import json + +class ARQDataFormatter: + def __init__(self): + pass + + def encapsulate(self, data, type_key="p2pmsg"): + """Encapsulate data into the specified format with the given type key.""" + formatted_data = {type_key: data} + return json.dumps(formatted_data) + + def decapsulate(self, byte_data): + """Decapsulate data from the specified format, returning both the data and the type.""" + try: + json_data = byte_data.decode('utf-8') # Decode byte array to string + parsed_data = json.loads(json_data) + if parsed_data and isinstance(parsed_data, dict): + for key, value in parsed_data.items(): + return key, value + return "raw", byte_data + except (json.JSONDecodeError, UnicodeDecodeError): + return "raw", byte_data diff --git a/modem/arq_received_data_dispatcher.py b/modem/arq_received_data_dispatcher.py new file mode 100644 index 00000000..b8572841 --- /dev/null +++ b/modem/arq_received_data_dispatcher.py @@ -0,0 +1,35 @@ +# File: arq_received_data_dispatcher.py + +import structlog +from arq_data_formatter import ARQDataFormatter + +class ARQReceivedDataDispatcher: + def __init__(self): + self.logger = structlog.get_logger(type(self).__name__) + self.arq_data_formatter = ARQDataFormatter() + self.endpoints = { + "p2pmsg": self.handle_p2pmsg, + "test": self.handle_test, + } + + def log(self, message, isWarning=False): + msg = f"[{type(self).__name__}]: {message}" + logger = self.logger.warn if isWarning else self.logger.info + logger(msg) + + def dispatch(self, byte_data): + """Use the data formatter to decapsulate and then dispatch data to the appropriate endpoint.""" + type_key, data = self.arq_data_formatter.decapsulate(byte_data) + if type_key in self.endpoints: + self.endpoints[type_key](data) + else: + self.handle_raw(data) + + def handle_p2pmsg(self, data): + self.log(f"Handling p2pmsg: {data}") + + def handle_raw(self, data): + self.log(f"Handling raw data: {data}") + + def handle_test(self, data): + self.log(f"Handling test data: {data}") diff --git a/modem/arq_session.py b/modem/arq_session.py index 9df7cc68..71ef28e3 100644 --- a/modem/arq_session.py +++ b/modem/arq_session.py @@ -5,6 +5,8 @@ import structlog from event_manager import EventManager from modem_frametypes import FRAME_TYPE import time +from arq_received_data_dispatcher import ARQReceivedDataDispatcher + class ARQSession(): @@ -44,6 +46,7 @@ class ARQSession(): self.frame_factory = data_frame_factory.DataFrameFactory(self.config) self.event_frame_received = threading.Event() + self.arq_received_data_dispatcher = ARQReceivedDataDispatcher() self.id = None self.session_started = time.time() self.session_ended = 0 @@ -88,10 +91,13 @@ class ARQSession(): if self.state in self.STATE_TRANSITION: if frame_type in self.STATE_TRANSITION[self.state]: action_name = self.STATE_TRANSITION[self.state][frame_type] - getattr(self, action_name)(frame) + received_data = getattr(self, action_name)(frame) + if received_data: + self.arq_received_data_dispatcher.dispatch(received_data) + return - self.log(f"Ignoring unknow transition from state {self.state.name} with frame {frame['frame_type']}") + self.log(f"Ignoring unknown transition from state {self.state.name} with frame {frame['frame_type']}") def is_session_outdated(self): session_alivetime = time.time() - self.session_max_age diff --git a/modem/arq_session_irs.py b/modem/arq_session_irs.py index cbdb02e1..7eb7f821 100644 --- a/modem/arq_session_irs.py +++ b/modem/arq_session_irs.py @@ -5,7 +5,6 @@ from modem_frametypes import FRAME_TYPE from codec2 import FREEDV_MODE from enum import Enum import time -from data_dispatcher import DataDispatcher class IRS_State(Enum): NEW = 0 @@ -193,7 +192,7 @@ class ARQSessionIRS(arq_session.ARQSession): self.set_state(IRS_State.ENDED) self.event_manager.send_arq_session_finished( False, self.id, self.dxcall, True, self.state.name, data=self.received_data, statistics=self.calculate_session_statistics()) - DataDispatcher().dispatch(self.received_data) + return self.received_data else: ack = self.frame_factory.build_arq_burst_ack(self.id, @@ -209,7 +208,7 @@ class ARQSessionIRS(arq_session.ARQSession): self.set_state(IRS_State.FAILED) self.event_manager.send_arq_session_finished( False, self.id, self.dxcall, False, self.state.name, statistics=self.calculate_session_statistics()) - + return False def calibrate_speed_settings(self): self.speed_level = 0 # for now stay at lowest speed level diff --git a/modem/data_dispatcher.py b/modem/data_dispatcher.py deleted file mode 100644 index f3593f05..00000000 --- a/modem/data_dispatcher.py +++ /dev/null @@ -1,51 +0,0 @@ -import json -import structlog -class DataDispatcher: - def __init__(self): - self.logger = structlog.get_logger(type(self).__name__) - - # endpoints - self.endpoints = { - "p2pmsg": self.handle_p2pmsg, - "test": self.handle_test, - } - - def log(self, message, isWarning = False): - msg = f"[{type(self).__name__}]: {message}" - logger = self.logger.warn if isWarning else self.logger.info - logger(msg) - - def encapsulate(self, data, type_key="p2pmsg"): - """Encapsulate data into the specified format with the given type key.""" - formatted_data = {type_key: data} - return json.dumps(formatted_data) - - def decapsulate(self, byte_data): - """Decapsulate data from the specified format, returning both the data and the type.""" - try: - json_data = byte_data.decode('utf-8') # Decode byte array to string - parsed_data = json.loads(json_data) - if parsed_data and isinstance(parsed_data, dict): - for key, value in parsed_data.items(): - return key, value # Return type and data - return "raw", byte_data # Treat as raw data if no matching type is found - except (json.JSONDecodeError, UnicodeDecodeError): - return "raw", byte_data # Return original data as raw if there's an error - - def dispatch(self, byte_data): - """Decapsulate and dispatch data to the appropriate endpoint based on its type.""" - type_key, data = self.decapsulate(byte_data) - if type_key in self.endpoints: - self.endpoints[type_key](data) - else: - # Use the default handler for unrecognized types - self.handle_raw(data) - - def handle_p2pmsg(self, data): - self.log(f"Handling p2pmsg: {data}") - - def handle_raw(self, data): - self.log(f"Handling raw data: {data}") - - def handle_test(self, data): - self.log(f"Handling test data: {data}") \ No newline at end of file diff --git a/tests/test_data_dispatcher.py b/tests/test_data_dispatcher.py index 6b30c8e8..90b64fa9 100644 --- a/tests/test_data_dispatcher.py +++ b/tests/test_data_dispatcher.py @@ -2,21 +2,23 @@ import sys sys.path.append('modem') import unittest -from data_dispatcher import DataDispatcher +from arq_data_formatter import ARQDataFormatter +from arq_received_data_dispatcher import ARQReceivedDataDispatcher class TestDispatcher(unittest.TestCase): @classmethod def setUpClass(cls): - cls.data_dispatcher = DataDispatcher() + cls.data_dispatcher = ARQReceivedDataDispatcher() + cls.data_formatter = ARQDataFormatter() def testEncapsulator(self): message_type = "p2pmsg" message_data = {"message": "Hello, P2P World!"} - encapsulated = self.data_dispatcher.encapsulate(message_data, message_type) - type, decapsulated = self.data_dispatcher.decapsulate(encapsulated.encode('utf-8')) + encapsulated = self.data_formatter.encapsulate(message_data, message_type) + type, decapsulated = self.data_formatter.decapsulate(encapsulated.encode('utf-8')) self.assertEqual(type, message_type) self.assertEqual(decapsulated, message_data) @@ -24,7 +26,7 @@ class TestDispatcher(unittest.TestCase): message_type = "test" message_data = {"message": "Hello, P2P World!"} - encapsulated = self.data_dispatcher.encapsulate(message_data, message_type) + encapsulated = self.data_formatter.encapsulate(message_data, message_type) self.data_dispatcher.dispatch(encapsulated.encode('utf-8')) From 857916285d399ec822e351910cf26389db245917 Mon Sep 17 00:00:00 2001 From: DJ2LS Date: Sun, 21 Jan 2024 20:34:01 +0100 Subject: [PATCH 008/131] changed dispatcher to a data type handler --- modem/arq_data_type_handler.py | 83 +++++++++++++++++++++++++++ modem/arq_received_data_dispatcher.py | 35 ----------- modem/arq_session.py | 10 ++-- modem/arq_session_irs.py | 15 +++-- modem/arq_session_iss.py | 18 ++++-- modem/command.py | 3 + modem/command_arq_raw.py | 9 ++- modem/data_frame_factory.py | 10 ++-- modem/frame_handler.py | 1 - tests/test_arq_session.py | 4 +- tests/test_data_dispatcher.py | 35 ----------- tests/test_data_type_handler.py | 37 ++++++++++++ 12 files changed, 165 insertions(+), 95 deletions(-) create mode 100644 modem/arq_data_type_handler.py delete mode 100644 modem/arq_received_data_dispatcher.py delete mode 100644 tests/test_data_dispatcher.py create mode 100644 tests/test_data_type_handler.py diff --git a/modem/arq_data_type_handler.py b/modem/arq_data_type_handler.py new file mode 100644 index 00000000..8b724844 --- /dev/null +++ b/modem/arq_data_type_handler.py @@ -0,0 +1,83 @@ +# File: arq_data_type_handler.py + +import structlog +import lzma +import gzip + +class ARQDataTypeHandler: + def __init__(self): + self.logger = structlog.get_logger(type(self).__name__) + self.handlers = { + "raw": { + 'prepare': self.prepare_raw, + 'handle': self.handle_raw + }, + "raw_lzma": { + 'prepare': self.prepare_raw_lzma, + 'handle': self.handle_raw_lzma + }, + "raw_gzip": { + 'prepare': self.prepare_raw_gzip, + 'handle': self.handle_raw_gzip + }, + "p2pmsg_lzma": { + 'prepare': self.prepare_p2pmsg_lzma, + 'handle': self.handle_p2pmsg_lzma + }, + } + + def dispatch(self, type_byte: int, data: bytearray): + endpoint_name = list(self.handlers.keys())[type_byte] + if endpoint_name in self.handlers and 'handle' in self.handlers[endpoint_name]: + return self.handlers[endpoint_name]['handle'](data) + else: + self.log(f"Unknown handling endpoint: {endpoint_name}", isWarning=True) + + def prepare(self, data: bytearray, endpoint_name="raw" ): + if endpoint_name in self.handlers and 'prepare' in self.handlers[endpoint_name]: + return self.handlers[endpoint_name]['prepare'](data), list(self.handlers.keys()).index(endpoint_name) + else: + self.log(f"Unknown preparation endpoint: {endpoint_name}", isWarning=True) + + def log(self, message, isWarning=False): + msg = f"[{type(self).__name__}]: {message}" + logger = self.logger.warn if isWarning else self.logger.info + logger(msg) + + def prepare_raw(self, data): + self.log(f"Preparing uncompressed data: {len(data)} Bytes") + return data + + def handle_raw(self, data): + self.log(f"Handling uncompressed data: {len(data)} Bytes") + return data + + def prepare_raw_lzma(self, data): + compressed_data = lzma.compress(data) + self.log(f"Preparing LZMA compressed data: {len(data)} Bytes >>> {len(compressed_data)} Bytes") + return compressed_data + + def handle_raw_lzma(self, data): + decompressed_data = lzma.decompress(data) + self.log(f"Handling LZMA compressed data: {len(decompressed_data)} Bytes from {len(data)} Bytes") + return decompressed_data + + def prepare_raw_gzip(self, data): + compressed_data = gzip.compress(data) + self.log(f"Preparing GZIP compressed data: {len(data)} Bytes >>> {len(compressed_data)} Bytes") + return compressed_data + + def handle_raw_gzip(self, data): + decompressed_data = gzip.decompress(data) + self.log(f"Handling GZIP compressed data: {len(decompressed_data)} Bytes from {len(data)} Bytes") + return decompressed_data + + def prepare_p2pmsg_lzma(self, data): + compressed_data = lzma.compress(data) + self.log(f"Preparing LZMA compressed P2PMSG data: {len(data)} Bytes >>> {len(compressed_data)} Bytes") + return compressed_data + + def handle_p2pmsg_lzma(self, data): + decompressed_data = lzma.decompress(data) + self.log(f"Handling LZMA compressed P2PMSG data: {len(decompressed_data)} Bytes from {len(data)} Bytes") + return decompressed_data diff --git a/modem/arq_received_data_dispatcher.py b/modem/arq_received_data_dispatcher.py deleted file mode 100644 index b8572841..00000000 --- a/modem/arq_received_data_dispatcher.py +++ /dev/null @@ -1,35 +0,0 @@ -# File: arq_received_data_dispatcher.py - -import structlog -from arq_data_formatter import ARQDataFormatter - -class ARQReceivedDataDispatcher: - def __init__(self): - self.logger = structlog.get_logger(type(self).__name__) - self.arq_data_formatter = ARQDataFormatter() - self.endpoints = { - "p2pmsg": self.handle_p2pmsg, - "test": self.handle_test, - } - - def log(self, message, isWarning=False): - msg = f"[{type(self).__name__}]: {message}" - logger = self.logger.warn if isWarning else self.logger.info - logger(msg) - - def dispatch(self, byte_data): - """Use the data formatter to decapsulate and then dispatch data to the appropriate endpoint.""" - type_key, data = self.arq_data_formatter.decapsulate(byte_data) - if type_key in self.endpoints: - self.endpoints[type_key](data) - else: - self.handle_raw(data) - - def handle_p2pmsg(self, data): - self.log(f"Handling p2pmsg: {data}") - - def handle_raw(self, data): - self.log(f"Handling raw data: {data}") - - def handle_test(self, data): - self.log(f"Handling test data: {data}") diff --git a/modem/arq_session.py b/modem/arq_session.py index 71ef28e3..26756d90 100644 --- a/modem/arq_session.py +++ b/modem/arq_session.py @@ -5,7 +5,7 @@ import structlog from event_manager import EventManager from modem_frametypes import FRAME_TYPE import time -from arq_received_data_dispatcher import ARQReceivedDataDispatcher +from arq_data_type_handler import ARQDataTypeHandler class ARQSession(): @@ -46,7 +46,7 @@ class ARQSession(): self.frame_factory = data_frame_factory.DataFrameFactory(self.config) self.event_frame_received = threading.Event() - self.arq_received_data_dispatcher = ARQReceivedDataDispatcher() + self.arq_data_type_handler = ARQDataTypeHandler() self.id = None self.session_started = time.time() self.session_ended = 0 @@ -91,9 +91,9 @@ class ARQSession(): if self.state in self.STATE_TRANSITION: if frame_type in self.STATE_TRANSITION[self.state]: action_name = self.STATE_TRANSITION[self.state][frame_type] - received_data = getattr(self, action_name)(frame) - if received_data: - self.arq_received_data_dispatcher.dispatch(received_data) + received_data, type_byte = getattr(self, action_name)(frame) + if isinstance(received_data, bytearray) and isinstance(type_byte, int): + self.arq_data_type_handler.dispatch(type_byte, received_data) return diff --git a/modem/arq_session_irs.py b/modem/arq_session_irs.py index 7eb7f821..8e0b461f 100644 --- a/modem/arq_session_irs.py +++ b/modem/arq_session_irs.py @@ -69,6 +69,7 @@ class ARQSessionIRS(arq_session.ARQSession): self.state = IRS_State.NEW self.state_enum = IRS_State # needed for access State enum from outside + self.type_byte = None self.total_length = 0 self.total_crc = '' self.received_data = None @@ -115,6 +116,7 @@ class ARQSessionIRS(arq_session.ARQSession): self.launch_transmit_and_wait(ack_frame, self.TIMEOUT_CONNECT, mode=FREEDV_MODE.signalling) if not self.abort: self.set_state(IRS_State.OPEN_ACK_SENT) + return None, None def send_info_ack(self, info_frame): # Get session info from ISS @@ -122,6 +124,7 @@ class ARQSessionIRS(arq_session.ARQSession): self.total_length = info_frame['total_length'] self.total_crc = info_frame['total_crc'] self.dx_snr.append(info_frame['snr']) + self.type_byte = info_frame['type'] self.log(f"New transfer of {self.total_length} bytes") self.event_manager.send_arq_session_new(False, self.id, self.dxcall, self.total_length, self.state.name) @@ -135,7 +138,7 @@ class ARQSessionIRS(arq_session.ARQSession): self.launch_transmit_and_wait(info_ack, self.TIMEOUT_CONNECT, mode=FREEDV_MODE.signalling) if not self.abort: self.set_state(IRS_State.INFO_ACK_SENT) - + return None, None def process_incoming_data(self, frame): if frame['offset'] != self.received_bytes: @@ -175,7 +178,7 @@ class ARQSessionIRS(arq_session.ARQSession): # self.transmitted_acks += 1 self.set_state(IRS_State.BURST_REPLY_SENT) self.launch_transmit_and_wait(ack, self.TIMEOUT_DATA, mode=FREEDV_MODE.signalling) - return + return None, None if self.final_crc_matches(): self.log("All data received successfully!") @@ -192,7 +195,8 @@ class ARQSessionIRS(arq_session.ARQSession): self.set_state(IRS_State.ENDED) self.event_manager.send_arq_session_finished( False, self.id, self.dxcall, True, self.state.name, data=self.received_data, statistics=self.calculate_session_statistics()) - return self.received_data + + return self.received_data, self.type_byte else: ack = self.frame_factory.build_arq_burst_ack(self.id, @@ -208,7 +212,7 @@ class ARQSessionIRS(arq_session.ARQSession): self.set_state(IRS_State.FAILED) self.event_manager.send_arq_session_finished( False, self.id, self.dxcall, False, self.state.name, statistics=self.calculate_session_statistics()) - return False + return False, False def calibrate_speed_settings(self): self.speed_level = 0 # for now stay at lowest speed level @@ -231,4 +235,5 @@ class ARQSessionIRS(arq_session.ARQSession): self.launch_transmit_and_wait(stop_ack, self.TIMEOUT_CONNECT, mode=FREEDV_MODE.signalling) self.set_state(IRS_State.ABORTED) self.event_manager.send_arq_session_finished( - False, self.id, self.dxcall, False, self.state.name, statistics=self.calculate_session_statistics()) \ No newline at end of file + False, self.id, self.dxcall, False, self.state.name, statistics=self.calculate_session_statistics()) + return None, None \ No newline at end of file diff --git a/modem/arq_session_iss.py b/modem/arq_session_iss.py index 5edc47e4..14970262 100644 --- a/modem/arq_session_iss.py +++ b/modem/arq_session_iss.py @@ -53,13 +53,13 @@ class ARQSessionISS(arq_session.ARQSession): } } - def __init__(self, config: dict, modem, dxcall: str, data: bytearray, state_manager): + def __init__(self, config: dict, modem, dxcall: str, state_manager, data: bytearray, type_byte: bytes): super().__init__(config, modem, dxcall) self.state_manager = state_manager self.data = data self.total_length = len(data) self.data_crc = '' - + self.type_byte = type_byte self.confirmed_bytes = 0 self.state = ISS_State.NEW @@ -119,11 +119,13 @@ class ARQSessionISS(arq_session.ARQSession): info_frame = self.frame_factory.build_arq_session_info(self.id, self.total_length, helpers.get_crc_32(self.data), - self.snr[0]) + self.snr[0], self.type_byte) self.launch_twr(info_frame, self.TIMEOUT_CONNECT_ACK, self.RETRIES_CONNECT, mode=FREEDV_MODE.signalling) self.set_state(ISS_State.INFO_SENT) + return None, None + def send_data(self, irs_frame): self.set_speed_and_frames_per_burst(irs_frame) @@ -137,15 +139,15 @@ class ARQSessionISS(arq_session.ARQSession): # check if we received an abort flag if irs_frame["flag"]["ABORT"]: self.transmission_aborted(irs_frame) - return + return None, None if irs_frame["flag"]["FINAL"]: if self.confirmed_bytes == self.total_length and irs_frame["flag"]["CHECKSUM"]: self.transmission_ended(irs_frame) - return + else: self.transmission_failed() - return + return None, None payload_size = self.get_data_payload_size() burst = [] @@ -158,6 +160,7 @@ class ARQSessionISS(arq_session.ARQSession): burst.append(data_frame) self.launch_twr(burst, self.TIMEOUT_TRANSFER, self.RETRIES_CONNECT, mode='auto') self.set_state(ISS_State.BURST_SENT) + return None, None def transmission_ended(self, irs_frame): # final function for sucessfully ended transmissions @@ -166,6 +169,7 @@ class ARQSessionISS(arq_session.ARQSession): self.log(f"All data transfered! flag_final={irs_frame['flag']['FINAL']}, flag_checksum={irs_frame['flag']['CHECKSUM']}") self.event_manager.send_arq_session_finished(True, self.id, self.dxcall,True, self.state.name, statistics=self.calculate_session_statistics()) self.state_manager.remove_arq_iss_session(self.id) + return None, None def transmission_failed(self, irs_frame=None): # final function for failed transmissions @@ -173,6 +177,7 @@ class ARQSessionISS(arq_session.ARQSession): self.set_state(ISS_State.FAILED) self.log(f"Transmission failed!") self.event_manager.send_arq_session_finished(True, self.id, self.dxcall,False, self.state.name, statistics=self.calculate_session_statistics()) + return None, None def abort_transmission(self, irs_frame=None): # function for starting the abort sequence @@ -202,4 +207,5 @@ class ARQSessionISS(arq_session.ARQSession): self.event_manager.send_arq_session_finished( True, self.id, self.dxcall, False, self.state.name, statistics=self.calculate_session_statistics()) self.state_manager.remove_arq_iss_session(self.id) + return None, None diff --git a/modem/command.py b/modem/command.py index 9bcb76f4..331e3fa8 100644 --- a/modem/command.py +++ b/modem/command.py @@ -3,6 +3,8 @@ import queue from codec2 import FREEDV_MODE import structlog from state_manager import StateManager +from arq_data_type_handler import ARQDataTypeHandler + class TxCommand(): @@ -13,6 +15,7 @@ class TxCommand(): self.event_manager = event_manager self.set_params_from_api(apiParams) self.frame_factory = DataFrameFactory(config) + self.arq_data_type_handler = ARQDataTypeHandler() def set_params_from_api(self, apiParams): pass diff --git a/modem/command_arq_raw.py b/modem/command_arq_raw.py index 7544db71..4d640bd0 100644 --- a/modem/command_arq_raw.py +++ b/modem/command_arq_raw.py @@ -13,13 +13,20 @@ class ARQRawCommand(TxCommand): if not api_validations.validate_freedata_callsign(self.dxcall): self.dxcall = f"{self.dxcall}-0" + try: + self.type = apiParams['type'] + except KeyError: + self.type = "raw" + self.data = base64.b64decode(apiParams['data']) def run(self, event_queue: Queue, modem): self.emit_event(event_queue) self.logger.info(self.log_message()) - iss = ARQSessionISS(self.config, modem, self.dxcall, self.data, self.state_manager) + prepared_data, type_byte = self.arq_data_type_handler.prepare(self.data, self.type) + + iss = ARQSessionISS(self.config, modem, self.dxcall, self.state_manager, prepared_data, type_byte) if iss.id: self.state_manager.register_arq_iss_session(iss) iss.start() diff --git a/modem/data_frame_factory.py b/modem/data_frame_factory.py index 29c2f460..b62ba11b 100644 --- a/modem/data_frame_factory.py +++ b/modem/data_frame_factory.py @@ -15,7 +15,6 @@ class DataFrameFactory: 'FINAL': 0, # Bit-position for indicating the FINAL state 'ABORT': 1, # Bit-position for indicating the ABORT request 'CHECKSUM': 2, # Bit-position for indicating the CHECKSUM is correct or not - 'ENABLE_COMPRESSION': 3 # Bit-position for indicating compression is enabled } def __init__(self, config): @@ -118,6 +117,7 @@ class DataFrameFactory: "total_crc": 4, "snr": 1, "flag": 1, + "type": 1, } self.template_list[FR_TYPE.ARQ_SESSION_INFO_ACK.value] = { @@ -218,7 +218,7 @@ class DataFrameFactory: elif key in ["session_id", "speed_level", "frames_per_burst", "version", - "offset", "total_length", "state"]: + "offset", "total_length", "state", "type"]: extracted_data[key] = int.from_bytes(data, 'big') elif key in ["snr"]: @@ -350,10 +350,8 @@ class DataFrameFactory: } return self.construct(FR_TYPE.ARQ_SESSION_OPEN_ACK, payload) - def build_arq_session_info(self, session_id: int, total_length: int, total_crc: bytes, snr, flag_compression=False): + def build_arq_session_info(self, session_id: int, total_length: int, total_crc: bytes, snr, type): flag = 0b00000000 - if flag_compression: - flag = helpers.set_flag(flag, 'ENABLE_COMPRESSION', True, self.ARQ_FLAGS) payload = { "session_id": session_id.to_bytes(1, 'big'), @@ -361,6 +359,7 @@ class DataFrameFactory: "total_crc": total_crc, "snr": helpers.snr_to_bytes(1), "flag": flag.to_bytes(1, 'big'), + "type": type.to_bytes(1, 'big'), } return self.construct(FR_TYPE.ARQ_SESSION_INFO, payload) @@ -377,7 +376,6 @@ class DataFrameFactory: } return self.construct(FR_TYPE.ARQ_STOP_ACK, payload) - def build_arq_session_info_ack(self, session_id, total_crc, snr, speed_level, frames_per_burst, flag_final=False, flag_abort=False): flag = 0b00000000 if flag_final: diff --git a/modem/frame_handler.py b/modem/frame_handler.py index 3d454782..d11ba742 100644 --- a/modem/frame_handler.py +++ b/modem/frame_handler.py @@ -31,7 +31,6 @@ class FrameHandler(): def is_frame_for_me(self): call_with_ssid = self.config['STATION']['mycall'] + "-" + str(self.config['STATION']['myssid']) ft = self.details['frame']['frame_type'] - print(self.details) valid = False # Check for callsign checksum if ft in ['ARQ_SESSION_OPEN', 'ARQ_SESSION_OPEN_ACK', 'PING', 'PING_ACK']: diff --git a/tests/test_arq_session.py b/tests/test_arq_session.py index 4bf66ad9..ecfc0b02 100644 --- a/tests/test_arq_session.py +++ b/tests/test_arq_session.py @@ -126,12 +126,13 @@ class TestARQSession(unittest.TestCase): def testARQSessionSmallPayload(self): # set Packet Error Rate (PER) / frame loss probability - self.loss_probability = 50 + self.loss_probability = 0 self.establishChannels() params = { 'dxcall': "XX1XXX-1", 'data': base64.b64encode(bytes("Hello world!", encoding="utf-8")), + 'type': "raw_lzma" } cmd = ARQRawCommand(self.config, self.iss_state_manager, self.iss_event_queue, params) cmd.run(self.iss_event_queue, self.iss_modem) @@ -146,6 +147,7 @@ class TestARQSession(unittest.TestCase): params = { 'dxcall': "XX1XXX-1", 'data': base64.b64encode(np.random.bytes(1000)), + 'type': "raw_lzma" } cmd = ARQRawCommand(self.config, self.iss_state_manager, self.iss_event_queue, params) cmd.run(self.iss_event_queue, self.iss_modem) diff --git a/tests/test_data_dispatcher.py b/tests/test_data_dispatcher.py deleted file mode 100644 index 90b64fa9..00000000 --- a/tests/test_data_dispatcher.py +++ /dev/null @@ -1,35 +0,0 @@ -import sys -sys.path.append('modem') - -import unittest -from arq_data_formatter import ARQDataFormatter -from arq_received_data_dispatcher import ARQReceivedDataDispatcher - -class TestDispatcher(unittest.TestCase): - - @classmethod - def setUpClass(cls): - cls.data_dispatcher = ARQReceivedDataDispatcher() - cls.data_formatter = ARQDataFormatter() - - - def testEncapsulator(self): - message_type = "p2pmsg" - message_data = {"message": "Hello, P2P World!"} - - encapsulated = self.data_formatter.encapsulate(message_data, message_type) - type, decapsulated = self.data_formatter.decapsulate(encapsulated.encode('utf-8')) - self.assertEqual(type, message_type) - self.assertEqual(decapsulated, message_data) - - def testDispatcher(self): - message_type = "test" - message_data = {"message": "Hello, P2P World!"} - - encapsulated = self.data_formatter.encapsulate(message_data, message_type) - self.data_dispatcher.dispatch(encapsulated.encode('utf-8')) - - - -if __name__ == '__main__': - unittest.main() diff --git a/tests/test_data_type_handler.py b/tests/test_data_type_handler.py new file mode 100644 index 00000000..b7b8cc26 --- /dev/null +++ b/tests/test_data_type_handler.py @@ -0,0 +1,37 @@ +import sys +sys.path.append('modem') + +import unittest +from arq_data_type_handler import ARQDataTypeHandler + +class TestDispatcher(unittest.TestCase): + + @classmethod + def setUpClass(cls): + cls.arq_data_type_handler = ARQDataTypeHandler() + + + def testDataTypeHandlerRaw(self): + # Example usage + example_data = b"Hello FreeDATA!" + formatted_data, type_byte = self.arq_data_type_handler.prepare(example_data, "raw") + dispatched_data = self.arq_data_type_handler.dispatch(type_byte, formatted_data) + self.assertEqual(example_data, dispatched_data) + + def testDataTypeHandlerLZMA(self): + # Example usage + example_data = b"Hello FreeDATA!" + formatted_data, type_byte = self.arq_data_type_handler.prepare(example_data, "raw_lzma") + dispatched_data = self.arq_data_type_handler.dispatch(type_byte, formatted_data) + self.assertEqual(example_data, dispatched_data) + + def testDataTypeHandlerGZIP(self): + # Example usage + example_data = b"Hello FreeDATA!" + formatted_data, type_byte = self.arq_data_type_handler.prepare(example_data, "raw_gzip") + dispatched_data = self.arq_data_type_handler.dispatch(type_byte, formatted_data) + self.assertEqual(example_data, dispatched_data) + + +if __name__ == '__main__': + unittest.main() From f83751cc8086df07f38e7a11f65d068968ce8fc0 Mon Sep 17 00:00:00 2001 From: DJ2LS Date: Tue, 23 Jan 2024 07:32:03 +0100 Subject: [PATCH 009/131] removed data formatter --- modem/arq_data_formatter.py | 24 ------------------------ 1 file changed, 24 deletions(-) delete mode 100644 modem/arq_data_formatter.py diff --git a/modem/arq_data_formatter.py b/modem/arq_data_formatter.py deleted file mode 100644 index d81b1fd6..00000000 --- a/modem/arq_data_formatter.py +++ /dev/null @@ -1,24 +0,0 @@ -# File: arq_data_formatter.py - -import json - -class ARQDataFormatter: - def __init__(self): - pass - - def encapsulate(self, data, type_key="p2pmsg"): - """Encapsulate data into the specified format with the given type key.""" - formatted_data = {type_key: data} - return json.dumps(formatted_data) - - def decapsulate(self, byte_data): - """Decapsulate data from the specified format, returning both the data and the type.""" - try: - json_data = byte_data.decode('utf-8') # Decode byte array to string - parsed_data = json.loads(json_data) - if parsed_data and isinstance(parsed_data, dict): - for key, value in parsed_data.items(): - return key, value - return "raw", byte_data - except (json.JSONDecodeError, UnicodeDecodeError): - return "raw", byte_data From 965dd5e29d5151a51728ff6612f09babbc2f491e Mon Sep 17 00:00:00 2001 From: DJ2LS Date: Tue, 23 Jan 2024 11:39:16 +0100 Subject: [PATCH 010/131] attempt fixing github test --- gui/src/js/freedata.ts | 6 ++++++ gui/src/store/settingsStore.js | 8 +++++++- 2 files changed, 13 insertions(+), 1 deletion(-) diff --git a/gui/src/js/freedata.ts b/gui/src/js/freedata.ts index 5d9158f9..c26154cf 100644 --- a/gui/src/js/freedata.ts +++ b/gui/src/js/freedata.ts @@ -105,6 +105,12 @@ export function getAppDataPath() { const platform = os.platform(); let appDataPath; + // Check if running in GitHub Actions + const isGitHubActions = process.env.GITHUB_ACTIONS === 'true'; + if (isGitHubActions) { + return "/home/runner/work/FreeDATA/FreeDATA/gui/config"; + } + switch (platform) { case "darwin": // macOS appDataPath = path.join(os.homedir(), "Library", "Application Support"); diff --git a/gui/src/store/settingsStore.js b/gui/src/store/settingsStore.js index 421853cf..a73db5b0 100644 --- a/gui/src/store/settingsStore.js +++ b/gui/src/store/settingsStore.js @@ -7,7 +7,13 @@ const nconf = require("nconf"); var appDataPath = getAppDataPath(); var configFolder = path.join(appDataPath, "FreeDATA"); -var configPath = path.join(configFolder, "config.json"); + +let configFile = "config.json" +const isGitHubActions = process.env.GITHUB_ACTIONS === 'true'; +if (isGitHubActions) { + configFile = "example.json"; +} +var configPath = path.join(configFolder, configFile); console.log("AppData Path:", appDataPath); console.log(configFolder); From 9d2332477f757b7e707d5ccd936d1d9600166dfd Mon Sep 17 00:00:00 2001 From: DJ2LS Date: Tue, 23 Jan 2024 11:42:47 +0100 Subject: [PATCH 011/131] attempt fixing github test --- gui/src/store/settingsStore.js | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/gui/src/store/settingsStore.js b/gui/src/store/settingsStore.js index a73db5b0..a73eaabc 100644 --- a/gui/src/store/settingsStore.js +++ b/gui/src/store/settingsStore.js @@ -7,12 +7,14 @@ const nconf = require("nconf"); var appDataPath = getAppDataPath(); var configFolder = path.join(appDataPath, "FreeDATA"); - let configFile = "config.json" + const isGitHubActions = process.env.GITHUB_ACTIONS === 'true'; if (isGitHubActions) { configFile = "example.json"; + configFolder = appDataPath; } + var configPath = path.join(configFolder, configFile); console.log("AppData Path:", appDataPath); From 53a34eaaa27b442285831ef28dc9b4b67ad77780 Mon Sep 17 00:00:00 2001 From: codefactor-io Date: Tue, 23 Jan 2024 10:44:34 +0000 Subject: [PATCH 012/131] [CodeFactor] Apply fixes --- gui/src/js/freedata.ts | 4 ++-- gui/src/store/settingsStore.js | 8 ++++---- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/gui/src/js/freedata.ts b/gui/src/js/freedata.ts index c26154cf..17c816bd 100644 --- a/gui/src/js/freedata.ts +++ b/gui/src/js/freedata.ts @@ -105,8 +105,8 @@ export function getAppDataPath() { const platform = os.platform(); let appDataPath; - // Check if running in GitHub Actions - const isGitHubActions = process.env.GITHUB_ACTIONS === 'true'; + // Check if running in GitHub Actions + const isGitHubActions = process.env.GITHUB_ACTIONS === "true"; if (isGitHubActions) { return "/home/runner/work/FreeDATA/FreeDATA/gui/config"; } diff --git a/gui/src/store/settingsStore.js b/gui/src/store/settingsStore.js index a73eaabc..f7ff0865 100644 --- a/gui/src/store/settingsStore.js +++ b/gui/src/store/settingsStore.js @@ -7,12 +7,12 @@ const nconf = require("nconf"); var appDataPath = getAppDataPath(); var configFolder = path.join(appDataPath, "FreeDATA"); -let configFile = "config.json" +let configFile = "config.json"; -const isGitHubActions = process.env.GITHUB_ACTIONS === 'true'; +const isGitHubActions = process.env.GITHUB_ACTIONS === "true"; if (isGitHubActions) { - configFile = "example.json"; - configFolder = appDataPath; + configFile = "example.json"; + configFolder = appDataPath; } var configPath = path.join(configFolder, configFile); From f4de64d3be199d868369dfd1c140c88dbc8bb0a9 Mon Sep 17 00:00:00 2001 From: DJ2LS Date: Tue, 23 Jan 2024 19:12:04 +0100 Subject: [PATCH 013/131] attempt fixing gui build process --- gui/electron-builder.json5 | 14 -------------- 1 file changed, 14 deletions(-) diff --git a/gui/electron-builder.json5 b/gui/electron-builder.json5 index 5bcf15e6..13f06861 100644 --- a/gui/electron-builder.json5 +++ b/gui/electron-builder.json5 @@ -19,22 +19,8 @@ "files": [ "dist", "dist-electron", - "../modem/server.dist/", ], - "extraResources": [ - - { - "from": "../modem/server.dist/", - "to": "modem", - "filter": [ - "**/*", - "!**/.git" - ] - - } - ], - "mac": { "target": [ From 53fcc6cc56c8251e5f4a0a738f57bed1978524d9 Mon Sep 17 00:00:00 2001 From: Pedro Date: Thu, 18 Jan 2024 11:35:44 +0100 Subject: [PATCH 014/131] Add P2P send message command. --- modem/api_validations.py | 7 ++++ modem/command_message_send.py | 23 +++++++++++++ modem/message_p2p.py | 62 +++++++++++++++++++++++++++++++++++ modem/server.py | 8 +++++ tests/test_message_p2p.py | 37 +++++++++++++++++++++ 5 files changed, 137 insertions(+) create mode 100644 modem/command_message_send.py create mode 100644 modem/message_p2p.py create mode 100755 tests/test_message_p2p.py diff --git a/modem/api_validations.py b/modem/api_validations.py index 81f9d2d3..e3b50021 100644 --- a/modem/api_validations.py +++ b/modem/api_validations.py @@ -3,3 +3,10 @@ import re def validate_freedata_callsign(callsign): regexp = "^[a-zA-Z]+\d+\w+-\d{1,2}$" return re.compile(regexp).match(callsign) is not None + +def validate_message_attachment(attachment): + for field in ['name', 'type', 'data']: + if field not in attachment: + raise ValueError(f"Attachment missing '{field}'") + if len(attachment[field]) < 1: + raise ValueError(f"Attachment has empty '{field}'") diff --git a/modem/command_message_send.py b/modem/command_message_send.py new file mode 100644 index 00000000..4915cf5e --- /dev/null +++ b/modem/command_message_send.py @@ -0,0 +1,23 @@ +from command import TxCommand +import api_validations +import base64 +from queue import Queue +from arq_session_iss import ARQSessionISS +from message_p2p import MessageP2P + +class SendMessageCommand(TxCommand): + """Command to send a P2P message using an ARQ transfer session + """ + + def set_params_from_api(self, apiParams): + origin = f"{self.config['STATION']['mycall']}-{self.config['STATION']['myssid']}" + self.message = MessageP2P.from_api_params(origin, apiParams) + + def transmit(self, modem): + iss = ARQSessionISS(self.config, modem, + self.message.destination, + self.message.to_payload(), + self.state_manager) + + self.state_manager.register_arq_iss_session(iss) + iss.start() diff --git a/modem/message_p2p.py b/modem/message_p2p.py new file mode 100644 index 00000000..30b7d7a7 --- /dev/null +++ b/modem/message_p2p.py @@ -0,0 +1,62 @@ +import datetime +import api_validations +import base64 +import json +import lzma + + +class MessageP2P: + def __init__(self, origin: str, destination: str, body: str, attachments: list) -> None: + self.timestamp = datetime.datetime.now().isoformat() + self.origin = origin + self.destination = destination + self.body = body + self.attachments = attachments + + @classmethod + def from_api_params(cls, origin: str, params: dict): + + dxcall = params['dxcall'] + if not api_validations.validate_freedata_callsign(dxcall): + dxcall = f"{dxcall}-0" + + if not api_validations.validate_freedata_callsign(dxcall): + raise ValueError(f"Invalid dxcall given ({params['dxcall']})") + + body = params['body'] + if len(body) < 1: + raise ValueError(f"Body cannot be empty") + + attachments = [] + if 'attachments' in params: + for a in params['attachments']: + api_validations.validate_message_attachment(a) + attachments.append({ + 'name': a['name'], + 'type': a['type'], + 'data': base64.decode(a['data']), + }) + + return cls(origin, dxcall, body, attachments) + + def get_id(self) -> str: + return f"{self.origin}.{self.destination}.{self.timestamp}" + + def to_dict(self): + """Make a dictionary out of the message data + """ + message = { + 'id': self.get_id(), + 'origin': self.origin, + 'destination': self.destination, + 'body': self.body, + 'attachments': self.attachments, + } + return message + + def to_payload(self): + """Make a byte array ready to be sent out of the message data""" + json_string = json.dumps(self.to_dict()) + json_bytes = bytes(json_string, 'utf-8') + final_payload = lzma.compress(json_bytes) + return final_payload diff --git a/modem/server.py b/modem/server.py index 3a5e0971..7e55723b 100644 --- a/modem/server.py +++ b/modem/server.py @@ -16,6 +16,7 @@ import command_ping import command_feq import command_test import command_arq_raw +import command_message_send import event_manager app = Flask(__name__) @@ -234,6 +235,13 @@ def get_post_radio(): elif request.method == 'GET': return api_response(app.state_manager.get_radio_status()) +@app.route('/freedata/messages', methods=['POST']) +def post_freedata_message(): + if enqueue_tx_command(command_message_send.SendMessageCommand, request.json): + return api_response(request.json) + else: + api_abort('Error executing command...', 500) + # @app.route('/modem/arq_connect', methods=['POST']) # @app.route('/modem/arq_disconnect', methods=['POST']) # @app.route('/modem/send_raw', methods=['POST']) diff --git a/tests/test_message_p2p.py b/tests/test_message_p2p.py new file mode 100755 index 00000000..f1341e52 --- /dev/null +++ b/tests/test_message_p2p.py @@ -0,0 +1,37 @@ +import sys +sys.path.append('modem') + +import unittest +from config import CONFIG +from message_p2p import MessageP2P + +class TestDataFrameFactory(unittest.TestCase): + + @classmethod + def setUpClass(cls): + config_manager = CONFIG('modem/config.ini.example') + cls.config = config_manager.read() + cls.mycall = f"{cls.config['STATION']['mycall']}-{cls.config['STATION']['myssid']}" + + + def testFromApiParams(self): + api_params = { + 'dxcall': 'DJ2LS-3', + 'body': 'Hello World!', + } + message = MessageP2P.from_api_params(self.mycall, api_params) + self.assertEqual(message.destination, api_params['dxcall']) + self.assertEqual(message.body, api_params['body']) + + def testToPayload(self): + api_params = { + 'dxcall': 'DJ2LS-3', + 'body': 'Hello World!', + } + message = MessageP2P.from_api_params(self.mycall, api_params) + payload = message.to_payload() + self.assertGreater(len(payload), 0) + self.assertIsInstance(payload, bytes) + +if __name__ == '__main__': + unittest.main() From ac77e1edbd0a08299dc0b1e52ef14566f5d8dfbe Mon Sep 17 00:00:00 2001 From: Pedro Date: Sat, 20 Jan 2024 14:41:51 +0100 Subject: [PATCH 015/131] Add MessageP2P attachment encoding/decoding --- modem/message_p2p.py | 29 +++++++++++++++++++++-------- tests/test_message_p2p.py | 20 +++++++++++++------- 2 files changed, 34 insertions(+), 15 deletions(-) diff --git a/modem/message_p2p.py b/modem/message_p2p.py index 30b7d7a7..bf0b0454 100644 --- a/modem/message_p2p.py +++ b/modem/message_p2p.py @@ -31,28 +31,41 @@ class MessageP2P: if 'attachments' in params: for a in params['attachments']: api_validations.validate_message_attachment(a) - attachments.append({ - 'name': a['name'], - 'type': a['type'], - 'data': base64.decode(a['data']), - }) + attachments.append(cls.__decode_attachment__(a)) return cls(origin, dxcall, body, attachments) + @classmethod + def from_payload(cls, payload): + json_string = str(lzma.decompress(payload), 'utf-8') + payload_message = json.loads(json_string) + attachments = list(map(cls.__decode_attachment__, payload_message['attachments'])) + return cls(payload_message['origin'], payload_message['destination'], + payload_message['body'], attachments) + def get_id(self) -> str: return f"{self.origin}.{self.destination}.{self.timestamp}" + + def __encode_attachment__(self, binary_attachment: dict): + encoded_attachment = binary_attachment.copy() + encoded_attachment['data'] = str(base64.b64encode(binary_attachment['data']), 'utf-8') + return encoded_attachment + def __decode_attachment__(encoded_attachment: dict): + decoded_attachment = encoded_attachment.copy() + decoded_attachment['data'] = base64.b64decode(encoded_attachment['data']) + return decoded_attachment + def to_dict(self): """Make a dictionary out of the message data """ - message = { + return { 'id': self.get_id(), 'origin': self.origin, 'destination': self.destination, 'body': self.body, - 'attachments': self.attachments, + 'attachments': list(map(self.__encode_attachment__, self.attachments)), } - return message def to_payload(self): """Make a byte array ready to be sent out of the message data""" diff --git a/tests/test_message_p2p.py b/tests/test_message_p2p.py index f1341e52..d5612921 100755 --- a/tests/test_message_p2p.py +++ b/tests/test_message_p2p.py @@ -1,5 +1,6 @@ import sys sys.path.append('modem') +import numpy as np import unittest from config import CONFIG @@ -23,15 +24,20 @@ class TestDataFrameFactory(unittest.TestCase): self.assertEqual(message.destination, api_params['dxcall']) self.assertEqual(message.body, api_params['body']) - def testToPayload(self): - api_params = { - 'dxcall': 'DJ2LS-3', - 'body': 'Hello World!', + def testToPayloadWithAttachment(self): + attachment = { + 'name': 'test.gif', + 'type': 'image/gif', + 'data': np.random.bytes(1024) } - message = MessageP2P.from_api_params(self.mycall, api_params) + message = MessageP2P(self.mycall, 'DJ2LS-3', 'Hello World!', [attachment]) payload = message.to_payload() - self.assertGreater(len(payload), 0) - self.assertIsInstance(payload, bytes) + + received_message = MessageP2P.from_payload(payload) + self.assertEqual(message.origin, received_message.origin) + self.assertEqual(message.destination, received_message.destination) + self.assertCountEqual(message.attachments, received_message.attachments) + self.assertEqual(attachment['data'], received_message.attachments[0]['data']) if __name__ == '__main__': unittest.main() From b62d3e3dc9f20d8f722d5c5cf8ccad322682799c Mon Sep 17 00:00:00 2001 From: Pedro Monteiro Date: Mon, 22 Jan 2024 17:08:47 +0100 Subject: [PATCH 016/131] Remove compression from MessageP2P payload from/to methods. --- modem/message_p2p.py | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/modem/message_p2p.py b/modem/message_p2p.py index bf0b0454..300a9c46 100644 --- a/modem/message_p2p.py +++ b/modem/message_p2p.py @@ -2,7 +2,6 @@ import datetime import api_validations import base64 import json -import lzma class MessageP2P: @@ -37,8 +36,7 @@ class MessageP2P: @classmethod def from_payload(cls, payload): - json_string = str(lzma.decompress(payload), 'utf-8') - payload_message = json.loads(json_string) + payload_message = json.loads(payload) attachments = list(map(cls.__decode_attachment__, payload_message['attachments'])) return cls(payload_message['origin'], payload_message['destination'], payload_message['body'], attachments) @@ -70,6 +68,4 @@ class MessageP2P: def to_payload(self): """Make a byte array ready to be sent out of the message data""" json_string = json.dumps(self.to_dict()) - json_bytes = bytes(json_string, 'utf-8') - final_payload = lzma.compress(json_bytes) - return final_payload + return json_string From 1f280745666b28326c6b0ceca2f57536d026c167 Mon Sep 17 00:00:00 2001 From: Pedro Date: Wed, 24 Jan 2024 17:45:21 +0100 Subject: [PATCH 017/131] Use arq_data_type_handler when sending P2P messages --- modem/command_message_send.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/modem/command_message_send.py b/modem/command_message_send.py index 4915cf5e..36e92ae0 100644 --- a/modem/command_message_send.py +++ b/modem/command_message_send.py @@ -4,6 +4,7 @@ import base64 from queue import Queue from arq_session_iss import ARQSessionISS from message_p2p import MessageP2P +from arq_data_type_handler import ARQDataTypeHandler class SendMessageCommand(TxCommand): """Command to send a P2P message using an ARQ transfer session @@ -14,10 +15,12 @@ class SendMessageCommand(TxCommand): self.message = MessageP2P.from_api_params(origin, apiParams) def transmit(self, modem): + data, data_type = self.arq_data_type_handler.prepare(self.message.to_payload, 'p2pmsg_lzma') iss = ARQSessionISS(self.config, modem, self.message.destination, - self.message.to_payload(), - self.state_manager) + data, + self.state_manager, + data_type) self.state_manager.register_arq_iss_session(iss) iss.start() From 67001ac842eec8c8860ade1bae3883613b331be2 Mon Sep 17 00:00:00 2001 From: Pedro Date: Thu, 18 Jan 2024 11:35:44 +0100 Subject: [PATCH 018/131] Add P2P send message command. --- modem/api_validations.py | 7 ++++ modem/command_message_send.py | 23 +++++++++++++ modem/message_p2p.py | 62 +++++++++++++++++++++++++++++++++++ modem/server.py | 8 +++++ tests/test_message_p2p.py | 37 +++++++++++++++++++++ 5 files changed, 137 insertions(+) create mode 100644 modem/command_message_send.py create mode 100644 modem/message_p2p.py create mode 100755 tests/test_message_p2p.py diff --git a/modem/api_validations.py b/modem/api_validations.py index 81f9d2d3..e3b50021 100644 --- a/modem/api_validations.py +++ b/modem/api_validations.py @@ -3,3 +3,10 @@ import re def validate_freedata_callsign(callsign): regexp = "^[a-zA-Z]+\d+\w+-\d{1,2}$" return re.compile(regexp).match(callsign) is not None + +def validate_message_attachment(attachment): + for field in ['name', 'type', 'data']: + if field not in attachment: + raise ValueError(f"Attachment missing '{field}'") + if len(attachment[field]) < 1: + raise ValueError(f"Attachment has empty '{field}'") diff --git a/modem/command_message_send.py b/modem/command_message_send.py new file mode 100644 index 00000000..4915cf5e --- /dev/null +++ b/modem/command_message_send.py @@ -0,0 +1,23 @@ +from command import TxCommand +import api_validations +import base64 +from queue import Queue +from arq_session_iss import ARQSessionISS +from message_p2p import MessageP2P + +class SendMessageCommand(TxCommand): + """Command to send a P2P message using an ARQ transfer session + """ + + def set_params_from_api(self, apiParams): + origin = f"{self.config['STATION']['mycall']}-{self.config['STATION']['myssid']}" + self.message = MessageP2P.from_api_params(origin, apiParams) + + def transmit(self, modem): + iss = ARQSessionISS(self.config, modem, + self.message.destination, + self.message.to_payload(), + self.state_manager) + + self.state_manager.register_arq_iss_session(iss) + iss.start() diff --git a/modem/message_p2p.py b/modem/message_p2p.py new file mode 100644 index 00000000..30b7d7a7 --- /dev/null +++ b/modem/message_p2p.py @@ -0,0 +1,62 @@ +import datetime +import api_validations +import base64 +import json +import lzma + + +class MessageP2P: + def __init__(self, origin: str, destination: str, body: str, attachments: list) -> None: + self.timestamp = datetime.datetime.now().isoformat() + self.origin = origin + self.destination = destination + self.body = body + self.attachments = attachments + + @classmethod + def from_api_params(cls, origin: str, params: dict): + + dxcall = params['dxcall'] + if not api_validations.validate_freedata_callsign(dxcall): + dxcall = f"{dxcall}-0" + + if not api_validations.validate_freedata_callsign(dxcall): + raise ValueError(f"Invalid dxcall given ({params['dxcall']})") + + body = params['body'] + if len(body) < 1: + raise ValueError(f"Body cannot be empty") + + attachments = [] + if 'attachments' in params: + for a in params['attachments']: + api_validations.validate_message_attachment(a) + attachments.append({ + 'name': a['name'], + 'type': a['type'], + 'data': base64.decode(a['data']), + }) + + return cls(origin, dxcall, body, attachments) + + def get_id(self) -> str: + return f"{self.origin}.{self.destination}.{self.timestamp}" + + def to_dict(self): + """Make a dictionary out of the message data + """ + message = { + 'id': self.get_id(), + 'origin': self.origin, + 'destination': self.destination, + 'body': self.body, + 'attachments': self.attachments, + } + return message + + def to_payload(self): + """Make a byte array ready to be sent out of the message data""" + json_string = json.dumps(self.to_dict()) + json_bytes = bytes(json_string, 'utf-8') + final_payload = lzma.compress(json_bytes) + return final_payload diff --git a/modem/server.py b/modem/server.py index 3a5e0971..7e55723b 100644 --- a/modem/server.py +++ b/modem/server.py @@ -16,6 +16,7 @@ import command_ping import command_feq import command_test import command_arq_raw +import command_message_send import event_manager app = Flask(__name__) @@ -234,6 +235,13 @@ def get_post_radio(): elif request.method == 'GET': return api_response(app.state_manager.get_radio_status()) +@app.route('/freedata/messages', methods=['POST']) +def post_freedata_message(): + if enqueue_tx_command(command_message_send.SendMessageCommand, request.json): + return api_response(request.json) + else: + api_abort('Error executing command...', 500) + # @app.route('/modem/arq_connect', methods=['POST']) # @app.route('/modem/arq_disconnect', methods=['POST']) # @app.route('/modem/send_raw', methods=['POST']) diff --git a/tests/test_message_p2p.py b/tests/test_message_p2p.py new file mode 100755 index 00000000..f1341e52 --- /dev/null +++ b/tests/test_message_p2p.py @@ -0,0 +1,37 @@ +import sys +sys.path.append('modem') + +import unittest +from config import CONFIG +from message_p2p import MessageP2P + +class TestDataFrameFactory(unittest.TestCase): + + @classmethod + def setUpClass(cls): + config_manager = CONFIG('modem/config.ini.example') + cls.config = config_manager.read() + cls.mycall = f"{cls.config['STATION']['mycall']}-{cls.config['STATION']['myssid']}" + + + def testFromApiParams(self): + api_params = { + 'dxcall': 'DJ2LS-3', + 'body': 'Hello World!', + } + message = MessageP2P.from_api_params(self.mycall, api_params) + self.assertEqual(message.destination, api_params['dxcall']) + self.assertEqual(message.body, api_params['body']) + + def testToPayload(self): + api_params = { + 'dxcall': 'DJ2LS-3', + 'body': 'Hello World!', + } + message = MessageP2P.from_api_params(self.mycall, api_params) + payload = message.to_payload() + self.assertGreater(len(payload), 0) + self.assertIsInstance(payload, bytes) + +if __name__ == '__main__': + unittest.main() From 7a9ee28bf7a82c7d00e1862912ac29d2132ada5a Mon Sep 17 00:00:00 2001 From: Pedro Date: Sat, 20 Jan 2024 14:41:51 +0100 Subject: [PATCH 019/131] Add MessageP2P attachment encoding/decoding --- modem/message_p2p.py | 29 +++++++++++++++++++++-------- tests/test_message_p2p.py | 20 +++++++++++++------- 2 files changed, 34 insertions(+), 15 deletions(-) diff --git a/modem/message_p2p.py b/modem/message_p2p.py index 30b7d7a7..bf0b0454 100644 --- a/modem/message_p2p.py +++ b/modem/message_p2p.py @@ -31,28 +31,41 @@ class MessageP2P: if 'attachments' in params: for a in params['attachments']: api_validations.validate_message_attachment(a) - attachments.append({ - 'name': a['name'], - 'type': a['type'], - 'data': base64.decode(a['data']), - }) + attachments.append(cls.__decode_attachment__(a)) return cls(origin, dxcall, body, attachments) + @classmethod + def from_payload(cls, payload): + json_string = str(lzma.decompress(payload), 'utf-8') + payload_message = json.loads(json_string) + attachments = list(map(cls.__decode_attachment__, payload_message['attachments'])) + return cls(payload_message['origin'], payload_message['destination'], + payload_message['body'], attachments) + def get_id(self) -> str: return f"{self.origin}.{self.destination}.{self.timestamp}" + + def __encode_attachment__(self, binary_attachment: dict): + encoded_attachment = binary_attachment.copy() + encoded_attachment['data'] = str(base64.b64encode(binary_attachment['data']), 'utf-8') + return encoded_attachment + def __decode_attachment__(encoded_attachment: dict): + decoded_attachment = encoded_attachment.copy() + decoded_attachment['data'] = base64.b64decode(encoded_attachment['data']) + return decoded_attachment + def to_dict(self): """Make a dictionary out of the message data """ - message = { + return { 'id': self.get_id(), 'origin': self.origin, 'destination': self.destination, 'body': self.body, - 'attachments': self.attachments, + 'attachments': list(map(self.__encode_attachment__, self.attachments)), } - return message def to_payload(self): """Make a byte array ready to be sent out of the message data""" diff --git a/tests/test_message_p2p.py b/tests/test_message_p2p.py index f1341e52..d5612921 100755 --- a/tests/test_message_p2p.py +++ b/tests/test_message_p2p.py @@ -1,5 +1,6 @@ import sys sys.path.append('modem') +import numpy as np import unittest from config import CONFIG @@ -23,15 +24,20 @@ class TestDataFrameFactory(unittest.TestCase): self.assertEqual(message.destination, api_params['dxcall']) self.assertEqual(message.body, api_params['body']) - def testToPayload(self): - api_params = { - 'dxcall': 'DJ2LS-3', - 'body': 'Hello World!', + def testToPayloadWithAttachment(self): + attachment = { + 'name': 'test.gif', + 'type': 'image/gif', + 'data': np.random.bytes(1024) } - message = MessageP2P.from_api_params(self.mycall, api_params) + message = MessageP2P(self.mycall, 'DJ2LS-3', 'Hello World!', [attachment]) payload = message.to_payload() - self.assertGreater(len(payload), 0) - self.assertIsInstance(payload, bytes) + + received_message = MessageP2P.from_payload(payload) + self.assertEqual(message.origin, received_message.origin) + self.assertEqual(message.destination, received_message.destination) + self.assertCountEqual(message.attachments, received_message.attachments) + self.assertEqual(attachment['data'], received_message.attachments[0]['data']) if __name__ == '__main__': unittest.main() From 6928ab14cc290e87247a5239f43033c9f43a350e Mon Sep 17 00:00:00 2001 From: Pedro Monteiro Date: Mon, 22 Jan 2024 17:08:47 +0100 Subject: [PATCH 020/131] Remove compression from MessageP2P payload from/to methods. --- modem/message_p2p.py | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/modem/message_p2p.py b/modem/message_p2p.py index bf0b0454..300a9c46 100644 --- a/modem/message_p2p.py +++ b/modem/message_p2p.py @@ -2,7 +2,6 @@ import datetime import api_validations import base64 import json -import lzma class MessageP2P: @@ -37,8 +36,7 @@ class MessageP2P: @classmethod def from_payload(cls, payload): - json_string = str(lzma.decompress(payload), 'utf-8') - payload_message = json.loads(json_string) + payload_message = json.loads(payload) attachments = list(map(cls.__decode_attachment__, payload_message['attachments'])) return cls(payload_message['origin'], payload_message['destination'], payload_message['body'], attachments) @@ -70,6 +68,4 @@ class MessageP2P: def to_payload(self): """Make a byte array ready to be sent out of the message data""" json_string = json.dumps(self.to_dict()) - json_bytes = bytes(json_string, 'utf-8') - final_payload = lzma.compress(json_bytes) - return final_payload + return json_string From eb3a74e146039485d0e1f9ab78a8b137f5c8e422 Mon Sep 17 00:00:00 2001 From: DJ2LS Date: Thu, 25 Jan 2024 15:17:38 +0100 Subject: [PATCH 021/131] crazy commit with --memory-- database stuff... --- modem/arq_data_type_handler.py | 6 ++ modem/message_p2p.py | 11 +++- modem/message_system_db_manager.py | 94 ++++++++++++++++++++++++++++++ modem/message_system_db_model.py | 63 ++++++++++++++++++++ modem/server.py | 7 ++- requirements.txt | 3 +- tests/test_message_p2p.py | 10 +++- 7 files changed, 189 insertions(+), 5 deletions(-) create mode 100644 modem/message_system_db_manager.py create mode 100644 modem/message_system_db_model.py diff --git a/modem/arq_data_type_handler.py b/modem/arq_data_type_handler.py index 8b724844..487a88e4 100644 --- a/modem/arq_data_type_handler.py +++ b/modem/arq_data_type_handler.py @@ -3,6 +3,8 @@ import structlog import lzma import gzip +from message_p2p import MessageP2P +from message_system_db_manager import DatabaseManager class ARQDataTypeHandler: def __init__(self): @@ -80,4 +82,8 @@ class ARQDataTypeHandler: def handle_p2pmsg_lzma(self, data): decompressed_data = lzma.decompress(data) self.log(f"Handling LZMA compressed P2PMSG data: {len(decompressed_data)} Bytes from {len(data)} Bytes") + received_message_obj = MessageP2P.from_payload(decompressed_data) + received_message_dict = MessageP2P.to_dict(received_message_obj, received=True) + DatabaseManager(uri='sqlite:///:memory:').add_message(received_message_dict) + return decompressed_data diff --git a/modem/message_p2p.py b/modem/message_p2p.py index 300a9c46..ef8d8680 100644 --- a/modem/message_p2p.py +++ b/modem/message_p2p.py @@ -42,7 +42,7 @@ class MessageP2P: payload_message['body'], attachments) def get_id(self) -> str: - return f"{self.origin}.{self.destination}.{self.timestamp}" + return f"{self.origin}_{self.destination}_{self.timestamp}" def __encode_attachment__(self, binary_attachment: dict): encoded_attachment = binary_attachment.copy() @@ -54,14 +54,21 @@ class MessageP2P: decoded_attachment['data'] = base64.b64decode(encoded_attachment['data']) return decoded_attachment - def to_dict(self): + def to_dict(self, received=False): """Make a dictionary out of the message data """ + + if received: + direction = 'receive' + else: + direction = 'transmit' + return { 'id': self.get_id(), 'origin': self.origin, 'destination': self.destination, 'body': self.body, + 'direction': direction, 'attachments': list(map(self.__encode_attachment__, self.attachments)), } diff --git a/modem/message_system_db_manager.py b/modem/message_system_db_manager.py new file mode 100644 index 00000000..b5c07455 --- /dev/null +++ b/modem/message_system_db_manager.py @@ -0,0 +1,94 @@ +# database_manager.py + +from sqlalchemy import create_engine +from sqlalchemy.orm import scoped_session, sessionmaker +from threading import local +from message_system_db_model import Base, Station, Status, Attachment, P2PMessage +from datetime import datetime +import json + +class DatabaseManager: + def __init__(self, uri='sqlite:///freedata-messages.db'): + self.engine = create_engine(uri, echo=False) + self.thread_local = local() + self.session_factory = sessionmaker(bind=self.engine) + Base.metadata.create_all(self.engine) + + def get_thread_scoped_session(self): + if not hasattr(self.thread_local, "session"): + self.thread_local.session = scoped_session(self.session_factory) + return self.thread_local.session + + def get_or_create_station(self, session, callsign): + station = session.query(Station).filter_by(callsign=callsign).first() + if not station: + station = Station(callsign=callsign) + session.add(station) + session.flush() # To get the callsign immediately + return station + + def get_or_create_status(self, session, status_name): + status = session.query(Status).filter_by(name=status_name).first() + if not status: + status = Status(name=status_name) + session.add(status) + session.flush() # To get the ID immediately + return status + + def add_message(self, message_data): + session = self.get_thread_scoped_session() + try: + # Create and add the origin and destination Stations + origin = self.get_or_create_station(session, message_data['origin']) + destination = self.get_or_create_station(session, message_data['destination']) + + # Create and add Status if provided + status = None + if 'status' in message_data: + status = self.get_or_create_status(session, message_data['status']) + + # Parse the timestamp from the message ID + timestamp = datetime.fromisoformat(message_data['id'].split('_')[2]) + + # Create the P2PMessage instance + new_message = P2PMessage( + id=message_data['id'], + origin_callsign=origin.callsign, + destination_callsign=destination.callsign, + body=message_data['body'], + timestamp=timestamp, + status_id=status.id if status else None + ) + + # Process and add attachments + for attachment_data in message_data.get('attachments', []): + attachment = Attachment( + name=attachment_data['name'], + data_type=attachment_data['type'], + data=attachment_data['data'] + ) + new_message.attachments.append(attachment) + + session.add(new_message) + session.commit() + return new_message.id + except Exception as e: + session.rollback() + raise e + finally: + session.remove() + + def get_all_messages(self): + session = self.get_thread_scoped_session() + try: + messages = session.query(P2PMessage).all() + return [message.to_dict() for message in messages] + except Exception as e: + raise e + finally: + session.remove() + + def get_all_messages_json(self): + messages_dict = self.get_all_messages() + messages_with_header = {'total_messages' : len(messages_dict), 'messages' : messages_dict} + return json.dumps(messages_with_header) # Convert to JSON string diff --git a/modem/message_system_db_model.py b/modem/message_system_db_model.py new file mode 100644 index 00000000..8609c1af --- /dev/null +++ b/modem/message_system_db_model.py @@ -0,0 +1,63 @@ +# models.py + +from sqlalchemy import Column, String, Integer, JSON, ForeignKey, DateTime +from sqlalchemy.orm import declarative_base, relationship + +Base = declarative_base() + +class Station(Base): + __tablename__ = 'station' + callsign = Column(String, primary_key=True) + location = Column(String, nullable=True) + info = Column(String, nullable=True) + +class Status(Base): + __tablename__ = 'status' + id = Column(Integer, primary_key=True) + name = Column(String, unique=True) + +class P2PMessage(Base): + __tablename__ = 'p2p_message' + id = Column(String, primary_key=True) + origin_callsign = Column(String, ForeignKey('station.callsign')) + via = Column(String, nullable=True) + destination_callsign = Column(String, ForeignKey('station.callsign')) + body = Column(String) + attachments = relationship('Attachment', backref='p2p_message') + timestamp = Column(DateTime) + timestamp_sent = Column(DateTime, nullable=True) + status_id = Column(Integer, ForeignKey('status.id'), nullable=True) + status = relationship('Status', backref='p2p_messages') + direction = Column(String, nullable=True) + statistics = Column(JSON, nullable=True) + + def to_dict(self): + return { + 'id': self.id, + 'timestamp': self.timestamp.isoformat() if self.timestamp else None, + 'origin': self.origin_callsign, + 'via': self.via, + 'destination': self.destination_callsign, + 'direction': self.direction, + 'body': self.body, + 'attachments': [attachment.to_dict() for attachment in self.attachments], + 'timestamp_sent': self.timestamp_sent.isoformat() if self.timestamp_sent else None, + 'status': self.status.name if self.status else None, + 'statistics': self.statistics + } + +class Attachment(Base): + __tablename__ = 'attachment' + id = Column(Integer, primary_key=True) + name = Column(String) + data_type = Column(String) + data = Column(String) + message_id = Column(String, ForeignKey('p2p_message.id')) + + def to_dict(self): + return { + 'id': self.id, + 'name': self.name, + 'data_type': self.data_type, + 'data': self.data # Be cautious with large binary data + } diff --git a/modem/server.py b/modem/server.py index 7e55723b..397846ef 100644 --- a/modem/server.py +++ b/modem/server.py @@ -18,6 +18,8 @@ import command_test import command_arq_raw import command_message_send import event_manager +from message_system_db_manager import DatabaseManager + app = Flask(__name__) CORS(app) @@ -235,8 +237,11 @@ def get_post_radio(): elif request.method == 'GET': return api_response(app.state_manager.get_radio_status()) -@app.route('/freedata/messages', methods=['POST']) +@app.route('/freedata/messages', methods=['POST', 'GET']) def post_freedata_message(): + if request.method in ['GET']: + result = DatabaseManager(uri='sqlite:///:memory:').get_all_messages_json() + return api_response(result) if enqueue_tx_command(command_message_send.SendMessageCommand, request.json): return api_response(request.json) else: diff --git a/requirements.txt b/requirements.txt index 5072d1e8..18cee53c 100644 --- a/requirements.txt +++ b/requirements.txt @@ -27,4 +27,5 @@ pytest-cov pytest-cover pytest-coverage pytest-rerunfailures -pick \ No newline at end of file +pick +sqlalchemy \ No newline at end of file diff --git a/tests/test_message_p2p.py b/tests/test_message_p2p.py index d5612921..4ba1b2e2 100755 --- a/tests/test_message_p2p.py +++ b/tests/test_message_p2p.py @@ -5,6 +5,8 @@ import numpy as np import unittest from config import CONFIG from message_p2p import MessageP2P +from message_system_db_manager import DatabaseManager + class TestDataFrameFactory(unittest.TestCase): @@ -13,7 +15,7 @@ class TestDataFrameFactory(unittest.TestCase): config_manager = CONFIG('modem/config.ini.example') cls.config = config_manager.read() cls.mycall = f"{cls.config['STATION']['mycall']}-{cls.config['STATION']['myssid']}" - + cls.database_manager = DatabaseManager(uri='sqlite:///:memory:') def testFromApiParams(self): api_params = { @@ -34,10 +36,16 @@ class TestDataFrameFactory(unittest.TestCase): payload = message.to_payload() received_message = MessageP2P.from_payload(payload) + received_message_dict = MessageP2P.to_dict(received_message, received=True) + self.database_manager.add_message(received_message_dict) + self.assertEqual(message.origin, received_message.origin) self.assertEqual(message.destination, received_message.destination) self.assertCountEqual(message.attachments, received_message.attachments) self.assertEqual(attachment['data'], received_message.attachments[0]['data']) + result = self.database_manager.get_all_messages() + self.assertEqual(result[0]["destination"], message.destination) + if __name__ == '__main__': unittest.main() From 2a98731b5d62b9533ce418103c69bf0941234cd1 Mon Sep 17 00:00:00 2001 From: DJ2LS Date: Thu, 25 Jan 2024 15:48:00 +0100 Subject: [PATCH 022/131] some fixes to transmit messages.. --- modem/arq_data_type_handler.py | 5 +++-- modem/command_message_send.py | 15 ++++++++++----- modem/message_system_db_manager.py | 9 +++++++++ modem/server.py | 2 +- tests/test_message_p2p.py | 2 +- 5 files changed, 24 insertions(+), 9 deletions(-) diff --git a/modem/arq_data_type_handler.py b/modem/arq_data_type_handler.py index 487a88e4..7edc5133 100644 --- a/modem/arq_data_type_handler.py +++ b/modem/arq_data_type_handler.py @@ -82,8 +82,9 @@ class ARQDataTypeHandler: def handle_p2pmsg_lzma(self, data): decompressed_data = lzma.decompress(data) self.log(f"Handling LZMA compressed P2PMSG data: {len(decompressed_data)} Bytes from {len(data)} Bytes") - received_message_obj = MessageP2P.from_payload(decompressed_data) + decompressed_json_string = decompressed_data.decode('utf-8') + received_message_obj = MessageP2P.from_payload(decompressed_json_string) received_message_dict = MessageP2P.to_dict(received_message_obj, received=True) - DatabaseManager(uri='sqlite:///:memory:').add_message(received_message_dict) + result = DatabaseManager(uri='sqlite:///:memory:').add_message(received_message_dict) return decompressed_data diff --git a/modem/command_message_send.py b/modem/command_message_send.py index 36e92ae0..062230b1 100644 --- a/modem/command_message_send.py +++ b/modem/command_message_send.py @@ -15,12 +15,17 @@ class SendMessageCommand(TxCommand): self.message = MessageP2P.from_api_params(origin, apiParams) def transmit(self, modem): - data, data_type = self.arq_data_type_handler.prepare(self.message.to_payload, 'p2pmsg_lzma') - iss = ARQSessionISS(self.config, modem, - self.message.destination, - data, + # Convert JSON string to bytes (using UTF-8 encoding) + payload = self.message.to_payload().encode('utf-8') + json_bytearray = bytearray(payload) + data, data_type = self.arq_data_type_handler.prepare(json_bytearray, 'p2pmsg_lzma') + iss = ARQSessionISS(self.config, + modem, + self.message.destination, self.state_manager, - data_type) + data, + data_type + ) self.state_manager.register_arq_iss_session(iss) iss.start() diff --git a/modem/message_system_db_manager.py b/modem/message_system_db_manager.py index b5c07455..39bc37eb 100644 --- a/modem/message_system_db_manager.py +++ b/modem/message_system_db_manager.py @@ -6,6 +6,8 @@ from threading import local from message_system_db_model import Base, Station, Status, Attachment, P2PMessage from datetime import datetime import json +import structlog + class DatabaseManager: def __init__(self, uri='sqlite:///freedata-messages.db'): @@ -14,6 +16,11 @@ class DatabaseManager: self.session_factory = sessionmaker(bind=self.engine) Base.metadata.create_all(self.engine) + def log(self, message, isWarning=False): + msg = f"[{type(self).__name__}]: {message}" + logger = self.logger.warn if isWarning else self.logger.info + logger(msg) + def get_thread_scoped_session(self): if not hasattr(self.thread_local, "session"): self.thread_local.session = scoped_session(self.session_factory) @@ -71,6 +78,8 @@ class DatabaseManager: session.add(new_message) session.commit() + + self.log(f"Added data to database: {new_message.id}") return new_message.id except Exception as e: session.rollback() diff --git a/modem/server.py b/modem/server.py index 397846ef..180f33c1 100644 --- a/modem/server.py +++ b/modem/server.py @@ -238,7 +238,7 @@ def get_post_radio(): return api_response(app.state_manager.get_radio_status()) @app.route('/freedata/messages', methods=['POST', 'GET']) -def post_freedata_message(): +def get_post_freedata_message(): if request.method in ['GET']: result = DatabaseManager(uri='sqlite:///:memory:').get_all_messages_json() return api_response(result) diff --git a/tests/test_message_p2p.py b/tests/test_message_p2p.py index 4ba1b2e2..3c0fe285 100755 --- a/tests/test_message_p2p.py +++ b/tests/test_message_p2p.py @@ -34,7 +34,7 @@ class TestDataFrameFactory(unittest.TestCase): } message = MessageP2P(self.mycall, 'DJ2LS-3', 'Hello World!', [attachment]) payload = message.to_payload() - + print(payload) received_message = MessageP2P.from_payload(payload) received_message_dict = MessageP2P.to_dict(received_message, received=True) self.database_manager.add_message(received_message_dict) From bc0e0fceb5e937b0afec0b398a7bd1741cdb3919 Mon Sep 17 00:00:00 2001 From: DJ2LS Date: Thu, 25 Jan 2024 15:49:37 +0100 Subject: [PATCH 023/131] some fixes to transmit messages.. --- modem/message_system_db_manager.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/modem/message_system_db_manager.py b/modem/message_system_db_manager.py index 39bc37eb..d7598a6b 100644 --- a/modem/message_system_db_manager.py +++ b/modem/message_system_db_manager.py @@ -16,6 +16,9 @@ class DatabaseManager: self.session_factory = sessionmaker(bind=self.engine) Base.metadata.create_all(self.engine) + self.logger = structlog.get_logger(type(self).__name__) + + def log(self, message, isWarning=False): msg = f"[{type(self).__name__}]: {message}" logger = self.logger.warn if isWarning else self.logger.info From beec229360681196870689feac50ede5acb986a3 Mon Sep 17 00:00:00 2001 From: DJ2LS Date: Thu, 25 Jan 2024 15:54:11 +0100 Subject: [PATCH 024/131] removed memory database for real testing.. --- modem/arq_data_type_handler.py | 2 +- modem/server.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/modem/arq_data_type_handler.py b/modem/arq_data_type_handler.py index 7edc5133..89b15fb8 100644 --- a/modem/arq_data_type_handler.py +++ b/modem/arq_data_type_handler.py @@ -85,6 +85,6 @@ class ARQDataTypeHandler: decompressed_json_string = decompressed_data.decode('utf-8') received_message_obj = MessageP2P.from_payload(decompressed_json_string) received_message_dict = MessageP2P.to_dict(received_message_obj, received=True) - result = DatabaseManager(uri='sqlite:///:memory:').add_message(received_message_dict) + result = DatabaseManager().add_message(received_message_dict) return decompressed_data diff --git a/modem/server.py b/modem/server.py index 180f33c1..f13f39c4 100644 --- a/modem/server.py +++ b/modem/server.py @@ -240,7 +240,7 @@ def get_post_radio(): @app.route('/freedata/messages', methods=['POST', 'GET']) def get_post_freedata_message(): if request.method in ['GET']: - result = DatabaseManager(uri='sqlite:///:memory:').get_all_messages_json() + result = DatabaseManager().get_all_messages_json() return api_response(result) if enqueue_tx_command(command_message_send.SendMessageCommand, request.json): return api_response(request.json) From 2685c7d5d57e9195987acb90addde1e485fa54ea Mon Sep 17 00:00:00 2001 From: DJ2LS Date: Sat, 27 Jan 2024 12:07:07 +0100 Subject: [PATCH 025/131] adjusted some db related things --- modem/arq_data_type_handler.py | 12 +++----- modem/arq_session.py | 2 +- modem/command.py | 2 +- modem/command_message_send.py | 7 +++++ modem/event_manager.py | 5 +++- modem/message_p2p.py | 16 ++++++---- modem/message_system_db_manager.py | 48 ++++++++++++++++++++++++++---- modem/message_system_db_model.py | 14 ++++----- modem/server.py | 4 ++- tests/test_data_type_handler.py | 9 ++++-- tests/test_message_p2p.py | 24 +++++++++++++-- 11 files changed, 106 insertions(+), 37 deletions(-) diff --git a/modem/arq_data_type_handler.py b/modem/arq_data_type_handler.py index 89b15fb8..6a0c9921 100644 --- a/modem/arq_data_type_handler.py +++ b/modem/arq_data_type_handler.py @@ -3,12 +3,12 @@ import structlog import lzma import gzip -from message_p2p import MessageP2P -from message_system_db_manager import DatabaseManager +from message_p2p import message_received class ARQDataTypeHandler: - def __init__(self): + def __init__(self, event_manager): self.logger = structlog.get_logger(type(self).__name__) + self.event_manager = event_manager self.handlers = { "raw": { 'prepare': self.prepare_raw, @@ -82,9 +82,5 @@ class ARQDataTypeHandler: def handle_p2pmsg_lzma(self, data): decompressed_data = lzma.decompress(data) self.log(f"Handling LZMA compressed P2PMSG data: {len(decompressed_data)} Bytes from {len(data)} Bytes") - decompressed_json_string = decompressed_data.decode('utf-8') - received_message_obj = MessageP2P.from_payload(decompressed_json_string) - received_message_dict = MessageP2P.to_dict(received_message_obj, received=True) - result = DatabaseManager().add_message(received_message_dict) - + message_received(self.event_manager, decompressed_data) return decompressed_data diff --git a/modem/arq_session.py b/modem/arq_session.py index 26756d90..370a81e6 100644 --- a/modem/arq_session.py +++ b/modem/arq_session.py @@ -46,7 +46,7 @@ class ARQSession(): self.frame_factory = data_frame_factory.DataFrameFactory(self.config) self.event_frame_received = threading.Event() - self.arq_data_type_handler = ARQDataTypeHandler() + self.arq_data_type_handler = ARQDataTypeHandler(self.event_manager) self.id = None self.session_started = time.time() self.session_ended = 0 diff --git a/modem/command.py b/modem/command.py index 331e3fa8..1ef552e8 100644 --- a/modem/command.py +++ b/modem/command.py @@ -15,7 +15,7 @@ class TxCommand(): self.event_manager = event_manager self.set_params_from_api(apiParams) self.frame_factory = DataFrameFactory(config) - self.arq_data_type_handler = ARQDataTypeHandler() + self.arq_data_type_handler = ARQDataTypeHandler(event_manager) def set_params_from_api(self, apiParams): pass diff --git a/modem/command_message_send.py b/modem/command_message_send.py index 062230b1..6a7d670a 100644 --- a/modem/command_message_send.py +++ b/modem/command_message_send.py @@ -5,6 +5,7 @@ from queue import Queue from arq_session_iss import ARQSessionISS from message_p2p import MessageP2P from arq_data_type_handler import ARQDataTypeHandler +from message_system_db_manager import DatabaseManager class SendMessageCommand(TxCommand): """Command to send a P2P message using an ARQ transfer session @@ -16,9 +17,15 @@ class SendMessageCommand(TxCommand): def transmit(self, modem): # Convert JSON string to bytes (using UTF-8 encoding) + + DatabaseManager().add_message(self.message.to_dict()) + payload = self.message.to_payload().encode('utf-8') json_bytearray = bytearray(payload) data, data_type = self.arq_data_type_handler.prepare(json_bytearray, 'p2pmsg_lzma') + + + iss = ARQSessionISS(self.config, modem, self.message.destination, diff --git a/modem/event_manager.py b/modem/event_manager.py index 15983718..21482ee3 100644 --- a/modem/event_manager.py +++ b/modem/event_manager.py @@ -89,4 +89,7 @@ class EventManager: def modem_failed(self): event = {"modem": "failed"} - self.broadcast(event) \ No newline at end of file + self.broadcast(event) + + def freedata_message_db_change(self): + self.broadcast({"message-db": "changed"}) \ No newline at end of file diff --git a/modem/message_p2p.py b/modem/message_p2p.py index ef8d8680..9e25607d 100644 --- a/modem/message_p2p.py +++ b/modem/message_p2p.py @@ -2,6 +2,14 @@ import datetime import api_validations import base64 import json +from message_system_db_manager import DatabaseManager + + +def message_received(event_manager, data): + decompressed_json_string = data.decode('utf-8') + received_message_obj = MessageP2P.from_payload(decompressed_json_string) + received_message_dict = MessageP2P.to_dict(received_message_obj, received=True) + DatabaseManager(event_manager).add_message(received_message_dict) class MessageP2P: @@ -58,17 +66,12 @@ class MessageP2P: """Make a dictionary out of the message data """ - if received: - direction = 'receive' - else: - direction = 'transmit' - return { 'id': self.get_id(), 'origin': self.origin, 'destination': self.destination, 'body': self.body, - 'direction': direction, + 'direction': 'receive' if received else 'transmit', 'attachments': list(map(self.__encode_attachment__, self.attachments)), } @@ -76,3 +79,4 @@ class MessageP2P: """Make a byte array ready to be sent out of the message data""" json_string = json.dumps(self.to_dict()) return json_string + diff --git a/modem/message_system_db_manager.py b/modem/message_system_db_manager.py index d7598a6b..8e75f759 100644 --- a/modem/message_system_db_manager.py +++ b/modem/message_system_db_manager.py @@ -1,4 +1,5 @@ # database_manager.py +import sqlite3 from sqlalchemy import create_engine from sqlalchemy.orm import scoped_session, sessionmaker @@ -8,9 +9,10 @@ from datetime import datetime import json import structlog - class DatabaseManager: - def __init__(self, uri='sqlite:///freedata-messages.db'): + def __init__(self, event_manger, uri='sqlite:///freedata-messages.db'): + self.event_manager = event_manger + self.engine = create_engine(uri, echo=False) self.thread_local = local() self.session_factory = sessionmaker(bind=self.engine) @@ -18,6 +20,32 @@ class DatabaseManager: self.logger = structlog.get_logger(type(self).__name__) + def initialize_default_values(self): + session = self.get_thread_scoped_session() + try: + statuses = [ + "transmitting", + "transmitted", + "received", + "failed", + "failed_checksum", + "aborted" + ] + + # Add default statuses if they don't exist + for status_name in statuses: + existing_status = session.query(Status).filter_by(name=status_name).first() + if not existing_status: + new_status = Status(name=status_name) + session.add(new_status) + + session.commit() + self.log("Initialized database") + except Exception as e: + session.rollback() + self.log(f"An error occurred while initializing default values: {e}", isWarning=True) + finally: + session.remove() def log(self, message, isWarning=False): msg = f"[{type(self).__name__}]: {message}" @@ -59,7 +87,6 @@ class DatabaseManager: # Parse the timestamp from the message ID timestamp = datetime.fromisoformat(message_data['id'].split('_')[2]) - # Create the P2PMessage instance new_message = P2PMessage( id=message_data['id'], @@ -67,6 +94,7 @@ class DatabaseManager: destination_callsign=destination.callsign, body=message_data['body'], timestamp=timestamp, + direction=message_data['direction'], status_id=status.id if status else None ) @@ -83,6 +111,7 @@ class DatabaseManager: session.commit() self.log(f"Added data to database: {new_message.id}") + self.event_manager.freedata_message_db_change() return new_message.id except Exception as e: session.rollback() @@ -95,12 +124,19 @@ class DatabaseManager: try: messages = session.query(P2PMessage).all() return [message.to_dict() for message in messages] + except Exception as e: - raise e + self.log(f"error fetching database messages with error: {e}", isWarning=True) + self.log(f"---> please delete or update existing database", isWarning=True) + + return False + finally: session.remove() def get_all_messages_json(self): messages_dict = self.get_all_messages() - messages_with_header = {'total_messages' : len(messages_dict), 'messages' : messages_dict} - return json.dumps(messages_with_header) # Convert to JSON string + if messages_dict: + messages_with_header = {'total_messages' : len(messages_dict), 'messages' : messages_dict} + return json.dumps(messages_with_header) # Convert to JSON string + return json.dumps({'error': 'fetching messages from database'}) \ No newline at end of file diff --git a/modem/message_system_db_model.py b/modem/message_system_db_model.py index 8609c1af..6388e52e 100644 --- a/modem/message_system_db_model.py +++ b/modem/message_system_db_model.py @@ -8,8 +8,8 @@ Base = declarative_base() class Station(Base): __tablename__ = 'station' callsign = Column(String, primary_key=True) - location = Column(String, nullable=True) - info = Column(String, nullable=True) + location = Column(JSON, nullable=True) + info = Column(JSON, nullable=True) class Status(Base): __tablename__ = 'status' @@ -20,15 +20,14 @@ class P2PMessage(Base): __tablename__ = 'p2p_message' id = Column(String, primary_key=True) origin_callsign = Column(String, ForeignKey('station.callsign')) - via = Column(String, nullable=True) + via_callsign = Column(String, ForeignKey('station.callsign'), nullable=True) destination_callsign = Column(String, ForeignKey('station.callsign')) - body = Column(String) + body = Column(String, nullable=True) attachments = relationship('Attachment', backref='p2p_message') timestamp = Column(DateTime) - timestamp_sent = Column(DateTime, nullable=True) status_id = Column(Integer, ForeignKey('status.id'), nullable=True) status = relationship('Status', backref='p2p_messages') - direction = Column(String, nullable=True) + direction = Column(String) statistics = Column(JSON, nullable=True) def to_dict(self): @@ -36,12 +35,11 @@ class P2PMessage(Base): 'id': self.id, 'timestamp': self.timestamp.isoformat() if self.timestamp else None, 'origin': self.origin_callsign, - 'via': self.via, + 'via': self.via_callsign, 'destination': self.destination_callsign, 'direction': self.direction, 'body': self.body, 'attachments': [attachment.to_dict() for attachment in self.attachments], - 'timestamp_sent': self.timestamp_sent.isoformat() if self.timestamp_sent else None, 'status': self.status.name if self.status else None, 'statistics': self.statistics } diff --git a/modem/server.py b/modem/server.py index f13f39c4..c20cb597 100644 --- a/modem/server.py +++ b/modem/server.py @@ -240,7 +240,7 @@ def get_post_radio(): @app.route('/freedata/messages', methods=['POST', 'GET']) def get_post_freedata_message(): if request.method in ['GET']: - result = DatabaseManager().get_all_messages_json() + result = DatabaseManager(app.event_manager).get_all_messages_json() return api_response(result) if enqueue_tx_command(command_message_send.SendMessageCommand, request.json): return api_response(request.json) @@ -296,6 +296,8 @@ if __name__ == "__main__": app.service_manager = service_manager.SM(app) # start modem service app.modem_service.put("start") + # initialize databse default values + DatabaseManager(app.event_manager).initialize_default_values() wsm.startThreads(app) app.run() diff --git a/tests/test_data_type_handler.py b/tests/test_data_type_handler.py index b7b8cc26..652fd881 100644 --- a/tests/test_data_type_handler.py +++ b/tests/test_data_type_handler.py @@ -2,16 +2,21 @@ import sys sys.path.append('modem') import unittest +import queue from arq_data_type_handler import ARQDataTypeHandler +from event_manager import EventManager + class TestDispatcher(unittest.TestCase): @classmethod def setUpClass(cls): - cls.arq_data_type_handler = ARQDataTypeHandler() + cls.event_queue = queue.Queue() + cls.event_manager = EventManager([cls.event_queue]) + cls.arq_data_type_handler = ARQDataTypeHandler(cls.event_manager) - def testDataTypeHandlerRaw(self): + def testDataTypeHevent_managerandlerRaw(self): # Example usage example_data = b"Hello FreeDATA!" formatted_data, type_byte = self.arq_data_type_handler.prepare(example_data, "raw") diff --git a/tests/test_message_p2p.py b/tests/test_message_p2p.py index 3c0fe285..4cedfbaa 100755 --- a/tests/test_message_p2p.py +++ b/tests/test_message_p2p.py @@ -6,7 +6,8 @@ import unittest from config import CONFIG from message_p2p import MessageP2P from message_system_db_manager import DatabaseManager - +from event_manager import EventManager +import queue class TestDataFrameFactory(unittest.TestCase): @@ -14,8 +15,11 @@ class TestDataFrameFactory(unittest.TestCase): def setUpClass(cls): config_manager = CONFIG('modem/config.ini.example') cls.config = config_manager.read() + + cls.event_queue = queue.Queue() + cls.event_manager = EventManager([cls.event_queue]) cls.mycall = f"{cls.config['STATION']['mycall']}-{cls.config['STATION']['myssid']}" - cls.database_manager = DatabaseManager(uri='sqlite:///:memory:') + cls.database_manager = DatabaseManager(cls.event_manager, uri='sqlite:///:memory:') def testFromApiParams(self): api_params = { @@ -34,7 +38,20 @@ class TestDataFrameFactory(unittest.TestCase): } message = MessageP2P(self.mycall, 'DJ2LS-3', 'Hello World!', [attachment]) payload = message.to_payload() - print(payload) + received_message = MessageP2P.from_payload(payload) + self.assertEqual(message.origin, received_message.origin) + self.assertEqual(message.destination, received_message.destination) + self.assertCountEqual(message.attachments, received_message.attachments) + self.assertEqual(attachment['data'], received_message.attachments[0]['data']) + + def testToPayloadWithAttachmentAndDatabase(self): + attachment = { + 'name': 'test.gif', + 'type': 'image/gif', + 'data': np.random.bytes(1024) + } + message = MessageP2P(self.mycall, 'DJ2LS-3', 'Hello World!', [attachment]) + payload = message.to_payload() received_message = MessageP2P.from_payload(payload) received_message_dict = MessageP2P.to_dict(received_message, received=True) self.database_manager.add_message(received_message_dict) @@ -47,5 +64,6 @@ class TestDataFrameFactory(unittest.TestCase): result = self.database_manager.get_all_messages() self.assertEqual(result[0]["destination"], message.destination) + if __name__ == '__main__': unittest.main() From 5d4544f4369fd5394ed91d7e271a75e6c59437a0 Mon Sep 17 00:00:00 2001 From: DJ2LS Date: Sat, 27 Jan 2024 12:09:59 +0100 Subject: [PATCH 026/131] adjusted api in case of 0 messages in db --- modem/message_system_db_manager.py | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/modem/message_system_db_manager.py b/modem/message_system_db_manager.py index 8e75f759..8c36e228 100644 --- a/modem/message_system_db_manager.py +++ b/modem/message_system_db_manager.py @@ -129,14 +129,12 @@ class DatabaseManager: self.log(f"error fetching database messages with error: {e}", isWarning=True) self.log(f"---> please delete or update existing database", isWarning=True) - return False + return [] finally: session.remove() def get_all_messages_json(self): messages_dict = self.get_all_messages() - if messages_dict: - messages_with_header = {'total_messages' : len(messages_dict), 'messages' : messages_dict} - return json.dumps(messages_with_header) # Convert to JSON string - return json.dumps({'error': 'fetching messages from database'}) \ No newline at end of file + messages_with_header = {'total_messages' : len(messages_dict), 'messages' : messages_dict} + return json.dumps(messages_with_header) # Convert to JSON string From b1d25bdc44c3e8a4e0e6eb35c43bb932997ecb3f Mon Sep 17 00:00:00 2001 From: DJ2LS Date: Sat, 27 Jan 2024 12:15:12 +0100 Subject: [PATCH 027/131] fixed missing event manager --- modem/command_message_send.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modem/command_message_send.py b/modem/command_message_send.py index 6a7d670a..48c6ad85 100644 --- a/modem/command_message_send.py +++ b/modem/command_message_send.py @@ -18,7 +18,7 @@ class SendMessageCommand(TxCommand): def transmit(self, modem): # Convert JSON string to bytes (using UTF-8 encoding) - DatabaseManager().add_message(self.message.to_dict()) + DatabaseManager(self.event_manager).add_message(self.message.to_dict()) payload = self.message.to_payload().encode('utf-8') json_bytearray = bytearray(payload) From 6104799d783ccc651a6acc127a3f3086225dacbc Mon Sep 17 00:00:00 2001 From: DJ2LS Date: Sat, 27 Jan 2024 12:15:37 +0100 Subject: [PATCH 028/131] version update --- modem/server.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modem/server.py b/modem/server.py index c20cb597..a8d50a65 100644 --- a/modem/server.py +++ b/modem/server.py @@ -25,7 +25,7 @@ app = Flask(__name__) CORS(app) CORS(app, resources={r"/*": {"origins": "*"}}) sock = Sock(app) -MODEM_VERSION = "0.12.1-alpha" +MODEM_VERSION = "0.13.0-alpha" # set config file to use def set_config(): From 30bb0ba828d8cab03a27e3b25f9dd8e24fa7e431 Mon Sep 17 00:00:00 2001 From: DJ2LS Date: Sat, 27 Jan 2024 12:17:31 +0100 Subject: [PATCH 029/131] additional error message --- modem/message_system_db_manager.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/modem/message_system_db_manager.py b/modem/message_system_db_manager.py index 8c36e228..ddaa03b5 100644 --- a/modem/message_system_db_manager.py +++ b/modem/message_system_db_manager.py @@ -115,7 +115,8 @@ class DatabaseManager: return new_message.id except Exception as e: session.rollback() - raise e + self.log(f"error adding new message to databse with error: {e}", isWarning=True) + self.log(f"---> please delete or update existing database", isWarning=True) finally: session.remove() From 2e9344bba7600c22f955951524cfe19a14773705 Mon Sep 17 00:00:00 2001 From: DJ2LS Date: Sat, 27 Jan 2024 17:44:18 +0100 Subject: [PATCH 030/131] cleanup and first implementation of server side message sytem --- gui/src/components/chat_conversations.vue | 49 +- gui/src/components/chat_messages.vue | 32 +- gui/src/components/chat_messages_received.vue | 11 +- gui/src/components/chat_navbar.vue | 42 +- gui/src/components/chat_new_message.vue | 10 +- gui/src/components/main_modals.vue | 92 +- gui/src/js/api.js | 19 +- gui/src/js/chatHandler.ts | 1110 ----------------- gui/src/js/deprecated_sock.js | 886 ------------- gui/src/js/eventHandler.js | 15 + gui/src/js/messagesHandler.ts | 66 + gui/src/store/chatStore.js | 16 +- 12 files changed, 237 insertions(+), 2111 deletions(-) delete mode 100644 gui/src/js/chatHandler.ts delete mode 100644 gui/src/js/deprecated_sock.js create mode 100644 gui/src/js/messagesHandler.ts diff --git a/gui/src/components/chat_conversations.vue b/gui/src/components/chat_conversations.vue index 29884103..d6eec968 100644 --- a/gui/src/components/chat_conversations.vue +++ b/gui/src/components/chat_conversations.vue @@ -6,14 +6,9 @@ setActivePinia(pinia); import { useChatStore } from "../store/chatStore.js"; const chat = useChatStore(pinia); -import { - getNewMessagesByDXCallsign, - resetIsNewMessage, -} from "../js/chatHandler"; function chatSelected(callsign) { chat.selectedCallsign = callsign.toUpperCase(); - // scroll message container to bottom var messageBody = document.getElementById("message-container"); if (messageBody != null) { @@ -21,27 +16,6 @@ function chatSelected(callsign) { messageBody.scrollTop = messageBody.scrollHeight - messageBody.clientHeight; } - if (getNewMessagesByDXCallsign(callsign)[1] > 0) { - let messageArray = getNewMessagesByDXCallsign(callsign)[2]; - console.log(messageArray); - - for (const key in messageArray) { - resetIsNewMessage(messageArray[key].uuid, false); - } - } - - try { - chat.beaconLabelArray = Object.values( - chat.sorted_beacon_list[chat.selectedCallsign].timestamp, - ); - chat.beaconDataArray = Object.values( - chat.sorted_beacon_list[chat.selectedCallsign].snr, - ); - } catch (e) { - console.log("beacon data not fetched: " + e); - chat.beaconLabelArray = []; - chat.beaconDataArray = []; - } }