From 836f4b99d8707f07cb29122912f86ae066050a1d Mon Sep 17 00:00:00 2001
From: DJ2LS <dj2ls@proton.me>
Date: Thu, 22 Feb 2024 13:02:23 +0100
Subject: [PATCH 01/29] fixed toasts z index

---
 gui/src/components/dynamic_components.vue | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/gui/src/components/dynamic_components.vue b/gui/src/components/dynamic_components.vue
index 3f521873..abd6a3af 100644
--- a/gui/src/components/dynamic_components.vue
+++ b/gui/src/components/dynamic_components.vue
@@ -485,8 +485,8 @@ function quickfill() {
     <i class="bi bi-grip-vertical h5"></i>
   </button>
 
-  <div class="grid-container" style="height: calc(100vh - 51px);">
-    <div class="grid-stack">
+  <div class="grid-container z-0" style="height: calc(100vh - 51px);">
+    <div class="grid-stack z-0">
       <div
         v-for="(w, indexs) in items"
         class="grid-stack-item"

From 22f0226600003d489de8c62348d5d7a91ace344a Mon Sep 17 00:00:00 2001
From: DJ2LS <dj2ls@proton.me>
Date: Thu, 22 Feb 2024 15:05:54 +0100
Subject: [PATCH 02/29] gear shifting test

---
 modem/arq_session.py        | 11 +++++-
 modem/arq_session_irs.py    | 73 +++++++++++++++++++++++++------------
 modem/arq_session_iss.py    | 26 ++++++++++---
 modem/data_frame_factory.py | 11 +++++-
 modem/demodulator.py        | 20 +++++-----
 modem/frame_handler.py      |  1 -
 modem/server.py             |  2 +-
 tests/test_arq_session.py   |  2 +-
 8 files changed, 102 insertions(+), 44 deletions(-)

diff --git a/modem/arq_session.py b/modem/arq_session.py
index 8bae69af..75334d6a 100644
--- a/modem/arq_session.py
+++ b/modem/arq_session.py
@@ -127,4 +127,13 @@ class ARQSession():
                 'total_bytes': total_bytes,
                 'duration': duration,
                 'bytes_per_minute': bytes_per_minute
-            }
\ No newline at end of file
+            }
+
+    def get_appropriate_speed_level(self, snr):
+        # Start with the lowest speed level as default
+        # In case of a not fitting SNR, we return the lowest speed level
+        appropriate_speed_level = min(self.SPEED_LEVEL_DICT.keys())
+        for level, details in self.SPEED_LEVEL_DICT.items():
+            if snr >= details['min_snr'] and level > appropriate_speed_level:
+                appropriate_speed_level = level
+        return appropriate_speed_level
\ No newline at end of file
diff --git a/modem/arq_session_irs.py b/modem/arq_session_irs.py
index 1c853568..fbc460b8 100644
--- a/modem/arq_session_irs.py
+++ b/modem/arq_session_irs.py
@@ -76,13 +76,10 @@ class ARQSessionIRS(arq_session.ARQSession):
         self.received_bytes = 0
         self.received_crc = None
 
-        self.transmitted_acks = 0
+        self.speed_level = 0
 
         self.abort = False
 
-    def set_decode_mode(self):
-        self.modem.demodulator.set_decode_mode(self.get_mode_by_speed_level(self.speed_level))
-
     def all_data_received(self):
         return self.total_length == self.received_bytes
 
@@ -127,9 +124,6 @@ class ARQSessionIRS(arq_session.ARQSession):
         self.log(f"New transfer of {self.total_length} bytes")
         self.event_manager.send_arq_session_new(False, self.id, self.dxcall, self.total_length, self.state.name)
 
-        self.calibrate_speed_settings()
-        self.set_decode_mode()
-
         info_ack = self.frame_factory.build_arq_session_info_ack(
             self.id, self.total_crc, self.snr[0],
             self.speed_level, self.frames_per_burst, flag_abort=self.abort)
@@ -163,17 +157,20 @@ class ARQSessionIRS(arq_session.ARQSession):
 
     def receive_data(self, burst_frame):
         self.process_incoming_data(burst_frame)
-        self.calibrate_speed_settings()
 
         if not self.all_data_received():
+            downshift, upshift = self.calibrate_speed_settings()
             ack = self.frame_factory.build_arq_burst_ack(
-                                                         self.id, self.received_bytes,
-                                                         self.speed_level, self.frames_per_burst, self.snr[0], flag_abort=self.abort)
+                self.id,
+                self.received_bytes,
+                self.speed_level,
+                self.frames_per_burst,
+                self.snr[0],
+                flag_abort=self.abort,
+                flag_upshift=upshift,
+                flag_downshift=downshift
+            )
 
-            self.set_decode_mode()
-
-            # increase ack counter
-            # self.transmitted_acks += 1
             self.set_state(IRS_State.BURST_REPLY_SENT)
             self.launch_transmit_and_wait(ack, self.TIMEOUT_DATA, mode=FREEDV_MODE.signalling)
             return None, None
@@ -209,16 +206,46 @@ class ARQSessionIRS(arq_session.ARQSession):
             self.transmission_failed()
 
     def calibrate_speed_settings(self):
-        self.speed_level = 0 # for now stay at lowest speed level
-        return
-        # if we have two ACKS, then consider increasing speed level
-        if self.transmitted_acks >= 2:
-            self.transmitted_acks = 0
-            new_speed_level = min(self.speed_level + 1, len(self.SPEED_LEVEL_DICT) - 1)
+        latest_snr = self.snr[-1] if self.snr else -10
+        appropriate_speed_level = self.get_appropriate_speed_level(latest_snr)
+        modes_to_decode = {}
 
-            # check first if the next mode supports the actual snr
-            if self.snr[0] >= self.SPEED_LEVEL_DICT[new_speed_level]["min_snr"]:
-                self.speed_level = new_speed_level
+        # Log the latest SNR, current, and appropriate speed levels for clarity
+        self.log(
+            f"Latest SNR: {latest_snr}, Current Speed Level: {self.speed_level}, Appropriate Speed Level: {appropriate_speed_level}",
+            isWarning=True)
+
+        # Always decode the current mode
+        current_mode = self.get_mode_by_speed_level(self.speed_level).value
+        modes_to_decode[current_mode] = True
+
+        # Initialize shift flags
+        upshift = False
+        downshift = False
+
+        # Check if a shift is needed
+        if appropriate_speed_level != self.speed_level:
+            # Determine the direction of the shift
+            upshift = appropriate_speed_level > self.speed_level
+            downshift = appropriate_speed_level < self.speed_level
+
+            # Update the speed level
+            self.speed_level = appropriate_speed_level
+            self.log(f"Updated Speed Level: {self.speed_level}", isWarning=True)
+
+            # Determine additional modes based on the direction of the shift
+            if upshift and self.speed_level < len(self.SPEED_LEVEL_DICT) - 1:
+                next_mode = self.get_mode_by_speed_level(self.speed_level).value
+                modes_to_decode[next_mode] = True
+            elif downshift and self.speed_level > 0:
+                prev_mode = self.get_mode_by_speed_level(self.speed_level).value
+                modes_to_decode[prev_mode] = True
+
+        self.log(f"Modes to Decode: {modes_to_decode}", isWarning=True)
+        # Apply the new decode mode based on the updated speed level
+        self.modem.demodulator.set_decode_mode(modes_to_decode)
+
+        return downshift, upshift
 
     def abort_transmission(self):
         self.log(f"Aborting transmission... setting abort flag")
diff --git a/modem/arq_session_iss.py b/modem/arq_session_iss.py
index 9c69522c..7ff769f3 100644
--- a/modem/arq_session_iss.py
+++ b/modem/arq_session_iss.py
@@ -105,11 +105,25 @@ class ARQSessionISS(arq_session.ARQSession):
         self.launch_twr(session_open_frame, self.TIMEOUT_CONNECT_ACK, self.RETRIES_CONNECT, mode=FREEDV_MODE.signalling)
         self.set_state(ISS_State.OPEN_SENT)
 
-    def set_speed_and_frames_per_burst(self, frame):
-        self.speed_level = frame['speed_level']
-        self.log(f"Speed level set to {self.speed_level}")
-        self.frames_per_burst = frame['frames_per_burst']
-        self.log(f"Frames per burst set to {self.frames_per_burst}")
+    def update_speed_level(self, frame):
+        self.log("---------------------------------------------------------", isWarning=True)
+
+        # Log the received frame for debugging
+        self.log(f"Received frame: {frame}", isWarning=True)
+
+        # Safely extract upshift and downshift flags with default to False if not present
+        upshift = frame['flag'].get('UPSHIFT', False)
+        downshift = frame['flag'].get('DOWNSHIFT', False)
+
+        # Check for UPSHIFT frame and ensure speed level does not exceed max limit
+        if upshift and not downshift and self.speed_level < len(self.SPEED_LEVEL_DICT) - 1:
+            self.speed_level += 1
+            self.log(f"Upshifting. New speed level: {self.speed_level}")
+
+        # Check for DOWNSHIFT frame and ensure speed level does not go below 0
+        elif downshift and not upshift and self.speed_level > 0:
+            self.speed_level -= 1
+            self.log(f"Downshifting. New speed level: {self.speed_level}")
 
     def send_info(self, irs_frame):
         # check if we received an abort flag
@@ -128,7 +142,7 @@ class ARQSessionISS(arq_session.ARQSession):
 
     def send_data(self, irs_frame):
 
-        self.set_speed_and_frames_per_burst(irs_frame)
+        self.update_speed_level(irs_frame)
 
         if 'offset' in irs_frame:
             self.confirmed_bytes = irs_frame['offset']
diff --git a/modem/data_frame_factory.py b/modem/data_frame_factory.py
index b62ba11b..894e2d1c 100644
--- a/modem/data_frame_factory.py
+++ b/modem/data_frame_factory.py
@@ -15,6 +15,8 @@ class DataFrameFactory:
         'FINAL': 0,  # Bit-position for indicating the FINAL state
         'ABORT': 1, # Bit-position for indicating the ABORT request
         'CHECKSUM': 2,  # Bit-position for indicating the CHECKSUM is correct or not
+        'DOWNSHIFT': 3, # Bit-position for indicating a requested ARQ DOWNSHIFT
+        'UPSHIFT': 4, # Bit-position for indicating a requested ARQ UPSHIFT
     }
 
     def __init__(self, config):
@@ -404,7 +406,7 @@ class DataFrameFactory:
         return frame
 
     def build_arq_burst_ack(self, session_id: bytes, offset, speed_level: int, 
-                            frames_per_burst: int, snr: int, flag_final=False, flag_checksum=False, flag_abort=False):
+                            frames_per_burst: int, snr: int, flag_final=False, flag_checksum=False, flag_abort=False, flag_downshift=False, flag_upshift=False):
         flag = 0b00000000
         if flag_final:
             flag = helpers.set_flag(flag, 'FINAL', True, self.ARQ_FLAGS)
@@ -415,6 +417,13 @@ class DataFrameFactory:
         if flag_abort:
             flag = helpers.set_flag(flag, 'ABORT', True, self.ARQ_FLAGS)
 
+        if flag_downshift:
+            flag = helpers.set_flag(flag, 'DOWNSHIFT', True, self.ARQ_FLAGS)
+            flag = helpers.set_flag(flag, 'UPSHIFT', False, self.ARQ_FLAGS)
+
+        if flag_upshift:
+            flag = helpers.set_flag(flag, 'DOWNSHIFT', False, self.ARQ_FLAGS)
+            flag = helpers.set_flag(flag, 'UPSHIFT', True, self.ARQ_FLAGS)
 
         payload = {
             "session_id": session_id.to_bytes(1, 'big'),
diff --git a/modem/demodulator.py b/modem/demodulator.py
index b22e99c0..01358174 100644
--- a/modem/demodulator.py
+++ b/modem/demodulator.py
@@ -382,15 +382,15 @@ class Demodulator():
         for mode in self.MODE_DICT:
             codec2.api.freedv_set_sync(self.MODE_DICT[mode]["instance"], 0)
 
-    def set_decode_mode(self, mode):
+    def set_decode_mode(self, modes_to_decode):
+        # Reset all modes to not decode
+        for m in self.MODE_DICT:
+            self.MODE_DICT[m]["decode"] = False
 
-        for m in self.MODE_DICT: self.MODE_DICT[m]["decode"] = False
+        # Enable specified modes
+        for mode, decode in modes_to_decode.items():
+            if mode in self.MODE_DICT:
+                self.MODE_DICT[mode]["decode"] = decode
 
-        # signalling is always true
-        self.MODE_DICT[codec2.FREEDV_MODE.signalling.value]["decode"] = True
-
-        # Enable mode based on speed_level
-        self.MODE_DICT[mode.value]["decode"] = True
-        self.log.info(f"[MDM] [demod_audio] set data mode: {mode.name}")
-
-        return
+        enabled_modes = [mode.name for mode, details in self.MODE_DICT.items() if details["decode"]]
+        self.log.info(f"[MDM] [demod_audio] Enabled decode modes: {', '.join(enabled_modes)}")
diff --git a/modem/frame_handler.py b/modem/frame_handler.py
index 246d3084..46347d1e 100644
--- a/modem/frame_handler.py
+++ b/modem/frame_handler.py
@@ -91,7 +91,6 @@ class FrameHandler():
     def add_to_heard_stations(self):
         frame = self.details['frame']
 
-        print(frame)
         if 'origin' not in frame:
             return
 
diff --git a/modem/server.py b/modem/server.py
index 912e4206..f425b37f 100644
--- a/modem/server.py
+++ b/modem/server.py
@@ -29,7 +29,7 @@ app = Flask(__name__)
 CORS(app)
 CORS(app, resources={r"/*": {"origins": "*"}})
 sock = Sock(app)
-MODEM_VERSION = "0.13.7-alpha"
+MODEM_VERSION = "0.14.0-alpha-exp"
 
 # set config file to use
 def set_config():
diff --git a/tests/test_arq_session.py b/tests/test_arq_session.py
index 185e90e3..8309565d 100644
--- a/tests/test_arq_session.py
+++ b/tests/test_arq_session.py
@@ -144,7 +144,7 @@ class TestARQSession(unittest.TestCase):
         self.waitAndCloseChannels()
         del cmd
 
-    def DisabledtestARQSessionLargePayload(self):
+    def testARQSessionLargePayload(self):
         # set Packet Error Rate (PER) / frame loss probability
         self.loss_probability = 0
 

From e64d71b1354b1a2a3ffe3c3f246098f217c3925d Mon Sep 17 00:00:00 2001
From: DJ2LS <dj2ls@proton.me>
Date: Thu, 22 Feb 2024 15:18:25 +0100
Subject: [PATCH 03/29] set initial speed level

---
 modem/arq_session_irs.py | 1 +
 modem/arq_session_iss.py | 4 ++++
 2 files changed, 5 insertions(+)

diff --git a/modem/arq_session_irs.py b/modem/arq_session_irs.py
index fbc460b8..ef5a70d1 100644
--- a/modem/arq_session_irs.py
+++ b/modem/arq_session_irs.py
@@ -130,6 +130,7 @@ class ARQSessionIRS(arq_session.ARQSession):
         self.launch_transmit_and_wait(info_ack, self.TIMEOUT_CONNECT, mode=FREEDV_MODE.signalling)
         if not self.abort:
             self.set_state(IRS_State.INFO_ACK_SENT)
+            self.calibrate_speed_settings()
         return None, None
 
     def process_incoming_data(self, frame):
diff --git a/modem/arq_session_iss.py b/modem/arq_session_iss.py
index 7ff769f3..1f1d34b1 100644
--- a/modem/arq_session_iss.py
+++ b/modem/arq_session_iss.py
@@ -68,6 +68,8 @@ class ARQSessionISS(arq_session.ARQSession):
 
         self.frame_factory = data_frame_factory.DataFrameFactory(self.config)
 
+        self.speed_level = 0
+
     def generate_id(self):
         while True:
             random_int = random.randint(1,255)
@@ -125,6 +127,8 @@ class ARQSessionISS(arq_session.ARQSession):
             self.speed_level -= 1
             self.log(f"Downshifting. New speed level: {self.speed_level}")
 
+
+
     def send_info(self, irs_frame):
         # check if we received an abort flag
         if irs_frame["flag"]["ABORT"]:

From 451ec404e99421633b383959f3cb184d3cd00502 Mon Sep 17 00:00:00 2001
From: DJ2LS <dj2ls@proton.me>
Date: Thu, 22 Feb 2024 15:30:21 +0100
Subject: [PATCH 04/29] adjusted gear shifting

---
 modem/arq_session_irs.py | 31 +++++++++++++++----------------
 modem/demodulator.py     |  3 ---
 2 files changed, 15 insertions(+), 19 deletions(-)

diff --git a/modem/arq_session_irs.py b/modem/arq_session_irs.py
index ef5a70d1..9788d10b 100644
--- a/modem/arq_session_irs.py
+++ b/modem/arq_session_irs.py
@@ -224,23 +224,22 @@ class ARQSessionIRS(arq_session.ARQSession):
         upshift = False
         downshift = False
 
-        # Check if a shift is needed
-        if appropriate_speed_level != self.speed_level:
-            # Determine the direction of the shift
-            upshift = appropriate_speed_level > self.speed_level
-            downshift = appropriate_speed_level < self.speed_level
+        # Determine if we need to shift
+        if appropriate_speed_level > self.speed_level and self.speed_level < len(self.SPEED_LEVEL_DICT) - 1:
+            # Upshift by one level
+            self.speed_level += 1
+            upshift = True
+            self.log(f"Upshifting. New speed level: {self.speed_level}", isWarning=True)
+        elif appropriate_speed_level < self.speed_level and self.speed_level > 0:
+            # Downshift by one level
+            self.speed_level -= 1
+            downshift = True
+            self.log(f"Downshifting. New speed level: {self.speed_level}", isWarning=True)
 
-            # Update the speed level
-            self.speed_level = appropriate_speed_level
-            self.log(f"Updated Speed Level: {self.speed_level}", isWarning=True)
-
-            # Determine additional modes based on the direction of the shift
-            if upshift and self.speed_level < len(self.SPEED_LEVEL_DICT) - 1:
-                next_mode = self.get_mode_by_speed_level(self.speed_level).value
-                modes_to_decode[next_mode] = True
-            elif downshift and self.speed_level > 0:
-                prev_mode = self.get_mode_by_speed_level(self.speed_level).value
-                modes_to_decode[prev_mode] = True
+        # Add the new speed level mode to decode, if we have shifted
+        if upshift or downshift:
+            new_mode = self.get_mode_by_speed_level(self.speed_level).value
+            modes_to_decode[new_mode] = True
 
         self.log(f"Modes to Decode: {modes_to_decode}", isWarning=True)
         # Apply the new decode mode based on the updated speed level
diff --git a/modem/demodulator.py b/modem/demodulator.py
index 01358174..7a65342e 100644
--- a/modem/demodulator.py
+++ b/modem/demodulator.py
@@ -391,6 +391,3 @@ class Demodulator():
         for mode, decode in modes_to_decode.items():
             if mode in self.MODE_DICT:
                 self.MODE_DICT[mode]["decode"] = decode
-
-        enabled_modes = [mode.name for mode, details in self.MODE_DICT.items() if details["decode"]]
-        self.log.info(f"[MDM] [demod_audio] Enabled decode modes: {', '.join(enabled_modes)}")

From 7eb9fa1dc5965bd2ac6b5d50289f8e22882b5e10 Mon Sep 17 00:00:00 2001
From: DJ2LS <dj2ls@proton.me>
Date: Thu, 22 Feb 2024 15:39:18 +0100
Subject: [PATCH 05/29] adjusted gear shifting

---
 modem/demodulator.py | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/modem/demodulator.py b/modem/demodulator.py
index 7a65342e..e3ec3620 100644
--- a/modem/demodulator.py
+++ b/modem/demodulator.py
@@ -387,6 +387,9 @@ class Demodulator():
         for m in self.MODE_DICT:
             self.MODE_DICT[m]["decode"] = False
 
+        # signalling is always true
+        self.MODE_DICT[codec2.FREEDV_MODE.signalling.value]["decode"] = True
+
         # Enable specified modes
         for mode, decode in modes_to_decode.items():
             if mode in self.MODE_DICT:

From 89f61c15fd89ce60cbb44ed64d807f3fd70d06f8 Mon Sep 17 00:00:00 2001
From: DJ2LS <dj2ls@proton.me>
Date: Thu, 22 Feb 2024 15:46:15 +0100
Subject: [PATCH 06/29] adjusted gear shifting

---
 modem/arq_session_irs.py | 25 +++++++++++++++----------
 1 file changed, 15 insertions(+), 10 deletions(-)

diff --git a/modem/arq_session_irs.py b/modem/arq_session_irs.py
index 9788d10b..252fe3fc 100644
--- a/modem/arq_session_irs.py
+++ b/modem/arq_session_irs.py
@@ -216,32 +216,37 @@ class ARQSessionIRS(arq_session.ARQSession):
             f"Latest SNR: {latest_snr}, Current Speed Level: {self.speed_level}, Appropriate Speed Level: {appropriate_speed_level}",
             isWarning=True)
 
-        # Always decode the current mode
-        current_mode = self.get_mode_by_speed_level(self.speed_level).value
-        modes_to_decode[current_mode] = True
-
         # Initialize shift flags
         upshift = False
         downshift = False
 
         # Determine if we need to shift
         if appropriate_speed_level > self.speed_level and self.speed_level < len(self.SPEED_LEVEL_DICT) - 1:
-            # Upshift by one level
+            # Upshift by one level, but remember to listen on the current level as well in case of loosing ACK
+            previous_speed_level = self.speed_level
             self.speed_level += 1
             upshift = True
             self.log(f"Upshifting. New speed level: {self.speed_level}", isWarning=True)
         elif appropriate_speed_level < self.speed_level and self.speed_level > 0:
-            # Downshift by one level
+            # Downshift by one level, but remember to listen on the current level as well in case of loosing ACK
+            previous_speed_level = self.speed_level
             self.speed_level -= 1
             downshift = True
             self.log(f"Downshifting. New speed level: {self.speed_level}", isWarning=True)
+        else:
+            # No shift needed, set previous to current for correct mode decoding setup
+            previous_speed_level = self.speed_level
 
-        # Add the new speed level mode to decode, if we have shifted
+        # Decode the current mode
+        current_mode = self.get_mode_by_speed_level(self.speed_level).value
+        modes_to_decode[current_mode] = True
+
+        # Additionally, decode the previous speed level mode if it has changed
         if upshift or downshift:
-            new_mode = self.get_mode_by_speed_level(self.speed_level).value
-            modes_to_decode[new_mode] = True
+            previous_mode = self.get_mode_by_speed_level(previous_speed_level).value
+            modes_to_decode[previous_mode] = True
 
-        self.log(f"Modes to Decode: {modes_to_decode}", isWarning=True)
+        self.log(f"Modes to Decode: {modes_to_decode.keys()}", isWarning=True)
         # Apply the new decode mode based on the updated speed level
         self.modem.demodulator.set_decode_mode(modes_to_decode)
 

From 0100104afbe0b3ba92772daf7d7565b55059f07f Mon Sep 17 00:00:00 2001
From: DJ2LS <dj2ls@proton.me>
Date: Thu, 22 Feb 2024 16:03:28 +0100
Subject: [PATCH 07/29] fixed missing arq abort state

---
 modem/arq_session_irs.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/modem/arq_session_irs.py b/modem/arq_session_irs.py
index 252fe3fc..9141d50f 100644
--- a/modem/arq_session_irs.py
+++ b/modem/arq_session_irs.py
@@ -260,6 +260,7 @@ class ARQSessionIRS(arq_session.ARQSession):
         stop_ack = self.frame_factory.build_arq_stop_ack(self.id)
         self.launch_transmit_and_wait(stop_ack, self.TIMEOUT_CONNECT, mode=FREEDV_MODE.signalling)
         self.set_state(IRS_State.ABORTED)
+        self.states.setARQ(False)
         self.event_manager.send_arq_session_finished(
                 False, self.id, self.dxcall, False, self.state.name, statistics=self.calculate_session_statistics())
         return None, None

From 3574f76a79c1543491e871a8b8959b6796a34fa0 Mon Sep 17 00:00:00 2001
From: DJ2LS <dj2ls@proton.me>
Date: Thu, 22 Feb 2024 21:15:43 +0100
Subject: [PATCH 08/29] moved from upshift downshift to speed level int

---
 modem/arq_session.py        |  3 +-
 modem/arq_session_irs.py    | 59 +++++++++++++++----------------------
 modem/arq_session_iss.py    | 43 +++++++++++++++++----------
 modem/data_frame_factory.py | 16 +++-------
 4 files changed, 57 insertions(+), 64 deletions(-)

diff --git a/modem/arq_session.py b/modem/arq_session.py
index 75334d6a..4b259cd6 100644
--- a/modem/arq_session.py
+++ b/modem/arq_session.py
@@ -44,6 +44,8 @@ class ARQSession():
 
         self.modem = modem
         self.speed_level = 0
+        self.previous_speed_level = 0
+
         self.frames_per_burst = 1
 
         self.frame_factory = data_frame_factory.DataFrameFactory(self.config)
@@ -97,7 +99,6 @@ class ARQSession():
                 received_data, type_byte = getattr(self, action_name)(frame)
                 if isinstance(received_data, bytearray) and isinstance(type_byte, int):
                     self.arq_data_type_handler.dispatch(type_byte, received_data)
-
                 return
         
         self.log(f"Ignoring unknown transition from state {self.state.name} with frame {frame['frame_type']}")
diff --git a/modem/arq_session_irs.py b/modem/arq_session_irs.py
index 9141d50f..52dd8fa4 100644
--- a/modem/arq_session_irs.py
+++ b/modem/arq_session_irs.py
@@ -76,8 +76,6 @@ class ARQSessionIRS(arq_session.ARQSession):
         self.received_bytes = 0
         self.received_crc = None
 
-        self.speed_level = 0
-
         self.abort = False
 
     def all_data_received(self):
@@ -160,16 +158,14 @@ class ARQSessionIRS(arq_session.ARQSession):
         self.process_incoming_data(burst_frame)
 
         if not self.all_data_received():
-            downshift, upshift = self.calibrate_speed_settings()
+            self.calibrate_speed_settings(burst_frame=burst_frame)
             ack = self.frame_factory.build_arq_burst_ack(
                 self.id,
                 self.received_bytes,
                 self.speed_level,
                 self.frames_per_burst,
                 self.snr[0],
-                flag_abort=self.abort,
-                flag_upshift=upshift,
-                flag_downshift=downshift
+                flag_abort=self.abort
             )
 
             self.set_state(IRS_State.BURST_REPLY_SENT)
@@ -204,53 +200,46 @@ class ARQSessionIRS(arq_session.ARQSession):
                                                          flag_checksum=False)
             self.transmit_frame(ack, mode=FREEDV_MODE.signalling)
             self.log("CRC fail at the end of transmission!")
-            self.transmission_failed()
+            return self.transmission_failed()
+
+    def calibrate_speed_settings(self, burst_frame=None):
+        if burst_frame:
+            received_speed_level = burst_frame['speed_level']
+        else:
+            received_speed_level = 0
 
-    def calibrate_speed_settings(self):
         latest_snr = self.snr[-1] if self.snr else -10
         appropriate_speed_level = self.get_appropriate_speed_level(latest_snr)
         modes_to_decode = {}
 
-        # Log the latest SNR, current, and appropriate speed levels for clarity
+        # Log the latest SNR, current, appropriate speed levels, and the previous speed level
         self.log(
-            f"Latest SNR: {latest_snr}, Current Speed Level: {self.speed_level}, Appropriate Speed Level: {appropriate_speed_level}",
+            f"Latest SNR: {latest_snr}, Current Speed Level: {self.speed_level}, Appropriate Speed Level: {appropriate_speed_level}, Previous Speed Level: {self.previous_speed_level}",
             isWarning=True)
 
-        # Initialize shift flags
-        upshift = False
-        downshift = False
-
-        # Determine if we need to shift
+        # Adjust the speed level by one step towards the appropriate level, if needed
         if appropriate_speed_level > self.speed_level and self.speed_level < len(self.SPEED_LEVEL_DICT) - 1:
-            # Upshift by one level, but remember to listen on the current level as well in case of loosing ACK
-            previous_speed_level = self.speed_level
-            self.speed_level += 1
-            upshift = True
-            self.log(f"Upshifting. New speed level: {self.speed_level}", isWarning=True)
+            if received_speed_level == self.speed_level:
+                self.speed_level += 1
         elif appropriate_speed_level < self.speed_level and self.speed_level > 0:
-            # Downshift by one level, but remember to listen on the current level as well in case of loosing ACK
-            previous_speed_level = self.speed_level
-            self.speed_level -= 1
-            downshift = True
-            self.log(f"Downshifting. New speed level: {self.speed_level}", isWarning=True)
-        else:
-            # No shift needed, set previous to current for correct mode decoding setup
-            previous_speed_level = self.speed_level
+            if received_speed_level == self.speed_level:
+                self.speed_level -= 1
 
-        # Decode the current mode
+        # Always decode the current mode
         current_mode = self.get_mode_by_speed_level(self.speed_level).value
         modes_to_decode[current_mode] = True
 
-        # Additionally, decode the previous speed level mode if it has changed
-        if upshift or downshift:
-            previous_mode = self.get_mode_by_speed_level(previous_speed_level).value
+        # Decode the previous speed level mode
+        if self.previous_speed_level != self.speed_level:
+            previous_mode = self.get_mode_by_speed_level(self.previous_speed_level).value
             modes_to_decode[previous_mode] = True
+            self.previous_speed_level = self.speed_level  # Update the previous speed level
 
-        self.log(f"Modes to Decode: {modes_to_decode.keys()}", isWarning=True)
-        # Apply the new decode mode based on the updated speed level
+        self.log(f"Modes to Decode: {list(modes_to_decode.keys())}", isWarning=True)
+        # Apply the new decode mode based on the updated and previous speed levels
         self.modem.demodulator.set_decode_mode(modes_to_decode)
 
-        return downshift, upshift
+        return self.speed_level
 
     def abort_transmission(self):
         self.log(f"Aborting transmission... setting abort flag")
diff --git a/modem/arq_session_iss.py b/modem/arq_session_iss.py
index 1f1d34b1..8308067d 100644
--- a/modem/arq_session_iss.py
+++ b/modem/arq_session_iss.py
@@ -68,8 +68,6 @@ class ARQSessionISS(arq_session.ARQSession):
 
         self.frame_factory = data_frame_factory.DataFrameFactory(self.config)
 
-        self.speed_level = 0
-
     def generate_id(self):
         while True:
             random_int = random.randint(1,255)
@@ -113,21 +111,34 @@ class ARQSessionISS(arq_session.ARQSession):
         # Log the received frame for debugging
         self.log(f"Received frame: {frame}", isWarning=True)
 
-        # Safely extract upshift and downshift flags with default to False if not present
-        upshift = frame['flag'].get('UPSHIFT', False)
-        downshift = frame['flag'].get('DOWNSHIFT', False)
-
-        # Check for UPSHIFT frame and ensure speed level does not exceed max limit
-        if upshift and not downshift and self.speed_level < len(self.SPEED_LEVEL_DICT) - 1:
-            self.speed_level += 1
-            self.log(f"Upshifting. New speed level: {self.speed_level}")
-
-        # Check for DOWNSHIFT frame and ensure speed level does not go below 0
-        elif downshift and not upshift and self.speed_level > 0:
-            self.speed_level -= 1
-            self.log(f"Downshifting. New speed level: {self.speed_level}")
+        # Extract the speed_level directly from the frame
+        if 'speed_level' in frame:
+            new_speed_level = frame['speed_level']
+            # Ensure the new speed level is within the allowable range
+            if 0 <= new_speed_level < len(self.SPEED_LEVEL_DICT):
+                # Log the speed level change if it's different from the current speed level
+                if new_speed_level != self.speed_level:
+                    self.log(f"Changing speed level from {self.speed_level} to {new_speed_level}", isWarning=True)
+                    self.previous_speed_level = self.speed_level  # Store the current speed level as previous
+                    self.speed_level = new_speed_level  # Update the current speed level
+                else:
+                    self.log("Received speed level is the same as the current speed level.", isWarning=True)
+            else:
+                self.log(f"Received speed level {new_speed_level} is out of allowable range.", isWarning=True)
+        else:
+            self.log("No speed level specified in the received frame.", isWarning=True)
 
+        # Apply the new decode mode based on the updated speed level, including the previous speed level
+        modes_to_decode = {
+            self.get_mode_by_speed_level(self.speed_level).value: True,
+        }
+        # Include the previous speed level mode if it's different from the current
+        if hasattr(self, 'previous_speed_level') and self.previous_speed_level != self.speed_level:
+            modes_to_decode[self.get_mode_by_speed_level(self.previous_speed_level).value] = True
 
+        self.log(f"Modes to Decode: {list(modes_to_decode.keys())}", isWarning=True)
+        # Apply the new decode mode based on the current and previous speed levels
+        self.modem.demodulator.set_decode_mode(modes_to_decode)
 
     def send_info(self, irs_frame):
         # check if we received an abort flag
@@ -174,7 +185,7 @@ class ARQSessionISS(arq_session.ARQSession):
             payload = self.data[offset : offset + payload_size]
             data_frame = self.frame_factory.build_arq_burst_frame(
                 self.SPEED_LEVEL_DICT[self.speed_level]["mode"],
-                self.id, self.confirmed_bytes, payload)
+                self.id, self.confirmed_bytes, payload, self.speed_level)
             burst.append(data_frame)
         self.launch_twr(burst, self.TIMEOUT_TRANSFER, self.RETRIES_CONNECT, mode='auto')
         self.set_state(ISS_State.BURST_SENT)
diff --git a/modem/data_frame_factory.py b/modem/data_frame_factory.py
index 894e2d1c..3fb4c9fb 100644
--- a/modem/data_frame_factory.py
+++ b/modem/data_frame_factory.py
@@ -15,8 +15,6 @@ class DataFrameFactory:
         'FINAL': 0,  # Bit-position for indicating the FINAL state
         'ABORT': 1, # Bit-position for indicating the ABORT request
         'CHECKSUM': 2,  # Bit-position for indicating the CHECKSUM is correct or not
-        'DOWNSHIFT': 3, # Bit-position for indicating a requested ARQ DOWNSHIFT
-        'UPSHIFT': 4, # Bit-position for indicating a requested ARQ UPSHIFT
     }
 
     def __init__(self, config):
@@ -146,6 +144,7 @@ class DataFrameFactory:
         self.template_list[FR_TYPE.ARQ_BURST_FRAME.value] = {
             "frame_length": None,
             "session_id": 1,
+            "speed_level": 1,
             "offset": 4,
             "data": "dynamic",
         }
@@ -396,9 +395,10 @@ class DataFrameFactory:
         }        
         return self.construct(FR_TYPE.ARQ_SESSION_INFO_ACK, payload)
 
-    def build_arq_burst_frame(self, freedv_mode: codec2.FREEDV_MODE, session_id: int, offset: int, data: bytes):
+    def build_arq_burst_frame(self, freedv_mode: codec2.FREEDV_MODE, session_id: int, offset: int, data: bytes, speed_level: int):
         payload = {
             "session_id": session_id.to_bytes(1, 'big'),
+            "speed_level": speed_level.to_bytes(4, 'big'),
             "offset": offset.to_bytes(4, 'big'),
             "data": data,
         }
@@ -406,7 +406,7 @@ class DataFrameFactory:
         return frame
 
     def build_arq_burst_ack(self, session_id: bytes, offset, speed_level: int, 
-                            frames_per_burst: int, snr: int, flag_final=False, flag_checksum=False, flag_abort=False, flag_downshift=False, flag_upshift=False):
+                            frames_per_burst: int, snr: int, flag_final=False, flag_checksum=False, flag_abort=False):
         flag = 0b00000000
         if flag_final:
             flag = helpers.set_flag(flag, 'FINAL', True, self.ARQ_FLAGS)
@@ -417,14 +417,6 @@ class DataFrameFactory:
         if flag_abort:
             flag = helpers.set_flag(flag, 'ABORT', True, self.ARQ_FLAGS)
 
-        if flag_downshift:
-            flag = helpers.set_flag(flag, 'DOWNSHIFT', True, self.ARQ_FLAGS)
-            flag = helpers.set_flag(flag, 'UPSHIFT', False, self.ARQ_FLAGS)
-
-        if flag_upshift:
-            flag = helpers.set_flag(flag, 'DOWNSHIFT', False, self.ARQ_FLAGS)
-            flag = helpers.set_flag(flag, 'UPSHIFT', True, self.ARQ_FLAGS)
-
         payload = {
             "session_id": session_id.to_bytes(1, 'big'),
             "offset": offset.to_bytes(4, 'big'),

From 33ad50fbe28f7aca7a0dd64f1db2f8bae9b90362 Mon Sep 17 00:00:00 2001
From: DJ2LS <dj2ls@proton.me>
Date: Thu, 22 Feb 2024 21:49:57 +0100
Subject: [PATCH 09/29] moved from upshift downshift to speed level int

---
 modem/arq_session_irs.py |  2 ++
 modem/arq_session_iss.py | 13 -------------
 2 files changed, 2 insertions(+), 13 deletions(-)

diff --git a/modem/arq_session_irs.py b/modem/arq_session_irs.py
index 52dd8fa4..97c4a5a7 100644
--- a/modem/arq_session_irs.py
+++ b/modem/arq_session_irs.py
@@ -219,9 +219,11 @@ class ARQSessionIRS(arq_session.ARQSession):
 
         # Adjust the speed level by one step towards the appropriate level, if needed
         if appropriate_speed_level > self.speed_level and self.speed_level < len(self.SPEED_LEVEL_DICT) - 1:
+            # we need to ensure, the received data is equal to our speed level before changing it
             if received_speed_level == self.speed_level:
                 self.speed_level += 1
         elif appropriate_speed_level < self.speed_level and self.speed_level > 0:
+            # we need to ensure, the received data is equal to our speed level before changing it
             if received_speed_level == self.speed_level:
                 self.speed_level -= 1
 
diff --git a/modem/arq_session_iss.py b/modem/arq_session_iss.py
index 8308067d..f481c97b 100644
--- a/modem/arq_session_iss.py
+++ b/modem/arq_session_iss.py
@@ -119,7 +119,6 @@ class ARQSessionISS(arq_session.ARQSession):
                 # Log the speed level change if it's different from the current speed level
                 if new_speed_level != self.speed_level:
                     self.log(f"Changing speed level from {self.speed_level} to {new_speed_level}", isWarning=True)
-                    self.previous_speed_level = self.speed_level  # Store the current speed level as previous
                     self.speed_level = new_speed_level  # Update the current speed level
                 else:
                     self.log("Received speed level is the same as the current speed level.", isWarning=True)
@@ -128,18 +127,6 @@ class ARQSessionISS(arq_session.ARQSession):
         else:
             self.log("No speed level specified in the received frame.", isWarning=True)
 
-        # Apply the new decode mode based on the updated speed level, including the previous speed level
-        modes_to_decode = {
-            self.get_mode_by_speed_level(self.speed_level).value: True,
-        }
-        # Include the previous speed level mode if it's different from the current
-        if hasattr(self, 'previous_speed_level') and self.previous_speed_level != self.speed_level:
-            modes_to_decode[self.get_mode_by_speed_level(self.previous_speed_level).value] = True
-
-        self.log(f"Modes to Decode: {list(modes_to_decode.keys())}", isWarning=True)
-        # Apply the new decode mode based on the current and previous speed levels
-        self.modem.demodulator.set_decode_mode(modes_to_decode)
-
     def send_info(self, irs_frame):
         # check if we received an abort flag
         if irs_frame["flag"]["ABORT"]:

From 7ecccabcc0fc283844175697439d2e281629337e Mon Sep 17 00:00:00 2001
From: DJ2LS <dj2ls@proton.me>
Date: Fri, 23 Feb 2024 08:59:03 +0100
Subject: [PATCH 10/29] fixed arq burst

---
 modem/data_frame_factory.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/modem/data_frame_factory.py b/modem/data_frame_factory.py
index 3fb4c9fb..2acc837c 100644
--- a/modem/data_frame_factory.py
+++ b/modem/data_frame_factory.py
@@ -398,7 +398,7 @@ class DataFrameFactory:
     def build_arq_burst_frame(self, freedv_mode: codec2.FREEDV_MODE, session_id: int, offset: int, data: bytes, speed_level: int):
         payload = {
             "session_id": session_id.to_bytes(1, 'big'),
-            "speed_level": speed_level.to_bytes(4, 'big'),
+            "speed_level": speed_level.to_bytes(1, 'big'),
             "offset": offset.to_bytes(4, 'big'),
             "data": data,
         }

From f307ed779fc93a1b0c20b2636da74e3f6baa8986 Mon Sep 17 00:00:00 2001
From: DJ2LS <dj2ls@proton.me>
Date: Fri, 23 Feb 2024 13:55:15 +0100
Subject: [PATCH 11/29] attempt of sending in previous speed level

---
 modem/arq_session_iss.py | 11 +++++++++--
 1 file changed, 9 insertions(+), 2 deletions(-)

diff --git a/modem/arq_session_iss.py b/modem/arq_session_iss.py
index f481c97b..0bfb53f4 100644
--- a/modem/arq_session_iss.py
+++ b/modem/arq_session_iss.py
@@ -76,7 +76,6 @@ class ARQSessionISS(arq_session.ARQSession):
             if len(self.state_manager.arq_iss_sessions) >= 255:
                 return False
 
-
     def transmit_wait_and_retry(self, frame_or_burst, timeout, retries, mode):
         while retries > 0:
             self.event_frame_received = threading.Event()
@@ -90,7 +89,15 @@ class ARQSessionISS(arq_session.ARQSession):
                 return
             self.log("Timeout!")
             retries = retries - 1
-        
+
+            # TODO TEMPORARY TEST FOR SENDING IN LOWER SPEED LEVEL IF WE HAVE TWO FAILED TRANSMISSIONS!!!
+            if retries == 8:
+                self.log("SENDING IN LOWER SPEED LEVEL at", isWarning=True)
+                if self.speed_level > 0:
+                    self.speed_level -= 1
+                self.send_data({'flag':{'ABORT': False, 'FINAL': False}, 'speed_level': self.speed_level})
+                return
+
         self.set_state(ISS_State.FAILED)
         self.transmission_failed()
 

From d3d09d4019e2ca46f876ac4264baceeb1eb60aaa Mon Sep 17 00:00:00 2001
From: DJ2LS <dj2ls@proton.me>
Date: Fri, 23 Feb 2024 13:59:47 +0100
Subject: [PATCH 12/29] attempt of sending in previous speed level

---
 modem/arq_session_iss.py | 10 +++++-----
 1 file changed, 5 insertions(+), 5 deletions(-)

diff --git a/modem/arq_session_iss.py b/modem/arq_session_iss.py
index 0bfb53f4..e79ce3e1 100644
--- a/modem/arq_session_iss.py
+++ b/modem/arq_session_iss.py
@@ -76,7 +76,7 @@ class ARQSessionISS(arq_session.ARQSession):
             if len(self.state_manager.arq_iss_sessions) >= 255:
                 return False
 
-    def transmit_wait_and_retry(self, frame_or_burst, timeout, retries, mode):
+    def transmit_wait_and_retry(self, frame_or_burst, timeout, retries, mode, isARQBurst=False):
         while retries > 0:
             self.event_frame_received = threading.Event()
             if isinstance(frame_or_burst, list): burst = frame_or_burst
@@ -91,7 +91,7 @@ class ARQSessionISS(arq_session.ARQSession):
             retries = retries - 1
 
             # TODO TEMPORARY TEST FOR SENDING IN LOWER SPEED LEVEL IF WE HAVE TWO FAILED TRANSMISSIONS!!!
-            if retries == 8:
+            if retries == 8 and isARQBurst:
                 self.log("SENDING IN LOWER SPEED LEVEL at", isWarning=True)
                 if self.speed_level > 0:
                     self.speed_level -= 1
@@ -101,8 +101,8 @@ class ARQSessionISS(arq_session.ARQSession):
         self.set_state(ISS_State.FAILED)
         self.transmission_failed()
 
-    def launch_twr(self, frame_or_burst, timeout, retries, mode):
-        twr = threading.Thread(target = self.transmit_wait_and_retry, args=[frame_or_burst, timeout, retries, mode], daemon=True)
+    def launch_twr(self, frame_or_burst, timeout, retries, mode, isARQBurst=False):
+        twr = threading.Thread(target = self.transmit_wait_and_retry, args=[frame_or_burst, timeout, retries, mode, isARQBurst], daemon=True)
         twr.start()
 
     def start(self):
@@ -181,7 +181,7 @@ class ARQSessionISS(arq_session.ARQSession):
                 self.SPEED_LEVEL_DICT[self.speed_level]["mode"],
                 self.id, self.confirmed_bytes, payload, self.speed_level)
             burst.append(data_frame)
-        self.launch_twr(burst, self.TIMEOUT_TRANSFER, self.RETRIES_CONNECT, mode='auto')
+        self.launch_twr(burst, self.TIMEOUT_TRANSFER, self.RETRIES_CONNECT, mode='auto', isARQBurst=True)
         self.set_state(ISS_State.BURST_SENT)
         return None, None
 

From 36890fe13172f006ec61fac23acc49cd42f6d054 Mon Sep 17 00:00:00 2001
From: DJ2LS <dj2ls@proton.me>
Date: Fri, 23 Feb 2024 14:38:08 +0100
Subject: [PATCH 13/29] adding fallback speed level

---
 modem/arq_session_iss.py | 9 ++++-----
 1 file changed, 4 insertions(+), 5 deletions(-)

diff --git a/modem/arq_session_iss.py b/modem/arq_session_iss.py
index e79ce3e1..61442ea2 100644
--- a/modem/arq_session_iss.py
+++ b/modem/arq_session_iss.py
@@ -76,7 +76,7 @@ class ARQSessionISS(arq_session.ARQSession):
             if len(self.state_manager.arq_iss_sessions) >= 255:
                 return False
 
-    def transmit_wait_and_retry(self, frame_or_burst, timeout, retries, mode, isARQBurst=False):
+    def transmit_wait_and_retry(self, frame_or_burst, timeout, retries, mode, isARQBurst=False, ):
         while retries > 0:
             self.event_frame_received = threading.Event()
             if isinstance(frame_or_burst, list): burst = frame_or_burst
@@ -91,10 +91,9 @@ class ARQSessionISS(arq_session.ARQSession):
             retries = retries - 1
 
             # TODO TEMPORARY TEST FOR SENDING IN LOWER SPEED LEVEL IF WE HAVE TWO FAILED TRANSMISSIONS!!!
-            if retries == 8 and isARQBurst:
-                self.log("SENDING IN LOWER SPEED LEVEL at", isWarning=True)
-                if self.speed_level > 0:
-                    self.speed_level -= 1
+            if retries == 8 and isARQBurst and self.speed_level > 0:
+                self.log("SENDING IN FALLBACK SPEED LEVEL", isWarning=True)
+                self.speed_level = 0
                 self.send_data({'flag':{'ABORT': False, 'FINAL': False}, 'speed_level': self.speed_level})
                 return
 

From 374f400f307da299ff34aa312287dc4b150b559e Mon Sep 17 00:00:00 2001
From: DJ2LS <dj2ls@proton.me>
Date: Fri, 23 Feb 2024 15:21:52 +0100
Subject: [PATCH 14/29] adding fallback speed level

---
 modem/demodulator.py | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/modem/demodulator.py b/modem/demodulator.py
index e3ec3620..554ca0ef 100644
--- a/modem/demodulator.py
+++ b/modem/demodulator.py
@@ -390,6 +390,9 @@ class Demodulator():
         # signalling is always true
         self.MODE_DICT[codec2.FREEDV_MODE.signalling.value]["decode"] = True
 
+        # lowest speed level is alwys true
+        self.MODE_DICT[codec2.FREEDV_MODE.datac4.value]["decode"] = True
+
         # Enable specified modes
         for mode, decode in modes_to_decode.items():
             if mode in self.MODE_DICT:

From 956cede5935d2ca99cc488a15e3fb524098ae802 Mon Sep 17 00:00:00 2001
From: DJ2LS <dj2ls@proton.me>
Date: Sat, 24 Feb 2024 20:34:53 +0100
Subject: [PATCH 15/29] try except for schedule manager

---
 modem/schedule_manager.py | 13 ++++++++-----
 1 file changed, 8 insertions(+), 5 deletions(-)

diff --git a/modem/schedule_manager.py b/modem/schedule_manager.py
index 2bc22adf..f238b911 100644
--- a/modem/schedule_manager.py
+++ b/modem/schedule_manager.py
@@ -79,10 +79,13 @@ class ScheduleManager:
 
     def check_for_queued_messages(self):
         if not self.state_manager.getARQ():
-            if DatabaseManagerMessages(self.event_manager).get_first_queued_message():
-                params = DatabaseManagerMessages(self.event_manager).get_first_queued_message()
-                command = command_message_send.SendMessageCommand(self.config_manager.read(), self.state_manager, self.event_manager, params)
-                command.transmit(self.modem)
-
+            try:
+                if first_queued_message := DatabaseManagerMessages(
+                    self.event_manager
+                ).get_first_queued_message():
+                    command = command_message_send.SendMessageCommand(self.config_manager.read(), self.state_manager, self.event_manager, first_queued_message)
+                    command.transmit(self.modem)
+            except Exception as e:
+                print(e)
         return
 

From dd4ca1903b0b887acc64dea9070a1f58fc7d6131 Mon Sep 17 00:00:00 2001
From: DJ2LS <dj2ls@proton.me>
Date: Sat, 24 Feb 2024 21:49:53 +0100
Subject: [PATCH 16/29] added session statistics

---
 modem/arq_data_type_handler.py      | 42 ++++++++++++++---------------
 modem/arq_session.py                | 35 +++++++++++++++++++-----
 modem/arq_session_irs.py            | 18 +++++++------
 modem/arq_session_iss.py            |  9 ++++---
 modem/command_message_send.py       |  2 +-
 modem/event_manager.py              |  6 ++++-
 modem/message_p2p.py                | 12 ++++-----
 modem/message_system_db_messages.py |  8 ++++--
 8 files changed, 83 insertions(+), 49 deletions(-)

diff --git a/modem/arq_data_type_handler.py b/modem/arq_data_type_handler.py
index 8745056a..573d4077 100644
--- a/modem/arq_data_type_handler.py
+++ b/modem/arq_data_type_handler.py
@@ -52,23 +52,23 @@ class ARQDataTypeHandler:
                 return session_type
         return None
 
-    def dispatch(self, type_byte: int, data: bytearray):
+    def dispatch(self, type_byte: int, data: bytearray, statistics: dict):
         session_type = self.get_session_type_from_value(type_byte)
 
         self.state_manager.setARQ(False)
 
         if session_type and session_type in self.handlers and 'handle' in self.handlers[session_type]:
-            return self.handlers[session_type]['handle'](data)
+            return self.handlers[session_type]['handle'](data, statistics)
         else:
             self.log(f"Unknown handling endpoint for type: {type_byte}", isWarning=True)
 
-    def failed(self, type_byte: int, data: bytearray):
+    def failed(self, type_byte: int, data: bytearray, statistics: dict):
         session_type = self.get_session_type_from_value(type_byte)
 
         self.state_manager.setARQ(False)
 
         if session_type in self.handlers and 'failed' in self.handlers[session_type]:
-            return self.handlers[session_type]['failed'](data)
+            return self.handlers[session_type]['failed'](data, statistics)
         else:
             self.log(f"Unknown handling endpoint: {session_type}", isWarning=True)
 
@@ -78,13 +78,13 @@ class ARQDataTypeHandler:
         else:
             self.log(f"Unknown preparation endpoint: {session_type}", isWarning=True)
 
-    def transmitted(self, type_byte: int, data: bytearray):
+    def transmitted(self, type_byte: int, data: bytearray, statistics: dict):
         session_type = self.get_session_type_from_value(type_byte)
 
         self.state_manager.setARQ(False)
 
         if session_type in self.handlers and 'transmitted' in self.handlers[session_type]:
-            return self.handlers[session_type]['transmitted'](data)
+            return self.handlers[session_type]['transmitted'](data, statistics)
         else:
             self.log(f"Unknown handling endpoint: {session_type}", isWarning=True)
 
@@ -97,14 +97,14 @@ class ARQDataTypeHandler:
         self.log(f"Preparing uncompressed data: {len(data)} Bytes")
         return data
 
-    def handle_raw(self, data):
+    def handle_raw(self, data, statistics):
         self.log(f"Handling uncompressed data: {len(data)} Bytes")
         return data
 
-    def failed_raw(self, data):
+    def failed_raw(self, data, statistics):
         return
 
-    def transmitted_raw(self, data):
+    def transmitted_raw(self, data, statistics):
         return data
 
     def prepare_raw_lzma(self, data):
@@ -112,15 +112,15 @@ class ARQDataTypeHandler:
         self.log(f"Preparing LZMA compressed data: {len(data)} Bytes >>> {len(compressed_data)} Bytes")
         return compressed_data
 
-    def handle_raw_lzma(self, data):
+    def handle_raw_lzma(self, data, statistics):
         decompressed_data = lzma.decompress(data)
         self.log(f"Handling LZMA compressed data: {len(decompressed_data)} Bytes from {len(data)} Bytes")
         return decompressed_data
 
-    def failed_raw_lzma(self, data):
+    def failed_raw_lzma(self, data, statistics):
         return
 
-    def transmitted_raw_lzma(self, data):
+    def transmitted_raw_lzma(self, data, statistics):
         decompressed_data = lzma.decompress(data)
         return decompressed_data
 
@@ -129,15 +129,15 @@ class ARQDataTypeHandler:
         self.log(f"Preparing GZIP compressed data: {len(data)} Bytes >>> {len(compressed_data)} Bytes")
         return compressed_data
 
-    def handle_raw_gzip(self, data):
+    def handle_raw_gzip(self, data, statistics):
         decompressed_data = gzip.decompress(data)
         self.log(f"Handling GZIP compressed data: {len(decompressed_data)} Bytes from {len(data)} Bytes")
         return decompressed_data
 
-    def failed_raw_gzip(self, data):
+    def failed_raw_gzip(self, data, statistics):
         return
 
-    def transmitted_raw_gzip(self, data):
+    def transmitted_raw_gzip(self, data, statistics):
         decompressed_data = gzip.decompress(data)
         return decompressed_data
 
@@ -146,19 +146,19 @@ class ARQDataTypeHandler:
         self.log(f"Preparing LZMA compressed P2PMSG data: {len(data)} Bytes >>> {len(compressed_data)} Bytes")
         return compressed_data
 
-    def handle_p2pmsg_lzma(self, data):
+    def handle_p2pmsg_lzma(self, data, statistics):
         decompressed_data = lzma.decompress(data)
         self.log(f"Handling LZMA compressed P2PMSG data: {len(decompressed_data)} Bytes from {len(data)} Bytes")
-        message_received(self.event_manager, self.state_manager, decompressed_data)
+        message_received(self.event_manager, self.state_manager, decompressed_data, statistics)
         return decompressed_data
 
-    def failed_p2pmsg_lzma(self, data):
+    def failed_p2pmsg_lzma(self, data, statistics):
         decompressed_data = lzma.decompress(data)
         self.log(f"Handling failed LZMA compressed P2PMSG data: {len(decompressed_data)} Bytes from {len(data)} Bytes", isWarning=True)
-        message_failed(self.event_manager, self.state_manager, decompressed_data)
+        message_failed(self.event_manager, self.state_manager, decompressed_data, statistics)
         return decompressed_data
 
-    def transmitted_p2pmsg_lzma(self, data):
+    def transmitted_p2pmsg_lzma(self, data, statistics):
         decompressed_data = lzma.decompress(data)
-        message_transmitted(self.event_manager, self.state_manager, decompressed_data)
+        message_transmitted(self.event_manager, self.state_manager, decompressed_data, statistics)
         return decompressed_data
\ No newline at end of file
diff --git a/modem/arq_session.py b/modem/arq_session.py
index 4b259cd6..1e750331 100644
--- a/modem/arq_session.py
+++ b/modem/arq_session.py
@@ -1,3 +1,4 @@
+import datetime
 import queue, threading
 import codec2
 import data_frame_factory
@@ -57,6 +58,11 @@ class ARQSession():
         self.session_ended = 0
         self.session_max_age = 500
 
+        # histogram lists for storing statistics
+        self.snr_histogram = []
+        self.bpm_histogram = []
+        self.time_histogram = []
+
     def log(self, message, isWarning = False):
         msg = f"[{type(self).__name__}][id={self.id}][state={self.state}]: {message}"
         logger = self.logger.warn if isWarning else self.logger.info
@@ -86,7 +92,7 @@ class ARQSession():
             )
 
     def set_details(self, snr, frequency_offset):
-        self.snr.append(snr)
+        self.snr = snr
         self.frequency_offset = frequency_offset
 
     def on_frame_received(self, frame):
@@ -98,7 +104,7 @@ class ARQSession():
                 action_name = self.STATE_TRANSITION[self.state][frame_type]
                 received_data, type_byte = getattr(self, action_name)(frame)
                 if isinstance(received_data, bytearray) and isinstance(type_byte, int):
-                    self.arq_data_type_handler.dispatch(type_byte, received_data)
+                    self.arq_data_type_handler.dispatch(type_byte, received_data, self.calculate_session_statistics())
                 return
         
         self.log(f"Ignoring unknown transition from state {self.state.name} with frame {frame['frame_type']}")
@@ -110,6 +116,9 @@ class ARQSession():
         return False
 
     def calculate_session_duration(self):
+        if self.session_ended == 0:
+            return time.time() - self.session_started
+
         return self.session_ended - self.session_started
 
     def calculate_session_statistics(self):
@@ -124,11 +133,25 @@ class ARQSession():
         else:
             bytes_per_minute = 0
 
+        # Convert histograms lists to dictionaries
+        time_histogram_dict = {i: timestamp for i, timestamp in enumerate(self.time_histogram)}
+        snr_histogram_dict = {i: snr for i, snr in enumerate(self.snr_histogram)}
+        bpm_histogram_dict = {i: bpm for i, bpm in enumerate(self.bpm_histogram)}
+
         return {
-                'total_bytes': total_bytes,
-                'duration': duration,
-                'bytes_per_minute': bytes_per_minute
-            }
+            'total_bytes': total_bytes,
+            'duration': duration,
+            'bytes_per_minute': bytes_per_minute,
+            'time_histogram': time_histogram_dict,
+            'snr_histogram': snr_histogram_dict,
+            'bpm_histogram': bpm_histogram_dict,
+        }
+
+    def update_histograms(self):
+        stats = self.calculate_session_statistics()
+        self.snr_histogram.append(self.snr)
+        self.bpm_histogram.append(stats['bytes_per_minute'])
+        self.time_histogram.append(datetime.datetime.now().isoformat())
 
     def get_appropriate_speed_level(self, snr):
         # Start with the lowest speed level as default
diff --git a/modem/arq_session_irs.py b/modem/arq_session_irs.py
index 97c4a5a7..74ad0b4a 100644
--- a/modem/arq_session_irs.py
+++ b/modem/arq_session_irs.py
@@ -105,7 +105,7 @@ class ARQSessionIRS(arq_session.ARQSession):
             self.id,
             self.dxcall, 
             self.version,
-            self.snr[0], flag_abort=self.abort)
+            self.snr, flag_abort=self.abort)
         self.launch_transmit_and_wait(ack_frame, self.TIMEOUT_CONNECT, mode=FREEDV_MODE.signalling)
         if not self.abort:
             self.set_state(IRS_State.OPEN_ACK_SENT)
@@ -123,7 +123,7 @@ class ARQSessionIRS(arq_session.ARQSession):
         self.event_manager.send_arq_session_new(False, self.id, self.dxcall, self.total_length, self.state.name)
 
         info_ack = self.frame_factory.build_arq_session_info_ack(
-            self.id, self.total_crc, self.snr[0],
+            self.id, self.total_crc, self.snr,
             self.speed_level, self.frames_per_burst, flag_abort=self.abort)
         self.launch_transmit_and_wait(info_ack, self.TIMEOUT_CONNECT, mode=FREEDV_MODE.signalling)
         if not self.abort:
@@ -150,13 +150,15 @@ class ARQSessionIRS(arq_session.ARQSession):
         self.received_bytes += len(data_part)
         self.log(f"Received {self.received_bytes}/{self.total_length} bytes")
         self.event_manager.send_arq_session_progress(
-            False, self.id, self.dxcall, self.received_bytes, self.total_length, self.state.name)
+            False, self.id, self.dxcall, self.received_bytes, self.total_length, self.state.name, self.calculate_session_statistics())
 
         return True
 
     def receive_data(self, burst_frame):
         self.process_incoming_data(burst_frame)
-
+        # update statistics
+        self.update_histograms()
+        
         if not self.all_data_received():
             self.calibrate_speed_settings(burst_frame=burst_frame)
             ack = self.frame_factory.build_arq_burst_ack(
@@ -164,7 +166,7 @@ class ARQSessionIRS(arq_session.ARQSession):
                 self.received_bytes,
                 self.speed_level,
                 self.frames_per_burst,
-                self.snr[0],
+                self.snr,
                 flag_abort=self.abort
             )
 
@@ -178,7 +180,7 @@ class ARQSessionIRS(arq_session.ARQSession):
                                                          self.received_bytes,
                                                          self.speed_level,
                                                          self.frames_per_burst,
-                                                         self.snr[0],
+                                                         self.snr,
                                                          flag_final=True,
                                                          flag_checksum=True)
             self.transmit_frame(ack, mode=FREEDV_MODE.signalling)
@@ -195,7 +197,7 @@ class ARQSessionIRS(arq_session.ARQSession):
                                                          self.received_bytes,
                                                          self.speed_level,
                                                          self.frames_per_burst,
-                                                         self.snr[0],
+                                                         self.snr,
                                                          flag_final=True,
                                                          flag_checksum=False)
             self.transmit_frame(ack, mode=FREEDV_MODE.signalling)
@@ -208,7 +210,7 @@ class ARQSessionIRS(arq_session.ARQSession):
         else:
             received_speed_level = 0
 
-        latest_snr = self.snr[-1] if self.snr else -10
+        latest_snr = self.snr if self.snr else -10
         appropriate_speed_level = self.get_appropriate_speed_level(latest_snr)
         modes_to_decode = {}
 
diff --git a/modem/arq_session_iss.py b/modem/arq_session_iss.py
index 61442ea2..eab75ae4 100644
--- a/modem/arq_session_iss.py
+++ b/modem/arq_session_iss.py
@@ -141,7 +141,7 @@ class ARQSessionISS(arq_session.ARQSession):
 
         info_frame = self.frame_factory.build_arq_session_info(self.id, self.total_length,
                                                                helpers.get_crc_32(self.data), 
-                                                               self.snr[0], self.type_byte)
+                                                               self.snr, self.type_byte)
 
         self.launch_twr(info_frame, self.TIMEOUT_CONNECT_ACK, self.RETRIES_CONNECT, mode=FREEDV_MODE.signalling)
         self.set_state(ISS_State.INFO_SENT)
@@ -149,14 +149,15 @@ class ARQSessionISS(arq_session.ARQSession):
         return None, None
 
     def send_data(self, irs_frame):
+        # update statistics
+        self.update_histograms()
 
         self.update_speed_level(irs_frame)
-
         if 'offset' in irs_frame:
             self.confirmed_bytes = irs_frame['offset']
             self.log(f"IRS confirmed {self.confirmed_bytes}/{self.total_length} bytes")
             self.event_manager.send_arq_session_progress(
-                True, self.id, self.dxcall, self.confirmed_bytes, self.total_length, self.state.name)
+                True, self.id, self.dxcall, self.confirmed_bytes, self.total_length, self.state.name, statistics=self.calculate_session_statistics())
 
         # check if we received an abort flag
         if irs_frame["flag"]["ABORT"]:
@@ -190,9 +191,9 @@ class ARQSessionISS(arq_session.ARQSession):
         self.set_state(ISS_State.ENDED)
         self.log(f"All data transfered! flag_final={irs_frame['flag']['FINAL']}, flag_checksum={irs_frame['flag']['CHECKSUM']}")
         self.event_manager.send_arq_session_finished(True, self.id, self.dxcall,True, self.state.name, statistics=self.calculate_session_statistics())
+        self.arq_data_type_handler.transmitted(self.type_byte, self.data, self.calculate_session_statistics())
         self.state_manager.remove_arq_iss_session(self.id)
         self.states.setARQ(False)
-        self.arq_data_type_handler.transmitted(self.type_byte, self.data)
         return None, None
 
     def transmission_failed(self, irs_frame=None):
diff --git a/modem/command_message_send.py b/modem/command_message_send.py
index ad6fb6cc..1923208d 100644
--- a/modem/command_message_send.py
+++ b/modem/command_message_send.py
@@ -15,7 +15,7 @@ class SendMessageCommand(TxCommand):
     def set_params_from_api(self, apiParams):
         origin = f"{self.config['STATION']['mycall']}-{self.config['STATION']['myssid']}"
         self.message = MessageP2P.from_api_params(origin, apiParams)
-        DatabaseManagerMessages(self.event_manager).add_message(self.message.to_dict(), direction='transmit', status='queued')
+        DatabaseManagerMessages(self.event_manager).add_message(self.message.to_dict(), statistics={}, direction='transmit', status='queued')
 
     def transmit(self, modem):
 
diff --git a/modem/event_manager.py b/modem/event_manager.py
index 21482ee3..ab19eff0 100644
--- a/modem/event_manager.py
+++ b/modem/event_manager.py
@@ -42,7 +42,10 @@ class EventManager:
         }
         self.broadcast(event)
 
-    def send_arq_session_progress(self, outbound: bool, session_id, dxcall, received_bytes, total_bytes, state):
+    def send_arq_session_progress(self, outbound: bool, session_id, dxcall, received_bytes, total_bytes, state, statistics=None):
+        if statistics is None:
+            statistics = {}
+
         direction = 'outbound' if outbound else 'inbound'
         event = {
                 "type": "arq",
@@ -52,6 +55,7 @@ class EventManager:
                 'received_bytes': received_bytes,
                 'total_bytes': total_bytes,
                 'state': state,
+                'statistics': statistics,
             }
         }
         self.broadcast(event)
diff --git a/modem/message_p2p.py b/modem/message_p2p.py
index fafd2c25..44fcde96 100644
--- a/modem/message_p2p.py
+++ b/modem/message_p2p.py
@@ -7,23 +7,23 @@ from message_system_db_messages import DatabaseManagerMessages
 #import command_message_send
 
 
-def message_received(event_manager, state_manager, data):
+def message_received(event_manager, state_manager, data, statistics):
     decompressed_json_string = data.decode('utf-8')
     received_message_obj = MessageP2P.from_payload(decompressed_json_string)
     received_message_dict = MessageP2P.to_dict(received_message_obj)
-    DatabaseManagerMessages(event_manager).add_message(received_message_dict, direction='receive', status='received', is_read=False)
+    DatabaseManagerMessages(event_manager).add_message(received_message_dict, statistics, direction='receive', status='received', is_read=False)
 
-def message_transmitted(event_manager, state_manager, data):
+def message_transmitted(event_manager, state_manager, data, statistics):
     decompressed_json_string = data.decode('utf-8')
     payload_message_obj = MessageP2P.from_payload(decompressed_json_string)
     payload_message = MessageP2P.to_dict(payload_message_obj)
-    DatabaseManagerMessages(event_manager).update_message(payload_message["id"], update_data={'status': 'transmitted'})
+    DatabaseManagerMessages(event_manager).update_message(payload_message["id"], update_data={'status': 'transmitted', 'statistics': statistics})
 
-def message_failed(event_manager, state_manager, data):
+def message_failed(event_manager, state_manager, data, statistics):
     decompressed_json_string = data.decode('utf-8')
     payload_message_obj = MessageP2P.from_payload(decompressed_json_string)
     payload_message = MessageP2P.to_dict(payload_message_obj)
-    DatabaseManagerMessages(event_manager).update_message(payload_message["id"], update_data={'status': 'failed'})
+    DatabaseManagerMessages(event_manager).update_message(payload_message["id"], statistics, update_data={'status': 'failed', 'statistics': statistics})
 
 class MessageP2P:
     def __init__(self, id: str, origin: str, destination: str, body: str, attachments: list) -> None:
diff --git a/modem/message_system_db_messages.py b/modem/message_system_db_messages.py
index 639ac92f..6f6739e1 100644
--- a/modem/message_system_db_messages.py
+++ b/modem/message_system_db_messages.py
@@ -11,7 +11,7 @@ class DatabaseManagerMessages(DatabaseManager):
         super().__init__(uri)
         self.attachments_manager = DatabaseManagerAttachments(uri)
 
-    def add_message(self, message_data, direction='receive', status=None, is_read=True):
+    def add_message(self, message_data, statistics, direction='receive', status=None, is_read=True):
         session = self.get_thread_scoped_session()
         try:
             # Create and add the origin and destination Stations
@@ -34,7 +34,8 @@ class DatabaseManagerMessages(DatabaseManager):
                 direction=direction,
                 status_id=status.id if status else None,
                 is_read=is_read,
-                attempt=0
+                attempt=0,
+                statistics=statistics
             )
 
             session.add(new_message)
@@ -131,6 +132,9 @@ class DatabaseManagerMessages(DatabaseManager):
                 if 'status' in update_data:
                     message.status = self.get_or_create_status(session, update_data['status'])
 
+                if 'statistics' in update_data:
+                    message.statistics = update_data['statistics']
+
                 session.commit()
                 self.log(f"Updated: {message_id}")
                 self.event_manager.freedata_message_db_change()

From e7ce198fa16ddfd5942e7719668270e9c90e3aef Mon Sep 17 00:00:00 2001
From: DJ2LS <dj2ls@proton.me>
Date: Sat, 24 Feb 2024 22:01:15 +0100
Subject: [PATCH 17/29] WIP gui stats

---
 gui/src/js/eventHandler.js  | 7 +++++++
 gui/src/store/stateStore.js | 1 +
 modem/arq_session_iss.py    | 2 +-
 3 files changed, 9 insertions(+), 1 deletion(-)

diff --git a/gui/src/js/eventHandler.js b/gui/src/js/eventHandler.js
index d91370b6..e6feb96b 100644
--- a/gui/src/js/eventHandler.js
+++ b/gui/src/js/eventHandler.js
@@ -190,6 +190,9 @@ export function eventDispatcher(data) {
               100;
             stateStore.arq_total_bytes =
               data["arq-transfer-outbound"].received_bytes;
+            stateStore.arq_speed_list_timestamp = data["arq-transfer-outbound"].statistics.time_histogram
+            stateStore.arq_speed_list_bpm = data["arq-transfer-outbound"].statistics.bpm_histogram
+            stateStore.arq_speed_list_snr = data["arq-transfer-outbound"].statistics.snr_histogram
             return;
 
           case "ABORTING":
@@ -232,6 +235,10 @@ export function eventDispatcher(data) {
             stateStore.dxcallsign = data["arq-transfer-inbound"].dxcall;
             stateStore.arq_transmission_percent = 0;
             stateStore.arq_total_bytes = 0;
+            stateStore.arq_speed_list_timestamp = data["arq-transfer-inbound"].statistics.time_histogram
+            stateStore.arq_speed_list_bpm = data["arq-transfer-inbound"].statistics.bpm_histogram
+            stateStore.arq_speed_list_snr = data["arq-transfer-inbound"].statistics.snr_histogram
+
             return;
 
           case "OPEN_ACK_SENT":
diff --git a/gui/src/store/stateStore.js b/gui/src/store/stateStore.js
index 1ed77aec..3632496a 100644
--- a/gui/src/store/stateStore.js
+++ b/gui/src/store/stateStore.js
@@ -53,6 +53,7 @@ export const useStateStore = defineStore("stateStore", () => {
   var arq_speed_list_bpm = ref([]);
   var arq_speed_list_snr = ref([]);
 
+    /* TODO Those 3 can be removed I guess , DJ2LS*/
   var arq_seconds_until_finish = ref();
   var arq_seconds_until_timeout = ref();
   var arq_seconds_until_timeout_percent = ref();
diff --git a/modem/arq_session_iss.py b/modem/arq_session_iss.py
index eab75ae4..d6992759 100644
--- a/modem/arq_session_iss.py
+++ b/modem/arq_session_iss.py
@@ -204,7 +204,7 @@ class ARQSessionISS(arq_session.ARQSession):
         self.event_manager.send_arq_session_finished(True, self.id, self.dxcall,False, self.state.name, statistics=self.calculate_session_statistics())
         self.states.setARQ(False)
 
-        self.arq_data_type_handler.failed(self.type_byte, self.data)
+        self.arq_data_type_handler.failed(self.type_byte, self.data, self.calculate_session_statistics())
         return None, None
 
     def abort_transmission(self, irs_frame=None):

From 0c322bacf8791a9d3964f42a169b01bc321c1b6d Mon Sep 17 00:00:00 2001
From: DJ2LS <dj2ls@proton.me>
Date: Sat, 24 Feb 2024 22:10:41 +0100
Subject: [PATCH 18/29] WIP gui stats

---
 modem/message_p2p.py                | 9 +++++++--
 modem/message_system_db_messages.py | 1 -
 2 files changed, 7 insertions(+), 3 deletions(-)

diff --git a/modem/message_p2p.py b/modem/message_p2p.py
index 44fcde96..d0ee6c23 100644
--- a/modem/message_p2p.py
+++ b/modem/message_p2p.py
@@ -17,13 +17,18 @@ def message_transmitted(event_manager, state_manager, data, statistics):
     decompressed_json_string = data.decode('utf-8')
     payload_message_obj = MessageP2P.from_payload(decompressed_json_string)
     payload_message = MessageP2P.to_dict(payload_message_obj)
-    DatabaseManagerMessages(event_manager).update_message(payload_message["id"], update_data={'status': 'transmitted', 'statistics': statistics})
+    # Todo we need to optimize this - WIP
+    DatabaseManagerMessages(event_manager).update_message(payload_message["id"], update_data={'status': 'transmitted'})
+    DatabaseManagerMessages(event_manager).update_message(payload_message["id"], update_data={'statistics': statistics})
+
 
 def message_failed(event_manager, state_manager, data, statistics):
     decompressed_json_string = data.decode('utf-8')
     payload_message_obj = MessageP2P.from_payload(decompressed_json_string)
     payload_message = MessageP2P.to_dict(payload_message_obj)
-    DatabaseManagerMessages(event_manager).update_message(payload_message["id"], statistics, update_data={'status': 'failed', 'statistics': statistics})
+    # Todo we need to optimize this - WIP
+    DatabaseManagerMessages(event_manager).update_message(payload_message["id"], update_data={'status': 'failed'})
+    DatabaseManagerMessages(event_manager).update_message(payload_message["id"], update_data={'statistics': statistics})
 
 class MessageP2P:
     def __init__(self, id: str, origin: str, destination: str, body: str, attachments: list) -> None:
diff --git a/modem/message_system_db_messages.py b/modem/message_system_db_messages.py
index 6f6739e1..4a1eb312 100644
--- a/modem/message_system_db_messages.py
+++ b/modem/message_system_db_messages.py
@@ -131,7 +131,6 @@ class DatabaseManagerMessages(DatabaseManager):
                     message.body = update_data['body']
                 if 'status' in update_data:
                     message.status = self.get_or_create_status(session, update_data['status'])
-
                 if 'statistics' in update_data:
                     message.statistics = update_data['statistics']
 

From 59778165bfa5f0f3a4e75348ed3ca8832916fef3 Mon Sep 17 00:00:00 2001
From: DJ2LS <dj2ls@proton.me>
Date: Sat, 24 Feb 2024 22:27:01 +0100
Subject: [PATCH 19/29] fixed some tests

---
 tests/test_data_frame_factory.py |  6 +++---
 tests/test_data_type_handler.py  |  6 +++---
 tests/test_message_database.py   | 12 ++++++------
 tests/test_message_p2p.py        |  2 +-
 4 files changed, 13 insertions(+), 13 deletions(-)

diff --git a/tests/test_data_frame_factory.py b/tests/test_data_frame_factory.py
index 6cb4a974..5f505662 100755
--- a/tests/test_data_frame_factory.py
+++ b/tests/test_data_frame_factory.py
@@ -49,7 +49,7 @@ class TestDataFrameFactory(unittest.TestCase):
         offset = 40
         payload = b'Hello World!'
         frame = self.factory.build_arq_burst_frame(FREEDV_MODE.datac3, 
-                                                session_id, offset, payload)
+                                                session_id, offset, payload, 0)
         frame_data = self.factory.deconstruct(frame)
         self.assertEqual(frame_data['session_id'], session_id)
         self.assertEqual(frame_data['offset'], offset)
@@ -58,11 +58,11 @@ class TestDataFrameFactory(unittest.TestCase):
 
         payload = payload * 1000
         self.assertRaises(OverflowError, self.factory.build_arq_burst_frame,
-            FREEDV_MODE.datac3, session_id, offset, payload)
+            FREEDV_MODE.datac3, session_id, offset, payload, 0)
         
     def testAvailablePayload(self):
         avail = self.factory.get_available_data_payload_for_mode(FRAME_TYPE.ARQ_BURST_FRAME, FREEDV_MODE.datac3)
-        self.assertEqual(avail, 120) # 128 bytes datac3 frame payload - BURST frame overhead
+        self.assertEqual(avail, 119) # 128 bytes datac3 frame payload - BURST frame overhead
 
 if __name__ == '__main__':
     unittest.main()
diff --git a/tests/test_data_type_handler.py b/tests/test_data_type_handler.py
index 773c11a9..7fafb58c 100644
--- a/tests/test_data_type_handler.py
+++ b/tests/test_data_type_handler.py
@@ -22,21 +22,21 @@ class TestDispatcher(unittest.TestCase):
         # Example usage
         example_data = b"Hello FreeDATA!"
         formatted_data, type_byte = self.arq_data_type_handler.prepare(example_data, ARQ_SESSION_TYPES.raw)
-        dispatched_data = self.arq_data_type_handler.dispatch(type_byte, formatted_data)
+        dispatched_data = self.arq_data_type_handler.dispatch(type_byte, formatted_data, statistics={})
         self.assertEqual(example_data, dispatched_data)
 
     def testDataTypeHandlerLZMA(self):
         # Example usage
         example_data = b"Hello FreeDATA!"
         formatted_data, type_byte = self.arq_data_type_handler.prepare(example_data, ARQ_SESSION_TYPES.raw_lzma)
-        dispatched_data = self.arq_data_type_handler.dispatch(type_byte, formatted_data)
+        dispatched_data = self.arq_data_type_handler.dispatch(type_byte, formatted_data, statistics={})
         self.assertEqual(example_data, dispatched_data)
 
     def testDataTypeHandlerGZIP(self):
         # Example usage
         example_data = b"Hello FreeDATA!"
         formatted_data, type_byte = self.arq_data_type_handler.prepare(example_data, ARQ_SESSION_TYPES.raw_gzip)
-        dispatched_data = self.arq_data_type_handler.dispatch(type_byte, formatted_data)
+        dispatched_data = self.arq_data_type_handler.dispatch(type_byte, formatted_data, statistics={})
         self.assertEqual(example_data, dispatched_data)
 
 
diff --git a/tests/test_message_database.py b/tests/test_message_database.py
index 6a20cae1..086bb85c 100644
--- a/tests/test_message_database.py
+++ b/tests/test_message_database.py
@@ -37,7 +37,7 @@ class TestDataFrameFactory(unittest.TestCase):
         payload = message.to_payload()
         received_message = MessageP2P.from_payload(payload)
         received_message_dict = MessageP2P.to_dict(received_message)
-        self.database_manager.add_message(received_message_dict)
+        self.database_manager.add_message(received_message_dict, statistics={})
         result = self.database_manager.get_message_by_id(message.id)
 
         self.assertEqual(result["destination"], message.destination)
@@ -53,7 +53,7 @@ class TestDataFrameFactory(unittest.TestCase):
         payload = message.to_payload()
         received_message = MessageP2P.from_payload(payload)
         received_message_dict = MessageP2P.to_dict(received_message)
-        self.database_manager.add_message(received_message_dict)
+        self.database_manager.add_message(received_message_dict, statistics={})
 
         result = self.database_manager.get_all_messages()
         message_id = result[0]["id"]
@@ -75,7 +75,7 @@ class TestDataFrameFactory(unittest.TestCase):
         received_message = MessageP2P.from_payload(payload)
         received_message_dict = MessageP2P.to_dict(received_message)
         print(received_message_dict)
-        message_id = self.database_manager.add_message(received_message_dict, direction='receive')
+        message_id = self.database_manager.add_message(received_message_dict, statistics={}, direction='receive')
         print(message_id)
         self.database_manager.update_message(message_id, {'body' : 'hello123'})
 
@@ -103,7 +103,7 @@ class TestDataFrameFactory(unittest.TestCase):
         payload = message.to_payload()
         received_message = MessageP2P.from_payload(payload)
         received_message_dict = MessageP2P.to_dict(received_message)
-        message_id = self.database_manager.add_message(received_message_dict)
+        message_id = self.database_manager.add_message(received_message_dict, statistics={})
         result = self.database_manager_attachments.get_attachments_by_message_id(message_id)
         attachment_names = [attachment['name'] for attachment in result]
         self.assertIn('test1.gif', attachment_names)
@@ -116,7 +116,7 @@ class TestDataFrameFactory(unittest.TestCase):
         payload = message.to_payload()
         received_message = MessageP2P.from_payload(payload)
         received_message_dict = MessageP2P.to_dict(received_message)
-        message_id = self.database_manager.add_message(received_message_dict)
+        message_id = self.database_manager.add_message(received_message_dict,statistics={},)
         self.database_manager.increment_message_attempts(message_id)
 
 
@@ -129,7 +129,7 @@ class TestDataFrameFactory(unittest.TestCase):
         payload = message.to_payload()
         received_message = MessageP2P.from_payload(payload)
         received_message_dict = MessageP2P.to_dict(received_message)
-        message_id = self.database_manager.add_message(received_message_dict, is_read=False)
+        message_id = self.database_manager.add_message(received_message_dict, statistics={},is_read=False)
         self.database_manager.mark_message_as_read(message_id)
 
         result = self.database_manager.get_message_by_id(message_id)
diff --git a/tests/test_message_p2p.py b/tests/test_message_p2p.py
index 93782235..5ea115af 100755
--- a/tests/test_message_p2p.py
+++ b/tests/test_message_p2p.py
@@ -60,7 +60,7 @@ class TestDataFrameFactory(unittest.TestCase):
         payload = message.to_payload()
         received_message = MessageP2P.from_payload(payload)
         received_message_dict = MessageP2P.to_dict(received_message)
-        self.database_manager.add_message(received_message_dict)
+        self.database_manager.add_message(received_message_dict, statistics={})
 
         self.assertEqual(message.origin, received_message.origin)
         self.assertEqual(message.destination, received_message.destination)

From 419f7732dfae5322ee2caf392f59e95624cc0e3c Mon Sep 17 00:00:00 2001
From: DJ2LS <dj2ls@proton.me>
Date: Sun, 25 Feb 2024 20:44:05 +0100
Subject: [PATCH 20/29] more try/except for schedule manager

---
 modem/schedule_manager.py | 19 ++++++++++++++-----
 1 file changed, 14 insertions(+), 5 deletions(-)

diff --git a/modem/schedule_manager.py b/modem/schedule_manager.py
index f238b911..5b8abbc9 100644
--- a/modem/schedule_manager.py
+++ b/modem/schedule_manager.py
@@ -65,17 +65,26 @@ class ScheduleManager:
             self.scheduler_thread.join()
 
     def transmit_beacon(self):
-        if not self.state_manager.getARQ() and self.state_manager.is_beacon_running:
-                cmd = command_beacon.BeaconCommand(self.config, self.state_manager, self.event_manager)
-                cmd.run(self.event_manager, self.modem)
+        try:
+            if not self.state_manager.getARQ() and self.state_manager.is_beacon_running:
+                    cmd = command_beacon.BeaconCommand(self.config, self.state_manager, self.event_manager)
+                    cmd.run(self.event_manager, self.modem)
+        except Exception as e:
+            print(e)
 
     def delete_beacons(self):
-        DatabaseManagerBeacon(self.event_manager).beacon_cleanup_older_than_days(2)
+        try:
+            DatabaseManagerBeacon(self.event_manager).beacon_cleanup_older_than_days(2)
+        except Exception as e:
+            print(e)
 
     def push_to_explorer(self):
         self.config = self.config_manager.read()
         if self.config['STATION']['enable_explorer']:
-            explorer.explorer(self.modem_version, self.config_manager, self.state_manager).push()
+            try:
+                explorer.explorer(self.modem_version, self.config_manager, self.state_manager).push()
+            except Exception as e:
+                print(e)
 
     def check_for_queued_messages(self):
         if not self.state_manager.getARQ():

From 4a34386c26fc427a1a17f4f0029fa95654bbea32 Mon Sep 17 00:00:00 2001
From: DJ2LS <dj2ls@proton.me>
Date: Sun, 25 Feb 2024 21:08:45 +0100
Subject: [PATCH 21/29] small adjustments to api validation, hopefully catching
 mygrid 4/6 error

---
 modem/api_validations.py | 11 +++++++++++
 modem/server.py          |  3 +++
 2 files changed, 14 insertions(+)

diff --git a/modem/api_validations.py b/modem/api_validations.py
index b8257d9c..a5cbc70e 100644
--- a/modem/api_validations.py
+++ b/modem/api_validations.py
@@ -1,5 +1,16 @@
 import re
 
+
+def validate_remote_config(config):
+    if not config:
+        return
+
+    mygrid = config["STATION"]["mygrid"]
+    if len(mygrid) != 6:
+        raise ValueError(f"Gridsquare must be 6 characters!")
+
+    return True
+
 def validate_freedata_callsign(callsign):
     #regexp = "^[a-zA-Z]+\d+\w+-\d{1,2}$"
     regexp = "^[A-Za-z0-9]{1,7}-[0-9]{1,3}$" # still broken - we need to allow all ssids form 0 - 255
diff --git a/modem/server.py b/modem/server.py
index f425b37f..8b7acd3e 100644
--- a/modem/server.py
+++ b/modem/server.py
@@ -96,6 +96,9 @@ def index():
 @app.route('/config', methods=['GET', 'POST'])
 def config():
     if request.method in ['POST']:
+
+        if not validations.validate_remote_config(request.json):
+            return api_abort("wrong config", 500)
         # check if config already exists
         if app.config_manager.read() == request.json:
             return api_response(request.json)

From 97062609333f5a6dd2cc566c23bc2a6ca3ccb9b7 Mon Sep 17 00:00:00 2001
From: DJ2LS <dj2ls@proton.me>
Date: Sun, 25 Feb 2024 21:23:15 +0100
Subject: [PATCH 22/29] adjust speed level with info ack

---
 modem/arq_session_irs.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/modem/arq_session_irs.py b/modem/arq_session_irs.py
index 74ad0b4a..8e431585 100644
--- a/modem/arq_session_irs.py
+++ b/modem/arq_session_irs.py
@@ -119,6 +119,8 @@ class ARQSessionIRS(arq_session.ARQSession):
         self.dx_snr.append(info_frame['snr'])
         self.type_byte = info_frame['type']
 
+        self.calibrate_speed_settings()
+
         self.log(f"New transfer of {self.total_length} bytes")
         self.event_manager.send_arq_session_new(False, self.id, self.dxcall, self.total_length, self.state.name)
 
@@ -128,7 +130,6 @@ class ARQSessionIRS(arq_session.ARQSession):
         self.launch_transmit_and_wait(info_ack, self.TIMEOUT_CONNECT, mode=FREEDV_MODE.signalling)
         if not self.abort:
             self.set_state(IRS_State.INFO_ACK_SENT)
-            self.calibrate_speed_settings()
         return None, None
 
     def process_incoming_data(self, frame):

From fd9fb81fa297d9ddee57b67cac6db83631fa641a Mon Sep 17 00:00:00 2001
From: DJ2LS <dj2ls@proton.me>
Date: Tue, 27 Feb 2024 22:39:45 +0100
Subject: [PATCH 23/29] adjusted stats

---
 modem/arq_session.py     | 13 +++++++------
 modem/arq_session_irs.py | 10 +++++-----
 modem/arq_session_iss.py | 16 ++++++++--------
 3 files changed, 20 insertions(+), 19 deletions(-)

diff --git a/modem/arq_session.py b/modem/arq_session.py
index 1e750331..67992fc7 100644
--- a/modem/arq_session.py
+++ b/modem/arq_session.py
@@ -104,7 +104,7 @@ class ARQSession():
                 action_name = self.STATE_TRANSITION[self.state][frame_type]
                 received_data, type_byte = getattr(self, action_name)(frame)
                 if isinstance(received_data, bytearray) and isinstance(type_byte, int):
-                    self.arq_data_type_handler.dispatch(type_byte, received_data, self.calculate_session_statistics())
+                    self.arq_data_type_handler.dispatch(type_byte, received_data, self.update_histograms(len(received_data), len(received_data)))
                 return
         
         self.log(f"Ignoring unknown transition from state {self.state.name} with frame {frame['frame_type']}")
@@ -121,15 +121,15 @@ class ARQSession():
 
         return self.session_ended - self.session_started
 
-    def calculate_session_statistics(self):
+    def calculate_session_statistics(self, confirmed_bytes, total_bytes):
         duration = self.calculate_session_duration()
-        total_bytes = self.total_length
+        #total_bytes = self.total_length
         # self.total_length
         duration_in_minutes = duration / 60  # Convert duration from seconds to minutes
 
         # Calculate bytes per minute
         if duration_in_minutes > 0:
-            bytes_per_minute = int(total_bytes / duration_in_minutes)
+            bytes_per_minute = int(confirmed_bytes / duration_in_minutes)
         else:
             bytes_per_minute = 0
 
@@ -147,11 +147,12 @@ class ARQSession():
             'bpm_histogram': bpm_histogram_dict,
         }
 
-    def update_histograms(self):
-        stats = self.calculate_session_statistics()
+    def update_histograms(self, confirmed_bytes, total_bytes):
+        stats = self.calculate_session_statistics(confirmed_bytes, total_bytes)
         self.snr_histogram.append(self.snr)
         self.bpm_histogram.append(stats['bytes_per_minute'])
         self.time_histogram.append(datetime.datetime.now().isoformat())
+        return stats
 
     def get_appropriate_speed_level(self, snr):
         # Start with the lowest speed level as default
diff --git a/modem/arq_session_irs.py b/modem/arq_session_irs.py
index 8e431585..fa6409fb 100644
--- a/modem/arq_session_irs.py
+++ b/modem/arq_session_irs.py
@@ -151,14 +151,14 @@ class ARQSessionIRS(arq_session.ARQSession):
         self.received_bytes += len(data_part)
         self.log(f"Received {self.received_bytes}/{self.total_length} bytes")
         self.event_manager.send_arq_session_progress(
-            False, self.id, self.dxcall, self.received_bytes, self.total_length, self.state.name, self.calculate_session_statistics())
+            False, self.id, self.dxcall, self.received_bytes, self.total_length, self.state.name, self.calculate_session_statistics(self.received_bytes, self.total_length))
 
         return True
 
     def receive_data(self, burst_frame):
         self.process_incoming_data(burst_frame)
         # update statistics
-        self.update_histograms()
+        self.update_histograms(self.received_bytes, self.total_length)
         
         if not self.all_data_received():
             self.calibrate_speed_settings(burst_frame=burst_frame)
@@ -189,7 +189,7 @@ class ARQSessionIRS(arq_session.ARQSession):
             self.session_ended = time.time()
             self.set_state(IRS_State.ENDED)
             self.event_manager.send_arq_session_finished(
-                False, self.id, self.dxcall, True, self.state.name, data=self.received_data, statistics=self.calculate_session_statistics())
+                False, self.id, self.dxcall, True, self.state.name, data=self.received_data, statistics=self.calculate_session_statistics(self.received_bytes, self.total_length))
 
             return self.received_data, self.type_byte
         else:
@@ -256,7 +256,7 @@ class ARQSessionIRS(arq_session.ARQSession):
         self.set_state(IRS_State.ABORTED)
         self.states.setARQ(False)
         self.event_manager.send_arq_session_finished(
-                False, self.id, self.dxcall, False, self.state.name, statistics=self.calculate_session_statistics())
+                False, self.id, self.dxcall, False, self.state.name, statistics=self.calculate_session_statistics(self.received_bytes, self.total_length))
         return None, None
 
     def transmission_failed(self, irs_frame=None):
@@ -264,6 +264,6 @@ class ARQSessionIRS(arq_session.ARQSession):
         self.session_ended = time.time()
         self.set_state(IRS_State.FAILED)
         self.log(f"Transmission failed!")
-        self.event_manager.send_arq_session_finished(True, self.id, self.dxcall,False, self.state.name, statistics=self.calculate_session_statistics())
+        self.event_manager.send_arq_session_finished(True, self.id, self.dxcall,False, self.state.name, statistics=self.calculate_session_statistics(self.received_bytes, self.total_length))
         self.states.setARQ(False)
         return None, None
diff --git a/modem/arq_session_iss.py b/modem/arq_session_iss.py
index d6992759..611f591e 100644
--- a/modem/arq_session_iss.py
+++ b/modem/arq_session_iss.py
@@ -150,14 +150,14 @@ class ARQSessionISS(arq_session.ARQSession):
 
     def send_data(self, irs_frame):
         # update statistics
-        self.update_histograms()
+        self.update_histograms(self.confirmed_bytes, self.total_length)
 
         self.update_speed_level(irs_frame)
         if 'offset' in irs_frame:
             self.confirmed_bytes = irs_frame['offset']
             self.log(f"IRS confirmed {self.confirmed_bytes}/{self.total_length} bytes")
             self.event_manager.send_arq_session_progress(
-                True, self.id, self.dxcall, self.confirmed_bytes, self.total_length, self.state.name, statistics=self.calculate_session_statistics())
+                True, self.id, self.dxcall, self.confirmed_bytes, self.total_length, self.state.name, statistics=self.calculate_session_statistics(self.confirmed_bytes, self.total_length))
 
         # check if we received an abort flag
         if irs_frame["flag"]["ABORT"]:
@@ -190,8 +190,8 @@ class ARQSessionISS(arq_session.ARQSession):
         self.session_ended = time.time()
         self.set_state(ISS_State.ENDED)
         self.log(f"All data transfered! flag_final={irs_frame['flag']['FINAL']}, flag_checksum={irs_frame['flag']['CHECKSUM']}")
-        self.event_manager.send_arq_session_finished(True, self.id, self.dxcall,True, self.state.name, statistics=self.calculate_session_statistics())
-        self.arq_data_type_handler.transmitted(self.type_byte, self.data, self.calculate_session_statistics())
+        self.event_manager.send_arq_session_finished(True, self.id, self.dxcall,True, self.state.name, statistics=self.calculate_session_statistics(self.confirmed_bytes, self.total_length))
+        self.arq_data_type_handler.transmitted(self.type_byte, self.data, self.calculate_session_statistics(self.confirmed_bytes, self.total_length))
         self.state_manager.remove_arq_iss_session(self.id)
         self.states.setARQ(False)
         return None, None
@@ -201,10 +201,10 @@ class ARQSessionISS(arq_session.ARQSession):
         self.session_ended = time.time()
         self.set_state(ISS_State.FAILED)
         self.log(f"Transmission failed!")
-        self.event_manager.send_arq_session_finished(True, self.id, self.dxcall,False, self.state.name, statistics=self.calculate_session_statistics())
+        self.event_manager.send_arq_session_finished(True, self.id, self.dxcall,False, self.state.name, statistics=self.calculate_session_statistics(self.confirmed_bytes, self.total_length))
         self.states.setARQ(False)
 
-        self.arq_data_type_handler.failed(self.type_byte, self.data, self.calculate_session_statistics())
+        self.arq_data_type_handler.failed(self.type_byte, self.data, self.calculate_session_statistics(self.confirmed_bytes, self.total_length))
         return None, None
 
     def abort_transmission(self, irs_frame=None):
@@ -213,7 +213,7 @@ class ARQSessionISS(arq_session.ARQSession):
         self.set_state(ISS_State.ABORTING)
 
         self.event_manager.send_arq_session_finished(
-            True, self.id, self.dxcall, False, self.state.name, statistics=self.calculate_session_statistics())
+            True, self.id, self.dxcall, False, self.state.name, statistics=self.calculate_session_statistics(self.confirmed_bytes, self.total_length))
 
         # break actual retries
         self.event_frame_received.set()
@@ -233,7 +233,7 @@ class ARQSessionISS(arq_session.ARQSession):
         self.event_frame_received.set()
 
         self.event_manager.send_arq_session_finished(
-            True, self.id, self.dxcall, False, self.state.name, statistics=self.calculate_session_statistics())
+            True, self.id, self.dxcall, False, self.state.name, statistics=self.calculate_session_statistics(self.confirmed_bytes, self.total_length))
         self.state_manager.remove_arq_iss_session(self.id)
         self.states.setARQ(False)
         return None, None

From 2f07441fc6ce4158076345ec9c1a52d058934812 Mon Sep 17 00:00:00 2001
From: codefactor-io <support@codefactor.io>
Date: Wed, 28 Feb 2024 08:27:09 +0000
Subject: [PATCH 24/29] [CodeFactor] Apply fixes

---
 gui/src/js/eventHandler.js  | 18 ++++++++++++------
 gui/src/store/stateStore.js |  2 +-
 2 files changed, 13 insertions(+), 7 deletions(-)

diff --git a/gui/src/js/eventHandler.js b/gui/src/js/eventHandler.js
index e6feb96b..f1e5921d 100644
--- a/gui/src/js/eventHandler.js
+++ b/gui/src/js/eventHandler.js
@@ -190,9 +190,12 @@ export function eventDispatcher(data) {
               100;
             stateStore.arq_total_bytes =
               data["arq-transfer-outbound"].received_bytes;
-            stateStore.arq_speed_list_timestamp = data["arq-transfer-outbound"].statistics.time_histogram
-            stateStore.arq_speed_list_bpm = data["arq-transfer-outbound"].statistics.bpm_histogram
-            stateStore.arq_speed_list_snr = data["arq-transfer-outbound"].statistics.snr_histogram
+            stateStore.arq_speed_list_timestamp =
+              data["arq-transfer-outbound"].statistics.time_histogram;
+            stateStore.arq_speed_list_bpm =
+              data["arq-transfer-outbound"].statistics.bpm_histogram;
+            stateStore.arq_speed_list_snr =
+              data["arq-transfer-outbound"].statistics.snr_histogram;
             return;
 
           case "ABORTING":
@@ -235,9 +238,12 @@ export function eventDispatcher(data) {
             stateStore.dxcallsign = data["arq-transfer-inbound"].dxcall;
             stateStore.arq_transmission_percent = 0;
             stateStore.arq_total_bytes = 0;
-            stateStore.arq_speed_list_timestamp = data["arq-transfer-inbound"].statistics.time_histogram
-            stateStore.arq_speed_list_bpm = data["arq-transfer-inbound"].statistics.bpm_histogram
-            stateStore.arq_speed_list_snr = data["arq-transfer-inbound"].statistics.snr_histogram
+            stateStore.arq_speed_list_timestamp =
+              data["arq-transfer-inbound"].statistics.time_histogram;
+            stateStore.arq_speed_list_bpm =
+              data["arq-transfer-inbound"].statistics.bpm_histogram;
+            stateStore.arq_speed_list_snr =
+              data["arq-transfer-inbound"].statistics.snr_histogram;
 
             return;
 
diff --git a/gui/src/store/stateStore.js b/gui/src/store/stateStore.js
index 3632496a..e65a331c 100644
--- a/gui/src/store/stateStore.js
+++ b/gui/src/store/stateStore.js
@@ -53,7 +53,7 @@ export const useStateStore = defineStore("stateStore", () => {
   var arq_speed_list_bpm = ref([]);
   var arq_speed_list_snr = ref([]);
 
-    /* TODO Those 3 can be removed I guess , DJ2LS*/
+  /* TODO Those 3 can be removed I guess , DJ2LS*/
   var arq_seconds_until_finish = ref();
   var arq_seconds_until_timeout = ref();
   var arq_seconds_until_timeout_percent = ref();

From 25de30097026e8480f10b6410286f79e8758f80c Mon Sep 17 00:00:00 2001
From: DJ2LS <dj2ls@proton.me>
Date: Wed, 28 Feb 2024 09:56:28 +0100
Subject: [PATCH 25/29] fixed test

---
 tests/test_server.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tests/test_server.py b/tests/test_server.py
index 18ee7475..2adb132a 100755
--- a/tests/test_server.py
+++ b/tests/test_server.py
@@ -48,7 +48,7 @@ class TestIntegration(unittest.TestCase):
         self.assertIn('RADIO', config)
 
     def test_config_post(self):
-        config = {'NETWORK': {'modemport' : 3050}}
+        config = {'STATION': {'mygrid' : 'JN48ea'}}
         r = requests.post(self.url + '/config', 
                           headers={'Content-type': 'application/json'},
                           data = json.dumps(config))

From bdaed0e873783c2b8142af04f6172aa9a1dc2880 Mon Sep 17 00:00:00 2001
From: DJ2LS <dj2ls@proton.me>
Date: Wed, 28 Feb 2024 19:41:06 +0100
Subject: [PATCH 26/29] fixed test

---
 tests/test_server.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tests/test_server.py b/tests/test_server.py
index 2adb132a..c4a0e47a 100755
--- a/tests/test_server.py
+++ b/tests/test_server.py
@@ -57,7 +57,7 @@ class TestIntegration(unittest.TestCase):
         r = requests.get(self.url + '/config')
         self.assertEqual(r.status_code, 200)
         config = r.json()
-        self.assertEqual(config['NETWORK']['modemport'], 3050)
+        self.assertEqual(config['NETWORK']['modemport'], 5000)
 
 if __name__ == '__main__':
     unittest.main()

From 05a274edeb5705e327584835858f9b164c3faa2b Mon Sep 17 00:00:00 2001
From: DJ2LS <dj2ls@proton.me>
Date: Wed, 28 Feb 2024 19:44:50 +0100
Subject: [PATCH 27/29] attempt fixing macos server build

---
 .github/workflows/build_server.yml | 1 +
 1 file changed, 1 insertion(+)

diff --git a/.github/workflows/build_server.yml b/.github/workflows/build_server.yml
index a9472a3d..efa1f73e 100644
--- a/.github/workflows/build_server.yml
+++ b/.github/workflows/build_server.yml
@@ -48,6 +48,7 @@ jobs:
           brew install portaudio
           python -m pip install --upgrade pip
           pip3 install pyaudio
+          export PYTHONPATH=/opt/homebrew/opt/portaudio/lib/:$PYTHONPATH
 
       - name: Install Python dependencies
         run: |

From c8e6e11d84be228398fb6df3f0a53357f41912e4 Mon Sep 17 00:00:00 2001
From: DJ2LS <dj2ls@proton.me>
Date: Wed, 28 Feb 2024 20:01:26 +0100
Subject: [PATCH 28/29] removed ujson dependency

---
 modem/config.ini.example | 2 +-
 modem/explorer.py        | 2 +-
 modem/server.py          | 2 +-
 modem/state_manager.py   | 1 -
 modem/stats.py           | 2 +-
 requirements.txt         | 1 -
 6 files changed, 4 insertions(+), 6 deletions(-)

diff --git a/modem/config.ini.example b/modem/config.ini.example
index b82b5ab5..9897b0ab 100644
--- a/modem/config.ini.example
+++ b/modem/config.ini.example
@@ -3,7 +3,7 @@ modemport = 5000
 
 [STATION]
 mycall = AA1AAA
-mygrid = AA12aa
+mygrid = JN48ea
 myssid = 1
 ssid_list = [0, 1, 2, 3, 4, 5, 6, 7, 8, 9]
 enable_explorer = True
diff --git a/modem/explorer.py b/modem/explorer.py
index d7c4ccc6..994b76ea 100644
--- a/modem/explorer.py
+++ b/modem/explorer.py
@@ -9,7 +9,7 @@ Created on 05.11.23
 
 import requests
 import threading
-import ujson as json
+import json
 import structlog
 import sched
 import time
diff --git a/modem/server.py b/modem/server.py
index 8b7acd3e..6a845da8 100644
--- a/modem/server.py
+++ b/modem/server.py
@@ -9,7 +9,7 @@ import audio
 import queue
 import service_manager
 import state_manager
-import ujson as json
+import json
 import websocket_manager as wsm
 import api_validations as validations
 import command_cq
diff --git a/modem/state_manager.py b/modem/state_manager.py
index 9cd93694..69b6b44a 100644
--- a/modem/state_manager.py
+++ b/modem/state_manager.py
@@ -1,5 +1,4 @@
 import time
-import ujson as json
 import threading
 import numpy as np
 class StateManager:
diff --git a/modem/stats.py b/modem/stats.py
index d47e826c..50e92d57 100644
--- a/modem/stats.py
+++ b/modem/stats.py
@@ -8,7 +8,7 @@ Created on 05.11.23
 # pylint: disable=import-outside-toplevel, attribute-defined-outside-init
 
 import requests
-import ujson as json
+import json
 import structlog
 
 log = structlog.get_logger("stats")
diff --git a/requirements.txt b/requirements.txt
index 18cee53c..ab2f827d 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -5,7 +5,6 @@ PyAudio
 pyserial
 sounddevice
 structlog
-ujson
 requests
 chardet
 colorama

From 2b21aab26b74390b6cbe6aa8791caf0a13dcd843 Mon Sep 17 00:00:00 2001
From: DJ2LS <dj2ls@proton.me>
Date: Wed, 28 Feb 2024 21:18:43 +0100
Subject: [PATCH 29/29] bump version

---
 gui/package.json | 2 +-
 modem/server.py  | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/gui/package.json b/gui/package.json
index e78bcf83..0a4216be 100644
--- a/gui/package.json
+++ b/gui/package.json
@@ -2,7 +2,7 @@
   "name": "FreeDATA",
   "description": "FreeDATA Client application for connecting to FreeDATA server",
   "private": true,
-  "version": "0.13.7-alpha",
+  "version": "0.14.0-alpha",
   "main": "dist-electron/main/index.js",
   "scripts": {
     "start": "vite",
diff --git a/modem/server.py b/modem/server.py
index 6a845da8..87bd73e8 100644
--- a/modem/server.py
+++ b/modem/server.py
@@ -29,7 +29,7 @@ app = Flask(__name__)
 CORS(app)
 CORS(app, resources={r"/*": {"origins": "*"}})
 sock = Sock(app)
-MODEM_VERSION = "0.14.0-alpha-exp"
+MODEM_VERSION = "0.14.0-alpha"
 
 # set config file to use
 def set_config():