Merge pull request #628 from DJ2LS/develop

.github/workflows/build_server.yml

@@ -32,20 +32,10 @@ jobs:
           repository: DJ2LS/FreeDATA
 
       - name: Set up Python 3.11
-        uses: actions/setup-python@v4
+        uses: actions/setup-python@v5
         with:
           python-version: "3.11"
 
-      - name: Create modem/dist
-        working-directory: modem
-        run: |
-          mkdir -p dist
-
-      - name: Create modem/dist/modem
-        working-directory: modem
-        run: |
-          mkdir -p dist/modem
-
       - name: Install Linux dependencies
         # if: matrix.os == 'ubuntu-20.04'
         if: ${{startsWith(matrix.os, 'ubuntu')}}

.github/workflows/build_server_nsis.yml

@@ -0,0 +1,50 @@
+name: Build and Release NSIS Installer
+on: [push]
+
+jobs:
+  build-and-release:
+    runs-on: windows-latest
+
+    steps:
+      - name: Check out repository
+        uses: actions/checkout@v4
+
+      - name: Set up Python 3.11
+        uses: actions/setup-python@v5
+        with:
+          python-version: "3.11"
+
+      - name: Install Python dependencies
+        run: |
+          python -m pip install --upgrade pip
+          pip install -r requirements.txt
+
+      - name: Build binaries
+        working-directory: modem
+        run: |
+          python3 -m nuitka  --remove-output --assume-yes-for-downloads --follow-imports --include-data-dir=lib=lib --include-data-files=lib/codec2/*=lib/codec2/ --include-data-files=config.ini.example=config.ini --standalone server.py --output-filename=freedata-server
+
+      - name: LIST ALL FILES
+        run: ls -R
+
+      - name: Create installer
+        uses: joncloud/makensis-action@v4
+        with:
+          script-file: "freedata-server-nsis-config.nsi"
+          arguments: '/V3'
+
+      - name: LIST ALL FILES
+        run: ls -R
+
+      - name: Upload artifact
+        uses: actions/upload-artifact@v4
+        with:
+          name: 'FreeData-Server-Installer'
+          path: ./FreeData-Server-Installer.exe
+
+      - name: Upload Installer to Release
+        uses: softprops/action-gh-release@v1
+        if: startsWith(github.ref, 'refs/tags/v')
+        with:
+          draft: true
+          files: ./FreeData-Server-Installer.exe

freedata-server-nsis-config.nsi

@@ -0,0 +1,102 @@
+!include "MUI2.nsh"
+
+; Request administrative rights
+RequestExecutionLevel admin
+
+; The name and file name of the installer
+Name "FreeData Server"
+OutFile "FreeData-Server-Installer.exe"
+
+; Default installation directory
+; InstallDir "$PROGRAMFILES\FreeData\freedata-server"
+
+InstallDir "$LOCALAPPDATA\FreeData\freedata-server"
+
+; Registry key to store the installation directory
+InstallDirRegKey HKCU "Software\FreeData\freedata-server" "Install_Dir"
+
+; Modern UI settings
+!define MUI_ABORTWARNING
+
+; Installer interface settings
+!define MUI_ICON "documentation\icon.ico"
+!define MUI_UNICON "documentation\icon.ico" ; Icon for the uninstaller
+
+
+; Define the welcome page text
+!define MUI_WELCOMEPAGE_TEXT "Welcome to the FreeData Server Setup Wizard. This wizard will guide you through the installation process."
+!define MUI_FINISHPAGE_TEXT "Folder: $INSTDIR"
+
+
+!define MUI_DIRECTORYPAGE_TEXT_TOP "Please select the installation folder. Its recommended using the suggested one for avoiding permission problems."
+
+
+; Pages
+!insertmacro MUI_PAGE_WELCOME
+!insertmacro MUI_PAGE_LICENSE "LICENSE"
+;!insertmacro MUI_PAGE_COMPONENTS
+!insertmacro MUI_PAGE_DIRECTORY
+!insertmacro MUI_PAGE_INSTFILES
+!insertmacro MUI_PAGE_FINISH
+
+; Uninstaller
+!insertmacro MUI_UNPAGE_WELCOME
+!insertmacro MUI_UNPAGE_CONFIRM
+!insertmacro MUI_UNPAGE_INSTFILES
+!insertmacro MUI_UNPAGE_FINISH
+
+; Language (you can choose and configure the language(s) you want)
+!insertmacro MUI_LANGUAGE "English"
+
+; Installer Sections
+Section "FreeData Server" SEC01
+
+  ; Set output path to the installation directory
+  SetOutPath $INSTDIR
+
+  ; Check if "config.ini" exists and back it up
+  IfFileExists $INSTDIR\config.ini backupConfig
+
+doneBackup:
+  ; Add your application files here
+  File /r "modem\server.dist\*.*"
+
+  ; Restore the original "config.ini" if it was backed up
+  IfFileExists $INSTDIR\config.ini.bak restoreConfig
+
+
+
+  ; Create a shortcut in the user's desktop
+  CreateShortCut "$DESKTOP\FreeData Server.lnk" "$INSTDIR\freedata-server.exe"
+
+  ; Create Uninstaller
+  WriteUninstaller "$INSTDIR\Uninstall.exe"
+
+; Backup "config.ini" before overwriting files
+backupConfig:
+  Rename $INSTDIR\config.ini $INSTDIR\config.ini.bak
+  Goto doneBackup
+
+; Restore the original "config.ini"
+restoreConfig:
+  Delete $INSTDIR\config.ini
+  Rename $INSTDIR\config.ini.bak $INSTDIR\config.ini
+
+
+SectionEnd
+
+; Uninstaller Section
+Section "Uninstall"
+
+  ; Delete files and directories
+  Delete $INSTDIR\freedata-server.exe
+  RMDir /r $INSTDIR
+
+  ; Remove the shortcut
+  Delete "$DESKTOP\FreeData Server.lnk"
+
+  ; Additional uninstallation commands here
+
+SectionEnd
+
+

gui/electron-builder.json5

@@ -19,22 +19,8 @@
   "files": [
     "dist",
     "dist-electron",
-    "../modem/server.dist/",
   ],
 
-    "extraResources": [
-
-      {
-        "from": "../modem/server.dist/",
-        "to": "modem",
-        "filter": [
-          "**/*",
-          "!**/.git"
-        ]
-
-      }
-    ],
-
 
   "mac": {
     "target": [

gui/package.json

@@ -1,8 +1,8 @@
 {
   "name": "FreeDATA",
-  "description": "FreeDATA",
+  "description": "FreeDATA Client application for connecting to FreeDATA server",
   "private": true,
-  "version": "0.12.1-alpha",
+  "version": "0.13.2-alpha",
   "main": "dist-electron/main/index.js",
   "scripts": {
     "start": "vite",
@@ -32,43 +32,38 @@
   },
   "homepage": "https://freedata.app",
   "dependencies": {
-    "@electron/asar": "3.2.7",
-    "@electron/notarize": "2.2.0",
-    "@electron/universal": "2.0.0",
+    "@electron/notarize": "2.2.1",
+    "@electron/universal": "2.0.1",
     "@popperjs/core": "2.11.8",
-    "@vueuse/electron": "10.7.1",
+    "@vueuse/electron": "10.7.2",
     "blob-util": "2.0.2",
     "bootstrap": "5.3.2",
-    "bootstrap-icons": "1.11.2",
+    "bootstrap-icons": "1.11.3",
     "bootswatch": "5.3.2",
     "browser-image-compression": "2.0.2",
     "chart.js": "4.4.1",
     "chartjs-plugin-annotation": "3.0.1",
-    "electron-log": "5.0.3",
+    "electron-log": "5.1.1",
     "electron-updater": "6.1.7",
-    "emoji-picker-element": "1.20.1",
+    "emoji-picker-element": "1.21.0",
     "emoji-picker-element-data": "1.6.0",
     "file-saver": "2.0.5",
     "gridstack": "10.0.1",
     "mime": "4.0.1",
     "nconf": "^0.12.1",
+    "noto-color-emoji": "^1.0.1",
     "pinia": "2.1.7",
-    "pouchdb": "8.0.1",
-    "pouchdb-browser": "8.0.1",
-    "pouchdb-find": "8.0.1",
-    "pouchdb-upsert": "2.2.0",
     "qth-locator": "2.1.0",
-    "sass": "1.66.1",
     "socket.io": "4.7.2",
-    "uuid": "9.0.1",
-    "vue": "3.3.12",
+    "uuid": "^9.0.1",
+    "vue": "3.4.15",
     "vue-chartjs": "5.3.0",
     "vuemoji-picker": "0.2.0"
   },
   "devDependencies": {
     "@types/nconf": "^0.10.6",
-    "@typescript-eslint/eslint-plugin": "6.17.0",
-    "@vitejs/plugin-vue": "4.5.2",
+    "@typescript-eslint/eslint-plugin": "6.19.1",
+    "@vitejs/plugin-vue": "5.0.3",
     "electron": "28.1.3",
     "electron-builder": "24.9.1",
     "eslint": "8.56.0",
@@ -84,7 +79,7 @@
     "vite-plugin-electron": "0.28.0",
     "vite-plugin-electron-renderer": "0.14.5",
     "vitest": "1.0.2",
-    "vue": "3.3.12",
+    "vue": "3.4.15",
     "vue-tsc": "1.8.27"
   }
 }

gui/src/components/chat.vue

@@ -1,22 +1,114 @@
 <script setup lang="ts">
-import chat_navbar from "./chat_navbar.vue";
+// @ts-nocheck
+// disable typescript check beacuse of error with beacon histogram options
+
 import chat_conversations from "./chat_conversations.vue";
 import chat_messages from "./chat_messages.vue";
 import chat_new_message from "./chat_new_message.vue";
 
-//updateAllChat();
+import { setActivePinia } from "pinia";
+import pinia from "../store/index";
+setActivePinia(pinia);
+
+import { useChatStore } from "../store/chatStore.js";
+const chat = useChatStore(pinia);
+
+import {
+  Chart as ChartJS,
+  CategoryScale,
+  LinearScale,
+  PointElement,
+  Title,
+  Tooltip,
+  Legend,
+  BarElement,
+} from "chart.js";
+
+import { Bar } from "vue-chartjs";
+import { ref, computed } from "vue";
+import annotationPlugin from "chartjs-plugin-annotation";
+
+ChartJS.register(
+  CategoryScale,
+  LinearScale,
+  PointElement,
+  Title,
+  Tooltip,
+  Legend,
+  BarElement,
+  annotationPlugin,
+);
+
+var beaconHistogramOptions = {
+  type: "bar",
+  bezierCurve: false, //remove curves from your plot
+  scaleShowLabels: false, //remove labels
+  tooltipEvents: [], //remove trigger from tooltips so they will'nt be show
+  pointDot: false, //remove the points markers
+  scaleShowGridLines: true, //set to false to remove the grids background
+  maintainAspectRatio: true,
+  plugins: {
+    legend: {
+      display: false,
+    },
+    annotation: {
+      annotations: [
+        {
+          type: "line",
+          mode: "horizontal",
+          scaleID: "y",
+          value: 0,
+          borderColor: "darkgrey", // Set the color to dark grey for the zero line
+          borderWidth: 0.5, // Set the line width
+        },
+      ],
+    },
+  },
+
+  scales: {
+    x: {
+      position: "bottom",
+      display: false,
+      min: -10,
+      max: 15,
+      ticks: {
+        display: false,
+      },
+    },
+    y: {
+      display: false,
+      min: -5,
+      max: 10,
+      ticks: {
+        display: false,
+      },
+    },
+  },
+};
+
+const beaconHistogramData = computed(() => ({
+  labels: chat.beaconLabelArray,
+  datasets: [
+    {
+      data: chat.beaconDataArray,
+      tension: 0.1,
+      borderColor: "rgb(0, 255, 0)",
+
+      backgroundColor: function (context) {
+        var value = context.dataset.data[context.dataIndex];
+        return value >= 0 ? "green" : "red";
+      },
+    },
+  ],
+}));
 </script>
 
 <template>
   <div class="container-fluid m-0 p-0">
-    <!------ chat navbar ---------------------------------------------------------------------->
-
-    <chat_navbar />
-
     <div class="row h-100 ms-0 mt-0 me-1">
       <div class="col-3 m-0 p-0 h-100 bg-light">
         <!------Chats area ---------------------------------------------------------------------->
-        <div class="container-fluid vh-100 overflow-scroll m-0 p-0">
+        <div class="container-fluid vh-100 overflow-auto m-0 p-0">
           <chat_conversations />
         </div>
         <div class="h-100">
@@ -29,18 +121,36 @@ import chat_new_message from "./chat_new_message.vue";
         </div>
       </div>
       <div class="col-9 border-start vh-100 p-0">
-        <!------messages area ---------------------------------------------------------------------->
-        <div
-          class="container overflow-auto"
-          id="message-container"
-          style="height: calc(100% - 225px)"
-        >
-          <chat_messages />
+        <div class="d-flex flex-column vh-100">
+          <!-- Top Navbar -->
+          <nav class="navbar sticky-top bg-body-tertiary shadow">
+            <div class="input-group mb-0 p-0 w-25">
+              <button type="button" class="btn btn-outline-secondary" disabled>
+                Beacons
+              </button>
+
+              <div
+                class="form-floating border border-secondary-subtle border-1 rounded-end"
+              >
+                <Bar
+                  :data="beaconHistogramData"
+                  :options="beaconHistogramOptions"
+                  width="300"
+                  height="50"
+                />
+              </div>
+            </div>
+          </nav>
+
+          <!-- Chat Messages Area -->
+          <div class="flex-grow-1 overflow-auto">
+            <chat_messages />
+          </div>
+
+          <chat_new_message />
         </div>
 
         <!------ new message area ---------------------------------------------------------------------->
-
-        <chat_new_message />
       </div>
     </div>
   </div>

gui/src/components/chat_conversations.vue

@@ -4,16 +4,15 @@ import pinia from "../store/index";
 setActivePinia(pinia);
 
 import { useChatStore } from "../store/chatStore.js";
-const chat = useChatStore(pinia);
+import { getBeaconDataByCallsign } from "../js/api.js";
 
-import {
-  getNewMessagesByDXCallsign,
-  resetIsNewMessage,
-} from "../js/chatHandler";
+import { ref, computed } from "vue";
+
+
+const chat = useChatStore(pinia);
 
 function chatSelected(callsign) {
   chat.selectedCallsign = callsign.toUpperCase();
-
   // scroll message container to bottom
   var messageBody = document.getElementById("message-container");
   if (messageBody != null) {
@@ -21,62 +20,83 @@ function chatSelected(callsign) {
     messageBody.scrollTop = messageBody.scrollHeight - messageBody.clientHeight;
   }
 
-  if (getNewMessagesByDXCallsign(callsign)[1] > 0) {
-    let messageArray = getNewMessagesByDXCallsign(callsign)[2];
-    console.log(messageArray);
-
-    for (const key in messageArray) {
-      resetIsNewMessage(messageArray[key].uuid, false);
-    }
-  }
-
-  try {
-    chat.beaconLabelArray = Object.values(
-      chat.sorted_beacon_list[chat.selectedCallsign].timestamp,
-    );
-    chat.beaconDataArray = Object.values(
-      chat.sorted_beacon_list[chat.selectedCallsign].snr,
-    );
-  } catch (e) {
-    console.log("beacon data not fetched: " + e);
-    chat.beaconLabelArray = [];
-    chat.beaconDataArray = [];
-  }
+  processBeaconData(callsign);
 }
+
+async function processBeaconData(callsign) {
+  // fetch beacon data when selecting a callsign
+  let beacons = await getBeaconDataByCallsign(callsign);
+  chat.beaconLabelArray = beacons.map((entry) => entry.timestamp);
+  chat.beaconDataArray = beacons.map((entry) => entry.snr);
+}
+
+function getDateTime(timestamp) {
+  let date = new Date(timestamp);
+  let hours = date.getHours().toString().padStart(2, "0");
+  let minutes = date.getMinutes().toString().padStart(2, "0");
+  let seconds = date.getSeconds().toString().padStart(2, "0");
+  return `${hours}:${minutes}`;
+}
+
+const newChatCall = ref(null);
+
+function newChat() {
+  let callsign = this.newChatCall.value;
+  callsign = callsign.toUpperCase().trim();
+  if (callsign === "") return;
+  this.newChatCall.value = "";
+}
+
+
 </script>
 <template>
+
+<nav class="navbar sticky-top bg-body-tertiary shadow">
+
+<button
+            class="btn btn-outline-primary w-100"
+            data-bs-target="#newChatModal"
+            data-bs-toggle="modal"
+          >
+            <i class="bi bi-pencil-square"> Start a new chat</i>
+          </button>
+
+
+</nav>
+
+
   <div
     class="list-group bg-body-tertiary m-0 p-1"
     id="chat-list-tab"
     role="chat-tablist"
   >
-    <template v-for="(item, key) in chat.callsign_list" :key="item.dxcallsign">
+    <template
+      v-for="(details, callsign, key) in chat.callsign_list"
+      :key="callsign"
+    >
       <a
-        class="list-group-item list-group-item-action list-group-item-dark rounded-2 border-0 mb-2"
+        class="list-group-item list-group-item-action list-group-item-secondary rounded-2 border-0 mb-2"
         :class="{ active: key == 0 }"
-        :id="`list-chat-list-${item}`"
+        :id="`list-chat-list-${callsign}`"
         data-bs-toggle="list"
-        :href="`#list-${item}-messages`"
+        :href="`#list-${callsign}-messages`"
         role="tab"
-        aria-controls="list-{{item}}-messages"
-        @click="chatSelected(item)"
+        aria-controls="list-{{callsign}}-messages"
+        @click="chatSelected(callsign)"
       >
         <div class="row">
-          <div class="col-9">
-            <strong>{{ item }}</strong>
-            <span
-              class="badge rounded-pill bg-danger"
-              v-if="getNewMessagesByDXCallsign(item)[1] > 0"
-            >
-              {{ getNewMessagesByDXCallsign(item)[1] }} new messages
-            </span>
+          <div class="col-9 text-truncate">
+            <strong>{{ callsign }}</strong>
+            <br />
+            <small> {{ details.body }} </small>
           </div>
           <div class="col-3">
+            <small> {{ getDateTime(details.timestamp) }} </small>
             <button
               class="btn btn-sm btn-outline-secondary ms-2 border-0"
               data-bs-target="#deleteChatModal"
               data-bs-toggle="modal"
-              @click="chatSelected(item)"
+              @click="chatSelected(callsign)"
             >
               <i class="bi bi-three-dots-vertical"></i>
             </button>

gui/src/components/chat_messages.vue

@@ -15,22 +15,24 @@ import SentBroadcastMessage from "./chat_messages_broadcast_sent.vue"; // Import
 var prevChatMessageDay = "";
 
 function getDateTime(timestampRaw) {
-  var datetime = new Date(timestampRaw * 1000).toLocaleString(
-    navigator.language,
-    {
-      hourCycle: "h23",
-      year: "numeric",
-      month: "2-digit",
-      day: "2-digit",
-    },
-  );
-  return datetime;
+  let date = new Date(timestampRaw);
+  let year = date.getFullYear();
+  let month = (date.getMonth() + 1).toString().padStart(2, "0"); // Months are zero-indexed
+  let day = date.getDate().toString().padStart(2, "0");
+  return `${year}-${month}-${day}`;
 }
+
+
+
 </script>
 
 <template>
-  <div class="tab-content" id="nav-tabContent-chat-messages">
-    <template v-for="(callsign, key) in chat.callsign_list">
+
+  <div class="tab-content p-3" id="nav-tabContent-chat-messages">
+    <template
+      v-for="(details, callsign, key) in chat.callsign_list"
+      :key="callsign"
+    >
       <div
         class="tab-pane fade show"
         :class="{ active: key == 0 }"
@@ -38,42 +40,36 @@ function getDateTime(timestampRaw) {
         role="tabpanel"
         :aria-labelledby="`list-chat-list-${callsign}`"
       >
-        <template
-          v-for="item in chat.sorted_chat_list[callsign]"
-          :key="item._id"
-        >
+        <template v-for="item in chat.sorted_chat_list[callsign]">
           <div v-if="prevChatMessageDay !== getDateTime(item.timestamp)">
             <div class="separator my-2">
               {{ (prevChatMessageDay = getDateTime(item.timestamp)) }}
             </div>
           </div>
 
-          <div v-if="item.type === 'beacon' && item.status === 'received'">
-            <!-- {{ item }} -->
-          </div>
-
-          <div v-if="item.type === 'ping'">{{ item.snr }} dB ping received</div>
-
-          <div v-if="item.type === 'ping-ack'">
-            {{ item.snr }} dB ping-ack received
-          </div>
-
-          <div v-if="item.type === 'transmit'">
+          <div v-if="item.direction === 'transmit'">
             <sent-message :message="item" />
           </div>
-          <div v-else-if="item.type === 'received'">
+
+          <div v-else-if="item.direction === 'receive'">
             <received-message :message="item" />
           </div>
+          <!--
           <div v-if="item.type === 'broadcast_transmit'">
             <sent-broadcast-message :message="item" />
           </div>
           <div v-else-if="item.type === 'broadcast_received'">
             <received-broadcast-message :message="item" />
           </div>
+          -->
         </template>
       </div>
     </template>
   </div>
+
+
+
+
 </template>
 
 <style>

gui/src/components/chat_messages_received.vue

@@ -2,16 +2,28 @@
   <div class="row justify-content-start mb-2">
     <div :class="messageWidthClass">
       <div class="card bg-light border-0 text-dark">
-        <div class="card-header" v-if="getFileContent['filesize'] !== 0">
-          <p class="card-text">
-            {{ getFileContent["filename"] }} |
-            {{ getFileContent["filesize"] }} Bytes |
-            {{ getFileContent["filetype"] }}
-          </p>
+        <div
+          v-for="attachment in message.attachments"
+          :key="attachment.id"
+          class="card-header"
+        >
+          <div class="btn-group w-100" role="group">
+            <button class="btn btn-light text-truncate" disabled>
+              {{ attachment.name }}
+            </button>
+            <button
+              @click="
+                downloadAttachment(attachment.hash_sha512, attachment.name)
+              "
+              class="btn btn-light w-25"
+            >
+              <i class="bi bi-download strong"></i>
+            </button>
+          </div>
         </div>
 
         <div class="card-body">
-          <p class="card-text">{{ message.msg }}</p>
+          <p class="card-text">{{ message.body }}</p>
         </div>
 
         <div class="card-footer p-0 bg-light border-top-0">
@@ -24,6 +36,7 @@
     <!-- Delete button outside of the card -->
     <div class="col-auto">
       <button
+        disabled
         class="btn btn-outline-secondary border-0 me-1"
         @click="showMessageInfo"
         data-bs-target="#messageInfoModal"
@@ -32,14 +45,6 @@
         <i class="bi bi-info-circle"></i>
       </button>
 
-      <button
-        v-if="getFileContent['filesize'] !== 0"
-        class="btn btn-outline-secondary border-0 me-1"
-        @click="downloadAttachment"
-      >
-        <i class="bi bi-download"></i>
-      </button>
-
       <button class="btn btn-outline-secondary border-0" @click="deleteMessage">
         <i class="bi bi-trash"></i>
       </button>
@@ -52,14 +57,13 @@ import {
   deleteMessageFromDB,
   requestMessageInfo,
   getMessageAttachment,
-} from "../js/chatHandler";
+} from "../js/messagesHandler";
 import { atob_FD } from "../js/freedata";
 
 // pinia store setup
 import { setActivePinia } from "pinia";
 import pinia from "../store/index";
 setActivePinia(pinia);
-import { saveAs } from "file-saver";
 
 import { useChatStore } from "../store/chatStore.js";
 const chat = useChatStore(pinia);
@@ -70,50 +74,55 @@ export default {
   },
   methods: {
     showMessageInfo() {
-      requestMessageInfo(this.message._id);
+      requestMessageInfo(this.message.id);
       //let infoModal = Modal.getOrCreateInstance(document.getElementById('messageInfoModal'))
       //console.log(this.infoModal)
       //this.infoModal.show()
     },
     deleteMessage() {
-      deleteMessageFromDB(this.message._id);
+      deleteMessageFromDB(this.message.id);
     },
-    async downloadAttachment() {
+    async downloadAttachment(hash_sha512, fileName) {
       try {
-        // reset file store
-        chat.downloadFileFromDB = [];
+        const jsondata = await getMessageAttachment(hash_sha512);
+        const byteCharacters = atob(jsondata.data);
+        const byteArrays = [];
 
-        const attachment = await getMessageAttachment(this.message._id);
-        const blob = new Blob([atob_FD(attachment[2])], {
-          type: `${attachment[1]};charset=utf-8`,
-        });
-        window.focus();
-        saveAs(blob, attachment[0]);
+        for (let offset = 0; offset < byteCharacters.length; offset += 512) {
+          const slice = byteCharacters.slice(offset, offset + 512);
+          const byteNumbers = new Array(slice.length);
+          for (let i = 0; i < slice.length; i++) {
+            byteNumbers[i] = slice.charCodeAt(i);
+          }
+          const byteArray = new Uint8Array(byteNumbers);
+          byteArrays.push(byteArray);
+        }
+
+        const blob = new Blob(byteArrays, { type: jsondata.type });
+        const url = URL.createObjectURL(blob);
+
+        // Creating a temporary anchor element to download the file
+        const anchor = document.createElement("a");
+        anchor.href = url;
+        anchor.download = fileName;
+        document.body.appendChild(anchor);
+        anchor.click();
+
+        // Cleanup
+        document.body.removeChild(anchor);
+        URL.revokeObjectURL(url);
       } catch (error) {
-        console.error("Failed to download attachment:", error);
+        console.error("Failed to download the attachment:", error);
       }
     },
   },
   computed: {
-    getFileContent() {
-      try {
-        var filename = Object.keys(this.message._attachments)[0];
-        var filesize = this.message._attachments[filename]["length"];
-        var filetype = filename.split(".")[1];
-
-        return { filename: filename, filesize: filesize, filetype: filetype };
-      } catch (e) {
-        console.log("file not loaded from database - empty?");
-        // we are only checking against filesize for displaying attachments
-        return { filesize: 0 };
-      }
-    },
     messageWidthClass() {
       // Calculate a Bootstrap grid class based on message length
       // Adjust the logic as needed to fit your requirements
-      if (this.message.msg.length <= 50) {
+      if (this.message.body.length <= 50) {
         return "col-4";
-      } else if (this.message.msg.length <= 100) {
+      } else if (this.message.body.length <= 100) {
         return "col-6";
       } else {
         return "col-9";
@@ -121,14 +130,11 @@ export default {
     },
 
     getDateTime() {
-      var datetime = new Date(this.message.timestamp * 1000).toLocaleString(
-        navigator.language,
-        {
-          hour: "2-digit",
-          minute: "2-digit",
-        },
-      );
-      return datetime;
+      let date = new Date(this.message.timestamp);
+      let hours = date.getHours().toString().padStart(2, "0");
+      let minutes = date.getMinutes().toString().padStart(2, "0");
+      let seconds = date.getSeconds().toString().padStart(2, "0");
+      return `${hours}:${minutes}:${seconds}`;
     },
   },
 };

gui/src/components/chat_messages_sent.vue

@@ -2,14 +2,6 @@
   <div class="row justify-content-end mb-2">
     <!-- control area -->
     <div class="col-auto p-0 m-0">
-      <button
-        v-if="getFileContent['filesize'] !== 0"
-        class="btn btn-outline-secondary border-0 me-1"
-        @click="downloadAttachment"
-      >
-        <i class="bi bi-download"></i>
-      </button>
-
       <button
         class="btn btn-outline-secondary border-0 me-1"
         @click="repeatMessage"
@@ -18,6 +10,7 @@
       </button>
 
       <button
+        disabled
         class="btn btn-outline-secondary border-0 me-1"
         @click="showMessageInfo"
         data-bs-target="#messageInfoModal"
@@ -30,24 +23,37 @@
         <i class="bi bi-trash"></i>
       </button>
     </div>
-
     <!-- message area -->
     <div :class="messageWidthClass">
       <div class="card bg-secondary text-white">
-        <div class="card-header" v-if="getFileContent['filesize'] !== 0">
-          <p class="card-text">
-            {{ getFileContent["filename"] }} |
-            {{ getFileContent["filesize"] }} Bytes |
-            {{ getFileContent["filetype"] }}
-          </p>
+        <div
+          v-for="attachment in message.attachments"
+          :key="attachment.id"
+          class="card-header"
+        >
+          <div class="btn-group w-100" role="group">
+            <button class="btn btn-light text-truncate" disabled>
+              {{ attachment.name }}
+            </button>
+            <button
+              @click="
+                downloadAttachment(attachment.hash_sha512, attachment.name)
+              "
+              class="btn btn-light w-25"
+            >
+              <i class="bi bi-download strong"></i>
+            </button>
+          </div>
         </div>
 
         <div class="card-body">
-          <p class="card-text">{{ message.msg }}</p>
+          <p class="card-text">{{ message.body }}</p>
         </div>
 
         <div class="card-footer p-0 bg-secondary border-top-0">
-          <p class="text p-0 m-0 me-1 text-end">{{ getDateTime }}</p>
+          <p class="text p-0 m-0 me-1 text-end">
+            {{ message.status }} | {{ getDateTime }}
+          </p>
           <!-- Display formatted timestamp in card-footer -->
         </div>
 
@@ -91,13 +97,12 @@ import {
   deleteMessageFromDB,
   requestMessageInfo,
   getMessageAttachment,
-} from "../js/chatHandler";
+} from "../js/messagesHandler";
 
 // pinia store setup
 import { setActivePinia } from "pinia";
 import pinia from "../store/index";
 setActivePinia(pinia);
-import { saveAs } from "file-saver";
 
 import { useChatStore } from "../store/chatStore.js";
 const chat = useChatStore(pinia);
@@ -109,57 +114,60 @@ export default {
 
   methods: {
     repeatMessage() {
-      repeatMessageTransmission(this.message._id);
+      repeatMessageTransmission(this.message.id);
     },
 
     deleteMessage() {
-      deleteMessageFromDB(this.message._id);
+      deleteMessageFromDB(this.message.id);
     },
     showMessageInfo() {
       console.log("requesting message info.....");
-      requestMessageInfo(this.message._id);
+      requestMessageInfo(this.message.id);
       //let infoModal = Modal.getOrCreateInstance(document.getElementById('messageInfoModal'))
       //console.log(this.infoModal)
       //this.infoModal.show()
     },
-    async downloadAttachment() {
+    async downloadAttachment(hash_sha512, fileName) {
       try {
-        // reset file store
-        chat.downloadFileFromDB = [];
+        const jsondata = await getMessageAttachment(hash_sha512);
+        const byteCharacters = atob(jsondata.data);
+        const byteArrays = [];
 
-        const attachment = await getMessageAttachment(this.message._id);
-        const blob = new Blob([atob_FD(attachment[2])], {
-          type: `${attachment[1]};charset=utf-8`,
-        });
-        saveAs(blob, attachment[0]);
+        for (let offset = 0; offset < byteCharacters.length; offset += 512) {
+          const slice = byteCharacters.slice(offset, offset + 512);
+          const byteNumbers = new Array(slice.length);
+          for (let i = 0; i < slice.length; i++) {
+            byteNumbers[i] = slice.charCodeAt(i);
+          }
+          const byteArray = new Uint8Array(byteNumbers);
+          byteArrays.push(byteArray);
+        }
+
+        const blob = new Blob(byteArrays, { type: jsondata.type });
+        const url = URL.createObjectURL(blob);
+
+        // Creating a temporary anchor element to download the file
+        const anchor = document.createElement("a");
+        anchor.href = url;
+        anchor.download = fileName;
+        document.body.appendChild(anchor);
+        anchor.click();
+
+        // Cleanup
+        document.body.removeChild(anchor);
+        URL.revokeObjectURL(url);
       } catch (error) {
-        console.error("Failed to download attachment:", error);
+        console.error("Failed to download the attachment:", error);
       }
     },
   },
   computed: {
-    getFileContent() {
-      var filename = Object.keys(this.message._attachments)[0];
-      var filesize = this.message._attachments[filename]["length"];
-      var filetype = filename.split(".")[1];
-
-      // ensure filesize is 0 for hiding message header if no data is available
-      if (
-        typeof filename === "undefined" ||
-        filename === "" ||
-        filename === "-"
-      ) {
-        filesize = 0;
-      }
-
-      return { filename: filename, filesize: filesize, filetype: filetype };
-    },
     messageWidthClass() {
       // Calculate a Bootstrap grid class based on message length
       // Adjust the logic as needed to fit your requirements
-      if (this.message.msg.length <= 50) {
+      if (this.message.body.length <= 50) {
         return "col-4";
-      } else if (this.message.msg.length <= 100) {
+      } else if (this.message.body.length <= 100) {
         return "col-6";
       } else {
         return "col-9";
@@ -167,14 +175,11 @@ export default {
     },
 
     getDateTime() {
-      var datetime = new Date(this.message.timestamp * 1000).toLocaleString(
-        navigator.language,
-        {
-          hour: "2-digit",
-          minute: "2-digit",
-        },
-      );
-      return datetime;
+      let date = new Date(this.message.timestamp);
+      let hours = date.getHours().toString().padStart(2, "0");
+      let minutes = date.getMinutes().toString().padStart(2, "0");
+      let seconds = date.getSeconds().toString().padStart(2, "0");
+      return `${hours}:${minutes}:${seconds}`;
     },
   },
 };

gui/src/components/chat_navbar.vue

@@ -1,182 +0,0 @@
-<script setup lang="ts">
-// @ts-nocheck
-
-import { setActivePinia } from "pinia";
-import pinia from "../store/index";
-setActivePinia(pinia);
-
-import { useStateStore } from "../store/stateStore.js";
-const state = useStateStore(pinia);
-
-import { useChatStore } from "../store/chatStore.js";
-const chat = useChatStore(pinia);
-
-import { startChatWithNewStation } from "../js/chatHandler";
-
-import {
-  Chart as ChartJS,
-  CategoryScale,
-  LinearScale,
-  PointElement,
-  Title,
-  Tooltip,
-  Legend,
-  BarElement,
-} from "chart.js";
-
-import { Bar } from "vue-chartjs";
-import { ref, computed } from "vue";
-import annotationPlugin from "chartjs-plugin-annotation";
-
-const newChatCall = ref(null);
-
-ChartJS.register(
-  CategoryScale,
-  LinearScale,
-  PointElement,
-  Title,
-  Tooltip,
-  Legend,
-  BarElement,
-  annotationPlugin,
-);
-
-var beaconHistogramOptions = {
-  type: "bar",
-  bezierCurve: false, //remove curves from your plot
-  scaleShowLabels: false, //remove labels
-  tooltipEvents: [], //remove trigger from tooltips so they will'nt be show
-  pointDot: false, //remove the points markers
-  scaleShowGridLines: true, //set to false to remove the grids background
-  maintainAspectRatio: true,
-  plugins: {
-    legend: {
-      display: false,
-    },
-    annotation: {
-      annotations: [
-        {
-          type: "line",
-          mode: "horizontal",
-          scaleID: "y",
-          value: 0,
-          borderColor: "darkgrey", // Set the color to dark grey for the zero line
-          borderWidth: 0.5, // Set the line width
-        },
-      ],
-    },
-  },
-
-  scales: {
-    x: {
-      position: "bottom",
-      display: false,
-      min: -10,
-      max: 15,
-      ticks: {
-        display: false,
-      },
-    },
-    y: {
-      display: false,
-      min: -5,
-      max: 10,
-      ticks: {
-        display: false,
-      },
-    },
-  },
-};
-
-const beaconHistogramData = computed(() => ({
-  labels: chat.beaconLabelArray,
-  datasets: [
-    {
-      data: chat.beaconDataArray,
-      tension: 0.1,
-      borderColor: "rgb(0, 255, 0)",
-
-      backgroundColor: function (context) {
-        var value = context.dataset.data[context.dataIndex];
-        return value >= 0 ? "green" : "red";
-      },
-    },
-  ],
-}));
-
-function newChat() {
-  let callsign = this.newChatCall.value;
-  callsign = callsign.toUpperCase().trim();
-  if (callsign === "") return;
-  startChatWithNewStation(callsign);
-  //updateAllChat(false);
-  this.newChatCall.value = "";
-}
-
-function syncWithModem() {
-  getRxBuffer();
-}
-</script>
-
-<template>
-  <nav class="navbar bg-body-tertiary border-bottom">
-    <div class="container">
-      <div class="row w-100">
-        <div class="col-3 p-0 me-2">
-          <div class="input-group bottom-0 m-0 ms-1">
-            <input
-              class="form-control"
-              maxlength="9"
-              style="text-transform: uppercase"
-              placeholder="callsign"
-              @keypress.enter="newChat()"
-              ref="newChatCall"
-            />
-            <button
-              class="btn btn-sm btn-outline-success"
-              id="createNewChatButton"
-              type="button"
-              title="Start a new chat (enter dx call sign first)"
-              @click="newChat()"
-            >
-              new chat
-              <i class="bi bi-pencil-square" style="font-size: 1.2rem"></i>
-            </button>
-          </div>
-        </div>
-        <div class="col-5 ms-2 p-0">
-          <!-- right side of chat nav bar-->
-
-          <div class="input-group mb-0 p-0 w-50">
-            <button type="button" class="btn btn-outline-secondary" disabled>
-              Beacons
-            </button>
-
-            <div
-              class="form-floating border border-secondary-subtle border-1 rounded-end"
-            >
-              <Bar
-                :data="beaconHistogramData"
-                :options="beaconHistogramOptions"
-                width="300"
-                height="50"
-              />
-            </div>
-          </div>
-        </div>
-
-        <div class="col-2 ms-2 p-0">
-          <div class="input-group mb-0 p-0">
-            <button
-              type="button"
-              class="btn btn-outline-secondary"
-              @click="syncWithModem()"
-            >
-              Modem Sync {{ state.rx_buffer_length }}
-            </button>
-          </div>
-        </div>
-      </div>
-    </div>
-  </nav>
-</template>

gui/src/components/chat_new_message.vue

@@ -17,8 +17,7 @@ import chat_navbar from './chat_navbar.vue'
 import chat_conversations from './chat_conversations.vue'
 import chat_messages from './chat_messages.vue'
 
-import {updateAllChat, newMessage, newBroadcast} from '../js/chatHandler'
-
+import { newMessage } from '../js/messagesHandler.ts'
 
 import {
   Chart as ChartJS,
@@ -45,56 +44,94 @@ chat.inputText += detail.unicode
 const chatModuleMessage=ref(null);
 
 
+// Function to trigger the hidden file input
+function triggerFileInput() {
+  fileInput.value.click();
+}
 
 
-function transmitNewMessage(){
+// Use a ref for storing multiple files
+const selectedFiles = ref([]);
+const fileInput = ref(null);
+
+function handleFileSelection(event) {
+    // Reset previously selected files
+    selectedFiles.value = [];
+
+    // Process each file
+    for (let file of event.target.files) {
+        const reader = new FileReader();
+        reader.onload = () => {
+            // Convert file content to base64
+            const base64Content = btoa(reader.result); // Adjust this line if necessary
+            selectedFiles.value.push({
+                name: file.name,
+                size: file.size,
+                type: file.type,
+                content: base64Content, // Store base64 encoded content
+            });
+        };
+        reader.readAsBinaryString(file); // Read the file content as binary string
+    }
+}
+
+function removeFile(index) {
+    selectedFiles.value.splice(index, 1);
+     // Check if the selectedFiles array is empty
+    if (selectedFiles.value.length === 0) {
+        // Reset the file input if there are no files left
+        resetFile();
+    }
+}
+
+function transmitNewMessage() {
+    // Check if a callsign is selected, default to the first one if not
+    if (typeof(chat.selectedCallsign) === 'undefined') {
+        chat.selectedCallsign = Object.keys(chat.callsign_list)[0];
+    }
 
     chat.inputText = chat.inputText.trim();
-    if (chat.inputText.length==0 && chat.inputFileName == "-")
-      return;
+
+    // Proceed only if there is text or files selected
+    if (chat.inputText.length === 0 && selectedFiles.value.length === 0) return;
+
+    const attachments = selectedFiles.value.map(file => {
+        return {
+            name: file.name,
+            type: file.type,
+            data: file.content
+        };
+    });
+
     if (chat.selectedCallsign.startsWith("BC-")) {
-
-        newBroadcast(chat.selectedCallsign, chat.inputText)
-
+        // Handle broadcast message differently if needed
+        return "new broadcast";
     } else {
-        newMessage(chat.selectedCallsign, chat.inputText, chat.inputFile, chat.inputFileName, chat.inputFileSize, chat.inputFileType)
+        // If there are attachments, send them along with the message
+        if (attachments.length > 0) {
+            newMessage(chat.selectedCallsign, chat.inputText, attachments);
+        } else {
+            // Send text only if no attachments are selected
+            newMessage(chat.selectedCallsign, chat.inputText);
+        }
     }
-    // finally do a cleanup
-    //chatModuleMessage.reset();
+
+    // Cleanup after sending message
     chat.inputText = '';
-    chatModuleMessage.value="";
-    // @ts-expect-error
+    chatModuleMessage.value = "";
     resetFile()
 }
 
 function resetFile(event){
-    chat.inputFileName = '-'
-    chat.inputFileSize = '-'
-    chat.inputFileType = '-'
-
+    if (fileInput.value) {
+        fileInput.value.value = ''; // Reset the file input
+    }
+    // Clear the selected files array to reset the state of attachments
+    selectedFiles.value = [];
 }
 
 
-function readFile(event) {
-    const reader = new FileReader();
 
-    reader.onload = () => {
-        console.log(reader.result);
-        chat.inputFileName = event.target.files[0].name
-        chat.inputFileSize = event.target.files[0].size
-        chat.inputFileType = event.target.files[0].type
-
-        chat.inputFile = reader.result
-        calculateTimeNeeded()
-
-//        String.fromCharCode.apply(null, Array.from(chatFile))
-
-
-      };
-
-    reader.readAsArrayBuffer(event.target.files[0]);
-
-}
 
 
 
@@ -133,9 +170,9 @@ function calculateTimeNeeded(){
             return obj.snr === snrList[i].snr
           })
 
-        calculatedSpeedPerMinutePER0.push(chat.inputFileSize / result.bpm)
-        calculatedSpeedPerMinutePER25.push(chat.inputFileSize / (result.bpm * 0.75))
-        calculatedSpeedPerMinutePER75.push(chat.inputFileSize / (result.bpm * 0.25))
+        calculatedSpeedPerMinutePER0.push(totalSize / result.bpm)
+        calculatedSpeedPerMinutePER25.push(totalSize / (result.bpm * 0.75))
+        calculatedSpeedPerMinutePER75.push(totalSize / (result.bpm * 0.25))
 
     }
 
@@ -162,22 +199,51 @@ const speedChartData = computed(() => ({
 
 <template>
 
-<div class="container-fluid mt-2 p-0">
-                        <input
-                          type="checkbox"
-                          id="expand_textarea"
-                          class="btn-check"
-                          autocomplete="off"
-                        />
-                        <label
-                          class="btn d-flex justify-content-center"
-                          id="expand_textarea_label"
-                          for="expand_textarea"
-                          ><i
-                            id="expand_textarea_button"
-                            class="bi bi-chevron-compact-up"
-                          ></i
-                        ></label>
+
+  <nav class="navbar sticky-bottom bg-body-tertiary border-top mb-5">
+<div class="container-fluid p-0">
+
+
+
+    <!-- Hidden file input -->
+    <input type="file" multiple ref="fileInput" @change="handleFileSelection" style="display: none;" />
+
+
+
+  <div class="container-fluid px-0">
+    <div class="d-flex flex-row overflow-auto bg-light">
+      <div v-for="(file, index) in selectedFiles" :key="index" class="pe-2">
+        <div class="card" style=" min-width: 10rem; max-width: 10rem;">
+          <!-- Card Header with Remove Button -->
+          <div class="card-header d-flex justify-content-between align-items-center">
+            <span class="text-truncate">{{ file.name }}</span>
+            <button class="btn btn-close" @click="removeFile(index)"></button>
+          </div>
+          <div class="card-body">
+            <p class="card-text">...</p>
+          </div>
+          <div class="card-footer text-muted">
+            {{ file.type }}
+          </div>
+          <div class="card-footer text-muted">
+            {{ file.size }} bytes
+          </div>
+        </div>
+      </div>
+    </div>
+  </div>
+
+<!--
+<Line :data="speedChartData" />
+-->
+
+
+
+
+
+
+
+
 
                         <div class="input-group bottom-0 ms-2">
 
@@ -195,10 +261,11 @@ const speedChartData = computed(() => ({
 
 
                                         <!-- trigger file selection modal -->
-                            <button type="button" class="btn btn-outline-secondary border-0 rounded-pill me-1" data-bs-toggle="modal" data-bs-target="#fileSelectionModal">
-                              <i class="bi bi-paperclip" style="font-size: 1.2rem"></i>
-                            </button>
 
+                           <button type="button" class="btn btn-outline-secondary border-0 rounded-pill me-1" @click="triggerFileInput">
+                              <i class="bi bi-paperclip" style="font-size: 1.2rem"></i>
+
+                            </button>
 
                           <textarea
                             class="form-control border rounded-pill"
@@ -226,76 +293,17 @@ const speedChartData = computed(() => ({
                         </div>
                       </div>
 
-
-
-
-                      <!-- select file modal -->
-
-                <div
-                  class="modal fade"
-                  id="fileSelectionModal"
-                  tabindex="-1"
-                  aria-labelledby="fileSelectionModalLabel"
-                  aria-hidden="true"
-                >
-
-                <div class="modal-dialog">
-    <div class="modal-content">
-      <div class="modal-header">
-        <h5 class="modal-title" id="staticBackdropLabel">File Attachment</h5>
-        <button type="button" class="btn-close" data-bs-dismiss="modal" aria-label="Close" @click="resetFile"></button>
-      </div>
-      <div class="modal-body">
-
-
-     <div class="alert alert-warning d-flex align-items-center" role="alert">
- <i class="bi bi-exclamation-triangle-fill ms-2 me-2"></i>
-  <div>
-    Transmission speed over HF channels is very limited!
-  </div>
-</div>
-
-           <div class="input-group-text mb-3">
-                <input class="" type="file" ref="doc" @change="readFile" />
-           </div>
-
-
-
-<div class="btn-group me-2" role="group" aria-label="Basic outlined example">
-  <button type="button" class="btn btn-secondary">Type</button>
-  <button type="button" class="btn btn-secondary disabled">{{chat.inputFileType}}</button>
-</div>
-
-<div class="btn-group me-2" role="group" aria-label="Basic outlined example">
-  <button type="button" class="btn btn-secondary">Size</button>
-  <button type="button" class="btn btn-secondary disabled">{{chat.inputFileSize}}</button>
-</div>
-
-<Line :data="speedChartData" />
-
-      </div>
-      <div class="modal-footer">
-        <button type="button" class="btn btn-secondary" data-bs-dismiss="modal" @click="resetFile">Reset</button>
-        <button type="button" class="btn btn-primary" data-bs-dismiss="modal">Append</button>
-      </div>
-    </div>
-  </div>
-                </div>
-
+</nav>
 <!-- Emoji Picker Modal -->
-<div class="modal fade" id="emojiPickerModal" tabindex="-1" aria-hidden="true">
+<div class="modal fade" id="emojiPickerModal" aria-hidden="true" >
   <div class="modal-dialog modal-dialog-centered modal-sm">
     <div class="modal-content">
       <div class="modal-body p-0">
-        <VuemojiPicker @emojiClick="handleEmojiClick" />
+        <VuemojiPicker @emojiClick="handleEmojiClick"/>
       </div>
     </div>
   </div>
 </div>
 
-
-
-
-
 </template>
 

gui/src/components/infoScreen.vue

@@ -192,7 +192,7 @@ onMounted(shuffleCards);
       <div class="row row-cols-1 row-cols-sm-2 row-cols-md-4 row-cols-lg-6">
         <div class="d-inline-block" v-for="card in cards" :key="card.titleName">
           <div class="col">
-            <div class="card border-dark m-2" style="max-width: 15rem">
+            <div class="card border-dark m-1" style="max-width: 10rem">
               <img :src="card.imgSrc" class="card-img-top grayscale" />
               <div class="card-body">
                 <p class="card-text text-center">{{ card.role }}</p>

gui/src/components/main.vue

@@ -20,12 +20,7 @@ import infoScreen from "./infoScreen.vue";
 import main_modem_healthcheck from "./main_modem_healthcheck.vue";
 import Dynamic_components from "./dynamic_components.vue";
 
-import { stopTransmission } from "../js/api";
-
-function stopAllTransmissions() {
-  console.log("stopping transmissions");
-  stopTransmission();
-}
+import { getFreedataMessages } from "../js/api";
 </script>
 
 <template>
@@ -44,7 +39,7 @@ function stopAllTransmissions() {
 
   <div class="container-fluid">
     <div class="row">
-      <div class="col-sm-auto bg-body-secondary border-end">
+      <div class="col-1 p-0 bg-body-secondary border-end">
         <div
           class="d-flex flex-sm-column flex-row flex-nowrap align-items-center sticky-top"
         >
@@ -75,6 +70,7 @@ function stopAllTransmissions() {
               role="tab"
               aria-controls="list-chat"
               title="Chat"
+              @click="getFreedataMessages"
               ><i class="bi bi-chat-text h3"></i
             ></a>
 
@@ -121,7 +117,7 @@ function stopAllTransmissions() {
           </div>
         </div>
       </div>
-      <div class="col-sm min-vh-100 m-0 p-0">
+      <div class="col-11 min-vh-100 m-0 p-0">
         <!-- content -->
 
         <!-- TODO: Remove the top navbar entirely if not needed

gui/src/components/main_active_heard_stations.vue

@@ -28,10 +28,12 @@ function getDateTime(timestampRaw) {
 }
 
 function getMaidenheadDistance(dxGrid) {
-  try {
-    return parseInt(distance(settings.remote.STATION.mygrid, dxGrid));
-  } catch (e) {
-    console.warn(e);
+  if (typeof dxGrid != "undefined") {
+    try {
+      return parseInt(distance(settings.remote.STATION.mygrid, dxGrid));
+    } catch (e) {
+      console.warn(e);
+    }
   }
 }
 </script>

gui/src/components/main_footer_navbar.vue

@@ -8,49 +8,52 @@ const state = useStateStore(pinia);
 </script>
 
 <template>
-  <nav
-    class="navbar fixed-bottom navbar-expand-xl bg-body-tertiary border-top p-2"
-    style="margin-left: 87px"
-  >
-    <div class="col">
-      <div class="btn-toolbar" role="toolbar" style="margin-left: 2px">
-        <div class="btn-group btn-group-sm me-1" role="group">
-          <button
-            class="btn btn-sm btn-secondary me-1"
-            v-bind:class="{
-              'btn-danger': state.ptt_state == true,
-              'btn-secondary': state.ptt_state == false,
-            }"
-            id="ptt_state"
-            type="button"
-            style="pointer-events: auto"
-            disabled
-            data-bs-toggle="tooltip"
-            data-bs-placement="top"
-            data-bs-title="PTT trigger state"
-          >
-            <i class="bi bi-broadcast-pin" style="font-size: 0.8rem"></i>
-          </button>
-
-          <button
-            class="btn btn-sm btn-secondary me-1"
-            id="busy_state"
-            type="button"
-            data-bs-placement="top"
-            data-bs-toggle="tooltip"
-            data-bs-trigger="hover"
-            data-bs-html="true"
-            v-bind:class="{
-              'btn-danger': state.busy_state === 'BUSY',
-              'btn-secondary': state.busy_state === 'IDLE',
-            }"
-            data-bs-title="Modem state"
-            disabled
-            style="pointer-events: auto"
-          >
-            <i class="bi bi-cpu" style="font-size: 0.8rem"></i>
-          </button>
+  <div class="container-fluid">
+    <div class="row">
+      <div class="col-1">123</div>
+      <div class="col-11">
+        <nav
+          class="navbar fixed-bottom navbar-expand-xl bg-body-tertiary border-top p-2"
+        >
+          <div class="col">
+            <div class="btn-toolbar" role="toolbar">
+              <div class="btn-group btn-group-sm me-1" role="group">
+                <button
+                  class="btn btn-sm btn-secondary me-1"
+                  v-bind:class="{
+                    'btn-danger': state.ptt_state == true,
+                    'btn-secondary': state.ptt_state == false,
+                  }"
+                  id="ptt_state"
+                  type="button"
+                  style="pointer-events: auto"
+                  disabled
+                  data-bs-toggle="tooltip"
+                  data-bs-placement="top"
+                  data-bs-title="PTT trigger state"
+                >
+                  <i class="bi bi-broadcast-pin" style="font-size: 0.8rem"></i>
+                </button>
 
+                <button
+                  class="btn btn-sm btn-secondary me-1"
+                  id="busy_state"
+                  type="button"
+                  data-bs-placement="top"
+                  data-bs-toggle="tooltip"
+                  data-bs-trigger="hover"
+                  data-bs-html="true"
+                  v-bind:class="{
+                    'btn-danger': state.busy_state === true,
+                    'btn-secondary': state.busy_state === false,
+                  }"
+                  data-bs-title="Modem state"
+                  disabled
+                  style="pointer-events: auto"
+                >
+                  <i class="bi bi-cpu" style="font-size: 0.8rem"></i>
+                </button>
+                <!--
           <button
             class="btn btn-sm btn-secondary me-1"
             id="arq_session"
@@ -69,7 +72,8 @@ const state = useStateStore(pinia);
           >
             <i class="bi bi-arrow-left-right" style="font-size: 0.8rem"></i>
           </button>
-
+          -->
+                <!--
           <button
             class="btn btn-sm btn-secondary me-1"
             id="arq_state"
@@ -87,7 +91,8 @@ const state = useStateStore(pinia);
           >
             <i class="bi bi-file-earmark-binary" style="font-size: 0.8rem"></i>
           </button>
-          <!--
+          -->
+                <!--
               <button
                 class="btn btn-sm btn-secondary me-1"
                 id="rigctld_state"
@@ -101,7 +106,7 @@ const state = useStateStore(pinia);
                 <i class="bi bi-usb-symbol" style="font-size: 0.8rem"></i>
               </button>
 -->
-
+                <!--
           <button
             class="btn btn-sm btn-secondary disabled me-3"
             type="button"
@@ -118,143 +123,157 @@ const state = useStateStore(pinia);
           >
             <i class="bi bi-hourglass"></i>
           </button>
-        </div>
+            -->
+              </div>
 
-        <div class="btn-group btn-group-sm me-1" role="group">
-          <button
-            class="btn btn-sm btn-secondary me-4 disabled"
-            type="button"
-            data-bs-placement="top"
-            data-bs-toggle="tooltip"
-            data-bs-trigger="hover"
-            data-bs-title="What's the frequency, Kenneth?"
-            style="pointer-events: auto"
-          >
-            {{ state.frequency / 1000 }} kHz
-          </button>
-        </div>
+              <div class="btn-group btn-group-sm me-1" role="group">
+                <button
+                  class="btn btn-sm btn-secondary me-4 disabled"
+                  type="button"
+                  data-bs-placement="top"
+                  data-bs-toggle="tooltip"
+                  data-bs-trigger="hover"
+                  data-bs-title="What's the frequency, Kenneth?"
+                  style="pointer-events: auto"
+                >
+                  {{ state.frequency / 1000 }} kHz
+                </button>
+              </div>
 
-        <div class="btn-group btn-group-sm me-1" role="group">
-          <button
-            class="btn btn-sm btn-secondary me-0"
-            type="button"
-            title="Speed level"
-          >
-            <i class="bi bi-speedometer2" style="font-size: 1rem"></i>
-          </button>
+              <div class="btn-group btn-group-sm me-1" role="group">
+                <button
+                  class="btn btn-sm btn-secondary me-0"
+                  type="button"
+                  title="Speed level"
+                >
+                  <i class="bi bi-speedometer2" style="font-size: 1rem"></i>
+                </button>
 
-          <button
-            class="btn btn-sm btn-secondary me-4 disabled"
-            type="button"
-            data-bs-placement="top"
-            data-bs-toggle="tooltip"
-            data-bs-trigger="hover"
-            data-bs-html="true"
-            data-bs-titel="speed level"
-          >
-            <i
-              class="bi"
-              style="font-size: 1rem"
-              v-bind:class="{
-                'bi-reception-0': state.speed_level == 0,
-                'bi-reception-1': state.speed_level == 1,
-                'bi-reception-2': state.speed_level == 2,
-                'bi-reception-3': state.speed_level == 3,
-                'bi-reception-4': state.speed_level == 4,
-              }"
-            ></i>
-          </button>
-        </div>
-        <div class="btn-group btn-group-sm me-1" role="group">
-          <button
-            class="btn btn-sm btn-secondary me-0"
-            type="button"
-            title="Bytes transfered"
-          >
-            <i class="bi bi-file-earmark-binary" style="font-size: 1rem"></i>
-          </button>
+                <button
+                  class="btn btn-sm btn-secondary me-4 disabled"
+                  type="button"
+                  data-bs-placement="top"
+                  data-bs-toggle="tooltip"
+                  data-bs-trigger="hover"
+                  data-bs-html="true"
+                  data-bs-titel="speed level"
+                >
+                  <i
+                    class="bi"
+                    style="font-size: 1rem"
+                    v-bind:class="{
+                      'bi-reception-0': state.speed_level == 0,
+                      'bi-reception-1': state.speed_level == 1,
+                      'bi-reception-2': state.speed_level == 2,
+                      'bi-reception-3': state.speed_level == 3,
+                      'bi-reception-4': state.speed_level == 4,
+                    }"
+                  ></i>
+                </button>
+              </div>
+              <div class="btn-group btn-group-sm me-1" role="group">
+                <button
+                  class="btn btn-sm btn-secondary me-0"
+                  type="button"
+                  title="Bytes transfered"
+                >
+                  <i
+                    class="bi bi-file-earmark-binary"
+                    style="font-size: 1rem"
+                  ></i>
+                </button>
 
-          <button
-            class="btn btn-sm btn-secondary me-4 disabled"
-            type="button"
-            data-bs-placement="top"
-            data-bs-toggle="tooltip"
-            data-bs-trigger="hover"
-            data-bs-html="true"
-            data-bs-title="total bytes of transmission"
-          >
-            {{ state.arq_total_bytes }}
-          </button>
-        </div>
-        <div class="btn-group btn-group-sm me-1" role="group">
-          <button
-            class="btn btn-sm btn-secondary me-0"
-            type="button"
-            data-bs-placement="top"
-            data-bs-toggle="tooltip"
-            data-bs-trigger="hover"
-            data-bs-html="true"
-            data-bs-title="Current or last connected with station"
-          >
-            <i class="bi bi-file-earmark-binary" style="font-size: 1rem"></i>
-          </button>
+                <button
+                  class="btn btn-sm btn-secondary me-4 disabled"
+                  type="button"
+                  data-bs-placement="top"
+                  data-bs-toggle="tooltip"
+                  data-bs-trigger="hover"
+                  data-bs-html="true"
+                  data-bs-title="total bytes of transmission"
+                >
+                  {{ state.arq_total_bytes }}
+                </button>
+              </div>
+              <div class="btn-group btn-group-sm me-1" role="group">
+                <button
+                  class="btn btn-sm btn-secondary me-0"
+                  type="button"
+                  data-bs-placement="top"
+                  data-bs-toggle="tooltip"
+                  data-bs-trigger="hover"
+                  data-bs-html="true"
+                  data-bs-title="Current or last connected with station"
+                >
+                  <i
+                    class="bi bi-file-earmark-binary"
+                    style="font-size: 1rem"
+                  ></i>
+                </button>
 
-          <button
-            class="btn btn-sm btn-secondary disabled me-1"
-            type="button"
-            data-bs-placement="top"
-            data-bs-toggle="tooltip"
-            data-bs-trigger="hover"
-            data-bs-html="true"
-            data-bs-title="the dxcallsign of the connected station"
-          >
-            {{ state.dxcallsign }}
-          </button>
-        </div>
-      </div>
-    </div>
-    <div class="col-lg-4">
-      <div style="margin-right: 2px">
-        <div class="progress w-100" style="height: 20px; min-width: 200px">
-          <div
-            class="progress-bar progress-bar-striped bg-primary force-gpu"
-            id="transmission_progress"
-            role="progressbar"
-            :style="{ width: state.arq_transmission_percent + '%' }"
-            aria-valuenow="0"
-            aria-valuemin="0"
-            aria-valuemax="100"
-          ></div>
-          <p
-            class="justify-content-center m-0 d-flex position-absolute w-100 text-dark"
-          >
-            Message Progress
-          </p>
-        </div>
-
-        <div
-          hidden
-          class="progress mb-0 rounded-0 rounded-bottom"
-          style="height: 10px"
-        >
-          <div
-            class="progress-bar progress-bar-striped bg-warning"
-            id="transmission_timeleft"
-            role="progressbar"
-            :style="{ width: state.arq_seconds_until_timeout_percent + '%' }"
-            aria-valuenow="0"
-            aria-valuemin="0"
-            aria-valuemax="100"
-          >
-            <p
-              class="justify-content-center m-0 d-flex position-absolute w-100 text-dark"
-            >
-              timeout in {{ state.arq_seconds_until_timeout }}s
-            </p>
+                <button
+                  class="btn btn-sm btn-secondary disabled me-1"
+                  type="button"
+                  data-bs-placement="top"
+                  data-bs-toggle="tooltip"
+                  data-bs-trigger="hover"
+                  data-bs-html="true"
+                  data-bs-title="the dxcallsign of the connected station"
+                >
+                  {{ state.dxcallsign }}
+                </button>
+              </div>
+            </div>
           </div>
-        </div>
+          <div class="col-lg-4">
+            <div style="margin-right: 2px">
+              <div
+                class="progress w-100"
+                style="height: 20px; min-width: 200px"
+              >
+                <div
+                  class="progress-bar progress-bar-striped bg-primary force-gpu"
+                  id="transmission_progress"
+                  role="progressbar"
+                  :style="{ width: state.arq_transmission_percent + '%' }"
+                  aria-valuenow="0"
+                  aria-valuemin="0"
+                  aria-valuemax="100"
+                ></div>
+                <p
+                  class="justify-content-center m-0 d-flex position-absolute w-100 text-dark"
+                >
+                  Message Progress
+                </p>
+              </div>
+
+              <div
+                hidden
+                class="progress mb-0 rounded-0 rounded-bottom"
+                style="height: 10px"
+              >
+                <div
+                  class="progress-bar progress-bar-striped bg-warning"
+                  id="transmission_timeleft"
+                  role="progressbar"
+                  :style="{
+                    width: state.arq_seconds_until_timeout_percent + '%',
+                  }"
+                  aria-valuenow="0"
+                  aria-valuemin="0"
+                  aria-valuemax="100"
+                >
+                  <p
+                    class="justify-content-center m-0 d-flex position-absolute w-100 text-dark"
+                  >
+                    timeout in {{ state.arq_seconds_until_timeout }}s
+                  </p>
+                </div>
+              </div>
+            </div>
+          </div>
+        </nav>
       </div>
     </div>
-  </nav>
+  </div>
 </template>
-ww

gui/src/components/main_modals.vue

@@ -9,19 +9,20 @@ import { useChatStore } from "../store/chatStore.js";
 const chat = useChatStore(pinia);
 
 import { settingsStore as settings, onChange } from "../store/settingsStore.js";
-
 import { sendModemTestFrame } from "../js/api";
-
-import {
-  deleteChatByCallsign,
-  getNewMessagesByDXCallsign,
-} from "../js/chatHandler";
-
 import main_startup_check from "./main_startup_check.vue";
+import { newMessage, deleteCallsignFromDB } from "../js/messagesHandler.ts";
+
+function newChat() {
+  let newCallsign = chat.newChatCallsign.toUpperCase();
+  newMessage(newCallsign, chat.newChatMessage);
+
+  chat.newChatCallsign = "";
+  chat.newChatMessage = "";
+}
 
 function deleteChat() {
-  //console.log(chat.selectedCallsign)
-  deleteChatByCallsign(chat.selectedCallsign);
+  deleteCallsignFromDB(chat.selectedCallsign);
 }
 
 import {
@@ -181,16 +182,12 @@ const transmissionSpeedChartDataMessageInfo = computed(() => ({
             <span class="input-group-text" id="basic-addon1"
               >Total Messages</span
             >
-            <span class="input-group-text" id="basic-addon1">{{
-              getNewMessagesByDXCallsign(chat.selectedCallsign)[0]
-            }}</span>
+            <span class="input-group-text" id="basic-addon1">...</span>
           </div>
 
           <div class="input-group mb-3">
             <span class="input-group-text" id="basic-addon1">New Messages</span>
-            <span class="input-group-text" id="basic-addon1">{{
-              getNewMessagesByDXCallsign(chat.selectedCallsign)[1]
-            }}</span>
+            <span class="input-group-text" id="basic-addon1">...</span>
           </div>
         </div>
         <div class="modal-footer">
@@ -316,6 +313,86 @@ const transmissionSpeedChartDataMessageInfo = computed(() => ({
     </div>
   </div>
 
+  <div
+    class="modal fade"
+    ref="modalEle"
+    id="newChatModal"
+    tabindex="-1"
+    aria-hidden="true"
+  >
+    <div class="modal-dialog">
+      <div class="modal-content">
+        <div class="modal-header">
+          <h1 class="modal-title fs-5" id="deleteChatModalLabel">
+            Start a new chat
+          </h1>
+          <button
+            type="button"
+            class="btn-close"
+            data-bs-dismiss="modal"
+            aria-label="Close"
+          ></button>
+        </div>
+        <div class="modal-body">
+          <div class="alert alert-info" role="alert">
+            1. Enter destination callsign
+            <br />
+            2. Enter a first message
+            <br />
+            3. Pressing "START NEW CHAT"
+          </div>
+
+          <div class="form-floating mb-3">
+            <input
+              type="text"
+              class="form-control"
+              id="floatingInputDestination"
+              placeholder="dxcallsign / destination"
+              maxlength="9"
+              style="text-transform: uppercase"
+              @keypress.enter="newChat()"
+              v-model="chat.newChatCallsign"
+            />
+            <label for="floatingInputDestination"
+              >dxcallsign / destination</label
+            >
+          </div>
+
+          <div class="form-floating">
+            <textarea
+              class="form-control"
+              placeholder="Your first message"
+              id="floatingTextareaNewChatMessage"
+              style="height: 100px"
+              v-model="chat.newChatMessage"
+            ></textarea>
+            <label for="floatingTextareaNewChatMessage">First message</label>
+          </div>
+        </div>
+        <div class="modal-footer">
+          <button
+            type="button"
+            class="btn btn-secondary"
+            data-bs-dismiss="modal"
+          >
+            Close
+          </button>
+          <button
+            class="btn btn-sm btn-outline-success"
+            id="createNewChatButton"
+            type="button"
+            data-bs-dismiss="modal"
+            title="Start a new chat (enter dx call sign first)"
+            @click="newChat()"
+          >
+            START NEW CHAT
+            <i class="bi bi-pencil-square" style="font-size: 1.2rem"></i>
+          </button>
+        </div>
+      </div>
+    </div>
+  </div>
+
   <!-- HELP MODALS AUDIO -->
   <div
     class="modal fade"

gui/src/components/main_modem_healthcheck.vue

@@ -6,15 +6,7 @@ setActivePinia(pinia);
 import { useStateStore } from "../store/stateStore.js";
 const state = useStateStore(pinia);
 
-function getOverallHealth() {
-  //Return a number indicating health for icon bg color; lower the number the healthier
-  let health = 0;
-  if (state.modem_connection !== "connected") health += 5;
-  if (!state.is_modem_running) health += 3;
-  if (state.radio_status === false) health += 2;
-  if (process.env.FDUpdateAvail === "1") health += 1;
-  return health;
-}
+import { getOverallHealth } from "../js/eventHandler.js";
 </script>
 <template>
   <a

gui/src/components/main_startup_check.vue

@@ -34,10 +34,6 @@ onMounted(() => {
   new Modal("#modemCheck", {}).show();
 });
 
-function refreshModem() {
-  getModemState();
-}
-
 function getModemStateLocal() {
   // Returns active/inactive if modem is running for modem status label
   if (state.is_modem_running == true) return "Active";
@@ -168,7 +164,9 @@ function testHamlib() {
               <div id="modemStatusCollapse" class="accordion-collapse collapse">
                 <div class="accordion-body">
                   <div class="input-group input-group-sm mb-1">
-                    <label class="input-group-text w-25">Modem control</label>
+                    <label class="input-group-text w-50"
+                      >Manual modem restart</label
+                    >
                     <label class="input-group-text">
                       <button
                         type="button"
@@ -202,20 +200,6 @@ function testHamlib() {
                         <i class="bi bi-stop-fill"></i>
                       </button>
                     </label>
-                    <label class="input-group-text">
-                      <button
-                        type="button"
-                        id="refreshModem"
-                        class="btn btn-sm btn-outline-secondary"
-                        data-bs-toggle="tooltip"
-                        data-bs-trigger="hover"
-                        data-bs-html="false"
-                        title="Refresh modem status."
-                        @click="refreshModem"
-                      >
-                        <i class="bi bi-bullseye"></i>
-                      </button>
-                    </label>
                   </div>
                   <!-- Audio Input Device -->
                   <div class="input-group input-group-sm mb-1">

gui/src/components/settings_station.vue

@@ -9,6 +9,10 @@ import {
   validateCallsignWithoutSSID,
 } from "../js/freedata";
 function validateCall() {
+  //ensure callsign is uppercase:
+  let call = settings.remote.STATION.mycall;
+  settings.remote.STATION.mycall = call.toUpperCase();
+
   if (validateCallsignWithoutSSID(settings.remote.STATION.mycall))
     //Send new callsign to modem if valid
     onChange();
@@ -25,6 +29,7 @@ function validateCall() {
     <input
       type="text"
       class="form-control"
+      style="text-transform: uppercase"
       placeholder="Enter your callsign and save it"
       id="myCall"
       aria-label="Station Callsign"

gui/src/js/api.js

@@ -4,6 +4,8 @@ import {
   validateCallsignWithoutSSID,
 } from "./freedata";
 
+import { processFreedataMessages } from "./messagesHandler";
+
 function buildURL(params, endpoint) {
   const url = "http://" + params.host + ":" + params.port + endpoint;
   return url;
@@ -15,10 +17,9 @@ async function apiGet(endpoint) {
     if (!response.ok) {
       throw new Error(`REST response not ok: ${response.statusText}`);
     }
-    const data = await response.json();
-    return data;
+    return await response.json();
   } catch (error) {
-    console.error("Error getting from REST:", error);
+    //console.error("Error getting from REST:", error);
   }
 }
 
@@ -43,6 +44,27 @@ export async function apiPost(endpoint, payload = {}) {
   }
 }
 
+export async function apiDelete(endpoint, payload = {}) {
+  try {
+    const response = await fetch(buildURL(settings.local, endpoint), {
+      method: "DELETE",
+      headers: {
+        "Content-Type": "application/json",
+      },
+      body: JSON.stringify(payload),
+    });
+
+    if (!response.ok) {
+      throw new Error(`REST response not ok: ${response.statusText}`);
+    }
+
+    const data = await response.json();
+    return data;
+  } catch (error) {
+    console.error("Error deleting from REST:", error);
+  }
+}
+
 export async function getVersion() {
   let data = await apiGet("/version").then((res) => {
     return res;
@@ -52,30 +74,30 @@ export async function getVersion() {
 }
 
 export async function getConfig() {
-  return apiGet("/config");
+  return await apiGet("/config");
 }
 
-export function setConfig(config) {
-  return apiPost("/config", config);
+export async function setConfig(config) {
+  return await apiPost("/config", config);
 }
 
-export function getAudioDevices() {
-  return apiGet("/devices/audio");
+export async function getAudioDevices() {
+  return await apiGet("/devices/audio");
 }
 
-export function getSerialDevices() {
-  return apiGet("/devices/serial");
+export async function getSerialDevices() {
+  return await apiGet("/devices/serial");
 }
 
-export function setModemBeacon(enabled = false) {
-  return apiPost("/modem/beacon", { enabled: enabled });
+export async function setModemBeacon(enabled = false) {
+  return await apiPost("/modem/beacon", { enabled: enabled });
 }
 
-export function sendModemCQ() {
-  return apiPost("/modem/cqcqcq");
+export async function sendModemCQ() {
+  return await apiPost("/modem/cqcqcq");
 }
 
-export function sendModemPing(dxcall) {
+export async function sendModemPing(dxcall) {
   if (
     validateCallsignWithSSID(dxcall) === false &&
     validateCallsignWithoutSSID(dxcall) === true
@@ -85,11 +107,11 @@ export function sendModemPing(dxcall) {
   }
   dxcall = String(dxcall).toUpperCase().trim();
   if (validateCallsignWithSSID(dxcall))
-    return apiPost("/modem/ping_ping", { dxcall: dxcall });
+    return await apiPost("/modem/ping_ping", { dxcall: dxcall });
 }
 
-export function sendModemARQRaw(mycall, dxcall, data, uuid) {
-  return apiPost("/modem/send_arq_raw", {
+export async function sendModemARQRaw(mycall, dxcall, data, uuid) {
+  return await apiPost("/modem/send_arq_raw", {
     mycallsign: mycall,
     dxcall: dxcall,
     data: data,
@@ -97,33 +119,63 @@ export function sendModemARQRaw(mycall, dxcall, data, uuid) {
   });
 }
 
-export function stopTransmission() {
-  return apiPost("/modem/stop_transmission");
+export async function stopTransmission() {
+  return await apiPost("/modem/stop_transmission");
 }
 
-export function sendModemTestFrame() {
-  return apiPost("/modem/send_test_frame");
+export async function sendModemTestFrame() {
+  return await apiPost("/modem/send_test_frame");
 }
 
-export function startModem() {
-  return apiPost("/modem/start");
+export async function startModem() {
+  return await apiPost("/modem/start");
 }
 
-export function stopModem() {
-  return apiPost("/modem/stop");
+export async function stopModem() {
+  return await apiPost("/modem/stop");
 }
 
-export function getModemState() {
-  return apiGet("/modem/state");
+export async function getModemState() {
+  return await apiGet("/modem/state");
 }
 
-export function setRadioParameters(frequency, mode, rf_level) {
-  return apiPost("/radio", {
+export async function setRadioParameters(frequency, mode, rf_level) {
+  return await apiPost("/radio", {
     radio_frequency: frequency,
     radio_mode: mode,
     radio_rf_level: rf_level,
   });
 }
-export function getRadioStatus() {
-  return apiGet("/radio");
+export async function getRadioStatus() {
+  return await apiGet("/radio");
+}
+
+export async function getFreedataMessages() {
+  let res = await apiGet("/freedata/messages");
+  processFreedataMessages(res);
+}
+
+export async function getFreedataAttachmentBySha512(data_sha512) {
+  let res = await apiGet(`/freedata/messages/attachment/${data_sha512}`);
+  return res;
+}
+
+export async function sendFreedataMessage(destination, body, attachments) {
+  return await apiPost("/freedata/messages", {
+    destination: destination,
+    body: body,
+    attachments: attachments,
+  });
+}
+
+export async function retransmitFreedataMessage(id) {
+  return await apiPost(`/freedata/messages/${id}`);
+}
+
+export async function deleteFreedataMessage(id) {
+  return await apiDelete(`/freedata/messages/${id}`);
+}
+
+export async function getBeaconDataByCallsign(callsign) {
+  return await apiGet(`/freedata/beacons/${callsign}`);
 }

gui/src/js/deprecated_sock.js

@@ -1,886 +0,0 @@
-var net = require("net");
-import { atob_FD, btoa_FD, sortByPropertyDesc } from "./freedata";
-import { addDataToWaterfall } from "../js/waterfallHandler.js";
-
-import {
-  newMessageReceived,
-  newBeaconReceived,
-  updateTransmissionStatus,
-  setStateSuccess,
-  setStateFailed,
-} from "./chatHandler";
-import { displayToast } from "./popupHandler";
-
-// ----------------- init pinia stores -------------
-import { setActivePinia } from "pinia";
-import pinia from "../store/index";
-setActivePinia(pinia);
-import { useStateStore } from "../store/stateStore.js";
-const stateStore = useStateStore(pinia);
-
-import { settingsStore as settings } from "../store/settingsStore.js";
-
-var client = new net.Socket();
-var socketchunk = ""; // Current message, per connection.
-
-// split character
-//const split_char = "\0;\1;";
-const split_char = "0;1;";
-
-// global to keep track of Modem connection error emissions
-var modemShowConnectStateError = 1;
-var setTxAudioLevelOnce = true;
-var setRxAudioLevelOnce = true;
-
-// network connection Timeout
-//setTimeout(connectModem, 2000);
-
-function connectModem() {
-  //exports.connectModem = function(){
-  //console.log('connecting to Modem...')
-
-  //clear message buffer after reconnecting or initial connection
-  socketchunk = "";
-
-  client.connect(settings.modem_port, settings.modem_host);
-}
-
-client.on("connect", function () {
-  console.log("Modem connection established");
-  stateStore.modem_running_state = "running";
-  stateStore.busy_state = "-";
-  stateStore.arq_state = "-";
-  stateStore.frequency = "-";
-  stateStore.mode = "-";
-  stateStore.bandwidth = "-";
-  stateStore.dbfs_level = 0;
-  stateStore.updateTncState(client.readyState);
-
-  modemShowConnectStateError = 1;
-});
-
-client.on("error", function (err) {
-  if (modemShowConnectStateError == 1) {
-    console.log("Modem connection error");
-    console.log(err);
-    modemShowConnectStateError = 0;
-    stateStore.modem_running_state = "stopped";
-  }
-
-  setTimeout(connectModem, 500);
-  client.destroy();
-  stateStore.busy_state = "-";
-  stateStore.arq_state = "-";
-  stateStore.frequency = "-";
-  stateStore.mode = "-";
-  stateStore.bandwidth = "-";
-  stateStore.dbfs_level = 0;
-  stateStore.updateTncState(client.readyState);
-});
-
-/*
-client.on('close', function(data) {
-	console.log(' Modem connection closed');
-    setTimeout(connectModem, 2000)
-});
-*/
-
-client.on("end", function (data) {
-  console.log("Modem connection ended");
-  console.log(data);
-  stateStore.busy_state = "-";
-  stateStore.arq_state = "-";
-  stateStore.frequency = "-";
-  stateStore.mode = "-";
-  stateStore.bandwidth = "-";
-  stateStore.dbfs_level = 0;
-  stateStore.updateTncState(client.readyState);
-  client.destroy();
-  stateStore.modem_running_state = "stopped";
-
-  setTimeout(connectModem, 500);
-});
-
-function writeTncCommand(command) {
-  console.log(command);
-  // we use the writingCommand function to update our TCPIP state because we are calling this function a lot
-  // if socket opened, we are able to run commands
-
-  if (client.readyState == "open") {
-    client.write(command + "\n");
-  }
-
-  if (client.readyState == "closed") {
-    console.log("Modem SOCKET CONNECTION CLOSED!");
-  }
-
-  if (client.readyState == "opening") {
-    console.log("connecting to Modem...");
-  }
-}
-
-client.on("data", function (socketdata) {
-  stateStore.updateTncState(client.readyState);
-
-  /*
-    inspired by:
-    stackoverflow.com questions 9070700 nodejs-net-createserver-large-amount-of-data-coming-in
-    */
-
-  socketdata = socketdata.toString("utf8"); // convert data to string
-  socketchunk += socketdata; // append data to buffer so we can stick long data together
-
-  // check if we received begin and end of json data
-  if (socketchunk.startsWith('{"') && socketchunk.endsWith('"}\n')) {
-    var data = "";
-
-    // split data into chunks if we received multiple commands
-    socketchunk = socketchunk.split("\n");
-    //don't think this is needed anymore
-    //data = JSON.parse(socketchunk[0])
-
-    // search for empty entries in socketchunk and remove them
-    for (let i = 0; i < socketchunk.length; i++) {
-      if (socketchunk[i] === "") {
-        socketchunk.splice(i, 1);
-      }
-    }
-
-    //iterate through socketchunks array to execute multiple commands in row
-    for (let i = 0; i < socketchunk.length; i++) {
-      //check if data is not empty
-      if (socketchunk[i].length > 0) {
-        //try to parse JSON
-        try {
-          data = JSON.parse(socketchunk[i]);
-        } catch (e) {
-          console.log("Throwing away data!!!!\n" + e); // "SyntaxError
-          //console.log(e); // "SyntaxError
-          console.log(socketchunk[i]);
-          socketchunk = "";
-          //If we're here, I don't think we want to process any data that may be in data variable
-          continue;
-        }
-      }
-      //console.log(data)
-      if (data["command"] == "modem_state") {
-        //console.log(data)
-
-        stateStore.rx_buffer_length = data["rx_buffer_length"];
-        stateStore.frequency = data["frequency"];
-        stateStore.busy_state = data["modem_state"];
-        stateStore.arq_state = data["arq_state"];
-        stateStore.mode = data["mode"];
-        stateStore.bandwidth = data["bandwidth"];
-        stateStore.tx_audio_level = data["tx_audio_level"];
-        stateStore.rx_audio_level = data["rx_audio_level"];
-        // if audio level is different from config one, send new audio level to modem
-        //console.log(parseInt(stateStore.tx_audio_level))
-        //console.log(parseInt(settings.tx_audio_level))
-        if (
-          parseInt(stateStore.tx_audio_level) !==
-            parseInt(settings.tx_audio_level) &&
-          setTxAudioLevelOnce === true
-        ) {
-          setTxAudioLevelOnce = false;
-          console.log(setTxAudioLevelOnce);
-          setTxAudioLevel(settings.tx_audio_level);
-        }
-
-        if (
-          parseInt(stateStore.rx_audio_level) !==
-            parseInt(settings.rx_audio_level) &&
-          setRxAudioLevelOnce === true
-        ) {
-          setRxAudioLevelOnce = false;
-          console.log(setRxAudioLevelOnce);
-          setRxAudioLevel(settings.rx_audio_level);
-        }
-
-        stateStore.dbfs_level = data["audio_dbfs"];
-        stateStore.ptt_state = data["ptt_state"];
-        stateStore.speed_level = data["speed_level"];
-        stateStore.fft = JSON.parse(data["fft"]);
-        stateStore.channel_busy = data["channel_busy"];
-        stateStore.channel_busy_slot = data["channel_busy_slot"];
-
-        addDataToWaterfall(JSON.parse(data["fft"]));
-
-        if (data["scatter"].length > 0) {
-          stateStore.scatter = data["scatter"];
-        }
-        // s meter strength
-        stateStore.s_meter_strength_raw = data["strength"];
-        if (stateStore.s_meter_strength_raw == "") {
-          stateStore.s_meter_strength_raw = "Unsupported";
-          stateStore.s_meter_strength_percent = 0;
-        } else {
-          // https://www.moellerstudios.org/converting-amplitude-representations/
-          stateStore.s_meter_strength_percent = Math.round(
-            Math.pow(10, stateStore.s_meter_strength_raw / 20) * 100,
-          );
-        }
-
-        stateStore.dbfs_level_percent = Math.round(
-          Math.pow(10, stateStore.dbfs_level / 20) * 100,
-        );
-        stateStore.dbfs_level = Math.round(stateStore.dbfs_level);
-
-        stateStore.arq_total_bytes = data["total_bytes"];
-        stateStore.heard_stations = data["stations"].sort(
-          sortByPropertyDesc("timestamp"),
-        );
-        stateStore.dxcallsign = data["dxcallsign"];
-
-        stateStore.beacon_state = data["beacon_state"];
-        stateStore.audio_recording = data["audio_recording"];
-
-        stateStore.hamlib_status = data["hamlib_status"];
-        stateStore.alc = data["alc"];
-        stateStore.rf_level = data["rf_level"];
-
-        stateStore.is_codec2_traffic = data["is_codec2_traffic"];
-
-        stateStore.arq_session_state = data["arq_session"];
-        stateStore.arq_state = data["arq_state"];
-        stateStore.arq_transmission_percent = data["arq_transmission_percent"];
-        stateStore.arq_seconds_until_finish = data["arq_seconds_until_finish"];
-        stateStore.arq_seconds_until_timeout =
-          data["arq_seconds_until_timeout"];
-        stateStore.arq_seconds_until_timeout_percent =
-          (stateStore.arq_seconds_until_timeout / 180) * 100;
-
-        if (data["speed_list"].length > 0) {
-          prepareStatsDataForStore(data["speed_list"]);
-        }
-
-        // TODO: Remove ported objects
-        /*
-        let Data = {
-          mycallsign: data["mycallsign"],
-          mygrid: data["mygrid"],
-          //channel_state: data['CHANNEL_STATE'],
-
-          info: data["info"],
-          rx_msg_buffer_length: data["rx_msg_buffer_length"],
-          tx_n_max_retries: data["tx_n_max_retries"],
-          arq_tx_n_frames_per_burst: data["arq_tx_n_frames_per_burst"],
-          arq_tx_n_bursts: data["arq_tx_n_bursts"],
-          arq_tx_n_current_arq_frame: data["arq_tx_n_current_arq_frame"],
-          arq_tx_n_total_arq_frames: data["arq_tx_n_total_arq_frames"],
-          arq_rx_frame_n_bursts: data["arq_rx_frame_n_bursts"],
-          arq_rx_n_current_arq_frame: data["arq_rx_n_current_arq_frame"],
-          arq_n_arq_frames_per_data_frame:
-          data["arq_n_arq_frames_per_data_frame"],
-          arq_bytes_per_minute: data["arq_bytes_per_minute"],
-          arq_compression_factor: data["arq_compression_factor"],
-          routing_table: data["routing_table"],
-          mesh_signalling_table: data["mesh_signalling_table"],
-          listen: data["listen"],
-          //speed_table: [{"bpm" : 5200, "snr": -3, "timestamp":1673555399},{"bpm" : 2315, "snr": 12, "timestamp":1673555500}],
-        };
-        */
-        //continue to next for loop iteration, nothing else needs to be done here
-        continue;
-      }
-
-      // ----------- catch modem messages START -----------
-      //init message variable
-      var message = "";
-      if (data["freedata"] == "modem-message") {
-        // break early if we received a dummy callsign
-        // thats a kind of hotfix, as long as the modem isnt handling this better
-        if (
-          data["dxcallsign"] == "AA0AA-0" ||
-          data["dxcallsign"] == "ZZ9YY-0"
-        ) {
-          break;
-        }
-
-        console.log(data);
-
-        switch (data["fec"]) {
-          case "is_writing":
-            // RX'd FECiswriting
-            break;
-
-          case "broadcast":
-            // RX'd FEC BROADCAST
-            var encoded_data = atob_FD(data["data"]);
-            var splitted_data = encoded_data.split(split_char);
-            var messageArray = [];
-            if (splitted_data[0] == "m") {
-              messageArray.push(data);
-              console.log(data);
-            }
-            break;
-        }
-
-        switch (data["cq"]) {
-          case "transmitting":
-            // CQ TRANSMITTING
-            displayToast(
-              "success",
-              "bi-arrow-left-right",
-              "Transmitting CQ",
-              5000,
-            );
-            break;
-
-          case "received":
-            // CQ RECEIVED
-            message = "CQ from " + data["dxcallsign"];
-            displayToast("success", "bi-person-arms-up", message, 5000);
-            break;
-        }
-
-        switch (data["qrv"]) {
-          case "transmitting":
-            // QRV TRANSMITTING
-            displayToast(
-              "info",
-              "bi-person-raised-hand",
-              "Transmitting QRV ",
-              5000,
-            );
-            break;
-
-          case "received":
-            // QRV RECEIVED
-            message = "QRV from " + data["dxcallsign"] + " | " + data["dxgrid"];
-            displayToast("success", "bi-person-raised-hand", message, 5000);
-            break;
-        }
-
-        switch (data["beacon"]) {
-          case "transmitting":
-            // BEACON TRANSMITTING
-            displayToast(
-              "success",
-              "bi-broadcast-pin",
-              "Transmitting beacon",
-              5000,
-            );
-            break;
-
-          case "received":
-            // BEACON RECEIVED
-            newBeaconReceived(data);
-
-            message =
-              "Beacon from " + data["dxcallsign"] + " | " + data["dxgrid"];
-            displayToast("info", "bi-broadcast", message, 5000);
-            break;
-        }
-
-        switch (data["ping"]) {
-          case "transmitting":
-            // PING TRANSMITTING
-            message = "Sending ping to " + data["dxcallsign"];
-            displayToast("success", "bi-arrow-right", message, 5000);
-            break;
-
-          case "received":
-            // PING RECEIVED
-            message =
-              "Ping request from " +
-              data["dxcallsign"] +
-              " | " +
-              data["dxgrid"];
-            displayToast("success", "bi-arrow-right-short", message, 5000);
-            break;
-
-          case "acknowledge":
-            // PING ACKNOWLEDGE
-            message =
-              "Received ping-ack from " +
-              data["dxcallsign"] +
-              " | " +
-              data["dxgrid"];
-            displayToast("success", "bi-arrow-left-right", message, 5000);
-            break;
-        }
-
-        // ARQ SESSION && freedata == modem-message
-        if (data["arq"] == "session") {
-          switch (data["status"]) {
-            case "connecting":
-              // ARQ Open
-              break;
-
-            case "connected":
-              // ARQ Opening
-              break;
-
-            case "waiting":
-              // ARQ Opening
-              break;
-
-            case "close":
-              // ARQ Closing
-              break;
-
-            case "failed":
-              // ARQ Failed
-              break;
-          }
-        }
-        // ARQ TRANSMISSION && freedata == modem-message
-        if (data["arq"] == "transmission") {
-          switch (data["status"]) {
-            case "opened":
-              // ARQ Open
-              message = "ARQ session opened: " + data["dxcallsign"];
-              displayToast("success", "bi-arrow-left-right", message, 5000);
-              break;
-
-            case "opening":
-              // ARQ Opening IRS/ISS
-              if (data["irs"] == "False") {
-                message = "ARQ session opening: " + data["dxcallsign"];
-                displayToast("info", "bi-arrow-left-right", message, 5000);
-                break;
-              } else {
-                message = "ARQ sesson request from: " + data["dxcallsign"];
-                displayToast("success", "bi-arrow-left-right", message, 5000);
-                break;
-              }
-
-            case "waiting":
-              // ARQ waiting
-              message = "Channel busy | ARQ protocol is waiting";
-              displayToast("warning", "bi-hourglass-split", message, 5000);
-              break;
-
-            case "receiving":
-              // ARQ RX
-              break;
-
-            case "failed":
-              // ARQ TX Failed
-              if (data["reason"] == "protocol version missmatch") {
-                message = "Protocol version mismatch!";
-                displayToast("danger", "bi-chevron-bar-expand", message, 5000);
-                setStateFailed();
-
-                break;
-              } else {
-                message = "Transmission failed";
-                displayToast("danger", "bi-x-octagon", message, 5000);
-                updateTransmissionStatus(data);
-                setStateFailed();
-                break;
-              }
-              switch (data["irs"]) {
-                case "True":
-                  updateTransmissionStatus(data);
-
-                  break;
-                default:
-                  updateTransmissionStatus(data);
-                  break;
-              }
-              break;
-
-            case "received":
-              // ARQ data received
-
-              console.log(data);
-              // we need to encode here to do a deep check for checking if file or message
-              //var encoded_data = atob(data['data'])
-              var encoded_data = atob_FD(data["data"]);
-              var splitted_data = encoded_data.split(split_char);
-
-              // new message received
-              if (splitted_data[0] == "m") {
-                console.log(splitted_data);
-                newMessageReceived(splitted_data, data);
-              }
-
-              break;
-
-            case "transmitting":
-              // ARQ transmitting
-              updateTransmissionStatus(data);
-              break;
-
-            case "transmitted":
-              // ARQ transmitted
-              message = "Data transmitted";
-              displayToast("success", "bi-check-sqaure", message, 5000);
-              updateTransmissionStatus(data);
-              setStateSuccess();
-
-              break;
-          }
-        }
-      }
-    }
-
-    //finally delete message buffer
-    socketchunk = "";
-  }
-});
-
-// Send Ping
-//exports.sendPing = function (dxcallsign) {
-export function sendPing(dxcallsign) {
-  var command =
-    '{"type" : "ping", "command" : "ping", "dxcallsign" : "' +
-    dxcallsign +
-    '"}';
-  writeTncCommand(command);
-}
-
-// Send Mesh Ping
-//exports.sendMeshPing = function (dxcallsign) {
-function sendMeshPing(dxcallsign) {
-  var command =
-    '{"type" : "mesh", "command" : "ping", "dxcallsign" : "' +
-    dxcallsign +
-    '"}';
-  writeTncCommand(command);
-}
-
-// Send CQ
-//exports.sendCQ = function () {
-export function sendCQ() {
-  var command = '{"type" : "broadcast", "command" : "cqcqcq"}';
-  writeTncCommand(command);
-}
-
-// Set AUDIO Level
-export function setTxAudioLevel(value) {
-  var command =
-    '{"type" : "set", "command" : "tx_audio_level", "value" : "' + value + '"}';
-  writeTncCommand(command);
-}
-export function setRxAudioLevel(value) {
-  var command =
-    '{"type" : "set", "command" : "rx_audio_level", "value" : "' + value + '"}';
-  writeTncCommand(command);
-}
-
-// Send Message
-export function sendMessage(obj) {
-  let dxcallsign = obj.dxcallsign;
-  let checksum = obj.checksum;
-  let uuid = obj.uuid;
-  let command = obj.command;
-
-  let filename = Object.keys(obj._attachments)[0];
-  //let filetype = filename.split(".")[1]
-  let filetype = obj._attachments[filename].content_type;
-  let file = obj._attachments[filename].data;
-
-  //console.log(obj._attachments)
-  //console.log(filename)
-  //console.log(filetype)
-  //console.log(file)
-
-  let data_with_attachment =
-    obj.timestamp +
-    split_char +
-    obj.msg +
-    split_char +
-    filename +
-    split_char +
-    filetype +
-    split_char +
-    file;
-
-  let data = btoa_FD(
-    "m" +
-      split_char +
-      command +
-      split_char +
-      checksum +
-      split_char +
-      uuid +
-      split_char +
-      data_with_attachment,
-  );
-
-  // TODO: REMOVE mode and frames from Modem!
-  var mode = 255;
-  var frames = 5;
-
-  command =
-    '{"type" : "arq", "command" : "send_raw",  "uuid" : "' +
-    uuid +
-    '", "parameter" : [{"dxcallsign" : "' +
-    dxcallsign +
-    '", "mode" : "' +
-    mode +
-    '", "n_frames" : "' +
-    frames +
-    '", "data" : "' +
-    data +
-    '", "attempts": "10"}]}';
-  console.log(command);
-  writeTncCommand(command);
-}
-/*
-// Send Request message
-//It would be then „m + split + request + split + request-type“
-function sendRequest(dxcallsign, mode, frames, data, command) {
-  data = btoa_FD("m" + split_char + command + split_char + data);
-  command =
-    '{"type" : "arq", "command" : "send_raw", "parameter" : [{"dxcallsign" : "' +
-    dxcallsign +
-    '", "mode" : "' +
-    mode +
-    '", "n_frames" : "' +
-    frames +
-    '", "data" : "' +
-    data +
-    '", "attempts": "10"}]}';
-  console.log(command);
-  console.log("--------------REQ--------------------");
-  writeTncCommand(command);
-}
-
-// Send Response message
-//It would be then „m + split + request + split + request-type“
-function sendResponse(dxcallsign, mode, frames, data, command) {
-  data = btoa_FD("m" + split_char + command + split_char + data);
-  command =
-    '{"type" : "arq", "command" : "send_raw", "parameter" : [{"dxcallsign" : "' +
-    dxcallsign +
-    '", "mode" : "' +
-    mode +
-    '", "n_frames" : "' +
-    frames +
-    '", "data" : "' +
-    data +
-    '", "attempts": "10"}]}';
-  console.log(command);
-  console.log("--------------RES--------------------");
-  writeTncCommand(command);
-}
-
-//Send station info request
-//exports.sendRequestInfo = function (dxcallsign) {
-function sendRequestInfo(dxcallsign) {
-  //Command 0 = user/station information
-  //Command 1 = shared folder list
-  //Command 2 = shared file transfer
-  sendRequest(dxcallsign, 255, 1, "0", "req");
-}
-
-//Send shared folder file list request
-//exports.sendRequestSharedFolderList = function (dxcallsign) {
-function sendRequestSharedFolderList(dxcallsign) {
-  //Command 0 = user/station information
-  //Command 1 = shared folder list
-  //Command 2 = shared file transfer
-  sendRequest(dxcallsign, 255, 1, "1", "req");
-}
-
-//Send shared file request
-//exports.sendRequestSharedFile = function (dxcallsign, file) {
-function sendRequestSharedFile(dxcallsign, file) {
-  //Command 0 = user/station information
-  //Command 1 = shared folder list
-  //Command 2 = shared file transfer
-  sendRequest(dxcallsign, 255, 1, "2" + file, "req");
-}
-
-//Send station info response
-//exports.sendResponseInfo = function (dxcallsign, userinfo) {
-function sendResponseInfo(dxcallsign, userinfo) {
-  //Command 0 = user/station information
-  //Command 1 = shared folder list
-  //Command 2 = shared file transfer
-  sendResponse(dxcallsign, 255, 1, userinfo, "res-0");
-}
-
-//Send shared folder response
-//exports.sendResponseSharedFolderList = function (dxcallsign, sharedFolderList) {
-function sendResponseSharedFolderList(dxcallsign, sharedFolderList) {
-  //Command 0 = user/station information
-  //Command 1 = shared folder list
-  //Command 2 = shared file transfer
-  sendResponse(dxcallsign, 255, 1, sharedFolderList, "res-1");
-}
-
-//Send shared file response
-//exports.sendResponseSharedFile = function (
-function sendResponseSharedFile(dxcallsign, sharedFile, sharedFileData) {
-  console.log(
-    "In sendResponseSharedFile",
-    dxcallsign,
-    sharedFile,
-    sharedFileData,
-  );
-  //Command 0 = user/station information
-  //Command 1 = shared folder list
-  //Command 2 = shared file transfer
-  sendResponse(dxcallsign, 255, 1, sharedFile + "/" + sharedFileData, "res-2");
-}
-*/
-
-// Get RX BUffer
-export function getRxBuffer() {
-  var command = '{"type" : "get", "command" : "rx_buffer"}';
-
-  writeTncCommand(command);
-}
-
-// OPEN ARQ SESSION
-export function connectARQ(dxcallsign) {
-  var command =
-    '{"type" : "arq", "command" : "connect", "dxcallsign": "' +
-    dxcallsign +
-    '", "attempts": "10"}';
-  writeTncCommand(command);
-}
-
-// CLOSE ARQ SESSION
-export function disconnectARQ() {
-  var command = '{"type" : "arq", "command" : "disconnect"}';
-  writeTncCommand(command);
-}
-
-// SEND TEST FRAME
-export function sendTestFrame() {
-  var command = '{"type" : "set", "command" : "send_test_frame"}';
-  writeTncCommand(command);
-}
-
-// SEND FEC
-export function sendFEC(mode, payload) {
-  var command =
-    '{"type" : "fec", "command" : "transmit", "mode" : "' +
-    mode +
-    '", "payload" : "' +
-    payload +
-    '"}';
-  writeTncCommand(command);
-}
-
-// SEND FEC IS WRITING
-export function sendFecIsWriting(mycallsign) {
-  var command =
-    '{"type" : "fec", "command" : "transmit_is_writing", "mycallsign" : "' +
-    mycallsign +
-    '"}';
-  writeTncCommand(command);
-}
-
-// SEND FEC TO BROADCASTCHANNEL
-//export function sendBroadcastChannel(channel, data_out, uuid) {
-export function sendBroadcastChannel(obj) {
-  //let checksum = obj.checksum;
-  let command = obj.command;
-  let uuid = obj.uuid;
-  let channel = obj.dxcallsign;
-  let data_out = obj.msg;
-
-  let data = btoa_FD(
-    "m" +
-      split_char +
-      channel +
-      //split_char +
-      //checksum +
-      split_char +
-      uuid +
-      split_char +
-      data_out,
-  );
-  console.log(data.length);
-  let payload = data;
-  command =
-    '{"type" : "fec", "command" : "transmit", "mode": "datac4", "wakeup": "True", "payload" : "' +
-    payload +
-    '"}';
-  writeTncCommand(command);
-}
-
-// RECORD AUDIO
-export function record_audio() {
-  var command = '{"type" : "set", "command" : "record_audio"}';
-  writeTncCommand(command);
-}
-
-// SET FREQUENCY
-export function set_frequency(frequency) {
-  var command =
-    '{"type" : "set", "command" : "frequency", "frequency": ' + frequency + "}";
-  writeTncCommand(command);
-}
-
-// SET MODE
-export function set_mode(mode) {
-  var command = '{"type" : "set", "command" : "mode", "mode": "' + mode + '"}';
-  writeTncCommand(command);
-}
-
-// SET rf_level
-export function set_rf_level(rf_level) {
-  var command =
-    '{"type" : "set", "command" : "rf_level", "rf_level": "' + rf_level + '"}';
-  writeTncCommand(command);
-}
-
-// https://stackoverflow.com/a/50579690
-// crc32 calculation
-//console.log(crc32('abc'));
-//console.log(crc32('abc').toString(16).toUpperCase()); // hex
-/*
-function crc32(r) {
-  for (var a, o = [], c = 0; c < 256; c++) {
-    a = c;
-    for (var f = 0; f < 8; f++) a = 1 & a ? 3988292384 ^ (a >>> 1) : a >>> 1;
-    o[c] = a;
-  }
-  for (var n = -1, t = 0; t < r.length; t++)
-    n = (n >>> 8) ^ o[255 & (n ^ r.charCodeAt(t))];
-  return (-1 ^ n) >>> 0;
-};
-*/
-
-// TODO Maybe moving this to another module
-function prepareStatsDataForStore(data) {
-  // dummy data
-  //state.arq_speed_list = [{"snr":0.0,"bpm":104,"timestamp":1696189769},{"snr":0.0,"bpm":80,"timestamp":1696189778},{"snr":0.0,"bpm":70,"timestamp":1696189783},{"snr":0.0,"bpm":58,"timestamp":1696189792},{"snr":0.0,"bpm":52,"timestamp":1696189797},{"snr":"NaN","bpm":42,"timestamp":1696189811},{"snr":0.0,"bpm":22,"timestamp":1696189875},{"snr":0.0,"bpm":21,"timestamp":1696189881},{"snr":0.0,"bpm":17,"timestamp":1696189913},{"snr":0.0,"bpm":15,"timestamp":1696189932},{"snr":0.0,"bpm":15,"timestamp":1696189937},{"snr":0.0,"bpm":14,"timestamp":1696189946},{"snr":-6.1,"bpm":14,"timestamp":1696189954},{"snr":-6.1,"bpm":14,"timestamp":1696189955},{"snr":-5.5,"bpm":28,"timestamp":1696189963},{"snr":-5.5,"bpm":27,"timestamp":1696189963}]
-
-  var speed_listSize = 0;
-  if (typeof data == "undefined") {
-    speed_listSize = 0;
-  } else {
-    speed_listSize = data.length;
-  }
-
-  var speed_list_bpm = [];
-
-  for (let i = 0; i < speed_listSize; i++) {
-    speed_list_bpm.push(data[i].bpm);
-  }
-
-  var speed_list_timestamp = [];
-
-  for (let i = 0; i < speed_listSize; i++) {
-    let timestamp = data[i].timestamp * 1000;
-    let h = new Date(timestamp).getHours();
-    let m = new Date(timestamp).getMinutes();
-    let s = new Date(timestamp).getSeconds();
-    let time = h + ":" + m + ":" + s;
-    speed_list_timestamp.push(time);
-  }
-
-  var speed_list_snr = [];
-  for (let i = 0; i < speed_listSize; i++) {
-    let snr = NaN;
-    if (data[i].snr !== 0) {
-      snr = data[i].snr;
-    } else {
-      snr = NaN;
-    }
-    speed_list_snr.push(snr);
-  }
-
-  stateStore.arq_speed_list_bpm = speed_list_bpm;
-  stateStore.arq_speed_list_timestamp = speed_list_timestamp;
-  stateStore.arq_speed_list_snr = speed_list_snr;
-}

gui/src/js/eventHandler.js

@@ -1,3 +1,4 @@
+/*
 import {
   newMessageReceived,
   newBeaconReceived,
@@ -5,7 +6,16 @@ import {
   setStateSuccess,
   setStateFailed,
 } from "./chatHandler";
+*/
 import { displayToast } from "./popupHandler";
+import {
+  getFreedataMessages,
+  getConfig,
+  getAudioDevices,
+  getSerialDevices,
+  getModemState,
+} from "./api";
+import { processFreedataMessages } from "./messagesHandler.ts";
 
 // ----------------- init pinia stores -------------
 import { setActivePinia } from "pinia";
@@ -21,10 +31,13 @@ export function connectionFailed(endpoint, event) {
 }
 export function stateDispatcher(data) {
   data = JSON.parse(data);
+  //Leave commented when not needed, otherwise can lead to heap overflows due to the amount of data logged
   //console.debug(data);
   if (data["type"] == "state-change" || data["type"] == "state") {
     stateStore.modem_connection = "connected";
 
+    stateStore.busy_state = data["is_modem_busy"];
+
     stateStore.channel_busy = data["channel_busy"];
     stateStore.is_codec2_traffic = data["is_codec2_traffic"];
     stateStore.is_modem_running = data["is_modem_running"];
@@ -58,6 +71,14 @@ export function eventDispatcher(data) {
     return;
   }
 
+  switch (data["message-db"]) {
+    case "changed":
+      console.log("fetching new messages...");
+      var messages = getFreedataMessages();
+      processFreedataMessages(messages);
+      return;
+  }
+
   switch (data["ptt"]) {
     case true:
     case false:
@@ -70,14 +91,24 @@ export function eventDispatcher(data) {
   switch (data["modem"]) {
     case "started":
       displayToast("success", "bi-arrow-left-right", "Modem started", 5000);
+      getModemState();
+      getConfig();
+      getAudioDevices();
+      getSerialDevices();
+      getFreedataMessages();
       return;
 
     case "stopped":
-      displayToast("success", "bi-arrow-left-right", "Modem stopped", 5000);
+      displayToast("warning", "bi-arrow-left-right", "Modem stopped", 5000);
       return;
 
     case "restarted":
       displayToast("secondary", "bi-bootstrap-reboot", "Modem restarted", 5000);
+      getModemState();
+      getConfig();
+      getAudioDevices();
+      getSerialDevices();
+      getFreedataMessages();
       return;
 
     case "failed":
@@ -97,8 +128,28 @@ export function eventDispatcher(data) {
       message = "Connected to server";
       displayToast("success", "bi-ethernet", message, 5000);
       stateStore.modem_connection = "connected";
+      getModemState();
+      getOverallHealth();
+      getConfig();
+      getAudioDevices();
+      getSerialDevices();
+      getFreedataMessages();
+      processFreedataMessages();
+
       return;
 
+      switch (data["received"]) {
+        case "PING":
+          message = `Ping request from: ${data.dxcallsign}, SNR: ${data.snr}`;
+          displayToast("success", "bi-check-circle", message, 5000);
+          return;
+
+        case "PING_ACK":
+          message = `Ping acknowledged from: ${data.dxcallsign}, SNR: ${data.snr}`;
+          displayToast("success", "bi-check-circle", message, 5000);
+          return;
+      }
+
     case "arq":
       if (data["arq-transfer-outbound"]) {
         switch (data["arq-transfer-outbound"].state) {
@@ -268,3 +319,17 @@ function build_HSL() {
   }
   stateStore.heard_stations.sort((a, b) => b.timestamp - a.timestamp); // b - a for reverse sort
 }
+
+export function getOverallHealth() {
+  //Return a number indicating health for icon bg color; lower the number the healthier
+  let health = 0;
+  if (stateStore.modem_connection !== "connected") {
+    health += 5;
+    stateStore.is_modem_running = false;
+    stateStore.radio_status = false;
+  }
+  if (!stateStore.is_modem_running) health += 3;
+  if (stateStore.radio_status === false) health += 2;
+  if (process.env.FDUpdateAvail === "1") health += 1;
+  return health;
+}

gui/src/js/event_sock.js

@@ -19,7 +19,7 @@ function connect(endpoint, dispatcher) {
 
   // handle opening
   socket.addEventListener("open", function (event) {
-    console.log("Connected to the WebSocket server: " + endpoint);
+    //console.log("Connected to the WebSocket server: " + endpoint);
   });
 
   // handle data
@@ -30,18 +30,18 @@ function connect(endpoint, dispatcher) {
 
   // handle errors
   socket.addEventListener("error", function (event) {
-    console.error("WebSocket error:", event);
+    //console.error("WebSocket error:", event);
     connectionFailed(endpoint, event);
   });
 
   // handle closing and reconnect
   socket.addEventListener("close", function (event) {
-    console.log("WebSocket connection closed:", event.code);
+    //console.log("WebSocket connection closed:", event.code);
 
     // Reconnect handler
     if (!event.wasClean) {
       setTimeout(() => {
-        console.log("Reconnecting to websocket");
+        //console.log("Reconnecting to websocket");
         connect(endpoint, dispatcher);
       }, 1000);
     }

gui/src/js/freedata.ts

@@ -65,7 +65,7 @@ export function sortByPropertyDesc(property) {
  */
 export function validateCallsignWithSSID(callsign: string) {
   var patt = new RegExp("^[A-Z]+[0-9][A-Z]*-(1[0-5]|[0-9])$");
-
+  callsign = callsign;
   if (
     callsign === undefined ||
     callsign === "" ||
@@ -105,6 +105,12 @@ export function getAppDataPath() {
   const platform = os.platform();
   let appDataPath;
 
+  // Check if running in GitHub Actions
+  const isGitHubActions = process.env.GITHUB_ACTIONS === "true";
+  if (isGitHubActions) {
+    return "/home/runner/work/FreeDATA/FreeDATA/gui/config";
+  }
+
   switch (platform) {
     case "darwin": // macOS
       appDataPath = path.join(os.homedir(), "Library", "Application Support");

gui/src/js/messagesHandler.ts

@@ -0,0 +1,110 @@
+// pinia store setup
+import { setActivePinia } from "pinia";
+import pinia from "../store/index";
+setActivePinia(pinia);
+
+import { useChatStore } from "../store/chatStore.js";
+const chatStore = useChatStore(pinia);
+
+import {
+  sendFreedataMessage,
+  deleteFreedataMessage,
+  retransmitFreedataMessage,
+  getFreedataAttachmentBySha512,
+} from "./api";
+
+interface Message {
+  id: string;
+  timestamp: string;
+  origin: string;
+  destination: string;
+  direction: string;
+  body: string;
+  attachments: any[];
+  status: any;
+  statistics: any;
+}
+
+export async function processFreedataMessages(data) {
+  if (
+    typeof data !== "undefined" &&
+    typeof data.messages !== "undefined" &&
+    Array.isArray(data.messages)
+  ) {
+    chatStore.callsign_list = createCallsignListFromAPI(data);
+    chatStore.sorted_chat_list = createSortedMessagesList(data);
+  }
+}
+
+function createCallsignListFromAPI(data: {
+  total_messages: number;
+  messages: Message[];
+}): { [key: string]: { timestamp: string; body: string } } {
+  const callsignList: { [key: string]: { timestamp: string; body: string } } =
+    {};
+
+  data.messages.forEach((message) => {
+    let callsign =
+      message.direction === "receive" ? message.origin : message.destination;
+
+    if (
+      !callsignList[callsign] ||
+      callsignList[callsign].timestamp < message.timestamp
+    ) {
+      callsignList[callsign] = {
+        timestamp: message.timestamp,
+        body: message.body,
+      };
+    }
+  });
+
+  return callsignList;
+}
+
+function createSortedMessagesList(data: {
+  total_messages: number;
+  messages: Message[];
+}): { [key: string]: Message[] } {
+  const callsignMessages: { [key: string]: Message[] } = {};
+
+  data.messages.forEach((message) => {
+    let callsign =
+      message.direction === "receive" ? message.origin : message.destination;
+
+    if (!callsignMessages[callsign]) {
+      callsignMessages[callsign] = [];
+    }
+
+    callsignMessages[callsign].push(message);
+  });
+
+  return callsignMessages;
+}
+
+export function newMessage(dxcall, body, attachments) {
+  console.log(attachments);
+  sendFreedataMessage(dxcall, body, attachments);
+}
+
+/* ------ TEMPORARY DUMMY FUNCTIONS --- */
+export function repeatMessageTransmission(id) {
+  retransmitFreedataMessage(id);
+}
+
+export function deleteCallsignFromDB(callsign) {
+  for (var message of chatStore.sorted_chat_list[callsign]) {
+    deleteFreedataMessage(message["id"]);
+  }
+}
+
+export function deleteMessageFromDB(id) {
+  deleteFreedataMessage(id);
+}
+
+export function requestMessageInfo(id) {
+  return;
+}
+
+export async function getMessageAttachment(data_sha512) {
+  return await getFreedataAttachmentBySha512(data_sha512);
+}

gui/src/store/chatStore.js

@@ -2,6 +2,13 @@ import { defineStore } from "pinia";
 import { ref } from "vue";
 
 export const useChatStore = defineStore("chatStore", () => {
+  var callsign_list = ref();
+  var sorted_chat_list = ref();
+  var newChatCallsign = ref();
+  var newChatMessage = ref();
+
+  /* ------------------------------------------------ */
+
   var chat_filter = ref([
     { type: "newchat" },
     { type: "received" },
@@ -46,10 +53,6 @@ export const useChatStore = defineStore("chatStore", () => {
   var inputFileType = ref("-");
   var inputFileSize = ref("-");
 
-  var callsign_list = ref();
-  var sorted_chat_list = ref();
-  var unsorted_chat_list = ref([]);
-
   var sorted_beacon_list = ref({});
   var unsorted_beacon_list = ref({});
 
@@ -68,12 +71,13 @@ export const useChatStore = defineStore("chatStore", () => {
 
   return {
     selectedCallsign,
+    newChatCallsign,
+    newChatMessage,
     selectedMessageObject,
     inputText,
     chat_filter,
     callsign_list,
     sorted_chat_list,
-    unsorted_chat_list,
     inputFileName,
     inputFileSize,
     inputFileType,

gui/src/store/settingsStore.js

@@ -7,7 +7,15 @@ const nconf = require("nconf");
 
 var appDataPath = getAppDataPath();
 var configFolder = path.join(appDataPath, "FreeDATA");
-var configPath = path.join(configFolder, "config.json");
+let configFile = "config.json";
+
+const isGitHubActions = process.env.GITHUB_ACTIONS === "true";
+if (isGitHubActions) {
+  configFile = "example.json";
+  configFolder = appDataPath;
+}
+
+var configPath = path.join(configFolder, configFile);
 
 console.log("AppData Path:", appDataPath);
 console.log(configFolder);

gui/src/store/stateStore.js

@@ -3,7 +3,7 @@ import { ref } from "vue";
 import * as bootstrap from "bootstrap";
 
 export const useStateStore = defineStore("stateStore", () => {
-  var busy_state = ref("-");
+  var busy_state = ref();
   var arq_state = ref("-");
   var frequency = ref(0);
   var new_frequency = ref(14093000);

modem/api_validations.py

@@ -1,5 +1,16 @@
 import re
 
 def validate_freedata_callsign(callsign):
-    regexp = "^[a-zA-Z]+\d+\w+-\d{1,2}$"
+    #regexp = "^[a-zA-Z]+\d+\w+-\d{1,2}$"
+    regexp = "^[A-Za-z0-9]{1,7}-[0-9]{1,3}$" # still broken - we need to allow all ssids form 0 - 255
     return re.compile(regexp).match(callsign) is not None
+
+def validate_message_attachment(attachment):
+    for field in ['name', 'type', 'data']:
+        if field not in attachment:
+            raise ValueError(f"Attachment missing '{field}'")
+
+        # check for content length, except type
+        # there are some files out there, don't having a mime type
+        if len(attachment[field]) < 1 and field not in ["type"]:
+            raise ValueError(f"Attachment has empty '{field}'")

modem/arq_data_type_handler.py

@@ -0,0 +1,155 @@
+# File: arq_data_type_handler.py
+
+import structlog
+import lzma
+import gzip
+from message_p2p import message_received, message_failed, message_transmitted
+from enum import Enum
+
+class ARQ_SESSION_TYPES(Enum):
+    raw = 0
+    raw_lzma = 10
+    raw_gzip = 11
+    p2pmsg_lzma = 20
+
+class ARQDataTypeHandler:
+    def __init__(self, event_manager, state_manager):
+        self.logger = structlog.get_logger(type(self).__name__)
+        self.event_manager = event_manager
+        self.state_manager = state_manager
+
+        self.handlers = {
+            ARQ_SESSION_TYPES.raw: {
+                'prepare': self.prepare_raw,
+                'handle': self.handle_raw,
+                'failed': self.failed_raw,
+                'transmitted': self.transmitted_raw,
+            },
+            ARQ_SESSION_TYPES.raw_lzma: {
+                'prepare': self.prepare_raw_lzma,
+                'handle': self.handle_raw_lzma,
+                'failed': self.failed_raw_lzma,
+                'transmitted': self.transmitted_raw_lzma,
+            },
+            ARQ_SESSION_TYPES.raw_gzip: {
+                'prepare': self.prepare_raw_gzip,
+                'handle': self.handle_raw_gzip,
+                'failed': self.failed_raw_gzip,
+                'transmitted': self.transmitted_raw_gzip,
+            },
+            ARQ_SESSION_TYPES.p2pmsg_lzma: {
+                'prepare': self.prepare_p2pmsg_lzma,
+                'handle': self.handle_p2pmsg_lzma,
+                'failed' : self.failed_p2pmsg_lzma,
+                'transmitted': self.transmitted_p2pmsg_lzma,
+            },
+        }
+
+    @staticmethod
+    def get_session_type_from_value(value):
+        for session_type in ARQ_SESSION_TYPES:
+            if session_type.value == value:
+                return session_type
+        return None
+
+    def dispatch(self, type_byte: int, data: bytearray):
+        session_type = self.get_session_type_from_value(type_byte)
+        if session_type and session_type in self.handlers and 'handle' in self.handlers[session_type]:
+            return self.handlers[session_type]['handle'](data)
+        else:
+            self.log(f"Unknown handling endpoint for type: {type_byte}", isWarning=True)
+
+    def failed(self, type_byte: int, data: bytearray):
+        session_type = self.get_session_type_from_value(type_byte)
+        if session_type in self.handlers and 'failed' in self.handlers[session_type]:
+            return self.handlers[session_type]['failed'](data)
+        else:
+            self.log(f"Unknown handling endpoint: {session_type}", isWarning=True)
+
+    def prepare(self, data: bytearray, session_type=ARQ_SESSION_TYPES.raw):
+        if session_type in self.handlers and 'prepare' in self.handlers[session_type]:
+            return self.handlers[session_type]['prepare'](data), session_type.value
+        else:
+            self.log(f"Unknown preparation endpoint: {session_type}", isWarning=True)
+
+    def transmitted(self, type_byte: int, data: bytearray):
+        session_type = self.get_session_type_from_value(type_byte)
+        if session_type in self.handlers and 'transmitted' in self.handlers[session_type]:
+            return self.handlers[session_type]['transmitted'](data)
+        else:
+            self.log(f"Unknown handling endpoint: {session_type}", isWarning=True)
+
+    def log(self, message, isWarning=False):
+        msg = f"[{type(self).__name__}]: {message}"
+        logger = self.logger.warn if isWarning else self.logger.info
+        logger(msg)
+
+    def prepare_raw(self, data):
+        self.log(f"Preparing uncompressed data: {len(data)} Bytes")
+        return data
+
+    def handle_raw(self, data):
+        self.log(f"Handling uncompressed data: {len(data)} Bytes")
+        return data
+
+    def failed_raw(self, data):
+        return
+
+    def transmitted_raw(self, data):
+        return data
+
+    def prepare_raw_lzma(self, data):
+        compressed_data = lzma.compress(data)
+        self.log(f"Preparing LZMA compressed data: {len(data)} Bytes >>> {len(compressed_data)} Bytes")
+        return compressed_data
+
+    def handle_raw_lzma(self, data):
+        decompressed_data = lzma.decompress(data)
+        self.log(f"Handling LZMA compressed data: {len(decompressed_data)} Bytes from {len(data)} Bytes")
+        return decompressed_data
+
+    def failed_raw_lzma(self, data):
+        return
+
+    def transmitted_raw_lzma(self, data):
+        decompressed_data = lzma.decompress(data)
+        return decompressed_data
+
+    def prepare_raw_gzip(self, data):
+        compressed_data = gzip.compress(data)
+        self.log(f"Preparing GZIP compressed data: {len(data)} Bytes >>> {len(compressed_data)} Bytes")
+        return compressed_data
+
+    def handle_raw_gzip(self, data):
+        decompressed_data = gzip.decompress(data)
+        self.log(f"Handling GZIP compressed data: {len(decompressed_data)} Bytes from {len(data)} Bytes")
+        return decompressed_data
+
+    def failed_raw_gzip(self, data):
+        return
+
+    def transmitted_raw_gzip(self, data):
+        decompressed_data = gzip.decompress(data)
+        return decompressed_data
+
+    def prepare_p2pmsg_lzma(self, data):
+        compressed_data = lzma.compress(data)
+        self.log(f"Preparing LZMA compressed P2PMSG data: {len(data)} Bytes >>> {len(compressed_data)} Bytes")
+        return compressed_data
+
+    def handle_p2pmsg_lzma(self, data):
+        decompressed_data = lzma.decompress(data)
+        self.log(f"Handling LZMA compressed P2PMSG data: {len(decompressed_data)} Bytes from {len(data)} Bytes")
+        message_received(self.event_manager, self.state_manager, decompressed_data)
+        return decompressed_data
+
+    def failed_p2pmsg_lzma(self, data):
+        decompressed_data = lzma.decompress(data)
+        self.log(f"Handling failed LZMA compressed P2PMSG data: {len(decompressed_data)} Bytes from {len(data)} Bytes", isWarning=True)
+        message_failed(self.event_manager, self.state_manager, decompressed_data)
+        return decompressed_data
+
+    def transmitted_p2pmsg_lzma(self, data):
+        decompressed_data = lzma.decompress(data)
+        message_transmitted(self.event_manager, self.state_manager, decompressed_data)
+        return decompressed_data

modem/arq_session.py

@@ -5,6 +5,8 @@ import structlog
 from event_manager import EventManager
 from modem_frametypes import FRAME_TYPE
 import time
+from arq_data_type_handler import ARQDataTypeHandler
+
 
 class ARQSession():
 
@@ -31,6 +33,9 @@ class ARQSession():
         self.config = config
 
         self.event_manager: EventManager = modem.event_manager
+        self.states = modem.states
+
+        self.states.setARQ(True)
 
         self.snr = []
 
@@ -44,6 +49,7 @@ class ARQSession():
         self.frame_factory = data_frame_factory.DataFrameFactory(self.config)
         self.event_frame_received = threading.Event()
 
+        self.arq_data_type_handler = ARQDataTypeHandler(self.event_manager, self.states)
         self.id = None
         self.session_started = time.time()
         self.session_ended = 0
@@ -88,10 +94,14 @@ class ARQSession():
         if self.state in self.STATE_TRANSITION:
             if frame_type in self.STATE_TRANSITION[self.state]:
                 action_name = self.STATE_TRANSITION[self.state][frame_type]
-                getattr(self, action_name)(frame)
+                received_data, type_byte = getattr(self, action_name)(frame)
+                if isinstance(received_data, bytearray) and isinstance(type_byte, int):
+                    self.arq_data_type_handler.dispatch(type_byte, received_data)
+
+                self.states.setARQ(False)
                 return
         
-        self.log(f"Ignoring unknow transition from state {self.state.name} with frame {frame['frame_type']}")
+        self.log(f"Ignoring unknown transition from state {self.state.name} with frame {frame['frame_type']}")
 
     def is_session_outdated(self):
         session_alivetime = time.time() - self.session_max_age

modem/arq_session_irs.py

@@ -5,6 +5,7 @@ from modem_frametypes import FRAME_TYPE
 from codec2 import FREEDV_MODE
 from enum import Enum
 import time
+
 class IRS_State(Enum):
     NEW = 0
     OPEN_ACK_SENT = 1
@@ -68,6 +69,7 @@ class ARQSessionIRS(arq_session.ARQSession):
         self.state = IRS_State.NEW
         self.state_enum = IRS_State  # needed for access State enum from outside
 
+        self.type_byte = None
         self.total_length = 0
         self.total_crc = ''
         self.received_data = None
@@ -114,6 +116,7 @@ class ARQSessionIRS(arq_session.ARQSession):
         self.launch_transmit_and_wait(ack_frame, self.TIMEOUT_CONNECT, mode=FREEDV_MODE.signalling)
         if not self.abort:
             self.set_state(IRS_State.OPEN_ACK_SENT)
+        return None, None
 
     def send_info_ack(self, info_frame):
         # Get session info from ISS
@@ -121,6 +124,7 @@ class ARQSessionIRS(arq_session.ARQSession):
         self.total_length = info_frame['total_length']
         self.total_crc = info_frame['total_crc']
         self.dx_snr.append(info_frame['snr'])
+        self.type_byte = info_frame['type']
 
         self.log(f"New transfer of {self.total_length} bytes")
         self.event_manager.send_arq_session_new(False, self.id, self.dxcall, self.total_length, self.state.name)
@@ -134,7 +138,7 @@ class ARQSessionIRS(arq_session.ARQSession):
         self.launch_transmit_and_wait(info_ack, self.TIMEOUT_CONNECT, mode=FREEDV_MODE.signalling)
         if not self.abort:
             self.set_state(IRS_State.INFO_ACK_SENT)
-
+        return None, None
 
     def process_incoming_data(self, frame):
         if frame['offset'] != self.received_bytes:
@@ -174,7 +178,7 @@ class ARQSessionIRS(arq_session.ARQSession):
             # self.transmitted_acks += 1
             self.set_state(IRS_State.BURST_REPLY_SENT)
             self.launch_transmit_and_wait(ack, self.TIMEOUT_DATA, mode=FREEDV_MODE.signalling)
-            return
+            return None, None
 
         if self.final_crc_matches():
             self.log("All data received successfully!")
@@ -192,6 +196,7 @@ class ARQSessionIRS(arq_session.ARQSession):
             self.event_manager.send_arq_session_finished(
                 False, self.id, self.dxcall, True, self.state.name, data=self.received_data, statistics=self.calculate_session_statistics())
 
+            return self.received_data, self.type_byte
         else:
 
             ack = self.frame_factory.build_arq_burst_ack(self.id,
@@ -207,7 +212,7 @@ class ARQSessionIRS(arq_session.ARQSession):
             self.set_state(IRS_State.FAILED)
             self.event_manager.send_arq_session_finished(
                 False, self.id, self.dxcall, False, self.state.name, statistics=self.calculate_session_statistics())
-
+            return False, False
 
     def calibrate_speed_settings(self):
         self.speed_level = 0 # for now stay at lowest speed level
@@ -230,4 +235,5 @@ class ARQSessionIRS(arq_session.ARQSession):
         self.launch_transmit_and_wait(stop_ack, self.TIMEOUT_CONNECT, mode=FREEDV_MODE.signalling)
         self.set_state(IRS_State.ABORTED)
         self.event_manager.send_arq_session_finished(
-                False, self.id, self.dxcall, False, self.state.name, statistics=self.calculate_session_statistics())
+                False, self.id, self.dxcall, False, self.state.name, statistics=self.calculate_session_statistics())
+        return None, None

modem/arq_session_iss.py

@@ -53,13 +53,13 @@ class ARQSessionISS(arq_session.ARQSession):
         }
     }
 
-    def __init__(self, config: dict, modem, dxcall: str, data: bytearray, state_manager):
+    def __init__(self, config: dict, modem, dxcall: str, state_manager, data: bytearray, type_byte: bytes):
         super().__init__(config, modem, dxcall)
         self.state_manager = state_manager
         self.data = data
         self.total_length = len(data)
         self.data_crc = ''
-
+        self.type_byte = type_byte
         self.confirmed_bytes = 0
 
         self.state = ISS_State.NEW
@@ -119,11 +119,13 @@ class ARQSessionISS(arq_session.ARQSession):
 
         info_frame = self.frame_factory.build_arq_session_info(self.id, self.total_length,
                                                                helpers.get_crc_32(self.data), 
-                                                               self.snr[0])
+                                                               self.snr[0], self.type_byte)
 
         self.launch_twr(info_frame, self.TIMEOUT_CONNECT_ACK, self.RETRIES_CONNECT, mode=FREEDV_MODE.signalling)
         self.set_state(ISS_State.INFO_SENT)
 
+        return None, None
+
     def send_data(self, irs_frame):
 
         self.set_speed_and_frames_per_burst(irs_frame)
@@ -137,15 +139,15 @@ class ARQSessionISS(arq_session.ARQSession):
         # check if we received an abort flag
         if irs_frame["flag"]["ABORT"]:
             self.transmission_aborted(irs_frame)
-            return
+            return None, None
 
         if irs_frame["flag"]["FINAL"]:
             if self.confirmed_bytes == self.total_length and irs_frame["flag"]["CHECKSUM"]:
                 self.transmission_ended(irs_frame)
-                return
+
             else:
                 self.transmission_failed()
-                return
+            return None, None
 
         payload_size = self.get_data_payload_size()
         burst = []
@@ -158,6 +160,7 @@ class ARQSessionISS(arq_session.ARQSession):
             burst.append(data_frame)
         self.launch_twr(burst, self.TIMEOUT_TRANSFER, self.RETRIES_CONNECT, mode='auto')
         self.set_state(ISS_State.BURST_SENT)
+        return None, None
 
     def transmission_ended(self, irs_frame):
         # final function for sucessfully ended transmissions
@@ -166,6 +169,9 @@ class ARQSessionISS(arq_session.ARQSession):
         self.log(f"All data transfered! flag_final={irs_frame['flag']['FINAL']}, flag_checksum={irs_frame['flag']['CHECKSUM']}")
         self.event_manager.send_arq_session_finished(True, self.id, self.dxcall,True, self.state.name, statistics=self.calculate_session_statistics())
         self.state_manager.remove_arq_iss_session(self.id)
+        self.states.setARQ(False)
+        self.arq_data_type_handler.transmitted(self.type_byte, self.data)
+        return None, None
 
     def transmission_failed(self, irs_frame=None):
         # final function for failed transmissions
@@ -173,6 +179,11 @@ class ARQSessionISS(arq_session.ARQSession):
         self.set_state(ISS_State.FAILED)
         self.log(f"Transmission failed!")
         self.event_manager.send_arq_session_finished(True, self.id, self.dxcall,False, self.state.name, statistics=self.calculate_session_statistics())
+        self.states.setARQ(False)
+
+        self.arq_data_type_handler.failed(self.type_byte, self.data)
+
+        return None, None
 
     def abort_transmission(self, irs_frame=None):
         # function for starting the abort sequence
@@ -202,4 +213,6 @@ class ARQSessionISS(arq_session.ARQSession):
         self.event_manager.send_arq_session_finished(
             True, self.id, self.dxcall, False, self.state.name, statistics=self.calculate_session_statistics())
         self.state_manager.remove_arq_iss_session(self.id)
+        self.states.setARQ(False)
+        return None, None
 

modem/beacon.py

@@ -1,41 +0,0 @@
-import command_beacon
-import sched
-import time
-import threading
-
-class Beacon:
-    def __init__(self, config, states, event_manager, logger, modem):
-        self.config = config
-        self.states = states
-        self.event_manager = event_manager
-        self.log = logger
-        self.modem = modem
-
-        self.scheduler = sched.scheduler(time.time, time.sleep)
-        self.beacon_interval = self.config['MODEM']['beacon_interval']
-        self.beacon_enabled = False
-        self.event = threading.Event()
-
-    def start(self):
-        self.beacon_enabled = True
-        self.schedule_beacon()
-
-    def stop(self):
-        self.beacon_enabled = False
-
-    def schedule_beacon(self):
-        if self.beacon_enabled:
-            self.scheduler.enter(self.beacon_interval, 1, self.run_beacon)
-            threading.Thread(target=self.scheduler.run, daemon=True).start()
-
-    def run_beacon(self):
-        if self.beacon_enabled:
-            # Your beacon logic here
-            cmd = command_beacon.BeaconCommand(self.config, self.states, self.event_manager)
-            cmd.run(self.event_manager, self.modem)
-            self.schedule_beacon()  # Reschedule the next beacon
-
-    def refresh(self):
-        # Interrupt and reschedule the beacon
-        self.scheduler = sched.scheduler(time.time, time.sleep)
-        self.schedule_beacon()

modem/callsign_reverse_db.txt

@@ -1 +0,0 @@
-{"DJ2LS-0": "22864b", "IW2DHW-0": "fcef94", "BEACON": "5492c7", "EI7IG-0": "0975c8", "G0HWW-0": "2cb363", "LA3QMA-0": "2b9fac", "EA7KOH-0": "9e1c3e", "OK6MS-0": "5f75ed", "N1QM-0": "e045a0"}

modem/command.py

@@ -3,16 +3,24 @@ import queue
 from codec2 import FREEDV_MODE
 import structlog
 from state_manager import StateManager
+from arq_data_type_handler import ARQDataTypeHandler
+
 
 class TxCommand():
 
     def __init__(self, config: dict, state_manager: StateManager, event_manager, apiParams:dict = {}):
         self.config = config
-        self.logger = structlog.get_logger("Command")
+        self.logger = structlog.get_logger(type(self).__name__)
         self.state_manager = state_manager
         self.event_manager = event_manager
         self.set_params_from_api(apiParams)
         self.frame_factory = DataFrameFactory(config)
+        self.arq_data_type_handler = ARQDataTypeHandler(event_manager, state_manager)
+
+    def log(self, message, isWarning = False):
+        msg = f"[{type(self).__name__}]: {message}"
+        logger = self.logger.warn if isWarning else self.logger.info
+        logger(msg)
 
     def set_params_from_api(self, apiParams):
         pass

modem/command_arq_raw.py

@@ -4,6 +4,7 @@ import api_validations
 import base64
 from queue import Queue
 from arq_session_iss import ARQSessionISS
+from arq_data_type_handler import ARQ_SESSION_TYPES
 
 
 class ARQRawCommand(TxCommand):
@@ -13,13 +14,20 @@ class ARQRawCommand(TxCommand):
         if not api_validations.validate_freedata_callsign(self.dxcall):
             self.dxcall = f"{self.dxcall}-0"
 
+        try:
+            self.type = ARQ_SESSION_TYPES[apiParams['type']]
+        except KeyError:
+            self.type = ARQ_SESSION_TYPES.raw
+
         self.data = base64.b64decode(apiParams['data'])
 
     def run(self, event_queue: Queue, modem):
         self.emit_event(event_queue)
         self.logger.info(self.log_message())
 
-        iss = ARQSessionISS(self.config, modem, self.dxcall, self.data, self.state_manager)
+        prepared_data, type_byte = self.arq_data_type_handler.prepare(self.data, self.type)
+
+        iss = ARQSessionISS(self.config, modem, self.dxcall, self.state_manager, prepared_data, type_byte)
         if iss.id:
             self.state_manager.register_arq_iss_session(iss)
             iss.start()

modem/command_message_send.py

@@ -0,0 +1,50 @@
+from command import TxCommand
+import api_validations
+import base64
+from queue import Queue
+from arq_session_iss import ARQSessionISS
+from message_p2p import MessageP2P
+from arq_data_type_handler import ARQ_SESSION_TYPES
+from message_system_db_manager import DatabaseManager
+from message_system_db_messages import DatabaseManagerMessages
+
+class SendMessageCommand(TxCommand):
+    """Command to send a P2P message using an ARQ transfer session
+    """
+
+    def set_params_from_api(self, apiParams):
+        origin = f"{self.config['STATION']['mycall']}-{self.config['STATION']['myssid']}"
+        self.message = MessageP2P.from_api_params(origin, apiParams)
+        DatabaseManagerMessages(self.event_manager).add_message(self.message.to_dict(), direction='transmit', status='queued')
+
+    def transmit(self, modem):
+
+        if self.state_manager.getARQ():
+            self.log("Modem busy, waiting until ready...")
+            return
+
+        first_queued_message = DatabaseManagerMessages(self.event_manager).get_first_queued_message()
+        if not first_queued_message:
+            self.log("No queued message in database.")
+            return
+
+        self.log(f"Queued message found: {first_queued_message['id']}")
+        DatabaseManagerMessages(self.event_manager).update_message(first_queued_message["id"], update_data={'status': 'transmitting'})
+        message_dict = DatabaseManagerMessages(self.event_manager).get_message_by_id(first_queued_message["id"])
+        message = MessageP2P.from_api_params(message_dict['origin'], message_dict)
+
+        # Convert JSON string to bytes (using UTF-8 encoding)
+        payload = message.to_payload().encode('utf-8')
+        json_bytearray = bytearray(payload)
+        data, data_type = self.arq_data_type_handler.prepare(json_bytearray, ARQ_SESSION_TYPES.p2pmsg_lzma)
+
+        iss = ARQSessionISS(self.config,
+                            modem,
+                            self.message.destination,
+                            self.state_manager,
+                            data,
+                            data_type
+                            )
+        
+        self.state_manager.register_arq_iss_session(iss)
+        iss.start()

modem/command_ping.py

@@ -1,5 +1,7 @@
 from command import TxCommand
 import api_validations
+from message_system_db_manager import DatabaseManager
+
 
 class PingCommand(TxCommand):
 
@@ -8,6 +10,9 @@ class PingCommand(TxCommand):
         if not api_validations.validate_freedata_callsign(self.dxcall):
             self.dxcall = f"{self.dxcall}-0"
 
+        # update callsign database...
+        DatabaseManager(self.event_manager).get_or_create_station(self.dxcall)
+
         return super().set_params_from_api(apiParams)
 
     def build_frame(self):

modem/config.ini.example

@@ -5,7 +5,7 @@ modemport = 3050
 mycall = XX1XXX
 mygrid = JN18cv
 myssid = 6
-ssid_list = [1, 7]
+ssid_list = [0, 1, 2, 3, 4, 5, 6, 7, 8, 9]
 enable_explorer = True
 enable_stats = True
 

modem/data_frame_factory.py

@@ -15,7 +15,6 @@ class DataFrameFactory:
         'FINAL': 0,  # Bit-position for indicating the FINAL state
         'ABORT': 1, # Bit-position for indicating the ABORT request
         'CHECKSUM': 2,  # Bit-position for indicating the CHECKSUM is correct or not
-        'ENABLE_COMPRESSION': 3  # Bit-position for indicating compression is enabled
     }
 
     def __init__(self, config):
@@ -118,6 +117,7 @@ class DataFrameFactory:
             "total_crc": 4,
             "snr": 1,
             "flag": 1,
+            "type": 1,
         }
 
         self.template_list[FR_TYPE.ARQ_SESSION_INFO_ACK.value] = {
@@ -218,7 +218,7 @@ class DataFrameFactory:
 
             elif key in ["session_id", "speed_level", 
                             "frames_per_burst", "version",
-                            "offset", "total_length", "state"]:
+                            "offset", "total_length", "state", "type"]:
                 extracted_data[key] = int.from_bytes(data, 'big')
 
             elif key in ["snr"]:
@@ -350,10 +350,8 @@ class DataFrameFactory:
         }
         return self.construct(FR_TYPE.ARQ_SESSION_OPEN_ACK, payload)
     
-    def build_arq_session_info(self, session_id: int, total_length: int, total_crc: bytes, snr, flag_compression=False):
+    def build_arq_session_info(self, session_id: int, total_length: int, total_crc: bytes, snr, type):
         flag = 0b00000000
-        if flag_compression:
-            flag = helpers.set_flag(flag, 'ENABLE_COMPRESSION', True, self.ARQ_FLAGS)
 
         payload = {
             "session_id": session_id.to_bytes(1, 'big'),
@@ -361,6 +359,7 @@ class DataFrameFactory:
             "total_crc": total_crc,
             "snr": helpers.snr_to_bytes(1),
             "flag": flag.to_bytes(1, 'big'),
+            "type": type.to_bytes(1, 'big'),
 
         }
         return self.construct(FR_TYPE.ARQ_SESSION_INFO, payload)
@@ -377,7 +376,6 @@ class DataFrameFactory:
         }
         return self.construct(FR_TYPE.ARQ_STOP_ACK, payload)
 
-
     def build_arq_session_info_ack(self, session_id, total_crc, snr, speed_level, frames_per_burst, flag_final=False, flag_abort=False):
         flag = 0b00000000
         if flag_final:

modem/event_manager.py

@@ -89,4 +89,7 @@ class EventManager:
 
     def modem_failed(self):
         event = {"modem": "failed"}
-        self.broadcast(event)
+        self.broadcast(event)
+
+    def freedata_message_db_change(self):
+        self.broadcast({"message-db": "changed"})

modem/explorer.py

@@ -17,34 +17,33 @@ import time
 log = structlog.get_logger("explorer")
 
 class explorer():
-    def __init__(self, app, config, states):
-        self.config = config
-        self.app = app
+    def __init__(self, modem_version, config_manager, states):
+        self.modem_version = modem_version
+
+        self.config_manager = config_manager
+        self.config = self.config_manager.read()
         self.states = states
         self.explorer_url = "https://api.freedata.app/explorer.php"
         self.publish_interval = 120
 
-        self.scheduler = sched.scheduler(time.time, time.sleep)
-        self.schedule_thread = threading.Thread(target=self.run_scheduler)
-        self.schedule_thread.start()
-
-    def run_scheduler(self):
-        # Schedule the first execution of push
-        self.scheduler.enter(self.publish_interval, 1, self.push)
-        # Run the scheduler in a loop
-        self.scheduler.run()
-
     def push(self):
+        self.config = self.config_manager.read()
 
         frequency = 0 if self.states.radio_frequency is None else self.states.radio_frequency
         band = "USB"
         callsign = str(self.config['STATION']['mycall']) + "-" + str(self.config["STATION"]['myssid'])
         gridsquare = str(self.config['STATION']['mygrid'])
-        version = str(self.app.MODEM_VERSION)
+        version = str(self.modem_version)
         bandwidth = str(self.config['MODEM']['enable_low_bandwidth_mode'])
         beacon = str(self.states.is_beacon_running)
         strength = str(self.states.s_meter_strength)
 
+        # stop pushing if default callsign
+        if callsign in ['XX1XXX-6']:
+            # Reschedule the push method
+            self.scheduler.enter(self.publish_interval, 1, self.push)
+            return
+
         log.info("[EXPLORER] publish", frequency=frequency, band=band, callsign=callsign, gridsquare=gridsquare, version=version, bandwidth=bandwidth)
 
         headers = {"Content-Type": "application/json"}
@@ -70,11 +69,3 @@ class explorer():
 
         except Exception as e:
             log.warning("[EXPLORER] connection lost")
-
-        # Reschedule the push method
-        self.scheduler.enter(self.publish_interval, 1, self.push)
-
-        def shutdown(self):
-            # If there are other cleanup tasks, include them here
-            if self.schedule_thread:
-                self.schedule_thread.join()

modem/frame_dispatcher.py

@@ -13,6 +13,7 @@ from frame_handler import FrameHandler
 from frame_handler_ping import PingFrameHandler
 from frame_handler_cq import CQFrameHandler
 from frame_handler_arq_session import ARQFrameHandler
+from frame_handler_beacon import BeaconFrameHandler
 
 class DISPATCHER():
 
@@ -26,7 +27,7 @@ class DISPATCHER():
         FR_TYPE.ARQ_CONNECTION_OPEN.value: {"class": ARQFrameHandler, "name": "ARQ OPEN SESSION"},
         FR_TYPE.ARQ_STOP.value: {"class": ARQFrameHandler, "name": "ARQ STOP"},
         FR_TYPE.ARQ_STOP_ACK.value: {"class": ARQFrameHandler, "name": "ARQ STOP ACK"},
-        FR_TYPE.BEACON.value: {"class": FrameHandler, "name": "BEACON"},
+        FR_TYPE.BEACON.value: {"class": BeaconFrameHandler, "name": "BEACON"},
         FR_TYPE.ARQ_BURST_FRAME.value:{"class": ARQFrameHandler, "name": "BURST FRAME"},
         FR_TYPE.ARQ_BURST_ACK.value: {"class": ARQFrameHandler, "name":  "BURST ACK"},
         FR_TYPE.CQ.value: {"class": CQFrameHandler, "name":  "CQ"},

modem/frame_handler.py

@@ -5,6 +5,7 @@ from queue import Queue
 import structlog
 import time, uuid
 from codec2 import FREEDV_MODE
+from message_system_db_manager import DatabaseManager
 
 TESTMODE = False
 
@@ -31,7 +32,6 @@ class FrameHandler():
     def is_frame_for_me(self):
         call_with_ssid = self.config['STATION']['mycall'] + "-" + str(self.config['STATION']['myssid'])
         ft = self.details['frame']['frame_type']
-        print(self.details)
         valid = False
         # Check for callsign checksum
         if ft in ['ARQ_SESSION_OPEN', 'ARQ_SESSION_OPEN_ACK', 'PING', 'PING_ACK']:
@@ -91,6 +91,7 @@ class FrameHandler():
     def add_to_heard_stations(self):
         frame = self.details['frame']
 
+        print(frame)
         if 'origin' not in frame:
             return
 
@@ -106,6 +107,7 @@ class FrameHandler():
         )
 
     def make_event(self):
+
         event = {
             "type": "frame-handler",
             "received": self.details['frame']['frame_type'],
@@ -116,6 +118,7 @@ class FrameHandler():
         }
         if 'origin' in self.details['frame']:
             event['dxcallsign'] = self.details['frame']['origin']
+
         return event
 
     def emit_event(self):
@@ -144,6 +147,10 @@ class FrameHandler():
         self.details['freedv_inst'] = freedv_inst
         self.details['bytes_per_frame'] = bytes_per_frame
 
+        # look in database for a full callsign if only crc is present
+        if 'origin' not in frame and 'origin_crc' in frame:
+            self.details['frame']['origin'] = DatabaseManager(self.event_manager).get_callsign_by_checksum(frame['origin_crc'])
+
         self.log()
         self.add_to_heard_stations()
         self.add_to_activity_list()

modem/frame_handler_beacon.py

@@ -0,0 +1,17 @@
+import frame_handler_ping
+import helpers
+import data_frame_factory
+import frame_handler
+import datetime
+from message_system_db_beacon import DatabaseManagerBeacon
+
+
+from message_system_db_manager import DatabaseManager
+class BeaconFrameHandler(frame_handler.FrameHandler):
+
+    def follow_protocol(self):
+        DatabaseManagerBeacon(self.event_manager).add_beacon(datetime.datetime.now(),
+                                                             self.details['frame']["origin"],
+                                                             self.details["snr"],
+                                                             self.details['frame']["gridsquare"]
+                                                             )

modem/message_p2p.py

@@ -0,0 +1,99 @@
+import datetime
+import api_validations
+import base64
+import json
+from message_system_db_manager import DatabaseManager
+from message_system_db_messages import DatabaseManagerMessages
+#import command_message_send
+
+
+def message_received(event_manager, state_manager, data):
+    decompressed_json_string = data.decode('utf-8')
+    received_message_obj = MessageP2P.from_payload(decompressed_json_string)
+    received_message_dict = MessageP2P.to_dict(received_message_obj)
+    DatabaseManagerMessages(event_manager).add_message(received_message_dict, direction='receive', status='received', is_read=False)
+
+def message_transmitted(event_manager, state_manager, data):
+    decompressed_json_string = data.decode('utf-8')
+    payload_message_obj = MessageP2P.from_payload(decompressed_json_string)
+    payload_message = MessageP2P.to_dict(payload_message_obj)
+    DatabaseManagerMessages(event_manager).update_message(payload_message["id"], update_data={'status': 'transmitted'})
+
+def message_failed(event_manager, state_manager, data):
+    decompressed_json_string = data.decode('utf-8')
+    payload_message_obj = MessageP2P.from_payload(decompressed_json_string)
+    payload_message = MessageP2P.to_dict(payload_message_obj)
+    DatabaseManagerMessages(event_manager).update_message(payload_message["id"], update_data={'status': 'failed'})
+
+class MessageP2P:
+    def __init__(self, id: str, origin: str, destination: str, body: str, attachments: list) -> None:
+        self.id = id
+        self.timestamp = datetime.datetime.now().isoformat()
+        self.origin = origin
+        self.destination = destination
+        self.body = body
+        self.attachments = attachments
+
+    @classmethod
+    def from_api_params(cls, origin: str, params: dict):
+
+        destination = params['destination']
+        if not api_validations.validate_freedata_callsign(destination):
+            destination = f"{destination}-0"
+
+        if not api_validations.validate_freedata_callsign(destination):
+            raise ValueError(f"Invalid destination given ({params['destination']})")
+
+        body = params['body']
+
+        attachments = []
+        if 'attachments' in params: 
+            for a in params['attachments']:
+                api_validations.validate_message_attachment(a)
+                attachments.append(cls.__decode_attachment__(a))
+
+        timestamp = datetime.datetime.now().isoformat()
+        if 'id' not in params:
+            msg_id = f"{origin}_{destination}_{timestamp}"
+        else:
+            msg_id = params["id"]
+
+        return cls(msg_id, origin, destination, body, attachments)
+        
+    @classmethod
+    def from_payload(cls, payload):
+        payload_message = json.loads(payload)
+        attachments = list(map(cls.__decode_attachment__, payload_message['attachments']))
+        return cls(payload_message['id'], payload_message['origin'], payload_message['destination'],
+                   payload_message['body'], attachments)
+
+    def get_id(self) -> str:
+        return f"{self.origin}_{self.destination}_{self.timestamp}"
+
+    def __encode_attachment__(self, binary_attachment: dict):
+        encoded_attachment = binary_attachment.copy()
+        encoded_attachment['data'] = str(base64.b64encode(binary_attachment['data']), 'utf-8')
+        return encoded_attachment
+    
+    def __decode_attachment__(encoded_attachment: dict):
+        decoded_attachment = encoded_attachment.copy()
+        decoded_attachment['data'] = base64.b64decode(encoded_attachment['data'])
+        return decoded_attachment
+
+    def to_dict(self):
+        """Make a dictionary out of the message data
+        """
+
+        return {
+            'id': self.id,
+            'origin': self.origin,
+            'destination': self.destination,
+            'body': self.body,
+            'attachments': list(map(self.__encode_attachment__, self.attachments)),
+        }
+    
+    def to_payload(self):
+        """Make a byte array ready to be sent out of the message data"""
+        json_string = json.dumps(self.to_dict())
+        return json_string
+

modem/message_system_db_attachments.py

@@ -0,0 +1,73 @@
+from message_system_db_manager import DatabaseManager
+from message_system_db_model import MessageAttachment, Attachment, P2PMessage
+import json
+import hashlib
+
+
+
+class DatabaseManagerAttachments(DatabaseManager):
+    def __init__(self, uri='sqlite:///freedata-messages.db'):
+        super().__init__(uri)
+
+
+    def add_attachment(self, session, message, attachment_data):
+        hash_sha512 = hashlib.sha512(attachment_data['data'].encode()).hexdigest()
+        existing_attachment = session.query(Attachment).filter_by(hash_sha512=hash_sha512).first()
+
+        if not existing_attachment:
+            attachment = Attachment(
+                name=attachment_data['name'],
+                data_type=attachment_data['type'],
+                data=attachment_data['data'],
+                checksum_crc32=attachment_data.get('checksum_crc32', ''),
+                hash_sha512=hash_sha512
+            )
+            session.add(attachment)
+            session.flush()  # Ensure the attachment is persisted and has an ID
+            self.log(f"Added attachment to database: {attachment.name}")
+        else:
+            attachment = existing_attachment
+            self.log(f"Attachment {attachment.name} already exists in database")
+
+        # Link the message and the attachment through MessageAttachment
+        link = MessageAttachment(message=message, attachment=attachment)
+        session.add(link)
+        self.log(f"Linked message with attachment: {attachment.name}")
+
+        return attachment
+
+    def get_attachments_by_message_id(self, message_id):
+        session = self.get_thread_scoped_session()
+        try:
+            # Fetch the message by its ID
+            message = session.query(P2PMessage).filter_by(id=message_id).first()
+            if message:
+                # Navigate through the association to get attachments
+                attachments = [ma.attachment.to_dict() for ma in message.message_attachments]
+                return attachments
+            else:
+                return []
+        except Exception as e:
+            self.log(f"Error fetching attachments for message ID {message_id}: {e}", isWarning=True)
+            return []
+        finally:
+            session.remove()
+
+    def get_attachments_by_message_id_json(self, message_id):
+        attachments = self.get_attachments_by_message_id(message_id)
+        return json.dumps(attachments)
+
+    def get_attachment_by_sha512(self, hash_sha512):
+        session = self.get_thread_scoped_session()
+        try:
+            attachment = session.query(Attachment).filter_by(hash_sha512=hash_sha512).first()
+            if attachment:
+                return attachment.to_dict()
+            else:
+                self.log(f"No attachment found with SHA-512 hash: {hash_sha512}")
+                return None
+        except Exception as e:
+            self.log(f"Error fetching attachment with SHA-512 hash {hash_sha512}: {e}", isWarning=True)
+            return None
+        finally:
+            session.remove()

modem/message_system_db_beacon.py

@@ -0,0 +1,124 @@
+from message_system_db_manager import DatabaseManager
+from sqlalchemy import create_engine
+from sqlalchemy.orm import scoped_session, sessionmaker
+from threading import local
+from message_system_db_model import Base, Beacon, Station, Status, Attachment, P2PMessage
+from datetime import timezone, timedelta, datetime
+import json
+import structlog
+import helpers
+
+
+class DatabaseManagerBeacon(DatabaseManager):
+    def __init__(self, uri):
+        super().__init__(uri)
+
+    def add_beacon(self, timestamp, callsign, snr, gridsquare):
+        session = None
+        try:
+            session = self.get_thread_scoped_session()
+            # Ensure the station exists or create it, and get the station object
+            station = self.get_or_create_station(callsign, session)
+            # Initialize location as an empty dict if None
+            if not station.location:
+                station.location = {}
+
+            # Now, check and update the station's location with gridsquare if it has changed
+            if station.location.get('gridsquare') != gridsquare:
+                self.log(f"Updating location for {callsign}")
+                station.location['gridsquare'] = gridsquare  # Update directly without re-serialization
+                session.flush()
+
+            # Now, add the beacon
+            new_beacon = Beacon(timestamp=timestamp, snr=snr, callsign=callsign)
+            session.add(new_beacon)
+            session.commit()
+            self.log(f"Added beacon for {callsign}")
+
+        except Exception as e:
+            self.log(f"An error occurred while adding beacon for {callsign}: {e}", isWarning=True)
+            if session:
+                session.rollback()
+        finally:
+            if session and not session.is_active:
+                session.remove()
+
+    def get_beacons_by_callsign(self, callsign):
+        session = self.get_thread_scoped_session()
+        try:
+            # Query the station by callsign
+            station = session.query(Station).filter_by(callsign=callsign).first()
+            if station:
+                # Access the beacons directly thanks to the back_populated relationship
+                beacons = station.beacons
+                # Convert beacon objects to a dictionary if needed, or directly return the list
+                beacons_data = [{
+                    'id': beacon.id,
+                    'timestamp': beacon.timestamp.isoformat(),
+                    'snr': beacon.snr,
+                    'gridsquare': station.location.get('gridsquare') if station.location else None
+                } for beacon in beacons]
+                return beacons_data
+            else:
+                self.log(f"No station found with callsign {callsign}")
+                return []
+        except Exception as e:
+            self.log(f"An error occurred while fetching beacons for callsign {callsign}: {e}", isWarning=True)
+            return []
+        finally:
+            session.remove()
+
+    def get_all_beacons(self):
+        session = None
+        try:
+            session = self.get_thread_scoped_session()
+            # Query all beacons
+            beacons_query = session.query(Beacon).all()
+
+            # Optionally format the result for easier consumption
+            beacons_list = []
+            for beacon in beacons_query:
+                # Fetch the associated station for each beacon to get the 'gridsquare' information
+                station = session.query(Station).filter_by(callsign=beacon.callsign).first()
+                gridsquare = station.location.get('gridsquare') if station and station.location else None
+
+                beacons_list.append({
+                    'id': beacon.id,
+                    'timestamp': beacon.timestamp.isoformat(),
+                    'snr': beacon.snr,
+                    'callsign': beacon.callsign,
+                    'gridsquare': gridsquare
+                })
+
+            return beacons_list
+
+        except Exception as e:
+            self.log(f"An error occurred while fetching all beacons: {e}", isWarning=True)
+            return []  # Return an empty list or handle the error as appropriate
+        finally:
+            if session and not session.is_active:
+                session.remove()
+
+    def beacon_cleanup_older_than_days(self, days):
+        session = None
+        try:
+            session = self.get_thread_scoped_session()
+            # Calculate the threshold timestamp
+            older_than_timestamp = datetime.now(timezone.utc) - timedelta(days=days)
+
+            # Query and delete beacons older than the calculated timestamp
+            delete_query = session.query(Beacon).filter(Beacon.timestamp < older_than_timestamp)
+            deleted_count = delete_query.delete(synchronize_session='fetch')
+            session.commit()
+
+            self.log(f"Deleted {deleted_count} beacons older than {days} days")
+            return deleted_count
+
+        except Exception as e:
+            self.log(f"An error occurred during beacon cleanup: {e}", isWarning=True)
+            if session:
+                session.rollback()
+            return 0  # Return 0 or handle the error as appropriate
+        finally:
+            if session and not session.is_active:
+                session.remove()

modem/message_system_db_manager.py

@@ -0,0 +1,111 @@
+# database_manager.py
+import sqlite3
+
+from sqlalchemy import create_engine
+from sqlalchemy.orm import scoped_session, sessionmaker
+from threading import local
+from message_system_db_model import Base, Station, Status
+import structlog
+import helpers
+
+class DatabaseManager:
+    def __init__(self, event_manger, uri='sqlite:///freedata-messages.db'):
+        self.event_manager = event_manger
+
+        self.engine = create_engine(uri, echo=False)
+        self.thread_local = local()
+        self.session_factory = sessionmaker(bind=self.engine)
+        Base.metadata.create_all(self.engine)
+
+        self.logger = structlog.get_logger(type(self).__name__)
+
+    def initialize_default_values(self):
+        session = self.get_thread_scoped_session()
+        try:
+            statuses = [
+                "transmitting",
+                "transmitted",
+                "received",
+                "failed",
+                "failed_checksum",
+                "aborted",
+                "queued"
+            ]
+
+            # Add default statuses if they don't exist
+            for status_name in statuses:
+                existing_status = session.query(Status).filter_by(name=status_name).first()
+                if not existing_status:
+                    new_status = Status(name=status_name)
+                    session.add(new_status)
+
+            session.commit()
+            self.log("Initialized database")
+        except Exception as e:
+            session.rollback()
+            self.log(f"An error occurred while initializing default values: {e}", isWarning=True)
+        finally:
+            session.remove()
+
+    def log(self, message, isWarning=False):
+        msg = f"[{type(self).__name__}]: {message}"
+        logger = self.logger.warn if isWarning else self.logger.info
+        logger(msg)
+
+    def get_thread_scoped_session(self):
+        if not hasattr(self.thread_local, "session"):
+            self.thread_local.session = scoped_session(self.session_factory)
+        return self.thread_local.session
+
+    def get_or_create_station(self, callsign, session=None):
+        own_session = False
+        if not session:
+            session = self.get_thread_scoped_session()
+            own_session = True
+
+        try:
+            station = session.query(Station).filter_by(callsign=callsign).first()
+            if not station:
+                self.log(f"Updating station list with {callsign}")
+                station = Station(callsign=callsign, checksum=helpers.get_crc_24(callsign).hex())
+                session.add(station)
+                session.flush()
+
+            if own_session:
+                session.commit()  # Only commit if we created the session
+
+            return station
+
+        except Exception as e:
+
+            if own_session:
+                session.rollback()
+
+        finally:
+            if own_session:
+                session.remove()
+
+    def get_callsign_by_checksum(self, checksum):
+        session = self.get_thread_scoped_session()
+        try:
+            station = session.query(Station).filter_by(checksum=checksum).first()
+            if station:
+                self.log(f"Found callsign [{station.callsign}] for checksum [{station.checksum}]")
+                return station.callsign
+            else:
+                self.log(f"No callsign found for [{checksum}]")
+                return None
+        except Exception as e:
+            self.log(f"Error fetching callsign for checksum {checksum}: {e}", isWarning=True)
+            return {'status': 'failure', 'message': str(e)}
+        finally:
+            session.remove()
+
+    def get_or_create_status(self, session, status_name):
+        status = session.query(Status).filter_by(name=status_name).first()
+        if not status:
+            status = Status(name=status_name)
+            session.add(status)
+            session.flush()  # To get the ID immediately
+        return status
+

modem/message_system_db_messages.py

@@ -0,0 +1,204 @@
+from message_system_db_manager import DatabaseManager
+from message_system_db_attachments import DatabaseManagerAttachments
+from message_system_db_model import Status, P2PMessage
+from sqlalchemy.exc import IntegrityError
+from datetime import datetime
+import json
+
+
+class DatabaseManagerMessages(DatabaseManager):
+    def __init__(self, uri='sqlite:///freedata-messages.db'):
+        super().__init__(uri)
+        self.attachments_manager = DatabaseManagerAttachments(uri)
+
+    def add_message(self, message_data, direction='receive', status=None, is_read=True):
+        session = self.get_thread_scoped_session()
+        try:
+            # Create and add the origin and destination Stations
+            origin = self.get_or_create_station(message_data['origin'], session)
+            destination = self.get_or_create_station(message_data['destination'], session)
+
+            # Create and add Status if provided
+            if status:
+                status = self.get_or_create_status(session, status)
+
+            # Parse the timestamp from the message ID
+            timestamp = datetime.fromisoformat(message_data['id'].split('_')[2])
+            # Create the P2PMessage instance
+            new_message = P2PMessage(
+                id=message_data['id'],
+                origin_callsign=origin.callsign,
+                destination_callsign=destination.callsign,
+                body=message_data['body'],
+                timestamp=timestamp,
+                direction=direction,
+                status_id=status.id if status else None,
+                is_read=is_read,
+                attempt=0
+            )
+
+            session.add(new_message)
+
+            # Process and add attachments
+            if 'attachments' in message_data:
+                for attachment_data in message_data['attachments']:
+                    self.attachments_manager.add_attachment(session, new_message, attachment_data)
+
+            session.commit()
+            self.log(f"Added data to database: {new_message.id}")
+            self.event_manager.freedata_message_db_change()
+            return new_message.id
+        except IntegrityError as e:
+            session.rollback()  # Roll back the session to a clean state
+            self.log(f"Message with ID {message_data['id']} already exists in the database.", isWarning=True)
+            return None  # or you might return the existing message's ID or details
+
+
+        except Exception as e:
+            session.rollback()
+            self.log(f"error adding new message to database with error: {e}", isWarning=True)
+        finally:
+            session.remove()
+
+
+    def get_all_messages(self):
+        session = self.get_thread_scoped_session()
+        try:
+            messages = session.query(P2PMessage).all()
+            return [message.to_dict() for message in messages]
+
+        except Exception as e:
+            self.log(f"error fetching database messages with error: {e}", isWarning=True)
+            self.log(f"---> please delete or update existing database", isWarning=True)
+
+            return []
+
+        finally:
+            session.remove()
+
+    def get_all_messages_json(self):
+        messages_dict = self.get_all_messages()
+        messages_with_header = {'total_messages' : len(messages_dict), 'messages' : messages_dict}
+        return messages_with_header
+
+    def get_message_by_id(self, message_id):
+        session = self.get_thread_scoped_session()
+        try:
+            message = session.query(P2PMessage).filter_by(id=message_id).first()
+            if message:
+                return message.to_dict()
+            else:
+                return None
+        except Exception as e:
+            self.log(f"Error fetching message with ID {message_id}: {e}", isWarning=True)
+            return None
+        finally:
+            session.remove()
+
+    def get_message_by_id_json(self, message_id):
+        message_dict = self.get_message_by_id(message_id)
+        return json.dumps(message_dict)  # Convert to JSON string
+
+    def delete_message(self, message_id):
+        session = self.get_thread_scoped_session()
+        try:
+            message = session.query(P2PMessage).filter_by(id=message_id).first()
+            if message:
+                session.delete(message)
+                session.commit()
+                self.log(f"Deleted: {message_id}")
+                self.event_manager.freedata_message_db_change()
+                return {'status': 'success', 'message': f'Message {message_id} deleted'}
+            else:
+                return {'status': 'failure', 'message': 'Message not found'}
+
+        except Exception as e:
+            session.rollback()
+            self.log(f"Error deleting message with ID {message_id}: {e}", isWarning=True)
+            return {'status': 'failure', 'message': str(e)}
+
+        finally:
+            session.remove()
+
+    def update_message(self, message_id, update_data):
+        session = self.get_thread_scoped_session()
+        try:
+            message = session.query(P2PMessage).filter_by(id=message_id).first()
+            if message:
+                # Update fields of the message as per update_data
+                if 'body' in update_data:
+                    message.body = update_data['body']
+                if 'status' in update_data:
+                    message.status = self.get_or_create_status(session, update_data['status'])
+
+                session.commit()
+                self.log(f"Updated: {message_id}")
+                self.event_manager.freedata_message_db_change()
+                return {'status': 'success', 'message': f'Message {message_id} updated'}
+            else:
+                return {'status': 'failure', 'message': 'Message not found'}
+
+        except Exception as e:
+            session.rollback()
+            self.log(f"Error updating message with ID {message_id}: {e}", isWarning=True)
+            return {'status': 'failure', 'message': str(e)}
+
+        finally:
+            session.remove()
+
+    def get_first_queued_message(self):
+        session = self.get_thread_scoped_session()
+        try:
+            # Find the status object for "queued"
+            queued_status = session.query(Status).filter_by(name='queued').first()
+            if queued_status:
+                # Query for the first (oldest) message with status "queued"
+                message = session.query(P2PMessage)\
+                    .filter_by(status=queued_status)\
+                    .order_by(P2PMessage.timestamp)\
+                    .first()
+                if message:
+                    self.log(f"Found queued message with ID {message.id}")
+                    return message.to_dict()
+                else:
+                    return None
+            else:
+                self.log("Queued status not found", isWarning=True)
+                return None
+        except Exception as e:
+            self.log(f"Error fetching the first queued message: {e}", isWarning=True)
+            return None
+        finally:
+            session.remove()
+
+    def increment_message_attempts(self, message_id):
+        session = self.get_thread_scoped_session()
+        try:
+            message = session.query(P2PMessage).filter_by(id=message_id).first()
+            if message:
+                message.attempt += 1
+                session.commit()
+                self.log(f"Incremented attempt count for message {message_id}")
+            else:
+                self.log(f"Message with ID {message_id} not found")
+        except Exception as e:
+            session.rollback()
+            self.log(f"An error occurred while incrementing attempts for message {message_id}: {e}")
+        finally:
+            session.remove()
+
+    def mark_message_as_read(self, message_id):
+        session = self.get_thread_scoped_session()
+        try:
+            message = session.query(P2PMessage).filter_by(id=message_id).first()
+            if message:
+                message.is_read = True
+                session.commit()
+                self.log(f"Marked message {message_id} as read")
+            else:
+                self.log(f"Message with ID {message_id} not found")
+        except Exception as e:
+            session.rollback()
+            self.log(f"An error occurred while marking message {message_id} as read: {e}")
+        finally:
+            session.remove()

modem/message_system_db_model.py

@@ -0,0 +1,121 @@
+# models.py
+
+from sqlalchemy import Index, Boolean, Column, String, Integer, JSON, ForeignKey, DateTime
+from sqlalchemy.orm import declarative_base, relationship
+
+Base = declarative_base()
+
+class MessageAttachment(Base):
+    __tablename__ = 'message_attachment'
+    message_id = Column(String, ForeignKey('p2p_message.id', ondelete='CASCADE'), primary_key=True)
+    attachment_id = Column(Integer, ForeignKey('attachment.id', ondelete='CASCADE'), primary_key=True)
+
+    message = relationship('P2PMessage', back_populates='message_attachments')
+    attachment = relationship('Attachment', back_populates='message_attachments')
+
+class Config(Base):
+    __tablename__ = 'config'
+    db_variable = Column(String, primary_key=True)  # Unique identifier for the configuration setting
+    db_version = Column(String)
+
+    def to_dict(self):
+        return {
+            'db_variable': self.db_variable,
+            'db_settings': self.db_settings
+        }
+
+
+class Beacon(Base):
+    __tablename__ = 'beacon'
+    id = Column(Integer, primary_key=True)
+    timestamp = Column(DateTime)
+    snr = Column(Integer)
+    callsign = Column(String, ForeignKey('station.callsign'))
+    station = relationship("Station", back_populates="beacons")
+
+    Index('idx_beacon_callsign', 'callsign')
+
+class Station(Base):
+    __tablename__ = 'station'
+    callsign = Column(String, primary_key=True)
+    checksum = Column(String, nullable=True)
+    location = Column(JSON, nullable=True)
+    info = Column(JSON, nullable=True)
+    beacons = relationship("Beacon", order_by="Beacon.id", back_populates="station")
+
+    Index('idx_station_callsign_checksum', 'callsign', 'checksum')
+
+    def to_dict(self):
+        return {
+            'callsign': self.callsign,
+            'checksum': self.checksum,
+            'location': self.location,
+            'info': self.info,
+
+        }
+class Status(Base):
+    __tablename__ = 'status'
+    id = Column(Integer, primary_key=True)
+    name = Column(String, unique=True)
+
+class P2PMessage(Base):
+    __tablename__ = 'p2p_message'
+    id = Column(String, primary_key=True)
+    origin_callsign = Column(String, ForeignKey('station.callsign'))
+    via_callsign = Column(String, ForeignKey('station.callsign'), nullable=True)
+    destination_callsign = Column(String, ForeignKey('station.callsign'))
+    body = Column(String, nullable=True)
+    message_attachments = relationship('MessageAttachment',
+                                       back_populates='message',
+                                       cascade='all, delete-orphan')
+    attempt = Column(Integer, default=0)
+    timestamp = Column(DateTime)
+    status_id = Column(Integer, ForeignKey('status.id'), nullable=True)
+    status = relationship('Status', backref='p2p_messages')
+    priority = Column(Integer, default=10)
+    direction = Column(String)
+    statistics = Column(JSON, nullable=True)
+    is_read = Column(Boolean, default=True)
+
+    Index('idx_p2p_message_origin_timestamp', 'origin_callsign', 'via_callsign', 'destination_callsign', 'timestamp', 'attachments')
+
+    def to_dict(self):
+        attachments_list = [ma.attachment.to_dict() for ma in self.message_attachments]
+
+        return {
+            'id': self.id,
+            'timestamp': self.timestamp.isoformat() if self.timestamp else None,
+            'attempt': self.attempt,
+            'origin': self.origin_callsign,
+            'via': self.via_callsign,
+            'destination': self.destination_callsign,
+            'direction': self.direction,
+            'body': self.body,
+            'attachments': attachments_list,
+            'status': self.status.name if self.status else None,
+            'priority': self.priority,
+            'is_read': self.is_read,
+            'statistics': self.statistics
+        }
+
+class Attachment(Base):
+    __tablename__ = 'attachment'
+    id = Column(Integer, primary_key=True)
+    name = Column(String)
+    data_type = Column(String)
+    data = Column(String)
+    checksum_crc32 = Column(String)
+    hash_sha512 = Column(String)
+    message_attachments = relationship("MessageAttachment", back_populates="attachment")
+
+    Index('idx_attachments_id_message_id', 'id', 'hash_sha512')
+
+    def to_dict(self):
+        return {
+            'id': self.id,
+            'name': self.name,
+            'type': self.data_type,
+            'data': self.data,
+            'checksum_crc32': self.checksum_crc32,
+            'hash_sha512' : self.hash_sha512
+        }

modem/rigctld.py

@@ -62,7 +62,8 @@ class radio:
     def send_command(self, command) -> str:
         if self.connected:
             # wait if we have another command awaiting its response...
-            self.await_response.wait()
+            # we need to set a timeout for avoiding a blocking state
+            self.await_response.wait(timeout=1)
 
             try:
                 self.await_response = threading.Event()
@@ -73,8 +74,6 @@ class radio:
             except Exception as err:
                 self.log.warning(f"[RIGCTLD] Error sending command [{command}] to rigctld: {err}")
                 self.connected = False
-
-
         return ""
 
     def set_ptt(self, state):

modem/schedule_manager.py

@@ -0,0 +1,88 @@
+import sched
+import time
+import threading
+import command_message_send
+from message_system_db_manager import DatabaseManager
+from message_system_db_messages import DatabaseManagerMessages
+from message_system_db_beacon import DatabaseManagerBeacon
+import explorer
+import command_beacon
+
+
+class ScheduleManager:
+    def __init__(self, modem_version, config_manager, state_manger, event_manager):
+        self.modem_version = modem_version
+        self.config_manager = config_manager
+        self.state_manager = state_manger
+        self.event_manager = event_manager
+        self.config = self.config_manager.read()
+        self.beacon_interval = self.config['MODEM']['beacon_interval']
+
+        self.scheduler = sched.scheduler(time.time, time.sleep)
+        self.events = {
+            'check_for_queued_messages': {'function': self.check_for_queued_messages, 'interval': 10},
+            'explorer_publishing': {'function': self.push_to_explorer, 'interval': 120},
+            'transmitting_beacon': {'function': self.transmit_beacon, 'interval': self.beacon_interval},
+            'beacon_cleanup': {'function': self.delete_beacons, 'interval': 600},
+        }
+        self.running = False  # Flag to control the running state
+        self.scheduler_thread = None  # Reference to the scheduler thread
+
+        self.modem = None
+
+    def schedule_event(self, event_function, interval):
+        """Schedule an event and automatically reschedule it after execution."""
+        event_function()  # Execute the event function
+        if self.running:  # Only reschedule if still running
+            self.scheduler.enter(interval, 1, self.schedule_event, (event_function, interval))
+
+    def start(self, modem):
+        """Start scheduling events and run the scheduler in a separate thread."""
+
+        # wait some time
+        threading.Event().wait(timeout=10)
+
+        # get actual modem instance
+        self.modem = modem
+
+        self.running = True  # Set the running flag to True
+        for event_info in self.events.values():
+            # Schedule each event for the first time
+            self.scheduler.enter(0, 1, self.schedule_event, (event_info['function'], event_info['interval']))
+
+        # Run the scheduler in a separate thread
+        self.scheduler_thread = threading.Thread(target=self.scheduler.run, daemon=True)
+        self.scheduler_thread.start()
+
+    def stop(self):
+        """Stop scheduling new events and terminate the scheduler thread."""
+        self.running = False  # Clear the running flag to stop scheduling new events
+        # Clear scheduled events to break the scheduler out of its waiting state
+        for event in list(self.scheduler.queue):
+            self.scheduler.cancel(event)
+        # Wait for the scheduler thread to finish
+        if self.scheduler_thread:
+            self.scheduler_thread.join()
+
+    def transmit_beacon(self):
+        if not self.state_manager.getARQ() and self.state_manager.is_beacon_running:
+                cmd = command_beacon.BeaconCommand(self.config, self.state_manager, self.event_manager)
+                cmd.run(self.event_manager, self.modem)
+
+    def delete_beacons(self):
+        DatabaseManagerBeacon(self.event_manager).beacon_cleanup_older_than_days(14)
+
+    def push_to_explorer(self):
+        self.config = self.config_manager.read()
+        if self.config['STATION']['enable_explorer']:
+            explorer.explorer(self.modem_version, self.config_manager, self.state_manager).push()
+
+    def check_for_queued_messages(self):
+        if not self.state_manager.getARQ():
+            if DatabaseManagerMessages(self.event_manager).get_first_queued_message():
+                params = DatabaseManagerMessages(self.event_manager).get_first_queued_message()
+                command = command_message_send.SendMessageCommand(self.config_manager.read(), self.state_manager, self.event_manager, params)
+                command.transmit(self.modem)
+
+        return
+

modem/server.py

@@ -2,6 +2,7 @@ from flask import Flask, request, jsonify, make_response, abort, Response
 from flask_sock import Sock
 from flask_cors import CORS
 import os
+import sys
 import serial_ports
 from config import CONFIG
 import audio
@@ -16,13 +17,19 @@ import command_ping
 import command_feq
 import command_test
 import command_arq_raw
+import command_message_send
 import event_manager
+from message_system_db_manager import DatabaseManager
+from message_system_db_messages import DatabaseManagerMessages
+from message_system_db_attachments import DatabaseManagerAttachments
+from message_system_db_beacon import DatabaseManagerBeacon
+from schedule_manager import ScheduleManager
 
 app = Flask(__name__)
 CORS(app)
 CORS(app, resources={r"/*": {"origins": "*"}})
 sock = Sock(app)
-MODEM_VERSION = "0.12.1-alpha"
+MODEM_VERSION = "0.13.2-alpha"
 
 # set config file to use
 def set_config():
@@ -35,7 +42,7 @@ def set_config():
         print(f"Using config from {config_file}")
     else:
         print(f"Config file '{config_file}' not found. Exiting.")
-        exit(1)
+        sys.exit(1)
 
     return config_file
 
@@ -70,6 +77,7 @@ def enqueue_tx_command(cmd_class, params = {}):
     if command.run(app.modem_events, app.service_manager.modem): # TODO remove the app.modem_event custom queue
         return True
     return False
+
 ## REST API
 @app.route('/', methods=['GET'])
 def index():
@@ -131,10 +139,8 @@ def post_beacon():
 
     if not app.state_manager.is_beacon_running:
         app.state_manager.set('is_beacon_running', request.json['enabled'])
-        app.modem_service.put("start_beacon")
     else:
         app.state_manager.set('is_beacon_running', request.json['enabled'])
-        app.modem_service.put("stop_beacon")
 
     return api_response(request.json)
 
@@ -234,19 +240,54 @@ def get_post_radio():
     elif request.method == 'GET':
         return api_response(app.state_manager.get_radio_status())
 
-# @app.route('/modem/arq_connect', methods=['POST'])
-# @app.route('/modem/arq_disconnect', methods=['POST'])
-# @app.route('/modem/send_raw', methods=['POST'])
-# @app.route('/modem/record_audio', methods=['POST'])
-# @app.route('/modem/audio_levels', methods=['POST']) # tx and rx # not needed if we are restarting modem on changing settings
-# @app.route('/modem/mesh_ping', methods=['POST'])
-# @app.route('/mesh/routing_table', methods=['GET'])
-# @app.route('/modem/get_rx_buffer', methods=['GET'])
-# @app.route('/modem/del_rx_buffer', methods=['POST'])
-# @app.route('/rig/status', methods=['GET'])
-# @app.route('/rig/mode', methods=['POST'])
-# @app.route('/rig/frequency', methods=['POST'])
-# @app.route('/rig/test_hamlib', methods=['POST'])
+@app.route('/freedata/messages', methods=['POST', 'GET'])
+def get_post_freedata_message():
+    if request.method in ['GET']:
+        result = DatabaseManagerMessages(app.event_manager).get_all_messages_json()
+        return api_response(result, 200)
+    if enqueue_tx_command(command_message_send.SendMessageCommand, request.json):
+        return api_response(request.json, 200)
+    else:
+        api_abort('Error executing command...', 500)
+
+@app.route('/freedata/messages/<string:message_id>', methods=['GET', 'POST', 'PATCH', 'DELETE'])
+def handle_freedata_message(message_id):
+    if request.method == 'GET':
+        message = DatabaseManagerMessages(app.event_manager).get_message_by_id_json(message_id)
+        return message
+    elif request.method == 'POST':
+        result = DatabaseManagerMessages(app.event_manager).update_message(message_id, update_data={'status': 'queued'})
+        DatabaseManagerMessages(app.event_manager).increment_message_attempts(message_id)
+        return api_response(result)
+    elif request.method == 'PATCH':
+        # Fixme We need to adjust this
+        result = DatabaseManagerMessages(app.event_manager).mark_message_as_read(message_id)
+        return api_response(result)
+    elif request.method == 'DELETE':
+        result = DatabaseManagerMessages(app.event_manager).delete_message(message_id)
+        return api_response(result)
+    else:
+        api_abort('Error executing command...', 500)
+
+@app.route('/freedata/messages/<string:message_id>/attachments', methods=['GET'])
+def get_message_attachments(message_id):
+    attachments = DatabaseManagerAttachments(app.event_manager).get_attachments_by_message_id_json(message_id)
+    return api_response(attachments)
+
+@app.route('/freedata/messages/attachment/<string:data_sha512>', methods=['GET'])
+def get_message_attachment(data_sha512):
+    attachment = DatabaseManagerAttachments(app.event_manager).get_attachment_by_sha512(data_sha512)
+    return api_response(attachment)
+
+@app.route('/freedata/beacons', methods=['GET'])
+def get_all_beacons():
+    beacons = DatabaseManagerBeacon(app.event_manager).get_all_beacons()
+    return api_response(beacons)
+
+@app.route('/freedata/beacons/<string:callsign>', methods=['GET'])
+def get_beacons_by_callsign(callsign):
+    beacons = DatabaseManagerBeacon(app.event_manager).get_beacons_by_callsign(callsign)
+    return api_response(beacons)
 
 # Event websocket
 @sock.route('/events')
@@ -279,10 +320,13 @@ if __name__ == "__main__":
     app.event_manager = event_manager.EventManager([app.modem_events])  # TODO remove the app.modem_event custom queue
     # init state manager
     app.state_manager = state_manager.StateManager(app.state_queue)
+    # initialize message system schedule manager
+    app.schedule_manager = ScheduleManager(app.MODEM_VERSION, app.config_manager, app.state_manager, app.event_manager)
     # start service manager
     app.service_manager = service_manager.SM(app)
     # start modem service
     app.modem_service.put("start")
-
+    # initialize database default values
+    DatabaseManager(app.event_manager).initialize_default_values()
     wsm.startThreads(app)
     app.run()

modem/service_manager.py

@@ -3,9 +3,7 @@ import frame_dispatcher
 import modem
 import structlog
 import audio
-import ujson as json
-import explorer
-import beacon
+
 import radio_manager
 
 
@@ -14,14 +12,13 @@ class SM:
         self.log = structlog.get_logger("service")
         self.app = app
         self.modem = False
-        self.beacon = False
-        self.explorer = False
         self.app.radio_manager = False
         self.config = self.app.config_manager.read()
         self.modem_fft = app.modem_fft
         self.modem_service = app.modem_service
         self.state_manager = app.state_manager
         self.event_manager = app.event_manager
+        self.schedule_manager = app.schedule_manager
 
 
         runner_thread = threading.Thread(
@@ -33,23 +30,18 @@ class SM:
         while True:
             cmd = self.modem_service.get()
             if cmd in ['start'] and not self.modem:
-                self.log.info("------------------ FreeDATA ------------------")
-                self.log.info("------------------  MODEM   ------------------")
                 self.config = self.app.config_manager.read()
                 self.start_radio_manager()
                 self.start_modem()
-                self.start_explorer_publishing()
 
             elif cmd in ['stop'] and self.modem:
                 self.stop_modem()
-                self.stop_explorer_publishing()
                 self.stop_radio_manager()
                 # we need to wait a bit for avoiding a portaudio crash
                 threading.Event().wait(0.5)
 
             elif cmd in ['restart']:
                 self.stop_modem()
-                self.stop_explorer_publishing()
                 self.stop_radio_manager()
                 # we need to wait a bit for avoiding a portaudio crash
                 threading.Event().wait(0.5)
@@ -59,24 +51,22 @@ class SM:
 
                 if self.start_modem():
                     self.event_manager.modem_restarted()
-                    self.start_explorer_publishing()
-            elif cmd in ['start_beacon']:
-                self.start_beacon()
-
-            elif cmd in ['stop_beacon']:
-                self.stop_beacon()
-
-
 
             else:
                 self.log.warning("[SVC] modem command processing failed", cmd=cmd, state=self.state_manager.is_modem_running)
 
 
     def start_modem(self):
+
+        if self.config['STATION']['mycall'] in ['XX1XXX']:
+            self.log.warning("wrong callsign in config! interrupting startup")
+            return False
+
         if self.state_manager.is_modem_running:
             self.log.warning("modem already running")
             return False
 
+
         # test audio devices
         audio_test = self.test_audio()
 
@@ -98,6 +88,7 @@ class SM:
         self.event_manager.modem_started()
         self.state_manager.set("is_modem_running", True)
         self.modem.start_modem()
+        self.schedule_manager.start(self.modem)
 
         return True
         
@@ -106,6 +97,7 @@ class SM:
         del self.modem
         self.modem = False
         self.state_manager.set("is_modem_running", False)
+        self.schedule_manager.stop()
         self.event_manager.modem_stopped()
 
     def test_audio(self):
@@ -119,28 +111,6 @@ class SM:
             self.log.error("Error testing audio devices", e=e)
             return [False, False]
 
-    def start_beacon(self):
-        self.beacon = beacon.Beacon(self.config, self.state_manager, self.event_manager, self.log, self.modem)
-        self.beacon.start()
-
-    def stop_beacon(self):
-        self.beacon.stop()
-
-    def start_explorer_publishing(self):
-        try:
-            # optionally start explorer module
-            if self.config['STATION']['enable_explorer']:
-                self.explorer = explorer.explorer(self.app, self.config, self.state_manager)
-        except Exception as e:
-            self.log.warning("[EXPLORER] Publishing not started because of error", e=e)
-
-    def stop_explorer_publishing(self):
-        if self.config['STATION']['enable_explorer']:
-            try:
-                del self.explorer
-            except Exception as e:
-                self.log.info("[EXPLORER] Error while stopping...", e=e)
-
     def start_radio_manager(self):
         self.app.radio_manager = radio_manager.RadioManager(self.config, self.state_manager, self.event_manager)
 

modem/state_manager.py

@@ -19,7 +19,10 @@ class StateManager:
         self.channel_busy_condition_codec2 = threading.Event()
 
         self.is_modem_running = False
-        self.is_modem_busy = False
+
+        self.is_modem_busy = threading.Event()
+        self.setARQ(False)
+
         self.is_beacon_running = False
 
         # If true, any wait() call is blocking
@@ -30,13 +33,13 @@ class StateManager:
         self.dxcallsign: bytes = b"ZZ9YY-0"
         self.dxgrid: bytes = b"------"
 
-        self.heard_stations = []  # TODO remove it... heard stations list == deprecated
+        self.heard_stations = []
         self.activities_list = {}
 
         self.arq_iss_sessions = {}
         self.arq_irs_sessions = {}
 
-        self.mesh_routing_table = []
+        #self.mesh_routing_table = []
 
         self.radio_frequency = 0
         self.radio_mode = None
@@ -87,6 +90,7 @@ class StateManager:
             "channel_busy_slot": self.channel_busy_slot,
             "audio_dbfs": self.audio_dbfs,
             "activities": self.activities_list,
+            "is_modem_busy" : self.getARQ()
         }
     
     # .wait() blocks until the event is set
@@ -100,6 +104,15 @@ class StateManager:
         else:
             self.transmitting_event.set()
 
+    def setARQ(self, busy):
+        if busy:
+            self.is_modem_busy.clear()
+        else:
+            self.is_modem_busy.set()
+
+    def getARQ(self):
+        return not self.is_modem_busy.is_set()
+
     def waitForTransmission(self):
         self.transmitting_event.wait()
 
@@ -201,4 +214,4 @@ class StateManager:
             "radio_mode": self.radio_mode,
             "radio_rf_level": self.radio_rf_level,
             "s_meter_strength": self.s_meter_strength,
-        }
+        }

requirements.txt

@@ -27,4 +27,5 @@ pytest-cov
 pytest-cover
 pytest-coverage
 pytest-rerunfailures
-pick
+pick
+sqlalchemy

tests/test_arq_session.py

@@ -16,15 +16,17 @@ import random
 import structlog
 import numpy as np
 from event_manager import EventManager
+from state_manager import StateManager
 from data_frame_factory import DataFrameFactory
 import codec2
 import arq_session_irs
 class TestModem:
-    def __init__(self, event_q):
+    def __init__(self, event_q, state_q):
         self.data_queue_received = queue.Queue()
         self.demodulator = unittest.mock.Mock()
         self.event_manager = EventManager([event_q])
         self.logger = structlog.get_logger('Modem')
+        self.states = StateManager(state_q)
 
     def getFrameTransmissionTime(self, mode):
         samples = 0
@@ -61,7 +63,8 @@ class TestARQSession(unittest.TestCase):
         cls.iss_state_manager = StateManager(queue.Queue())
         cls.iss_event_manager = EventManager([queue.Queue()])
         cls.iss_event_queue = queue.Queue()
-        cls.iss_modem = TestModem(cls.iss_event_queue)
+        cls.iss_state_queue = queue.Queue()
+        cls.iss_modem = TestModem(cls.iss_event_queue, cls.iss_state_queue)
         cls.iss_frame_dispatcher = DISPATCHER(cls.config, 
                                           cls.iss_event_manager,
                                           cls.iss_state_manager, 
@@ -71,7 +74,8 @@ class TestARQSession(unittest.TestCase):
         cls.irs_state_manager = StateManager(queue.Queue())
         cls.irs_event_manager = EventManager([queue.Queue()])
         cls.irs_event_queue = queue.Queue()
-        cls.irs_modem = TestModem(cls.irs_event_queue)
+        cls.irs_state_queue = queue.Queue()
+        cls.irs_modem = TestModem(cls.irs_event_queue, cls.irs_state_queue)
         cls.irs_frame_dispatcher = DISPATCHER(cls.config, 
                                           cls.irs_event_manager,
                                           cls.irs_state_manager, 
@@ -126,12 +130,13 @@ class TestARQSession(unittest.TestCase):
 
     def testARQSessionSmallPayload(self):
         # set Packet Error Rate (PER) / frame loss probability
-        self.loss_probability = 50
+        self.loss_probability = 0
 
         self.establishChannels()
         params = {
             'dxcall': "XX1XXX-1",
             'data': base64.b64encode(bytes("Hello world!", encoding="utf-8")),
+            'type': "raw_lzma"
         }
         cmd = ARQRawCommand(self.config, self.iss_state_manager, self.iss_event_queue, params)
         cmd.run(self.iss_event_queue, self.iss_modem)
@@ -146,6 +151,7 @@ class TestARQSession(unittest.TestCase):
         params = {
             'dxcall': "XX1XXX-1",
             'data': base64.b64encode(np.random.bytes(1000)),
+            'type': "raw_lzma"
         }
         cmd = ARQRawCommand(self.config, self.iss_state_manager, self.iss_event_queue, params)
         cmd.run(self.iss_event_queue, self.iss_modem)

tests/test_data_type_handler.py

@@ -0,0 +1,44 @@
+import sys
+sys.path.append('modem')
+
+import unittest
+import queue
+from arq_data_type_handler import ARQDataTypeHandler, ARQ_SESSION_TYPES
+from event_manager import EventManager
+from state_manager import StateManager
+
+class TestDispatcher(unittest.TestCase):
+
+    @classmethod
+    def setUpClass(cls):
+        cls.event_queue = queue.Queue()
+        cls.state_queue = queue.Queue()
+        cls.event_manager = EventManager([cls.event_queue])
+        cls.state_manager = StateManager([cls.state_queue])
+        cls.arq_data_type_handler = ARQDataTypeHandler(cls.event_manager, cls.state_manager)
+
+
+    def testDataTypeHevent_managerandlerRaw(self):
+        # Example usage
+        example_data = b"Hello FreeDATA!"
+        formatted_data, type_byte = self.arq_data_type_handler.prepare(example_data, ARQ_SESSION_TYPES.raw)
+        dispatched_data = self.arq_data_type_handler.dispatch(type_byte, formatted_data)
+        self.assertEqual(example_data, dispatched_data)
+
+    def testDataTypeHandlerLZMA(self):
+        # Example usage
+        example_data = b"Hello FreeDATA!"
+        formatted_data, type_byte = self.arq_data_type_handler.prepare(example_data, ARQ_SESSION_TYPES.raw_lzma)
+        dispatched_data = self.arq_data_type_handler.dispatch(type_byte, formatted_data)
+        self.assertEqual(example_data, dispatched_data)
+
+    def testDataTypeHandlerGZIP(self):
+        # Example usage
+        example_data = b"Hello FreeDATA!"
+        formatted_data, type_byte = self.arq_data_type_handler.prepare(example_data, ARQ_SESSION_TYPES.raw_gzip)
+        dispatched_data = self.arq_data_type_handler.dispatch(type_byte, formatted_data)
+        self.assertEqual(example_data, dispatched_data)
+
+
+if __name__ == '__main__':
+    unittest.main()

tests/test_message_database.py

@@ -0,0 +1,139 @@
+import sys
+sys.path.append('modem')
+import numpy as np
+
+import unittest
+from config import CONFIG
+from message_p2p import MessageP2P
+from message_system_db_manager import DatabaseManager
+from message_system_db_messages import DatabaseManagerMessages
+from message_system_db_attachments import DatabaseManagerAttachments
+
+from event_manager import EventManager
+import queue
+import base64
+
+class TestDataFrameFactory(unittest.TestCase):
+
+    @classmethod
+    def setUpClass(cls):
+        config_manager = CONFIG('modem/config.ini.example')
+        cls.config = config_manager.read()
+
+        cls.event_queue = queue.Queue()
+        cls.event_manager = EventManager([cls.event_queue])
+        cls.mycall = f"{cls.config['STATION']['mycall']}-{cls.config['STATION']['myssid']}"
+        cls.database_manager = DatabaseManagerMessages(cls.event_manager)
+        cls.database_manager_attachments = DatabaseManagerAttachments(cls.event_manager)
+
+    def testAddToDatabase(self):
+        attachment = {
+            'name': 'test.gif',
+            'type': 'image/gif',
+            'data': str(base64.b64encode(np.random.bytes(1024)), 'utf-8')
+        }
+        apiParams = {'destination': 'DJ2LS-3', 'body': 'Hello World!', 'attachments': [attachment]}
+        message = MessageP2P.from_api_params(self.mycall, apiParams)
+        payload = message.to_payload()
+        received_message = MessageP2P.from_payload(payload)
+        received_message_dict = MessageP2P.to_dict(received_message)
+        self.database_manager.add_message(received_message_dict)
+        result = self.database_manager.get_message_by_id(message.id)
+
+        self.assertEqual(result["destination"], message.destination)
+
+    def testDeleteFromDatabase(self):
+        attachment = {
+            'name': 'test.gif',
+            'type': 'image/gif',
+            'data': str(base64.b64encode(np.random.bytes(1024)), 'utf-8')
+        }
+        apiParams = {'destination': 'DJ2LS-3', 'body': 'Hello World!', 'attachments': [attachment]}
+        message = MessageP2P.from_api_params(self.mycall, apiParams)
+        payload = message.to_payload()
+        received_message = MessageP2P.from_payload(payload)
+        received_message_dict = MessageP2P.to_dict(received_message)
+        self.database_manager.add_message(received_message_dict)
+
+        result = self.database_manager.get_all_messages()
+        message_id = result[0]["id"]
+        self.database_manager.delete_message(message_id)
+
+        result = self.database_manager.get_all_messages()
+        self.assertNotIn(message_id, result)
+
+    def testUpdateMessage(self):
+        attachment = {
+            'name': 'test.gif',
+            'type': 'image/gif',
+            'data': str(base64.b64encode(np.random.bytes(1024)), 'utf-8')
+        }
+
+        apiParams = {'destination': 'DJ2LS-3', 'body': 'Hello World!', 'attachments': [attachment]}
+        message = MessageP2P.from_api_params(self.mycall, apiParams)
+        payload = message.to_payload()
+        received_message = MessageP2P.from_payload(payload)
+        received_message_dict = MessageP2P.to_dict(received_message)
+        print(received_message_dict)
+        message_id = self.database_manager.add_message(received_message_dict, direction='receive')
+        print(message_id)
+        self.database_manager.update_message(message_id, {'body' : 'hello123'})
+
+        result = self.database_manager.get_message_by_id(message_id)
+        self.assertIn('hello123', result['body'])
+
+    def testGetAttachments(self):
+        attachment1 = {
+            'name': 'test1.gif',
+            'type': 'image/gif',
+            'data': str(base64.b64encode(np.random.bytes(1024)), 'utf-8')
+        }
+        attachment2 = {
+            'name': 'test2.gif',
+            'type': 'image/gif',
+            'data': str(base64.b64encode(np.random.bytes(1024)), 'utf-8')
+        }
+        attachment3 = {
+            'name': 'test3.gif',
+            'type': 'image/gif',
+            'data': str(base64.b64encode(np.random.bytes(1024)), 'utf-8')
+        }
+        apiParams = {'destination': 'DJ2LS-3', 'body': 'Hello World!', 'attachments': [attachment1, attachment2, attachment3]}
+        message = MessageP2P.from_api_params(self.mycall, apiParams)
+        payload = message.to_payload()
+        received_message = MessageP2P.from_payload(payload)
+        received_message_dict = MessageP2P.to_dict(received_message)
+        message_id = self.database_manager.add_message(received_message_dict)
+        result = self.database_manager_attachments.get_attachments_by_message_id(message_id)
+        attachment_names = [attachment['name'] for attachment in result]
+        self.assertIn('test1.gif', attachment_names)
+        self.assertIn('test2.gif', attachment_names)
+        self.assertIn('test3.gif', attachment_names)
+
+    def testIncrementAttempts(self):
+        apiParams = {'destination': 'DJ2LS-3', 'body': 'Hello World!', 'attachments': []}
+        message = MessageP2P.from_api_params(self.mycall, apiParams)
+        payload = message.to_payload()
+        received_message = MessageP2P.from_payload(payload)
+        received_message_dict = MessageP2P.to_dict(received_message)
+        message_id = self.database_manager.add_message(received_message_dict)
+        self.database_manager.increment_message_attempts(message_id)
+
+
+        result = self.database_manager.get_message_by_id(message_id)
+        self.assertEqual(result["attempt"], 1)
+
+    def testMarkAsRead(self):
+        apiParams = {'destination': 'DJ2LS-3', 'body': 'Hello World!', 'attachments': []}
+        message = MessageP2P.from_api_params(self.mycall, apiParams)
+        payload = message.to_payload()
+        received_message = MessageP2P.from_payload(payload)
+        received_message_dict = MessageP2P.to_dict(received_message)
+        message_id = self.database_manager.add_message(received_message_dict, is_read=False)
+        self.database_manager.mark_message_as_read(message_id)
+
+        result = self.database_manager.get_message_by_id(message_id)
+        self.assertEqual(result["is_read"], True)
+
+if __name__ == '__main__':
+    unittest.main()

tests/test_message_p2p.py

@@ -0,0 +1,77 @@
+import sys
+sys.path.append('modem')
+import numpy as np
+
+import unittest
+from config import CONFIG
+from message_p2p import MessageP2P
+from message_system_db_messages import DatabaseManagerMessages
+from event_manager import EventManager
+import queue
+import base64
+
+class TestDataFrameFactory(unittest.TestCase):
+
+    @classmethod
+    def setUpClass(cls):
+        config_manager = CONFIG('modem/config.ini.example')
+        cls.config = config_manager.read()
+
+        cls.event_queue = queue.Queue()
+        cls.event_manager = EventManager([cls.event_queue])
+        cls.mycall = f"{cls.config['STATION']['mycall']}-{cls.config['STATION']['myssid']}"
+        cls.database_manager = DatabaseManagerMessages(cls.event_manager)
+
+    def testFromApiParams(self):
+        api_params = {
+            'destination': 'DJ2LS-3',
+            'body': 'Hello World!',
+        }
+        message = MessageP2P.from_api_params(self.mycall, api_params)
+        self.assertEqual(message.destination, api_params['destination'])
+        self.assertEqual(message.body, api_params['body'])
+
+    def testToPayloadWithAttachment(self):
+        attachment = {
+            'name': 'test.gif',
+            'type': 'image/gif',
+            'data': str(base64.b64encode(np.random.bytes(1024)), 'utf-8')
+        }
+        apiParams = {'destination': 'DJ2LS-3', 'body': 'Hello World!', 'attachments': [attachment]}
+        message = MessageP2P.from_api_params(self.mycall, apiParams)
+
+        payload = message.to_payload()
+        received_message = MessageP2P.from_payload(payload)
+        self.assertEqual(message.origin, received_message.origin)
+        self.assertEqual(message.destination, received_message.destination)
+        self.assertCountEqual(message.attachments, received_message.attachments)
+        # FIXME...
+        #self.assertEqual(attachment['data'], received_message.attachments[0]['data'])
+
+    def testToPayloadWithAttachmentAndDatabase(self):
+        attachment = {
+            'name': 'test.gif',
+            'type': 'image/gif',
+            'data': str(base64.b64encode(np.random.bytes(1024)), 'utf-8')
+        }
+        apiParams = {'destination': 'DJ2LS-3', 'body': 'Hello World!', 'attachments': [attachment]}
+        message = MessageP2P.from_api_params(self.mycall, apiParams)
+
+        payload = message.to_payload()
+        received_message = MessageP2P.from_payload(payload)
+        received_message_dict = MessageP2P.to_dict(received_message)
+        self.database_manager.add_message(received_message_dict)
+
+        self.assertEqual(message.origin, received_message.origin)
+        self.assertEqual(message.destination, received_message.destination)
+        self.assertCountEqual(message.attachments, received_message.attachments)
+        result = self.database_manager.get_message_by_id(message.id)
+        self.assertEqual(result["is_read"], True)
+        self.assertEqual(result["destination"], message.destination)
+
+
+
+
+
+if __name__ == '__main__':
+    unittest.main()

tests/test_message_protocol.py

@@ -0,0 +1,153 @@
+import sys
+import time
+
+sys.path.append('modem')
+
+import unittest
+import unittest.mock
+from config import CONFIG
+import helpers
+import queue
+import threading
+import base64
+from command_arq_raw import ARQRawCommand
+from state_manager import StateManager
+from frame_dispatcher import DISPATCHER
+import random
+import structlog
+import numpy as np
+from event_manager import EventManager
+from state_manager import StateManager
+from data_frame_factory import DataFrameFactory
+import codec2
+import arq_session_irs
+from server import enqueue_tx_command
+import command_message_send
+
+
+class TestModem:
+    def __init__(self, event_q, state_q):
+        self.data_queue_received = queue.Queue()
+        self.demodulator = unittest.mock.Mock()
+        self.event_manager = EventManager([event_q])
+        self.logger = structlog.get_logger('Modem')
+        self.states = StateManager(state_q)
+
+    def getFrameTransmissionTime(self, mode):
+        samples = 0
+        c2instance = codec2.open_instance(mode.value)
+        samples += codec2.api.freedv_get_n_tx_preamble_modem_samples(c2instance)
+        samples += codec2.api.freedv_get_n_tx_modem_samples(c2instance)
+        samples += codec2.api.freedv_get_n_tx_postamble_modem_samples(c2instance)
+        time = samples / 8000
+        return time
+
+    def transmit(self, mode, repeats: int, repeat_delay: int, frames: bytearray) -> bool:
+        # Simulate transmission time
+        tx_time = self.getFrameTransmissionTime(mode) + 0.1  # PTT
+        self.logger.info(f"TX {tx_time} seconds...")
+        threading.Event().wait(tx_time)
+
+        transmission = {
+            'mode': mode,
+            'bytes': frames,
+        }
+        self.data_queue_received.put(transmission)
+
+
+class TestMessageProtocol(unittest.TestCase):
+
+    @classmethod
+    def setUpClass(cls):
+        config_manager = CONFIG('modem/config.ini.example')
+        cls.config = config_manager.read()
+        cls.logger = structlog.get_logger("TESTS")
+        cls.frame_factory = DataFrameFactory(cls.config)
+
+        # ISS
+        cls.iss_state_manager = StateManager(queue.Queue())
+        cls.iss_event_manager = EventManager([queue.Queue()])
+        cls.iss_event_queue = queue.Queue()
+        cls.iss_state_queue = queue.Queue()
+        cls.iss_modem = TestModem(cls.iss_event_queue, cls.iss_state_queue)
+        cls.iss_frame_dispatcher = DISPATCHER(cls.config,
+                                              cls.iss_event_manager,
+                                              cls.iss_state_manager,
+                                              cls.iss_modem)
+
+        # IRS
+        cls.irs_state_manager = StateManager(queue.Queue())
+        cls.irs_event_manager = EventManager([queue.Queue()])
+        cls.irs_event_queue = queue.Queue()
+        cls.irs_state_queue = queue.Queue()
+        cls.irs_modem = TestModem(cls.irs_event_queue, cls.irs_state_queue)
+        cls.irs_frame_dispatcher = DISPATCHER(cls.config,
+                                              cls.irs_event_manager,
+                                              cls.irs_state_manager,
+                                              cls.irs_modem)
+
+        # Frame loss probability in %
+        cls.loss_probability = 30
+
+        cls.channels_running = True
+
+    def channelWorker(self, modem_transmit_queue: queue.Queue, frame_dispatcher: DISPATCHER):
+        while self.channels_running:
+            # Transfer data between both parties
+            try:
+                transmission = modem_transmit_queue.get(timeout=1)
+                if random.randint(0, 100) < self.loss_probability:
+                    self.logger.info(f"[{threading.current_thread().name}] Frame lost...")
+                    continue
+
+                frame_bytes = transmission['bytes']
+                frame_dispatcher.new_process_data(frame_bytes, None, len(frame_bytes), 0, 0)
+            except queue.Empty:
+                continue
+        self.logger.info(f"[{threading.current_thread().name}] Channel closed.")
+
+    def waitForSession(self, q, outbound=False):
+        key = 'arq-transfer-outbound' if outbound else 'arq-transfer-inbound'
+        while True:
+            ev = q.get()
+            if key in ev and ('success' in ev[key] or 'ABORTED' in ev[key]):
+                self.logger.info(f"[{threading.current_thread().name}] {key} session ended.")
+                break
+
+    def establishChannels(self):
+        self.channels_running = True
+        self.iss_to_irs_channel = threading.Thread(target=self.channelWorker,
+                                                   args=[self.iss_modem.data_queue_received,
+                                                         self.irs_frame_dispatcher],
+                                                   name="ISS to IRS channel")
+        self.iss_to_irs_channel.start()
+
+        self.irs_to_iss_channel = threading.Thread(target=self.channelWorker,
+                                                   args=[self.irs_modem.data_queue_received,
+                                                         self.iss_frame_dispatcher],
+                                                   name="IRS to ISS channel")
+        self.irs_to_iss_channel.start()
+
+    def waitAndCloseChannels(self):
+        self.waitForSession(self.iss_event_queue, True)
+        self.waitForSession(self.irs_event_queue, False)
+        self.channels_running = False
+
+    def testMessageViaSession(self):
+        # set Packet Error Rate (PER) / frame loss probability
+        self.loss_probability = 0
+
+        self.establishChannels()
+        params = {
+            'destination': "XX1XXX-1",
+            'body': 'Hello World',
+        }
+
+        cmd_class = command_message_send.SendMessageCommand
+        command = cmd_class(self.config, self.iss_state_manager, self.iss_event_manager, params)
+        command.run(self.iss_event_manager, self.iss_modem)
+
+        self.waitAndCloseChannels()
+
+
+

tests/test_protocols.py

@@ -51,7 +51,7 @@ class TestProtocols(unittest.TestCase):
 
     def testPingWithAck(self):
         # Run ping command
-        api_params = { "dxcall": "XX1XXX-7"}
+        api_params = { "dxcall": "XX1XXX-6"}
         ping_cmd = PingCommand(self.config, self.state_manager, self.event_manager, api_params)
         #ping_cmd.run(self.event_queue, self.modem)
         frame = ping_cmd.test(self.event_queue)
@@ -64,7 +64,6 @@ class TestProtocols(unittest.TestCase):
         self.shortcutTransmission(event_frame)
         self.assertEventReceivedType('PING_ACK')
         print("PING/PING ACK CHECK SUCCESSFULLY")
-
     def testCQWithQRV(self):
         self.config['MODEM']['respond_to_cq'] = True
 

tools/Modem-Install-Requrements.bat

@@ -1,5 +0,0 @@
-REM Place this batch file in FreeData/modem and then run it
-REM ie. c:\FD-Src\modem
-
-python -m pip install -r ..\requirements.txt
-pause

tools/Modem-Launch.bat

@@ -1,10 +0,0 @@
-REM Place this batch file in FreeData/tnc and then run it
-REM ie. c:\FD-Src\tnc
-
-REM Set environment variable to let modem know where to find config, change if you need to specify a different config
-set FREEDATA_CONFIG=.\config.ini 
-
-REM launch modem
-flask --app server run
-
-pause

tools/Modem-list-audio-devs.bat

@@ -1,6 +0,0 @@
-@echo off
-REM PLace in modem directory and run to retrieve list of audio devices; you'll need the CRC for the config.ini
-
-python ..\tools\list_audio_devices.py
-
-pause

tools/Windows/Modem-Update-Requrements.bat

@@ -0,0 +1,5 @@
+REM Place this batch file in FreeData/modem and then run it to update requirements
+REM ie. c:\FD-Src\modem
+
+python -m pip install --upgrade -r ..\requirements.txt
+pause