diff --git a/README.md b/README.md index 9094ed1e..511a9138 100644 --- a/README.md +++ b/README.md @@ -9,17 +9,45 @@ My attempt to create a free and open-source TNC with a GUI for [codec2](https:// [![CodeFactor](https://www.codefactor.io/repository/github/dj2ls/freedata/badge)](https://www.codefactor.io/repository/github/dj2ls/freedata) -Please keep in mind, that this project is still a prototype with many issues which need to be solved. -Build steps for other OS than Ubuntu are provided, but not fully working, yet. +Please keep in mind, this project is still under development with many issues which need to be solved. -Please check the ['Releases'](https://github.com/DJ2LS/FreeDATA/releases) section for downloading nightly builds +### existing/planned TNC features +- [x] network based +- [x] raw data transfer +- [x] fft output +- [x] JSON based commands +- [x] speed levels +- [x] ARQ - stop and wait +- [x] SNR operation level SNR > 0dB MPP/MPD +- [x] file compression +- [x] auto updater +- [ ] channel measurement +- [ ] hybrid ARQ +- [ ] SNR operation level SNR @ -5dB MPP/MPD +- [ ] tbc... +### existing/planned Chat features + - [x] chat messages + - [x] file transfer + - [x] file transfer with chat message + - [x] database for not loosing messages + - [x] smileys + - [ ] database network sync + - [ ] voice messages + - [ ] image compression + - [ ] status messages + - [ ] avatars + - [ ] tbc... -## Preview -![preview](https://github.com/DJ2LS/FreeDATA/blob/main/documentation/FreeDATA_preview.gif?raw=true "Preview") +## Data Preview +![preview](https://github.com/DJ2LS/FreeDATA/blob/main/documentation/data_preview.gif?raw=true "Preview") + +## Chat Preview +![preview](https://github.com/DJ2LS/FreeDATA/blob/main/documentation/chat_preview_fast.gif?raw=true "Preview") ## Installation Please check the [wiki](https://wiki.freedata.app) for installation instructions +Please check the ['Releases'](https://github.com/DJ2LS/FreeDATA/releases) section for downloading precompiled builds ## Credits * David Rowe and the FreeDV team for developing the modem and libraries - diff --git a/documentation/chat_preview_fast.gif b/documentation/chat_preview_fast.gif new file mode 100644 index 00000000..b85193db Binary files /dev/null and b/documentation/chat_preview_fast.gif differ diff --git a/documentation/data_preview.gif b/documentation/data_preview.gif new file mode 100644 index 00000000..c54adb78 Binary files /dev/null and b/documentation/data_preview.gif differ diff --git a/gui/daemon.js b/gui/daemon.js index c22ec79c..3d5f79c4 100644 --- a/gui/daemon.js +++ b/gui/daemon.js @@ -217,7 +217,7 @@ exports.getDaemonState = function() { // START TNC // ` `== multi line string -exports.startTNC = function(mycall, mygrid, rx_audio, tx_audio, radiocontrol, devicename, deviceport, pttprotocol, pttport, serialspeed, data_bits, stop_bits, handshake, rigctld_ip, rigctld_port, enable_fft, enable_scatter, low_bandwidth_mode, tuning_range_fmin, tuning_range_fmax, enable_fsk, tx_audio_level, respond_to_cq) { +exports.startTNC = function(mycall, mygrid, rx_audio, tx_audio, radiocontrol, devicename, deviceport, pttprotocol, pttport, serialspeed, data_bits, stop_bits, handshake, rigctld_ip, rigctld_port, enable_fft, enable_scatter, low_bandwidth_mode, tuning_range_fmin, tuning_range_fmax, enable_fsk, tx_audio_level, respond_to_cq, rx_buffer_size) { var json_command = JSON.stringify({ type: 'set', command: 'start_tnc', @@ -244,7 +244,8 @@ exports.startTNC = function(mycall, mygrid, rx_audio, tx_audio, radiocontrol, de tuning_range_fmin : tuning_range_fmin, tuning_range_fmax : tuning_range_fmax, tx_audio_level : tx_audio_level, - respond_to_cq : respond_to_cq + respond_to_cq : respond_to_cq, + rx_buffer_size : rx_buffer_size }] }) diff --git a/gui/main.js b/gui/main.js index 3b889a78..74dcaf78 100644 --- a/gui/main.js +++ b/gui/main.js @@ -77,7 +77,8 @@ const configDefaultSettings = '{\ "received_files_folder" : "None",\ "tuning_range_fmin" : "-50.0",\ "tuning_range_fmax" : "50.0",\ - "respond_to_cq" : "True" \ + "respond_to_cq" : "True",\ + "rx_buffer_size" : "16" \ }'; if (!fs.existsSync(configPath)) { diff --git a/gui/preload-chat.js b/gui/preload-chat.js index 002a7c97..c9905661 100644 --- a/gui/preload-chat.js +++ b/gui/preload-chat.js @@ -187,7 +187,6 @@ window.addEventListener('DOMContentLoaded', () => { }) // NEW CHAT - document.getElementById("createNewChatButton").addEventListener("click", () => { var dxcallsign = document.getElementById('chatModuleNewDxCall').value; @@ -277,9 +276,6 @@ db.post({ update_chat_obj_by_uuid(uuid); - // scroll to bottom - var element = document.getElementById("message-container"); - element.scrollTo(0, element.scrollHeight); // clear input document.getElementById('chatModuleMessage').value = '' @@ -521,10 +517,10 @@ update_chat = function(obj) { //document.getElementById('chatModuleDxCall').value = dxcallsign; selected_callsign = dxcallsign; - // scroll to bottom - var element = document.getElementById("message-container"); - //console.log(element.scrollHeight) - element.scrollTo(0, element.scrollHeight); + + setTimeout(scrollMessagesToBottom, 200); + + }); @@ -662,9 +658,6 @@ update_chat = function(obj) { // CHECK CHECK CHECK --> This could be done better var id = "chat-" + obj.dxcallsign document.getElementById(id).insertAdjacentHTML("beforeend", new_message); - - //var element = document.getElementById("message-container"); - //console.log(element.scrollHeight) /* UPDATE EXISTING ELEMENTS */ } else if (document.getElementById('msg-' + obj._id)) { @@ -730,9 +723,6 @@ update_chat = function(obj) { // handle doc console.log(doc) - - - var filename = Object.keys(obj._attachments)[0] var filetype = filename.content_type @@ -774,6 +764,10 @@ update_chat = function(obj) { } //window.location = window.location + + // scroll to bottom on new message + scrollMessagesToBottom(); + } @@ -869,4 +863,11 @@ add_obj_to_database = function(obj){ }).catch(function(err) { console.log(err); }); +} + + +// Scroll to bottom of message-container +function scrollMessagesToBottom() { + var messageBody = document.getElementById('message-container'); + messageBody.scrollTop = messageBody.scrollHeight - messageBody.clientHeight; } \ No newline at end of file diff --git a/gui/preload-main.js b/gui/preload-main.js index 32775e83..c0c0d697 100644 --- a/gui/preload-main.js +++ b/gui/preload-main.js @@ -117,9 +117,7 @@ document.getElementById('openReceivedFilesFolder').addEventListener('click', () //document.getElementById("respondCQSwitch").value = config.respond_to_cq; document.getElementById("received_files_folder").value = config.received_files_folder; - - - + if(config.enable_scatter == 'True'){ document.getElementById("scatterSwitch").checked = true; } else { @@ -175,8 +173,10 @@ document.getElementById('openReceivedFilesFolder').addEventListener('click', () // Update TX Audio Level document.getElementById("audioLevelTXvalue").innerHTML = parseInt(config.tx_audio_level); document.getElementById("audioLevelTX").value = parseInt(config.tx_audio_level); - - + + // Update RX Buffer Size + document.getElementById("rx_buffer_size").value = config.rx_buffer_size; + if (config.spectrum == 'waterfall') { document.getElementById("waterfall-scatter-switch1").checked = true; document.getElementById("waterfall-scatter-switch2").checked = false; @@ -590,8 +590,13 @@ document.getElementById('openReceivedFilesFolder').addEventListener('click', () fs.writeFileSync(configPath, JSON.stringify(config, null, 2)); }); - - + + // rx buffer size selector clicked + document.getElementById("rx_buffer_size").addEventListener("click", () => { + var rx_buffer_size = document.getElementById("rx_buffer_size").value; + config.rx_buffer_size = rx_buffer_size; + fs.writeFileSync(configPath, JSON.stringify(config, null, 2)); + }); //screen size @@ -710,7 +715,7 @@ document.getElementById('openReceivedFilesFolder').addEventListener('click', () } var tx_audio_level = document.getElementById("audioLevelTX").value; - + var rx_buffer_size = document.getElementById("rx_buffer_size").value; config.radiocontrol = radiocontrol; config.mycall = callsign_ssid; @@ -735,6 +740,7 @@ document.getElementById('openReceivedFilesFolder').addEventListener('click', () config.low_bandwidth_mode = low_bandwidth_mode; config.tx_audio_level = tx_audio_level; config.respond_to_cq = respond_to_cq; + config.rx_buffer_size = rx_buffer_size; fs.writeFileSync(configPath, JSON.stringify(config, null, 2)); @@ -753,7 +759,7 @@ document.getElementById('openReceivedFilesFolder').addEventListener('click', () */ - daemon.startTNC(callsign_ssid, mygrid, rx_audio, tx_audio, radiocontrol, deviceid, deviceport, pttprotocol, pttport, serialspeed, data_bits, stop_bits, handshake, rigctld_ip, rigctld_port, enable_fft, enable_scatter, low_bandwidth_mode, tuning_range_fmin, tuning_range_fmax, enable_fsk, tx_audio_level, respond_to_cq); + daemon.startTNC(callsign_ssid, mygrid, rx_audio, tx_audio, radiocontrol, deviceid, deviceport, pttprotocol, pttport, serialspeed, data_bits, stop_bits, handshake, rigctld_ip, rigctld_port, enable_fft, enable_scatter, low_bandwidth_mode, tuning_range_fmin, tuning_range_fmax, enable_fsk, tx_audio_level, respond_to_cq, rx_buffer_size); }) diff --git a/gui/src/chat-module.html b/gui/src/chat-module.html index 683df8fe..6ab0ad32 100644 --- a/gui/src/chat-module.html +++ b/gui/src/chat-module.html @@ -28,7 +28,7 @@
-
+
@@ -41,7 +41,7 @@
-
+
diff --git a/gui/src/index.html b/gui/src/index.html index aa13cf12..d1dc53fb 100644 --- a/gui/src/index.html +++ b/gui/src/index.html @@ -1275,6 +1275,24 @@
+
+ + +
diff --git a/gui/src/styles.css b/gui/src/styles.css index 04038c8c..b8c6006e 100644 --- a/gui/src/styles.css +++ b/gui/src/styles.css @@ -1,5 +1,5 @@ -/* disable scrolling in main window */ -body { +/* disable scrolling in main window */ +body { padding-right: 0px !important; overflow-y: hidden !important; overflow-x: hidden !important; @@ -17,6 +17,10 @@ body { color: black; } +/* smooth scrolling */ +html { + scroll-behavior: smooth; +} /* hide scrollbar in callsign list */ #callsignlist::-webkit-scrollbar { diff --git a/tnc/daemon.py b/tnc/daemon.py index 89452384..dc638b47 100755 --- a/tnc/daemon.py +++ b/tnc/daemon.py @@ -32,7 +32,7 @@ import structlog import ujson as json -# signal handler for closing aplication +# signal handler for closing application def signal_handler(sig, frame): """ Signal handler for closing the network socket on app exit @@ -151,6 +151,8 @@ class DAEMON: # data[21] enable FSK # data[22] tx-audio-level # data[23] respond_to_cq + # data[24] rx_buffer_size + if data[0] == "STARTTNC": self.log.warning("[DMN] Starting TNC", rig=data[5], port=data[6]) @@ -237,6 +239,9 @@ class DAEMON: if data[23] == "True": options.append("--qrv") + options.append("--rx-buffer-size") + options.append(data[24]) + # Try running tnc from binary, else run from source # This helps running the tnc in a developer environment try: diff --git a/tnc/data_handler.py b/tnc/data_handler.py index f3314785..4c7a41ad 100644 --- a/tnc/data_handler.py +++ b/tnc/data_handler.py @@ -25,7 +25,7 @@ import structlog import ujson as json from codec2 import FREEDV_MODE from exceptions import NoCallsign -from queues import DATA_QUEUE_RECEIVED, DATA_QUEUE_TRANSMIT +from queues import DATA_QUEUE_RECEIVED, DATA_QUEUE_TRANSMIT, RX_BUFFER from static import FRAME_TYPE as FR_TYPE TESTMODE = False @@ -697,7 +697,27 @@ class DATA: # Re-code data_frame in base64, UTF-8 for JSON UI communication. base64_data = base64.b64encode(data_frame).decode("UTF-8") - static.RX_BUFFER.append( + + # check if RX_BUFFER isn't full + if not RX_BUFFER.full(): + # make sure we have always the correct buffer size + RX_BUFFER.maxsize = static.RX_BUFFER_SIZE + else: + # if full, free space by getting an item + self.log.info( + "[TNC] ARQ | RX | RX_BUFFER FULL - dropping old data", + buffer_size=RX_BUFFER.qsize(), + maxsize=static.RX_BUFFER_SIZE + ) + RX_BUFFER.get() + + # add item to RX_BUFFER + self.log.info( + "[TNC] ARQ | RX | saving data to rx buffer", + buffer_size=RX_BUFFER.qsize() + 1, + maxsize=RX_BUFFER.maxsize + ) + RX_BUFFER.put( [ self.transmission_uuid, timestamp, @@ -706,6 +726,7 @@ class DATA: base64_data, ] ) + self.send_data_to_socket_queue( freedata="tnc-message", arq="transmission", diff --git a/tnc/main.py b/tnc/main.py index 06e2ca46..64328580 100755 --- a/tnc/main.py +++ b/tnc/main.py @@ -224,7 +224,13 @@ if __name__ == "__main__": help="Set the tx audio level at an early stage", type=int, ) - + PARSER.add_argument( + "--rx-buffer-size", + dest="rx_buffer_size", + default=16, + help="Set the maximum size of rx buffer.", + type=int, + ) ARGS = PARSER.parse_args() # additional step for being sure our callsign is correctly @@ -260,6 +266,7 @@ if __name__ == "__main__": static.TUNING_RANGE_FMAX = ARGS.tuning_range_fmax static.TX_AUDIO_LEVEL = ARGS.tx_audio_level static.RESPOND_TO_CQ = ARGS.enable_respond_to_cq + static.RX_BUFFER_SIZE = ARGS.rx_buffer_size # we need to wait until we got all parameters from argparse first before we can load the other modules import sock diff --git a/tnc/queues.py b/tnc/queues.py index 259218f6..926db4e4 100644 --- a/tnc/queues.py +++ b/tnc/queues.py @@ -2,6 +2,7 @@ Hold queues used by more than one module to eliminate cyclic imports. """ import queue +import static DATA_QUEUE_TRANSMIT = queue.Queue() DATA_QUEUE_RECEIVED = queue.Queue() @@ -9,3 +10,6 @@ DATA_QUEUE_RECEIVED = queue.Queue() # Initialize FIFO queue to store received frames MODEM_RECEIVED_QUEUE = queue.Queue() MODEM_TRANSMIT_QUEUE = queue.Queue() + +# Initialize FIFO queue to finally store received data +RX_BUFFER = queue.Queue(maxsize=static.RX_BUFFER_SIZE) diff --git a/tnc/sock.py b/tnc/sock.py index f62a321f..9ef12714 100644 --- a/tnc/sock.py +++ b/tnc/sock.py @@ -30,7 +30,7 @@ import static import structlog import ujson as json from exceptions import NoCallsign -from queues import DATA_QUEUE_TRANSMIT +from queues import DATA_QUEUE_TRANSMIT, RX_BUFFER SOCKET_QUEUE = queue.Queue() DAEMON_QUEUE = queue.Queue() @@ -420,24 +420,22 @@ def process_tnc_commands(data): "data-array": [], } - for i in range(len(static.RX_BUFFER)): - # print(static.RX_BUFFER[i][4]) - # rawdata = json.loads(static.RX_BUFFER[i][4]) - base64_data = static.RX_BUFFER[i][4] - output["data-array"].append( - { - "uuid": static.RX_BUFFER[i][0], - "timestamp": static.RX_BUFFER[i][1], - "dxcallsign": str(static.RX_BUFFER[i][2], "utf-8"), - "dxgrid": str(static.RX_BUFFER[i][3], "utf-8"), - "data": base64_data, - } - ) - - jsondata = json.dumps(output) - # self.request.sendall(bytes(jsondata, encoding)) - SOCKET_QUEUE.put(jsondata) - command_response("rx_buffer", True) + if not RX_BUFFER.empty(): + for _buffer_length in range(RX_BUFFER.qsize()): + base64_data = RX_BUFFER.queue[_buffer_length][4] + output["data-array"].append( + { + "uuid": RX_BUFFER.queue[_buffer_length][0], + "timestamp": RX_BUFFER.queue[_buffer_length][1], + "dxcallsign": str(RX_BUFFER.queue[_buffer_length][2], "utf-8"), + "dxgrid": str(RX_BUFFER.queue[_buffer_length][3], "utf-8"), + "data": base64_data, + } + ) + jsondata = json.dumps(output) + # self.request.sendall(bytes(jsondata, encoding)) + SOCKET_QUEUE.put(jsondata) + command_response("rx_buffer", True) except Exception as err: command_response("rx_buffer", False) @@ -452,7 +450,7 @@ def process_tnc_commands(data): and received_json["command"] == "del_rx_buffer" ): try: - static.RX_BUFFER = [] + RX_BUFFER.queue.clear() command_response("del_rx_buffer", True) except Exception as err: command_response("del_rx_buffer", False) @@ -489,7 +487,7 @@ def send_tnc_state(): "fft": str(static.FFT), "channel_busy": str(static.CHANNEL_BUSY), "scatter": static.SCATTER, - "rx_buffer_length": str(len(static.RX_BUFFER)), + "rx_buffer_length": str(RX_BUFFER.qsize()), "rx_msg_buffer_length": str(len(static.RX_MSG_BUFFER)), "arq_bytes_per_minute": str(static.ARQ_BYTES_PER_MINUTE), "arq_bytes_per_minute_burst": str(static.ARQ_BYTES_PER_MINUTE_BURST), @@ -609,6 +607,7 @@ def process_daemon_commands(data): tuning_range_fmax = str(received_json["parameter"][0]["tuning_range_fmax"]) tx_audio_level = str(received_json["parameter"][0]["tx_audio_level"]) respond_to_cq = str(received_json["parameter"][0]["respond_to_cq"]) + rx_buffer_size = str(received_json["parameter"][0]["rx_buffer_size"]) # print some debugging parameters for item in received_json["parameter"][0]: @@ -643,6 +642,7 @@ def process_daemon_commands(data): enable_fsk, tx_audio_level, respond_to_cq, + rx_buffer_size, ] ) command_response("start_tnc", True) diff --git a/tnc/static.py b/tnc/static.py index fe276965..4351f5a2 100644 --- a/tnc/static.py +++ b/tnc/static.py @@ -109,11 +109,10 @@ BEACON_STATE: bool = False BEACON_PAUSE: bool = False # ------- RX BUFFER -RX_BUFFER: list = [] RX_MSG_BUFFER: list = [] RX_BURST_BUFFER: list = [] RX_FRAME_BUFFER: bytes = b"" -# RX_BUFFER_SIZE: int = 0 +RX_BUFFER_SIZE: int = 16 # ------- HEARD STATIONS BUFFER HEARD_STATIONS: list = []