Crash_Override/FreeDATA
1
0
Fork 0
mirror of https://github.com/DJ2LS/FreeDATA synced 2024-05-14 08:04:33 +00:00
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #242 from DJ2LS/ls_rx_buffer_improvement

Browse source
This commit is contained in:
DJ2LS 2022-09-10 19:41:46 +02:00 committed by GitHub
parent 206bc6f068 8deaa5756d
commit 68c430fcaa
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
16 changed files with 157 additions and 62 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

38
README.md
View file

@ -9,17 +9,45 @@ My attempt to create a free and open-source TNC with a GUI for [codec2](https://
[![CodeFactor](https://www.codefactor.io/repository/github/dj2ls/freedata/badge)](https://www.codefactor.io/repository/github/dj2ls/freedata)
Please keep in mind, that this project is still a prototype with many issues which need to be solved.
Build steps for other OS than Ubuntu are provided, but not fully working, yet.
Please keep in mind, this project is still under development with many issues which need to be solved.
Please check the ['Releases'](https://github.com/DJ2LS/FreeDATA/releases) section for downloading nightly builds
### existing/planned TNC features
- [x] network based
- [x] raw data transfer
- [x] fft output
- [x] JSON based commands
- [x] speed levels
- [x] ARQ - stop and wait
- [x] SNR operation level SNR > 0dB MPP/MPD
- [x] file compression
- [x] auto updater
- [ ] channel measurement
- [ ] hybrid ARQ
- [ ] SNR operation level SNR @ -5dB MPP/MPD
- [ ] tbc...
### existing/planned Chat features
- [x] chat messages
- [x] file transfer
- [x] file transfer with chat message
- [x] database for not loosing messages
- [x] smileys
- [ ] database network sync
- [ ] voice messages
- [ ] image compression
- [ ] status messages
- [ ] avatars
- [ ] tbc...
## Preview
![preview](https://github.com/DJ2LS/FreeDATA/blob/main/documentation/FreeDATA_preview.gif?raw=true "Preview")
## Data Preview
![preview](https://github.com/DJ2LS/FreeDATA/blob/main/documentation/data_preview.gif?raw=true "Preview")
## Chat Preview
![preview](https://github.com/DJ2LS/FreeDATA/blob/main/documentation/chat_preview_fast.gif?raw=true "Preview")
## Installation
Please check the [wiki](https://wiki.freedata.app) for installation instructions
Please check the ['Releases'](https://github.com/DJ2LS/FreeDATA/releases) section for downloading precompiled builds
## Credits
* David Rowe and the FreeDV team for developing the modem and libraries -

BIN
documentation/chat_preview_fast.gif Normal file
View file

Binary file not shown.
Side by side

After

Width:  |  Height:  |  Size: 384 KiB

BIN
documentation/data_preview.gif Normal file
View file

Binary file not shown.
Side by side

After

Width:  |  Height:  |  Size: 2.9 MiB

5
gui/daemon.js
View file

@ -217,7 +217,7 @@ exports.getDaemonState = function() {
// START TNC
// ` `== multi line string
exports.startTNC = function(mycall, mygrid, rx_audio, tx_audio, radiocontrol, devicename, deviceport, pttprotocol, pttport, serialspeed, data_bits, stop_bits, handshake, rigctld_ip, rigctld_port, enable_fft, enable_scatter, low_bandwidth_mode, tuning_range_fmin, tuning_range_fmax, enable_fsk, tx_audio_level, respond_to_cq) {
exports.startTNC = function(mycall, mygrid, rx_audio, tx_audio, radiocontrol, devicename, deviceport, pttprotocol, pttport, serialspeed, data_bits, stop_bits, handshake, rigctld_ip, rigctld_port, enable_fft, enable_scatter, low_bandwidth_mode, tuning_range_fmin, tuning_range_fmax, enable_fsk, tx_audio_level, respond_to_cq, rx_buffer_size) {
var json_command = JSON.stringify({
type: 'set',
command: 'start_tnc',
@ -244,7 +244,8 @@ exports.startTNC = function(mycall, mygrid, rx_audio, tx_audio, radiocontrol, de
tuning_range_fmin : tuning_range_fmin,
tuning_range_fmax : tuning_range_fmax,
tx_audio_level : tx_audio_level,
respond_to_cq : respond_to_cq
respond_to_cq : respond_to_cq,
rx_buffer_size : rx_buffer_size
}]
})

3
gui/main.js
View file

@ -77,7 +77,8 @@ const configDefaultSettings = '{\
"received_files_folder" : "None",\
"tuning_range_fmin" : "-50.0",\
"tuning_range_fmax" : "50.0",\
"respond_to_cq" : "True" \
"respond_to_cq" : "True",\
"rx_buffer_size" : "16" \
}';
if (!fs.existsSync(configPath)) {

29
gui/preload-chat.js
View file

@ -187,7 +187,6 @@ window.addEventListener('DOMContentLoaded', () => {
})
// NEW CHAT
document.getElementById("createNewChatButton").addEventListener("click", () => {
var dxcallsign = document.getElementById('chatModuleNewDxCall').value;
@ -277,9 +276,6 @@ db.post({
update_chat_obj_by_uuid(uuid);
// scroll to bottom
var element = document.getElementById("message-container");
element.scrollTo(0, element.scrollHeight);
// clear input
document.getElementById('chatModuleMessage').value = ''
@ -521,10 +517,10 @@ update_chat = function(obj) {
//document.getElementById('chatModuleDxCall').value = dxcallsign;
selected_callsign = dxcallsign;
// scroll to bottom
var element = document.getElementById("message-container");
//console.log(element.scrollHeight)
element.scrollTo(0, element.scrollHeight);
setTimeout(scrollMessagesToBottom, 200);
});
@ -662,9 +658,6 @@ update_chat = function(obj) {
// CHECK CHECK CHECK --> This could be done better
var id = "chat-" + obj.dxcallsign
document.getElementById(id).insertAdjacentHTML("beforeend", new_message);
//var element = document.getElementById("message-container");
//console.log(element.scrollHeight)
/* UPDATE EXISTING ELEMENTS */
} else if (document.getElementById('msg-' + obj._id)) {
@ -730,9 +723,6 @@ update_chat = function(obj) {
// handle doc
console.log(doc)
var filename = Object.keys(obj._attachments)[0]
var filetype = filename.content_type
@ -774,6 +764,10 @@ update_chat = function(obj) {
}
//window.location = window.location
// scroll to bottom on new message
scrollMessagesToBottom();
}
@ -869,4 +863,11 @@ add_obj_to_database = function(obj){
}).catch(function(err) {
console.log(err);
});
}
// Scroll to bottom of message-container
function scrollMessagesToBottom() {
var messageBody = document.getElementById('message-container');
messageBody.scrollTop = messageBody.scrollHeight - messageBody.clientHeight;
}

24
gui/preload-main.js
View file

@ -117,9 +117,7 @@ document.getElementById('openReceivedFilesFolder').addEventListener('click', ()
//document.getElementById("respondCQSwitch").value = config.respond_to_cq;
document.getElementById("received_files_folder").value = config.received_files_folder;
if(config.enable_scatter == 'True'){
document.getElementById("scatterSwitch").checked = true;
} else {
@ -175,8 +173,10 @@ document.getElementById('openReceivedFilesFolder').addEventListener('click', ()
// Update TX Audio Level
document.getElementById("audioLevelTXvalue").innerHTML = parseInt(config.tx_audio_level);
document.getElementById("audioLevelTX").value = parseInt(config.tx_audio_level);
// Update RX Buffer Size
document.getElementById("rx_buffer_size").value = config.rx_buffer_size;
if (config.spectrum == 'waterfall') {
document.getElementById("waterfall-scatter-switch1").checked = true;
document.getElementById("waterfall-scatter-switch2").checked = false;
@ -590,8 +590,13 @@ document.getElementById('openReceivedFilesFolder').addEventListener('click', ()
fs.writeFileSync(configPath, JSON.stringify(config, null, 2));
});
// rx buffer size selector clicked
document.getElementById("rx_buffer_size").addEventListener("click", () => {
var rx_buffer_size = document.getElementById("rx_buffer_size").value;
config.rx_buffer_size = rx_buffer_size;
fs.writeFileSync(configPath, JSON.stringify(config, null, 2));
});
//screen size
@ -710,7 +715,7 @@ document.getElementById('openReceivedFilesFolder').addEventListener('click', ()
}
var tx_audio_level = document.getElementById("audioLevelTX").value;
var rx_buffer_size = document.getElementById("rx_buffer_size").value;
config.radiocontrol = radiocontrol;
config.mycall = callsign_ssid;
@ -735,6 +740,7 @@ document.getElementById('openReceivedFilesFolder').addEventListener('click', ()
config.low_bandwidth_mode = low_bandwidth_mode;
config.tx_audio_level = tx_audio_level;
config.respond_to_cq = respond_to_cq;
config.rx_buffer_size = rx_buffer_size;
fs.writeFileSync(configPath, JSON.stringify(config, null, 2));
@ -753,7 +759,7 @@ document.getElementById('openReceivedFilesFolder').addEventListener('click', ()
*/
daemon.startTNC(callsign_ssid, mygrid, rx_audio, tx_audio, radiocontrol, deviceid, deviceport, pttprotocol, pttport, serialspeed, data_bits, stop_bits, handshake, rigctld_ip, rigctld_port, enable_fft, enable_scatter, low_bandwidth_mode, tuning_range_fmin, tuning_range_fmax, enable_fsk, tx_audio_level, respond_to_cq);
daemon.startTNC(callsign_ssid, mygrid, rx_audio, tx_audio, radiocontrol, deviceid, deviceport, pttprotocol, pttport, serialspeed, data_bits, stop_bits, handshake, rigctld_ip, rigctld_port, enable_fft, enable_scatter, low_bandwidth_mode, tuning_range_fmin, tuning_range_fmax, enable_fsk, tx_audio_level, respond_to_cq, rx_buffer_size);
})

4
gui/src/chat-module.html
View file

@ -28,7 +28,7 @@
</div>
<div class="container-fluid">
<div class="row h-100">
<div class="col-3 p-2 bg-light">
<div class="col-4 p-2 bg-light">
<! ------Chats area ---------------------------------------------------------------------->
<div class="container-fluid m-0 p-0">
<div class="input-group bottom-0 m-0 w-100">
@ -41,7 +41,7 @@
<div class="list-group" id="list-tab" role="tablist"> </div>
</div>
</div>
<div class="col-9 border vh-100 p-0">
<div class="col-8 border vh-100 p-0">
<! ------ chat navbar ---------------------------------------------------------------------->
<div class="container-fluid m-2 p-0">
<div class="input-group bottom-0">

18
gui/src/index.html
View file

@ -1275,6 +1275,24 @@
</div>
</label>
</div>
<div class="input-group input-group-sm mb-1">
<label class="input-group-text w-50">rx buffer size</label>
<label class="input-group-text bg-white w-50">
<select class="form-select form-select-sm" id="rx_buffer_size">
<option value="1">1</option>
<option value="2">2</option>
<option value="4">4</option>
<option value="8">8</option>
<option value="16">16</option>
<option value="32">32</option>
<option value="64">64</option>
<option value="128">128</option>
<option value="256">256</option>
<option value="512">512</option>
<option value="1024">1024</option>
</select>
</label>
</div>
</div>
</div>
</div>

8
gui/src/styles.css
View file

@ -1,5 +1,5 @@
/* disable scrolling in main window */
body {
/* disable scrolling in main window */
body {
padding-right: 0px !important;
overflow-y: hidden !important;
overflow-x: hidden !important;
@ -17,6 +17,10 @@ body {
color: black;
}
/* smooth scrolling */
html {
scroll-behavior: smooth;
}
/* hide scrollbar in callsign list */
#callsignlist::-webkit-scrollbar {

7
tnc/daemon.py
View file

@ -32,7 +32,7 @@ import structlog
import ujson as json
# signal handler for closing aplication
# signal handler for closing application
def signal_handler(sig, frame):
"""
Signal handler for closing the network socket on app exit
@ -151,6 +151,8 @@ class DAEMON:
# data[21] enable FSK
# data[22] tx-audio-level
# data[23] respond_to_cq
# data[24] rx_buffer_size
if data[0] == "STARTTNC":
self.log.warning("[DMN] Starting TNC", rig=data[5], port=data[6])
@ -237,6 +239,9 @@ class DAEMON:
if data[23] == "True":
options.append("--qrv")
options.append("--rx-buffer-size")
options.append(data[24])
# Try running tnc from binary, else run from source
# This helps running the tnc in a developer environment
try:

25
tnc/data_handler.py
View file

@ -25,7 +25,7 @@ import structlog
import ujson as json
from codec2 import FREEDV_MODE
from exceptions import NoCallsign
from queues import DATA_QUEUE_RECEIVED, DATA_QUEUE_TRANSMIT
from queues import DATA_QUEUE_RECEIVED, DATA_QUEUE_TRANSMIT, RX_BUFFER
from static import FRAME_TYPE as FR_TYPE
TESTMODE = False
@ -697,7 +697,27 @@ class DATA:
# Re-code data_frame in base64, UTF-8 for JSON UI communication.
base64_data = base64.b64encode(data_frame).decode("UTF-8")
static.RX_BUFFER.append(
# check if RX_BUFFER isn't full
if not RX_BUFFER.full():
# make sure we have always the correct buffer size
RX_BUFFER.maxsize = static.RX_BUFFER_SIZE
else:
# if full, free space by getting an item
self.log.info(
"[TNC] ARQ | RX | RX_BUFFER FULL - dropping old data",
buffer_size=RX_BUFFER.qsize(),
maxsize=static.RX_BUFFER_SIZE
)
RX_BUFFER.get()
# add item to RX_BUFFER
self.log.info(
"[TNC] ARQ | RX | saving data to rx buffer",
buffer_size=RX_BUFFER.qsize() + 1,
maxsize=RX_BUFFER.maxsize
)
RX_BUFFER.put(
[
self.transmission_uuid,
timestamp,
@ -706,6 +726,7 @@ class DATA:
base64_data,
]
)
self.send_data_to_socket_queue(
freedata="tnc-message",
arq="transmission",

9
tnc/main.py
View file

@ -224,7 +224,13 @@ if __name__ == "__main__":
help="Set the tx audio level at an early stage",
type=int,
)
PARSER.add_argument(
"--rx-buffer-size",
dest="rx_buffer_size",
default=16,
help="Set the maximum size of rx buffer.",
type=int,
)
ARGS = PARSER.parse_args()
# additional step for being sure our callsign is correctly
@ -260,6 +266,7 @@ if __name__ == "__main__":
static.TUNING_RANGE_FMAX = ARGS.tuning_range_fmax
static.TX_AUDIO_LEVEL = ARGS.tx_audio_level
static.RESPOND_TO_CQ = ARGS.enable_respond_to_cq
static.RX_BUFFER_SIZE = ARGS.rx_buffer_size
# we need to wait until we got all parameters from argparse first before we can load the other modules
import sock

4
tnc/queues.py
View file

@ -2,6 +2,7 @@
Hold queues used by more than one module to eliminate cyclic imports.
"""
import queue
import static
DATA_QUEUE_TRANSMIT = queue.Queue()
DATA_QUEUE_RECEIVED = queue.Queue()
@ -9,3 +10,6 @@ DATA_QUEUE_RECEIVED = queue.Queue()
# Initialize FIFO queue to store received frames
MODEM_RECEIVED_QUEUE = queue.Queue()
MODEM_TRANSMIT_QUEUE = queue.Queue()
# Initialize FIFO queue to finally store received data
RX_BUFFER = queue.Queue(maxsize=static.RX_BUFFER_SIZE)

42
tnc/sock.py
View file

@ -30,7 +30,7 @@ import static
import structlog
import ujson as json
from exceptions import NoCallsign
from queues import DATA_QUEUE_TRANSMIT
from queues import DATA_QUEUE_TRANSMIT, RX_BUFFER
SOCKET_QUEUE = queue.Queue()
DAEMON_QUEUE = queue.Queue()
@ -420,24 +420,22 @@ def process_tnc_commands(data):
"data-array": [],
}
for i in range(len(static.RX_BUFFER)):
# print(static.RX_BUFFER[i][4])
# rawdata = json.loads(static.RX_BUFFER[i][4])
base64_data = static.RX_BUFFER[i][4]
output["data-array"].append(
{
"uuid": static.RX_BUFFER[i][0],
"timestamp": static.RX_BUFFER[i][1],
"dxcallsign": str(static.RX_BUFFER[i][2], "utf-8"),
"dxgrid": str(static.RX_BUFFER[i][3], "utf-8"),
"data": base64_data,
}
)
jsondata = json.dumps(output)
# self.request.sendall(bytes(jsondata, encoding))
SOCKET_QUEUE.put(jsondata)
command_response("rx_buffer", True)
if not RX_BUFFER.empty():
for _buffer_length in range(RX_BUFFER.qsize()):
base64_data = RX_BUFFER.queue[_buffer_length][4]
output["data-array"].append(
{
"uuid": RX_BUFFER.queue[_buffer_length][0],
"timestamp": RX_BUFFER.queue[_buffer_length][1],
"dxcallsign": str(RX_BUFFER.queue[_buffer_length][2], "utf-8"),
"dxgrid": str(RX_BUFFER.queue[_buffer_length][3], "utf-8"),
"data": base64_data,
}
)
jsondata = json.dumps(output)
# self.request.sendall(bytes(jsondata, encoding))
SOCKET_QUEUE.put(jsondata)
command_response("rx_buffer", True)
except Exception as err:
command_response("rx_buffer", False)
@ -452,7 +450,7 @@ def process_tnc_commands(data):
and received_json["command"] == "del_rx_buffer"
):
try:
static.RX_BUFFER = []
RX_BUFFER.queue.clear()
command_response("del_rx_buffer", True)
except Exception as err:
command_response("del_rx_buffer", False)
@ -489,7 +487,7 @@ def send_tnc_state():
"fft": str(static.FFT),
"channel_busy": str(static.CHANNEL_BUSY),
"scatter": static.SCATTER,
"rx_buffer_length": str(len(static.RX_BUFFER)),
"rx_buffer_length": str(RX_BUFFER.qsize()),
"rx_msg_buffer_length": str(len(static.RX_MSG_BUFFER)),
"arq_bytes_per_minute": str(static.ARQ_BYTES_PER_MINUTE),
"arq_bytes_per_minute_burst": str(static.ARQ_BYTES_PER_MINUTE_BURST),
@ -609,6 +607,7 @@ def process_daemon_commands(data):
tuning_range_fmax = str(received_json["parameter"][0]["tuning_range_fmax"])
tx_audio_level = str(received_json["parameter"][0]["tx_audio_level"])
respond_to_cq = str(received_json["parameter"][0]["respond_to_cq"])
rx_buffer_size = str(received_json["parameter"][0]["rx_buffer_size"])
# print some debugging parameters
for item in received_json["parameter"][0]:
@ -643,6 +642,7 @@ def process_daemon_commands(data):
enable_fsk,
tx_audio_level,
respond_to_cq,
rx_buffer_size,
]
)
command_response("start_tnc", True)

3
tnc/static.py
View file

@ -109,11 +109,10 @@ BEACON_STATE: bool = False
BEACON_PAUSE: bool = False
# ------- RX BUFFER
RX_BUFFER: list = []
RX_MSG_BUFFER: list = []
RX_BURST_BUFFER: list = []
RX_FRAME_BUFFER: bytes = b""
# RX_BUFFER_SIZE: int = 0
RX_BUFFER_SIZE: int = 16
# ------- HEARD STATIONS BUFFER
HEARD_STATIONS: list = []