Crash_Override/FreeDATA
1
0
Fork 0
mirror of https://github.com/DJ2LS/FreeDATA synced 2024-05-14 08:04:33 +00:00
Code Issues Projects Releases Packages Wiki Activity

Merge branch 'develop' into dependabot/npm_and_yarn/gui/develop/gridstack-10.0.1

Browse source
This commit is contained in:
DJ2LS 2023-12-31 13:42:35 +01:00 committed by GitHub
commit 0801d0ac4d
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
95 changed files with 644 additions and 13658 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

37
.github/workflows/build_gui.yml vendored Normal file
View file

@ -0,0 +1,37 @@
name: build_gui
on: [push]
jobs:
build_i686_x64_release:
name: Build FreeDATA GUI
runs-on: ${{ matrix.os }}
strategy:
fail-fast: false
matrix:
os: [ubuntu-20.04, macos-latest, windows-latest]
include:
- os: ubuntu-20.04
electron_parameters: "-p always"
- os: macos-latest
electron_parameters: "-p always"
- os: windows-latest
electron_parameters: "-p always --x64 --ia32"
steps:
- name: Checkout code for ${{ matrix.platform.name }}
uses: actions/checkout@v4
with:
repository: DJ2LS/FreeDATA
- name: Electron Builder
env: # Setting environment variables for the entire job
GH_TOKEN: ${{ secrets.github_token }}
APPLE_ID: ${{ secrets.APPLE_ID }}
APPLE_ID_PASSWORD: ${{ secrets.APPLE_ID_PASSWORD }}
APPLE_APP_SPECIFIC_PASSWORD: ${{ secrets.APPLE_ID_PASSWORD }}
APPLE_TEAM_ID: ${{ secrets.APPLE_TEAM_ID }}
working-directory: gui
run: |
npm i
npm run release

351
.github/workflows/build_multiplatform.yml vendored
View file

@ -1,351 +0,0 @@
name: Build_Multiplatform
on: [push]
jobs:
BUILD_AMD64:
name: Build codec2 for x86/x64 devices
runs-on: ${{ matrix.os }}
strategy:
fail-fast: false
matrix:
os: [ubuntu-20.04, ubuntu-22.04, macos-latest, macos-12]
platform: [{name: "native"}, {name: "Windows", file: "dll"}]
architecture: [i686-w64-mingw32, x86_64-w64-mingw32]
include:
- os: ubuntu-20.04
libcodec2_name: libcodec2.so.1.2
libcodec2_os_name: libcodec2_ubuntu-2004
libcodec2_filetype: so
generator: Unix Makefiles
shell: bash
- os: ubuntu-22.04
libcodec2_name: libcodec2.so.1.2
libcodec2_os_name: libcodec2_ubuntu-2204
libcodec2_filetype: so
generator: Unix Makefiles
shell: bash
- os: macos-latest
libcodec2_name: libcodec2.1.2.dylib
libcodec2_os_name: libcodec2_macos-latest
libcodec2_filetype: dylib
generator: Unix Makefiles
shell: bash
- os: macos-12
libcodec2_name: libcodec2.1.2.dylib
libcodec2_os_name: libcodec2_macos-12
libcodec2_filetype: dylib
generator: Unix Makefiles
shell: bash
steps:
- name: Build codec2 on ${{ matrix.os }} for ${{ matrix.platform.name }}
if: ${{startsWith(matrix.platform.name, 'native') }}
run: |
git clone https://github.com/drowe67/codec2.git
cd codec2
mkdir build
mkdir tempfiles
cd build
cmake -DCMAKE_BUILD_TYPE=Release ../
make
mv src/${{ matrix.libcodec2_name }} ../tempfiles/libcodec2_${{ matrix.os }}_${{ matrix.platform.name }}.${{ matrix.libcodec2_filetype }}
- name: LIST ALL FILES ${{ github.workspace }}
run: ls -R ${{ github.workspace }}
- uses: actions/upload-artifact@v3
if: ${{startsWith(matrix.platform.name, 'native') }}
with:
name: libcodec2_${{ matrix.os }}_${{ matrix.platform.name }}.${{ matrix.libcodec2_filetype }}
# path: ${{ github.workspace }}/codec2/tempfiles/libcodec2_${{ matrix.os }}_${{ matrix.platform.name }}.${{ matrix.libcodec2_filetype }}
path: ${{ github.workspace }}/codec2/tempfiles/
- name: Build codec2 ${{ matrix.platform.name }} ${{ matrix.architecture }}
if: ${{startsWith(matrix.os, 'ubuntu-20') && !startsWith(matrix.platform.name, 'native') }}
run: |
sudo apt install build-essential mingw-w64 g++-mingw-w64 make cmake
git clone https://github.com/drowe67/codec2.git
cd codec2
mkdir tempfiles
mkdir build_w32
cd build_w32
echo 'set(CMAKE_SYSTEM_NAME ${{ matrix.platform.name }})' > toolchain-ubuntu-mingw32.cmake
echo 'set(CMAKE_C_COMPILER ${{ matrix.architecture }}-gcc)' >> toolchain-ubuntu-mingw32.cmake
echo 'set(CMAKE_CXX_COMPILER ${{ matrix.architecture }}-g++)' >> toolchain-ubuntu-mingw32.cmake
echo 'set(CMAKE_RC_COMPILER ${{ matrix.architecture }}-windres)' >> toolchain-ubuntu-mingw32.cmake
echo 'set(CMAKE_FIND_ROOT_PATH /usr/${{ matrix.architecture }})' >> toolchain-ubuntu-mingw32.cmake
echo 'set(CMAKE_FIND_ROOT_PATH_MODE_PROGRAM NEVER)' >> toolchain-ubuntu-mingw32.cmake
echo 'set(CMAKE_FIND_ROOT_PATH_MODE_LIBRARY ONLY)' >> toolchain-ubuntu-mingw32.cmake
echo 'set(CMAKE_FIND_ROOT_PATH_MODE_INCLUDE ONLY)' >> toolchain-ubuntu-mingw32.cmake
echo 'set(CMAKE_SHARED_LINKER_FLAGS "-static-libgcc -static-libstdc++ -static")' >> toolchain-ubuntu-mingw32.cmake
cmake -G "Unix Makefiles" -DCMAKE_BUILD_TYPE=Release -DCMAKE_TOOLCHAIN_FILE=toolchain-ubuntu-mingw32.cmake ..
make
mv src/libcodec2.${{ matrix.platform.file }} ../tempfiles/libcodec2_${{ matrix.platform.name }}_${{ matrix.architecture }}.${{ matrix.platform.file }}
- uses: actions/upload-artifact@v3
if: ${{startsWith(matrix.os, 'ubuntu-20') && !startsWith(matrix.platform.name, 'native') }}
with:
name: libcodec2_${{ matrix.os }}_${{ matrix.platform.name }}_${{ matrix.architecture }}.${{ matrix.platform.file }}
path: codec2/tempfiles/*
BUILD_ARM:
# The host should always be linux
runs-on: ubuntu-latest
name: Build codec2 for ARM devices
# Run steps on a matrix of 2 arch/distro combinations
strategy:
matrix:
include:
- arch: armv7
distro: bullseye
libcodec2_os_name: libcodec2_bullseye_armv7.so
- arch: armv7
distro: ubuntu_latest
libcodec2_os_name: libcodec2_ubuntu_latest_armv7.so
steps:
- uses: actions/checkout@v4
with:
ref: ${{ github.head_ref }}
- uses: uraimo/run-on-arch-action@v2
name: Build artifact
id: build
with:
arch: ${{ matrix.arch }}
distro: ${{ matrix.distro }}
# Not required, but speeds up builds
githubToken: ${{ github.token }}
# Create an artifacts directory
setup: |
mkdir -p "${PWD}/artifacts"
# Mount the artifacts directory as /artifacts in the container
dockerRunArgs: |
--volume "${PWD}/artifacts:/artifacts"
# Pass some environment variables to the container
env: | # YAML, but pipe character is necessary
artifact_name: ${{ matrix.libcodec2_os_name }}
# The shell to run commands with in the container
shell: /bin/sh
# Install some dependencies in the container. This speeds up builds if
# you are also using githubToken. Any dependencies installed here will
# be part of the container image that gets cached, so subsequent
# builds don't have to re-install them. The image layer is cached
# publicly in your project's package repository, so it is vital that
# no secrets are present in the container state or logs.
install: |
case "${{ matrix.distro }}" in
ubuntu*|jessie|stretch|buster|bullseye)
apt-get update -q -y
apt-get install -q -y git build-essential cmake gcc g++
cmake --version
;;
fedora*)
dnf -y update
dnf -y install git which make cmake gcc-c++ gcc
cmake --version
;;
alpine*)
apk update
apk add git cmake gcc g++
cmake --version
;;
esac
# Produce a binary artifact and place it in the mounted volume
run: |
git clone https://github.com/drowe67/codec2.git
cd codec2
git checkout main
mkdir build
cd build
cmake ../
make
mv ./src/libcodec2.so.1.2 /artifacts/${artifact_name}
- name: Show recursive PWD/artifacts
# Items placed in /artifacts in the container will be in
# ${PWD}/artifacts on the host.
run: ls -al "${PWD}/artifacts"
- uses: actions/upload-artifact@v3
with:
name: ${{ matrix.libcodec2_os_name }}
#path: $GITHUB_WORKSPACE/codec2/artifacts/*
path: artifacts/*
build_i686_x64_release:
needs: [BUILD_AMD64, BUILD_ARM]
name: Build FreeDATA packages
runs-on: ${{ matrix.os }}
strategy:
fail-fast: false
matrix:
os: [ubuntu-20.04, macos-latest, windows-latest]
include:
- os: ubuntu-20.04
zip_name: ubuntu_modem
generator: Unix Makefiles
daemon_binary_name: freedata-daemon
modem_binary_name: freedata-modem
electron_parameters: "-p always"
- os: macos-latest
zip_name: macos_modem
generator: Unix Makefiles
daemon_binary_name: freedata-daemon
modem_binary_name: freedata-modem
electron_parameters: "-p always"
- os: windows-latest
zip_name: windows_modem
generator: Visual Studio 16 2019
daemon_binary_name: freedata-daemon.exe
modem_binary_name: freedata-modem.exe
electron_parameters: "-p always --x64 --ia32"
steps:
- name: Checkout code for ${{ matrix.platform.name }}
uses: actions/checkout@v4
with:
repository: DJ2LS/FreeDATA
- name: Set up Python 3.11
uses: actions/setup-python@v4
with:
python-version: "3.11"
- name: Install Node.js, NPM and Yarn
uses: actions/setup-node@v4
with:
node-version: 18.17
- name: Create modem/dist
working-directory: modem
run: |
mkdir -p dist
- name: Create modem/dist/modem
working-directory: modem
run: |
mkdir -p dist/modem
##- name: Download libcodec2 artifact Modem DIST
## uses: actions/download-artifact@v3
## with:
## path: modem/dist/codec2
- name: create modem/lib/codec2
working-directory: modem/lib/
run: |
mkdir codec2
- name: Download libcodec2 artifact Modem LIB
uses: actions/download-artifact@v3
with:
path: modem/lib/codec2
- name: Install Linux dependencies
# if: matrix.os == 'ubuntu-20.04'
if: ${{startsWith(matrix.os, 'ubuntu')}}
run: |
sudo apt install -y portaudio19-dev libhamlib-dev libhamlib-utils build-essential cmake python3-libhamlib2 patchelf
- name: Install MacOS pyAudio
if: ${{startsWith(matrix.os, 'macos')}}
run: |
brew install portaudio
python -m pip install --upgrade pip
pip3 install pyaudio
- name: Install Python dependencies
run: |
python -m pip install --upgrade pip
pip install -r requirements.txt
- name: Add MacOS certs
if: ${{startsWith(matrix.os, 'macos')}}
run: chmod +x add-osx-cert.sh && ./add-osx-cert.sh
env:
CERTIFICATE_OSX_APPLICATION: ${{ secrets.CERTIFICATE_OSX_APPLICATION }}
CERTIFICATE_PASSWORD: ${{ secrets.CERTIFICATE_PASSWORD }}
- name: Build binaries Linux and Windows
working-directory: modem
run: |
python3 -m nuitka --enable-plugin=numpy --remove-output --assume-yes-for-downloads --standalone server.py
- name: LIST ALL FILES
run: ls -R
- name: Download Portaudio binaries Linux macOS
if: ${{!startsWith(matrix.os, 'windows')}}
working-directory: modem
run: |
if ! test -d "server.dist/modem/_sounddevice_data"; then
git clone https://github.com/spatialaudio/portaudio-binaries dist/modem/_sounddevice_data/portaudio-binaries
fi
- name: Download Portaudio binaries Windows
if: ${{startsWith(matrix.os, 'windows')}}
working-directory: modem
run: |
if(Test-Path -Path "server.dist/modem/_sounddevice_data"){
echo "sounddevice folder already exists"
} else {
git clone https://github.com/spatialaudio/portaudio-binaries dist/modem/_sounddevice_data/portaudio-binaries
}
- name: LIST ALL FILES
run: ls -R
- name: cleanup on macos before code signing
if: ${{startsWith(matrix.os, 'macos')}}
run: |
ls -l
# find . -type d -name .git -exec rm -r {} \;
find . -type d -o -name ".git" -delete
- name: Electron Builder
env: # Setting environment variables for the entire job
GH_TOKEN: ${{ secrets.github_token }}
APPLE_ID: ${{ secrets.APPLE_ID }}
APPLE_ID_PASSWORD: ${{ secrets.APPLE_ID_PASSWORD }}
APPLE_APP_SPECIFIC_PASSWORD: ${{ secrets.APPLE_ID_PASSWORD }}
APPLE_TEAM_ID: ${{ secrets.APPLE_TEAM_ID }}
working-directory: gui
run: |
npm i
npm run release
- name: Compress Modem
uses: thedoctor0/zip-release@master
with:
type: 'zip'
filename: '${{ matrix.zip_name }}.zip'
directory: ./modem/server.dist
path: .
# exclusions: '*.git* /*node_modules/* .editorconfig'
- name: Release Modem
uses: softprops/action-gh-release@v1
if: startsWith(github.ref, 'refs/tags/v')
with:
draft: true
files: ./modem/server.dist/${{ matrix.zip_name }}.zip
- name: LIST ALL FILES
run: ls -R

124
.github/workflows/build_server.yml vendored Normal file
View file

@ -0,0 +1,124 @@
name: build_server
on: [push]
jobs:
build_i686_x64_release:
name: Build FreeDATA packages
runs-on: ${{ matrix.os }}
strategy:
fail-fast: false
matrix:
os: [ubuntu-20.04, macos-latest, windows-latest]
include:
- os: ubuntu-20.04
zip_name: ubuntu_modem
generator: Unix Makefiles
modem_binary_name: freedata-server
- os: macos-latest
zip_name: macos_modem
generator: Unix Makefiles
modem_binary_name: freedata-server
- os: windows-latest
zip_name: windows_modem
generator: Visual Studio 16 2019
modem_binary_name: freedata-server.exe
steps:
- name: Checkout code for ${{ matrix.platform.name }}
uses: actions/checkout@v4
with:
repository: DJ2LS/FreeDATA
- name: Set up Python 3.11
uses: actions/setup-python@v4
with:
python-version: "3.11"
- name: Create modem/dist
working-directory: modem
run: |
mkdir -p dist
- name: Create modem/dist/modem
working-directory: modem
run: |
mkdir -p dist/modem
- name: Install Linux dependencies
# if: matrix.os == 'ubuntu-20.04'
if: ${{startsWith(matrix.os, 'ubuntu')}}
run: |
sudo apt install -y portaudio19-dev libhamlib-dev libhamlib-utils build-essential cmake python3-libhamlib2 patchelf
- name: Install MacOS pyAudio
if: ${{startsWith(matrix.os, 'macos')}}
run: |
brew install portaudio
python -m pip install --upgrade pip
pip3 install pyaudio
- name: Install Python dependencies
run: |
python -m pip install --upgrade pip
pip install -r requirements.txt
- name: Add MacOS certs
if: ${{startsWith(matrix.os, 'macos')}}
run: chmod +x add-osx-cert.sh && ./add-osx-cert.sh
env:
CERTIFICATE_OSX_APPLICATION: ${{ secrets.CERTIFICATE_OSX_APPLICATION }}
CERTIFICATE_PASSWORD: ${{ secrets.CERTIFICATE_PASSWORD }}
- name: Build binaries
working-directory: modem
run: |
python3 -m nuitka --enable-plugin=numpy --remove-output --assume-yes-for-downloads --standalone server.py
- name: Download Portaudio binaries Linux macOS
if: ${{!startsWith(matrix.os, 'windows')}}
working-directory: modem
run: |
if ! test -d "server.dist/modem/_sounddevice_data"; then
git clone https://github.com/spatialaudio/portaudio-binaries dist/modem/_sounddevice_data/portaudio-binaries
fi
- name: Download Portaudio binaries Windows
if: ${{startsWith(matrix.os, 'windows')}}
working-directory: modem
run: |
if(Test-Path -Path "server.dist/modem/_sounddevice_data"){
echo "sounddevice folder already exists"
} else {
git clone https://github.com/spatialaudio/portaudio-binaries dist/modem/_sounddevice_data/portaudio-binaries
}
- name: LIST ALL FILES
run: ls -R
- name: cleanup on macos before code signing
if: ${{startsWith(matrix.os, 'macos')}}
run: |
ls -l
# find . -type d -name .git -exec rm -r {} \;
find . -type d -o -name ".git" -delete
- name: Compress Modem
uses: thedoctor0/zip-release@master
with:
type: 'zip'
filename: '${{ matrix.zip_name }}.zip'
directory: ./modem/server.dist
path: .
# exclusions: '*.git* /*node_modules/* .editorconfig'
- name: Release Modem
uses: softprops/action-gh-release@v1
if: startsWith(github.ref, 'refs/tags/v')
with:
draft: true
files: ./modem/server.dist/${{ matrix.zip_name }}.zip
- name: LIST ALL FILES
run: ls -R

1
.github/workflows/gui_tests.yml vendored
View file

@ -16,7 +16,6 @@ jobs:
fail-fast: false
matrix:
include:
- node-version: "14"
- node-version: "16"
- node-version: "18"
- node-version: "20"

8
.github/workflows/modem_tests.yml vendored
View file

@ -43,14 +43,6 @@ jobs:
run: |
pip3 install -r requirements.txt
- name: Build codec2
shell: bash
working-directory: modem/lib/
run: |
git clone https://github.com/drowe67/codec2.git
cd codec2
mkdir -p build_linux && cd build_linux && cmake .. && make
- name: run config tests
shell: bash
run: |

2
gui/src/components/main.vue
View file

@ -20,7 +20,7 @@ import infoScreen from "./infoScreen.vue";
import main_modem_healthcheck from "./main_modem_healthcheck.vue";
import Dynamic_components2 from "./dynamic_components2.vue";
import { stopTransmission } from "../js/sock";
import { stopTransmission } from "../js/api";
function stopAllTransmissions() {
console.log("stopping transmissions");

4
gui/src/js/api.js
View file

@ -97,6 +97,10 @@ export function sendModemARQRaw(mycall, dxcall, data, uuid) {
});
}
export function stopTransmission() {
return apiPost("/modem/stop_transmission");
}
export function sendModemTestFrame() {
return apiPost("/modem/send_test_frame");
}

2
gui/src/js/eventHandler.js
View file

@ -23,7 +23,7 @@ export function connectionFailed(endpoint, event) {
}
export function stateDispatcher(data) {
data = JSON.parse(data);
//console.log(data);
console.log(data);
stateStore.modem_connection = "connected";

5
gui/src/js/sock.js
View file

@ -719,11 +719,6 @@ function sendResponseSharedFile(dxcallsign, sharedFile, sharedFileData) {
sendResponse(dxcallsign, 255, 1, sharedFile + "/" + sharedFileData, "res-2");
}
*/
//STOP TRANSMISSION
export function stopTransmission() {
var command = '{"type" : "arq", "command": "stop_transmission"}';
writeTncCommand(command);
}
// Get RX BUffer
export function getRxBuffer() {

5
modem/arq_session.py
View file

@ -46,7 +46,7 @@ class ARQSession():
self.id = None
def log(self, message, isWarning = False):
msg = f"[{type(self).__name__}]: {message}"
msg = f"[{type(self).__name__}][state={self.state}]: {message}"
logger = self.logger.warn if isWarning else self.logger.info
logger(msg)
@ -87,4 +87,5 @@ class ARQSession():
getattr(self, action_name)(frame)
return
self.log(f"Ignoring unknow transition from state {self.state} with frame {frame['frame_type']}")
self.log(f"Ignoring unknow transition from state {self.state.name} with frame {frame['frame_type']}")

58
modem/arq_session_irs.py
View file

@ -16,29 +16,42 @@ class IRS_State(Enum):
class ARQSessionIRS(arq_session.ARQSession):
RETRIES_CONNECT = 3
RETRIES_TRANSFER = 3 # we need to increase this
TIMEOUT_CONNECT = 3
TIMEOUT_DATA = 12
TIMEOUT_CONNECT = 55 #14.2
TIMEOUT_DATA = 60
STATE_TRANSITION = {
IRS_State.NEW: {
FRAME_TYPE.ARQ_SESSION_OPEN.value : 'send_open_ack',
FRAME_TYPE.ARQ_STOP.value: 'send_stop_ack'
},
IRS_State.OPEN_ACK_SENT: {
FRAME_TYPE.ARQ_SESSION_OPEN.value: 'send_open_ack',
FRAME_TYPE.ARQ_SESSION_INFO.value: 'send_info_ack',
FRAME_TYPE.ARQ_STOP.value: 'send_stop_ack'
},
IRS_State.INFO_ACK_SENT: {
FRAME_TYPE.ARQ_SESSION_INFO.value: 'send_info_ack',
FRAME_TYPE.ARQ_BURST_FRAME.value: 'receive_data',
FRAME_TYPE.ARQ_STOP.value: 'send_stop_ack'
},
IRS_State.BURST_REPLY_SENT: {
FRAME_TYPE.ARQ_BURST_FRAME.value: 'receive_data',
FRAME_TYPE.ARQ_STOP.value: 'send_stop_ack'
},
IRS_State.ENDED: {
FRAME_TYPE.ARQ_BURST_FRAME.value: 'receive_data',
FRAME_TYPE.ARQ_STOP.value: 'send_stop_ack'
},
IRS_State.FAILED: {
FRAME_TYPE.ARQ_BURST_FRAME.value: 'receive_data',
FRAME_TYPE.ARQ_STOP.value: 'send_stop_ack'
},
IRS_State.ABORTED: {
FRAME_TYPE.ARQ_STOP.value: 'send_stop_ack'
},
}
@ -59,6 +72,8 @@ class ARQSessionIRS(arq_session.ARQSession):
self.transmitted_acks = 0
self.abort = False
def set_decode_mode(self):
self.modem.demodulator.set_decode_mode(self.get_mode_by_speed_level(self.speed_level))
@ -76,7 +91,7 @@ class ARQSessionIRS(arq_session.ARQSession):
if not self.event_frame_received.wait(timeout):
self.log("Timeout waiting for ISS. Session failed.")
self.set_state(IRS_State.FAILED)
self.event_manager.send_arq_session_finished(False, self.id, self.dxcall, self.total_length, False)
self.event_manager.send_arq_session_finished(False, self.id, self.dxcall, self.total_length, False, self.state.name)
def launch_transmit_and_wait(self, frame, timeout, mode):
thread_wait = threading.Thread(target = self.transmit_and_wait,
@ -84,6 +99,7 @@ class ARQSessionIRS(arq_session.ARQSession):
thread_wait.start()
def send_open_ack(self, open_frame):
ack_frame = self.frame_factory.build_arq_session_open_ack(
self.id,
self.dxcall,
@ -100,23 +116,17 @@ class ARQSessionIRS(arq_session.ARQSession):
self.dx_snr.append(info_frame['snr'])
self.log(f"New transfer of {self.total_length} bytes")
self.event_manager.send_arq_session_new(False, self.id, self.dxcall, self.total_length)
self.event_manager.send_arq_session_new(False, self.id, self.dxcall, self.total_length, self.state.name)
self.calibrate_speed_settings()
self.set_decode_mode()
info_ack = self.frame_factory.build_arq_session_info_ack(
self.id, self.total_crc, self.snr[0],
self.speed_level, self.frames_per_burst)
self.speed_level, self.frames_per_burst, flag_abort=self.abort)
self.launch_transmit_and_wait(info_ack, self.TIMEOUT_CONNECT, mode=FREEDV_MODE.signalling)
self.set_state(IRS_State.INFO_ACK_SENT)
def send_burst_nack(self):
self.calibrate_speed_settings()
self.set_decode_mode()
nack = self.frame_factory.build_arq_burst_ack(self.id, self.received_bytes, self.speed_level, self.frames_per_burst, self.snr[0])
self.launch_transmit_and_wait(nack, self.TIMEOUT_DATA, mode=FREEDV_MODE.signalling)
self.log("NACK sent")
def process_incoming_data(self, frame):
if frame['offset'] != self.received_bytes:
@ -137,7 +147,7 @@ class ARQSessionIRS(arq_session.ARQSession):
self.received_bytes += len(data_part)
self.log(f"Received {self.received_bytes}/{self.total_length} bytes")
self.event_manager.send_arq_session_progress(
False, self.id, self.dxcall, self.received_bytes, self.total_length)
False, self.id, self.dxcall, self.received_bytes, self.total_length, self.state.name)
return True
@ -148,10 +158,10 @@ class ARQSessionIRS(arq_session.ARQSession):
if not self.all_data_received():
ack = self.frame_factory.build_arq_burst_ack(
self.id, self.received_bytes,
self.speed_level, self.frames_per_burst, self.snr[0])
self.speed_level, self.frames_per_burst, self.snr[0], flag_abort=self.abort)
# increase ack counter
self.transmitted_acks += 1
# self.transmitted_acks += 1
self.set_state(IRS_State.BURST_REPLY_SENT)
self.launch_transmit_and_wait(ack, self.TIMEOUT_DATA, mode=FREEDV_MODE.signalling)
return
@ -169,7 +179,7 @@ class ARQSessionIRS(arq_session.ARQSession):
self.log("ACK sent")
self.set_state(IRS_State.ENDED)
self.event_manager.send_arq_session_finished(
False, self.id, self.dxcall, self.total_length, True)
False, self.id, self.dxcall, self.total_length, True, self.state.name, data=self.received_data)
else:
@ -184,7 +194,7 @@ class ARQSessionIRS(arq_session.ARQSession):
self.log("CRC fail at the end of transmission!")
self.set_state(IRS_State.FAILED)
self.event_manager.send_arq_session_finished(
False, self.id, self.dxcall, self.total_length, False)
False, self.id, self.dxcall, self.total_length, False, self.state.name)
def calibrate_speed_settings(self):
@ -199,3 +209,13 @@ class ARQSessionIRS(arq_session.ARQSession):
if self.snr[0] >= self.SPEED_LEVEL_DICT[new_speed_level]["min_snr"]:
self.speed_level = new_speed_level
def abort_transmission(self):
self.log(f"Aborting transmission... setting abort flag")
self.abort = True
def send_stop_ack(self, stop_frame):
stop_ack = self.frame_factory.build_arq_stop_ack(self.id)
self.launch_transmit_and_wait(stop_ack, self.TIMEOUT_CONNECT, mode=FREEDV_MODE.signalling)
self.set_state(IRS_State.ABORTED)
self.event_manager.send_arq_session_finished(
False, self.id, self.dxcall, self.total_length, False, self.state.name)

80
modem/arq_session_iss.py
View file

@ -15,13 +15,19 @@ class ISS_State(Enum):
BURST_SENT = 3
ENDED = 4
FAILED = 5
ABORTED = 6
ABORTING = 6 # state while running abort sequence and waiting for stop ack
ABORTED = 7 # stop ack received
class ARQSessionISS(arq_session.ARQSession):
RETRIES_CONNECT = 10
TIMEOUT_CONNECT_ACK = 3
TIMEOUT_TRANSFER = 3
# DJ2LS: 3 seconds seems to be too small for radios with a too slow PTT toggle time
# DJ2LS: 3.5 seconds is working well WITHOUT a channel busy detection delay
TIMEOUT_CHANNEL_BUSY = 2
TIMEOUT_CONNECT_ACK = 3.5 + TIMEOUT_CHANNEL_BUSY
TIMEOUT_TRANSFER = 3.5 + TIMEOUT_CHANNEL_BUSY
TIMEOUT_STOP_ACK = 3.5 + TIMEOUT_CHANNEL_BUSY
STATE_TRANSITION = {
ISS_State.OPEN_SENT: {
@ -34,8 +40,16 @@ class ARQSessionISS(arq_session.ARQSession):
ISS_State.BURST_SENT: {
FRAME_TYPE.ARQ_SESSION_INFO_ACK.value: 'send_data',
FRAME_TYPE.ARQ_BURST_ACK.value: 'send_data',
FRAME_TYPE.ARQ_BURST_NACK.value: 'send_data',
},
ISS_State.FAILED:{
FRAME_TYPE.ARQ_STOP_ACK.value: 'transmission_aborted'
},
ISS_State.ABORTING: {
FRAME_TYPE.ARQ_STOP_ACK.value: 'transmission_aborted',
},
ISS_State.ABORTED: {
FRAME_TYPE.ARQ_STOP_ACK.value: 'transmission_aborted',
}
}
def __init__(self, config: dict, modem, dxcall: str, data: bytearray):
@ -47,7 +61,6 @@ class ARQSessionISS(arq_session.ARQSession):
self.state = ISS_State.NEW
self.id = self.generate_id()
self.frame_factory = data_frame_factory.DataFrameFactory(self.config)
def generate_id(self):
@ -66,9 +79,9 @@ class ARQSessionISS(arq_session.ARQSession):
return
self.log("Timeout!")
retries = retries - 1
self.set_state(ISS_State.FAILED)
self.log("Session failed")
self.event_manager.send_arq_session_finished(True, self.id, self.dxcall, len(self.data), False)
self.transmission_failed()
def launch_twr(self, frame_or_burst, timeout, retries, mode):
twr = threading.Thread(target = self.transmit_wait_and_retry, args=[frame_or_burst, timeout, retries, mode])
@ -94,20 +107,27 @@ class ARQSessionISS(arq_session.ARQSession):
self.set_state(ISS_State.INFO_SENT)
def send_data(self, irs_frame):
self.set_speed_and_frames_per_burst(irs_frame)
if 'offset' in irs_frame:
self.confirmed_bytes = irs_frame['offset']
self.log(f"IRS confirmed {self.confirmed_bytes}/{len(self.data)} bytes")
self.event_manager.send_arq_session_progress(
True, self.id, self.dxcall, self.confirmed_bytes, len(self.data))
True, self.id, self.dxcall, self.confirmed_bytes, len(self.data), self.state.name)
if self.confirmed_bytes == len(self.data) and irs_frame["flag"]["FINAL"]:
self.set_state(ISS_State.ENDED)
self.log("All data transfered!")
self.event_manager.send_arq_session_finished(True, self.id, self.dxcall, len(self.data), irs_frame["flag"]["CHECKSUM"])
if irs_frame["flag"]["ABORT"]:
self.transmission_aborted(irs_frame)
return
if irs_frame["flag"]["FINAL"]:
if self.confirmed_bytes == len(self.data) and irs_frame["flag"]["CHECKSUM"]:
self.transmission_ended(irs_frame)
return
else:
self.transmission_failed()
return
payload_size = self.get_data_payload_size()
burst = []
for f in range(0, self.frames_per_burst):
@ -119,3 +139,39 @@ class ARQSessionISS(arq_session.ARQSession):
burst.append(data_frame)
self.launch_twr(burst, self.TIMEOUT_TRANSFER, self.RETRIES_CONNECT, mode='auto')
self.set_state(ISS_State.BURST_SENT)
def transmission_ended(self, irs_frame):
# final function for sucessfully ended transmissions
self.set_state(ISS_State.ENDED)
self.log(f"All data transfered! flag_final={irs_frame['flag']['FINAL']}, flag_checksum={irs_frame['flag']['CHECKSUM']}")
self.event_manager.send_arq_session_finished(True, self.id, self.dxcall, len(self.data),True, self.state.name)
def transmission_failed(self, irs_frame=None):
# final function for failed transmissions
self.set_state(ISS_State.FAILED)
self.log(f"Transmission failed!")
self.event_manager.send_arq_session_finished(True, self.id, self.dxcall, len(self.data),False, self.state.name)
def abort_transmission(self, irs_frame=None):
# function for starting the abort sequence
self.log(f"aborting transmission...")
self.set_state(ISS_State.ABORTING)
self.event_manager.send_arq_session_finished(
True, self.id, self.dxcall, len(self.data), False, self.state.name)
# break actual retries
self.event_frame_received.set()
# start with abort sequence
self.send_stop()
def send_stop(self):
stop_frame = self.frame_factory.build_arq_stop(self.id)
self.launch_twr(stop_frame, self.TIMEOUT_STOP_ACK, self.RETRIES_CONNECT, mode=FREEDV_MODE.signalling)
def transmission_aborted(self, irs_frame):
self.log("session aborted")
self.set_state(ISS_State.ABORTED)
self.event_manager.send_arq_session_finished(
True, self.id, self.dxcall, len(self.data), False, self.state.name)

11
modem/audio.py
View file

@ -8,6 +8,7 @@ import sounddevice as sd
import structlog
import numpy as np
import queue
import threading
atexit.register(sd._terminate)
@ -313,19 +314,17 @@ def calculate_fft(data, fft_queue, states) -> None:
if addDelay:
# Limit delay counter to a maximum of 200. The higher this value,
# the longer we will wait until releasing state
states.set("channel_busy", True)
states.set_channel_busy_condition_traffic(True)
CHANNEL_BUSY_DELAY = min(CHANNEL_BUSY_DELAY + 10, 200)
else:
# Decrement channel busy counter if no signal has been detected.
CHANNEL_BUSY_DELAY = max(CHANNEL_BUSY_DELAY - 1, 0)
# When our channel busy counter reaches 0, toggle state to False
if CHANNEL_BUSY_DELAY == 0:
states.set("channel_busy", False)
# erase queue if greater than 10
if fft_queue.qsize() >= 10:
states.set_channel_busy_condition_traffic(False)
# erase queue if greater than 3
if fft_queue.qsize() >= 1:
fft_queue = queue.Queue()
fft_queue.put(dfftlist[:315]) # 315 --> bandwidth 3200
except Exception as err:
print(f"[MDM] calculate_fft: Exception: {err}")
print("[MDM] Setting fft=0")
fft_queue.put([0])

8
modem/beacon.py
View file

@ -6,13 +6,13 @@ class Beacon:
BEACON_LOOP_INTERVAL = 1
def __init__(self, config, states, event_queue, logger, modem_tx_queue):
def __init__(self, config, states, event_queue, logger, modem):
self.modem_config = config
self.states = states
self.event_queue = event_queue
self.log = logger
self.tx_frame_queue = modem_tx_queue
self.modem = modem
self.loop_running = True
self.paused = False
@ -39,8 +39,8 @@ class Beacon:
True):
#not self.states.channel_busy):
cmd = command_beacon.BeaconCommand(self.modem_config, self.log)
cmd.run(self.event_queue, self.tx_frame_queue)
cmd = command_beacon.BeaconCommand(self.modem_config, self.states, self.event_queue)
cmd.run(self.event_queue, self.modem)
self.event.wait(self.modem_config['MODEM']['beacon_interval'])
self.event.wait(self.BEACON_LOOP_INTERVAL)

2
modem/codec2.py
View file

@ -101,7 +101,7 @@ for file in files:
#log.info("[C2 ] Libcodec2 loaded", path=file)
break
except OSError as err:
log.warning("[C2 ] Error: Libcodec2 found but not loaded", path=file, e=err)
log.info("[C2 ] Error: Libcodec2 found but not loaded", path=file, e=err)
# Quit module if codec2 cant be loaded
if api is None or "api" not in locals():

2
modem/command_arq_raw.py
View file

@ -4,6 +4,8 @@ import api_validations
import base64
from queue import Queue
from arq_session_iss import ARQSessionISS
class ARQRawCommand(TxCommand):
def set_params_from_api(self, apiParams):

11
modem/command_beacon.py
View file

@ -5,8 +5,9 @@ class BeaconCommand(TxCommand):
def build_frame(self):
return self.frame_factory.build_beacon()
def transmit(self, modem):
super().transmit(modem)
if self.config['MODEM']['enable_morse_identifier']:
mycall = f"{self.config['STATION']['mycall']}-{self.config['STATION']['myssid']}"
modem.transmit_morse("morse", 1, 0, mycall)
#def transmit(self, modem):
# super().transmit(modem)
# if self.config['MODEM']['enable_morse_identifier']:
# mycall = f"{self.config['STATION']['mycall']}-{self.config['STATION']['myssid']}"
# modem.transmit_morse("morse", 1, 0, mycall)

218
modem/data cemetery/CMakeLists.txt
View file

@ -1,218 +0,0 @@
cmake_minimum_required(VERSION 3.0)
project (FreeDATA)
include(CTest)
enable_testing()
# Find codec2
if(CODEC2_BUILD_DIR)
find_package(codec2 REQUIRED
PATHS ${CODEC2_BUILD_DIR}
NO_DEFAULT_PATH
CONFIGS codec2.cmake
)
if(codec2_FOUND)
message(STATUS "Codec2 library found in build tree.")
endif()
else()
find_package(codec2 REQUIRED)
endif()
# test variables
set(FRAMESPERBURST 3)
set(BURSTS 1)
set(TESTFRAMES 3)
add_test(NAME audio_buffer
COMMAND sh -c "export LD_LIBRARY_PATH=${CODEC2_BUILD_DIR}/src;
export PYTHONPATH=../modem;
cd ${CMAKE_CURRENT_SOURCE_DIR}/test;
python3 test_audiobuffer.py")
set_tests_properties(audio_buffer PROPERTIES PASS_REGULAR_EXPRESSION "errors: 0")
add_test(NAME resampler
COMMAND sh -c "export LD_LIBRARY_PATH=${CODEC2_BUILD_DIR}/src;
export PYTHONPATH=../modem;
cd ${CMAKE_CURRENT_SOURCE_DIR}/test;
python3 test_resample_48_8.py")
set_tests_properties(resampler PROPERTIES PASS_REGULAR_EXPRESSION "PASS")
add_test(NAME modem_state_machine
COMMAND sh -c "export LD_LIBRARY_PATH=${CODEC2_BUILD_DIR}/src;
export PYTHONPATH=../modem;
cd ${CMAKE_CURRENT_SOURCE_DIR}/test;
python3 test_modem_states.py")
set_tests_properties(modem_state_machine PROPERTIES PASS_REGULAR_EXPRESSION "errors: 0")
add_test(NAME modem_irs_iss
COMMAND sh -c "export LD_LIBRARY_PATH=${CODEC2_BUILD_DIR}/src;
export PYTHONPATH=../modem;
cd ${CMAKE_CURRENT_SOURCE_DIR}/test;
python3 test_modem.py")
set_tests_properties(modem_irs_iss PROPERTIES PASS_REGULAR_EXPRESSION "errors: 0")
# disabled this test as its actually broken since we introduced session IDs
#add_test(NAME chat_text
# COMMAND sh -c "export LD_LIBRARY_PATH=${CODEC2_BUILD_DIR}/src;
# export PYTHONPATH=../modem;
# cd ${CMAKE_CURRENT_SOURCE_DIR}/test;
# python3 test_chat_text.py")
# set_tests_properties(chat_text PROPERTIES PASS_REGULAR_EXPRESSION "errors: 0")
add_test(NAME datac13_frames
COMMAND sh -c "export LD_LIBRARY_PATH=${CODEC2_BUILD_DIR}/src;
export PYTHONPATH=../modem;
cd ${CMAKE_CURRENT_SOURCE_DIR}/test;
python3 test_datac13.py")
set_tests_properties(datac13_frames PROPERTIES PASS_REGULAR_EXPRESSION "errors: 0")
# disabled this test as its actually broken since we introduced dataclasses
#add_test(NAME datac13_frames_negative
# COMMAND sh -c "export LD_LIBRARY_PATH=${CODEC2_BUILD_DIR}/src;
# export PYTHONPATH=../modem;
# cd ${CMAKE_CURRENT_SOURCE_DIR}/test;
# python3 test_datac13_negative.py")
# set_tests_properties(datac13_frames_negative PROPERTIES PASS_REGULAR_EXPRESSION "errors: 0")
add_test(NAME helper_routines
COMMAND sh -c "export LD_LIBRARY_PATH=${CODEC2_BUILD_DIR}/src;
export PYTHONPATH=../modem;
cd ${CMAKE_CURRENT_SOURCE_DIR}/test;
python3 test_helpers.py")
set_tests_properties(helper_routines PROPERTIES PASS_REGULAR_EXPRESSION "errors: 0")
add_test(NAME py_highsnr_stdio_P_P_multi
COMMAND sh -c "export LD_LIBRARY_PATH=${CODEC2_BUILD_DIR}/src;
export PYTHONPATH=../modem;
export BURSTS=${BURSTS};
export FRAMESPERBURST=${FRAMESPERBURST};
PATH=$PATH:${CODEC2_BUILD_DIR}/src;
cd ${CMAKE_CURRENT_SOURCE_DIR}/test;
python3 test_highsnr_stdio_P_P_multi.py")
set_tests_properties(py_highsnr_stdio_P_P_multi PROPERTIES PASS_REGULAR_EXPRESSION "DATAC13: ${BURSTS}/${FRAMESPERBURST} DATAC1: ${BURSTS}/${FRAMESPERBURST} DATAC3: ${BURSTS}/${FRAMESPERBURST}")
add_test(NAME py_highsnr_stdio_P_P_datacx
COMMAND sh -c "export LD_LIBRARY_PATH=${CODEC2_BUILD_DIR}/src;
export PYTHONPATH=../modem;
export BURSTS=${BURSTS};
export FRAMESPERBURST=${FRAMESPERBURST};
PATH=$PATH:${CODEC2_BUILD_DIR}/src;
cd ${CMAKE_CURRENT_SOURCE_DIR}/test;
python3 test_highsnr_stdio_P_P_datacx.py")
set_tests_properties(py_highsnr_stdio_P_P_datacx PROPERTIES PASS_REGULAR_EXPRESSION "RECEIVED BURSTS: ${BURSTS} RECEIVED FRAMES: ${FRAMESPERBURST}")
add_test(NAME py_highsnr_stdio_P_C_datacx
COMMAND sh -c "export LD_LIBRARY_PATH=${CODEC2_BUILD_DIR}/src;
export PYTHONPATH=../modem;
export BURSTS=${BURSTS};
export FRAMESPERBURST=${FRAMESPERBURST};
PATH=$PATH:${CODEC2_BUILD_DIR}/src;
cd ${CMAKE_CURRENT_SOURCE_DIR}/test;
python3 test_highsnr_stdio_P_C_datacx.py")
set_tests_properties(py_highsnr_stdio_P_C_datacx PROPERTIES PASS_REGULAR_EXPRESSION "HELLO WORLD")
add_test(NAME py_highsnr_stdio_C_P_datacx
COMMAND sh -c "export LD_LIBRARY_PATH=${CODEC2_BUILD_DIR}/src;
export PYTHONPATH=../modem;
export BURSTS=${BURSTS};
export FRAMESPERBURST=${FRAMESPERBURST};
export TESTFRAMES=${TESTFRAMES};
PATH=$PATH:${CODEC2_BUILD_DIR}/src;
cd ${CMAKE_CURRENT_SOURCE_DIR}/test;
python3 test_highsnr_stdio_C_P_datacx.py")
set_tests_properties(py_highsnr_stdio_C_P_datacx PROPERTIES PASS_REGULAR_EXPRESSION "RECEIVED BURSTS: ${BURSTS} RECEIVED FRAMES: ${FRAMESPERBURST}")
add_test(NAME highsnr_stdio_P_C_single
COMMAND sh -c "export LD_LIBRARY_PATH=${CODEC2_BUILD_DIR}/src;
PATH=$PATH:${CODEC2_BUILD_DIR}/src;
cd ${CMAKE_CURRENT_SOURCE_DIR}/test;
python3 util_tx.py --mode datac13 --delay 500 --framesperburst ${FRAMESPERBURST} --bursts ${BURSTS} |
sox -t .s16 -r 48000 -c 1 - -t .s16 -r 8000 -c 1 - |
freedv_data_raw_rx datac13 - - --framesperburst ${FRAMESPERBURST} | hexdump -C")
set_tests_properties(highsnr_stdio_P_C_single PROPERTIES PASS_REGULAR_EXPRESSION "HELLO WORLD")
add_test(NAME highsnr_stdio_C_P_single
COMMAND sh -c "export LD_LIBRARY_PATH=${CODEC2_BUILD_DIR}/src;
PATH=$PATH:${CODEC2_BUILD_DIR}/src;
cd ${CMAKE_CURRENT_SOURCE_DIR}/test;
freedv_data_raw_tx datac13 --testframes ${TESTFRAMES} --bursts ${BURSTS} --framesperburst ${FRAMESPERBURST} /dev/zero - |
sox -t .s16 -r 8000 -c 1 - -t .s16 -r 48000 -c 1 - |
python3 util_rx.py --mode datac13 --framesperburst ${FRAMESPERBURST} --bursts ${BURSTS}")
set_tests_properties(highsnr_stdio_C_P_single PROPERTIES PASS_REGULAR_EXPRESSION "RECEIVED BURSTS: ${BURSTS} RECEIVED FRAMES: ${FRAMESPERBURST}")
add_test(NAME highsnr_stdio_P_P_single
COMMAND sh -c "export LD_LIBRARY_PATH=${CODEC2_BUILD_DIR}/src;
PATH=$PATH:${CODEC2_BUILD_DIR}/src;
cd ${CMAKE_CURRENT_SOURCE_DIR}/test;
python3 util_tx.py --mode datac13 --delay 500 --framesperburst ${FRAMESPERBURST} --bursts ${BURSTS} |
python3 util_rx.py --debug --mode datac13 --framesperburst ${FRAMESPERBURST} --bursts ${BURSTS}")
set_tests_properties(highsnr_stdio_P_P_single PROPERTIES PASS_REGULAR_EXPRESSION "RECEIVED BURSTS: ${BURSTS} RECEIVED FRAMES: ${FRAMESPERBURST}")
add_test(NAME highsnr_stdio_P_P_multi
COMMAND sh -c "export LD_LIBRARY_PATH=${CODEC2_BUILD_DIR}/src;
PATH=$PATH:${CODEC2_BUILD_DIR}/src;
cd ${CMAKE_CURRENT_SOURCE_DIR}/test;
python3 util_multimode_tx.py --delay 500 --framesperburst ${FRAMESPERBURST} --bursts ${BURSTS} |
python3 util_multimode_rx.py --framesperburst ${FRAMESPERBURST} --bursts ${BURSTS} --timeout 60")
set_tests_properties(highsnr_stdio_P_P_multi PROPERTIES PASS_REGULAR_EXPRESSION "DATAC13: ${BURSTS}/${FRAMESPERBURST} DATAC1: ${BURSTS}/${FRAMESPERBURST} DATAC3: ${BURSTS}/${FRAMESPERBURST}")
# These tests can't run on GitHub actions as we don't have a virtual sound card
if(NOT DEFINED ENV{GITHUB_RUN_ID})
# uses aplay/arecord then pipe to Python
add_test(NAME highsnr_virtual1_P_P_single_alsa
COMMAND sh -c "export LD_LIBRARY_PATH=${CODEC2_BUILD_DIR}/src;
PATH=$PATH:${CODEC2_BUILD_DIR}/src;
cd ${CMAKE_CURRENT_SOURCE_DIR}/test;
./test_virtual1.sh")
set_tests_properties(highsnr_virtual1_P_P_single_alsa PROPERTIES PASS_REGULAR_EXPRESSION "RECEIVED BURSTS: 5 RECEIVED FRAMES: 10 RX_ERRORS: 0")
# let Python do audio I/O
add_test(NAME highsnr_virtual2_P_P_single
COMMAND sh -c "export LD_LIBRARY_PATH=${CODEC2_BUILD_DIR}/src;
PATH=$PATH:${CODEC2_BUILD_DIR}/src;
cd ${CMAKE_CURRENT_SOURCE_DIR}/test;
./test_virtual2.sh")
set_tests_properties(highsnr_virtual2_P_P_single PROPERTIES PASS_REGULAR_EXPRESSION "RECEIVED BURSTS: 3 RECEIVED FRAMES: 6 RX_ERRORS: 0")
# Multimode test with Python I/O
add_test(NAME highsnr_virtual3_P_P_multi
COMMAND sh -c "export LD_LIBRARY_PATH=${CODEC2_BUILD_DIR}/src;
PATH=$PATH:${CODEC2_BUILD_DIR}/src;
cd ${CMAKE_CURRENT_SOURCE_DIR}/test;
./test_virtual_mm.sh")
set_tests_properties(highsnr_virtual3_P_P_multi PROPERTIES PASS_REGULAR_EXPRESSION "DATAC13: 2/4 DATAC1: 2/4 DATAC3: 2/4")
# let Python do audio I/O via pyaudio callback mode
add_test(NAME highsnr_virtual4_P_P_single_callback
COMMAND sh -c "export LD_LIBRARY_PATH=${CODEC2_BUILD_DIR}/src;
PATH=$PATH:${CODEC2_BUILD_DIR}/src;
cd ${CMAKE_CURRENT_SOURCE_DIR}/test;
./test_virtual3a.sh")
set_tests_properties(highsnr_virtual4_P_P_single_callback PROPERTIES PASS_REGULAR_EXPRESSION "RECEIVED BURSTS: 3 RECEIVED FRAMES: 6 RX_ERRORS: 0")
# let Python do audio I/O via pyaudio callback mode with code outside of callback
add_test(NAME highsnr_virtual4_P_P_single_callback_outside
COMMAND sh -c "export LD_LIBRARY_PATH=${CODEC2_BUILD_DIR}/src;
PATH=$PATH:${CODEC2_BUILD_DIR}/src;
cd ${CMAKE_CURRENT_SOURCE_DIR}/test;
./test_virtual3b.sh")
set_tests_properties(highsnr_virtual4_P_P_single_callback_outside PROPERTIES PASS_REGULAR_EXPRESSION "RECEIVED BURSTS: 3 RECEIVED FRAMES: 6 RX_ERRORS: 0")
# let Python do audio I/O via pyaudio callback mode with code outside of callback
add_test(NAME highsnr_virtual5_P_P_multi_callback
COMMAND sh -c "export LD_LIBRARY_PATH=${CODEC2_BUILD_DIR}/src;
PATH=$PATH:${CODEC2_BUILD_DIR}/src;
cd ${CMAKE_CURRENT_SOURCE_DIR}/test;
./test_virtual4a.sh")
set_tests_properties(highsnr_virtual5_P_P_multi_callback PROPERTIES PASS_REGULAR_EXPRESSION "DATAC13: 2/4 DATAC1: 2/4 DATAC3: 2/4")
# let Python do audio I/O via pyaudio callback mode with code outside of callback
add_test(NAME highsnr_virtual5_P_P_multi_callback_outside
COMMAND sh -c "export LD_LIBRARY_PATH=${CODEC2_BUILD_DIR}/src;
PATH=$PATH:${CODEC2_BUILD_DIR}/src;
cd ${CMAKE_CURRENT_SOURCE_DIR}/test;
./test_virtual4b.sh")
set_tests_properties(highsnr_virtual5_P_P_multi_callback_outside PROPERTIES PASS_REGULAR_EXPRESSION "DATAC13: 2/4 DATAC1: 2/4 DATAC3: 2/4")
endif()

630
modem/data cemetery/deprecated_daemon.py
View file

@ -1,630 +0,0 @@
#!/usr/bin/python3
# -*- coding: utf-8 -*-
"""
deprecated_daemon.py
Author: DJ2LS, January 2022
daemon for providing basic information for the modem like audio or serial devices
"""
# pylint: disable=invalid-name, line-too-long, c-extension-no-member
# pylint: disable=import-outside-toplevel
import argparse
import atexit
import multiprocessing
import os
import signal
import socketserver
import subprocess
import sys
import threading
import time
import audio
import crcengine
import log_handler
import serial.tools.list_ports
import deprecated_sock
from global_instances import ARQ, AudioParam, Beacon, Channel, Daemon, HamlibParam, ModemParam, Station, Statistics, TCIParam, Modem
import structlog
import ujson as json
import config
# signal handler for closing application
def signal_handler(sig, frame):
"""
Signal handler for closing the network socket on app exit
Args:
sig:
frame:
Returns: system exit
"""
print("Closing daemon...")
sock.CLOSE_SIGNAL = True
sys.exit(0)
signal.signal(signal.SIGINT, signal_handler)
class DAEMON:
"""
Daemon class
"""
log = structlog.get_logger("DAEMON")
def __init__(self):
# load crc engine
self.crc_algorithm = crcengine.new("crc16-ccitt-false") # load crc8 library
self.daemon_queue = sock.DAEMON_QUEUE
update_audio_devices = threading.Thread(
target=self.update_audio_devices, name="UPDATE_AUDIO_DEVICES", daemon=True
)
update_audio_devices.start()
update_serial_devices = threading.Thread(
target=self.update_serial_devices, name="UPDATE_SERIAL_DEVICES", daemon=True
)
update_serial_devices.start()
worker = threading.Thread(target=self.worker, name="WORKER", daemon=True)
worker.start()
rigctld_watchdog_thread = threading.Thread(target=self.rigctld_watchdog, name="WORKER", daemon=True)
rigctld_watchdog_thread.start()
def rigctld_watchdog(self):
"""
Check for rigctld status
Returns:
"""
while True:
threading.Event().wait(0.01)
# only continue, if we have a process object initialized
if hasattr(Daemon.rigctldprocess, "returncode"):
if Daemon.rigctldprocess.returncode in [None, "None"] or not Daemon.rigctldstarted:
Daemon.rigctldstarted = True
# outs, errs = Daemon.rigctldprocess.communicate(timeout=10)
# print(f"outs: {outs}")
# print(f"errs: {errs}")
else:
self.log.warning("[DMN] [RIGCTLD] [Watchdog] returncode detected",process=Daemon.rigctldprocess)
Daemon.rigctldstarted = False
# triggering another kill
Daemon.rigctldprocess.kill()
# erase process object
Daemon.rigctldprocess = None
else:
Daemon.rigctldstarted = False
def update_audio_devices(self):
"""
Update audio devices and set to static
"""
while True:
try:
if not Daemon.modemstarted:
(
AudioParam.audio_input_devices,
AudioParam.audio_output_devices,
) = audio.get_audio_devices()
except Exception as err1:
self.log.error(
"[DMN] update_audio_devices: Exception gathering audio devices:",
e=err1,
)
threading.Event().wait(1)
def update_serial_devices(self):
"""
Update serial devices and set to static
"""
while True:
try:
serial_devices = []
ports = serial.tools.list_ports.comports()
for port, desc, hwid in ports:
# calculate hex of hwid if we have unique names
crc_hwid = self.crc_algorithm(bytes(hwid, encoding="utf-8"))
crc_hwid = crc_hwid.to_bytes(2, byteorder="big")
crc_hwid = crc_hwid.hex()
description = f"{desc} [{crc_hwid}]"
serial_devices.append(
{"port": str(port), "description": str(description)}
)
Daemon.serial_devices = serial_devices
threading.Event().wait(1)
except Exception as err1:
self.log.error(
"[DMN] update_serial_devices: Exception gathering serial devices:",
e=err1,
)
def worker(self):
"""
Worker to handle the received commands
"""
while True:
try:
data = self.daemon_queue.get()
# increase length of list for storing additional
# parameters starting at entry 64
data = data[:64] + [None] * (64 - len(data))
# data[1] mycall
# data[2] mygrid
# data[3] rx_audio
# data[4] tx_audio
# data[5] radiocontrol
# data[6] rigctld_ip
# data[7] rigctld_port
# data[8] send_scatter
# data[9] send_fft
# data[10] low_bandwidth_mode
# data[11] tuning_range_fmin
# data[12] tuning_range_fmax
# data[13] enable FSK
# data[14] tx-audio-level
# data[15] respond_to_cq
# data[16] rx_buffer_size
# data[17] explorer
# data[18] ssid_list
# data[19] auto_tune
# data[20] stats
# data[21] tx_delay
if data[0] == "STARTModem":
self.start_modem(data)
if data[0] == "TEST_HAMLIB":
# data[9] radiocontrol
# data[10] rigctld_ip
# data[11] rigctld_port
self.test_hamlib_ptt(data)
if data[0] == "START_RIGCTLD":
"""
data[0] START_RIGCTLD,
data[1] hamlib_deviceid,
data[2] hamlib_deviceport,
data[3] hamlib_stop_bits,
data[4] hamlib_data_bits,
data[5] hamlib_handshake,
data[6] hamlib_serialspeed,
data[7] hamlib_dtrstate,
data[8] hamlib_pttprotocol,
data[9] hamlib_ptt_port,
data[10] hamlib_dcd,
data[11] hamlbib_serialspeed_ptt,
data[12] hamlib_rigctld_port,
data[13] hamlib_rigctld_ip,
data[14] hamlib_rigctld_path,
data[15] hamlib_rigctld_server_port,
data[16] hamlib_rigctld_custom_args
"""
self.start_rigctld(data)
except Exception as err1:
self.log.error("[DMN] worker: Exception: ", e=err1)
def test_hamlib_ptt(self, data):
radiocontrol = data[1]
# check how we want to control the radio
if radiocontrol == "rigctld":
import rigctld as rig
rigctld_ip = data[2]
rigctld_port = data[3]
elif radiocontrol == "tci":
import tci as rig
rigctld_ip = data[22]
rigctld_port = data[23]
else:
import rigdummy as rig
rigctld_ip = '127.0.0.1'
rigctld_port = '0'
hamlib = rig.radio()
hamlib.open_rig(
rigctld_ip=rigctld_ip,
rigctld_port=rigctld_port,
)
# hamlib_version = rig.hamlib_version
hamlib.set_ptt(True)
#Allow a little time for network based rig to register PTT is active
time.sleep(.250)
if hamlib.get_ptt():
self.log.info("[DMN] Hamlib PTT", status="SUCCESS")
response = {"command": "test_hamlib", "result": "SUCCESS"}
else:
self.log.warning("[DMN] Hamlib PTT", status="NO SUCCESS")
response = {"command": "test_hamlib", "result": "NOSUCCESS"}
hamlib.set_ptt(False)
hamlib.close_rig()
jsondata = json.dumps(response)
sock.SOCKET_QUEUE.put(jsondata)
def start_rigctld(self, data):
# Seems to be working on Win
"""
data[0] START_RIGCTLD,
data[1] hamlib_deviceid,
data[2] hamlib_deviceport,
data[3] hamlib_stop_bits,
data[4] hamlib_data_bits,
data[5] hamlib_handshake,
data[6] hamlib_serialspeed,
data[7] hamlib_dtrstate,
data[8] hamlib_pttprotocol,
data[9] hamlib_ptt_port,
data[10] hamlib_dcd,
data[11] hamlbib_serialspeed_ptt,
data[12] hamlib_rigctld_port,
data[13] hamlib_rigctld_ip,
data[14] hamlib_rigctld_path,
data[15] hamlib_rigctld_server_port,
data[16] hamlib_rigctld_custom_args
"""
try:
command = []
isWin = False
if sys.platform in ["darwin"]:
if data[14] not in [""]:
# hamlib_rigctld_path
application_path = data[14]
else:
application_path = "rigctld"
command.append(f'{application_path}')
elif sys.platform in ["linux", "darwin"]:
if data[14] not in [""]:
# hamlib_rigctld_path
application_path = data[14]
else:
application_path = "rigctld"
command.append(f'{application_path}')
elif sys.platform in ["win32", "win64"]:
isWin=True
if data[13].lower() == "localhost":
data[13]="127.0.0.1"
if data[14] not in [""]:
# hamlib_rigctld_path
application_path = data[14]
else:
application_path = "rigctld.exe"
command.append(f'{application_path}')
options = []
# hamlib_deviceid
if data[1] not in [None, "None", "ignore"]:
options.append(("--model=" + data[1] ))
# hamlib_deviceport
if data[2] not in [None, "None", "ignore"]:
options.append(("--rig-file="+ data[2]))
# hamlib_stop_bits
if data[3] not in [None, "None", "ignore"]:
options.append(("--set-conf=stop_bits=" + data[3]))
# hamlib_data_bits
if data[4] not in [None, "None", "ignore"]:
options.append(("--set-conf=data_bits=" + data[4]))
# hamlib_handshake
if data[5] not in [None, "None", "ignore"]:
options.append(("--set-conf=serial_handshake=" + data[5]))
# hamlib_serialspeed
if data[6] not in [None, "None", "ignore"]:
options.append(("--serial-speed=" + data[6]))
# hamlib_dtrstate
if data[7] not in [None, "None", "ignore"]:
options.append(("--set-conf=dtr_state=" + data[7]))
# hamlib_pttprotocol
if data[8] not in [None, "None", "ignore"]:
options.append(("--ptt-type=" + data[8]))
# hamlib_ptt_port
if data[9] not in [None, "None", "ignore"]:
options.append(("--ptt-file=" + data[9]))
# hamlib_dcd
if data[10] not in [None, "None", "ignore"]:
options.append(("--dcd-type=" + data[10]))
# hamlbib_serialspeed_ptt
if data[11] not in [None, "None", "ignore"]:
# options.extend(("-m", data[11]))
pass
# hamlib_rigctld_port
# Using this ensures rigctld starts on port configured in GUI
if data[12] not in [None, "None", "ignore"]:
options.append(("--port="+ data[12]))
# hamlib_rigctld_ip
if data[13] not in [None, "None", "ignore"]:
options.append(("--listen-addr="+ data[13]))
# data[14] == hamlib_rigctld_path
# maybe at wrong place in list...
#Not needed for setting command line arguments
# hamlib_rigctld_server_port
# Ignore configured value and use value configured in GUI
#if data[15] not in [None, "None", "ignore"]:
# options.extend(("-m", data[15]))
# pass
# hamlib_rigctld_custom_args
if data[16] not in [None, "None", "ignore"]:
for o in data[16].split(" "):
options.append(o)
# append debugging paramter
# disabled as this could be set via gui
#options.append(("-vvv"))
command += options
self.log.info("[DMN] starting rigctld: ", param=command)
if not isWin:
# NOTE --> It seems Popen is non blocking, while run is blocking
#proc = subprocess.Popen(command, stdout=subprocess.PIPE, stderr=subprocess.PIPE, shell=True)
proc = subprocess.Popen(command)
#proc = subprocess.run(command, shell=False, check=True, text=True, capture_output=True)
else:
#On windows, open rigctld in new window for easier troubleshooting
proc = subprocess.Popen(command, creationflags=subprocess.CREATE_NEW_CONSOLE,close_fds=True)
Daemon.rigctldstarted = True
Daemon.rigctldprocess = proc
atexit.register(proc.kill)
except Exception as err:
self.log.warning("[DMN] err starting rigctld: ", e=err)
def start_modem(self, data):
self.log.warning("[DMN] Starting Modem", rig=data[5], port=data[6])
# list of parameters, necessary for running subprocess command as a list
options = ["--port", str(DAEMON.port - 1)]
# create an additional list entry for parameters not covered by gui
data[50] = int(DAEMON.port - 1)
options.append("--mycall")
options.extend((data[1], "--mygrid"))
options.extend((data[2], "--rx"))
options.extend((data[3], "--tx"))
options.append(data[4])
# if radiocontrol != disabled
# this should hopefully avoid a ton of problems if we are just running in
# disabled mode
if data[5] != "disabled":
options.append("--radiocontrol")
options.append(data[5])
if data[5] == "rigctld":
options.append("--rigctld_ip")
options.extend((data[6], "--rigctld_port"))
options.append(data[7])
if data[5] == "tci":
options.append("--tci-ip")
options.extend((data[22], "--tci-port"))
options.append(data[23])
if data[8] == "True":
options.append("--scatter")
if data[9] == "True":
options.append("--fft")
if data[10] == "True":
options.append("--500hz")
options.append("--tuning_range_fmin")
options.extend((data[11], "--tuning_range_fmax"))
options.extend((data[12], "--tx-audio-level"))
options.append(data[14])
if data[15] == "True":
options.append("--qrv")
options.append("--rx-buffer-size")
options.append(data[16])
if data[17] == "True":
options.append("--explorer")
options.append("--ssid")
options.extend(str(i) for i in data[18])
if data[19] == "True":
options.append("--tune")
if data[20] == "True":
options.append("--stats")
if data[13] == "True":
options.append("--fsk")
options.append("--tx-delay")
options.append(data[21])
#Mesh
if data[24] == "True":
options.append("--mesh")
options.append("--rx-audio-level")
options.append(data[25])
#Morse identifier
if data[26] == "True":
options.append("--morse")
# safe data to config file
config.write_entire_config(data)
# Try running modem from binary, else run from source
# This helps running the modem in a developer environment
try:
command = []
if (getattr(sys, 'frozen', False) or hasattr(sys, "_MEIPASS")) and sys.platform in ["darwin"]:
# If the application is run as a bundle, the PyInstaller bootloader
# extends the sys module by a flag frozen=True and sets the app
# path into variable _MEIPASS'.
application_path = sys._MEIPASS
command.append(f'{application_path}/freedata-modem')
elif sys.platform in ["linux", "darwin"]:
command.append("./freedata-modem")
elif sys.platform in ["win32", "win64"]:
command.append("freedata-modem.exe")
command += options
proc = subprocess.Popen(command)
atexit.register(proc.kill)
Daemon.modemprocess = proc
Daemon.modemstarted = True
self.log.info("[DMN] Modem started", path="binary")
except FileNotFoundError as err1:
try:
self.log.info("[DMN] worker: ", e=err1)
command = []
if sys.platform in ["linux", "darwin"]:
command.append("python3")
elif sys.platform in ["win32", "win64"]:
command.append("python")
command.append("deprecated_main.py")
command += options
proc = subprocess.Popen(command)
atexit.register(proc.kill)
self.log.info("[DMN] Modem started", path="source")
Daemon.modemprocess = proc
Daemon.modemstarted = True
except Exception as e:
self.log.error("[DMN] Modem not started", error=e)
Daemon.modemstarted = False
if __name__ == "__main__":
mainlog = structlog.get_logger(__file__)
# we need to run this on Windows for multiprocessing support
multiprocessing.freeze_support()
# --------------------------------------------GET PARAMETER INPUTS
PARSER = argparse.ArgumentParser(description="FreeDATA Daemon")
PARSER.add_argument(
"--port",
dest="socket_port",
default=3001,
help="Socket port in the range of 1024-65535",
type=int,
)
ARGS = PARSER.parse_args()
DAEMON.port = ARGS.socket_port
try:
if sys.platform == "linux":
logging_path = os.getenv("HOME") + "/.config/" + "FreeDATA/" + "daemon"
if sys.platform == "darwin":
logging_path = (
os.getenv("HOME")
+ "/Library/"
+ "Application Support/"
+ "FreeDATA/"
+ "daemon"
)
if sys.platform in ["win32", "win64"]:
logging_path = os.getenv("APPDATA") + "/" + "FreeDATA/" + "daemon"
if not os.path.exists(logging_path):
os.makedirs(logging_path)
log_handler.setup_logging(logging_path)
except Exception as err:
mainlog.error("[DMN] logger init error", exception=err)
# init config
config = config.CONFIG("config.ini")
try:
mainlog.info("[DMN] Starting TCP/IP socket", port=DAEMON.port)
# https://stackoverflow.com/a/16641793
socketserver.TCPServer.allow_reuse_address = True
cmdserver = deprecated_sock.ThreadedTCPServer(
(Modem.host, DAEMON.port), sock.ThreadedTCPRequestHandler
)
server_thread = threading.Thread(target=cmdserver.serve_forever)
server_thread.daemon = True
server_thread.start()
except Exception as err:
mainlog.error(
"[DMN] Starting TCP/IP socket failed", port=DAEMON.port, e=err
)
sys.exit(1)
daemon = DAEMON()
mainlog.info(
"[DMN] Starting FreeDATA Daemon",
author="DJ2LS",
year="2023",
version=Modem.version,
)
while True:
threading.Event().wait(1)

35
modem/data cemetery/deprecated_data_handler.py
View file

@ -1,35 +0,0 @@
# -*- coding: UTF-8 -*-
"""
Created on Sun Dec 27 20:43:40 2020
@author: DJ2LS
"""
# pylint: disable=invalid-name, line-too-long, c-extension-no-member
# pylint: disable=import-outside-toplevel, attribute-defined-outside-init
# pylint: disable=fixme
import threading
import helpers
import structlog
from modem_frametypes import FRAME_TYPE as FR_TYPE
import event_manager
TESTMODE = False
class DATA:
"""Terminal Node Controller for FreeDATA"""
log = structlog.get_logger("DATA")
def __init__(self, config, event_queue, states):
self.config = config
self.event_queue = event_queue
self.states = states

194
modem/data cemetery/deprecated_data_handler_broadcasts.py
View file

@ -1,194 +0,0 @@
import time
from modem_frametypes import FRAME_TYPE as FR_TYPE
from codec2 import FREEDV_MODE
from queues import MODEM_TRANSMIT_QUEUE
import helpers
from random import randrange
import uuid
import structlog
import event_manager
import command_qrv
from deprecated_data_handler import DATA
TESTMODE = False
class BROADCAST(DATA):
def __init__(self, config, event_queue, states):
super().__init__(config, event_queue, states)
self.log = structlog.get_logger("DHBC")
self.states = states
self.event_queue = event_queue
self.config = config
self.event_manager = event_manager.EventManager([event_queue])
# length of signalling frame
self.length_sig0_frame = 14
self.modem_frequency_offset = 0
# load config
self.mycallsign = config['STATION']['mycall']
self.myssid = config['STATION']['myssid']
self.mycallsign += "-" + str(self.myssid)
encoded_call = helpers.callsign_to_bytes(self.mycallsign)
self.mycallsign_bytes = helpers.bytes_to_callsign(encoded_call)
self.mygrid = config['STATION']['mygrid']
self.enable_fsk = config['MODEM']['enable_fsk']
self.respond_to_cq = config['MODEM']['respond_to_cq']
self.respond_to_call = True
self.duration_datac13 = 2.0
self.duration_sig1_frame = self.duration_datac13
def received_cq(self, frame_data, snr) -> None:
"""
Called when we receive a CQ frame
Args:
data_in:bytes:
Returns:
Nothing
"""
# here we add the received station to the heard stations buffer
dxcallsign = frame_data['origin']
self.log.debug("[Modem] received_cq:", dxcallsign=dxcallsign)
self.dxgrid = frame_data['gridsquare']
self.event_manager.send_custom_event(
freedata="modem-message",
cq="received",
mycallsign=self.mycallsign,
dxcallsign=dxcallsign,
dxgrid=self.dxgrid,
)
self.log.info(
"[Modem] CQ RCVD ["
+ dxcallsign
+ "]["
+ self.dxgrid
+ "] ",
snr=snr,
)
helpers.add_to_heard_stations(
dxcallsign,
self.dxgrid,
"CQ CQ CQ",
snr,
self.modem_frequency_offset,
self.states.radio_frequency,
self.states.heard_stations
)
# Sleep a random amount of time before responding to make it more likely to be
# heard when many stations respond. Each DATAC0 frame is 0.44 sec (440ms) in
# duration, plus overhead. Set the wait interval to be random between 0 and
# self.duration_sig1_frame * 4 == 4 slots
# in self.duration_sig1_frame increments.
if self.respond_to_cq and self.respond_to_call:
params = {'snr': snr, 'dxcall': dxcallsign}
cmd = command_qrv.QRVCommand(self.config, self.log, params)
cmd.run(self.event_queue, MODEM_TRANSMIT_QUEUE)
def received_qrv(self, data_in: bytes, snr) -> None:
"""
Called when we receive a QRV frame
Args:
data_in:bytes:
"""
# here we add the received station to the heard stations buffer
dxcallsign = helpers.bytes_to_callsign(bytes(data_in[1:7]))
self.dxgrid = bytes(helpers.decode_grid(data_in[7:11]), "UTF-8")
dxsnr = helpers.snr_from_bytes(data_in[11:12])
combined_snr = f"{snr}/{dxsnr}"
self.event_manager.send_custom_event(
freedata="modem-message",
qrv="received",
dxcallsign=str(dxcallsign, "UTF-8"),
dxgrid=str(self.dxgrid, "UTF-8"),
snr=str(snr),
dxsnr=str(dxsnr)
)
self.log.info(
"[Modem] QRV RCVD ["
+ str(dxcallsign, "UTF-8")
+ "]["
+ str(self.dxgrid, "UTF-8")
+ "] ",
snr=snr,
dxsnr=dxsnr
)
helpers.add_to_heard_stations(
dxcallsign,
self.dxgrid,
"QRV",
combined_snr,
self.modem_frequency_offset,
self.states.radio_frequency,
self.states.heard_stations
)
def received_is_writing(self, data_in: bytes, snr) -> None:
"""
Called when we receive a IS WRITING frame
Args:
data_in:bytes:
"""
# here we add the received station to the heard stations buffer
dxcallsign = helpers.bytes_to_callsign(bytes(data_in[1:7]))
self.event_manager.send_custom_event(
freedata="modem-message",
fec="is_writing",
dxcallsign=str(dxcallsign, "UTF-8")
)
self.log.info(
"[Modem] IS_WRITING RCVD ["
+ str(dxcallsign, "UTF-8")
+ "] ",
)
# ----------- BROADCASTS
def received_beacon(self, frame_data, snr) -> None:
"""
Called if we received a beacon
Args:
data_in:bytes:
"""
# here we add the received station to the heard stations buffer
beacon_callsign = frame_data['origin']
self.dxgrid = frame_data['gridsquare']
self.event_manager.send_custom_event(
freedata="modem-message",
beacon="received",
uuid=str(uuid.uuid4()),
timestamp=int(time.time()),
dxcallsign=beacon_callsign,
dxgrid=self.dxgrid,
snr=str(snr),
)
self.log.info(
f"[Modem] BEACON RCVD [{beacon_callsign}][{self.dxgrid}]",
snr=snr,
)
helpers.add_to_heard_stations(
beacon_callsign,
self.dxgrid,
"BEACON",
snr,
self.modem_frequency_offset,
self.states.radio_frequency,
self.states.heard_stations
)

122
modem/data cemetery/deprecated_data_handler_data_broadcasts.py
View file

@ -1,122 +0,0 @@
import structlog
import threading
import helpers
import time
import modem
import base64
import ujson as json
from deprecated_data_handler import DATA
class DATABROADCAST(DATA):
"""Terminal Node Controller for FreeDATA"""
log = structlog.get_logger("BROADCAST")
def __init__(self, config, event_queue, states) -> None:
super().__init__(config, event_queue, states)
self.log = structlog.get_logger("DHDBC")
self.states = states
self.event_queue = event_queue
self.mycallsign = config['STATION']['mycall']
self.fec_wakeup_callsign = bytes()
self.longest_duration = 6
self.wakeup_received = False
self.broadcast_timeout_reached = False
self.broadcast_payload_bursts = 1
self.broadcast_watchdog = threading.Thread(
target=self.watchdog, name="watchdog thread", daemon=True
)
self.broadcast_watchdog.start()
self.event_queue = event_queue
def received_fec_wakeup(self, data_in: bytes):
self.fec_wakeup_callsign = helpers.bytes_to_callsign(bytes(data_in[1:7]))
self.wakeup_mode = int.from_bytes(bytes(data_in[7:8]), "big")
bursts = int.from_bytes(bytes(data_in[8:9]), "big")
self.wakeup_received = True
modem.RECEIVE_DATAC4 = True
self.send_data_to_socket_queue(
freedata="modem-message",
fec="wakeup",
mode=self.wakeup_mode,
bursts=bursts,
dxcallsign=str(self.fec_wakeup_callsign, "UTF-8")
)
self.log.info(
"[Modem] FRAME WAKEUP RCVD ["
+ str(self.fec_wakeup_callsign, "UTF-8")
+ "] ", mode=self.wakeup_mode, bursts=bursts,
)
def received_fec(self, data_in: bytes):
print(self.fec_wakeup_callsign)
self.send_data_to_socket_queue(
freedata="modem-message",
fec="broadcast",
dxcallsign=str(self.fec_wakeup_callsign, "UTF-8"),
data=base64.b64encode(data_in[1:]).decode("UTF-8")
)
self.log.info("[Modem] FEC DATA RCVD")
def send_data_to_socket_queue(self, **jsondata):
"""
Send information to the UI via JSON and the sock.SOCKET_QUEUE.
Args:
Dictionary containing the data to be sent, in the format:
key=value, for each item. E.g.:
self.send_data_to_socket_queue(
freedata="modem-message",
arq="received",
status="success",
uuid=self.transmission_uuid,
timestamp=timestamp,
mycallsign=str(self.mycallsign, "UTF-8"),
dxcallsign=str(Station.dxcallsign, "UTF-8"),
dxgrid=str(Station.dxgrid, "UTF-8"),
data=base64_data,
)
"""
# add mycallsign and dxcallsign to network message if they not exist
# and make sure we are not overwrite them if they exist
try:
if "mycallsign" not in jsondata:
jsondata["mycallsign"] = str(self.mycallsign, "UTF-8")
except Exception as e:
self.log.debug("[Modem] error adding callsigns to network message", e=e)
# run json dumps
json_data_out = json.dumps(jsondata)
self.log.debug("[Modem] send_data_to_socket_queue:", jsondata=json_data_out)
# finally push data to our network queue
self.event_queue.put(json_data_out)
def watchdog(self):
while 1:
if self.wakeup_received:
timeout = time.time() + (self.longest_duration * self.broadcast_payload_bursts) + 2
while time.time() < timeout:
threading.Event().wait(0.01)
self.broadcast_timeout_reached = True
self.log.info(
"[Modem] closing broadcast slot ["
+ str(self.fec_wakeup_callsign, "UTF-8")
+ "] ", mode=self.wakeup_mode, bursts=self.broadcast_payload_bursts,
)
# TODO We need a dynamic way of modifying this
modem.RECEIVE_DATAC4 = False
self.fec_wakeup_callsign = bytes()
self.wakeup_received = False
else:
threading.Event().wait(0.01)

145
modem/data cemetery/deprecated_data_handler_ping.py
View file

@ -1,145 +0,0 @@
import time
from modem_frametypes import FRAME_TYPE as FR_TYPE
from codec2 import FREEDV_MODE
import helpers
import uuid
import structlog
from deprecated_data_handler import DATA
class PING(DATA):
def __init__(self, config, event_queue, states):
super().__init__(config, event_queue, states)
self.log = structlog.get_logger("DHPING")
self.states = states
self.event_queue = event_queue
self.config = config
def received_ping(self, deconstructed_frame: list, snr) -> None:
"""
Called if we received a ping
Args:
data_in:bytes:
"""
destination_crc = deconstructed_frame["destination_crc"]
origin_crc = deconstructed_frame["origin_crc"]
origin = deconstructed_frame["origin"]
# check if callsign ssid override
valid, mycallsign = helpers.check_callsign(self.config['STATION']['mycall'], destination_crc, self.config['STATION']['ssid_list'])
if not valid:
# PING packet not for me.
self.log.debug("[Modem] received_ping: ping not for this station.")
return
self.dxcallsign_crc = origin_crc
self.dxcallsign = origin
self.log.info(
f"[Modem] PING REQ from [{origin}] to [{mycallsign}]",
snr=snr,
)
self.dxgrid = ""
helpers.add_to_heard_stations(
origin,
self.dxgrid,
"PING",
snr,
-999, # TODO we don't have the offset available here yet...
self.states.radio_frequency,
self.states.heard_stations
)
self.event_queue.put({
"freedata": "modem-message",
"ping": "received",
"uuid": str(uuid.uuid4()),
"timestamp": int(time.time()),
"dxgrid": self.dxgrid,
"dxcallsign": origin,
"mycallsign": mycallsign,
"snr": str(snr),
})
self.transmit_ping_ack(snr)
def transmit_ping_ack(self, snr):
"""
transmit a ping ack frame
called by def received_ping
"""
self.log.warning('DATA_HANDLER_PING: REVISE PING ACK!')
return
ping_frame = bytearray(self.length_sig0_frame)
ping_frame[:1] = bytes([FR_TYPE.PING_ACK.value])
ping_frame[1:4] = self.dxcallsign_crc
ping_frame[4:7] = self.mycallsign_crc
ping_frame[7:11] = helpers.encode_grid(self.mygrid)
ping_frame[13:14] = helpers.snr_to_bytes(snr)
if self.enable_fsk:
self.enqueue_frame_for_tx([ping_frame], c2_mode=FREEDV_MODE.fsk_ldpc_0.value)
else:
self.enqueue_frame_for_tx([ping_frame], c2_mode=FREEDV_MODE.sig0.value)
def received_ping_ack(self, data_in: bytes, snr) -> None:
"""
Called if a PING ack has been received
Args:
data_in:bytes:
"""
# check if we received correct ping
# check if callsign ssid override
_valid, mycallsign = helpers.check_callsign(self.mycallsign, data_in[1:4], self.ssid_list)
if _valid:
self.dxgrid = bytes(helpers.decode_grid(data_in[7:11]), "UTF-8")
dxsnr = helpers.snr_from_bytes(data_in[13:14])
self.send_data_to_socket_queue(
freedata="modem-message",
ping="acknowledge",
uuid=str(uuid.uuid4()),
timestamp=int(time.time()),
dxgrid=str(self.dxgrid, "UTF-8"),
dxcallsign=str(self.dxcallsign, "UTF-8"),
mycallsign=str(mycallsign, "UTF-8"),
snr=str(snr),
dxsnr=str(dxsnr)
)
# combined_snr = own rx snr / snr on dx side
combined_snr = f"{snr}/{dxsnr}"
helpers.add_to_heard_stations(
self.dxcallsign,
self.dxgrid,
"PING-ACK",
combined_snr,
self.modem_frequency_offset,
self.states.radio_frequency,
self.states.heard_stations
)
self.log.info(
"[Modem] PING ACK ["
+ str(mycallsign, "UTF-8")
+ "] >|< ["
+ str(self.dxcallsign, "UTF-8")
+ "]",
snr=snr,
dxsnr=dxsnr,
)
self.states.set("is_modem_busy", False)
else:
self.log.info(
"[Modem] FOREIGN PING ACK ["
+ str(self.mycallsign, "UTF-8")
+ "] ??? ["
+ str(bytes(data_in[4:7]), "UTF-8")
+ "]",
snr=snr,
)

253
modem/data cemetery/deprecated_main.py
View file

@ -1,253 +0,0 @@
#!/usr/bin/python3
# -*- coding: utf-8 -*-
"""
Created on Tue Dec 22 16:58:45 2020
@author: DJ2LS
main module for running the modem
"""
# run modem self test on startup before we are doing other things
# import selftest
# selftest.TEST()
# continue if we passed the test
import multiprocessing
import os
import signal
import socketserver
import sys
import threading
import argparse
import config
import data_handler
import helpers
import log_handler
import modem
from global_instances import ARQ, AudioParam, Beacon, Channel, Daemon, HamlibParam, ModemParam, Station, Statistics, TCIParam, Modem, MeshParam
import structlog
import explorer
import json
import mesh
from os.path import exists
log = structlog.get_logger("main")
def signal_handler(sig, frame):
"""
a signal handler, which closes the network/socket when closing the application
Args:
sig: signal
frame:
Returns: system exit
"""
print("Closing Modem...")
deprecated_sock.CLOSE_SIGNAL = True
sys.exit(0)
signal.signal(signal.SIGINT, signal_handler)
# This is for Windows multiprocessing support
multiprocessing.freeze_support()
parser = argparse.ArgumentParser()
parser.add_argument("--use-config",
help = "Specify a config file",
default = 'config.ini')
args = parser.parse_args()
# init config
config_file = args.use_config
if not exists(config_file):
print("Config file %s not found. Exiting." % config_file)
exit(1)
conf = config.CONFIG(config_file)
try:
# additional step for being sure our callsign is correctly
# in case we are not getting a station ssid
# then we are forcing a station ssid = 0
mycallsign = bytes(conf.get('STATION', 'mycall', 'AA0AA'), "utf-8")
mycallsign = helpers.callsign_to_bytes(mycallsign)
Station.mycallsign = helpers.bytes_to_callsign(mycallsign)
Station.mycallsign_crc = helpers.get_crc_24(Station.mycallsign)
#json.loads = for converting str list to list
Station.ssid_list = json.loads(conf.get('STATION', 'ssid_list', '[0, 1, 2, 3, 4, 5, 6, 7, 8, 9]'))
# init config
conf = config.CONFIG(config_file)
try:
# additional step for being sure our callsign is correctly
# in case we are not getting a station ssid
# then we are forcing a station ssid = 0
mycallsign = bytes(conf.get('STATION', 'mycall', 'AA0AA'), "utf-8")
mycallsign = helpers.callsign_to_bytes(mycallsign)
Station.mycallsign = helpers.bytes_to_callsign(mycallsign)
Station.mycallsign_crc = helpers.get_crc_24(Station.mycallsign)
#json.loads = for converting str list to list
Station.ssid_list = json.loads(conf.get('STATION', 'ssid_list', '[0, 1, 2, 3, 4, 5, 6, 7, 8, 9]'))
Station.mygrid = bytes(conf.get('STATION', 'mygrid', 'JN12aa'), "utf-8")
# check if we have an int or str as device name
try:
AudioParam.audio_input_device = int(conf.get('AUDIO', 'rx', '0'))
except ValueError:
AudioParam.audio_input_device = conf.get('AUDIO', 'rx', '0')
try:
AudioParam.audio_output_device = int(conf.get('AUDIO', 'tx', '0'))
except ValueError:
AudioParam.audio_output_device = conf.get('AUDIO', 'tx', '0')
Modem.port = int(conf.get('NETWORK', 'modemport', '3000'))
HamlibParam.hamlib_radiocontrol = conf.get('RADIO', 'radiocontrol', 'disabled')
HamlibParam.hamlib_rigctld_ip = conf.get('RADIO', 'rigctld_ip', '127.0.0.1')
HamlibParam.hamlib_rigctld_port = str(conf.get('RADIO', 'rigctld_port', '4532'))
ModemParam.enable_scatter = conf.get('Modem', 'scatter', 'True')
AudioParam.enable_fft = conf.get('Modem', 'fft', 'True')
Modem.enable_fsk = conf.get('Modem', 'fsk', 'False')
Modem.low_bandwidth_mode = conf.get('Modem', 'narrowband', 'False')
ModemParam.tuning_range_fmin = float(conf.get('Modem', 'fmin', '-50.0'))
ModemParam.tuning_range_fmax = float(conf.get('Modem', 'fmax', '50.0'))
AudioParam.tx_audio_level = int(conf.get('AUDIO', 'txaudiolevel', '0'))
AudioParam.rx_audio_level = int(conf.get('AUDIO', 'rxaudiolevel', '0'))
Modem.respond_to_cq = conf.get('Modem', 'qrv', 'True')
ARQ.rx_buffer_size = int(conf.get('Modem', 'rx_buffer_size', '16'))
Modem.enable_explorer = conf.get('Modem', 'explorer', 'False')
AudioParam.audio_auto_tune = conf.get('AUDIO', 'auto_tune', 'False')
Modem.enable_stats = conf.get('Modem', 'stats', 'False')
TCIParam.ip = str(conf.get('TCI', 'tci_ip', 'localhost'))
TCIParam.port = int(conf.get('TCI', 'tci_port', '50001'))
ModemParam.tx_delay = int(conf.get('Modem', 'tx_delay', '0'))
MeshParam.enable_protocol = conf.get('MESH','mesh_enable','False')
MeshParam.transmit_morse_identifier = conf.get('Modem','transmit_morse_identifier','False')
except KeyError as e:
log.warning("[CFG] Error reading config file near", key=str(e))
except Exception as e:
log.warning("[CFG] Error", e=e)
# make sure the own ssid is always part of the ssid list
my_ssid = int(Station.mycallsign.split(b'-')[1])
if my_ssid not in Station.ssid_list:
Station.ssid_list.append(my_ssid)
Station.mygrid = bytes(conf.get('STATION', 'mygrid', 'JN12aa'), "utf-8")
# check if we have an int or str as device name
# we need to wait until we got all parameters from argparse first before we can load the other modules
import deprecated_sock
try:
AudioParam.audio_input_device = int(conf.get('AUDIO', 'rx', '0'))
except ValueError:
AudioParam.audio_input_device = conf.get('AUDIO', 'rx', '0')
try:
AudioParam.audio_output_device = int(conf.get('AUDIO', 'tx', '0'))
except ValueError:
AudioParam.audio_output_device = conf.get('AUDIO', 'tx', '0')
Modem.port = int(conf.get('NETWORK', 'modemport', '3000'))
HamlibParam.hamlib_radiocontrol = conf.get('RADIO', 'radiocontrol', 'disabled')
HamlibParam.hamlib_rigctld_ip = conf.get('RADIO', 'rigctld_ip', '127.0.0.1')
HamlibParam.hamlib_rigctld_port = str(conf.get('RADIO', 'rigctld_port', '4532'))
ModemParam.enable_scatter = conf.get('Modem', 'scatter', 'True')
AudioParam.enable_fft = conf.get('Modem', 'fft', 'True')
Modem.enable_fsk = conf.get('Modem', 'fsk', 'False')
Modem.low_bandwidth_mode = conf.get('Modem', 'narrowband', 'False')
ModemParam.tuning_range_fmin = float(conf.get('Modem', 'fmin', '-50.0'))
ModemParam.tuning_range_fmax = float(conf.get('Modem', 'fmax', '50.0'))
AudioParam.tx_audio_level = int(conf.get('AUDIO', 'txaudiolevel', '100'))
Modem.respond_to_cq = conf.get('Modem', 'qrv', 'True')
ARQ.rx_buffer_size = int(conf.get('Modem', 'rx_buffer_size', '16'))
ARQ.arq_save_to_folder = conf.get('Modem', 'save_to_folder', 'False')
Modem.enable_explorer = conf.get('Modem', 'explorer', 'False')
AudioParam.audio_auto_tune = conf.get('AUDIO', 'auto_tune', 'False')
Modem.enable_stats = conf.get('Modem', 'stats', 'False')
TCIParam.ip = str(conf.get('TCI', 'tci_ip', 'localhost'))
TCIParam.port = int(conf.get('TCI', 'tci_port', '50001'))
ModemParam.tx_delay = int(conf.get('Modem', 'tx_delay', '0'))
MeshParam.enable_protocol = conf.get('MESH','mesh_enable','False')
except KeyError as e:
log.warning("[CFG] Error reading config file near", key=str(e))
except Exception as e:
log.warning("[CFG] Error", e=e)
# make sure the own ssid is always part of the ssid list
my_ssid = int(Station.mycallsign.split(b'-')[1])
if my_ssid not in Station.ssid_list:
Station.ssid_list.append(my_ssid)
# we need to wait until we got all parameters from argparse first before we can load the other modules
import deprecated_sock
# config logging
try:
if sys.platform == "linux":
logging_path = os.getenv("HOME") + "/.config/" + "FreeDATA/" + "modem"
if sys.platform == "darwin":
logging_path = (
os.getenv("HOME")
+ "/Library/"
+ "Application Support/"
+ "FreeDATA/"
+ "modem"
)
if sys.platform in ["win32", "win64"]:
logging_path = os.getenv("APPDATA") + "/" + "FreeDATA/" + "modem"
if not os.path.exists(logging_path):
os.makedirs(logging_path)
log_handler.setup_logging(logging_path)
except Exception as err:
log.error("[DMN] logger init error", exception=err)
log.info(
"[Modem] Starting FreeDATA", author="DJ2LS", version=Modem.version
)
# start data handler
data_handler.DATA(conf.config)
# start modem
modem = modem.RF(conf.config)
# start mesh protocol only if enabled
if MeshParam.enable_protocol:
log.info("[MESH] loading module")
# start mesh module
mesh = mesh.MeshRouter()
# optionally start explorer module
if Modem.enable_explorer:
log.info("[EXPLORER] Publishing to https://explorer.freedata.app", state=Modem.enable_explorer)
explorer = explorer.explorer()
# --------------------------------------------START CMD SERVER
try:
log.info("[Modem] Starting TCP/IP socket", port=Modem.port)
# https://stackoverflow.com/a/16641793
socketserver.TCPServer.allow_reuse_address = True
cmdserver = sock.ThreadedTCPServer(
(Modem.host, Modem.port), sock.ThreadedTCPRequestHandler
)
server_thread = threading.Thread(target=cmdserver.serve_forever)
server_thread.daemon = True
server_thread.start()
except Exception as err:
log.error("[Modem] Starting TCP/IP socket failed", port=Modem.port, e=err)
sys.exit(1)
server_thread.join()

309
modem/data cemetery/deprecated_protocol_arq_connection.py
View file

@ -1,309 +0,0 @@
import time
import helpers
from codec2 import FREEDV_MODE
from modem_frametypes import FRAME_TYPE as FR_TYPE
from deprecated_protocol_arq_session import ARQ
class SESSION(ARQ):
def __init__(self, config, event_queue, states):
super().__init__(config, event_queue, states)
def received_session_close(self, data_in: bytes, snr):
"""
Closes the session when a close session frame is received and
the DXCALLSIGN_CRC matches the remote station participating in the session.
Args:
data_in:bytes:
"""
# We've arrived here from process_data which already checked that the frame
# is intended for this station.
# Close the session if the CRC matches the remote station in static.
_valid_crc, mycallsign = helpers.check_callsign(self.mycallsign, bytes(data_in[2:5]), self.ssid_list)
_valid_session = helpers.check_session_id(self.session_id, bytes(data_in[1:2]))
if (_valid_crc or _valid_session) and self.states.arq_session_state not in ["disconnected"]:
self.states.set("arq_session_state", "disconnected")
self.dxgrid = b'------'
helpers.add_to_heard_stations(
self.dxcallsign,
self.dxgrid,
"DATA",
snr,
self.modem_frequency_offset,
self.states.radio_frequency,
self.states.heard_stations
)
self.log.info(
"[Modem] SESSION ["
+ str(self.mycallsign, "UTF-8")
+ "]<<X>>["
+ str(self.dxcallsign, "UTF-8")
+ "]",
self.states.arq_session_state,
)
self.event_manager.send_custom_event(
freedata="modem-message",
arq="session",
status="close",
mycallsign=str(self.mycallsign, 'UTF-8'),
dxcallsign=str(self.dxcallsign, 'UTF-8'),
)
self.IS_ARQ_SESSION_MASTER = False
self.states.is_arq_session = False
self.arq_cleanup()
def transmit_session_heartbeat(self) -> None:
"""Send ARQ sesion heartbeat while connected"""
# self.states.is_arq_session = True
# self.states.set("is_modem_busy", True)
# self.states.set("arq_session_state", "connected")
connection_frame = bytearray(self.length_sig0_frame)
connection_frame[:1] = bytes([FR_TYPE.ARQ_SESSION_HB.value])
connection_frame[1:2] = self.session_id
self.event_manager.send_custom_event(
freedata="modem-message",
arq="session",
status="connected",
heartbeat="transmitting",
mycallsign=str(self.mycallsign, 'UTF-8'),
dxcallsign=str(self.dxcallsign, 'UTF-8'),
)
self.enqueue_frame_for_tx([connection_frame], c2_mode=FREEDV_MODE.sig0.value, copies=1, repeat_delay=0)
def received_session_heartbeat(self, data_in: bytes, snr) -> None:
"""
Received an ARQ session heartbeat, record and update state accordingly.
Args:
data_in:bytes:
"""
# Accept session data if the DXCALLSIGN_CRC matches the station in static or session id.
_valid_crc, _ = helpers.check_callsign(self.dxcallsign, bytes(data_in[4:7]), self.ssid_list)
_valid_session = helpers.check_session_id(self.session_id, bytes(data_in[1:2]))
if _valid_crc or _valid_session and self.states.arq_session_state in ["connected", "connecting"]:
self.log.debug("[Modem] Received session heartbeat")
self.dxgrid = b'------'
helpers.add_to_heard_stations(
self.dxcallsign,
self.dxgrid,
"SESSION-HB",
snr,
self.modem_frequency_offset,
self.states.radio_frequency,
self.states.heard_stations
)
self.event_manager.send_custom_event(
freedata="modem-message",
arq="session",
status="connected",
heartbeat="received",
mycallsign=str(self.mycallsign, 'UTF-8'),
dxcallsign=str(self.dxcallsign, 'UTF-8'),
)
self.states.is_arq_session = True
self.states.set("arq_session_state", "connected")
self.states.set("is_modem_busy", True)
# Update the timeout timestamps
self.arq_session_last_received = int(time.time())
self.data_channel_last_received = int(time.time())
# transmit session heartbeat only
# -> if not session master
# --> this will be triggered by heartbeat watchdog
# -> if not during a file transfer
# -> if ARQ_SESSION_STATE != disconnecting, disconnected, failed
# --> to avoid heartbeat toggle loops while disconnecting
if (
not self.IS_ARQ_SESSION_MASTER
and not self.arq_file_transfer
and self.states.arq_session_state != 'disconnecting'
and self.states.arq_session_state != 'disconnected'
and self.states.arq_session_state != 'failed'
):
self.transmit_session_heartbeat()
def close_session(self) -> None:
"""Close the ARQ session"""
self.states.set("arq_session_state", "disconnecting")
self.log.info(
"[Modem] SESSION ["
+ str(self.mycallsign, "UTF-8")
+ "]<<X>>["
+ str(self.dxcallsign, "UTF-8")
+ "]",
self.states.arq_session_state,
)
self.event_manager.send_custom_event(
freedata="modem-message",
arq="session",
status="close",
mycallsign=str(self.mycallsign, 'UTF-8'),
dxcallsign=str(self.dxcallsign, 'UTF-8'),
)
self.IS_ARQ_SESSION_MASTER = False
self.states.is_arq_session = False
# we need to send disconnect frame before doing arq cleanup
# we would lose our session id then
self.send_disconnect_frame()
# transmit morse identifier if configured
if self.enable_morse_identifier:
MODEM_TRANSMIT_QUEUE.put(["morse", 1, 0, self.mycallsign])
self.arq_cleanup()
def open_session(self) -> bool:
"""
Create and send the frame to request a connection.
Returns:
True if the session was opened successfully
False if the session open request failed
"""
self.IS_ARQ_SESSION_MASTER = True
self.states.set("arq_session_state", "connecting")
# create a random session id
self.session_id = np.random.bytes(1)
# build connection frame
connection_frame = self.frame_factory.build_arq_session_connect(
session_id=self.session_id,
destination_crc=self.dxcallsign_crc,
)
while not self.states.is_arq_session:
threading.Event().wait(0.01)
for attempt in range(self.session_connect_max_retries):
self.log.info(
"[Modem] SESSION ["
+ str(self.mycallsign, "UTF-8")
+ "]>>?<<["
+ str(self.dxcallsign, "UTF-8")
+ "]",
a=f"{str(attempt + 1)}/{str(self.session_connect_max_retries)}",
state=self.states.arq_session_state,
)
self.event_manager.send_custom_event(
freedata="modem-message",
arq="session",
status="connecting",
attempt=attempt + 1,
maxattempts=self.session_connect_max_retries,
mycallsign=str(self.mycallsign, 'UTF-8'),
dxcallsign=str(self.dxcallsign, 'UTF-8'),
)
self.enqueue_frame_for_tx([connection_frame], c2_mode=FREEDV_MODE.sig0.value, copies=1, repeat_delay=0)
# Wait for a time, looking to see if `self.states.is_arq_session`
# indicates we've received a positive response from the far station.
timeout = time.time() + 3
while time.time() < timeout:
threading.Event().wait(0.01)
# Stop waiting if data channel is opened
if self.states.is_arq_session:
return True
# Stop waiting and interrupt if data channel is getting closed while opening
if self.states.arq_session_state == "disconnecting":
# disabled this session close as its called twice
# self.close_session()
return False
# Session connect timeout, send close_session frame to
# attempt to clean up the far-side, if it received the
# open_session frame and can still hear us.
if not self.states.is_arq_session:
self.close_session()
return False
# Given the while condition, it will only exit when `self.states.is_arq_session` is True
self.event_manager.send_custom_event(
freedata="modem-message",
arq="session",
status="connected",
mycallsign=str(self.mycallsign, 'UTF-8'),
dxcallsign=str(self.dxcallsign, 'UTF-8'),
)
return True
def received_session_opener(self, data_in: bytes, snr) -> None:
"""
Received a session open request packet.
Args:
data_in:bytes:
"""
# if we don't want to respond to calls, return False
if not self.respond_to_call:
return False
# ignore channel opener if already in ARQ STATE
# use case: Station A is connecting to Station B while
# Station B already tries connecting to Station A.
# For avoiding ignoring repeated connect request in case of packet loss
# we are only ignoring packets in case we are ISS
if self.states.is_arq_session and self.IS_ARQ_SESSION_MASTER:
return False
self.IS_ARQ_SESSION_MASTER = False
self.states.set("arq_session_state", "connecting")
# Update arq_session timestamp
self.arq_session_last_received = int(time.time())
self.session_id = bytes(data_in[1:2])
self.dxcallsign_crc = bytes(data_in[5:8])
self.dxcallsign = helpers.bytes_to_callsign(bytes(data_in[8:14]))
self.states.set("dxcallsign", self.dxcallsign)
# check if callsign ssid override
valid, mycallsign = helpers.check_callsign(self.mycallsign, data_in[2:5], self.ssid_list)
self.mycallsign = mycallsign
self.dxgrid = b'------'
helpers.add_to_heard_stations(
self.dxcallsign,
self.dxgrid,
"DATA",
snr,
self.modem_frequency_offset,
self.states.radio_frequency,
self.states.heard_stations
)
self.log.info(
"[Modem] SESSION ["
+ str(self.mycallsign, "UTF-8")
+ "]>>|<<["
+ str(self.dxcallsign, "UTF-8")
+ "]",
self.states.arq_session_state,
)
self.states.is_arq_session = True
self.states.set("is_modem_busy", True)
self.event_manager.send_custom_event(
freedata="modem-message",
arq="session",
status="connected",
mycallsign=str(self.mycallsign, 'UTF-8'),
dxcallsign=str(self.dxcallsign, 'UTF-8'),
)
self.transmit_session_heartbeat()

736
modem/data cemetery/deprecated_protocol_arq_session.py
View file

@ -1,736 +0,0 @@
import threading
import time
import codec2
import helpers
import modem
import stats
import structlog
from data_frame_factory import DataFrameFactory
from codec2 import FREEDV_MODE, FREEDV_MODE_USED_SLOTS
from modem_frametypes import FRAME_TYPE as FR_TYPE
import event_manager
TESTMODE = False
class ARQ:
def __init__(self, config, event_queue, states):
self.log = structlog.get_logger("DHARQ")
self.event_queue = event_queue
self.states = states
self.event_manager = event_manager.EventManager([event_queue])
self.frame_factory = DataFrameFactory(config)
# ARQ PROTOCOL VERSION
# v.5 - signalling frame uses datac0
# v.6 - signalling frame uses datac13
# v.7 - adjusting ARQ timeout
# v.8 - adjusting ARQ structure
self.arq_protocol_version = 8
self.stats = stats.stats(config, event_queue, states)
# load config
self.mycallsign = config['STATION']['mycall']
self.myssid = config['STATION']['myssid']
self.mycallsign += "-" + str(self.myssid)
encoded_call = helpers.callsign_to_bytes(self.mycallsign)
self.mycallsign = helpers.bytes_to_callsign(encoded_call)
self.ssid_list = config['STATION']['ssid_list']
self.mycallsign_crc = helpers.get_crc_24(self.mycallsign)
self.mygrid = config['STATION']['mygrid']
self.enable_fsk = config['MODEM']['enable_fsk']
self.respond_to_cq = config['MODEM']['respond_to_cq']
self.enable_hmac = config['MODEM']['enable_hmac']
self.enable_stats = config['STATION']['enable_stats']
self.enable_morse_identifier = config['MODEM']['enable_morse_identifier']
self.arq_rx_buffer_size = config['MODEM']['rx_buffer_size']
self.enable_experimental_features = False
# flag to indicate if modem running in low bandwidth mode
self.low_bandwidth_mode = config["MODEM"]["enable_low_bandwidth_mode"]
# Enable general responding to channel openers for example
# this can be combined with a callsign blacklist for example
self.respond_to_call = True
self.modem_frequency_offset = 0
self.dxcallsign = b"ZZ9YY-0"
self.dxcallsign_crc = b''
self.dxgrid = b''
# length of signalling frame
self.length_sig0_frame = 14
self.length_sig1_frame = 14
# duration of frames
self.duration_datac4 = 5.17
self.duration_datac13 = 2.0
self.duration_datac1 = 4.18
self.duration_datac3 = 3.19
self.duration_sig0_frame = self.duration_datac13
self.duration_sig1_frame = self.duration_datac13
self.longest_duration = self.duration_datac4
# hold session id
self.session_id = bytes(1)
# ------- ARQ SESSION
self.arq_file_transfer = False
self.IS_ARQ_SESSION_MASTER = False
self.arq_session_last_received = 0
self.arq_session_timeout = 30
self.session_connect_max_retries = 10
self.arq_compression_factor = 0
self.transmission_uuid = ""
self.burst_last_received = 0.0 # time of last "live sign" of a burst
self.data_channel_last_received = 0.0 # time of last "live sign" of a frame
# Flag to indicate if we received an ACK frame for a burst
self.burst_ack = False
# Flag to indicate if we received an ACK frame for a data frame
self.data_frame_ack_received = False
# Flag to indicate if we received a request for repeater frames
self.rpt_request_received = False
self.rpt_request_buffer = [] # requested frames, saved in a list
self.burst_rpt_counter = 0
# 3 bytes for the BOF Beginning of File indicator in a data frame
self.data_frame_bof = b"BOF"
# 3 bytes for the EOF End of File indicator in a data frame
self.data_frame_eof = b"EOF"
self.n_retries_per_burst = 0
self.max_n_frames_per_burst = 1
# Flag to indicate if we received a low bandwidth mode channel opener
self.received_LOW_BANDWIDTH_MODE = False
self.data_channel_max_retries = 15
# event for checking arq_state_event
self.arq_state_event = threading.Event()
# -------------- AVAILABLE MODES START-----------
# IMPORTANT: LISTS MUST BE OF EQUAL LENGTH
# --------------------- LOW BANDWIDTH
# List of codec2 modes to use in "low bandwidth" mode.
self.mode_list_low_bw = [
FREEDV_MODE.datac4.value,
]
# List for minimum SNR operating level for the corresponding mode in self.mode_list
self.snr_list_low_bw = [-100]
# List for time to wait for corresponding mode in seconds
self.time_list_low_bw = [self.duration_datac4]
# --------------------- HIGH BANDWIDTH
# List of codec2 modes to use in "high bandwidth" mode.
self.mode_list_high_bw = [
FREEDV_MODE.datac4.value,
FREEDV_MODE.datac3.value,
FREEDV_MODE.datac1.value,
]
# List for minimum SNR operating level for the corresponding mode in self.mode_list
self.snr_list_high_bw = [-100, 0, 3]
# List for time to wait for corresponding mode in seconds
# test with 6,7 --> caused sometimes a frame timeout if ack frame takes longer
# TODO Need to check why ACK frames needs more time
# TODO Adjust these times
self.time_list_high_bw = [self.duration_datac4, self.duration_datac3, self.duration_datac1]
# -------------- AVAILABLE MODES END-----------
# Mode list for selecting between low bandwidth ( 500Hz ) and modes with higher bandwidth
# but ability to fall back to low bandwidth modes if needed.
if self.low_bandwidth_mode:
# List of codec2 modes to use in "low bandwidth" mode.
self.mode_list = self.mode_list_low_bw
# list of times to wait for corresponding mode in seconds
self.time_list = self.time_list_low_bw
else:
# List of codec2 modes to use in "high bandwidth" mode.
self.mode_list = self.mode_list_high_bw
# list of times to wait for corresponding mode in seconds
self.time_list = self.time_list_high_bw
self.speed_level = len(self.mode_list) - 1 # speed level for selecting mode
self.states.set("arq_speed_level", self.speed_level)
self.is_IRS = False
self.burst_nack = False
self.burst_nack_counter = 0
self.frame_nack_counter = 0
self.frame_received_counter = 0
# TIMEOUTS
self.transmission_timeout = 180 # transmission timeout in seconds
self.channel_busy_timeout = 3 # time how long we want to wait until channel busy state overrides
# START THE THREAD FOR THE TIMEOUT WATCHDOG
watchdog_thread = threading.Thread(
target=self.watchdog, name="watchdog", daemon=True
)
watchdog_thread.start()
arq_session_thread = threading.Thread(
target=self.heartbeat, name="watchdog", daemon=True
)
arq_session_thread.start()
def send_ident_frame(self, transmit) -> None:
"""Build and send IDENT frame """
ident_frame = bytearray(self.length_sig1_frame)
ident_frame[:1] = bytes([FR_TYPE.IDENT.value])
ident_frame[1:self.length_sig1_frame] = self.mycallsign
# Transmit frame
if transmit:
self.enqueue_frame_for_tx([ident_frame], c2_mode=FREEDV_MODE.sig0.value)
else:
return ident_frame
def send_disconnect_frame(self) -> None:
"""Build and send a disconnect frame"""
disconnection_frame = bytearray(self.length_sig1_frame)
disconnection_frame[:1] = bytes([FR_TYPE.ARQ_SESSION_CLOSE.value])
disconnection_frame[1:2] = self.session_id
disconnection_frame[2:5] = self.dxcallsign_crc
# wait if we have a channel busy condition
if self.states.channel_busy:
self.channel_busy_handler()
self.enqueue_frame_for_tx([disconnection_frame], c2_mode=FREEDV_MODE.sig0.value, copies=3, repeat_delay=0)
def check_if_mode_fits_to_busy_slot(self):
"""
Check if actual mode is fitting into given busy state
Returns:
"""
mode_name = FREEDV_MODE(self.mode_list[self.speed_level]).name
mode_slots = FREEDV_MODE_USED_SLOTS[mode_name].value
if mode_slots in [self.states.channel_busy_slot]:
self.log.warning(
"[Modem] busy slot detection",
slots=self.states.channel_busy_slot,
mode_slots=mode_slots,
)
return False
return True
def arq_calculate_speed_level(self, snr):
current_speed_level = self.speed_level
self.frame_received_counter += 1
# try increasing speed level only if we had two successful decodes
if self.frame_received_counter >= 2:
self.frame_received_counter = 0
# make sure new speed level isn't higher than available modes
new_speed_level = min(self.speed_level + 1, len(self.mode_list) - 1)
# check if actual snr is higher than minimum snr for next mode
if snr >= self.snr_list[new_speed_level]:
self.speed_level = new_speed_level
else:
self.log.info("[Modem] ARQ | increasing speed level not possible because of SNR limit",
given_snr=snr,
needed_snr=self.snr_list[new_speed_level]
)
# calculate if speed level fits to busy condition
if not self.check_if_mode_fits_to_busy_slot():
self.speed_level = current_speed_level
self.states.set("arq_speed_level", self.speed_level)
# Update modes we are listening to
self.set_listening_modes(False, True, self.mode_list[self.speed_level])
self.log.debug(
"[Modem] calculated speed level",
speed_level=self.speed_level,
given_snr=snr,
min_snr=self.snr_list[self.speed_level],
)
# for i in range(0, 6, 2):
# if not missing_area[i: i + 2].endswith(b"\x00\x00"):
# self.rpt_request_buffer.insert(0, missing_area[i: i + 2])
##########################################################################################################
# ARQ DATA CHANNEL HANDLER
##########################################################################################################
def stop_transmission(self) -> None:
"""
Force a stop of the running transmission
"""
self.log.warning("[Modem] Stopping transmission!")
self.event_manager.send_custom_event(
freedata="modem-message",
arq="transmission",
status="stopped",
mycallsign=str(self.mycallsign, 'UTF-8'),
dxcallsign=str(self.dxcallsign, 'UTF-8')
)
stop_frame = bytearray(self.length_sig0_frame)
stop_frame[:1] = bytes([FR_TYPE.ARQ_STOP.value])
stop_frame[1:4] = self.dxcallsign_crc
stop_frame[4:7] = self.mycallsign_crc
# TODO Not sure if we really need the session id when disconnecting
# stop_frame[1:2] = self.session_id
stop_frame[7:13] = helpers.callsign_to_bytes(self.mycallsign)
self.enqueue_frame_for_tx([stop_frame], c2_mode=FREEDV_MODE.sig1.value, copies=3, repeat_delay=0)
self.arq_cleanup()
def received_stop_transmission(
self, deconstructed_frame: list
) -> None: # pylint: disable=unused-argument
"""
Received a transmission stop
"""
self.log.warning("[Modem] Stopping transmission!")
self.states.set("is_modem_busy", False)
self.states.set("is_arq_state", False)
self.event_manager.send_custom_event(
freedata="modem-message",
arq="transmission",
status="stopped",
mycallsign=str(self.mycallsign, 'UTF-8'),
dxcallsign=str(self.dxcallsign, 'UTF-8'),
uuid=self.transmission_uuid
)
self.arq_cleanup()
def channel_busy_handler(self):
"""
function for handling the channel busy situation
Args:
Returns:
"""
self.log.warning("[Modem] Channel busy, waiting until free...")
self.event_manager.send_custom_event(
freedata="modem-message",
channel="busy",
status="waiting",
)
# wait while timeout not reached and our busy state is busy
channel_busy_timeout = time.time() + self.channel_busy_timeout
while self.states.channel_busy and time.time() < channel_busy_timeout and not self.check_if_mode_fits_to_busy_slot():
threading.Event().wait(0.01)
# ------------ CALCULATE TRANSFER RATES
def reset_statistics(self) -> None:
"""
Reset statistics
"""
# reset ARQ statistics
self.states.set("bytes_per_minute_burst", 0)
self.states.set("arq_total_bytes", 0)
self.states.set("self.states.arq_seconds_until_finish", 0)
self.states.set("arq_bits_per_second", 0)
self.states.set("bytes_per_minute", 0)
self.states.set("arq_transmission_percent", 0)
self.states.set("arq_compression_factor", 0)
# ----------------------CLEANUP AND RESET FUNCTIONS
def arq_cleanup(self) -> None:
"""
Cleanup function which clears all ARQ states
"""
# TODO
# We need to check if we are in a ARQ session
# Then we cant delete the session_id for now
self.states.delete_arq_instance_by_id(self.session_id)
if TESTMODE:
self.log.debug("[Modem] TESTMODE: arq_cleanup: Not performing cleanup.")
return
self.log.debug("[Modem] arq_cleanup")
# wait a second for smoother arq behaviour
helpers.wait(1.0)
self.rx_frame_bof_received = False
self.rx_frame_eof_received = False
self.burst_ack = False
self.rpt_request_received = False
self.burst_rpt_counter = 0
self.data_frame_ack_received = False
self.arq_rx_burst_buffer = []
self.arq_rx_frame_buffer = b""
self.burst_ack_snr = 0
self.arq_burst_last_payload = 0
self.rx_n_frame_of_burst = 0
self.rx_n_frames_per_burst = 0
# reset modem receiving state to reduce cpu load
modem.demodulator.RECEIVE_SIG0 = True
modem.demodulator.RECEIVE_SIG1 = False
modem.demodulator.RECEIVE_DATAC1 = False
modem.demodulator.RECEIVE_DATAC3 = False
modem.demodulator.RECEIVE_DATAC4 = False
# modem.demodulator.RECEIVE_FSK_LDPC_0 = False
modem.demodulator.RECEIVE_FSK_LDPC_1 = False
self.is_IRS = False
self.burst_nack = False
self.burst_nack_counter = 0
self.frame_nack_counter = 0
self.frame_received_counter = 0
self.speed_level = len(self.mode_list) - 1
self.states.set("arq_speed_level", self.speed_level)
# low bandwidth mode indicator
self.received_LOW_BANDWIDTH_MODE = False
# reset retry counter for rx channel / burst
self.n_retries_per_burst = 0
# reset max retries possibly overriden by api
self.session_connect_max_retries = 10
self.data_channel_max_retries = 10
self.states.set("arq_session_state", "disconnected")
self.states.arq_speed_list = []
self.states.set("is_arq_state", False)
self.arq_state_event = threading.Event()
self.arq_file_transfer = False
self.beacon_paused = False
# reset beacon interval timer for not directly starting beacon after ARQ
self.beacon_interval_timer = time.time() + self.beacon_interval
def arq_reset_ack(self, state: bool) -> None:
"""
Funktion for resetting acknowledge states
Args:
state:bool:
"""
self.burst_ack = state
self.rpt_request_received = state
self.data_frame_ack_received = state
def set_listening_modes(self, enable_sig0: bool, enable_sig1: bool, mode: int) -> None:
# sourcery skip: extract-duplicate-method
"""
Function for setting the data modes we are listening to for saving cpu power
Args:
enable_sig0:int: Enable/Disable signalling mode 0
enable_sig1:int: Enable/Disable signalling mode 1
mode:int: Codec2 mode to listen for
"""
# set modes we want to listen to
modem.RECEIVE_SIG0 = enable_sig0
modem.RECEIVE_SIG1 = enable_sig1
if mode == codec2.FREEDV_MODE.datac1.value:
modem.RECEIVE_DATAC1 = True
modem.RECEIVE_DATAC3 = False
modem.RECEIVE_DATAC4 = False
modem.RECEIVE_FSK_LDPC_1 = False
self.log.debug("[Modem] Changing listening data mode", mode="datac1")
elif mode == codec2.FREEDV_MODE.datac3.value:
modem.RECEIVE_DATAC1 = False
modem.RECEIVE_DATAC3 = True
modem.RECEIVE_DATAC4 = False
modem.RECEIVE_FSK_LDPC_1 = False
self.log.debug("[Modem] Changing listening data mode", mode="datac3")
elif mode == codec2.FREEDV_MODE.datac4.value:
modem.RECEIVE_DATAC1 = False
modem.RECEIVE_DATAC3 = False
modem.RECEIVE_DATAC4 = True
modem.RECEIVE_FSK_LDPC_1 = False
self.log.debug("[Modem] Changing listening data mode", mode="datac4")
elif mode == codec2.FREEDV_MODE.fsk_ldpc_1.value:
modem.RECEIVE_DATAC1 = False
modem.RECEIVE_DATAC3 = False
modem.RECEIVE_DATAC4 = False
modem.RECEIVE_FSK_LDPC_1 = True
self.log.debug("[Modem] Changing listening data mode", mode="fsk_ldpc_1")
else:
modem.RECEIVE_DATAC1 = True
modem.RECEIVE_DATAC3 = True
modem.RECEIVE_DATAC4 = True
modem.RECEIVE_FSK_LDPC_1 = True
self.log.debug(
"[Modem] Changing listening data mode", mode="datac1/datac3/fsk_ldpc"
)
# ------------------------- WATCHDOG FUNCTIONS FOR TIMER
def watchdog(self) -> None:
"""Author: DJ2LS
Watchdog master function. From here, "pet" the watchdogs
"""
while True:
threading.Event().wait(0.1)
self.data_channel_keep_alive_watchdog()
self.burst_watchdog()
self.arq_session_keep_alive_watchdog()
def burst_watchdog(self) -> None:
"""
Watchdog which checks if we are running into a connection timeout
DATA BURST
"""
# IRS SIDE
# TODO We need to redesign this part for cleaner state handling
# Return if not ARQ STATE and not ARQ SESSION STATE as they are different use cases
if (
not self.states.is_arq_state
and self.states.arq_session_state != "connected"
or not self.is_IRS
):
return
# get modem error state
modem_error_state = modem.get_modem_error_state()
# We want to reach this state only if connected ( == return above not called )
if self.rx_n_frames_per_burst > 1:
# uses case for IRS: reduce time for waiting by counting "None" in burst buffer
frames_left = self.arq_rx_burst_buffer.count(None)
elif self.rx_n_frame_of_burst == 0 and self.rx_n_frames_per_burst == 0:
# use case for IRS: We didn't receive a burst yet, because the first one got lost
# in this case we don't have any information about the expected burst length
# we must assume, we are getting a burst with max_n_frames_per_burst
frames_left = self.max_n_frames_per_burst
else:
frames_left = 1
# make sure we don't have a 0 here for avoiding too short timeouts
if frames_left == 0:
frames_left = 1
# timeout is reached, if we didnt receive data, while we waited
# for the corresponding data frame + the transmitted signalling frame of ack/nack
# + a small offset of about 1 second
timeout = \
(
self.burst_last_received
+ (self.time_list[self.speed_level] * frames_left)
+ self.duration_sig0_frame
+ self.channel_busy_timeout
+ 1
)
# override calculation
# if we reached 2/3 of the waiting time and didnt received a signal
# then send NACK earlier
time_left = timeout - time.time()
waiting_time = (self.time_list[
self.speed_level] * frames_left) + self.duration_sig0_frame + self.channel_busy_timeout + 1
timeout_percent = 100 - (time_left / waiting_time * 100)
# timeout_percent = 0
if timeout_percent >= 75 and not self.states.is_codec2_traffic and not self.states.isTransmitting():
override = True
else:
override = False
# TODO Enable this for development
print(
f"timeout expected in:{round(timeout - time.time())} | timeout percent: {timeout_percent} | frames left: {frames_left} of {self.rx_n_frames_per_burst} | speed level: {self.speed_level}")
# if timeout is expired, but we are receivingt codec2 data,
# better wait some more time because data might be important for us
# reason for this situation can be delays on IRS and ISS, maybe because both had a busy channel condition.
# Nevertheless, we need to keep timeouts short for efficiency
if timeout <= time.time() or modem_error_state and not self.states.is_codec2_traffic and not self.states.isTransmitting() or override:
self.log.warning(
"[Modem] Burst decoding error or timeout",
attempt=self.n_retries_per_burst,
max_attempts=self.rx_n_max_retries_per_burst,
speed_level=self.speed_level,
modem_error_state=modem_error_state
)
print(
f"frames_per_burst {self.rx_n_frame_of_burst} / {self.rx_n_frames_per_burst}, Repeats: {self.burst_rpt_counter} Nones: {self.arq_rx_burst_buffer.count(None)}")
# check if we have N frames per burst > 1
if self.rx_n_frames_per_burst > 1 and self.burst_rpt_counter < 3 and self.arq_rx_burst_buffer.count(
None) > 0:
# reset self.burst_last_received
self.burst_last_received = time.time() + self.time_list[self.speed_level] * frames_left
self.burst_rpt_counter += 1
self.send_retransmit_request_frame()
else:
# reset self.burst_last_received counter
self.burst_last_received = time.time()
# reduce speed level if nack counter increased
self.frame_received_counter = 0
self.burst_nack_counter += 1
if self.burst_nack_counter >= 2:
self.burst_nack_counter = 0
self.speed_level = max(self.speed_level - 1, 0)
self.states.set("arq_speed_level", self.speed_level)
# TODO Create better mechanisms for handling n frames per burst for bad channels
# reduce frames per burst
if self.burst_rpt_counter >= 2:
tx_n_frames_per_burst = max(self.rx_n_frames_per_burst - 1, 1)
else:
tx_n_frames_per_burst = self.rx_n_frames_per_burst
# Update modes we are listening to
self.set_listening_modes(True, True, self.mode_list[self.speed_level])
# TODO Does SNR make sense for NACK if we dont have an actual SNR information?
self.send_burst_nack_frame_watchdog(tx_n_frames_per_burst)
# Update data_channel timestamp
# TODO Disabled this one for testing.
# self.data_channel_last_received = time.time()
self.n_retries_per_burst += 1
else:
# debugging output
# print((self.data_channel_last_received + self.time_list[self.speed_level])-time.time())
pass
if self.n_retries_per_burst >= self.rx_n_max_retries_per_burst:
self.stop_transmission()
def data_channel_keep_alive_watchdog(self) -> None:
"""
watchdog which checks if we are running into a connection timeout
DATA CHANNEL
"""
# and not static.ARQ_SEND_KEEP_ALIVE:
if self.states.is_arq_state and self.states.is_modem_busy:
threading.Event().wait(0.01)
if (
self.data_channel_last_received + self.transmission_timeout
> time.time()
):
timeleft = int((self.data_channel_last_received + self.transmission_timeout) - time.time())
self.states.set("arq_seconds_until_timeout", timeleft)
if timeleft % 10 == 0:
self.log.debug("Time left until channel timeout", seconds=timeleft)
# threading.Event().wait(5)
# print(self.data_channel_last_received + self.transmission_timeout - time.time())
# pass
else:
# Clear the timeout timestamp
self.data_channel_last_received = 0
self.log.info(
"[Modem] DATA ["
+ str(self.mycallsign, "UTF-8")
+ "]<<T>>["
+ str(self.dxcallsign, "UTF-8")
+ "]"
)
self.event_manager.send_custom_event(
freedata="modem-message",
arq="transmission",
status="failed",
uuid=self.transmission_uuid,
mycallsign=str(self.mycallsign, 'UTF-8'),
dxcallsign=str(self.dxcallsign, 'UTF-8'),
irs=helpers.bool_to_string(self.is_IRS)
)
self.arq_cleanup()
def arq_session_keep_alive_watchdog(self) -> None:
"""
watchdog which checks if we are running into a connection timeout
ARQ SESSION
"""
if (
self.states.is_arq_session
and self.states.is_modem_busy
and not self.arq_file_transfer
):
if self.arq_session_last_received + self.arq_session_timeout > time.time():
threading.Event().wait(0.01)
else:
self.log.info(
"[Modem] SESSION ["
+ str(self.mycallsign, "UTF-8")
+ "]<<T>>["
+ str(self.dxcallsign, "UTF-8")
+ "]"
)
self.event_manager.send_custom_event(
freedata="modem-message",
arq="session",
status="failed",
reason="timeout",
mycallsign=str(self.mycallsign, 'UTF-8'),
dxcallsign=str(self.dxcallsign, 'UTF-8'),
)
self.close_session()
def heartbeat(self) -> None:
"""
Heartbeat thread which auto pauses and resumes the heartbeat signal when in an arq session
"""
while True:
threading.Event().wait(0.01)
# additional check for smoother stopping if heartbeat transmission
while not self.arq_file_transfer:
threading.Event().wait(0.01)
if (
self.states.is_arq_session
and self.IS_ARQ_SESSION_MASTER
and self.states.arq_session_state == "connected"
# and not self.arq_file_transfer
):
threading.Event().wait(1)
self.transmit_session_heartbeat()
threading.Event().wait(2)

853
modem/data cemetery/deprecated_protocol_arq_session_irs.py
View file

@ -1,853 +0,0 @@
import base64
import time
import uuid
import lzma
import helpers
import numpy as np
from codec2 import FREEDV_MODE
from queues import RX_BUFFER
from modem_frametypes import FRAME_TYPE as FR_TYPE
from deprecated_protocol_arq_session import ARQ
class IRS(ARQ):
def __init__(self, config, event_queue, states):
super().__init__(config, event_queue, states)
self.arq_rx_burst_buffer = []
self.arq_rx_frame_buffer = b""
self.rx_n_max_retries_per_burst = 40
self.rx_n_frame_of_burst = 0
self.rx_n_frames_per_burst = 0
self.rx_frame_bof_received = False
self.rx_frame_eof_received = False
self.rx_start_of_transmission = 0 # time of transmission start
# minimum payload for arq burst
# import for avoiding byteorder bug and buffer search area
self.arq_burst_header_size = 3
self.arq_burst_minimum_payload = 56 - self.arq_burst_header_size
self.arq_burst_maximum_payload = 510 - self.arq_burst_header_size
# save last used payload for optimising buffer search area
self.arq_burst_last_payload = self.arq_burst_maximum_payload
def arq_process_received_data_frame(self, data_frame, snr, signed):
"""
"""
# transmittion duration
signed = "True" if signed else "False"
duration = time.time() - self.rx_start_of_transmission
self.calculate_transfer_rate_rx(
self.rx_start_of_transmission, len(self.arq_rx_frame_buffer), snr
)
self.log.info("[Modem] ARQ | RX | DATA FRAME SUCCESSFULLY RECEIVED", nacks=self.frame_nack_counter,
bytesperminute=self.states.arq_bytes_per_minute, total_bytes=self.states.arq_total_bytes,
duration=duration, hmac_signed=signed)
# Decompress the data frame
data_frame_decompressed = lzma.decompress(data_frame)
self.arq_compression_factor = len(data_frame_decompressed) / len(
data_frame
)
data_frame = data_frame_decompressed
self.transmission_uuid = str(uuid.uuid4())
timestamp = int(time.time())
# Re-code data_frame in base64, UTF-8 for JSON UI communication.
base64_data = base64.b64encode(data_frame).decode("UTF-8")
# check if RX_BUFFER isn't full
if not RX_BUFFER.full():
# make sure we have always the correct buffer size
RX_BUFFER.maxsize = int(self.arq_rx_buffer_size)
else:
# if full, free space by getting an item
self.log.info(
"[Modem] ARQ | RX | RX_BUFFER FULL - dropping old data",
buffer_size=RX_BUFFER.qsize(),
maxsize=int(self.arq_rx_buffer_size)
)
RX_BUFFER.get()
# add item to RX_BUFFER
self.log.info(
"[Modem] ARQ | RX | saving data to rx buffer",
buffer_size=RX_BUFFER.qsize() + 1,
maxsize=RX_BUFFER.maxsize
)
try:
# RX_BUFFER[0] = transmission uuid
# RX_BUFFER[1] = timestamp
# RX_BUFFER[2] = dxcallsign
# RX_BUFFER[3] = dxgrid
# RX_BUFFER[4] = data
# RX_BUFFER[5] = hmac signed
# RX_BUFFER[6] = compression factor
# RX_BUFFER[7] = bytes per minute
# RX_BUFFER[8] = duration
# RX_BUFFER[9] = self.frame_nack_counter
# RX_BUFFER[10] = speed list stats
RX_BUFFER.put(
[
self.transmission_uuid,
timestamp,
self.dxcallsign,
self.dxgrid,
base64_data,
signed,
self.arq_compression_factor,
self.states.arq_bytes_per_minute,
duration,
self.frame_nack_counter,
self.states.arq_speed_list
]
)
except Exception as e:
# File "/usr/lib/python3.7/queue.py", line 133, in put
# if self.maxsize > 0
# TypeError: '>' not supported between instances of 'str' and 'int'
#
# Occurs on Raspberry Pi and Python 3.7
self.log.error(
"[Modem] ARQ | RX | error occurred when saving data!",
e=e,
uuid=self.transmission_uuid,
timestamp=timestamp,
dxcall=self.dxcallsign,
dxgrid=self.dxgrid,
data=base64_data
)
self.event_manager.send_custom_event(
freedata="modem-message",
arq="transmission",
status="received",
uuid=self.transmission_uuid,
percent=self.states.arq_transmission_percent,
bytesperminute=self.states.arq_bytes_per_minute,
compression=self.arq_compression_factor,
timestamp=timestamp,
finished=0,
mycallsign=str(self.mycallsign, "UTF-8"),
dxcallsign=str(self.dxcallsign, "UTF-8"),
dxgrid=str(self.dxgrid, "UTF-8"),
data=base64_data,
irs=helpers.bool_to_string(self.is_IRS),
hmac_signed=signed,
duration=duration,
nacks=self.frame_nack_counter,
speed_list=self.states.arq_speed_list
)
if self.enable_stats:
duration = time.time() - self.rx_start_of_transmission
self.stats.push(frame_nack_counter=self.frame_nack_counter, status="received", duration=duration)
self.log.info(
"[Modem] ARQ | RX | SENDING DATA FRAME ACK")
self.send_data_ack_frame(snr)
# Update statistics AFTER the frame ACK is sent
self.calculate_transfer_rate_rx(
self.rx_start_of_transmission, len(self.arq_rx_frame_buffer), snr
)
self.log.info(
"[Modem] | RX | DATACHANNEL ["
+ str(self.mycallsign, "UTF-8")
+ "]<< >>["
+ str(self.dxcallsign, "UTF-8")
+ "]",
snr=snr,
)
def arq_received_data_channel_opener(self, data_in: bytes, snr):
"""
Received request to open data channel frame
Args:
data_in:bytes:
"""
# We've arrived here from process_data which already checked that the frame
# is intended for this station.
# stop processing if we don't want to respond to a call when not in a arq session
if not self.respond_to_call and not self.states.is_arq_session:
return False
# stop processing if not in arq session, but modem state is busy and we have a different session id
# use-case we get a connection request while connecting to another station
if not self.states.is_arq_session and self.states.is_modem_busy and data_in[13:14] != self.session_id:
return False
self.arq_file_transfer = True
# check if callsign ssid override
_, self.mycallsign = helpers.check_callsign(self.mycallsign, data_in[1:4], self.ssid_list)
# ignore channel opener if already in ARQ STATE
# use case: Station A is connecting to Station B while
# Station B already tries connecting to Station A.
# For avoiding ignoring repeated connect request in case of packet loss
# we are only ignoring packets in case we are ISS
if self.arq_state_event.is_set() and not self.is_IRS:
return False
self.is_IRS = True
self.dxcallsign_crc = bytes(data_in[4:7])
self.dxcallsign = helpers.bytes_to_callsign(bytes(data_in[7:13]))
self.states.set("dxcallsign", self.dxcallsign)
self.event_manager.send_custom_event(
freedata="modem-message",
arq="transmission",
status="opening",
mycallsign=str(self.mycallsign, 'UTF-8'),
dxcallsign=str(self.dxcallsign, 'UTF-8'),
irs=helpers.bool_to_string(self.is_IRS)
)
frametype = int.from_bytes(bytes(data_in[:1]), "big")
# check if we received low bandwidth mode
# possible channel constellations
# ISS(w) <-> IRS(w)
# ISS(w) <-> IRS(n)
# ISS(n) <-> IRS(w)
# ISS(n) <-> IRS(n)
if frametype == FR_TYPE.ARQ_DC_OPEN_W.value and not self.low_bandwidth_mode:
# ISS(w) <-> IRS(w)
constellation = "ISS(w) <-> IRS(w)"
self.received_LOW_BANDWIDTH_MODE = False
self.mode_list = self.mode_list_high_bw
self.time_list = self.time_list_high_bw
self.snr_list = self.snr_list_high_bw
elif frametype == FR_TYPE.ARQ_DC_OPEN_W.value:
# ISS(w) <-> IRS(n)
constellation = "ISS(w) <-> IRS(n)"
self.received_LOW_BANDWIDTH_MODE = False
self.mode_list = self.mode_list_low_bw
self.time_list = self.time_list_low_bw
self.snr_list = self.snr_list_low_bw
elif frametype == FR_TYPE.ARQ_DC_OPEN_N.value and not self.low_bandwidth_mode:
# ISS(n) <-> IRS(w)
constellation = "ISS(n) <-> IRS(w)"
self.received_LOW_BANDWIDTH_MODE = True
self.mode_list = self.mode_list_low_bw
self.time_list = self.time_list_low_bw
self.snr_list = self.snr_list_low_bw
elif frametype == FR_TYPE.ARQ_DC_OPEN_N.value:
# ISS(n) <-> IRS(n)
constellation = "ISS(n) <-> IRS(n)"
self.received_LOW_BANDWIDTH_MODE = True
self.mode_list = self.mode_list_low_bw
self.time_list = self.time_list_low_bw
self.snr_list = self.snr_list_low_bw
else:
constellation = "not matched"
self.received_LOW_BANDWIDTH_MODE = True
self.mode_list = self.mode_list_low_bw
self.time_list = self.time_list_low_bw
self.snr_list = self.snr_list_low_bw
# get mode which fits to given SNR
# initially set speed_level 0 in case of bad SNR and no matching mode
self.speed_level = 0
# calculate initial speed level in correlation to latest known SNR
for i in range(len(self.mode_list)):
if snr >= self.snr_list[i]:
self.speed_level = i
# check if speed level fits to busy condition
if not self.check_if_mode_fits_to_busy_slot():
self.speed_level = 0
# Update modes we are listening to
self.set_listening_modes(True, True, self.mode_list[self.speed_level])
self.dxgrid = b'------'
helpers.add_to_heard_stations(
self.dxcallsign,
self.dxgrid,
"DATA",
snr,
self.modem_frequency_offset,
self.states.radio_frequency,
self.states.heard_stations
)
self.session_id = data_in[13:14]
# check again if callsign ssid override
_, self.mycallsign = helpers.check_callsign(self.mycallsign, data_in[1:4], self.ssid_list)
self.log.info(
"[Modem] ARQ | DATA | RX | ["
+ str(self.mycallsign, "UTF-8")
+ "]>> <<["
+ str(self.dxcallsign, "UTF-8")
+ "]",
channel_constellation=constellation,
)
# Reset data_channel/burst timestamps
# TIMING TEST
self.data_channel_last_received = int(time.time())
self.burst_last_received = int(time.time() + 10) # we might need some more time so lets increase this
# Set ARQ State AFTER resetting timeouts
# this avoids timeouts starting too early
self.states.set("is_arq_state", True)
self.states.set("is_modem_busy", True)
self.reset_statistics()
# Select the frame type based on the current Modem mode
if self.low_bandwidth_mode or self.received_LOW_BANDWIDTH_MODE:
frametype = bytes([FR_TYPE.ARQ_DC_OPEN_ACK_N.value])
self.log.debug("[Modem] Responding with low bandwidth mode")
else:
frametype = bytes([FR_TYPE.ARQ_DC_OPEN_ACK_W.value])
self.log.debug("[Modem] Responding with high bandwidth mode")
connection_ack_frame = self.frame_factory.build_arq_connect_ack(
session_id=self.session_id,
speed_level=self.speed_level,
arq_protocol_version=self.arq_protocol_version
)
self.enqueue_frame_for_tx([connection_ack_frame], c2_mode=FREEDV_MODE.sig0.value, copies=1, repeat_delay=0)
self.event_manager.send_custom_event(
freedata="modem-message",
arq="transmission",
status="opened",
mycallsign=str(self.mycallsign, 'UTF-8'),
dxcallsign=str(self.dxcallsign, 'UTF-8'),
irs=helpers.bool_to_string(self.is_IRS)
)
self.log.info(
"[Modem] ARQ | DATA | RX | ["
+ str(self.mycallsign, "UTF-8")
+ "]>>|<<["
+ str(self.dxcallsign, "UTF-8")
+ "]",
bandwidth="wide",
snr=snr,
)
# set start of transmission for our statistics
self.rx_start_of_transmission = time.time()
# TIMING TEST
# Reset data_channel/burst timestamps once again for avoiding running into timeout
# and therefore sending a NACK
self.data_channel_last_received = int(time.time())
self.burst_last_received = int(time.time() + 10) # we might need some more time so lets increase this
def calculate_transfer_rate_rx(
self, rx_start_of_transmission: float, receivedbytes: int, snr
) -> list:
"""
Calculate transfer rate for received data
Args:
rx_start_of_transmission:float:
receivedbytes:int:
Returns: List of:
bits_per_second: float,
bytes_per_minute: float,
transmission_percent: float
"""
try:
if self.states.arq_total_bytes == 0:
self.states.set("arq_total_bytes", 1)
arq_transmission_percent = min(
int(
(
receivedbytes
* self.arq_compression_factor
/ self.states.arq_total_bytes
)
* 100
),
100,
)
transmissiontime = time.time() - self.rx_start_of_transmission
if receivedbytes > 0:
arq_bits_per_second = int((receivedbytes * 8) / transmissiontime)
bytes_per_minute = int(
receivedbytes / (transmissiontime / 60)
)
arq_seconds_until_finish = int(((self.states.arq_total_bytes - receivedbytes) / (
bytes_per_minute * self.arq_compression_factor)) * 60) - 20 # offset because of frame ack/nack
speed_chart = {"snr": snr, "bpm": bytes_per_minute, "timestamp": int(time.time())}
# check if data already in list
if speed_chart not in self.states.arq_speed_list:
self.states.arq_speed_list.append(speed_chart)
else:
arq_bits_per_second = 0
bytes_per_minute = 0
arq_seconds_until_finish = 0
except Exception as err:
self.log.error(f"[Modem] calculate_transfer_rate_rx: Exception: {err}")
arq_transmission_percent = 0.0
arq_bits_per_second = 0
bytes_per_minute = 0
self.states.set("arq_bits_per_second", arq_bits_per_second)
self.states.set("bytes_per_minute", bytes_per_minute)
self.states.set("arq_transmission_percent", arq_transmission_percent)
self.states.set("arq_compression_factor", self.arq_compression_factor)
return [
arq_bits_per_second,
bytes_per_minute,
arq_transmission_percent,
]
def send_burst_nack_frame(self, snr: bytes) -> None:
"""Build and send NACK frame for received DATA frame"""
# nack_frame = bytearray(self.length_sig1_frame)
# nack_frame[:1] = bytes([FR_TYPE.FR_NACK.value])
# nack_frame[1:2] = self.session_id
# nack_frame[2:3] = helpers.snr_to_bytes(snr)
# nack_frame[3:4] = bytes([int(self.speed_level)])
# nack_frame[4:8] = len(self.arq_rx_frame_buffer).to_bytes(4, byteorder="big")
nack_frame = self.frame_factory.build_arq_frame_nack(session_id=self.session_id,
snr=snr,
speed_level=self.speed_level,
len_arq_rx_frame_buffer=len(self.arq_rx_frame_buffer)
)
# TRANSMIT NACK FRAME FOR BURST
# TODO Do we have to send ident frame?
# self.enqueue_frame_for_tx([ack_frame, self.send_ident_frame(False)], c2_mode=FREEDV_MODE.sig1.value, copies=3, repeat_delay=0)
# wait if we have a channel busy condition
if self.states.channel_busy:
self.channel_busy_handler()
self.enqueue_frame_for_tx([nack_frame], c2_mode=FREEDV_MODE.sig1.value, copies=3, repeat_delay=0)
# reset burst timeout in case we had to wait too long
self.burst_last_received = time.time()
def send_burst_nack_frame_watchdog(self, tx_n_frames_per_burst) -> None:
"""Build and send NACK frame for watchdog timeout"""
# increment nack counter for transmission stats
self.frame_nack_counter += 1
# we need to clear our rx burst buffer
self.arq_rx_burst_buffer = []
# Create and send ACK frame
self.log.info("[Modem] ARQ | RX | SENDING NACK")
# nack_frame = bytearray(self.length_sig1_frame)
# nack_frame[:1] = bytes([FR_TYPE.BURST_NACK.value])
# nack_frame[1:2] = self.session_id
# nack_frame[2:3] = helpers.snr_to_bytes(0)
# nack_frame[3:4] = bytes([int(self.speed_level)])
# nack_frame[4:5] = bytes([int(tx_n_frames_per_burst)])
# nack_frame[5:9] = len(self.arq_rx_frame_buffer).to_bytes(4, byteorder="big")
nack_frame = self.frame_factory.build_arq_burst_nack(session_id=self.session_id,
snr=0,
speed_level=self.speed_level,
len_arq_rx_frame_buffer=len(self.arq_rx_frame_buffer),
n_frames_per_burst=bytes([int(tx_n_frames_per_burst)])
)
# wait if we have a channel busy condition
if self.states.channel_busy:
self.channel_busy_handler()
# TRANSMIT NACK FRAME FOR BURST
self.enqueue_frame_for_tx([nack_frame], c2_mode=FREEDV_MODE.sig1.value, copies=1, repeat_delay=0)
# reset frame counter for not increasing speed level
self.frame_received_counter = 0
def arq_data_received(
self, deconstructed_frame: list, bytes_per_frame: int, snr: float, freedv
) -> None:
"""
Args:
data_in:bytes:
bytes_per_frame:int:
snr:float:
freedv:
Returns:
"""
# We've arrived here from process_data which already checked that the frame
# is intended for this station.
data_in = deconstructed_frame["data"]
# only process data if we are in ARQ and BUSY state else return to quit
if not self.states.is_arq_state and not self.states.is_modem_busy:
self.log.warning("[Modem] wrong modem state - dropping data", is_arq_state=self.states.is_arq_state,
modem_state=self.states.is_modem_busy)
return
self.arq_file_transfer = True
self.states.set("is_modem_busy", True)
self.states.set("is_arq_state", True)
# Update data_channel timestamp
self.data_channel_last_received = int(time.time())
self.burst_last_received = int(time.time())
# Extract some important data from the frame
# Get sequence number of burst frame
self.rx_n_frame_of_burst = int.from_bytes(bytes(data_in[:1]), "big") - 10
# Get number of bursts from received frame
self.rx_n_frames_per_burst = int.from_bytes(bytes(data_in[1:2]), "big")
# The RX burst buffer needs to have a fixed length filled with "None".
# We need this later for counting the "Nones" to detect missing data.
# Check if burst buffer has expected length else create it
if len(self.arq_rx_burst_buffer) != self.rx_n_frames_per_burst:
self.arq_rx_burst_buffer = [None] * self.rx_n_frames_per_burst
# Append data to rx burst buffer
self.arq_rx_burst_buffer[self.rx_n_frame_of_burst] = data_in[self.arq_burst_header_size:] # type: ignore
self.dxgrid = b'------'
helpers.add_to_heard_stations(
self.dxcallsign,
self.dxgrid,
"DATA",
snr,
self.modem_frequency_offset,
self.states.radio_frequency,
self.states.heard_stations
)
# Check if we received all frames in the burst by checking if burst buffer has no more "Nones"
# This is the ideal case because we received all data
if None not in self.arq_rx_burst_buffer:
# then iterate through burst buffer and stick the burst together
# the temp burst buffer is needed for checking, if we already received data
temp_burst_buffer = b""
for value in self.arq_rx_burst_buffer:
# self.arq_rx_frame_buffer += self.arq_rx_burst_buffer[i]
temp_burst_buffer += bytes(value) # type: ignore
# free up burst buffer
self.arq_rx_burst_buffer = []
# TODO Needs to be removed as soon as mode error is fixed
# catch possible modem error which leads into false byteorder
# modem possibly decodes too late - data then is pushed to buffer
# which leads into wrong byteorder
# Lets put this in try/except so we are not crashing modem as its highly experimental
# This might only work for datac1 and datac3
try:
# area_of_interest = (modem.get_bytes_per_frame(self.mode_list[speed_level] - 1) -3) * 2
if self.arq_rx_frame_buffer.endswith(temp_burst_buffer[:246]) and len(temp_burst_buffer) >= 246:
self.log.warning(
"[Modem] ARQ | RX | wrong byteorder received - dropping data"
)
# we need to run a return here, so we are not sending an ACK
# return
except Exception as e:
self.log.warning(
"[Modem] ARQ | RX | wrong byteorder check failed", e=e
)
self.log.debug("[Modem] temp_burst_buffer", buffer=temp_burst_buffer)
self.log.debug("[Modem] self.arq_rx_frame_buffer", buffer=self.arq_rx_frame_buffer)
# if frame buffer ends not with the current frame, we are going to append new data
# if data already exists, we received the frame correctly,
# but the ACK frame didn't receive its destination (ISS)
if self.arq_rx_frame_buffer.endswith(temp_burst_buffer):
self.log.info(
"[Modem] ARQ | RX | Frame already received - sending ACK again"
)
else:
# Here we are going to search for our data in the last received bytes.
# This reduces the chance we will lose the entire frame in the case of signalling frame loss
# self.arq_rx_frame_buffer --> existing data
# temp_burst_buffer --> new data
# search_area --> area where we want to search
search_area = self.arq_burst_last_payload * self.rx_n_frames_per_burst
search_position = len(self.arq_rx_frame_buffer) - search_area
# if search position < 0, then search position = 0
search_position = max(0, search_position)
# find position of data. returns -1 if nothing found in area else >= 0
# we are beginning from the end, so if data exists twice or more,
# only the last one should be replaced
# we are going to only check position against minimum data frame payload
# use case: receive data, which already contains received data
# while the payload of data received before is shorter than actual payload
get_position = self.arq_rx_frame_buffer[search_position:].rfind(
temp_burst_buffer[:self.arq_burst_minimum_payload]
)
# if we find data, replace it at this position with the new data and strip it
if get_position >= 0:
self.arq_rx_frame_buffer = self.arq_rx_frame_buffer[
: search_position + get_position
]
self.log.warning(
"[Modem] ARQ | RX | replacing existing buffer data",
area=search_area,
pos=get_position,
)
else:
self.log.debug("[Modem] ARQ | RX | appending data to buffer")
self.arq_rx_frame_buffer += temp_burst_buffer
self.arq_burst_last_payload = len(temp_burst_buffer)
# Check if we didn't receive a BOF and EOF yet to avoid sending
# ack frames if we already received all data
if (
not self.rx_frame_bof_received
and not self.rx_frame_eof_received
and data_in.find(self.data_frame_eof) < 0
):
self.arq_calculate_speed_level(snr)
# TIMING TEST
# self.data_channel_last_received = int(time.time()) + 6 + 6
# self.burst_last_received = int(time.time()) + 6 + 6
self.data_channel_last_received = int(time.time())
self.burst_last_received = int(time.time())
# Create and send ACK frame
self.log.info("[Modem] ARQ | RX | SENDING ACK", finished=self.states.arq_seconds_until_finish,
bytesperminute=self.states.arq_bytes_per_minute)
self.send_burst_ack_frame(snr)
# Reset n retries per burst counter
self.n_retries_per_burst = 0
# calculate statistics
self.calculate_transfer_rate_rx(
self.rx_start_of_transmission, len(self.arq_rx_frame_buffer), snr
)
# send a network message with information
self.event_manager.send_custom_event(
freedata="modem-message",
arq="transmission",
status="receiving",
uuid=self.transmission_uuid,
percent=self.states.arq_transmission_percent,
bytesperminute=self.states.arq_bytes_per_minute,
compression=self.arq_compression_factor,
mycallsign=str(self.mycallsign, 'UTF-8'),
dxcallsign=str(self.dxcallsign, 'UTF-8'),
finished=self.states.arq_seconds_until_finish,
irs=helpers.bool_to_string(self.is_IRS)
)
else:
self.log.warning(
"[Modem] data_handler: missing data in burst buffer...",
frame=self.rx_n_frame_of_burst + 1,
frames=self.rx_n_frames_per_burst
)
# We have a BOF and EOF flag in our data. If we received both we received our frame.
# In case of loosing data, but we received already a BOF and EOF we need to make sure, we
# received the complete last burst by checking it for Nones
bof_position = self.arq_rx_frame_buffer.find(self.data_frame_bof)
eof_position = self.arq_rx_frame_buffer.find(self.data_frame_eof)
# get total bytes per transmission information as soon we received a frame with a BOF
if bof_position >= 0:
self.arq_extract_statistics_from_data_frame(bof_position, eof_position, snr)
if (
bof_position >= 0
and eof_position > 0
and None not in self.arq_rx_burst_buffer
):
self.log.debug(
"[Modem] arq_data_received:",
bof_position=bof_position,
eof_position=eof_position,
)
self.rx_frame_bof_received = True
self.rx_frame_eof_received = True
# Extract raw data from buffer
payload = self.arq_rx_frame_buffer[
bof_position + len(self.data_frame_bof): eof_position
]
# Get the data frame crc
data_frame_crc = payload[:4] # 0:4 = 4 bytes
# Get the data frame length
frame_length = int.from_bytes(payload[4:8], "big") # 4:8 = 4 bytes
self.states.set("arq_total_bytes", frame_length)
# 8:9 = compression factor
data_frame = payload[9:]
data_frame_crc_received = helpers.get_crc_32(data_frame)
# check if hmac signing enabled
if self.enable_hmac:
self.log.info(
"[Modem] [HMAC] Enabled",
)
# now check if we have valid hmac signature - returns salt or bool
salt_found = helpers.search_hmac_salt(self.dxcallsign, self.mycallsign, data_frame_crc, data_frame,
token_iters=100)
if salt_found:
# hmac digest received
self.arq_process_received_data_frame(data_frame, snr, signed=True)
else:
# hmac signature wrong
self.arq_process_received_data_frame(data_frame, snr, signed=False)
elif data_frame_crc == data_frame_crc_received:
self.log.warning(
"[Modem] [HMAC] Disabled, using CRC",
)
self.arq_process_received_data_frame(data_frame, snr, signed=False)
else:
self.event_manager.send_custom_event(
freedata="modem-message",
arq="transmission",
status="failed",
uuid=self.transmission_uuid,
mycallsign=str(self.mycallsign, 'UTF-8'),
dxcallsign=str(self.dxcallsign, 'UTF-8'),
irs=helpers.bool_to_string(self.is_IRS)
)
duration = time.time() - self.rx_start_of_transmission
self.log.warning(
"[Modem] ARQ | RX | DATA FRAME NOT SUCCESSFULLY RECEIVED!",
e="wrong crc",
expected=data_frame_crc.hex(),
received=data_frame_crc_received.hex(),
nacks=self.frame_nack_counter,
duration=duration,
bytesperminute=self.states.arq_bytes_per_minute,
compression=self.arq_compression_factor,
data=data_frame,
)
if self.enable_stats:
self.stats.push(frame_nack_counter=self.frame_nack_counter, status="wrong_crc", duration=duration)
self.log.info("[Modem] ARQ | RX | Sending NACK", finished=self.states.arq_seconds_until_finish,
bytesperminute=self.states.arq_bytes_per_minute)
self.send_burst_nack_frame(snr)
# Update arq_session timestamp
self.arq_session_last_received = int(time.time())
# Finally cleanup our buffers and states,
self.arq_cleanup()
def arq_extract_statistics_from_data_frame(self, bof_position, eof_position, snr):
payload = self.arq_rx_frame_buffer[
bof_position + len(self.data_frame_bof): eof_position
]
frame_length = int.from_bytes(payload[4:8], "big") # 4:8 4bytes
self.states.set("arq_total_bytes", frame_length)
compression_factor = int.from_bytes(payload[8:9], "big") # 4:8 4bytes
# limit to max value of 255
compression_factor = np.clip(compression_factor, 0, 255)
self.arq_compression_factor = compression_factor / 10
self.calculate_transfer_rate_rx(
self.rx_start_of_transmission, len(self.arq_rx_frame_buffer), snr
)
def send_burst_ack_frame(self, snr) -> None:
"""Build and send ACK frame for burst DATA frame"""
# ack_frame = bytearray(self.length_sig1_frame)
# ack_frame[:1] = bytes([FR_TYPE.BURST_ACK.value])
# ack_frame[1:2] = self.session_id
# ack_frame[2:3] = helpers.snr_to_bytes(snr)
# ack_frame[3:4] = bytes([int(self.speed_level)])
# ack_frame[4:8] = len(self.arq_rx_frame_buffer).to_bytes(4, byteorder="big")
ack_frame = self.frame_factory.build_arq_burst_ack(session_id=self.session_id,
snr=snr,
speed_level=self.speed_level,
len_arq_rx_frame_buffer=len(self.arq_rx_frame_buffer)
)
# wait if we have a channel busy condition
if self.states.channel_busy:
self.channel_busy_handler()
# Transmit frame
self.enqueue_frame_for_tx([ack_frame], c2_mode=FREEDV_MODE.sig1.value)
def send_data_ack_frame(self, snr) -> None:
"""Build and send ACK frame for received DATA frame"""
# ack_frame = bytearray(self.length_sig1_frame)
# ack_frame[:1] = bytes([FR_TYPE.FR_ACK.value])
# ack_frame[1:2] = self.session_id
# ack_frame[2:3] = helpers.snr_to_bytes(snr)
ack_frame = self.frame_factory.build_arq_frame_ack(session_id=self.session_id, snr=snr)
# wait if we have a channel busy condition
if self.states.channel_busy:
self.channel_busy_handler()
# Transmit frame
self.enqueue_frame_for_tx([ack_frame], c2_mode=FREEDV_MODE.sig1.value, copies=3, repeat_delay=0)
def send_retransmit_request_frame(self) -> None:
# check where a None is in our burst buffer and do frame+1, because lists start at 0
# FIXME Check to see if there's a `frame - 1` in the receive portion. Remove both if there is.
missing_frames = [
frame + 1
for frame, element in enumerate(self.arq_rx_burst_buffer)
if element is None
]
rpt_frame = bytearray(self.length_sig1_frame)
rpt_frame[:1] = bytes([FR_TYPE.FR_REPEAT.value])
rpt_frame[1:2] = self.session_id
rpt_frame[2:2 + len(missing_frames)] = missing_frames
self.log.info("[Modem] ARQ | RX | Requesting", frames=missing_frames)
# Transmit frame
self.enqueue_frame_for_tx([rpt_frame], c2_mode=FREEDV_MODE.sig1.value, copies=1, repeat_delay=0)

938
modem/data cemetery/deprecated_protocol_arq_session_iss.py
View file

@ -1,938 +0,0 @@
import sys
import threading
import time
import lzma
from random import randrange
import hmac
import hashlib
import helpers
import modem
import numpy as np
from codec2 import FREEDV_MODE
from modem_frametypes import FRAME_TYPE as FR_TYPE
import event_manager
from deprecated_protocol_arq_session import ARQ
class ISS(ARQ):
def __init__(self, config, event_queue, states):
super().__init__(config, event_queue, states)
self.tx_n_max_retries_per_burst = 40
self.datachannel_opening_interval = self.duration_sig1_frame + self.channel_busy_timeout + 1 # time between attempts when opening data channel
self.irs_buffer_position = 0
# actual n retries of burst
self.tx_n_retry_of_burst = 0
self.burst_ack_snr = 0 # SNR from received burst ack frames
def arq_transmit(self, data_out: bytes, hmac_salt: bytes):
"""
Transmit ARQ frame
Args:
data_out:bytes:
"""
# set signalling modes we want to listen to
# we are in an ongoing arq transmission, so we don't need sig0 actually
modem.demodulator.RECEIVE_SIG0 = False
modem.demodulator.RECEIVE_SIG1 = True
self.tx_n_retry_of_burst = 0 # retries we already sent data
# Maximum number of retries to send before declaring a frame is lost
# save len of data_out to TOTAL_BYTES for our statistics
self.states.set("arq_total_bytes", len(data_out))
self.arq_file_transfer = True
frame_total_size = len(data_out).to_bytes(4, byteorder="big")
# Compress data frame
data_frame_compressed = lzma.compress(data_out)
compression_factor = len(data_out) / len(data_frame_compressed)
self.arq_compression_factor = np.clip(compression_factor, 0, 255)
compression_factor = bytes([int(self.arq_compression_factor * 10)])
self.event_manager.send_custom_event(
freedata="modem-message",
arq="transmission",
status="transmitting",
uuid=self.transmission_uuid,
percent=self.states.arq_transmission_percent,
bytesperminute=self.states.arq_bytes_per_minute,
compression=self.arq_compression_factor,
finished=self.states.arq_seconds_until_finish,
mycallsign=str(self.mycallsign, 'UTF-8'),
dxcallsign=str(self.dxcallsign, 'UTF-8'),
irs=helpers.bool_to_string(self.is_IRS)
)
self.log.info(
"[Modem] | TX | DATACHANNEL",
Bytes=self.states.arq_total_bytes,
)
data_out = data_frame_compressed
# Reset data transfer statistics
tx_start_of_transmission = time.time()
self.calculate_transfer_rate_tx(tx_start_of_transmission, 0, len(data_out))
# check if hmac signature is available
if hmac_salt not in ['', False]:
print(data_out)
# create hmac digest
hmac_digest = hmac.new(hmac_salt, data_out, hashlib.sha256).digest()
# truncate to 32bit
frame_payload_crc = hmac_digest[:4]
self.log.debug("[Modem] frame payload HMAC:", crc=frame_payload_crc.hex())
else:
# Append a crc at the beginning and end of file indicators
frame_payload_crc = helpers.get_crc_32(data_out)
self.log.debug("[Modem] frame payload CRC:", crc=frame_payload_crc.hex())
# Assemble the data frame
data_out = (
self.data_frame_bof
+ frame_payload_crc
+ frame_total_size
+ compression_factor
+ data_out
+ self.data_frame_eof
)
self.log.debug("[Modem] frame raw data:", data=data_out)
# Initial bufferposition is 0
bufferposition = 0
bufferposition_end = 0
bufferposition_burst_start = 0
# Iterate through data_out buffer
while not self.data_frame_ack_received and self.states.is_arq_state:
# we have self.tx_n_max_retries_per_burst attempts for sending a burst
for self.tx_n_retry_of_burst in range(self.tx_n_max_retries_per_burst):
# Bound speed level to:
# - minimum of either the speed or the length of mode list - 1
# - maximum of either the speed or zero
self.speed_level = min(self.speed_level, len(self.mode_list) - 1)
self.speed_level = max(self.speed_level, 0)
self.states.set("arq_speed_level", self.speed_level)
data_mode = self.mode_list[self.speed_level]
self.log.debug(
"[Modem] Speed-level:",
level=self.speed_level,
retry=self.tx_n_retry_of_burst,
mode=FREEDV_MODE(data_mode).name,
)
# Payload information
payload_per_frame = modem.get_bytes_per_frame(data_mode) - 2
self.log.info("[Modem] early buffer info",
bufferposition=bufferposition,
bufferposition_end=bufferposition_end,
bufferposition_burst_start=bufferposition_burst_start
)
# check for maximum frames per burst for remaining data
n_frames_per_burst = 1
if self.max_n_frames_per_burst > 1:
while (payload_per_frame * n_frames_per_burst) % len(data_out[bufferposition_burst_start:]) == (
payload_per_frame * n_frames_per_burst):
threading.Event().wait(0.01)
# print((payload_per_frame * n_frames_per_burst) % len(data_out))
n_frames_per_burst += 1
if n_frames_per_burst == self.max_n_frames_per_burst:
break
else:
n_frames_per_burst = 1
self.log.info("[Modem] calculated frames_per_burst:", n=n_frames_per_burst)
tempbuffer = []
self.rpt_request_buffer = []
# Append data frames with n_frames_per_burst to tempbuffer
for n_frame in range(n_frames_per_burst):
arqheader = bytearray()
arqheader[:1] = bytes([FR_TYPE.BURST_01.value + n_frame])
#####arqheader[:1] = bytes([FR_TYPE.BURST_01.value])
arqheader[1:2] = bytes([n_frames_per_burst])
arqheader[2:3] = self.session_id
# only check for buffer position if at least one NACK received
self.log.info("[Modem] ----- data buffer position:", iss_buffer_pos=bufferposition,
irs_bufferposition=self.irs_buffer_position)
if self.frame_nack_counter > 0 and self.irs_buffer_position != bufferposition:
self.log.error("[Modem] ----- data buffer offset:", iss_buffer_pos=bufferposition,
irs_bufferposition=self.irs_buffer_position)
# only adjust buffer position for experimental versions
if self.enable_experimental_features:
self.log.warning("[Modem] ----- data adjustment enabled!")
bufferposition = self.irs_buffer_position
bufferposition_end = bufferposition + payload_per_frame - len(arqheader)
# Normal condition
if bufferposition_end <= len(data_out):
frame = data_out[bufferposition:bufferposition_end]
frame = arqheader + frame
# Pad the last bytes of a frame
else:
extended_data_out = data_out[bufferposition:]
extended_data_out += bytes([0]) * (
payload_per_frame - len(extended_data_out) - len(arqheader)
)
frame = arqheader + extended_data_out
######tempbuffer = frame # [frame]
tempbuffer.append(frame)
# add data to our repeat request buffer for easy access if we received a request
self.rpt_request_buffer.append(frame)
# set new buffer position
bufferposition = bufferposition_end
self.log.debug("[Modem] tempbuffer:", tempbuffer=tempbuffer)
self.log.info(
"[Modem] ARQ | TX | FRAMES",
mode=FREEDV_MODE(data_mode).name,
fpb=n_frames_per_burst,
retry=self.tx_n_retry_of_burst,
)
self.enqueue_frame_for_tx(tempbuffer, c2_mode=data_mode)
# After transmission finished, wait for an ACK or RPT frame
while (
self.states.is_arq_state
and not self.burst_ack
and not self.burst_nack
and not self.rpt_request_received
and not self.data_frame_ack_received
):
threading.Event().wait(0.01)
# Once we receive a burst ack, reset its state and break the RETRIES loop
if self.burst_ack:
self.burst_ack = False # reset ack state
self.tx_n_retry_of_burst = 0 # reset retries
self.log.debug(
"[Modem] arq_transmit: Received BURST ACK. Sending next chunk."
, irs_snr=self.burst_ack_snr)
# update temp bufferposition for n frames per burst early calculation
bufferposition_burst_start = bufferposition_end
break # break retry loop
if self.data_frame_ack_received:
self.log.debug(
"[Modem] arq_transmit: Received FRAME ACK. Braking retry loop."
)
break # break retry loop
if self.burst_nack:
self.tx_n_retry_of_burst += 1
self.log.warning(
"[Modem] arq_transmit: Received BURST NACK. Resending data",
bufferposition_burst_start=bufferposition_burst_start,
bufferposition=bufferposition
)
bufferposition = bufferposition_burst_start
self.burst_nack = False # reset nack state
# We need this part for leaving the repeat loop
# self.states.is_arq_state == "DATA" --> when stopping transmission manually
if not self.states.is_arq_state:
self.log.debug(
"[Modem] arq_transmit: ARQ State changed to FALSE. Breaking retry loop."
)
break
self.calculate_transfer_rate_tx(
tx_start_of_transmission, bufferposition_end, len(data_out)
)
# NEXT ATTEMPT
self.log.debug(
"[Modem] ATTEMPT:",
retry=self.tx_n_retry_of_burst,
maxretries=self.tx_n_max_retries_per_burst,
)
# update buffer position
bufferposition = bufferposition_end
# update stats
self.calculate_transfer_rate_tx(
tx_start_of_transmission, bufferposition_end, len(data_out)
)
self.event_manager.send_custom_event(
freedata="modem-message",
arq="transmission",
status="transmitting",
uuid=self.transmission_uuid,
percent=self.states.arq_transmission_percent,
bytesperminute=self.states.arq_bytes_per_minute,
compression=self.arq_compression_factor,
finished=self.states.arq_seconds_until_finish,
irs_snr=self.burst_ack_snr,
mycallsign=str(self.mycallsign, 'UTF-8'),
dxcallsign=str(self.dxcallsign, 'UTF-8'),
irs=helpers.bool_to_string(self.is_IRS)
)
# Stay in the while loop until we receive a data_frame_ack. Otherwise,
# the loop exits after sending the last frame only once and doesn't
# wait for an acknowledgement.
if self.data_frame_ack_received and bufferposition > len(data_out):
self.log.debug("[Modem] arq_tx: Last fragment sent and acknowledged.")
break
# GOING TO NEXT ITERATION
if self.data_frame_ack_received:
self.arq_transmit_success()
else:
self.arq_transmit_failed()
if TESTMODE:
# Quit after transmission
self.log.debug("[Modem] TESTMODE: arq_transmit exiting.")
sys.exit(0)
def arq_transmit_success(self):
"""
will be called if we successfully transmitted all of queued data
"""
# we need to wait until sending "transmitted" state
# gui database is too slow for handling this within 0.001 seconds
# so let's sleep a little
threading.Event().wait(0.2)
self.event_manager.send_custom_event(
freedata="modem-message",
arq="transmission",
status="transmitted",
uuid=self.transmission_uuid,
percent=self.states.arq_transmission_percent,
bytesperminute=self.states.arq_bytes_per_minute,
compression=self.arq_compression_factor,
finished=self.states.arq_seconds_until_finish,
mycallsign=str(self.mycallsign, 'UTF-8'),
dxcallsign=str(self.dxcallsign, 'UTF-8'),
irs=helpers.bool_to_string(self.is_IRS),
nacks=self.frame_nack_counter,
speed_list=self.states.arq_speed_list
)
self.log.info(
"[Modem] ARQ | TX | DATA TRANSMITTED!",
BytesPerMinute=self.states.arq_bytes_per_minute,
total_bytes=self.states.arq_total_bytes,
BitsPerSecond=self.states.arq_bits_per_second,
)
# finally do an arq cleanup
self.arq_cleanup()
def arq_transmit_failed(self):
"""
will be called if we not successfully transmitted all of queued data
"""
self.event_manager.send_custom_event(
freedata="modem-message",
arq="transmission",
status="failed",
uuid=self.transmission_uuid,
percent=self.states.arq_transmission_percent,
bytesperminute=self.states.arq_bytes_per_minute,
compression=self.arq_compression_factor,
mycallsign=str(self.mycallsign, 'UTF-8'),
dxcallsign=str(self.dxcallsign, 'UTF-8'),
irs=helpers.bool_to_string(self.is_IRS),
nacks=self.frame_nack_counter,
speed_list=self.states.arq_speed_list
)
self.log.info(
"[Modem] ARQ | TX | TRANSMISSION FAILED OR TIME OUT!")
self.stop_transmission()
def burst_ack_nack_received(self, data_in: bytes, snr) -> None:
"""
Received an ACK/NACK for a transmitted frame, keep track and
make adjustments to speed level if needed.
Args:
data_in:bytes:
Returns:
"""
# Process data only if we are in ARQ and BUSY state
if self.states.is_arq_state:
self.dxgrid = b'------'
helpers.add_to_heard_stations(
self.dxcallsign,
self.dxgrid,
"DATA",
snr,
self.modem_frequency_offset,
self.states.radio_frequency,
self.states.heard_stations
)
frametype = int.from_bytes(bytes(data_in[:1]), "big")
if frametype == FR_TYPE.BURST_ACK.value:
# Increase speed level if we received a burst ack
# self.speed_level = min(self.speed_level + 1, len(self.mode_list) - 1)
# Force data retry loops of TX Modem to stop and continue with next frame
self.burst_ack = True
# Reset burst nack counter
self.burst_nack_counter = 0
# Reset n retries per burst counter
self.n_retries_per_burst = 0
self.irs_buffer_position = int.from_bytes(data_in[4:8], "big")
self.burst_ack_snr = helpers.snr_from_bytes(data_in[2:3])
else:
# Decrease speed level if we received a burst nack
# self.speed_level = max(self.speed_level - 1, 0)
# Set flag to retry frame again.
self.burst_nack = True
# Increment burst nack counter
self.burst_nack_counter += 1
self.burst_ack_snr = 'NaN'
self.irs_buffer_position = int.from_bytes(data_in[5:9], "big")
self.log.warning(
"[Modem] ARQ | TX | Burst NACK received",
burst_nack_counter=self.burst_nack_counter,
irs_buffer_position=self.irs_buffer_position,
)
# Update data_channel timestamp
self.data_channel_last_received = int(time.time())
# self.burst_ack_snr = int.from_bytes(bytes(data_in[2:3]), "big")
self.burst_ack_snr = helpers.snr_from_bytes(data_in[2:3])
# self.log.info("SNR ON IRS", snr=self.burst_ack_snr)
self.speed_level = int.from_bytes(bytes(data_in[3:4]), "big")
self.states.set("arq_speed_level", self.speed_level)
def frame_ack_received(
self, data_in: bytes, snr # pylint: disable=unused-argument,
) -> None:
"""Received an ACK for a transmitted frame"""
# Process data only if we are in ARQ and BUSY state
if self.states.is_arq_state:
self.dxgrid = b'------'
helpers.add_to_heard_stations(
self.dxcallsign,
self.dxgrid,
"DATA",
snr,
self.modem_frequency_offset,
self.states.radio_frequency,
self.states.heard_stations
)
# Force data loops of Modem to stop and continue with next frame
self.data_frame_ack_received = True
# Update arq_session and data_channel timestamp
self.data_channel_last_received = int(time.time())
self.arq_session_last_received = int(time.time())
def frame_nack_received(
self, data_in: bytes, snr # pylint: disable=unused-argument
) -> None:
"""
Received a NACK for a transmitted frame
Args:
data_in:bytes:
"""
self.log.warning("[Modem] ARQ FRAME NACK RECEIVED - cleanup!",
arq="transmission",
status="failed",
uuid=self.transmission_uuid,
percent=self.states.arq_transmission_percent,
bytesperminute=self.states.arq_bytes_per_minute,
mycallsign=str(self.mycallsign, 'UTF-8'),
dxcallsign=str(self.dxcallsign, 'UTF-8'),
irs=helpers.bool_to_string(self.is_IRS),
nacks=self.frame_nack_counter,
speed_list=self.states.arq_speed_list
)
self.dxgrid = b'------'
helpers.add_to_heard_stations(
self.dxcallsign,
self.dxgrid,
"DATA",
snr,
self.modem_frequency_offset,
self.states.radio_frequency,
self.states.heard_stations
)
self.event_manager.send_custom_event(
freedata="modem-message",
arq="transmission",
status="failed",
uuid=self.transmission_uuid,
percent=self.states.arq_transmission_percent,
bytesperminute=self.states.arq_bytes_per_minute,
compression=self.arq_compression_factor,
mycallsign=str(self.mycallsign, 'UTF-8'),
dxcallsign=str(self.dxcallsign, 'UTF-8'),
irs=helpers.bool_to_string(self.is_IRS),
nacks=self.frame_nack_counter,
speed_list=self.states.arq_speed_list
)
# Update data_channel timestamp
self.arq_session_last_received = int(time.time())
self.arq_cleanup()
def burst_rpt_received(self, data_in: bytes, snr):
"""
Repeat request frame received for transmitted frame
Args:
data_in:bytes:
"""
# Only process data if we are in ARQ and BUSY state
if not self.states.is_arq_state or not self.states.is_modem_busy:
return
self.dxgrid = b'------'
helpers.add_to_heard_stations(
self.dxcallsign,
self.dxgrid,
"DATA",
snr,
self.modem_frequency_offset,
self.states.radio_frequency,
self.states.heard_stations
)
self.log.info("[Modem] ARQ REPEAT RECEIVED")
# self.rpt_request_received = True
# Update data_channel timestamp
self.data_channel_last_received = int(time.time())
# self.rpt_request_buffer = []
missing_area = bytes(data_in[2:12]) # 1:9
missing_area = missing_area.strip(b"\x00")
print(missing_area)
print(self.rpt_request_buffer)
tempbuffer_rptframes = []
for i in range(len(missing_area)):
print(missing_area[i])
missing_frames_buffer_position = missing_area[i] - 1
tempbuffer_rptframes.append(self.rpt_request_buffer[missing_frames_buffer_position])
self.log.info("[Modem] SENDING REPEAT....")
data_mode = self.mode_list[self.speed_level]
self.enqueue_frame_for_tx(tempbuffer_rptframes, c2_mode=data_mode)
############################################################################################################
# ARQ SESSION HANDLER
############################################################################################################
def arq_session_handler(self, mycallsign, dxcallsign) -> bool:
"""
Create a session with `self.dxcallsign` and wait until the session is open.
Returns:
True if the session was opened successfully
False if the session open request failed
"""
encoded_call = helpers.callsign_to_bytes(mycallsign)
mycallsign = helpers.bytes_to_callsign(encoded_call)
encoded_call = helpers.callsign_to_bytes(dxcallsign)
dxcallsign = helpers.bytes_to_callsign(encoded_call)
self.states.set("dxcallsign", dxcallsign)
dxcallsign_crc = helpers.get_crc_24(dxcallsign)
self.log.info(
"[Modem] SESSION ["
+ str(mycallsign, "UTF-8")
+ "]>> <<["
+ str(dxcallsign, "UTF-8")
+ "]",
self.states.arq_session_state,
)
# wait if we have a channel busy condition
if self.states.channel_busy:
self.channel_busy_handler()
self.open_session()
# wait until data channel is open
while not self.states.is_arq_session and not self.arq_session_timeout:
threading.Event().wait(0.01)
self.states.set("arq_session_state", "connecting")
self.event_manager.send_custom_event(
freedata="modem-message",
arq="session",
status="connecting",
mycallsign=str(mycallsign, 'UTF-8'),
dxcallsign=str(dxcallsign, 'UTF-8'),
)
if self.states.is_arq_session and self.states.arq_session_state == "connected":
# self.states.set("arq_session_state", "connected")
self.event_manager.send_custom_event(
freedata="modem-message",
arq="session",
status="connected",
mycallsign=str(mycallsign, 'UTF-8'),
dxcallsign=str(dxcallsign, 'UTF-8'),
)
return True
self.log.warning(
"[Modem] SESSION FAILED ["
+ str(mycallsign, "UTF-8")
+ "]>>X<<["
+ str(dxcallsign, "UTF-8")
+ "]",
attempts=self.session_connect_max_retries, # Adjust for 0-based for user display
reason="maximum connection attempts reached",
state=self.states.arq_session_state,
)
self.states.set("arq_session_state", "failed")
self.event_manager.send_custom_event(
freedata="modem-message",
arq="session",
status="failed",
reason="timeout",
mycallsign=str(mycallsign, 'UTF-8'),
dxcallsign=str(dxcallsign, 'UTF-8'),
)
return False
def open_dc_and_transmit(
self,
data_out: bytes,
transmission_uuid: str,
mycallsign,
dxcallsign,
) -> bool:
"""
Open data channel and transmit data
Args:
data_out:bytes:
transmission_uuid:str:
mycallsign:bytes:
Returns:
True if the data session was opened and the data was sent
False if the data session was not opened
"""
self.mycallsign = mycallsign
# additional step for being sure our callsign is correctly
# in case we are not getting a station ssid
# then we are forcing a station ssid = 0
if not self.states.is_arq_session:
dxcallsign = helpers.callsign_to_bytes(dxcallsign)
dxcallsign = helpers.bytes_to_callsign(dxcallsign)
self.dxcallsign = dxcallsign
self.dxcallsign_crc = helpers.get_crc_24(self.dxcallsign)
# check if hmac hash is provided
try:
self.log.info("[SCK] [HMAC] Looking for salt/token", local=mycallsign, remote=dxcallsign)
hmac_salt = helpers.get_hmac_salt(dxcallsign, mycallsign)
self.log.info("[SCK] [HMAC] Salt info", local=mycallsign, remote=dxcallsign, salt=hmac_salt)
except Exception:
self.log.warning("[SCK] [HMAC] No salt/token found")
hmac_salt = ''
self.states.set("is_modem_busy", True)
self.arq_file_transfer = True
self.beacon_paused = True
self.transmission_uuid = transmission_uuid
# wait a moment for the case, a heartbeat is already on the way back to us
# this makes channel establishment more clean
if self.states.is_arq_session:
threading.Event().wait(2.5)
# init arq state event
self.arq_state_event = threading.Event()
# finally start the channel opening procedure
self.arq_open_data_channel(mycallsign)
# if data channel is open, return true else false
if self.arq_state_event.is_set():
# start arq transmission
self.arq_transmit(data_out, hmac_salt)
return True
else:
self.log.debug(
"[Modem] arq_open_data_channel:", transmission_uuid=self.transmission_uuid
)
self.event_manager.send_custom_event(
freedata="modem-message",
arq="transmission",
status="failed",
reason="unknown",
uuid=self.transmission_uuid,
percent=self.states.arq_transmission_percent,
bytesperminute=self.states.arq_bytes_per_minute,
compression=self.arq_compression_factor,
mycallsign=str(self.mycallsign, 'UTF-8'),
dxcallsign=str(self.dxcallsign, 'UTF-8'),
irs=helpers.bool_to_string(self.is_IRS),
nacks=self.frame_nack_counter,
speed_list=self.states.arq_speed_list
)
self.log.warning(
"[Modem] ARQ | TX | DATA ["
+ str(mycallsign, "UTF-8")
+ "]>>X<<["
+ str(self.dxcallsign, "UTF-8")
+ "]"
)
# Attempt to clean up the far-side, if it received the
# open_session frame and can still hear us.
self.close_session()
# release beacon pause
self.beacon_paused = False
# otherwise return false
return False
def arq_open_data_channel(
self, mycallsign
) -> bool:
"""
Open an ARQ data channel.
Args:
mycallsign:bytes:
Returns:
True if the data channel was opened successfully
False if the data channel failed to open
"""
# set IRS indicator to false, because we are IRS
self.is_IRS = False
# init a new random session id if we are not in an arq session
if not self.states.is_arq_session:
self.session_id = np.random.bytes(1)
# Update data_channel timestamp
self.data_channel_last_received = int(time.time())
# check if the Modem is running in low bandwidth mode
# then set the corresponding frametype and build frame
if self.low_bandwidth_mode:
frametype = bytes([FR_TYPE.ARQ_DC_OPEN_N.value])
self.log.debug("[Modem] Requesting low bandwidth mode")
else:
frametype = bytes([FR_TYPE.ARQ_DC_OPEN_W.value])
self.log.debug("[Modem] Requesting high bandwidth mode")
# build connection frame
connection_frame = self.frame_factory.build_arq_connect(
session_id=self.session_id,
destination_crc=self.dxcallsign_crc,
)
for attempt in range(self.data_channel_max_retries):
self.event_manager.send_custom_event(
freedata="modem-message",
arq="transmission",
status="opening",
mycallsign=mycallsign,
dxcallsign=str(self.dxcallsign, 'UTF-8'),
irs=helpers.bool_to_string(self.is_IRS)
)
self.log.info(
"[Modem] ARQ | DATA | TX | ["
+ mycallsign
+ "]>> <<["
+ str(self.dxcallsign, "UTF-8")
+ "]",
attempt=f"{str(attempt + 1)}/{str(self.data_channel_max_retries)}",
)
# Let's check if we have a busy channel and if we are not in a running arq session.
if self.states.channel_busy and not self.arq_state_event.is_set() or self.states.is_codec2_traffic:
self.channel_busy_handler()
# if channel free, enqueue frame for tx
if not self.arq_state_event.is_set():
self.enqueue_frame_for_tx([connection_frame], c2_mode=FREEDV_MODE.sig0.value, copies=1, repeat_delay=0)
# wait until timeout or event set
random_wait_time = randrange(int(self.duration_sig1_frame * 10),
int(self.datachannel_opening_interval * 10), 1) / 10
self.arq_state_event.wait(timeout=random_wait_time)
if self.arq_state_event.is_set():
return True
if not self.states.is_modem_busy:
return False
# `data_channel_max_retries` attempts have been sent. Aborting attempt & cleaning up
return False
def arq_received_channel_is_open(self, data_in: bytes, snr) -> None:
"""
Called if we received a data channel opener
Args:
data_in:bytes:
"""
protocol_version = int.from_bytes(bytes(data_in[13:14]), "big")
if protocol_version == self.arq_protocol_version:
self.event_manager.send_custom_event(
freedata="modem-message",
arq="transmission",
status="opened",
mycallsign=str(self.mycallsign, 'UTF-8'),
dxcallsign=str(self.dxcallsign, 'UTF-8'),
irs=helpers.bool_to_string(self.is_IRS)
)
frametype = int.from_bytes(bytes(data_in[:1]), "big")
if frametype == FR_TYPE.ARQ_DC_OPEN_ACK_N.value:
self.received_LOW_BANDWIDTH_MODE = True
self.mode_list = self.mode_list_low_bw
self.time_list = self.time_list_low_bw
self.log.debug("[Modem] low bandwidth mode", modes=self.mode_list)
else:
self.received_LOW_BANDWIDTH_MODE = False
self.mode_list = self.mode_list_high_bw
self.time_list = self.time_list_high_bw
self.log.debug("[Modem] high bandwidth mode", modes=self.mode_list)
# set speed level from session opener frame delegation
self.speed_level = int.from_bytes(bytes(data_in[8:9]), "big")
self.log.debug("[Modem] speed level selected for given SNR", speed_level=self.speed_level)
self.dxgrid = b'------'
helpers.add_to_heard_stations(
self.dxcallsign,
self.dxgrid,
"DATA",
snr,
self.modem_frequency_offset,
self.states.radio_frequency,
self.states.heard_stations
)
self.log.info(
"[Modem] ARQ | DATA | TX | ["
+ str(self.mycallsign, "UTF-8")
+ "]>>|<<["
+ str(self.dxcallsign, "UTF-8")
+ "]",
snr=snr,
)
# as soon as we set ARQ_STATE to True, transmission starts
self.states.set("is_arq_state", True)
# also set the ARQ event
self.arq_state_event.set()
# Update data_channel timestamp
self.data_channel_last_received = int(time.time())
else:
self.event_manager.send_custom_event(
freedata="modem-message",
arq="transmission",
status="failed",
reason="protocol version missmatch",
mycallsign=str(self.mycallsign, 'UTF-8'),
dxcallsign=str(self.dxcallsign, 'UTF-8'),
irs=helpers.bool_to_string(self.is_IRS)
)
self.log.warning(
"[Modem] protocol version mismatch:",
received=protocol_version,
own=self.arq_protocol_version,
)
self.stop_transmission()
def calculate_transfer_rate_tx(
self, tx_start_of_transmission: float, sentbytes: int, tx_buffer_length: int
) -> list:
"""
Calculate transfer rate for transmission
Args:
tx_start_of_transmission:float:
sentbytes:int:
tx_buffer_length:int:
Returns: List of:
bits_per_second: float,
bytes_per_minute: float,
transmission_percent: float
"""
try:
arq_transmission_percent = min(
int((sentbytes / tx_buffer_length) * 100), 100
)
transmissiontime = time.time() - tx_start_of_transmission
if sentbytes > 0:
arq_bits_per_second = int((sentbytes * 8) / transmissiontime)
bytes_per_minute = int(sentbytes / (transmissiontime / 60))
arq_seconds_until_finish = int(((tx_buffer_length - sentbytes) / (
bytes_per_minute * self.arq_compression_factor)) * 60)
speed_chart = {"snr": self.burst_ack_snr, "bpm": bytes_per_minute,
"timestamp": int(time.time())}
# check if data already in list
if speed_chart not in self.states.arq_speed_list:
self.states.arq_speed_list.append(speed_chart)
else:
arq_bits_per_second = 0
bytes_per_minute = 0
arq_seconds_until_finish = 0
except Exception as err:
self.log.error(f"[Modem] calculate_transfer_rate_tx: Exception: {err}")
arq_transmission_percent = 0.0
arq_bits_per_second = 0
bytes_per_minute = 0
self.states.set("arq_bits_per_second", arq_bits_per_second)
self.states.set("bytes_per_minute", bytes_per_minute)
self.states.set("arq_transmission_percent", arq_transmission_percent)
self.states.set("arq_compression_factor", self.arq_compression_factor)

1469
modem/data cemetery/deprecated_sock.py
View file

File diff suppressed because it is too large Load diff

23
modem/data cemetery/test/001_highsnr_stdio_audio/test_virtual3.sh
View file

@ -1,23 +0,0 @@
#!/bin/bash -x
# Run a test using the virtual sound cards, Python audio I/O
function check_alsa_loopback {
lsmod | grep snd_aloop >> /dev/null
if [ $? -eq 1 ]; then
echo "ALSA loopback device not present. Please install with:"
echo
echo " sudo modprobe snd-aloop index=1,2 enable=1,1 pcm_substreams=1,1 id=CHAT1,CHAT2"
exit 1
fi
}
check_alsa_loopback
# make sure all child processes are killed when we exit
trap 'jobs -p | xargs -r kill' EXIT
python3 util_callback_rx.py --mode datac13 --frames 2 --bursts 3 --audiodev -2 --debug &
rx_pid=$!
#sleep 1
python3 util_tx.py --mode datac13 --frames 2 --bursts 3 --audiodev -2
wait ${rx_pid}

103
modem/data cemetery/test/002_highsnr_ping_pong/README.md
View file

@ -1,103 +0,0 @@
# FreeDV-JATE [Just Another TNC Experiment]
## 002_HIGHSNR_PING_PONG
### INSTALL TEST SUITE
#### Install prerequierements
```
sudo apt update
sudo apt upgrade
sudo apt install git cmake build-essential python3-pip portaudio19-dev python3-pyaudio
pip3 install crcengine
pip3 install threading
```
Go into a directory of your choice
Run the following commands --> They will download and compile the latest codec2 ( dr-packet ) files and LPCNet as well into the directory of your choice
```
wget https://raw.githubusercontent.com/DJ2LS/FreeDV-JATE/002_HIGHSNR_PING_PONG/install_test_suite.sh
chmod +x install_test_suite.sh
./install_test_suite.sh
```
### PARAMETERS
| parameter | description | side |
| ---------------- | ------------------------------------------ | ----------------------- |
| - -txmode 12 | set the mode for FreeDV ( 10,11,12,14 ) | Terminal 1 & Terminal 2 |
| - -rxmode 14 | set the mode for FreeDV ( 10,11,12,14 ) | Terminal 1 & Terminal 2 |
| - -frames 1 | set the number of frames per burst | Terminal 1 |
| - -bursts 1 | set the number of bursts | Terminal 1 |
| - -audioinput 2 | set the audio device | Terminal 1 & Terminal 2 |
| - -audiooutput 1 | set the audio device | Terminal 1 & Terminal 2 |
| - -debug | if used, print additional debugging output | Terminal 1 & Terminal 2 |
### AUDIO TESTS VIA VIRTUAL AUDIO DEVICE
#### Create audio sinkhole and subdevices
Note: This command needs to be run again after every reboot
```
sudo modprobe snd-aloop index=1,2 enable=1,1 pcm_substreams=1,1 id=CHAT1,CHAT2
```
check if devices have been created
aplay -l
Output should be like this:
```
Karte 0: Intel [HDA Intel], Gerät 0: Generic Analog [Generic Analog]
Sub-Geräte: 1/1
Sub-Gerät #0: subdevice #0
Karte 1: CHAT1 [Loopback], Gerät 0: Loopback PCM [Loopback PCM]
Sub-Geräte: 1/1
Sub-Gerät #0: subdevice #0
Karte 1: CHAT1 [Loopback], Gerät 1: Loopback PCM [Loopback PCM]
Sub-Geräte: 1/1
Sub-Gerät #0: subdevice #0
Karte 2: CHAT2 [Loopback], Gerät 0: Loopback PCM [Loopback PCM]
Sub-Geräte: 1/1
Sub-Gerät #0: subdevice #0
Karte 2: CHAT2 [Loopback], Gerät 1: Loopback PCM [Loopback PCM]
Sub-Geräte: 1/1
Sub-Gerät #0: subdevice #0
```
### Run tests:
#### Terminal 1: Ping
```
python3 PING.py --txmode 12 --rxmode 14 --audioinput 2 --audiooutput 2 --frames 1 --bursts 2
```
Output
```
BURSTS: 2 FRAMES: 1
-----------------------------------------------------------------
TX | PING | BURST [1/2] FRAME [1/1]
RX | PONG | BURST [1/2] FRAME [1/1]
-----------------------------------------------------------------
TX | PING | BURST [2/2] FRAME [1/1]
RX | PONG | BURST [2/2] FRAME [1/1]
```
#### Terminal 2: Pong
```
python3 PONG.py --txmode 14 --rxmode 12 --audioinput 2 --audiooutput 2
```
Output
```
RX | BURST [1/2] FRAME [1/1] >>> SENDING PONG
RX | BURST [2/2] FRAME [1/1] >>> SENDING PONG
```

165
modem/data cemetery/test/README.md
View file

@ -1,165 +0,0 @@
# Unit Test Menu
The following `CTest` tests cover some TNC functionality and the interface to codec2:
1. Name: `audio_buffer`
Tests the thread safety of the audio buffer routines.
1. Name: `resampler`
Tests FreeDATA audio resampling from 48KHz to 8KHz.
1. Name: `tnc_state_machine`
Tests TNC transitions between states. This needs to be expanded.
1. Name: `tnc_irs_iss`
Tests TNC modem queue interaction. This needs to be expanded.
1. Name: `helper_routines`
Tests various helper routines.
1. Name: `py_highsnr_stdio_P_P_multi`
Tests a high signal-to-noise ratio (good quality) audio path using multiple codecs. (Pure python.)
1. Name: `py_highsnr_stdio_P_P_datacx`
Tests a high signal-to-noise ratio audio path using multiple individual codecs.
1. Name: `py_highsnr_stdio_P_C_datacx`
Tests a high signal-to-noise ratio audio path using multiple individual codecs.
1. Name: `py_highsnr_stdio_C_P_datacx`
Tests a high signal-to-noise ratio audio path using multiple individual codecs.
1. Name: `highsnr_stdio_P_C_single`
Tests compatibility with FreeDATA's transmit and freedv's raw data receive.
1. Name: `highsnr_stdio_C_P_single`
Tests compatibility with freedv's raw data transmit and FreeDATA's receive.
1. Name: `highsnr_stdio_P_P_single`
Tests a high signal-to-noise ratio audio path using multiple codecs. (Requires POSIX system.)
1. Name: `highsnr_stdio_P_P_multi`
Tests a high signal-to-noise ratio audio path using multiple codecs. (Requires POSIX system.)
The following tests can not currently be run with GitHub's pipeline as they require the ALSA dummy device
kernel module to be installed. They also do not perform reliably. These tests are slowly being
replaced with equivalent pipeline-compatible tests.
1. Name: `highsnr_virtual1_P_P_single_alsa`
Tests a high signal-to-noise ratio audio path using a single codec directly over an ALSA dummy device.
1. Name: `highsnr_virtual2_P_P_single`
Tests a high signal-to-noise ratio audio path using a single codec over an ALSA dummy device.
**Not functional** due to an incompatibility between the two scripts in the way they determine audio devices.
1. Name: `highsnr_virtual3_P_P_multi`
Tests a high signal-to-noise ratio audio path using multiple codecs over an ALSA dummy device.
1. Name: `highsnr_virtual4_P_P_single_callback`
**Not functional** due to an incompatibility between the two scripts in the way they determine audio devices.
1. Name: `highsnr_virtual4_P_P_single_callback_outside`
**Not functional** due to an incompatibility between the two scripts in the way they determine audio devices.
1. Name: `highsnr_virtual5_P_P_multi_callback`
1. Name: `highsnr_virtual5_P_P_multi_callback_outside`
# Instructions
1. Install:
```
cd FreeDATA
mkdir build
cd build
cmake -DCODEC2_BUILD_DIR=$HOME/codec2/build_linux ..
```
2. List available tests:
```
ctest -N
Test project /home/david/FreeDATA/build
Test #1: 000_audio_tests
Test #2: 001_highsnr_stdio_audio
Total Tests: 2
```
3. Run tests:
```
ctest --output-on-failure
```
or, to include only GitHub-compatible tests:
```
GITHUB_RUN_ID=0 ctest --output-on-failure
```
4. Run tests verbosely:
```
ctest -V
```
# 001_HIGHSNR_STDIO_AUDIO TEST SUITE
1. Install
```
sudo apt update
sudo apt upgrade
sudo apt install git cmake build-essential python3-pip portaudio19-dev python3-pyaudio
pip3 install crcengine
pip3 install threading
```
1. Install codec2, and set up the `libcodec2.so` shared library path, for example
```
export LD_LIBRARY_PATH=${HOME}/codec2/build_linux/src
```
## STDIO tests
Pipes are used to move audio samples from the Tx to Rx:
```
python3 util_tx.py --mode datac1 --delay 500 --frames 2 --bursts 1 | python3 util_rx.py --mode datac1 --frames 2 --bursts 1
```
## Moderate signal-to-noise ratio (SNR)
Tests need to be written that test a low SNR data path so that the TNC performance when packets are lost can be evaluated.
## AUDIO test via virtual audio devices
### Important:
The virtual audio devices are great for testing, but they are also a little tricky to handle. So there's a high chance, the tests will fail, if you are running them via virtual audio devices. You should run the tests several times, while keeping this in mind. Most time the ctest is working even if it is failing.
1. Create virtual audio devices. Note: This command needs to be run again after every reboot
```
sudo modprobe snd-aloop index=1,2 enable=1,1 pcm_substreams=1,1 id=CHAT1,CHAT2
```
1. Check if devices have been created
```
aplay -l
Karte 0: Intel [HDA Intel], Gerät 0: Generic Analog [Generic Analog]
Sub-Geräte: 1/1
Sub-Gerät #0: subdevice #0
Karte 1: CHAT1 [Loopback], Gerät 0: Loopback PCM [Loopback PCM]
Sub-Geräte: 1/1
Sub-Gerät #0: subdevice #0
Karte 1: CHAT1 [Loopback], Gerät 1: Loopback PCM [Loopback PCM]
Sub-Geräte: 1/1
Sub-Gerät #0: subdevice #0
Karte 2: CHAT2 [Loopback], Gerät 0: Loopback PCM [Loopback PCM]
Sub-Geräte: 1/1
Sub-Gerät #0: subdevice #0
Karte 2: CHAT2 [Loopback], Gerät 1: Loopback PCM [Loopback PCM]
Sub-Geräte: 1/1
Sub-Gerät #0: subdevice #0
```
1. Determine the audio device number you would like to use:
```
python3 util_rx.py --list
<snip>
audiodev: 0 HDA Intel PCH: ALC269VC Analog (hw:0,0)
audiodev: 1 HDA Intel PCH: HDMI 0 (hw:0,3)
audiodev: 2 HDA Intel PCH: HDMI 1 (hw:0,7)
audiodev: 3 HDA Intel PCH: HDMI 2 (hw:0,8)
audiodev: 4 Loopback: PCM (hw:1,0)
audiodev: 5 Loopback: PCM (hw:1,1)
audiodev: 6 Loopback: PCM (hw:2,0)
audiodev: 7 Loopback: PCM (hw:2,1)
```
In this case we choose audiodev 4 for the RX and 5 for the Tx.
1. Start the Rx first, then Tx in separate consoles:
```
python3 util_rx.py --mode datac0 --frames 2 --bursts 1 --audiodev 4 --debug
python3 util_tx.py --mode datac0 --frames 2 --bursts 1 --audiodev 5
```

121
modem/data cemetery/test/hamlib-test.py
View file

@ -1,121 +0,0 @@
#!/usr/bin/env python3
# -*- coding: utf-8 -*-
import ctypes
from ctypes import *
import pathlib
from enum import Enum
class DEBUGLEVEL(Enum):
RIG_DEBUG_NONE = 0
RIG_DEBUG_BUG = 1
RIG_DEBUG_ERR = 2
RIG_DEBUG_WARN = 3
RIG_DEBUG_VERBOSE = 4
RIG_DEBUG_TRACE = 5
RIG_DEBUG_CACHE = 6
class RETCODE(Enum):
RIG_OK = 0
RIG_EINVAL = 1
RIG_ECONF = 2
RIG_ENOMEM = 3
RIG_ENIMPL = 4
RIG_ETIMEOUT = 5
RIG_EIO = 6
RIG_EINTERNAL = 7
RIG_EPROTO = 8
RIG_ERJCTED = 9
RIG_ETRUNC = 10
RIG_ENAVAIL = 11
RIG_ENTARGET = 12
RIG_BUSERROR = 13
RIG_BUSBUSY = 14
RIG_EARG = 15
RIG_EVFO = 16
RIG_EDOM = 17
libname = pathlib.Path("../modem/lib/hamlib/linux/libhamlib.so")
hamlib = ctypes.CDLL(libname)
class SERIAL(ctypes.Structure):
_fields_ = [
("data_bits", ctypes.c_int),
("stop_bits", ctypes.c_int),
("rate", ctypes.c_int),
("parity", ctypes.c_int),
("handshake", ctypes.c_void_p),
]
class PARM(ctypes.Structure):
_fields_ = [
("serial", SERIAL),
]
class TYPE(ctypes.Structure):
_fields_ = [
("rig", ctypes.c_void_p),
]
class MYPORT(ctypes.Structure):
_fields_ = [
("pathname", ctypes.c_char),
("model", ctypes.c_int),
("parm", PARM),
("type", TYPE),
]
hamlib.rig_set_debug(9) # 6
myrig_model = 3085 # 3085 = ICOM 6 = DUMMY
myport = MYPORT()
myport.parm.serial.data_bits = 7
myport.parm.serial.stop_bits = 2
myport.parm.serial.rate = 9600
rig = hamlib.rig_init(myrig_model)
retcode = hamlib.rig_set_parm(rig, 'stop_bits', 5)
print(retcode)
'''
parameter = create_string_buffer(16)
retcode = hamlib.rig_get_parm(rig, 0, parameter)
print(retcode)
print(bytes(parameter))
'''
# attempt to access global vars. Maybe we can access structures as well?
# https://github.com/Hamlib/Hamlib/blob/f5b229f9dc4b4364d2f40e0b0b415e92c9a371ce/src/rig.c#L95
hamlib_version = ctypes.cast(hamlib.hamlib_version, ctypes.POINTER(ctypes.c_char*21))
print(hamlib_version.contents.value)
'''
retcode = hamlib.rig_has_get_parm(rig, 7)
print(retcode)
'''
'''
retcode = hamlib.rig_open(rig)
print(retcode)
hamlib.rig_close(rig)
'''
# riginfo = create_string_buffer(1024)
# retcode = hamlib.rig_get_rig_info(rig, riginfo, 1024);
'''
char riginfo[1024];
retcode = rig_get_rig_info(rig, riginfo, sizeof(riginfo));
'''

214
modem/data cemetery/test/ping.py
View file

@ -1,214 +0,0 @@
#!/usr/bin/env python3
# -*- coding: utf-8 -*-
import argparse
import ctypes
import pathlib
import threading
import time
import pyaudio
# --------------------------------------------GET PARAMETER INPUTS
parser = argparse.ArgumentParser(description="Simons TEST TNC")
parser.add_argument("--bursts", dest="N_BURSTS", default=0, type=int)
parser.add_argument("--frames", dest="N_FRAMES_PER_BURST", default=0, type=int)
parser.add_argument("--delay", dest="DELAY_BETWEEN_BURSTS", default=0, type=int)
parser.add_argument("--txmode", dest="FREEDV_TX_MODE", default=0, type=int)
parser.add_argument("--rxmode", dest="FREEDV_RX_MODE", default=0, type=int)
parser.add_argument("--audiooutput", dest="AUDIO_OUTPUT", default=0, type=int)
parser.add_argument("--audioinput", dest="AUDIO_INPUT", default=0, type=int)
parser.add_argument("--debug", dest="DEBUGGING_MODE", action="store_true")
args, _ = parser.parse_known_args()
N_BURSTS = args.N_BURSTS
N_FRAMES_PER_BURST = args.N_FRAMES_PER_BURST
DELAY_BETWEEN_BURSTS = args.DELAY_BETWEEN_BURSTS / 1000
AUDIO_OUTPUT_DEVICE = args.AUDIO_OUTPUT
AUDIO_INPUT_DEVICE = args.AUDIO_INPUT
# 1024 good for mode 6
AUDIO_FRAMES_PER_BUFFER = 2048
MODEM_SAMPLE_RATE = 8000
FREEDV_TX_MODE = args.FREEDV_TX_MODE
FREEDV_RX_MODE = args.FREEDV_RX_MODE
DEBUGGING_MODE = args.DEBUGGING_MODE
# -------------------------------------------- LOAD FREEDV
libname = pathlib.Path().absolute() / "codec2/build_linux/src/libcodec2.so"
c_lib = ctypes.CDLL(str(libname))
# --------------------------------------------CREATE PYAUDIO INSTANCE
p = pyaudio.PyAudio()
# --------------------------------------------GET SUPPORTED SAMPLE RATES FROM SOUND DEVICE
# AUDIO_SAMPLE_RATE_TX = int(p.get_device_info_by_index(AUDIO_OUTPUT_DEVICE)['defaultSampleRate'])
# AUDIO_SAMPLE_RATE_RX = int(p.get_device_info_by_index(AUDIO_INPUT_DEVICE)['defaultSampleRate'])
AUDIO_SAMPLE_RATE_TX = 8000
AUDIO_SAMPLE_RATE_RX = 8000
# --------------------------------------------OPEN AUDIO CHANNEL TX
stream_tx = p.open(
format=pyaudio.paInt16,
channels=1,
rate=AUDIO_SAMPLE_RATE_TX,
frames_per_buffer=AUDIO_FRAMES_PER_BUFFER, # n_nom_modem_samples
output=True,
output_device_index=AUDIO_OUTPUT_DEVICE,
)
stream_rx = p.open(
format=pyaudio.paInt16,
channels=1,
rate=AUDIO_SAMPLE_RATE_RX,
frames_per_buffer=AUDIO_FRAMES_PER_BUFFER,
input=True,
input_device_index=AUDIO_INPUT_DEVICE,
)
def receive():
c_lib.freedv_open.restype = ctypes.POINTER(ctypes.c_ubyte)
freedv = c_lib.freedv_open(FREEDV_RX_MODE)
bytes_per_frame = int(c_lib.freedv_get_bits_per_modem_frame(freedv) / 8)
payload_per_frame = bytes_per_frame - 2
n_nom_modem_samples = c_lib.freedv_get_n_nom_modem_samples(freedv)
n_tx_modem_samples = c_lib.freedv_get_n_tx_modem_samples(
freedv
) # get n_tx_modem_samples which defines the size of the modulation object # --> *2
bytes_out = ctypes.c_ubyte * bytes_per_frame # bytes_per_frame
bytes_out = bytes_out() # get pointer from bytes_out
rx_total_frames = 0
rx_frames = 0
rx_bursts = 0
receive = True
total_n_bytes = 0
while receive:
time.sleep(0.01)
nin = c_lib.freedv_nin(freedv)
nin_converted = int(nin * (AUDIO_SAMPLE_RATE_RX / MODEM_SAMPLE_RATE))
if DEBUGGING_MODE:
print("-----------------------------")
print(f"NIN: {str(nin)} [ {nin_converted} ]")
data_in = stream_rx.read(nin_converted, exception_on_overflow=False)
data_in = data_in.rstrip(b"\x00")
c_lib.freedv_rawdatarx.argtype = [
ctypes.POINTER(ctypes.c_ubyte),
bytes_out,
data_in,
] # check if really neccessary
nbytes = c_lib.freedv_rawdatarx(freedv, bytes_out, data_in) # demodulate audio
total_n_bytes = total_n_bytes + nbytes
if DEBUGGING_MODE:
print(f"SYNC: {str(c_lib.freedv_get_rx_status(freedv))}")
if nbytes == bytes_per_frame:
rx_total_frames = rx_total_frames + 1
rx_frames = rx_frames + 1
if rx_frames == N_FRAMES_PER_BURST:
rx_frames = 0
rx_bursts = rx_bursts + 1
c_lib.freedv_set_sync(freedv, 0)
burst = bytes_out[0]
n_total_burst = bytes_out[1]
frame = bytes_out[2]
n_total_frame = bytes_out[3]
print(
f"RX | PONG | BURST [{str(burst)}/{str(n_total_burst)}] FRAME [{str(frame)}/{str(n_total_frame)}]"
)
print("-----------------------------------------------------------------")
c_lib.freedv_set_sync(freedv, 0)
if rx_bursts == N_BURSTS:
receive = False
RECEIVE = threading.Thread(target=receive, name="RECEIVE THREAD")
RECEIVE.start()
c_lib.freedv_open.restype = ctypes.POINTER(ctypes.c_ubyte)
freedv = c_lib.freedv_open(FREEDV_TX_MODE)
bytes_per_frame = int(c_lib.freedv_get_bits_per_modem_frame(freedv) / 8)
n_nom_modem_samples = c_lib.freedv_get_n_nom_modem_samples(freedv)
n_tx_modem_samples = c_lib.freedv_get_n_tx_modem_samples(
freedv
) # get n_tx_modem_samples which defines the size of the modulation object # --> *2
mod_out = ctypes.c_short * n_tx_modem_samples
mod_out = mod_out()
mod_out_preamble = ctypes.c_short * (
1760 * 2
) # 1760 for mode 10,11,12 #4000 for mode 9
mod_out_preamble = mod_out_preamble()
print(f"BURSTS: {str(N_BURSTS)} FRAMES: {str(N_FRAMES_PER_BURST)}")
print("-----------------------------------------------------------------")
payload_per_frame = bytes_per_frame - 2
for i in range(N_BURSTS):
c_lib.freedv_rawdatapreambletx(freedv, mod_out_preamble)
txbuffer = bytearray()
txbuffer += bytes(mod_out_preamble)
for n in range(N_FRAMES_PER_BURST):
data_out = bytearray()
data_out += bytes([i + 1])
data_out += bytes([N_BURSTS])
data_out += bytes([n + 1])
data_out += bytes([N_FRAMES_PER_BURST])
buffer = bytearray(
payload_per_frame
) # use this if CRC16 checksum is required ( DATA1-3)
buffer[
: len(data_out)
] = data_out # set buffer size to length of data which will be sent
crc = ctypes.c_ushort(
c_lib.freedv_gen_crc16(bytes(buffer), payload_per_frame)
) # generate CRC16
crc = crc.value.to_bytes(2, byteorder="big") # convert crc to 2 byte hex string
buffer += crc # append crc16 to buffer
data = (ctypes.c_ubyte * bytes_per_frame).from_buffer_copy(buffer)
c_lib.freedv_rawdatatx(
freedv, mod_out, data
) # modulate DATA and safe it into mod_out pointer
txbuffer += bytes(mod_out)
print(
f"TX | PING | BURST [{str(i + 1)}/{str(N_BURSTS)}] FRAME [{str(n + 1)}/{str(N_FRAMES_PER_BURST)}]"
)
stream_tx.write(bytes(txbuffer))
ACK_TIMEOUT = time.time() + 3
txbuffer = bytearray()
# time.sleep(DELAY_BETWEEN_BURSTS)
# WAIT UNTIL WE RECEIVD AN ACK/datac13 FRAME
while ACK_TIMEOUT >= time.time():
time.sleep(0.01)
time.sleep(1)
stream_tx.close()
p.terminate()

166
modem/data cemetery/test/pong.py
View file

@ -1,166 +0,0 @@
#!/usr/bin/env python3
# -*- coding: utf-8 -*-
"""
Created on Wed Dec 23 07:04:24 2020
@author: DJ2LS
"""
import ctypes
from ctypes import *
import pathlib
import pyaudio
import sys
import logging
import time
import threading
import sys
import argparse
# --------------------------------------------GET PARAMETER INPUTS
parser = argparse.ArgumentParser(description='Simons TEST TNC')
parser.add_argument('--bursts', dest="N_BURSTS", default=0, type=int)
parser.add_argument('--frames', dest="N_FRAMES_PER_BURST", default=0, type=int)
parser.add_argument('--txmode', dest="FREEDV_TX_MODE", default=0, type=int)
parser.add_argument('--rxmode', dest="FREEDV_RX_MODE", default=0, type=int)
parser.add_argument('--audioinput', dest="AUDIO_INPUT", default=0, type=int)
parser.add_argument('--audiooutput', dest="AUDIO_OUTPUT", default=0, type=int)
parser.add_argument('--debug', dest="DEBUGGING_MODE", action="store_true")
args, _ = parser.parse_known_args()
N_BURSTS = args.N_BURSTS
N_FRAMES_PER_BURST = args.N_FRAMES_PER_BURST
AUDIO_OUTPUT_DEVICE = args.AUDIO_OUTPUT
AUDIO_INPUT_DEVICE = args.AUDIO_INPUT
FREEDV_TX_MODE = args.FREEDV_TX_MODE
FREEDV_RX_MODE = args.FREEDV_RX_MODE
DEBUGGING_MODE = args.DEBUGGING_MODE
# 1024 good for mode 6
AUDIO_FRAMES_PER_BUFFER = 2048
MODEM_SAMPLE_RATE = 8000
# -------------------------------------------- LOAD FREEDV
libname = pathlib.Path().absolute() / "codec2/build_linux/src/libcodec2.so"
c_lib = ctypes.CDLL(libname)
# --------------------------------------------CREATE PYAUDIO INSTANCE
p = pyaudio.PyAudio()
# --------------------------------------------GET SUPPORTED SAMPLE RATES FROM SOUND DEVICE
# AUDIO_SAMPLE_RATE_TX = int(p.get_device_info_by_index(AUDIO_OUTPUT_DEVICE)['defaultSampleRate'])
# AUDIO_SAMPLE_RATE_RX = int(p.get_device_info_by_index(AUDIO_INPUT_DEVICE)['defaultSampleRate'])
AUDIO_SAMPLE_RATE_TX = 8000
AUDIO_SAMPLE_RATE_RX = 8000
# --------------------------------------------OPEN AUDIO CHANNEL RX
stream_tx = p.open(format=pyaudio.paInt16,
channels=1,
rate=AUDIO_SAMPLE_RATE_TX,
frames_per_buffer=AUDIO_FRAMES_PER_BUFFER, # n_nom_modem_samples
output=True,
output_device_index=AUDIO_OUTPUT_DEVICE,
)
stream_rx = p.open(format=pyaudio.paInt16,
channels=1,
rate=AUDIO_SAMPLE_RATE_RX,
frames_per_buffer=AUDIO_FRAMES_PER_BUFFER,
input=True,
input_device_index=AUDIO_INPUT_DEVICE,
)
# GENERAL PARAMETERS
c_lib.freedv_open.restype = ctypes.POINTER(ctypes.c_ubyte)
def send_pong(burst,n_total_burst,frame,n_total_frame):
data_out = bytearray()
data_out[:1] = bytes([burst])
data_out[1:2] = bytes([n_total_burst])
data_out[2:3] = bytes([frame])
data_out[4:5] = bytes([n_total_frame])
c_lib.freedv_open.restype = ctypes.POINTER(ctypes.c_ubyte)
freedv = c_lib.freedv_open(FREEDV_TX_MODE)
bytes_per_frame = int(c_lib.freedv_get_bits_per_modem_frame(freedv)/8)
payload_per_frame = bytes_per_frame -2
n_nom_modem_samples = c_lib.freedv_get_n_nom_modem_samples(freedv)
n_tx_modem_samples = c_lib.freedv_get_n_tx_modem_samples(freedv) # get n_tx_modem_samples which defines the size of the modulation object # --> *2
mod_out = ctypes.c_short * n_tx_modem_samples
mod_out = mod_out()
mod_out_preamble = ctypes.c_short * (1760*2) # 1760 for mode 10,11,12 #4000 for mode 9
mod_out_preamble = mod_out_preamble()
buffer = bytearray(payload_per_frame) # use this if CRC16 checksum is required ( DATA1-3)
buffer[:len(data_out)] = data_out # set buffer size to length of data which will be sent
crc = ctypes.c_ushort(c_lib.freedv_gen_crc16(bytes(buffer), payload_per_frame)) # generate CRC16
crc = crc.value.to_bytes(2, byteorder='big') # convert crc to 2 byte hex string
buffer += crc # append crc16 to buffer
c_lib.freedv_rawdatapreambletx(freedv, mod_out_preamble);
txbuffer = bytearray()
txbuffer += bytes(mod_out_preamble)
data = (ctypes.c_ubyte * bytes_per_frame).from_buffer_copy(buffer)
c_lib.freedv_rawdatatx(freedv,mod_out,data) # modulate DATA and safe it into mod_out pointer
txbuffer += bytes(mod_out)
stream_tx.write(bytes(txbuffer))
txbuffer = bytearray()
# DATA CHANNEL INITIALISATION
freedv = c_lib.freedv_open(FREEDV_RX_MODE)
bytes_per_frame = int(c_lib.freedv_get_bits_per_modem_frame(freedv)/8)
n_max_modem_samples = c_lib.freedv_get_n_max_modem_samples(freedv)
bytes_out = (ctypes.c_ubyte * bytes_per_frame) # bytes_per_frame
bytes_out = bytes_out() # get pointer from bytes_out
receive = True
while receive:
time.sleep(0.01)
data_in = b''
nin = c_lib.freedv_nin(freedv)
nin_converted = int(nin*(AUDIO_SAMPLE_RATE_RX/MODEM_SAMPLE_RATE))
if DEBUGGING_MODE:
print("-----------------------------")
print(f"NIN: {str(nin)} [ {nin_converted} ]")
data_in = stream_rx.read(nin_converted, exception_on_overflow = False)
data_in = data_in.rstrip(b'\x00')
c_lib.freedv_rawdatarx.argtype = [ctypes.POINTER(ctypes.c_ubyte), bytes_out, data_in] # check if really neccessary
nbytes = c_lib.freedv_rawdatarx(freedv, bytes_out, data_in) # demodulate audio
if DEBUGGING_MODE:
print(f"SYNC: {str(c_lib.freedv_get_rx_status(freedv))}")
if nbytes == bytes_per_frame:
burst = bytes_out[0]
n_total_burst = bytes_out[1]
frame = bytes_out[2]
n_total_frame = bytes_out[3]
print(
f"RX | BURST [{str(burst)}/{str(n_total_burst)}] FRAME [{str(frame)}/{str(n_total_frame)}] >>> SENDING PONG"
)
TRANSMIT_PONG = threading.Thread(target=send_pong, args=[burst,n_total_burst,frame,n_total_frame], name="SEND PONG")
TRANSMIT_PONG.start()
c_lib.freedv_set_sync(freedv,0)

88
modem/data cemetery/test/test_audiobuffer.py
View file

@ -1,88 +0,0 @@
#!/usr/bin/env python3
# -*- coding: utf-8 -*-
#
# tests audio buffer thread safety
# pylint: disable=global-statement, invalid-name
import sys
import threading
from time import sleep
import codec2
import numpy as np
import pytest
BUFFER_SZ = 1024
N_MAX = 100 # write a repeating sequence of 0....N_MAX-1
WRITE_SZ = 10 # different read and write sized buffers
READ_SZ = 8
NTESTS = 10000
running = True
audio_buffer = codec2.audio_buffer(BUFFER_SZ)
n_write = 0
def t_writer():
"""
Subprocess to handle writes to the NumPY audio "device."
"""
global n_write
print("writer starting")
n = 0
buf = np.zeros(WRITE_SZ, dtype=np.int16)
while running:
nfree = audio_buffer.size - audio_buffer.nbuffer
if nfree >= WRITE_SZ:
for index in range(WRITE_SZ):
buf[index] = n
n += 1
if n == N_MAX:
n = 0
n_write += 1
audio_buffer.push(buf)
def test_audiobuffer():
"""
Test for the audiobuffer
"""
global running
# Start the writer in a new thread.
writer_thread = threading.Thread(target=t_writer)
writer_thread.start()
n_out = n_read = errors = 0
for _ in range(NTESTS):
while audio_buffer.nbuffer < READ_SZ:
sleep(0.001)
for i in range(READ_SZ):
if audio_buffer.buffer[i] != n_out:
errors += 1
n_out += 1
if n_out == N_MAX:
n_out = 0
n_read += 1
audio_buffer.pop(READ_SZ)
print(f"n_write: {n_write} n_read: {n_read} errors: {errors} ")
# Indirectly stop the thread
running = False
sleep(0.1)
assert not writer_thread.is_alive()
assert n_write - n_read < BUFFER_SZ
assert errors == 0
if __name__ == "__main__":
# Run pytest with the current script as the filename.
ecode = pytest.main(["-v", sys.argv[0]])
if ecode == 0:
print("errors: 0")
else:
print(ecode)

237
modem/data cemetery/test/test_chat_text.py
View file

@ -1,237 +0,0 @@
#!/usr/bin/env python3
# -*- coding: utf-8 -*-
"""
Test small (single-frame) and large (multi-frame) messages over a high quality
simulated audio channel.
Near end-to-end test for sending / receiving data through the TNC and modem
and back through on the other station. Data injection initiates from the
queue used by the daemon process into and out of the TNC. Tests both low- and
high-bandwidth data frames (datac3 and datac1 respectively) from Codec2.
Can be invoked from CMake, pytest, coverage or directly.
Uses util_chat_test_[12].py in separate processes to perform the data transfer.
@author: N2KIQ
"""
import contextlib
import multiprocessing
import os
import sys
import threading
import time
import zlib
import helpers
import log_handler
import pytest
import structlog
try:
import test.util_chat_text_1 as util1
import test.util_chat_text_2 as util2
except ImportError:
import util_chat_text_1 as util1
import util_chat_text_2 as util2
STATIONS = ["AA2BB", "ZZ9YY"]
bytes_out = b'{"dt":"f","fn":"zeit.txt","ft":"text\\/plain","d":"data:text\\/plain;base64,MyBtb2Rlcywgb2huZSBjbGFzcwowLjAwMDk2OTQ4MTE4MDk5MTg0MTcKCjIgbW9kZXMsIG9obmUgY2xhc3MKMC4wMDA5NjY1NDUxODkxMjI1Mzk0CgoxIG1vZGUsIG9obmUgY2xhc3MKMC4wMDA5NjY5NzY1NTU4Nzc4MjA5CgMyBtb2Rlcywgb2huZSBjbGFzcwowLjAwMDk2OTQ4MTE4MDk5MTg0MTcKCjIgbW9kZXMsIG9obmUgY2xhc3MKMC4wMDA5NjY1NDUxODkxMjI1Mzk0CgoxIG1vZGUsIG9obmUgY2xhc3MKMC4wMDA5NjY5NzY1NTU4Nzc4MjA5Cg=MyBtb2Rlcywgb2huZSBjbGFzcwowLjAwMDk2OTQ4MTE4MDk5MTg0MTcKCjIgbW9kZXMsIG9obmUgY2xhc3MKMC4wMDA5NjY1NDUxODkxMjI1Mzk0CgoxIG1vZGUsIG9obmUgY2xhc3MKMC4wMDA5NjY5NzY1NTU4Nzc4MjA5CgMyBtb2Rlcywgb2huZSBjbGFzcwowLjAwMDk2OTQ4MTE4MDk5MTg0MTcKCjIgbW9kZXMsIG9obmUgY2xhc3MKMC4wMDA5NjY1NDUxODkxMjI1Mzk0CgoxIG1vZGUsIG9obmUgY2xhc3MKMC4wMDA5NjY5NzY1NTU4Nzc4MjA5CgMyBtb2Rlcywgb2huZSBjbGFzcwowLjAwMDk2OTQ4MTE4MDk5MTg0MTcKCjIgbW9kZXMsIG9obmUgY2xhc3MKMC4wMDA5NjY1NDUxODkxMjI1Mzk0CgoxIG1vZGUsIG9obmUgY2xhc3MKMC4wMDA5NjY5NzY1NTU4Nzc4MjA5Cg=","crc":"123123123"}'
messages = [
"This is a test chat...",
"This is a much longer message, hopefully longer than each of the datac1 and "
"datac3 frames available to use in this modem. This should be long enough, "
"but to err on the side of completeness this will string on for many more "
"words before coming to the long awaited conclusion. We are not at the "
"concluding point just yet because there is still more space to be taken up "
"in the datac3 frame. Perhaps now would be a good place to terminate this test "
"message, but perhaps not because we need a few more bytes. Here then we stop. "
"This compresses so well that I need more data, even more stuff than is already "
"here and included in the unreadable diatribe below, or is it a soliloquy? "
"MyBtb2Rlcywgb2huZSBjbGFzcwowLjAwMDk2OTQ4MTE4MDk5MTg0MTcKCjIgbW9kZXMsIG9obm"
"UgY2xhc3MKMC4wMDA5NjY1NDUxODkxMjI1Mzk0CgoxIG1vZGUsIG9obmUgY2xhc3MKMC4wMDA5"
"NjY5NzY1NTU4Nzc4MjA5CgMyBtb2Rlcywgb2huZSBjbGFzcwowLjAwMDk2OTQ4MTE4MDk5MTg0"
"MTcKCjIgbW9kZXMsIG9obmUgY2xhc3MKMC4wMDA5NjY1NDUxODkxMjI1Mzk0CgoxIG1vZGUsIG"
"9obmUgY2xhc3MKMC4wMDA5NjY5NzY1NTU4Nzc4MjA5Cg=MyBtb2Rlcywgb2huZSBjbGFzcwowL"
"jAwMDk2OTQ4MTE4MDk5MTg0MTcKCjIgbW9kZXMsIG9obmUgY2xhc3MKMC4wMDA5NjY1NDUxODk"
"xMjI1Mzk0CgoxIG1vZGUsIG9obmUgY2xhc3MKMC4wMDA5NjY5NzY1NTU4Nzc4MjA5CgMyBtb2R"
"lcywgb2huZSBjbGFzcwowLjAwMDk2OTQ4MTE4MDk5MTg0MTcKCjIgbW9kZXMsIG9obmUgY2xhc"
"3MKMC4wMDA5NjY1NDUxODkxMjI1Mzk0CgoxIG1vZGUsIG9obmUgY2xhc3MKMC4wMDA5NjY5NzY"
"1NTU4Nzc4MjA5CgMyBtb2Rlcywgb2huZSBjbGFzcwowLjAwMDk2OTQ4MTE4MDk5MTg0MTcKCjI"
"gbW9kZXMsIG9obmUgY2xhc3MKMC4wMDA5NjY1NDUxODkxMjI1Mzk0CgoxIG1vZGUsIG9obmUgY"
"2xhc3MKMC4wMDA5NjY5NzY1NTU4Nzc4MjA5Cg=",
]
PIPE_THREAD_RUNNING = True
def locate_data_with_crc(source_list: list, text: str, data: bytes, frametype: str):
"""Try to locate data in source_list."""
log = structlog.get_logger("locate_data_with_crc")
if data in source_list:
with contextlib.suppress():
data = zlib.decompress(data[2:])
log.info(f"analyze_results: {text} no CRC", _frametype=frametype, data=data)
elif data + helpers.get_crc_8(data) in source_list:
with contextlib.suppress(zlib.error):
data = zlib.decompress(data[2:-1])
log.info(f"analyze_results: {text} CRC 8", _frametype=frametype, data=data)
elif data + helpers.get_crc_16(data) in source_list:
with contextlib.suppress(zlib.error):
data = zlib.decompress(data[2:-2])
log.info(f"analyze_results: {text} CRC16", _frametype=frametype, data=data)
elif data + helpers.get_crc_24(data) in source_list:
with contextlib.suppress(zlib.error):
data = zlib.decompress(data[2:-3])
log.info(f"analyze_results: {text} CRC24", _frametype=frametype, data=data)
elif data + helpers.get_crc_32(data) in source_list:
with contextlib.suppress(zlib.error):
data = zlib.decompress(data[2:-4])
log.info(f"analyze_results: {text} CRC32", _frametype=frametype, data=data)
else:
log.info(
f"analyze_results: {text} not received:",
_frame=frametype,
data=data,
)
def analyze_results(station1: list, station2: list, call_list: list):
"""Examine the information retrieved from the sub-processes."""
# Data in these lists is either a series of bytes of received data,
# or a bytearray of transmitted data from the station.
log = structlog.get_logger("analyze_results")
# Check that each station's transmitted data was received by the other.
for s1, s2, text in [
(station1, station2, call_list[0]),
(station2, station1, call_list[1]),
]:
for s1_item in s1:
if not isinstance(s1_item, list):
continue
data = bytes(s1_item[0])
frametypeno = int.from_bytes(data[:1], "big")
# frametype = static.FRAME_TYPE(frametypeno).name
frametype = str(frametypeno)
s1_crc = helpers.decode_call(helpers.bytes_to_callsign(data[1:4]))
s2_crc = helpers.decode_call(helpers.bytes_to_callsign(data[2:5]))
log.info(
"analyze_results: callsign CRCs:",
tx_station=text,
s1_crc=s1_crc,
s2_crc=s2_crc,
)
locate_data_with_crc(s2, text, data, frametype)
@pytest.mark.parametrize("freedv_mode", ["datac1", "datac3"])
@pytest.mark.parametrize("n_frames_per_burst", [1]) # Higher fpb is broken.
@pytest.mark.parametrize("message_no", range(len(messages)))
@pytest.mark.flaky(reruns=3)
def test_chat_text(
freedv_mode: str, n_frames_per_burst: int, message_no: int, tmp_path
):
log_handler.setup_logging(filename=tmp_path / "test_chat_text", level="INFO")
log = structlog.get_logger("test_chat_text")
s1_data = []
s2_data = []
def recv_data(buffer: list, pipe):
while PIPE_THREAD_RUNNING:
if pipe.poll(0.1):
buffer.append(pipe.recv())
else:
time.sleep(0.1)
def recv_from_pipes(s1_rx, s1_pipe, s2_rx, s2_pipe) -> list:
processes = [
threading.Thread(target=recv_data, args=(s1_rx, s1_pipe)),
threading.Thread(target=recv_data, args=(s2_rx, s2_pipe)),
]
for item in processes:
item.start()
return processes
# This sufficiently separates the two halves of the test. This is needed
# because both scripts change global state. They would conflict if running in
# the same process.
from_s1, s1_send = multiprocessing.Pipe()
from_s2, s2_send = multiprocessing.Pipe()
proc = [
multiprocessing.Process(
target=util1.t_highsnr_arq_short_station1,
args=(
s1_send,
freedv_mode,
n_frames_per_burst,
STATIONS[0],
STATIONS[1],
messages[message_no],
freedv_mode == "datac3", # == low bandwidth mode
tmp_path,
),
daemon=True,
),
multiprocessing.Process(
target=util2.t_highsnr_arq_short_station2,
args=(
s2_send,
freedv_mode,
n_frames_per_burst,
STATIONS[1],
STATIONS[0],
messages[message_no],
freedv_mode == "datac3", # == low bandwidth mode
tmp_path,
),
daemon=True,
),
]
pipe_receivers = recv_from_pipes(s1_data, from_s1, s2_data, from_s2)
# log.debug("Creating ")
# print("Starting threads.")
for p_item in proc:
p_item.start()
# This relies on each process exiting when its job is complete!
# print("Waiting for threads to exit.")
for p_item in proc:
p_item.join()
global PIPE_THREAD_RUNNING # pylint: disable=global-statement
PIPE_THREAD_RUNNING = False
for pipe_recv in pipe_receivers:
pipe_recv.join()
for idx in range(2):
try:
os.unlink(tmp_path / f"hfchannel{idx+1}")
except FileNotFoundError as fnfe:
log.debug(f"Unlinking pipe: {fnfe}")
for p_item in proc:
assert p_item.exitcode == 0
# p_item.close() # Python 3.7+ only
p_item.terminate()
p_item.join()
analyze_results(s1_data, s2_data, STATIONS)
if __name__ == "__main__":
# Run pytest with the current script as the filename.
ecode = pytest.main(["-s", "-v", sys.argv[0]])
if ecode == 0:
print("errors: 0")
else:
print(ecode)

291
modem/data cemetery/test/test_datac13.py
View file

@ -1,291 +0,0 @@
#!/usr/bin/env python3
# -*- coding: utf-8 -*-
"""
Test control frame commands over a high quality simulated audio channel.
Near end-to-end test for sending / receiving select control frames through the
TNC and modem and back through on the other station. Data injection initiates from the
queue used by the daemon process into and out of the TNC.
Can be invoked from CMake, pytest, coverage or directly.
Uses util_datac13.py in separate process to perform the data transfer.
@author: N2KIQ
"""
import contextlib
import multiprocessing
import os
import sys
import threading
import time
import zlib
import helpers
import log_handler
import pytest
import structlog
try:
import test.util_datac13 as util
except ImportError:
import util_datac13 as util
STATIONS = ["AA2BB", "ZZ9YY"]
PIPE_THREAD_RUNNING = True
def parameters() -> dict:
# Construct message to start beacon.
beacon_data = {"type": "command", "command": "start_beacon", "parameter": "5"}
# Construct message to start cq.
cq_data = {"type": "command", "command": "cqcqcq"}
# Construct message to start ping.
ping_data = {"type": "ping", "command": "ping", "dxcallsign": "ZZ9YY-0"}
connect_data = {"type": "arq", "command": "connect", "dxcallsign": "ZZ9YY-0"}
stop_data = {"type": "arq", "command": "stop_transmission", "dxcallsign": "ZZ9YY-0"}
beacon_timeout = 1
ping_timeout = 1
cq_timeout = 1
connect_timeout = 1
stop_timeout = 1
beacon_tx_check = '"beacon":"transmitting"'
cq_tx_check = '"qrv":"received"'
ping_tx_check = '"ping":"transmitting"'
connect_tx_check = '"session":"connecting"'
stop_tx_check = '"status":"stopped"'
beacon_rx_check = '"beacon":"received"'
cq_rx_check = '"cq":"received"'
ping_rx_check = '"ping":"received"'
connect_rx_check = '"connect":"received"'
stop_rx_check = '"status":"stopped"'
beacon_final_tx_check = [beacon_tx_check]
cq_final_tx_check = ['"cq":"transmitting"', cq_tx_check]
ping_final_tx_check = [ping_tx_check, '"ping":"acknowledge"']
connect_final_tx_check = ['"status":"connected"', '"connect":"acknowledge"']
stop_final_tx_check = [stop_tx_check]
beacon_final_rx_check = [beacon_rx_check]
cq_final_rx_check = [cq_rx_check, '"qrv":"transmitting"']
ping_final_rx_check = [ping_rx_check]
connect_final_rx_check = [connect_rx_check]
stop_final_rx_check = [stop_rx_check]
return {
"beacon": (
beacon_data,
beacon_timeout,
beacon_tx_check,
beacon_rx_check,
beacon_final_tx_check,
beacon_final_rx_check,
),
"connect": (
connect_data,
connect_timeout,
connect_tx_check,
connect_rx_check,
connect_final_tx_check,
connect_final_rx_check,
),
"cq": (
cq_data,
cq_timeout,
cq_tx_check,
cq_rx_check,
cq_final_tx_check,
cq_final_rx_check,
),
"ping": (
ping_data,
ping_timeout,
ping_tx_check,
ping_rx_check,
ping_final_tx_check,
ping_final_rx_check,
),
"stop": (
stop_data,
stop_timeout,
stop_tx_check,
stop_rx_check,
stop_final_tx_check,
stop_final_rx_check,
),
}
def locate_data_with_crc(source_list: list, text: str, data: bytes, frametype: str):
"""Try to locate data in source_list."""
log = structlog.get_logger("locate_data_with_crc")
if data in source_list:
with contextlib.suppress():
data = zlib.decompress(data[2:])
log.info(f"analyze_results: {text} no CRC", _frametype=frametype, data=data)
elif data + helpers.get_crc_8(data) in source_list:
with contextlib.suppress(zlib.error):
data = zlib.decompress(data[2:-1])
log.info(f"analyze_results: {text} CRC 8", _frametype=frametype, data=data)
elif data + helpers.get_crc_16(data) in source_list:
with contextlib.suppress(zlib.error):
data = zlib.decompress(data[2:-2])
log.info(f"analyze_results: {text} CRC16", _frametype=frametype, data=data)
elif data + helpers.get_crc_24(data) in source_list:
with contextlib.suppress(zlib.error):
data = zlib.decompress(data[2:-3])
log.info(f"analyze_results: {text} CRC24", _frametype=frametype, data=data)
elif data + helpers.get_crc_32(data) in source_list:
with contextlib.suppress(zlib.error):
data = zlib.decompress(data[2:-4])
log.info(f"analyze_results: {text} CRC32", _frametype=frametype, data=data)
else:
log.info(
f"analyze_results: {text} not received:",
_frame=frametype,
data=data,
)
def analyze_results(station1: list, station2: list, call_list: list):
"""Examine the information retrieved from the sub-processes."""
# Data in these lists is either a series of bytes of received data,
# or a bytearray of transmitted data from the station.
log = structlog.get_logger("analyze_results")
# Check that each station's transmitted data was received by the other.
for s1, s2, text in [
(station1, station2, call_list[0]),
(station2, station1, call_list[1]),
]:
for s1_item in s1:
if not isinstance(s1_item, list):
continue
data = bytes(s1_item[0])
frametypeno = int.from_bytes(data[:1], "big")
# frametype = static.FRAME_TYPE(frametypeno).name
frametype = str(frametypeno)
s1_crc = helpers.decode_call(helpers.bytes_to_callsign(data[1:4]))
s2_crc = helpers.decode_call(helpers.bytes_to_callsign(data[2:5]))
log.info(
"analyze_results: callsign CRCs:",
tx_station=text,
s1_crc=s1_crc,
s2_crc=s2_crc,
)
locate_data_with_crc(s2, text, data, frametype)
# frame_type "connect" doesn't work 2022-Jun-16. Missing / incomplete SOCKET_QUEUE data.
# frame_type "cq" is overly flaky. Can't get the timing right / FIFO not delivering data.
@pytest.mark.parametrize(
"frame_type",
[
pytest.param("beacon", marks=pytest.mark.flaky(reruns=2)),
pytest.param("ping", marks=pytest.mark.flaky(reruns=2)),
# FIXME: pytest.param("cq", marks=pytest.mark.flaky(reruns=20)),
#pytest.param("cq", marks=pytest.mark.xfail(reason="Too unstable for CI")),
pytest.param("stop", marks=pytest.mark.flaky(reruns=2)),
],
)
def test_datac13(frame_type: str, tmp_path):
log_handler.setup_logging(filename=tmp_path / "test_datac13", level="DEBUG")
log = structlog.get_logger("test_datac13")
s1_data = []
s2_data = []
def recv_data(buffer: list, pipe):
while PIPE_THREAD_RUNNING:
if pipe.poll(0.1):
buffer.append(pipe.recv())
else:
time.sleep(0.1)
def recv_from_pipes(s1_rx, s1_pipe, s2_rx, s2_pipe) -> list:
processes = [
threading.Thread(target=recv_data, args=(s1_rx, s1_pipe)),
threading.Thread(target=recv_data, args=(s2_rx, s2_pipe)),
]
for item in processes:
item.start()
return processes
# This sufficiently separates the two halves of the test. This is needed
# because both scripts change global state. They would conflict if running in
# the same process.
from_s1, s1_send = multiprocessing.Pipe()
from_s2, s2_send = multiprocessing.Pipe()
proc = [
multiprocessing.Process(
target=util.t_datac13_1,
args=(
s1_send,
STATIONS[0],
STATIONS[1],
parameters()[frame_type],
tmp_path,
),
daemon=True,
),
multiprocessing.Process(
target=util.t_datac13_2,
args=(
s2_send,
STATIONS[1],
STATIONS[0],
parameters()[frame_type],
tmp_path,
),
daemon=True,
),
]
pipe_receivers = recv_from_pipes(s1_data, from_s1, s2_data, from_s2)
# log.debug("Creating ")
# print("Starting threads.")
for p_item in proc:
p_item.start()
# This relies on each process exiting when its job is complete!
# print("Waiting for threads to exit.")
for p_item in proc:
p_item.join()
global PIPE_THREAD_RUNNING # pylint: disable=global-statement
PIPE_THREAD_RUNNING = False
for pipe_recv in pipe_receivers:
pipe_recv.join()
for idx in range(2):
try:
os.unlink(tmp_path / f"hfchannel{idx+1}")
except FileNotFoundError as fnfe:
log.debug(f"Unlinking pipe: {fnfe}")
for p_item in proc:
assert p_item.exitcode == 0
# p_item.close() # Python 3.7+ only
p_item.terminate()
p_item.join()
analyze_results(s1_data, s2_data, STATIONS)
if __name__ == "__main__":
# Run pytest with the current script as the filename.
ecode = pytest.main(["-s", "-v", sys.argv[0]])
if ecode == 0:
print("errors: 0")
else:
print(ecode)

261
modem/data cemetery/test/test_datac13_negative.py
View file

@ -1,261 +0,0 @@
#!/usr/bin/env python3
# -*- coding: utf-8 -*-
"""
Negative tests for datac13 frames.
@author: kronenpj
"""
import contextlib
import multiprocessing
import os
import sys
import threading
import time
import zlib
import helpers
import log_handler
import pytest
import structlog
try:
import test.util_datac13_negative as util
except ImportError:
import util_datac13_negative as util
STATIONS = ["AA2BB", "ZZ9YY"]
PIPE_THREAD_RUNNING = True
def parameters() -> dict:
# Construct message to start beacon.
beacon_data = {"type": "command", "command": "start_beacon", "parameter": "-5"}
# Construct message to start ping.
ping_data = {"type": "ping", "command": "ping", "dxcallsign": ""}
connect_data = {"type": "arq", "command": "connect", "dxcallsign": ""}
stop_data = {"type": "arq", "command": "stop_transmission", "dxcallsign": "DD5GG-3"}
beacon_timeout = 1
ping_timeout = 1
connect_timeout = 1
stop_timeout = 1
beacon_tx_check = '"status":"Failed"'
ping_tx_check = '"ping","status":"Failed"'
connect_tx_check = '"status":"Failed"'
stop_tx_check = '"status":"stopped"'
beacon_rx_check = '"beacon":"received"'
ping_rx_check = '"ping":"received"'
connect_rx_check = '"connect":"received"'
stop_rx_check = '"status":"stopped"'
beacon_final_tx_check = [beacon_tx_check]
ping_final_tx_check = [ping_tx_check]
connect_final_tx_check = [connect_tx_check]
stop_final_tx_check = [stop_tx_check]
beacon_final_rx_check = [beacon_rx_check]
ping_final_rx_check = [ping_rx_check]
connect_final_rx_check = [connect_rx_check]
stop_final_rx_check = [stop_rx_check]
return {
"beacon": (
beacon_data,
beacon_timeout,
beacon_tx_check,
beacon_rx_check,
beacon_final_tx_check,
beacon_final_rx_check,
),
"connect": (
connect_data,
connect_timeout,
connect_tx_check,
connect_rx_check,
connect_final_tx_check,
connect_final_rx_check,
),
"ping": (
ping_data,
ping_timeout,
ping_tx_check,
ping_rx_check,
ping_final_tx_check,
ping_final_rx_check,
),
"stop": (
stop_data,
stop_timeout,
stop_tx_check,
stop_rx_check,
stop_final_tx_check,
stop_final_rx_check,
),
}
def locate_data_with_crc(source_list: list, text: str, data: bytes, frametype: str):
"""Try to locate data in source_list."""
log = structlog.get_logger("locate_data_with_crc")
if data in source_list:
with contextlib.suppress():
data = zlib.decompress(data[2:])
log.info(f"analyze_results: {text} no CRC", _frametype=frametype, data=data)
elif data + helpers.get_crc_8(data) in source_list:
with contextlib.suppress(zlib.error):
data = zlib.decompress(data[2:-1])
log.info(f"analyze_results: {text} CRC 8", _frametype=frametype, data=data)
elif data + helpers.get_crc_16(data) in source_list:
with contextlib.suppress(zlib.error):
data = zlib.decompress(data[2:-2])
log.info(f"analyze_results: {text} CRC16", _frametype=frametype, data=data)
elif data + helpers.get_crc_24(data) in source_list:
with contextlib.suppress(zlib.error):
data = zlib.decompress(data[2:-3])
log.info(f"analyze_results: {text} CRC24", _frametype=frametype, data=data)
elif data + helpers.get_crc_32(data) in source_list:
with contextlib.suppress(zlib.error):
data = zlib.decompress(data[2:-4])
log.info(f"analyze_results: {text} CRC32", _frametype=frametype, data=data)
else:
log.info(
f"analyze_results: {text} not received:",
_frame=frametype,
data=data,
)
def analyze_results(station1: list, station2: list, call_list: list):
"""Examine the information retrieved from the sub-processes."""
# Data in these lists is either a series of bytes of received data,
# or a bytearray of transmitted data from the station.
log = structlog.get_logger("analyze_results")
# Check that each station's transmitted data was received by the other.
for s1, s2, text in [
(station1, station2, call_list[0]),
(station2, station1, call_list[1]),
]:
for s1_item in s1:
if not isinstance(s1_item, list):
continue
data = bytes(s1_item[0])
frametypeno = int.from_bytes(data[:1], "big")
# frametype = static.FRAME_TYPE(frametypeno).name
frametype = str(frametypeno)
s1_crc = helpers.decode_call(helpers.bytes_to_callsign(data[1:4]))
s2_crc = helpers.decode_call(helpers.bytes_to_callsign(data[2:5]))
log.info(
"analyze_results: callsign CRCs:",
tx_station=text,
s1_crc=s1_crc,
s2_crc=s2_crc,
)
locate_data_with_crc(s2, text, data, frametype)
# @pytest.mark.parametrize("frame_type", ["beacon", "connect", "ping"])
@pytest.mark.parametrize("frame_type", [
"ping",
pytest.param("stop", marks=pytest.mark.flaky(reruns=10))
])
def test_datac13_negative(frame_type: str, tmp_path):
log_handler.setup_logging(filename=tmp_path / "test_datac13", level="DEBUG")
log = structlog.get_logger("test_datac13")
s1_data = []
s2_data = []
def recv_data(buffer: list, pipe):
while PIPE_THREAD_RUNNING:
if pipe.poll(0.1):
buffer.append(pipe.recv())
else:
time.sleep(0.1)
def recv_from_pipes(s1_rx, s1_pipe, s2_rx, s2_pipe) -> list:
processes = [
threading.Thread(target=recv_data, args=(s1_rx, s1_pipe)),
threading.Thread(target=recv_data, args=(s2_rx, s2_pipe)),
]
for item in processes:
item.start()
return processes
# This sufficiently separates the two halves of the test. This is needed
# because both scripts change global state. They would conflict if running in
# the same process.
from_s1, s1_send = multiprocessing.Pipe()
from_s2, s2_send = multiprocessing.Pipe()
proc = [
multiprocessing.Process(
target=util.t_datac13_1,
args=(
s1_send,
STATIONS[0],
STATIONS[1],
parameters()[frame_type],
tmp_path,
),
daemon=True,
),
multiprocessing.Process(
target=util.t_datac13_2,
args=(
s2_send,
STATIONS[1],
STATIONS[0],
parameters()[frame_type],
tmp_path,
),
daemon=True,
),
]
pipe_receivers = recv_from_pipes(s1_data, from_s1, s2_data, from_s2)
# log.debug("Creating ")
# print("Starting threads.")
for p_item in proc:
p_item.start()
# This relies on each process exiting when its job is complete!
# print("Waiting for threads to exit.")
for p_item in proc:
p_item.join()
global PIPE_THREAD_RUNNING # pylint: disable=global-statement
PIPE_THREAD_RUNNING = False
for pipe_recv in pipe_receivers:
pipe_recv.join()
for idx in range(2):
try:
os.unlink(tmp_path / f"hfchannel{idx+1}")
except FileNotFoundError as fnfe:
log.debug(f"Unlinking pipe: {fnfe}")
for p_item in proc:
assert p_item.exitcode == 0
# p_item.close() # Python 3.7+ only
p_item.terminate()
p_item.join()
analyze_results(s1_data, s2_data, STATIONS)
if __name__ == "__main__":
# Run pytest with the current script as the filename.
ecode = pytest.main(["-s", "-v", sys.argv[0]])
if ecode == 0:
print("errors: 0")
else:
print(ecode)

98
modem/data cemetery/test/test_helpers.py
View file

@ -1,98 +0,0 @@
#!/usr/bin/env python3
# -*- coding: utf-8 -*-
"""
Unit test common helper routines used throughout the Modem.
Can be invoked from CMake, pytest, coverage or directly.
Uses no other files.
@author: N2KIQ
"""
import sys
import helpers
import pytest
from static import ARQ, AudioParam, Beacon, Channel, Daemon, HamlibParam, ModemParam, Station, Statistics, TCIParam, Modem
@pytest.mark.parametrize("callsign", ["AA1AA", "DE2DE", "E4AWQ-4"])
def test_check_callsign(callsign: str):
"""
Execute test to demonstrate how to create and verify callsign checksums.
"""
Station.ssid_list = [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15]
t_callsign_bytes = helpers.callsign_to_bytes(callsign)
t_callsign = helpers.bytes_to_callsign(t_callsign_bytes)
t_callsign_crc = helpers.get_crc_24(t_callsign)
dxcallsign_bytes = helpers.callsign_to_bytes("ZZ9ZZA-0")
dxcallsign = helpers.bytes_to_callsign(dxcallsign_bytes)
dxcallsign_crc = helpers.get_crc_24(dxcallsign)
assert helpers.check_callsign(t_callsign, t_callsign_crc)[0] is True
assert helpers.check_callsign(t_callsign, dxcallsign_crc)[0] is False
@pytest.mark.parametrize("callsign", ["AA1AA-2", "DE2DE-0", "E4AWQ-4"])
def test_callsign_to_bytes(callsign: str):
"""
Execute test to demonsrate symmetry when converting callsigns to and from byte arrays.
"""
Station.ssid_list = [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15]
t_callsign_crc = helpers.get_crc_24(bytes(callsign, "UTF-8"))
t_callsign_bytes = helpers.callsign_to_bytes(callsign)
t_callsign = helpers.bytes_to_callsign(t_callsign_bytes)
assert helpers.check_callsign(t_callsign, t_callsign_crc)[0] is True
assert helpers.check_callsign(t_callsign, t_callsign_crc)[1] == bytes(
callsign, "UTF-8"
)
@pytest.mark.parametrize("callsign", ["AA1AA-2", "DE2DE-0", "e4awq-4"])
def test_encode_callsign(callsign: str):
"""
Execute test to demonsrate symmetry when encoding and decoding callsigns.
"""
callenc = helpers.encode_call(callsign)
calldec = helpers.decode_call(callenc)
assert callsign.upper() != calldec
@pytest.mark.parametrize("gridsq", ["EM98dc", "DE01GG", "EF42sW"])
def test_encode_grid(gridsq: str):
"""
Execute test to demonsrate symmetry when encoding and decoding grid squares.
"""
gridenc = helpers.encode_grid(gridsq)
griddec = helpers.decode_grid(gridenc)
assert gridsq.upper() == griddec
@pytest.mark.parametrize("gridsq", ["SM98dc", "DE01GZ", "EV42sY"])
@pytest.mark.xfail(reason="Invalid gridsquare provided")
def test_invalid_grid(gridsq: str):
"""
Execute test to demonsrate symmetry when encoding and decoding grid squares.
"""
gridenc = helpers.encode_grid(gridsq)
griddec = helpers.decode_grid(gridenc)
assert gridsq.upper() != griddec
if __name__ == "__main__":
# Run pytest with the current script as the filename.
ecode = pytest.main(["-v", sys.argv[0]])
if ecode == 0:
print("errors: 0")
else:
print(ecode)

192
modem/data cemetery/test/test_highsnr_stdio_C_P_datacx.py
View file

@ -1,192 +0,0 @@
#!/usr/bin/env python3
# -*- coding: utf-8 -*-
"""
Test small multiple-burst messages over a high quality simulated audio channel.
Legacy test for sending / receiving frames through the codec2 modem
and back through on the other station. Data injection initiates directly into
codec2 API. Tests all three codec2 data frames.
Can be invoked from CMake, pytest, coverage or directly.
Uses util_rx.py, sox and freedv_data_raw_tx in separate processeses to perform
the audio tests.
@author: DJ2LS, N2KIQ
"""
# pylint: disable=global-statement, invalid-name, unused-import
import contextlib
import glob
import multiprocessing
import os
import subprocess
import sys
import time
import pytest
BURSTS = 1
FRAMESPERBURST = 1
TESTFRAMES = 3
with contextlib.suppress(KeyError):
BURSTS = int(os.environ["BURSTS"])
with contextlib.suppress(KeyError):
FRAMESPERBURST = int(os.environ["FRAMESPERBURST"])
with contextlib.suppress(KeyError):
TESTFRAMES = int(os.environ["TESTFRAMES"])
# For some reason, sometimes, this test requires the current directory to be `test`.
# Try to adapt dynamically. I still want to figure out why but as a workaround,
# I'm not completely dissatisfied.
if os.path.exists("test"):
os.chdir("test")
def t_HighSNR_C_P_DATACx(
bursts: int, frames_per_burst: int, testframes: int, mode: str
):
"""
Test a high signal-to-noise ratio path with datac13.
:param bursts: Number of bursts
:type bursts: str
:param frames_per_burst: Number of frames transmitted per burst
:type frames_per_burst: str
:param testframes: Number of test frames to transmit
:type testframes: str
"""
# Facilitate running from main directory as well as inside test/
rx_side = "util_rx.py"
if os.path.exists("test") and os.path.exists(os.path.join("test", rx_side)):
rx_side = os.path.join("test", rx_side)
os.environ["PYTHONPATH"] += ":."
tx_side = "freedv_data_raw_tx"
_txpaths = (
os.path.join("..", "modem")
if os.path.exists(os.path.join("..", "modem"))
else "modem"
)
_txpaths = glob.glob(rf"{_txpaths}/**/{tx_side}", recursive=True)
for path in _txpaths:
tx_side = path
break
print(f"tx_side={tx_side} / rx_side={rx_side}")
with subprocess.Popen(
args=[
tx_side,
mode,
"--testframes",
f"{testframes}",
"--bursts",
f"{bursts}",
"--framesperburst",
f"{frames_per_burst}",
"/dev/zero",
"-",
],
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
) as transmit:
with subprocess.Popen(
args=[
"sox",
"-t",
".s16",
"-r",
"8000",
"-c",
"1",
"-",
"-t",
".s16",
"-r",
"48000",
"-c",
"1",
"-",
],
stdin=transmit.stdout,
stdout=subprocess.PIPE,
stderr=subprocess.STDOUT,
) as sox_filter:
with subprocess.Popen(
args=[
"python3",
rx_side,
"--mode",
mode,
"--framesperburst",
str(frames_per_burst),
"--bursts",
str(bursts),
],
stdin=sox_filter.stdout,
stdout=subprocess.PIPE,
stderr=subprocess.STDOUT,
) as receive:
assert receive.stdout
lastline = "".join(
[
str(line, "UTF-8")
for line in receive.stdout.readlines()
if "RECEIVED " in str(line, "UTF-8")
]
)
assert f"RECEIVED BURSTS: {bursts}" in lastline
assert f"RECEIVED FRAMES: {frames_per_burst * bursts}" in lastline
assert "RX_ERRORS: 0" in lastline
print(lastline)
# @pytest.mark.parametrize("bursts", [BURSTS, 2, 3])
# @pytest.mark.parametrize("frames_per_burst", [FRAMESPERBURST, 2, 3])
# @pytest.mark.parametrize("testframes", [TESTFRAMES, 2, 1])
@pytest.mark.parametrize("bursts", [BURSTS])
@pytest.mark.parametrize("frames_per_burst", [FRAMESPERBURST])
@pytest.mark.parametrize("testframes", [TESTFRAMES])
@pytest.mark.parametrize("mode", ["datac13", "datac1", "datac3"])
def test_HighSNR_C_P_DATACx(
bursts: int, frames_per_burst: int, testframes: int, mode: str
):
proc = multiprocessing.Process(
target=t_HighSNR_C_P_DATACx,
args=[bursts, frames_per_burst, testframes, mode],
daemon=True,
)
proc.start()
# Set timeout
timeout = time.time() + 5
while time.time() < timeout:
time.sleep(0.1)
if proc.is_alive():
proc.terminate()
assert 0, "Timeout waiting for test to complete."
proc.join()
proc.terminate()
assert proc.exitcode == 0
# proc.close() # Python 3.7+ only
proc.terminate()
proc.join()
if __name__ == "__main__":
# Run pytest with the current script as the filename.
ecode = pytest.main(["-v", "-s", sys.argv[0]])
if ecode == 0:
print("errors: 0")
else:
print(ecode)

192
modem/data cemetery/test/test_highsnr_stdio_P_C_datacx.py
View file

@ -1,192 +0,0 @@
#!/usr/bin/env python3
# -*- coding: utf-8 -*-
"""
Test small multiple-burst messages over a high quality simulated audio channel.
Legacy test for sending / receiving frames through the codec2 modem
and back through on the other station. Data injection initiates directly into
codec2 API. Tests all three codec2 data frames.
Can be invoked from CMake, pytest, coverage or directly.
Uses util_tx.py, sox, freedv_data_raw_rx and hexdump in separate processeses to
perform the audio tests.
@author: N2KIQ
"""
# pylint: disable=global-statement, invalid-name, unused-import
import contextlib
import glob
import multiprocessing
import os
import subprocess
import sys
import time
import pytest
BURSTS = 1
FRAMESPERBURST = 1
TESTFRAMES = 3
with contextlib.suppress(KeyError):
BURSTS = int(os.environ["BURSTS"])
with contextlib.suppress(KeyError):
FRAMESPERBURST = int(os.environ["FRAMESPERBURST"])
with contextlib.suppress(KeyError):
TESTFRAMES = int(os.environ["TESTFRAMES"])
# For some reason, sometimes, this test requires the current directory to be `test`.
# Try to adapt dynamically. I still want to figure out why but as a workaround,
# I'm not completely dissatisfied.
if os.path.exists("test"):
os.chdir("test")
def t_HighSNR_P_C_DATACx(bursts: int, frames_per_burst: int, mode: str):
"""
Test a high signal-to-noise ratio path with datac13.
:param bursts: Number of bursts
:type bursts: str
:param frames_per_burst: Number of frames transmitted per burst
:type frames_per_burst: str
:param testframes: Number of test frames to transmit
:type testframes: str
"""
# Facilitate running from main directory as well as inside test/
rx_side = "freedv_data_raw_rx"
_rxpath = (
os.path.join("..", "modem")
if os.path.exists(os.path.join("..", "modem"))
else "modem"
)
_rxpaths = glob.glob(rf"{_rxpath}/**/{rx_side}", recursive=True)
for path in _rxpaths:
rx_side = path
break
tx_side = "util_tx.py"
if os.path.exists("test") and os.path.exists(os.path.join("test", tx_side)):
tx_side = os.path.join("test", tx_side)
os.environ["PYTHONPATH"] += ":."
print(f"tx_side={tx_side} / rx_side={rx_side}")
with subprocess.Popen(
args=[
"python3",
tx_side,
"--mode",
mode,
"--delay",
"500",
"--framesperburst",
f"{frames_per_burst}",
"--bursts",
f"{bursts}",
],
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
) as transmit:
with subprocess.Popen(
args=[
"sox",
"-t",
".s16",
"-r",
"48000",
"-c",
"1",
"-",
"-t",
".s16",
"-r",
"8000",
"-c",
"1",
"-",
],
stdin=transmit.stdout,
stdout=subprocess.PIPE,
stderr=subprocess.STDOUT,
) as sox_filter:
with subprocess.Popen(
args=[
rx_side,
mode,
"-",
"-",
"--framesperburst",
str(frames_per_burst),
],
stdin=sox_filter.stdout,
stdout=subprocess.PIPE,
stderr=subprocess.STDOUT,
) as receive:
with subprocess.Popen(
args=[
"hexdump",
"-C",
],
stdin=receive.stdout,
stdout=subprocess.PIPE,
stderr=subprocess.STDOUT,
) as hexdump:
assert hexdump.stdout
lastline = "".join(
[
str(line, "UTF-8")
for line in hexdump.stdout.readlines()
if "HELLO" in str(line, "UTF-8")
]
)
assert "HELLO WORLD!" in lastline
print(lastline)
# @pytest.mark.parametrize("bursts", [BURSTS, 2, 3])
# @pytest.mark.parametrize("frames_per_burst", [FRAMESPERBURST, 2, 3])
@pytest.mark.parametrize("bursts", [BURSTS])
@pytest.mark.parametrize("frames_per_burst", [FRAMESPERBURST])
@pytest.mark.parametrize("mode", ["datac13", "datac1", "datac3"])
def test_HighSNR_P_C_DATACx(bursts: int, frames_per_burst: int, mode: str):
proc = multiprocessing.Process(
target=t_HighSNR_P_C_DATACx,
args=[bursts, frames_per_burst, mode],
daemon=True,
)
proc.start()
# Set timeout
timeout = time.time() + 5
while time.time() < timeout:
time.sleep(0.1)
if proc.is_alive():
proc.terminate()
assert 0, "Timeout waiting for test to complete."
proc.join()
proc.terminate()
assert proc.exitcode == 0
# proc.close() # Python 3.7+ only
proc.terminate()
proc.join()
if __name__ == "__main__":
# Run pytest with the current script as the filename.
ecode = pytest.main(["-v", "-s", sys.argv[0]])
if ecode == 0:
print("errors: 0")
else:
print(ecode)

153
modem/data cemetery/test/test_highsnr_stdio_P_P_datacx.py
View file

@ -1,153 +0,0 @@
#!/usr/bin/env python3
# -*- coding: utf-8 -*-
"""
Test small multiple-burst messages over a high quality simulated audio channel.
Legacy test for sending / receiving frames through the codec2 modem
and back through on the other station. Data injection initiates directly into
codec2 API. Tests all three codec2 data frames.
Can be invoked from CMake, pytest, coverage or directly.
Uses util_rx.py and util_tx.py in separate processeses to perform
the audio tests.
@author: N2KIQ
"""
# pylint: disable=global-statement, invalid-name, unused-import
import contextlib
import multiprocessing
import os
import subprocess
import sys
import time
import pytest
BURSTS = 1
FRAMESPERBURST = 1
TESTFRAMES = 3
with contextlib.suppress(KeyError):
BURSTS = int(os.environ["BURSTS"])
with contextlib.suppress(KeyError):
FRAMESPERBURST = int(os.environ["FRAMESPERBURST"])
with contextlib.suppress(KeyError):
TESTFRAMES = int(os.environ["TESTFRAMES"])
# For some reason, sometimes, this test requires the current directory to be `test`.
# Try to adapt dynamically. I still want to figure out why but as a workaround,
# I'm not completely dissatisfied.
if os.path.exists("test"):
os.chdir("test")
def t_HighSNR_P_P_DATACx(bursts: int, frames_per_burst: int, mode: str):
"""
Test a high signal-to-noise ratio path with Codec2 modes.
:param bursts: Number of bursts
:type bursts: str
:param frames_per_burst: Number of frames transmitted per burst
:type frames_per_burst: str
"""
# Facilitate running from main directory as well as inside test/
tx_side = "util_tx.py"
rx_side = "util_rx.py"
if os.path.exists("test") and os.path.exists(os.path.join("test", tx_side)):
tx_side = os.path.join("test", tx_side)
rx_side = os.path.join("test", rx_side)
os.environ["PYTHONPATH"] += ":."
print(f"tx_side={tx_side} / rx_side={rx_side}")
with subprocess.Popen(
args=[
"python3",
tx_side,
"--mode",
mode,
"--delay",
"500",
"--framesperburst",
str(frames_per_burst),
"--bursts",
str(bursts),
],
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
) as transmit:
with subprocess.Popen(
args=[
"python3",
rx_side,
"--mode",
mode,
"--framesperburst",
str(frames_per_burst),
"--bursts",
str(bursts),
"--timeout",
"20",
],
stdin=transmit.stdout,
stdout=subprocess.PIPE,
stderr=subprocess.STDOUT,
) as receive:
assert receive.stdout
lastline = "".join(
[
str(line, "UTF-8")
for line in receive.stdout.readlines()
if "RECEIVED " in str(line, "UTF-8")
]
)
assert f"RECEIVED BURSTS: {bursts}" in lastline
assert f"RECEIVED FRAMES: {frames_per_burst * bursts}" in lastline
assert "RX_ERRORS: 0" in lastline
print(lastline)
# @pytest.mark.parametrize("bursts", [BURSTS, 2, 3])
# @pytest.mark.parametrize("frames_per_burst", [FRAMESPERBURST, 2, 3])
@pytest.mark.parametrize("bursts", [BURSTS])
@pytest.mark.parametrize("frames_per_burst", [FRAMESPERBURST])
@pytest.mark.parametrize("mode", ["datac13", "datac1", "datac3"])
def test_HighSNR_P_P_DATACx(bursts: int, frames_per_burst: int, mode: str):
proc = multiprocessing.Process(
target=t_HighSNR_P_P_DATACx,
args=[bursts, frames_per_burst, mode],
daemon=True,
)
proc.start()
# Set timeout
timeout = time.time() + 5
while time.time() < timeout:
time.sleep(0.1)
if proc.is_alive():
proc.terminate()
assert 0, "Timeout waiting for test to complete."
proc.join()
proc.terminate()
assert proc.exitcode == 0
# proc.close() # Python 3.7+ only
proc.terminate()
proc.join()
if __name__ == "__main__":
# Run pytest with the current script as the filename.
ecode = pytest.main(["-v", "-s", sys.argv[0]])
if ecode == 0:
print("errors: 0")
else:
print(ecode)

148
modem/data cemetery/test/test_highsnr_stdio_P_P_multi.py
View file

@ -1,148 +0,0 @@
#!/usr/bin/env python3
# -*- coding: utf-8 -*-
"""
Test small multiple-burst messages over a high quality simulated audio channel.
Legacy test for sending / receiving frames through the codec2 modem
and back through on the other station. Data injection initiates directly into
codec2 API. Tests all three codec2 data frames simultaneously.
Can be invoked from CMake, pytest, coverage or directly.
Uses util_multimode_tx.py and util_multimode_tx in separate processeses to perform
the audio tests.
@author: N2KIQ
"""
# pylint: disable=global-statement, invalid-name, unused-import
import contextlib
import multiprocessing
import os
import subprocess
import sys
import time
import pytest
BURSTS = 1
FRAMESPERBURST = 1
TESTFRAMES = 3
with contextlib.suppress(KeyError):
BURSTS = int(os.environ["BURSTS"])
with contextlib.suppress(KeyError):
FRAMESPERBURST = int(os.environ["FRAMESPERBURST"])
with contextlib.suppress(KeyError):
TESTFRAMES = int(os.environ["TESTFRAMES"])
# For some reason, sometimes, this test requires the current directory to be `test`.
# Try to adapt dynamically. I still want to figure out why but as a workaround,
# I'm not completely dissatisfied.
if os.path.exists("test"):
os.chdir("test")
def t_HighSNR_P_P_Multi(bursts: int, frames_per_burst: int):
"""
Test a high signal-to-noise ratio path with datac13, DATAC1 and DATAC3.
:param bursts: Number of bursts
:type bursts: int
:param frames_per_burst: Number of frames transmitted per burst
:type frames_per_burst: int
"""
# Facilitate running from main directory as well as inside test/
tx_side = "util_multimode_tx.py"
rx_side = "util_multimode_rx.py"
if os.path.exists("test") and os.path.exists(os.path.join("test", tx_side)):
tx_side = os.path.join("test", tx_side)
rx_side = os.path.join("test", rx_side)
os.environ["PYTHONPATH"] += ":."
print(f"tx_side={tx_side} / rx_side={rx_side}")
with subprocess.Popen(
args=[
"python3",
tx_side,
"--delay",
"500",
"--framesperburst",
str(frames_per_burst),
"--bursts",
str(bursts),
],
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
) as transmit:
with subprocess.Popen(
args=[
"python3",
rx_side,
"--framesperburst",
str(frames_per_burst),
"--bursts",
str(bursts),
"--timeout",
"20",
],
stdin=transmit.stdout,
stdout=subprocess.PIPE,
stderr=subprocess.STDOUT,
) as receive:
assert receive.stdout
lastline = "".join(
[
str(line, "UTF-8")
for line in receive.stdout.readlines()
if "DATAC" in str(line, "UTF-8")
]
)
assert f"DATAC13: {bursts}/{frames_per_burst * bursts}" in lastline
assert f"DATAC1: {bursts}/{frames_per_burst * bursts}" in lastline
assert f"DATAC3: {bursts}/{frames_per_burst * bursts}" in lastline
print(lastline)
# @pytest.mark.parametrize("bursts", [BURSTS, 2, 3])
# @pytest.mark.parametrize("frames_per_burst", [FRAMESPERBURST, 2, 3])
@pytest.mark.parametrize("bursts", [BURSTS])
@pytest.mark.parametrize("frames_per_burst", [FRAMESPERBURST])
def test_HighSNR_P_P_multi(bursts: int, frames_per_burst: int):
proc = multiprocessing.Process(
target=t_HighSNR_P_P_Multi,
args=[bursts, frames_per_burst],
daemon=True,
)
proc.start()
# Set timeout
timeout = time.time() + 5
while time.time() < timeout:
time.sleep(0.1)
if proc.is_alive():
proc.terminate()
assert 0, "Timeout waiting for test to complete."
proc.join()
proc.terminate()
assert proc.exitcode == 0
# proc.close() # Python 3.7+ only
proc.terminate()
proc.join()
if __name__ == "__main__":
# Run pytest with the current script as the filename.
ecode = pytest.main(["-v", "-s", sys.argv[0]])
if ecode == 0:
print("errors: 0")
else:
print(ecode)

71
modem/data cemetery/test/test_modem.py
View file

@ -1,71 +0,0 @@
#!/usr/bin/env python3
# -*- coding: utf-8 -*-
"""
Test connect frame commands over a high quality simulated audio channel.
Near end-to-end test for sending / receiving connection control frames through the
Modem and modem and back through on the other station. Data injection initiates from the
queue used by the daemon process into and out of the Modem.
Can be invoked from CMake, pytest, coverage or directly.
Uses util_modem_I[RS]S.py in separate process to perform the data transfer.
@author: DJ2LS, N2KIQ
"""
import multiprocessing
import os
import sys
import time
import log_handler
import pytest
import structlog
try:
import test.util_modem_IRS as irs
import test.util_modem_ISS as iss
except ImportError:
import util_modem_IRS as irs
import util_modem_ISS as iss
# This test is currently a little inconsistent.
@pytest.mark.parametrize("command", ["CONNECT"])
@pytest.mark.flaky(reruns=2)
def test_modem(command, tmp_path):
log_handler.setup_logging(filename=tmp_path / "test_modem", level="INFO")
log = structlog.get_logger("test_modem")
iss_proc = multiprocessing.Process(target=iss.t_arq_iss, args=[command, tmp_path])
irs_proc = multiprocessing.Process(target=irs.t_arq_irs, args=[command, tmp_path])
log.debug("Starting threads.")
iss_proc.start()
irs_proc.start()
time.sleep(12)
log.debug("Terminating threads.")
irs_proc.terminate()
iss_proc.terminate()
irs_proc.join()
iss_proc.join()
for idx in range(2):
try:
os.unlink(tmp_path / f"hfchannel{idx+1}")
except FileNotFoundError as fnfe:
log.debug(f"Unlinking pipe: {fnfe}")
assert iss_proc.exitcode in [0, -15], f"Transmit side failed test. {iss_proc}"
assert irs_proc.exitcode in [0, -15], f"Receive side failed test. {irs_proc}"
if __name__ == "__main__":
# Run pytest with the current script as the filename.
ecode = pytest.main(["-s", "-v", sys.argv[0]])
if ecode == 0:
print("errors: 0")
else:
print(ecode)

326
modem/data cemetery/test/test_modem_states.py
View file

@ -1,326 +0,0 @@
#!/usr/bin/env python3
# -*- coding: utf-8 -*-
"""
Test control frame messages over a high quality simulated audio channel.
Near end-to-end test for sending / receiving select control frames through the
Modem and modem and back through on the other station. Data injection initiates from the
queue used by the daemon process into and out of the Modem.
Can be invoked from CMake, pytest, coverage or directly.
Uses util_datac13.py in separate process to perform the data transfer.
@author: N2KIQ
"""
import multiprocessing
import numpy as np
import sys
import time
import pytest
# pylint: disable=wrong-import-position
sys.path.insert(0, "..")
sys.path.insert(0, "../modem")
import data_handler
import helpers
from global_instances import ARQ, AudioParam, Beacon, Channel, Daemon, HamlibParam, ModemParam, Station, Statistics, TCIParam, Modem
def print_frame(data: bytearray):
"""
Pretty-print the provided frame.
:param data: Frame to be output
:type data: bytearray
"""
print(f"Type : {int(data[0])}")
print(f"DXCRC : {bytes(data[1:4])}")
print(f"CallCRC: {bytes(data[4:7])}")
print(f"Call : {helpers.bytes_to_callsign(data[7:13])}")
def t_create_frame(frame_type: int, mycall: str, dxcall: str) -> bytearray:
"""
Generate the requested frame.
:param frame_type: The numerical type of the desired frame.
:type frame_type: int
:param mycall: Callsign of the near station
:type mycall: str
:param dxcall: Callsign of the far station
:type dxcall: str
:return: Bytearray of the requested frame
:rtype: bytearray
"""
mycallsign_bytes = helpers.callsign_to_bytes(mycall)
mycallsign = helpers.bytes_to_callsign(mycallsign_bytes)
mycallsign_crc = helpers.get_crc_24(mycallsign)
dxcallsign_bytes = helpers.callsign_to_bytes(dxcall)
dxcallsign = helpers.bytes_to_callsign(dxcallsign_bytes)
dxcallsign_crc = helpers.get_crc_24(dxcallsign)
# frame = bytearray(14)
# frame[:1] = bytes([frame_type])
# frame[1:4] = dxcallsign_crc
# frame[4:7] = mycallsign_crc
# frame[7:13] = mycallsign_bytes
session_id = np.random.bytes(1)
frame = bytearray(14)
frame[:1] = bytes([frame_type])
frame[1:2] = session_id
frame[2:5] = dxcallsign_crc
frame[5:8] = mycallsign_crc
frame[8:14] = mycallsign_bytes
return frame
def t_create_session_close_old(mycall: str, dxcall: str) -> bytearray:
"""
Generate the session_close frame.
:param mycall: Callsign of the near station
:type mycall: str
:param dxcall: Callsign of the far station
:type dxcall: str
:return: Bytearray of the requested frame
:rtype: bytearray
"""
return t_create_frame(223, mycall, dxcall)
def t_create_session_close(session_id: bytes, dxcall: str) -> bytearray:
"""
Generate the session_close frame.
:param session_id: Session to close
:type mycall: int
:return: Bytearray of the requested frame
:rtype: bytearray
"""
dxcallsign_bytes = helpers.callsign_to_bytes(dxcall)
dxcallsign = helpers.bytes_to_callsign(dxcallsign_bytes)
dxcallsign_crc = helpers.get_crc_24(dxcallsign)
# return t_create_frame(223, mycall, dxcall)
frame = bytearray(14)
frame[:1] = bytes([223])
frame[1:2] = session_id
frame[2:5] = dxcallsign_crc
return frame
def t_create_start_session(mycall: str, dxcall: str) -> bytearray:
"""
Generate the create_session frame.
:param mycall: Callsign of the near station
:type mycall: str
:param dxcall: Callsign of the far station
:type dxcall: str
:return: Bytearray of the requested frame
:rtype: bytearray
"""
return t_create_frame(221, mycall, dxcall)
def t_tsh_dummy():
"""Replacement function for transmit_session_heartbeat"""
print("In transmit_session_heartbeat")
def t_foreign_disconnect(mycall: str, dxcall: str):
"""
Execute test to validate that receiving a session open frame sets the correct machine
state.
:param mycall: Callsign of the near station
:type mycall: str
:param dxcall: Callsign of the far station
:type dxcall: str
:return: Bytearray of the requested frame
:rtype: bytearray
"""
# Set the SSIDs we'll use for this test.
Station.ssid_list = [0, 1, 2, 3, 4]
# Setup the static parameters for the connection.
mycallsign_bytes = helpers.callsign_to_bytes(mycall)
mycallsign = helpers.bytes_to_callsign(mycallsign_bytes)
Station.mycallsign = mycallsign
Station.mycallsign_crc = helpers.get_crc_24(mycallsign)
dxcallsign_bytes = helpers.callsign_to_bytes(dxcall)
dxcallsign = helpers.bytes_to_callsign(dxcallsign_bytes)
Station.dxcallsign = dxcallsign
Station.dxcallsign_crc = helpers.get_crc_24(dxcallsign)
# Create the Modem
modem = data_handler.DATA()
modem.arq_cleanup()
# Replace the heartbeat transmit routine with a No-Op.
modem.transmit_session_heartbeat = t_tsh_dummy
# Create frame to be 'received' by this station.
create_frame = t_create_start_session(mycall=dxcall, dxcall=mycall)
print_frame(create_frame)
modem.received_session_opener(create_frame)
assert helpers.callsign_to_bytes(Station.mycallsign) == mycallsign_bytes
assert helpers.callsign_to_bytes(Station.dxcallsign) == dxcallsign_bytes
assert ARQ.arq_session is True
assert Modem.modem_state == "BUSY"
assert ARQ.arq_session_state == "connecting"
# Set up a frame from a non-associated station.
# foreigncall_bytes = helpers.callsign_to_bytes("ZZ0ZZ-0")
# foreigncall = helpers.bytes_to_callsign(foreigncall_bytes)
# close_frame = t_create_session_close_old("ZZ0ZZ-0", "ZZ0ZZ-0")
open_session = create_frame[1:2]
wrong_session = np.random.bytes(1)
while wrong_session == open_session:
wrong_session = np.random.bytes(1)
close_frame = t_create_session_close(wrong_session, dxcall)
print_frame(close_frame)
# assert (
# helpers.check_callsign(Station.dxcallsign, bytes(close_frame[4:7]))[0] is False
# ), f"{helpers.get_crc_24(Station.dxcallsign)} == {bytes(close_frame[4:7])} but should be not equal."
# assert (
# helpers.check_callsign(foreigncall, bytes(close_frame[4:7]))[0] is True
# ), f"{helpers.get_crc_24(foreigncall)} != {bytes(close_frame[4:7])} but should be equal."
# Send the non-associated session close frame to the Modem
modem.received_session_close(close_frame)
assert helpers.callsign_to_bytes(Station.mycallsign) == helpers.callsign_to_bytes(
mycall
), f"{Station.mycallsign} != {mycall} but should equal."
assert helpers.callsign_to_bytes(Station.dxcallsign) == helpers.callsign_to_bytes(
dxcall
), f"{Station.dxcallsign} != {dxcall} but should equal."
assert ARQ.arq_session is True
assert Modem.modem_state == "BUSY"
assert ARQ.arq_session_state == "connecting"
def t_valid_disconnect(mycall: str, dxcall: str):
"""
Execute test to validate that receiving a session open frame sets the correct machine
state.
:param mycall: Callsign of the near station
:type mycall: str
:param dxcall: Callsign of the far station
:type dxcall: str
:return: Bytearray of the requested frame
:rtype: bytearray
"""
# Set the SSIDs we'll use for this test.
Station.ssid_list = [0, 1, 2, 3, 4]
# Setup the static parameters for the connection.
mycallsign_bytes = helpers.callsign_to_bytes(mycall)
mycallsign = helpers.bytes_to_callsign(mycallsign_bytes)
Station.mycallsign = mycallsign
Station.mycallsign_crc = helpers.get_crc_24(mycallsign)
dxcallsign_bytes = helpers.callsign_to_bytes(dxcall)
dxcallsign = helpers.bytes_to_callsign(dxcallsign_bytes)
Station.dxcallsign = dxcallsign
Station.dxcallsign_crc = helpers.get_crc_24(dxcallsign)
# Create the Modem
modem = data_handler.DATA()
modem.arq_cleanup()
# Replace the heartbeat transmit routine with our own, a No-Op.
modem.transmit_session_heartbeat = t_tsh_dummy
# Create packet to be 'received' by this station.
create_frame = t_create_start_session(mycall=dxcall, dxcall=mycall)
print_frame(create_frame)
modem.received_session_opener(create_frame)
print(ARQ.arq_session)
assert ARQ.arq_session is True
assert Modem.modem_state == "BUSY"
assert ARQ.arq_session_state == "connecting"
# Create packet to be 'received' by this station.
# close_frame = t_create_session_close_old(mycall=dxcall, dxcall=mycall)
open_session = create_frame[1:2]
print(dxcall)
print("#####################################################")
close_frame = t_create_session_close(open_session, mycall)
print(close_frame[2:5])
print_frame(close_frame)
modem.received_session_close(close_frame)
assert helpers.callsign_to_bytes(Station.mycallsign) == mycallsign_bytes
assert helpers.callsign_to_bytes(Station.dxcallsign) == dxcallsign_bytes
assert ARQ.arq_session is False
assert Modem.modem_state == "IDLE"
assert ARQ.arq_session_state == "disconnected"
# These tests are pushed into separate processes as a workaround. These tests
# change the state of one of the static parts of the system. Unfortunately the
# specific state(s) maintained across tests in the same interpreter are not yet known.
# The other tests affected are: `test_modem.py` and the ARQ tests.
@pytest.mark.parametrize("mycall", ["AA1AA-2", "DE2DE-0", "E4AWQ-4"])
@pytest.mark.parametrize("dxcall", ["AA9AA-1", "DE2ED-0", "F6QWE-3"])
# @pytest.mark.flaky(reruns=2)
def test_foreign_disconnect(mycall: str, dxcall: str):
proc = multiprocessing.Process(target=t_foreign_disconnect, args=(mycall, dxcall))
# print("Starting threads.")
proc.start()
time.sleep(5.05)
# print("Terminating threads.")
proc.terminate()
proc.join()
# print(f"\nproc.exitcode={proc.exitcode}")
assert proc.exitcode == 0
@pytest.mark.parametrize("mycall", ["AA1AA-2", "DE2DE-0", "M4AWQ-4"])
@pytest.mark.parametrize("dxcall", ["AA9AA-1", "DE2ED-0", "F6QWE-3"])
@pytest.mark.flaky(reruns=2)
def test_valid_disconnect(mycall: str, dxcall: str):
proc = multiprocessing.Process(target=t_valid_disconnect, args=(mycall, dxcall))
# print("Starting threads.")
proc.start()
time.sleep(5.05)
# print("Terminating threads.")
proc.terminate()
proc.join()
# print(f"\nproc.exitcode={proc.exitcode}")
assert proc.exitcode == 0
if __name__ == "__main__":
# Run pytest with the current script as the filename.
ecode = pytest.main(["-v", sys.argv[0]])
if ecode == 0:
print("errors: 0")
else:
print(ecode)

48
modem/data cemetery/test/test_pa.py
View file

@ -1,48 +0,0 @@
#!/usr/bin/env python3
# -*- coding: utf-8 -*-
#
# Throw away test program to help understand the care and feeding of PyAudio
import numpy as np
import pyaudio
CHUNK = 1024
FS48 = 48000
FTEST = 800
AMP = 16000
def test_pa():
# 1. play sine wave out of default sound device
p_audio = pyaudio.PyAudio()
stream = p_audio.open(
format=pyaudio.paInt16,
channels=1,
rate=FS48,
frames_per_buffer=CHUNK,
output=True,
)
with open("out48.raw", mode="wb") as f48:
temp = 0
for _ in range(50):
sine_48k = (
AMP * np.cos(2 * np.pi * np.arange(temp, temp + CHUNK) * FTEST / FS48)
).astype(np.int16)
temp += CHUNK
sine_48k.tofile(f48)
stream.write(sine_48k.tobytes())
sil_48k = np.zeros(CHUNK, dtype=np.int16)
for _ in range(50):
sil_48k.tofile(f48)
stream.write(sil_48k)
stream.stop_stream()
stream.close()
p_audio.terminate()
if __name__ == "__main__":
test_pa()

120
modem/data cemetery/test/test_resample_48_8.py
View file

@ -1,120 +0,0 @@
#!/usr/bin/env python3
# -*- coding: utf-8 -*-
#
# Unit test for FreeDV API resampler functions, from
# codec2/unittest/t48_8_short.c - generate a sine wave at 8 KHz,
# upsample to 48 kHz, add an interferer, then downsample back to 8 kHz
#
# You can listen to the output files with:
#
# aplay -f S16_LE in8.raw
# aplay -r 48000 -f S16_LE out48.raw
# aplay -f S16_LE out8.raw
#
# They should sound like clean sine waves
# pylint: disable=global-statement, invalid-name, unused-import
import os
import sys
import codec2
import numpy as np
import pytest
# dig some constants out
FDMDV_OS_48 = codec2.api.FDMDV_OS_48
FDMDV_OS_TAPS_48K = codec2.api.FDMDV_OS_TAPS_48K
FDMDV_OS_TAPS_48_8K = codec2.api.FDMDV_OS_TAPS_48_8K
N8 = 180 # processing buffer size at 8 kHz
N48 = N8 * FDMDV_OS_48 # processing buffer size at 48 kHz
MEM8 = FDMDV_OS_TAPS_48_8K # 8kHz signal filter memory
MEM48 = FDMDV_OS_TAPS_48K # 48kHz signal filter memory
FRAMES = 50 # number of frames to test
FS8 = 8000
FS48 = 48000
AMP = 16000 # sine wave amplitude
FTEST8 = 800 # input test frequency at FS=8kHz
FINTER48 = 10000 # interferer frequency at FS=48kHz
# Due to the design of these resamplers, the processing buffer (at 8kHz)
# must be an integer multiple of oversampling ratio
assert N8 % FDMDV_OS_48 == 0
def test_resampler():
"""
Test for the codec2 audio resampling routine
"""
# time indexes, we advance every frame
t = 0
t1 = 0
# output files to listen to/evaluate result
with open("in8.raw", mode="wb") as fin8:
with open("out48.raw", mode="wb") as f48:
with open("out8.raw", mode="wb") as fout8:
resampler = codec2.resampler()
# Generate FRAMES of a sine wave
for _ in range(FRAMES):
# Primary sine wave, which the down-sampling filter should retain.
sine_in8k = (
AMP * np.cos(2 * np.pi * np.arange(t, t + N8) * FTEST8 / FS8)
).astype(np.int16)
t += N8
sine_in8k.tofile(fin8)
sine_out48k = resampler.resample8_to_48(sine_in8k)
sine_out48k.tofile(f48)
# Add an interfering sine wave, which the down-sampling filter should (mostly) remove
sine_in48k = (
sine_out48k
+ (AMP / 2)
* np.cos(2 * np.pi * np.arange(t1, t1 + N48) * FINTER48 / FS48)
).astype(np.int16)
t1 += N48
sine_out8k = resampler.resample48_to_8(sine_in48k)
sine_out8k.tofile(fout8)
# os.unlink("out48.raw")
# Automated test evaluation --------------------------------------------
# The input and output signals will not be time aligned due to the filter
# delays, so compare the magnitude spectrum
# Read the raw audio files
in8k = np.fromfile("in8.raw", dtype=np.int16)
out8k = np.fromfile("out8.raw", dtype=np.int16)
assert len(in8k) == len(out8k)
# os.unlink("in8.raw")
# os.unlink("out8.raw")
# Apply hanning filter to raw input data samples
h = np.hanning(len(in8k))
S1 = np.abs(np.fft.fft(in8k * h))
S2 = np.abs(np.fft.fft(out8k * h))
# Calculate the ratio between signal and noise (error energy).
error = S1 - S2
error_energy = np.dot(error, error)
ratio = error_energy / np.dot(S1, S1)
ratio_dB = 10 * np.log10(ratio)
# Establish -40.0 as the noise ratio ceiling
threshdB = -40.0
print(f"ratio_dB: {ratio_dB:4.2}" % (ratio_dB))
assert ratio_dB < threshdB
if __name__ == "__main__":
# Run pytest with the current script as the filename.
ecode = pytest.main(["-v", sys.argv[0]])
if ecode == 0:
print("PASS")
else:
print("FAIL")

27
modem/data cemetery/test/test_virtual1.sh
View file

@ -1,27 +0,0 @@
#!/bin/bash -x
# Run a test using the virtual sound cards, sound I/O performed by aplay/arecord at
# Fs=8000 Hz, and we pipe to Python utilities
function check_alsa_loopback {
lsmod | grep snd_aloop >> /dev/null
if [ $? -eq 1 ]; then
echo "ALSA loopback device not present. Please install with:"
echo
echo " sudo modprobe snd-aloop index=1,2 enable=1,1 pcm_substreams=1,1 id=CHAT1,CHAT2"
exit 1
fi
}
check_alsa_loopback
RX_LOG=$(mktemp)
MAX_RUN_TIME=2700
# make sure all child processes are killed when we exit
trap 'jobs -p | xargs -r kill' EXIT
arecord --device="plughw:CARD=CHAT2,DEV=0" -r 48000 -f S16_LE -d $MAX_RUN_TIME | python3 util_rx.py --mode datac13 --frames 2 --bursts 5 --debug &
rx_pid=$!
sleep 1
python3 util_tx.py --mode datac13 --frames 2 --bursts 5 --delay 500 | aplay --device="plughw:CARD=CHAT2,DEV=1" -r 48000 -f S16_LE
wait ${rx_pid}

16
modem/data cemetery/test/test_virtual1a.sh
View file

@ -1,16 +0,0 @@
#!/bin/bash -x
# Run a test using the virtual sound cards, tx sound I/O performed by aplay
# and arecord at Fs=48000Hz, we pipe to Python utilities
MAX_RUN_TIME=2600
# make sure all child processes are killed when we exit
trap 'jobs -p | xargs -r kill' EXIT
arecord -r 48000 --device="plughw:CARD=CHAT1,DEV=0" -f S16_LE -d $MAX_RUN_TIME | \
python3 util_rx.py --mode datac13 --frames 2 --bursts 5 --debug &
rx_pid=$!
sleep 1
python3 util_tx.py --mode datac13 --frames 2 --bursts 5 --delay 500 | \
aplay -r 48000 --device="plughw:CARD=CHAT1,DEV=1" -f S16_LE
wait ${rx_pid}

15
modem/data cemetery/test/test_virtual1b.sh
View file

@ -1,15 +0,0 @@
#!/bin/bash -x
# Run a test using the virtual sound cards, tx sound I/O performed by Python,
# rx using arecord, Fs=48000Hz
MAX_RUN_TIME=2600
# make sure all child processes are killed when we exit
trap 'jobs -p | xargs -r kill' EXIT
arecord -r 48000 --device="plughw:CARD=CHAT1,DEV=0" -f S16_LE -d $MAX_RUN_TIME | \
python3 util_rx.py --mode datac13 --frames 2 --bursts 5 --debug --timeout 20 &
rx_pid=$!
sleep 1
python3 util_tx.py --mode datac13 --frames 2 --bursts 5 --delay 2000 --audiodev -2
wait ${rx_pid}

15
modem/data cemetery/test/test_virtual1c.sh
View file

@ -1,15 +0,0 @@
#!/bin/bash -x
# Run a test using the virtual sound cards, tx sound I/O performed by aplay,
# rx sound I/O by Python, Fs=48000Hz.
MAX_RUN_TIME=2600
# make sure all child processes are killed when we exit
trap 'jobs -p | xargs -r kill' EXIT
python3 util_rx.py --mode datac13 --frames 2 --bursts 5 --debug --audiodev -2 &
rx_pid=$!
sleep 1
python3 util_tx.py --mode datac13 --frames 2 --bursts 5 | \
aplay -r 48000 --device="plughw:CARD=CHAT1,DEV=1" -f S16_LE
wait ${rx_pid}

23
modem/data cemetery/test/test_virtual2.sh
View file

@ -1,23 +0,0 @@
#!/bin/bash -x
# Run a test using the virtual sound cards, Python audio I/O
function check_alsa_loopback {
lsmod | grep snd_aloop >> /dev/null
if [ $? -eq 1 ]; then
echo "ALSA loopback device not present. Please install with:"
echo
echo " sudo modprobe snd-aloop index=1,2 enable=1,1 pcm_substreams=1,1 id=CHAT1,CHAT2"
exit 1
fi
}
check_alsa_loopback
# make sure all child processes are killed when we exit
trap 'jobs -p | xargs -r kill' EXIT
python3 util_callback_rx.py --mode datac13 --frames 2 --bursts 3 --audiodev -2 --debug &
rx_pid=$!
sleep 1
python3 util_tx.py --mode datac13 --frames 2 --bursts 3 --audiodev -2
wait ${rx_pid}

23
modem/data cemetery/test/test_virtual3a.sh
View file

@ -1,23 +0,0 @@
#!/bin/bash -x
# Run a test using the virtual sound cards, Python audio I/O
function check_alsa_loopback {
lsmod | grep snd_aloop >> /dev/null
if [ $? -eq 1 ]; then
echo "ALSA loopback device not present. Please install with:"
echo
echo " sudo modprobe snd-aloop index=1,2 enable=1,1 pcm_substreams=1,1 id=CHAT1,CHAT2"
exit 1
fi
}
check_alsa_loopback
# make sure all child processes are killed when we exit
trap 'jobs -p | xargs -r kill' EXIT
python3 util_callback_rx.py --mode datac13 --frames 2 --bursts 3 --audiodev -2 --debug &
rx_pid=$!
#sleep 1
python3 util_tx.py --mode datac13 --frames 2 --bursts 3 --audiodev -2
wait ${rx_pid}

23
modem/data cemetery/test/test_virtual3b.sh
View file

@ -1,23 +0,0 @@
#!/bin/bash -x
# Run a test using the virtual sound cards, Python audio I/O
function check_alsa_loopback {
lsmod | grep snd_aloop >> /dev/null
if [ $? -eq 1 ]; then
echo "ALSA loopback device not present. Please install with:"
echo
echo " sudo modprobe snd-aloop index=1,2 enable=1,1 pcm_substreams=1,1 id=CHAT1,CHAT2"
exit 1
fi
}
check_alsa_loopback
# make sure all child processes are killed when we exit
trap 'jobs -p | xargs -r kill' EXIT
python3 util_callback_rx_outside.py --mode datac13 --frames 2 --bursts 3 --audiodev -2 --debug &
rx_pid=$!
#sleep 1
python3 util_tx.py --mode datac13 --frames 2 --bursts 3 --audiodev -2
wait ${rx_pid}

32
modem/data cemetery/test/test_virtual4a.sh
View file

@ -1,32 +0,0 @@
#!/bin/bash -x
# Run a test using the virtual sound cards
function check_alsa_loopback {
lsmod | grep snd_aloop >> /dev/null
if [ $? -eq 1 ]; then
echo "ALSA loopback device not present. Please install with:"
echo
echo " sudo modprobe snd-aloop index=1,2 enable=1,1 pcm_substreams=1,1 id=CHAT1,CHAT2"
exit 1
fi
}
myInterruptHandler()
{
exit 1
}
check_alsa_loopback
RX_LOG=$(mktemp)
trap myInterruptHandler SIGINT
# make sure all child processes are killed when we exit
trap 'jobs -p | xargs -r kill' EXIT
python3 util_callback_multimode_rx.py --timeout 60 --framesperburst 2 --bursts 2 --audiodev -2 --debug &
rx_pid=$!
sleep 1
python3 util_multimode_tx.py --framesperburst 2 --bursts 2 --audiodev -2 --delay 500
wait ${rx_pid}

32
modem/data cemetery/test/test_virtual4b.sh
View file

@ -1,32 +0,0 @@
#!/bin/bash -x
# Run a test using the virtual sound cards
function check_alsa_loopback {
lsmod | grep snd_aloop >> /dev/null
if [ $? -eq 1 ]; then
echo "ALSA loopback device not present. Please install with:"
echo
echo " sudo modprobe snd-aloop index=1,2 enable=1,1 pcm_substreams=1,1 id=CHAT1,CHAT2"
exit 1
fi
}
myInterruptHandler()
{
exit 1
}
check_alsa_loopback
RX_LOG=$(mktemp)
trap myInterruptHandler SIGINT
# make sure all child processes are killed when we exit
trap 'jobs -p | xargs -r kill' EXIT
python3 util_callback_multimode_rx_outside.py --timeout 60 --framesperburst 2 --bursts 2 --audiodev -2 --debug &
rx_pid=$!
sleep 1
python3 util_multimode_tx.py --framesperburst 2 --bursts 2 --audiodev -2 --delay 500
wait ${rx_pid}

32
modem/data cemetery/test/test_virtual_mm.sh
View file

@ -1,32 +0,0 @@
#!/bin/bash -x
# Run a test using the virtual sound cards
function check_alsa_loopback {
lsmod | grep snd_aloop >> /dev/null
if [ $? -eq 1 ]; then
echo "ALSA loopback device not present. Please install with:"
echo
echo " sudo modprobe snd-aloop index=1,2 enable=1,1 pcm_substreams=1,1 id=CHAT1,CHAT2"
exit 1
fi
}
myInterruptHandler()
{
exit 1
}
check_alsa_loopback
RX_LOG=$(mktemp)
trap myInterruptHandler SIGINT
# make sure all child processes are killed when we exit
trap 'jobs -p | xargs -r kill' EXIT
python3 util_multimode_rx.py --timeout 60 --framesperburst 2 --bursts 2 --audiodev -2 --debug &
rx_pid=$!
sleep 1
python3 util_multimode_tx.py --framesperburst 2 --bursts 2 --audiodev -2 --delay 500
wait ${rx_pid}

323
modem/data cemetery/test/util_callback_multimode_rx.py
View file

@ -1,323 +0,0 @@
#!/usr/bin/env python3
# -*- coding: utf-8 -*-
"""
Created on Wed Dec 23 07:04:24 2020
@author: DJ2LS
"""
import argparse
import ctypes
import sys
import threading
import time
import numpy as np
import pyaudio
sys.path.insert(0, "..")
from modem import codec2
# --------------------------------------------GET PARAMETER INPUTS
parser = argparse.ArgumentParser(description="FreeDATA audio test")
parser.add_argument("--bursts", dest="N_BURSTS", default=1, type=int)
parser.add_argument("--framesperburst", dest="N_FRAMES_PER_BURST", default=1, type=int)
parser.add_argument(
"--audiodev",
dest="AUDIO_INPUT_DEVICE",
default=-1,
type=int,
help="audio device number to use",
)
parser.add_argument("--debug", dest="DEBUGGING_MODE", action="store_true")
parser.add_argument(
"--list",
dest="LIST",
action="store_true",
help="list audio devices by number and exit",
)
parser.add_argument(
"--timeout",
dest="TIMEOUT",
default=10,
type=int,
help="Timeout (seconds) before test ends",
)
args, _ = parser.parse_known_args()
if args.LIST:
p = pyaudio.PyAudio()
for dev in range(p.get_device_count()):
print("audiodev: ", dev, p.get_device_info_by_index(dev)["name"])
sys.exit()
class Test:
def __init__(self):
self.N_BURSTS = args.N_BURSTS
self.N_FRAMES_PER_BURST = args.N_FRAMES_PER_BURST
self.AUDIO_INPUT_DEVICE = args.AUDIO_INPUT_DEVICE
self.DEBUGGING_MODE = args.DEBUGGING_MODE
self.TIMEOUT = args.TIMEOUT
# AUDIO PARAMETERS
# v-- consider increasing if you get nread_exceptions > 0
self.AUDIO_FRAMES_PER_BUFFER = 2400 * 2
self.MODEM_SAMPLE_RATE = codec2.api.FREEDV_FS_8000
self.AUDIO_SAMPLE_RATE_RX = 48000
# make sure our resampler will work
assert (
self.AUDIO_SAMPLE_RATE_RX / self.MODEM_SAMPLE_RATE
) == codec2.api.FDMDV_OS_48
# check if we want to use an audio device then do a pyaudio init
if self.AUDIO_INPUT_DEVICE != -1:
self.p = pyaudio.PyAudio()
# auto search for loopback devices
if self.AUDIO_INPUT_DEVICE == -2:
loopback_list = [
dev
for dev in range(self.p.get_device_count())
if "Loopback: PCM"
in self.p.get_device_info_by_index(dev)["name"]
]
if len(loopback_list) >= 2:
# 0 = RX 1 = TX
self.AUDIO_INPUT_DEVICE = loopback_list[0]
print(f"loopback_list rx: {loopback_list}", file=sys.stderr)
else:
sys.exit()
print(
f"AUDIO INPUT DEVICE: {self.AUDIO_INPUT_DEVICE} DEVICE: {self.p.get_device_info_by_index(self.AUDIO_INPUT_DEVICE)['name']} \
AUDIO SAMPLE RATE: {self.AUDIO_SAMPLE_RATE_RX}",
file=sys.stderr,
)
self.stream_rx = self.p.open(
format=pyaudio.paInt16,
channels=1,
rate=self.AUDIO_SAMPLE_RATE_RX,
frames_per_buffer=self.AUDIO_FRAMES_PER_BUFFER,
input=True,
output=False,
input_device_index=self.AUDIO_INPUT_DEVICE,
stream_callback=self.callback,
)
else:
print("test_callback_multimode_rx: Not written for STDIN usage.")
print("Exiting.")
sys.exit()
# open codec2 instance
self.datac13_freedv = ctypes.cast(
codec2.api.freedv_open(codec2.FREEDV_MODE.datac13.value), ctypes.c_void_p
)
self.datac13_bytes_per_frame = int(
codec2.api.freedv_get_bits_per_modem_frame(self.datac13_freedv) / 8
)
self.datac13_bytes_out = ctypes.create_string_buffer(self.datac13_bytes_per_frame)
codec2.api.freedv_set_frames_per_burst(
self.datac13_freedv, self.N_FRAMES_PER_BURST
)
self.datac13_buffer = codec2.audio_buffer(2 * self.AUDIO_FRAMES_PER_BUFFER)
self.datac1_freedv = ctypes.cast(
codec2.api.freedv_open(codec2.FREEDV_MODE.datac1.value), ctypes.c_void_p
)
self.datac1_bytes_per_frame = int(
codec2.api.freedv_get_bits_per_modem_frame(self.datac1_freedv) / 8
)
self.datac1_bytes_out = ctypes.create_string_buffer(self.datac1_bytes_per_frame)
codec2.api.freedv_set_frames_per_burst(
self.datac1_freedv, self.N_FRAMES_PER_BURST
)
self.datac1_buffer = codec2.audio_buffer(2 * self.AUDIO_FRAMES_PER_BUFFER)
self.datac3_freedv = ctypes.cast(
codec2.api.freedv_open(codec2.FREEDV_MODE.datac3.value), ctypes.c_void_p
)
self.datac3_bytes_per_frame = int(
codec2.api.freedv_get_bits_per_modem_frame(self.datac3_freedv) / 8
)
self.datac3_bytes_out = ctypes.create_string_buffer(self.datac3_bytes_per_frame)
codec2.api.freedv_set_frames_per_burst(
self.datac3_freedv, self.N_FRAMES_PER_BURST
)
self.datac3_buffer = codec2.audio_buffer(2 * self.AUDIO_FRAMES_PER_BUFFER)
# SET COUNTERS
self.rx_total_frames_datac13 = 0
self.rx_frames_datac13 = 0
self.rx_bursts_datac13 = 0
self.rx_total_frames_datac1 = 0
self.rx_frames_datac1 = 0
self.rx_bursts_datac1 = 0
self.rx_total_frames_datac3 = 0
self.rx_frames_datac3 = 0
self.rx_bursts_datac3 = 0
self.rx_errors = 0
self.nread_exceptions = 0
self.timeout = time.time() + self.TIMEOUT
self.receive = True
self.resampler = codec2.resampler()
# Copy received 48 kHz to a file. Listen to this file with:
# aplay -r 48000 -f S16_LE rx48_callback.raw
# Corruption of this file is a good way to detect audio card issues
self.frx = open("rx48_callback_multimode.raw", mode="wb")
# initial nin values
self.datac13_nin = codec2.api.freedv_nin(self.datac13_freedv)
self.datac1_nin = codec2.api.freedv_nin(self.datac1_freedv)
self.datac3_nin = codec2.api.freedv_nin(self.datac3_freedv)
self.LOGGER_THREAD = threading.Thread(
target=self.print_stats, name="LOGGER_THREAD"
)
self.LOGGER_THREAD.start()
def callback(self, data_in48k, frame_count, time_info, status):
x = np.frombuffer(data_in48k, dtype=np.int16)
x.tofile(self.frx)
x = self.resampler.resample48_to_8(x)
self.datac13_buffer.push(x)
self.datac1_buffer.push(x)
self.datac3_buffer.push(x)
while self.datac13_buffer.nbuffer >= self.datac13_nin:
# demodulate audio
nbytes = codec2.api.freedv_rawdatarx(
self.datac13_freedv,
self.datac13_bytes_out,
self.datac13_buffer.buffer.ctypes,
)
self.datac13_buffer.pop(self.datac13_nin)
self.datac13_nin = codec2.api.freedv_nin(self.datac13_freedv)
if nbytes == self.datac13_bytes_per_frame:
self.rx_total_frames_datac13 = self.rx_total_frames_datac13 + 1
self.rx_frames_datac13 = self.rx_frames_datac13 + 1
if self.rx_frames_datac13 == self.N_FRAMES_PER_BURST:
self.rx_frames_datac13 = 0
self.rx_bursts_datac13 = self.rx_bursts_datac13 + 1
while self.datac1_buffer.nbuffer >= self.datac1_nin:
# demodulate audio
nbytes = codec2.api.freedv_rawdatarx(
self.datac1_freedv,
self.datac1_bytes_out,
self.datac1_buffer.buffer.ctypes,
)
self.datac1_buffer.pop(self.datac1_nin)
self.datac1_nin = codec2.api.freedv_nin(self.datac1_freedv)
if nbytes == self.datac1_bytes_per_frame:
self.rx_total_frames_datac1 = self.rx_total_frames_datac1 + 1
self.rx_frames_datac1 = self.rx_frames_datac1 + 1
if self.rx_frames_datac1 == self.N_FRAMES_PER_BURST:
self.rx_frames_datac1 = 0
self.rx_bursts_datac1 = self.rx_bursts_datac1 + 1
while self.datac3_buffer.nbuffer >= self.datac3_nin:
# demodulate audio
nbytes = codec2.api.freedv_rawdatarx(
self.datac3_freedv,
self.datac3_bytes_out,
self.datac3_buffer.buffer.ctypes,
)
self.datac3_buffer.pop(self.datac3_nin)
self.datac3_nin = codec2.api.freedv_nin(self.datac3_freedv)
if nbytes == self.datac3_bytes_per_frame:
self.rx_total_frames_datac3 = self.rx_total_frames_datac3 + 1
self.rx_frames_datac3 = self.rx_frames_datac3 + 1
if self.rx_frames_datac3 == self.N_FRAMES_PER_BURST:
self.rx_frames_datac3 = 0
self.rx_bursts_datac3 = self.rx_bursts_datac3 + 1
if (
self.rx_bursts_datac13 and self.rx_bursts_datac1 and self.rx_bursts_datac3
) == self.N_BURSTS:
self.receive = False
return (None, pyaudio.paContinue)
def print_stats(self):
while self.receive:
time.sleep(0.01)
if self.DEBUGGING_MODE:
self.datac13_rxstatus = codec2.api.freedv_get_rx_status(
self.datac13_freedv
)
self.datac13_rxstatus = codec2.api.rx_sync_flags_to_text[
self.datac13_rxstatus
]
self.datac1_rxstatus = codec2.api.freedv_get_rx_status(
self.datac1_freedv
)
self.datac1_rxstatus = codec2.api.rx_sync_flags_to_text[
self.datac1_rxstatus
]
self.datac3_rxstatus = codec2.api.freedv_get_rx_status(
self.datac3_freedv
)
self.datac3_rxstatus = codec2.api.rx_sync_flags_to_text[
self.datac3_rxstatus
]
print(
"NIN0: %5d RX_STATUS0: %4s NIN1: %5d RX_STATUS1: %4s NIN3: %5d RX_STATUS3: %4s"
% (
self.datac13_nin,
self.datac13_rxstatus,
self.datac1_nin,
self.datac1_rxstatus,
self.datac3_nin,
self.datac3_rxstatus,
),
file=sys.stderr,
)
def run_audio(self):
try:
print("starting pyaudio callback", file=sys.stderr)
self.stream_rx.start_stream()
except Exception as e:
print(f"pyAudio error: {e}", file=sys.stderr)
while self.receive and time.time() < self.timeout:
time.sleep(1)
if time.time() >= self.timeout and self.stream_rx.is_active():
print("TIMEOUT REACHED")
self.receive = False
if self.nread_exceptions:
print(
"nread_exceptions %d - receive audio lost! Consider increasing Pyaudio frames_per_buffer..."
% self.nread_exceptions,
file=sys.stderr,
)
print(
f"datac13: {self.rx_bursts_datac13}/{self.rx_total_frames_datac13} DATAC1: {self.rx_bursts_datac1}/{self.rx_total_frames_datac1} DATAC3: {self.rx_bursts_datac3}/{self.rx_total_frames_datac3}",
file=sys.stderr,
)
self.frx.close()
# cloese pyaudio instance
self.stream_rx.close()
self.p.terminate()
test = Test()
test.run_audio()

311
modem/data cemetery/test/util_callback_multimode_rx_outside.py
View file

@ -1,311 +0,0 @@
#!/usr/bin/env python3
# -*- coding: utf-8 -*-
"""
Created on Wed Dec 23 07:04:24 2020
@author: DJ2LS
"""
import argparse
import ctypes
import sys
import time
import numpy as np
import pyaudio
sys.path.insert(0, "..")
from modem import codec2
# --------------------------------------------GET PARAMETER INPUTS
parser = argparse.ArgumentParser(description="FreeDATA audio test")
parser.add_argument("--bursts", dest="N_BURSTS", default=1, type=int)
parser.add_argument("--framesperburst", dest="N_FRAMES_PER_BURST", default=1, type=int)
parser.add_argument(
"--audiodev",
dest="AUDIO_INPUT_DEVICE",
default=-1,
type=int,
help="audio device number to use",
)
parser.add_argument("--debug", dest="DEBUGGING_MODE", action="store_true")
parser.add_argument(
"--list",
dest="LIST",
action="store_true",
help="list audio devices by number and exit",
)
parser.add_argument(
"--timeout",
dest="TIMEOUT",
default=10,
type=int,
help="Timeout (seconds) before test ends",
)
args, _ = parser.parse_known_args()
if args.LIST:
p = pyaudio.PyAudio()
for dev in range(p.get_device_count()):
print("audiodev: ", dev, p.get_device_info_by_index(dev)["name"])
class Test:
def __init__(self):
self.N_BURSTS = args.N_BURSTS
self.N_FRAMES_PER_BURST = args.N_FRAMES_PER_BURST
self.AUDIO_INPUT_DEVICE = args.AUDIO_INPUT_DEVICE
self.DEBUGGING_MODE = args.DEBUGGING_MODE
self.TIMEOUT = args.TIMEOUT
# AUDIO PARAMETERS
self.AUDIO_FRAMES_PER_BUFFER = (
2400 * 2
) # <- consider increasing if you get nread_exceptions > 0
self.MODEM_SAMPLE_RATE = codec2.api.FREEDV_FS_8000
self.AUDIO_SAMPLE_RATE_RX = 48000
# make sure our resampler will work
assert (
self.AUDIO_SAMPLE_RATE_RX / self.MODEM_SAMPLE_RATE
) == codec2.api.FDMDV_OS_48
# check if we want to use an audio device then do a pyaudio init
if self.AUDIO_INPUT_DEVICE != -1:
self.p = pyaudio.PyAudio()
# auto search for loopback devices
if self.AUDIO_INPUT_DEVICE == -2:
loopback_list = [
dev
for dev in range(self.p.get_device_count())
if "Loopback: PCM" in self.p.get_device_info_by_index(dev)["name"]
]
if len(loopback_list) >= 2:
self.AUDIO_INPUT_DEVICE = loopback_list[0] # 0 = RX 1 = TX
print(f"loopback_list rx: {loopback_list}", file=sys.stderr)
else:
sys.exit()
print(
f"AUDIO INPUT DEVICE: {self.AUDIO_INPUT_DEVICE} DEVICE: {self.p.get_device_info_by_index(self.AUDIO_INPUT_DEVICE)['name']} \
AUDIO SAMPLE RATE: {self.AUDIO_SAMPLE_RATE_RX}",
file=sys.stderr,
)
self.stream_rx = self.p.open(
format=pyaudio.paInt16,
channels=1,
rate=self.AUDIO_SAMPLE_RATE_RX,
frames_per_buffer=self.AUDIO_FRAMES_PER_BUFFER,
input=True,
output=False,
input_device_index=self.AUDIO_INPUT_DEVICE,
stream_callback=self.callback,
)
else:
print("test_callback_multimode_rx_outside: Not written for STDIN usage.")
print("Exiting.")
sys.exit()
# open codec2 instance
self.datac13_freedv = ctypes.cast(
codec2.api.freedv_open(codec2.FREEDV_MODE.datac13.value), ctypes.c_void_p
)
self.datac13_bytes_per_frame = int(
codec2.api.freedv_get_bits_per_modem_frame(self.datac13_freedv) / 8
)
self.datac13_bytes_out = ctypes.create_string_buffer(self.datac13_bytes_per_frame)
codec2.api.freedv_set_frames_per_burst(
self.datac13_freedv, self.N_FRAMES_PER_BURST
)
self.datac13_buffer = codec2.audio_buffer(2 * self.AUDIO_FRAMES_PER_BUFFER)
self.datac1_freedv = ctypes.cast(
codec2.api.freedv_open(codec2.FREEDV_MODE.datac1.value), ctypes.c_void_p
)
self.datac1_bytes_per_frame = int(
codec2.api.freedv_get_bits_per_modem_frame(self.datac1_freedv) / 8
)
self.datac1_bytes_out = ctypes.create_string_buffer(self.datac1_bytes_per_frame)
codec2.api.freedv_set_frames_per_burst(
self.datac1_freedv, self.N_FRAMES_PER_BURST
)
self.datac1_buffer = codec2.audio_buffer(2 * self.AUDIO_FRAMES_PER_BUFFER)
self.datac3_freedv = ctypes.cast(
codec2.api.freedv_open(codec2.FREEDV_MODE.datac3.value), ctypes.c_void_p
)
self.datac3_bytes_per_frame = int(
codec2.api.freedv_get_bits_per_modem_frame(self.datac3_freedv) / 8
)
self.datac3_bytes_out = ctypes.create_string_buffer(self.datac3_bytes_per_frame)
codec2.api.freedv_set_frames_per_burst(
self.datac3_freedv, self.N_FRAMES_PER_BURST
)
self.datac3_buffer = codec2.audio_buffer(2 * self.AUDIO_FRAMES_PER_BUFFER)
# SET COUNTERS
self.rx_total_frames_datac13 = 0
self.rx_frames_datac13 = 0
self.rx_bursts_datac13 = 0
self.rx_total_frames_datac1 = 0
self.rx_frames_datac1 = 0
self.rx_bursts_datac1 = 0
self.rx_total_frames_datac3 = 0
self.rx_frames_datac3 = 0
self.rx_bursts_datac3 = 0
self.rx_errors = 0
self.nread_exceptions = 0
self.timeout = time.time() + self.TIMEOUT
self.receive = True
self.resampler = codec2.resampler()
# Copy received 48 kHz to a file. Listen to this file with:
# aplay -r 48000 -f S16_LE rx48_callback.raw
# Corruption of this file is a good way to detect audio card issues
self.frx = open("rx48_callback_multimode.raw", mode="wb")
# initial nin values
self.datac13_nin = codec2.api.freedv_nin(self.datac13_freedv)
self.datac1_nin = codec2.api.freedv_nin(self.datac1_freedv)
self.datac3_nin = codec2.api.freedv_nin(self.datac3_freedv)
def callback(self, data_in48k, frame_count, time_info, status):
x = np.frombuffer(data_in48k, dtype=np.int16)
x.tofile(self.frx)
x = self.resampler.resample48_to_8(x)
self.datac13_buffer.push(x)
self.datac1_buffer.push(x)
self.datac3_buffer.push(x)
return (None, pyaudio.paContinue)
def print_stats(self):
if self.DEBUGGING_MODE:
self.datac13_rxstatus = codec2.api.freedv_get_rx_status(self.datac13_freedv)
self.datac13_rxstatus = codec2.api.rx_sync_flags_to_text[
self.datac13_rxstatus
]
self.datac1_rxstatus = codec2.api.freedv_get_rx_status(self.datac1_freedv)
self.datac1_rxstatus = codec2.api.rx_sync_flags_to_text[
self.datac1_rxstatus
]
self.datac3_rxstatus = codec2.api.freedv_get_rx_status(self.datac3_freedv)
self.datac3_rxstatus = codec2.api.rx_sync_flags_to_text[
self.datac3_rxstatus
]
print(
"NIN0: %5d RX_STATUS0: %4s NIN1: %5d RX_STATUS1: %4s NIN3: %5d RX_STATUS3: %4s"
% (
self.datac13_nin,
self.datac13_rxstatus,
self.datac1_nin,
self.datac1_rxstatus,
self.datac3_nin,
self.datac3_rxstatus,
),
file=sys.stderr,
)
def run_audio(self):
try:
print("starting pyaudio callback", file=sys.stderr)
self.stream_rx.start_stream()
except Exception as e:
print(f"pyAudio error: {e}", file=sys.stderr)
while self.receive and time.time() < self.timeout:
while self.datac13_buffer.nbuffer >= self.datac13_nin:
# demodulate audio
nbytes = codec2.api.freedv_rawdatarx(
self.datac13_freedv,
self.datac13_bytes_out,
self.datac13_buffer.buffer.ctypes,
)
self.datac13_buffer.pop(self.datac13_nin)
self.datac13_nin = codec2.api.freedv_nin(self.datac13_freedv)
if nbytes == self.datac13_bytes_per_frame:
self.rx_total_frames_datac13 = self.rx_total_frames_datac13 + 1
self.rx_frames_datac13 = self.rx_frames_datac13 + 1
if self.rx_frames_datac13 == self.N_FRAMES_PER_BURST:
self.rx_frames_datac13 = 0
self.rx_bursts_datac13 = self.rx_bursts_datac13 + 1
self.print_stats()
while self.datac1_buffer.nbuffer >= self.datac1_nin:
# demodulate audio
nbytes = codec2.api.freedv_rawdatarx(
self.datac1_freedv,
self.datac1_bytes_out,
self.datac1_buffer.buffer.ctypes,
)
self.datac1_buffer.pop(self.datac1_nin)
self.datac1_nin = codec2.api.freedv_nin(self.datac1_freedv)
if nbytes == self.datac1_bytes_per_frame:
self.rx_total_frames_datac1 = self.rx_total_frames_datac1 + 1
self.rx_frames_datac1 = self.rx_frames_datac1 + 1
if self.rx_frames_datac1 == self.N_FRAMES_PER_BURST:
self.rx_frames_datac1 = 0
self.rx_bursts_datac1 = self.rx_bursts_datac1 + 1
self.print_stats()
while self.datac3_buffer.nbuffer >= self.datac3_nin:
# demodulate audio
nbytes = codec2.api.freedv_rawdatarx(
self.datac3_freedv,
self.datac3_bytes_out,
self.datac3_buffer.buffer.ctypes,
)
self.datac3_buffer.pop(self.datac3_nin)
self.datac3_nin = codec2.api.freedv_nin(self.datac3_freedv)
if nbytes == self.datac3_bytes_per_frame:
self.rx_total_frames_datac3 = self.rx_total_frames_datac3 + 1
self.rx_frames_datac3 = self.rx_frames_datac3 + 1
if self.rx_frames_datac3 == self.N_FRAMES_PER_BURST:
self.rx_frames_datac3 = 0
self.rx_bursts_datac3 = self.rx_bursts_datac3 + 1
self.print_stats()
if (
self.rx_bursts_datac13
and self.rx_bursts_datac1
and self.rx_bursts_datac3
) == self.N_BURSTS:
self.receive = False
if time.time() >= self.timeout:
print("TIMEOUT REACHED")
if self.nread_exceptions:
print(
"nread_exceptions %d - receive audio lost! Consider increasing Pyaudio frames_per_buffer..."
% self.nread_exceptions,
file=sys.stderr,
)
print(
f"datac13: {self.rx_bursts_datac13}/{self.rx_total_frames_datac13} DATAC1: {self.rx_bursts_datac1}/{self.rx_total_frames_datac1} DATAC3: {self.rx_bursts_datac3}/{self.rx_total_frames_datac3}",
file=sys.stderr,
)
self.frx.close()
# cloese pyaudio instance
self.stream_rx.close()
self.p.terminate()
test = Test()
test.run_audio()

256
modem/data cemetery/test/util_callback_multimode_tx.py
View file

@ -1,256 +0,0 @@
#!/usr/bin/env python3
# -*- coding: utf-8 -*-
"""
Created on Wed Dec 23 07:04:24 2020
@author: DJ2LS
"""
import argparse
import ctypes
import queue
import sys
import time
import numpy as np
import pyaudio
sys.path.insert(0, "..")
from modem import codec2
# --------------------------------------------GET PARAMETER INPUTS
parser = argparse.ArgumentParser(description="FreeDATA audio test")
parser.add_argument("--bursts", dest="N_BURSTS", default=1, type=int)
parser.add_argument("--framesperburst", dest="N_FRAMES_PER_BURST", default=1, type=int)
parser.add_argument("--delay", dest="DELAY_BETWEEN_BURSTS", default=500, type=int)
parser.add_argument(
"--audiodev",
dest="AUDIO_OUTPUT_DEVICE",
default=-1,
type=int,
help="audio output device number to use",
)
parser.add_argument(
"--list",
dest="LIST",
action="store_true",
help="list audio devices by number and exit",
)
parser.add_argument(
"--testframes",
dest="TESTFRAMES",
action="store_true",
default=False,
help="generate testframes",
)
args, _ = parser.parse_known_args()
if args.LIST:
p = pyaudio.PyAudio()
for dev in range(p.get_device_count()):
print("audiodev: ", dev, p.get_device_info_by_index(dev)["name"])
sys.exit()
class Test:
def __init__(self):
self.dataqueue = queue.Queue()
self.N_BURSTS = args.N_BURSTS
self.N_FRAMES_PER_BURST = args.N_FRAMES_PER_BURST
self.AUDIO_OUTPUT_DEVICE = args.AUDIO_OUTPUT_DEVICE
self.DELAY_BETWEEN_BURSTS = args.DELAY_BETWEEN_BURSTS / 1000
# AUDIO PARAMETERS
# v-- consider increasing if you get nread_exceptions > 0
self.AUDIO_FRAMES_PER_BUFFER = 2400
self.MODEM_SAMPLE_RATE = codec2.api.FREEDV_FS_8000
self.AUDIO_SAMPLE_RATE_TX = 48000
# make sure our resampler will work
assert (
self.AUDIO_SAMPLE_RATE_TX / self.MODEM_SAMPLE_RATE
) == codec2.api.FDMDV_OS_48
self.transmit = True
self.resampler = codec2.resampler()
# check if we want to use an audio device then do a pyaudio init
if self.AUDIO_OUTPUT_DEVICE != -1:
self.p = pyaudio.PyAudio()
# auto search for loopback devices
if self.AUDIO_OUTPUT_DEVICE == -2:
loopback_list = [
dev
for dev in range(self.p.get_device_count())
if "Loopback: PCM" in self.p.get_device_info_by_index(dev)["name"]
]
if len(loopback_list) >= 2:
self.AUDIO_OUTPUT_DEVICE = loopback_list[0] # 0 = RX 1 = TX
print(f"loopback_list rx: {loopback_list}", file=sys.stderr)
else:
sys.exit()
print(
f"AUDIO OUTPUT DEVICE: {self.AUDIO_OUTPUT_DEVICE} "
f"DEVICE: {self.p.get_device_info_by_index(self.AUDIO_OUTPUT_DEVICE)['name']} "
f"AUDIO SAMPLE RATE: {self.AUDIO_SAMPLE_RATE_TX}",
file=sys.stderr,
)
self.stream_tx = self.p.open(
format=pyaudio.paInt16,
channels=1,
rate=self.AUDIO_SAMPLE_RATE_TX,
frames_per_buffer=self.AUDIO_FRAMES_PER_BUFFER,
input=False,
output=True,
output_device_index=self.AUDIO_OUTPUT_DEVICE,
stream_callback=self.callback,
)
else:
print("test_callback_multimode_tx: Not written for STDOUT usage.")
print("Exiting.")
sys.exit()
# Copy received 48 kHz to a file. Listen to this file with:
# aplay -r 48000 -f S16_LE rx48_callback.raw
# Corruption of this file is a good way to detect audio card issues
self.ftx = open("tx48_callback.raw", mode="wb")
# data binary string
if args.TESTFRAMES:
self.data_out = bytearray(14)
self.data_out[:1] = bytes([255])
self.data_out[1:2] = bytes([1])
self.data_out[2:] = b"HELLO WORLD"
else:
self.data_out = b"HELLO WORLD!"
def callback(self, data_in48k, frame_count, time_info, status):
data_out48k = self.dataqueue.get()
return (data_out48k, pyaudio.paContinue)
def run_audio(self):
try:
print("starting pyaudio callback", file=sys.stderr)
self.stream_tx.start_stream()
except Exception as e:
print(f"pyAudio error: {e}", file=sys.stderr)
sheeps = 0
while self.transmit:
time.sleep(1)
sheeps = sheeps + 1
print(f"counting sheeps...{sheeps}")
self.ftx.close()
# close pyaudio instance
self.stream_tx.close()
self.p.terminate()
def create_modulation(self):
modes = [
codec2.FREEDV_MODE.datac13.value,
codec2.FREEDV_MODE.datac1.value,
codec2.FREEDV_MODE.datac3.value,
]
for m in modes:
freedv = ctypes.cast(codec2.api.freedv_open(m), ctypes.c_void_p)
n_tx_modem_samples = codec2.api.freedv_get_n_tx_modem_samples(freedv)
mod_out = ctypes.create_string_buffer(2 * n_tx_modem_samples)
n_tx_preamble_modem_samples = (
codec2.api.freedv_get_n_tx_preamble_modem_samples(freedv)
)
mod_out_preamble = ctypes.create_string_buffer(
2 * n_tx_preamble_modem_samples
)
n_tx_postamble_modem_samples = (
codec2.api.freedv_get_n_tx_postamble_modem_samples(freedv)
)
mod_out_postamble = ctypes.create_string_buffer(
2 * n_tx_postamble_modem_samples
)
bytes_per_frame = int(
codec2.api.freedv_get_bits_per_modem_frame(freedv) / 8
)
payload_per_frame = bytes_per_frame - 2
buffer = bytearray(payload_per_frame)
# set buffer size to length of data which will be sent
buffer[: len(self.data_out)] = self.data_out
crc = ctypes.c_ushort(
codec2.api.freedv_gen_crc16(bytes(buffer), payload_per_frame)
) # generate CRC16
# convert crc to 2 byte hex string
crc = crc.value.to_bytes(2, byteorder="big")
buffer += crc # append crc16 to buffer
data = (ctypes.c_ubyte * bytes_per_frame).from_buffer_copy(buffer)
for i in range(1, self.N_BURSTS + 1):
# write preamble to txbuffer
codec2.api.freedv_rawdatapreambletx(freedv, mod_out_preamble)
txbuffer = bytes(mod_out_preamble)
# create modulaton for N = FRAMESPERBURST and append it to txbuffer
for n in range(1, self.N_FRAMES_PER_BURST + 1):
data = (ctypes.c_ubyte * bytes_per_frame).from_buffer_copy(buffer)
codec2.api.freedv_rawdatatx(
freedv, mod_out, data
) # modulate DATA and save it into mod_out pointer
txbuffer += bytes(mod_out)
print(
f"GENERATING TX BURST: {i}/{self.N_BURSTS} FRAME: {n}/{self.N_FRAMES_PER_BURST}",
file=sys.stderr,
)
# append postamble to txbuffer
codec2.api.freedv_rawdatapostambletx(freedv, mod_out_postamble)
txbuffer += bytes(mod_out_postamble)
# append a delay between bursts as audio silence
samples_delay = int(self.MODEM_SAMPLE_RATE * self.DELAY_BETWEEN_BURSTS)
mod_out_silence = ctypes.create_string_buffer(samples_delay * 2)
txbuffer += bytes(mod_out_silence)
# resample up to 48k (resampler works on np.int16)
x = np.frombuffer(txbuffer, dtype=np.int16)
txbuffer_48k = self.resampler.resample8_to_48(x)
# split modulated audio to chunks
# https://newbedev.com/how-to-split-a-byte-string-into-separate-bytes-in-python
txbuffer_48k = bytes(txbuffer_48k)
chunk = [
txbuffer_48k[i : i + self.AUDIO_FRAMES_PER_BUFFER * 2]
for i in range(
0, len(txbuffer_48k), self.AUDIO_FRAMES_PER_BUFFER * 2
)
]
# add modulated chunks to fifo buffer
for c in chunk:
# if data is shorter than the expcected audio frames per buffer we need to append 0
# to prevent the callback from stucking/crashing
if len(c) < self.AUDIO_FRAMES_PER_BUFFER * 2:
c += bytes(self.AUDIO_FRAMES_PER_BUFFER * 2 - len(c))
self.dataqueue.put(c)
test = Test()
test.create_modulation()
test.run_audio()

219
modem/data cemetery/test/util_callback_rx.py
View file

@ -1,219 +0,0 @@
#!/usr/bin/env python3
# -*- coding: utf-8 -*-
"""
Created on Wed Dec 23 07:04:24 2020
@author: DJ2LS
"""
import argparse
import ctypes
import sys
import time
import numpy as np
import pyaudio
sys.path.insert(0, "..")
from modem import codec2
# --------------------------------------------GET PARAMETER INPUTS
parser = argparse.ArgumentParser(description="FreeDATA audio test")
parser.add_argument("--bursts", dest="N_BURSTS", default=1, type=int)
parser.add_argument("--framesperburst", dest="N_FRAMES_PER_BURST", default=1, type=int)
parser.add_argument(
"--mode", dest="FREEDV_MODE", type=str, choices=["datac13", "datac1", "datac3"]
)
parser.add_argument(
"--audiodev",
dest="AUDIO_INPUT_DEVICE",
default=-1,
type=int,
help="audio device number to use, use -2 to automatically select a loopback device",
)
parser.add_argument("--debug", dest="DEBUGGING_MODE", action="store_true")
parser.add_argument(
"--timeout",
dest="TIMEOUT",
default=60,
type=int,
help="Timeout (seconds) before test ends",
)
parser.add_argument(
"--list",
dest="LIST",
action="store_true",
help="list audio devices by number and exit",
)
args, _ = parser.parse_known_args()
if args.LIST:
p = pyaudio.PyAudio()
for dev in range(p.get_device_count()):
print("audiodev: ", dev, p.get_device_info_by_index(dev)["name"])
sys.exit()
class Test:
def __init__(self):
self.N_BURSTS = args.N_BURSTS
self.N_FRAMES_PER_BURST = args.N_FRAMES_PER_BURST
self.AUDIO_INPUT_DEVICE = args.AUDIO_INPUT_DEVICE
self.MODE = codec2.FREEDV_MODE[args.FREEDV_MODE].value
self.DEBUGGING_MODE = args.DEBUGGING_MODE
self.TIMEOUT = args.TIMEOUT
# AUDIO PARAMETERS
self.AUDIO_FRAMES_PER_BUFFER = (
2400 * 2
) # <- consider increasing if you get nread_exceptions > 0
self.MODEM_SAMPLE_RATE = codec2.api.FREEDV_FS_8000
self.AUDIO_SAMPLE_RATE_RX = 48000
# make sure our resampler will work
assert (
self.AUDIO_SAMPLE_RATE_RX / self.MODEM_SAMPLE_RATE
) == codec2.api.FDMDV_OS_48
# check if we want to use an audio device then do a pyaudio init
if self.AUDIO_INPUT_DEVICE != -1:
self.p = pyaudio.PyAudio()
# auto search for loopback devices
if self.AUDIO_INPUT_DEVICE == -2:
loopback_list = [
dev
for dev in range(self.p.get_device_count())
if "Loopback: PCM"
in self.p.get_device_info_by_index(dev)["name"]
]
if len(loopback_list) >= 2:
self.AUDIO_INPUT_DEVICE = loopback_list[0] # 0 = RX 1 = TX
print(f"loopback_list rx: {loopback_list}", file=sys.stderr)
else:
sys.exit()
print(
f"AUDIO INPUT DEVICE: {self.AUDIO_INPUT_DEVICE} DEVICE: {self.p.get_device_info_by_index(self.AUDIO_INPUT_DEVICE)['name']} \
AUDIO SAMPLE RATE: {self.AUDIO_SAMPLE_RATE_RX}",
file=sys.stderr,
)
self.stream_rx = self.p.open(
format=pyaudio.paInt16,
channels=1,
rate=self.AUDIO_SAMPLE_RATE_RX,
frames_per_buffer=self.AUDIO_FRAMES_PER_BUFFER,
input=True,
output=False,
input_device_index=self.AUDIO_INPUT_DEVICE,
stream_callback=self.callback,
)
# open codec2 instance
self.freedv = ctypes.cast(codec2.api.freedv_open(self.MODE), ctypes.c_void_p)
# get number of bytes per frame for mode
self.bytes_per_frame = int(
codec2.api.freedv_get_bits_per_modem_frame(self.freedv) / 8
)
self.bytes_out = ctypes.create_string_buffer(self.bytes_per_frame)
codec2.api.freedv_set_frames_per_burst(self.freedv, self.N_FRAMES_PER_BURST)
self.total_n_bytes = 0
self.rx_total_frames = 0
self.rx_frames = 0
self.rx_bursts = 0
self.rx_errors = 0
self.nread_exceptions = 0
self.timeout = time.time() + self.TIMEOUT
self.receive = True
self.audio_buffer = codec2.audio_buffer(self.AUDIO_FRAMES_PER_BUFFER * 2)
self.resampler = codec2.resampler()
# Copy received 48 kHz to a file. Listen to this file with:
# aplay -r 48000 -f S16_LE rx48_callback.raw
# Corruption of this file is a good way to detect audio card issues
self.frx = open("rx48_callback.raw", mode="wb")
def callback(self, data_in48k, frame_count, time_info, status):
x = np.frombuffer(data_in48k, dtype=np.int16)
x.tofile(self.frx)
x = self.resampler.resample48_to_8(x)
self.audio_buffer.push(x)
# when we have enough samples call FreeDV Rx
nin = codec2.api.freedv_nin(self.freedv)
while self.audio_buffer.nbuffer >= nin:
# demodulate audio
nbytes = codec2.api.freedv_rawdatarx(
self.freedv, self.bytes_out, self.audio_buffer.buffer.ctypes
)
self.audio_buffer.pop(nin)
# call me on every loop!
nin = codec2.api.freedv_nin(self.freedv)
rx_status = codec2.api.freedv_get_rx_status(self.freedv)
if rx_status & codec2.api.FREEDV_RX_BIT_ERRORS:
self.rx_errors = self.rx_errors + 1
if self.DEBUGGING_MODE:
rx_status = codec2.api.rx_sync_flags_to_text[rx_status]
print(
"nin: %5d rx_status: %4s naudio_buffer: %4d"
% (nin, rx_status, self.audio_buffer.nbuffer),
file=sys.stderr,
)
if nbytes:
self.total_n_bytes = self.total_n_bytes + nbytes
if nbytes == self.bytes_per_frame:
self.rx_total_frames = self.rx_total_frames + 1
self.rx_frames = self.rx_frames + 1
if self.rx_frames == self.N_FRAMES_PER_BURST:
self.rx_frames = 0
self.rx_bursts = self.rx_bursts + 1
if self.rx_bursts == self.N_BURSTS:
self.receive = False
return (None, pyaudio.paContinue)
def run_audio(self):
try:
print("starting pyaudio callback", file=sys.stderr)
self.stream_rx.start_stream()
except Exception as e:
print(f"pyAudio error: {e}", file=sys.stderr)
while self.receive and time.time() < self.timeout:
time.sleep(1)
if time.time() >= self.timeout:
print("TIMEOUT REACHED")
if self.nread_exceptions:
print(
"nread_exceptions %d - receive audio lost! Consider increasing Pyaudio frames_per_buffer..."
% self.nread_exceptions,
file=sys.stderr,
)
print(
f"RECEIVED BURSTS: {self.rx_bursts} RECEIVED FRAMES: {self.rx_total_frames} RX_ERRORS: {self.rx_errors}",
file=sys.stderr,
)
self.frx.close()
# cloese pyaudio instance
self.stream_rx.close()
self.p.terminate()
test = Test()
test.run_audio()

217
modem/data cemetery/test/util_callback_rx_outside.py
View file

@ -1,217 +0,0 @@
#!/usr/bin/env python3
# -*- coding: utf-8 -*-
"""
Created on Wed Dec 23 07:04:24 2020
@author: DJ2LS
"""
import argparse
import ctypes
import sys
import time
import numpy as np
import pyaudio
sys.path.insert(0, "..")
from modem import codec2
# --------------------------------------------GET PARAMETER INPUTS
parser = argparse.ArgumentParser(description="FreeDATA audio test")
parser.add_argument("--bursts", dest="N_BURSTS", default=1, type=int)
parser.add_argument("--framesperburst", dest="N_FRAMES_PER_BURST", default=1, type=int)
parser.add_argument(
"--mode", dest="FREEDV_MODE", type=str, choices=["datac13", "datac1", "datac3"]
)
parser.add_argument(
"--audiodev",
dest="AUDIO_INPUT_DEVICE",
default=-1,
type=int,
help="audio device number to use, use -2 to automatically select a loopback device",
)
parser.add_argument("--debug", dest="DEBUGGING_MODE", action="store_true")
parser.add_argument(
"--timeout",
dest="TIMEOUT",
default=10,
type=int,
help="Timeout (seconds) before test ends",
)
parser.add_argument(
"--list",
dest="LIST",
action="store_true",
help="list audio devices by number and exit",
)
args, _ = parser.parse_known_args()
if args.LIST:
p = pyaudio.PyAudio()
for dev in range(p.get_device_count()):
print("audiodev: ", dev, p.get_device_info_by_index(dev)["name"])
sys.exit()
class Test:
def __init__(self):
self.N_BURSTS = args.N_BURSTS
self.N_FRAMES_PER_BURST = args.N_FRAMES_PER_BURST
self.AUDIO_INPUT_DEVICE = args.AUDIO_INPUT_DEVICE
self.MODE = codec2.FREEDV_MODE[args.FREEDV_MODE].value
self.DEBUGGING_MODE = args.DEBUGGING_MODE
self.TIMEOUT = args.TIMEOUT
# AUDIO PARAMETERS
self.AUDIO_FRAMES_PER_BUFFER = (
2400 * 2
) # <- consider increasing if you get nread_exceptions > 0
self.MODEM_SAMPLE_RATE = codec2.api.FREEDV_FS_8000
self.AUDIO_SAMPLE_RATE_RX = 48000
# make sure our resampler will work
assert (
self.AUDIO_SAMPLE_RATE_RX / self.MODEM_SAMPLE_RATE
) == codec2.api.FDMDV_OS_48
# check if we want to use an audio device then do a pyaudio init
if self.AUDIO_INPUT_DEVICE != -1:
self.p = pyaudio.PyAudio()
# auto search for loopback devices
if self.AUDIO_INPUT_DEVICE == -2:
loopback_list = [
dev
for dev in range(self.p.get_device_count())
if "Loopback: PCM"
in self.p.get_device_info_by_index(dev)["name"]
]
if len(loopback_list) >= 2:
self.AUDIO_INPUT_DEVICE = loopback_list[0] # 0 = RX 1 = TX
print(f"loopback_list rx: {loopback_list}", file=sys.stderr)
else:
sys.exit()
print(
f"AUDIO INPUT DEVICE: {self.AUDIO_INPUT_DEVICE} DEVICE: {self.p.get_device_info_by_index(self.AUDIO_INPUT_DEVICE)['name']} \
AUDIO SAMPLE RATE: {self.AUDIO_SAMPLE_RATE_RX}",
file=sys.stderr,
)
self.stream_rx = self.p.open(
format=pyaudio.paInt16,
channels=1,
rate=self.AUDIO_SAMPLE_RATE_RX,
frames_per_buffer=self.AUDIO_FRAMES_PER_BUFFER,
input=True,
output=False,
input_device_index=self.AUDIO_INPUT_DEVICE,
stream_callback=self.callback,
)
# open codec2 instance
self.freedv = ctypes.cast(codec2.api.freedv_open(self.MODE), ctypes.c_void_p)
# get number of bytes per frame for mode
self.bytes_per_frame = int(
codec2.api.freedv_get_bits_per_modem_frame(self.freedv) / 8
)
self.bytes_out = ctypes.create_string_buffer(self.bytes_per_frame * 2)
codec2.api.freedv_set_frames_per_burst(self.freedv, self.N_FRAMES_PER_BURST)
self.total_n_bytes = 0
self.rx_total_frames = 0
self.rx_frames = 0
self.rx_bursts = 0
self.rx_errors = 0
self.nread_exceptions = 0
self.timeout = time.time() + self.TIMEOUT
self.receive = True
self.audio_buffer = codec2.audio_buffer(self.AUDIO_FRAMES_PER_BUFFER * 2)
self.resampler = codec2.resampler()
# Copy received 48 kHz to a file. Listen to this file with:
# aplay -r 48000 -f S16_LE rx48_callback.raw
# Corruption of this file is a good way to detect audio card issues
self.frx = open("rx48_callback.raw", mode="wb")
def callback(self, data_in48k, frame_count, time_info, status):
x = np.frombuffer(data_in48k, dtype=np.int16)
x.tofile(self.frx)
x = self.resampler.resample48_to_8(x)
self.audio_buffer.push(x)
return (None, pyaudio.paContinue)
def run_audio(self):
try:
print("starting pyaudio callback", file=sys.stderr)
self.stream_rx.start_stream()
except Exception as e:
print(f"pyAudio error: {e}", file=sys.stderr)
while self.receive and time.time() < self.timeout:
# time.sleep(1)
# when we have enough samples call FreeDV Rx
nin = codec2.api.freedv_nin(self.freedv)
while self.audio_buffer.nbuffer >= nin:
# demodulate audio
nbytes = codec2.api.freedv_rawdatarx(
self.freedv, self.bytes_out, self.audio_buffer.buffer.ctypes
)
self.audio_buffer.pop(nin)
# call me on every loop!
nin = codec2.api.freedv_nin(self.freedv)
rx_status = codec2.api.freedv_get_rx_status(self.freedv)
if rx_status & codec2.api.FREEDV_RX_BIT_ERRORS:
self.rx_errors = self.rx_errors + 1
if self.DEBUGGING_MODE:
rx_status = codec2.api.rx_sync_flags_to_text[rx_status]
print(
"nin: %5d rx_status: %4s naudio_buffer: %4d"
% (nin, rx_status, self.audio_buffer.nbuffer),
file=sys.stderr,
)
if nbytes:
self.total_n_bytes = self.total_n_bytes + nbytes
if nbytes == self.bytes_per_frame:
self.rx_total_frames = self.rx_total_frames + 1
self.rx_frames = self.rx_frames + 1
if self.rx_frames == self.N_FRAMES_PER_BURST:
self.rx_frames = 0
self.rx_bursts = self.rx_bursts + 1
if self.rx_bursts == self.N_BURSTS:
self.receive = False
if time.time() >= self.timeout:
print("TIMEOUT REACHED")
if self.nread_exceptions:
print(
"nread_exceptions %d - receive audio lost! Consider increasing Pyaudio frames_per_buffer..."
% self.nread_exceptions,
file=sys.stderr,
)
print(
f"RECEIVED BURSTS: {self.rx_bursts} RECEIVED FRAMES: {self.rx_total_frames} RX_ERRORS: {self.rx_errors}",
file=sys.stderr,
)
self.frx.close()
# cloese pyaudio instance
self.stream_rx.close()
self.p.terminate()
test = Test()
test.run_audio()

272
modem/data cemetery/test/util_callback_tx.py
View file

@ -1,272 +0,0 @@
#!/usr/bin/env python3
# -*- coding: utf-8 -*-
"""
Created on Wed Dec 23 07:04:24 2020
@author: DJ2LS
"""
import argparse
import ctypes
import queue
import sys
import time
import numpy as np
import pyaudio
sys.path.insert(0, "..")
from modem import codec2
# --------------------------------------------GET PARAMETER INPUTS
parser = argparse.ArgumentParser(description="FreeDATA audio test")
parser.add_argument("--bursts", dest="N_BURSTS", default=1, type=int)
parser.add_argument("--framesperburst", dest="N_FRAMES_PER_BURST", default=1, type=int)
parser.add_argument("--delay", dest="DELAY_BETWEEN_BURSTS", default=500, type=int)
parser.add_argument(
"--mode", dest="FREEDV_MODE", type=str, choices=["datac13", "datac1", "datac3"]
)
parser.add_argument(
"--audiodev",
dest="AUDIO_OUTPUT_DEVICE",
default=-1,
type=int,
help="audio device number to use, use -2 to automatically select a loopback device",
)
parser.add_argument(
"--list",
dest="LIST",
action="store_true",
help="list audio devices by number and exit",
)
parser.add_argument(
"--testframes",
dest="TESTFRAMES",
action="store_true",
default=False,
help="generate testframes",
)
args, _ = parser.parse_known_args()
if args.LIST:
p = pyaudio.PyAudio()
for dev in range(p.get_device_count()):
print("audiodev: ", dev, p.get_device_info_by_index(dev)["name"])
sys.exit()
class Test:
def __init__(self):
self.dataqueue = queue.Queue()
self.N_BURSTS = args.N_BURSTS
self.N_FRAMES_PER_BURST = args.N_FRAMES_PER_BURST
self.AUDIO_OUTPUT_DEVICE = args.AUDIO_OUTPUT_DEVICE
self.MODE = codec2.FREEDV_MODE[args.FREEDV_MODE].value
self.DELAY_BETWEEN_BURSTS = args.DELAY_BETWEEN_BURSTS / 1000
# AUDIO PARAMETERS
# v-- consider increasing if you get nread_exceptions > 0
self.AUDIO_FRAMES_PER_BUFFER = 2400
self.MODEM_SAMPLE_RATE = codec2.api.FREEDV_FS_8000
self.AUDIO_SAMPLE_RATE_TX = 48000
# make sure our resampler will work
assert (
self.AUDIO_SAMPLE_RATE_TX / self.MODEM_SAMPLE_RATE
) == codec2.api.FDMDV_OS_48
self.transmit = True
self.resampler = codec2.resampler()
# check if we want to use an audio device then do a pyaudio init
if self.AUDIO_OUTPUT_DEVICE != -1:
self.p = pyaudio.PyAudio()
# auto search for loopback devices
if self.AUDIO_OUTPUT_DEVICE == -2:
loopback_list = [
dev
for dev in range(self.p.get_device_count())
if "Loopback: PCM"
in self.p.get_device_info_by_index(dev)["name"]
]
if len(loopback_list) >= 2:
self.AUDIO_OUTPUT_DEVICE = loopback_list[0] # 0 = RX 1 = TX
print(f"loopback_list rx: {loopback_list}", file=sys.stderr)
else:
sys.exit()
print(
f"AUDIO OUTPUT DEVICE: {self.AUDIO_OUTPUT_DEVICE} DEVICE: {self.p.get_device_info_by_index(self.AUDIO_OUTPUT_DEVICE)['name']} \
AUDIO SAMPLE RATE: {self.AUDIO_SAMPLE_RATE_TX}",
file=sys.stderr,
)
self.stream_tx = self.p.open(
format=pyaudio.paInt16,
channels=1,
rate=self.AUDIO_SAMPLE_RATE_TX,
frames_per_buffer=self.AUDIO_FRAMES_PER_BUFFER,
input=False,
output=True,
output_device_index=self.AUDIO_OUTPUT_DEVICE,
stream_callback=self.callback,
)
# open codec2 instance
self.freedv = ctypes.cast(codec2.api.freedv_open(self.MODE), ctypes.c_void_p)
# get number of bytes per frame for mode
self.bytes_per_frame = int(
codec2.api.freedv_get_bits_per_modem_frame(self.freedv) / 8
)
self.bytes_out = ctypes.create_string_buffer(self.bytes_per_frame)
codec2.api.freedv_set_frames_per_burst(self.freedv, self.N_FRAMES_PER_BURST)
# Copy received 48 kHz to a file. Listen to this file with:
# aplay -r 48000 -f S16_LE rx48_callback.raw
# Corruption of this file is a good way to detect audio card issues
self.ftx = open("tx48_callback.raw", mode="wb")
# data binary string
if args.TESTFRAMES:
self.data_out = bytearray(14)
self.data_out[:1] = bytes([255])
self.data_out[1:2] = bytes([1])
self.data_out[2:] = b"HELLO WORLD"
else:
self.data_out = b"HELLO WORLD!"
def callback(self, data_in48k, frame_count, time_info, status):
data_out48k = self.dataqueue.get()
return (data_out48k, pyaudio.paContinue)
def run_audio(self):
try:
print("starting pyaudio callback", file=sys.stderr)
self.stream_tx.start_stream()
except Exception as e:
print(f"pyAudio error: {e}", file=sys.stderr)
sheeps = 0
while self.transmit:
time.sleep(1)
sheeps = sheeps + 1
print(f"counting sheeps...{sheeps}")
self.ftx.close()
# close pyaudio instance
self.stream_tx.close()
self.p.terminate()
def create_modulation(self):
# open codec2 instance
freedv = ctypes.cast(codec2.api.freedv_open(self.MODE), ctypes.c_void_p)
# get number of bytes per frame for mode
bytes_per_frame = int(codec2.api.freedv_get_bits_per_modem_frame(freedv) / 8)
payload_bytes_per_frame = bytes_per_frame - 2
# init buffer for data
n_tx_modem_samples = codec2.api.freedv_get_n_tx_modem_samples(freedv)
mod_out = ctypes.create_string_buffer(n_tx_modem_samples * 2)
# init buffer for preample
n_tx_preamble_modem_samples = codec2.api.freedv_get_n_tx_preamble_modem_samples(
freedv
)
mod_out_preamble = ctypes.create_string_buffer(n_tx_preamble_modem_samples * 2)
# init buffer for postamble
n_tx_postamble_modem_samples = (
codec2.api.freedv_get_n_tx_postamble_modem_samples(freedv)
)
mod_out_postamble = ctypes.create_string_buffer(n_tx_postamble_modem_samples * 2)
# create buffer for data
buffer = bytearray(
payload_bytes_per_frame
) # use this if CRC16 checksum is required ( DATA1-3)
buffer[
: len(self.data_out)
] = self.data_out # set buffer size to length of data which will be sent
# create crc for data frame - we are using the crc function shipped with codec2 to avoid
# crc algorithm incompatibilities
crc = ctypes.c_ushort(
codec2.api.freedv_gen_crc16(bytes(buffer), payload_bytes_per_frame)
) # generate CRC16
crc = crc.value.to_bytes(2, byteorder="big") # convert crc to 2 byte hex string
buffer += crc # append crc16 to buffer
print(
f"TOTAL BURSTS: {self.N_BURSTS} TOTAL FRAMES_PER_BURST: {self.N_FRAMES_PER_BURST}",
file=sys.stderr,
)
for i in range(1, self.N_BURSTS + 1):
# write preamble to txbuffer
codec2.api.freedv_rawdatapreambletx(freedv, mod_out_preamble)
txbuffer = bytes(mod_out_preamble)
# create modulaton for N = FRAMESPERBURST and append it to txbuffer
for n in range(1, self.N_FRAMES_PER_BURST + 1):
data = (ctypes.c_ubyte * bytes_per_frame).from_buffer_copy(buffer)
codec2.api.freedv_rawdatatx(
freedv, mod_out, data
) # modulate DATA and save it into mod_out pointer
txbuffer += bytes(mod_out)
print(
f" GENERATING TX BURST: {i}/{self.N_BURSTS} FRAME: {n}/{self.N_FRAMES_PER_BURST}",
file=sys.stderr,
)
# append postamble to txbuffer
codec2.api.freedv_rawdatapostambletx(freedv, mod_out_postamble)
txbuffer += bytes(mod_out_postamble)
# append a delay between bursts as audio silence
samples_delay = int(self.MODEM_SAMPLE_RATE * self.DELAY_BETWEEN_BURSTS)
mod_out_silence = ctypes.create_string_buffer(samples_delay * 2)
txbuffer += bytes(mod_out_silence)
print(
f"samples_delay: {samples_delay} DELAY_BETWEEN_BURSTS: {self.DELAY_BETWEEN_BURSTS}",
file=sys.stderr,
)
# resample up to 48k (resampler works on np.int16)
x = np.frombuffer(txbuffer, dtype=np.int16)
txbuffer_48k = self.resampler.resample8_to_48(x)
# split modualted audio to chunks
# https://newbedev.com/how-to-split-a-byte-string-into-separate-bytes-in-python
txbuffer_48k = bytes(txbuffer_48k)
chunk = [
txbuffer_48k[i : i + self.AUDIO_FRAMES_PER_BUFFER * 2]
for i in range( len(txbuffer_48k), self.AUDIO_FRAMES_PER_BUFFER * 2)
]
# add modulated chunks to fifo buffer
for c in chunk:
# if data is shorter than the expcected audio frames per buffer we need to append 0
# to prevent the callback from stucking/crashing
if len(c) < self.AUDIO_FRAMES_PER_BUFFER * 2:
c += bytes(self.AUDIO_FRAMES_PER_BUFFER * 2 - len(c))
self.dataqueue.put(c)
test = Test()
test.create_modulation()
test.run_audio()

219
modem/data cemetery/test/util_chat_text_1.py
View file

@ -1,219 +0,0 @@
# -*- coding: utf-8 -*-
"""
Send-side station emulator for connect frame tests over a high quality simulated audio channel.
Near end-to-end test for sending / receiving connection control frames through the
Modem and modem and back through on the other station. Data injection initiates from the
queue used by the daemon process into and out of the Modem.
Invoked from test_chat_text.py.
@author: N2KIQ
"""
import base64
import json
import time
from pprint import pformat
from typing import Callable
import codec2
import data_handler
import helpers
import modem
import sock
from static import ARQ, AudioParam, Beacon, Channel, Daemon, HamlibParam, ModemParam, Station, Statistics, TCIParam, Modem
import structlog
def t_setup(
mycall: str,
dxcall: str,
lowbwmode: bool,
t_transmit,
t_process_data,
tmp_path,
):
# Disable data_handler testmode - This is required to test a conversation.
data_handler.TESTMODE = False
modem.RXCHANNEL = tmp_path / "hfchannel1"
modem.TESTMODE = True
modem.TXCHANNEL = tmp_path / "hfchannel2"
HamlibParam.hamlib_radiocontrol = "disabled"
Modem.low_bandwidth_mode = lowbwmode
Station.mygrid = bytes("AA12aa", "utf-8")
Modem.respond_to_cq = True
Station.ssid_list = [0, 1, 2, 3, 4, 5, 6, 7, 8, 9]
# override ARQ SESSION STATE for allowing disconnect command
ARQ.arq_session_state = "connected"
mycallsign = helpers.callsign_to_bytes(mycall)
mycallsign = helpers.bytes_to_callsign(mycallsign)
Station.mycallsign = mycallsign
Station.mycallsign_crc = helpers.get_crc_24(Station.mycallsign)
dxcallsign = helpers.callsign_to_bytes(dxcall)
dxcallsign = helpers.bytes_to_callsign(dxcallsign)
Station.dxcallsign = dxcallsign
Station.dxcallsign_crc = helpers.get_crc_24(Station.dxcallsign)
# Create the Modem
modem = data_handler.DATA()
orig_rx_func = data_handler.DATA.process_data
data_handler.DATA.process_data = t_process_data
modem.log = structlog.get_logger("station1_DATA")
# Limit the frame-ack timeout
modem.time_list_low_bw = [3, 1, 1]
modem.time_list_high_bw = [3, 1, 1]
modem.time_list = [3, 1, 1]
# Limit number of retries
modem.rx_n_max_retries_per_burst = 5
# Create the modem
t_modem = modem.RF()
orig_tx_func = modem.RF.transmit
modem.RF.transmit = t_transmit
t_modem.log = structlog.get_logger("station1_RF")
return modem, orig_rx_func, orig_tx_func
def t_highsnr_arq_short_station1(
parent_pipe,
freedv_mode: str,
n_frames_per_burst: int,
mycall: str,
dxcall: str,
message: str,
lowbwmode: bool,
tmp_path,
):
log = structlog.get_logger("station1")
orig_tx_func: Callable
orig_rx_func: Callable
log.info("t_highsnr_arq_short_station1:", TMP_PATH=tmp_path)
def t_transmit(self, mode, repeats: int, repeat_delay: int, frames: bytearray):
"""'Wrap' RF.transmit function to extract the arguments."""
nonlocal orig_tx_func, parent_pipe
t_frames = frames
parent_pipe.send(t_frames)
# log.info("S1 TX: ", frames=t_frames)
for item in t_frames:
frametype = int.from_bytes(item[:1], "big") # type: ignore
log.info("S1 TX: ", mode=static.FRAME_TYPE(frametype).name)
if (
Modem.low_bandwidth_mode
and frametype == static.FRAME_TYPE.ARQ_DC_OPEN_W.value
):
mesg = (
"enqueue_frame_for_tx: Low BW global "
"but DC Open narrow frame NOT chosen."
)
# Low bandwidth data type, wide bandwidth frame type
log.error(mesg)
assert False, mesg
if (
not Modem.low_bandwidth_mode
and frametype == static.FRAME_TYPE.ARQ_DC_OPEN_N.value
):
mesg = (
"enqueue_frame_for_tx: High BW global "
"but DC Open wide frame NOT chosen."
)
# High bandwidth data type, low bandwidth frame type
log.error(mesg)
assert False, mesg
# Apologies for the Python "magic." "orig_func" is a pointer to the
# original function captured before this one was put in place.
orig_tx_func(self, mode, repeats, repeat_delay, frames) # type: ignore
def t_process_data(self, bytes_out, freedv, bytes_per_frame: int):
"""'Wrap' DATA.process_data function to extract the arguments."""
nonlocal orig_rx_func, parent_pipe
t_bytes_out = bytes(bytes_out)
parent_pipe.send(t_bytes_out)
log.debug(
"S1 RX: ",
bytes_out=t_bytes_out,
bytes_per_frame=bytes_per_frame,
)
frametype = int.from_bytes(t_bytes_out[:1], "big")
log.info("S1 RX: ", RX=frametype)
# Apologies for the Python "magic." "orig_func" is a pointer to the
# original function captured before this one was put in place.
orig_rx_func(self, bytes_out, freedv, bytes_per_frame) # type: ignore
modem, orig_rx_func, orig_tx_func = t_setup(
mycall, dxcall, lowbwmode, t_transmit, t_process_data, tmp_path
)
log.info("t_highsnr_arq_short_station1:", RXCHANNEL=modem.RXCHANNEL)
log.info("t_highsnr_arq_short_station1:", TXCHANNEL=modem.TXCHANNEL)
# Construct message to dxstation.
b64_str = str(base64.b64encode(bytes(message, "UTF-8")), "UTF-8").strip()
data = {
"type": "arq",
"command": "send_raw",
"parameter": [
{
"data": b64_str,
"dxcallsign": dxcall,
"mode": codec2.FREEDV_MODE[freedv_mode].value,
"n_frames": n_frames_per_burst,
}
],
}
sock.process_modem_commands(json.dumps(data, indent=None))
# Assure the test completes.
timeout = time.time() + 25
# Compare with the string conversion instead of repeatedly dumping
# the queue to an object for comparisons.
while '"arq":"transmission","status":"transmitted"' not in str(
sock.SOCKET_QUEUE.queue
):
if time.time() > timeout:
log.warning("station1 TIMEOUT", first=True)
break
time.sleep(0.1)
log.info("station1, first", arq_state=pformat(ARQ.arq_state))
data = {"type": "arq", "command": "disconnect", "dxcallsign": dxcall}
sock.process_modem_commands(json.dumps(data, indent=None))
time.sleep(0.5)
# override ARQ SESSION STATE for allowing disconnect command
ARQ.arq_session_state = "connected"
sock.process_modem_commands(json.dumps(data, indent=None))
# Allow enough time for this side to process the disconnect frame.
timeout = time.time() + 20
while ARQ.arq_state or modem.data_queue_transmit.queue:
if time.time() > timeout:
log.error("station1", TIMEOUT=True)
break
time.sleep(0.5)
log.info("station1", arq_state=pformat(ARQ.arq_state))
# log.info("S1 DQT: ", DQ_Tx=pformat(modem.data_queue_transmit.queue))
# log.info("S1 DQR: ", DQ_Rx=pformat(modem.data_queue_received.queue))
log.info("S1 Socket: ", socket_queue=pformat(sock.SOCKET_QUEUE.queue))
assert '"arq":"transmission","status":"transmitting"' in str(
sock.SOCKET_QUEUE.queue
)
assert '"arq":"transmission","status":"transmitted"' in str(sock.SOCKET_QUEUE.queue)
assert '"arq":"transmission","status":"failed"' not in str(sock.SOCKET_QUEUE.queue)
assert '"percent":100' in str(sock.SOCKET_QUEUE.queue)
assert '"command_response":"disconnect","status":"OK"' in str(
sock.SOCKET_QUEUE.queue
)
log.error("station1: Exiting!")

163
modem/data cemetery/test/util_chat_text_2.py
View file

@ -1,163 +0,0 @@
# -*- coding: utf-8 -*-
"""
Receive-side station emulator for connect frame tests over a high quality simulated audio channel.
Near end-to-end test for sending / receiving connection control frames through the
Modem and modem and back through on the other station. Data injection initiates from the
queue used by the daemon process into and out of the Modem.
Invoked from test_chat_text.py.
@author: N2KIQ
"""
import time
from pprint import pformat
from typing import Callable
import data_handler
import helpers
import modem
import sock
from static import ARQ, AudioParam, Beacon, Channel, Daemon, HamlibParam, ModemParam, Station, Statistics, TCIParam, Modem
import structlog
def t_setup(
mycall: str,
dxcall: str,
lowbwmode: bool,
t_transmit,
t_process_data,
tmp_path,
):
# Disable data_handler testmode - This is required to test a conversation.
data_handler.TESTMODE = False
modem.RXCHANNEL = tmp_path / "hfchannel2"
modem.TESTMODE = True
modem.TXCHANNEL = tmp_path / "hfchannel1"
HamlibParam.hamlib_radiocontrol = "disabled"
Modem.low_bandwidth_mode = lowbwmode
Station.mygrid = bytes("AA12aa", "utf-8")
Modem.respond_to_cq = True
Station.ssid_list = [0, 1, 2, 3, 4, 5, 6, 7, 8, 9]
# override ARQ SESSION STATE for allowing disconnect command
ARQ.arq_session_state = "connected"
mycallsign = helpers.callsign_to_bytes(mycall)
mycallsign = helpers.bytes_to_callsign(mycallsign)
Station.mycallsign = mycallsign
Station.mycallsign_crc = helpers.get_crc_24(Station.mycallsign)
dxcallsign = helpers.callsign_to_bytes(dxcall)
dxcallsign = helpers.bytes_to_callsign(dxcallsign)
Station.dxcallsign = dxcallsign
Station.dxcallsign_crc = helpers.get_crc_24(Station.dxcallsign)
# Create the Modem
modem = data_handler.DATA()
orig_rx_func = data_handler.DATA.process_data
data_handler.DATA.process_data = t_process_data
modem.log = structlog.get_logger("station2_DATA")
# Limit the frame-ack timeout
modem.time_list_low_bw = [1, 1, 1]
modem.time_list_high_bw = [1, 1, 1]
modem.time_list = [1, 1, 1]
# Limit number of retries
modem.rx_n_max_retries_per_burst = 5
# Create the modem
t_modem = modem.RF()
orig_tx_func = modem.RF.transmit
modem.RF.transmit = t_transmit
t_modem.log = structlog.get_logger("station2_RF")
return modem, orig_rx_func, orig_tx_func
def t_highsnr_arq_short_station2(
parent_pipe,
freedv_mode: str,
n_frames_per_burst: int,
mycall: str,
dxcall: str,
message: str,
lowbwmode: bool,
tmp_path,
):
log = structlog.get_logger("station2")
orig_tx_func: Callable
orig_rx_func: Callable
log.info("t_highsnr_arq_short_station2:", TMP_PATH=tmp_path)
def t_transmit(self, mode, repeats: int, repeat_delay: int, frames: bytearray):
"""'Wrap' RF.transmit function to extract the arguments."""
nonlocal orig_tx_func, parent_pipe
t_frames = frames
parent_pipe.send(t_frames)
# log.info("S2 TX: ", frames=t_frames)
for item in t_frames:
frametype = int.from_bytes(item[:1], "big") # type: ignore
log.info("S2 TX: ", TX=frametype)
# Apologies for the Python "magic." "orig_func" is a pointer to the
# original function captured before this one was put in place.
orig_tx_func(self, mode, repeats, repeat_delay, frames) # type: ignore
def t_process_data(self, bytes_out, freedv, bytes_per_frame: int):
"""'Wrap' DATA.process_data function to extract the arguments."""
nonlocal orig_rx_func, parent_pipe
t_bytes_out = bytes(bytes_out)
parent_pipe.send(t_bytes_out)
log.debug(
"S2 RX: ",
bytes_out=t_bytes_out,
bytes_per_frame=bytes_per_frame,
)
frametype = int.from_bytes(t_bytes_out[:1], "big")
log.info("S2 RX: ", RX=frametype)
# Apologies for the Python "magic." "orig_func" is a pointer to the
# original function captured before this one was put in place.
orig_rx_func(self, bytes_out, freedv, bytes_per_frame) # type: ignore
modem, orig_rx_func, orig_tx_func = t_setup(
mycall, dxcall, lowbwmode, t_transmit, t_process_data, tmp_path
)
log.info("t_highsnr_arq_short_station2:", RXCHANNEL=modem.RXCHANNEL)
log.info("t_highsnr_arq_short_station2:", TXCHANNEL=modem.TXCHANNEL)
# Assure the test completes.
timeout = time.time() + 25
# Compare with the string conversion instead of repeatedly dumping
# the queue to an object for comparisons.
while (
'"arq":"transmission","status":"received"' not in str(sock.SOCKET_QUEUE.queue)
or ARQ.arq_state
):
if time.time() > timeout:
log.warning("station2 TIMEOUT", first=True)
break
time.sleep(0.5)
log.info("station2, first", arq_state=pformat(ARQ.arq_state))
# Allow enough time for this side to receive the disconnect frame.
timeout = time.time() + 20
while '"arq":"session","status":"close"' not in str(sock.SOCKET_QUEUE.queue):
if time.time() > timeout:
log.warning("station2", TIMEOUT=True)
break
time.sleep(0.5)
log.info("station2", arq_state=pformat(ARQ.arq_state))
# log.info("S2 DQT: ", DQ_Tx=pformat(modem.data_queue_transmit.queue))
# log.info("S2 DQR: ", DQ_Rx=pformat(modem.data_queue_received.queue))
log.info("S2 Socket: ", socket_queue=pformat(sock.SOCKET_QUEUE.queue))
assert '"arq":"transmission","status":"received"' in str(sock.SOCKET_QUEUE.queue)
assert '"arq":"session","status":"close"' in str(sock.SOCKET_QUEUE.queue)
log.warning("station2: Exiting!")

309
modem/data cemetery/test/util_datac13.py
View file

@ -1,309 +0,0 @@
# -*- coding: utf-8 -*-
"""
Send- and receive-side station emulator for control frame tests over a high quality
simulated audio channel.
Near end-to-end test for sending / receiving control frames through the Modem and modem
and back through on the other station. Data injection initiates from the queue used
by the daemon process into and out of the ModemParam.
Invoked from test_datac13.py.
@author: N2KIQ
"""
import json
import time
from pprint import pformat
from typing import Callable, Tuple
import data_handler
import helpers
import modem
import sock
from global_instances import ARQ, AudioParam, Beacon, Channel, Daemon, HamlibParam, ModemParam, Station, Statistics, TCIParam, Modem
from static import FRAME_TYPE as FR_TYPE
import structlog
#from static import FRAME_TYPE as FR_TYPE
def t_setup(
station: int,
mycall: str,
dxcall: str,
rx_channel: str,
tx_channel: str,
lowbwmode: bool,
t_transmit,
t_process_data,
tmp_path,
):
# Disable data_handler testmode - This is required to test a conversation.
data_handler.TESTMODE = True
# Enable socket testmode for overriding socket class
sock.TESTMODE = True
modem.RXCHANNEL = tmp_path / rx_channel
modem.TESTMODE = True
modem.TXCHANNEL = tmp_path / tx_channel
HamlibParam.hamlib_radiocontrol = "disabled"
ModemParam.low_bandwidth_mode = lowbwmode or True
Station.mygrid = bytes("AA12aa", "utf-8")
ModemParam.respond_to_cq = True
Station.ssid_list = [0, 1, 2, 3, 4, 5, 6, 7, 8, 9]
mycallsign = helpers.callsign_to_bytes(mycall)
mycallsign = helpers.bytes_to_callsign(mycallsign)
Station.mycallsign = mycallsign
Station.mycallsign_crc = helpers.get_crc_24(Station.mycallsign)
dxcallsign = helpers.callsign_to_bytes(dxcall)
dxcallsign = helpers.bytes_to_callsign(dxcallsign)
Station.dxcallsign = dxcallsign
Station.dxcallsign_crc = helpers.get_crc_24(Station.dxcallsign)
# Create the Modem
modem_data_handler = data_handler.DATA()
orig_rx_func = data_handler.DATA.process_data
data_handler.DATA.process_data = t_process_data
modem_data_handler.log = structlog.get_logger(f"station{station}_DATA")
# Limit the frame-ack timeout
modem_data_handler.time_list_low_bw = [8, 8, 8]
modem_data_handler.time_list_high_bw = [8, 8, 8]
modem_data_handler.time_list = [8, 8, 8]
# Limit number of retries
modem_data_handler.rx_n_max_retries_per_burst = 4
ModemParam.tx_delay = 50 # add additional delay time for passing test
# Create the modem
t_modem = modem.RF()
orig_tx_func = modem.RF.transmit
modem.RF.transmit = t_transmit
t_modem.log = structlog.get_logger(f"station{station}_RF")
return modem_data_handler, orig_rx_func, orig_tx_func
def t_datac13_1(
parent_pipe,
mycall: str,
dxcall: str,
config: Tuple,
tmp_path,
):
log = structlog.get_logger("station1")
orig_tx_func: Callable
orig_rx_func: Callable
log.debug("t_datac13_1:", TMP_PATH=tmp_path)
# Unpack tuple
data, timeout_duration, tx_check, _, final_tx_check, _ = config
def t_transmit(self, mode, repeats: int, repeat_delay: int, frames: bytearray):
"""'Wrap' RF.transmit function to extract the arguments."""
nonlocal orig_tx_func, parent_pipe
t_frames = frames
parent_pipe.send(t_frames)
# log.info("S1 TX: ", frames=t_frames)
for item in t_frames:
frametype = int.from_bytes(item[:1], "big") # type: ignore
log.info("S1 TX: ", TX=FR_TYPE(frametype).name)
# Apologies for the Python "magic." "orig_func" is a pointer to the
# original function captured before this one was put in place.
orig_tx_func(self, mode, repeats, repeat_delay, frames) # type: ignore
def t_process_data(self, bytes_out, freedv, bytes_per_frame: int):
"""'Wrap' DATA.process_data function to extract the arguments."""
nonlocal orig_rx_func, parent_pipe
t_bytes_out = bytes(bytes_out)
parent_pipe.send(t_bytes_out)
log.debug(
"S1 RX: ",
bytes_out=t_bytes_out,
bytes_per_frame=bytes_per_frame,
)
frametype = int.from_bytes(t_bytes_out[:1], "big")
log.info("S1 RX: ", RX=FR_TYPE(frametype).name)
# Apologies for the Python "magic." "orig_func" is a pointer to the
# original function captured before this one was put in place.
orig_rx_func(self, bytes_out, freedv, bytes_per_frame) # type: ignore
modem_data_handler, orig_rx_func, orig_tx_func = t_setup(
1,
mycall,
dxcall,
"hfchannel1",
"hfchannel2",
True,
t_transmit,
t_process_data,
tmp_path,
)
log.info("t_datac13_1:", RXCHANNEL=modem.RXCHANNEL)
log.info("t_datac13_1:", TXCHANNEL=modem.TXCHANNEL)
time.sleep(0.5)
if "stop" in data["command"]:
log.debug("t_datac13_1: STOP test, setting Modem state")
ModemParam.modem_state = "BUSY"
ARQ.arq_state = True
sock.ThreadedTCPRequestHandler.process_modem_commands(None,json.dumps(data, indent=None))
time.sleep(0.5)
sock.ThreadedTCPRequestHandler.process_modem_commands(None,json.dumps(data, indent=None))
# Assure the test completes.
timeout = time.time() + timeout_duration + 5
while tx_check not in str(sock.SOCKET_QUEUE.queue):
if time.time() > timeout:
log.warning(
"station1 TIMEOUT",
first=True,
queue=str(sock.SOCKET_QUEUE.queue),
tx_check=tx_check,
)
break
time.sleep(0.5)
log.info("station1, first")
# override ARQ SESSION STATE for allowing disconnect command
ARQ.arq_session_state = "connected"
data = {"type": "arq", "command": "disconnect", "dxcallsign": dxcall}
sock.ThreadedTCPRequestHandler.process_modem_commands(None,json.dumps(data, indent=None))
time.sleep(0.5)
# Allow enough time for this side to process the disconnect frame.
timeout = time.time() + timeout_duration
while modem_data_handler.data_queue_transmit.queue:
if time.time() > timeout:
log.warning("station1", TIMEOUT=True, dq_tx=modem_data_handler.data_queue_transmit.queue)
break
time.sleep(0.5)
log.info("station1, final")
# log.info("S1 DQT: ", DQ_Tx=pformat(ModemParam.data_queue_transmit.queue))
# log.info("S1 DQR: ", DQ_Rx=pformat(ModemParam.data_queue_received.queue))
log.debug("S1 Socket: ", socket_queue=pformat(sock.SOCKET_QUEUE.queue))
for item in final_tx_check:
assert item in str(
sock.SOCKET_QUEUE.queue
), f"{item} not found in {str(sock.SOCKET_QUEUE.queue)}"
assert ':"failed"' not in str(sock.SOCKET_QUEUE.queue)
assert '"command_response":"disconnect","status":"OK"' in str(
sock.SOCKET_QUEUE.queue
)
log.warning("station1: Exiting!")
def t_datac13_2(
parent_pipe,
mycall: str,
dxcall: str,
config: Tuple,
tmp_path,
):
log = structlog.get_logger("station2")
orig_tx_func: Callable
orig_rx_func: Callable
log.debug("t_datac13_2:", TMP_PATH=tmp_path)
# Unpack tuple
data, timeout_duration, _, rx_check, _, final_rx_check = config
def t_transmit(self, mode, repeats: int, repeat_delay: int, frames: bytearray):
"""'Wrap' RF.transmit function to extract the arguments."""
nonlocal orig_tx_func, parent_pipe
t_frames = frames
parent_pipe.send(t_frames)
# log.info("S2 TX: ", frames=t_frames)
for item in t_frames:
frametype = int.from_bytes(item[:1], "big") # type: ignore
log.info("S2 TX: ", TX=FR_TYPE(frametype).name)
# Apologies for the Python "magic." "orig_func" is a pointer to the
# original function captured before this one was put in place.
orig_tx_func(self, mode, repeats, repeat_delay, frames) # type: ignore
def t_process_data(self, bytes_out, freedv, bytes_per_frame: int):
"""'Wrap' DATA.process_data function to extract the arguments."""
nonlocal orig_rx_func, parent_pipe
t_bytes_out = bytes(bytes_out)
parent_pipe.send(t_bytes_out)
log.debug(
"S2 RX: ",
bytes_out=t_bytes_out,
bytes_per_frame=bytes_per_frame,
)
frametype = int.from_bytes(t_bytes_out[:1], "big")
log.info("S2 RX: ", RX=FR_TYPE(frametype).name)
# Apologies for the Python "magic." "orig_func" is a pointer to the
# original function captured before this one was put in place.
orig_rx_func(self, bytes_out, freedv, bytes_per_frame) # type: ignore
_, orig_rx_func, orig_tx_func = t_setup(
2,
mycall,
dxcall,
"hfchannel2",
"hfchannel1",
True,
t_transmit,
t_process_data,
tmp_path,
)
log.info("t_datac13_2:", RXCHANNEL=modem.RXCHANNEL)
log.info("t_datac13_2:", TXCHANNEL=modem.TXCHANNEL)
# TODO Why do we need this when calling CQ?
#if "cq" in data:
# t_data = {"type": "arq", "command": "stop_transmission"}
# sock.ThreadedTCPRequestHandler.process_modem_commands(None,json.dumps(t_data, indent=None))
# sock.ThreadedTCPRequestHandler.process_modem_commands(None,json.dumps(t_data, indent=None))
# Assure the test completes.
timeout = time.time() + timeout_duration
# Compare with the string conversion instead of repeatedly dumping
# the queue to an object for comparisons.
while rx_check not in str(sock.SOCKET_QUEUE.queue):
if time.time() > timeout:
log.warning(
"station2 TIMEOUT",
first=True,
queue=str(sock.SOCKET_QUEUE.queue),
rx_check=rx_check,
)
break
time.sleep(0.5)
log.info("station2, first")
# Allow enough time for this side to receive the disconnect frame.
timeout = time.time() + timeout_duration
while '"arq":"session","status":"close"' not in str(sock.SOCKET_QUEUE.queue):
if time.time() > timeout:
log.warning("station2", TIMEOUT=True, queue=str(sock.SOCKET_QUEUE.queue))
break
time.sleep(0.5)
log.info("station2, final")
# log.info("S2 DQT: ", DQ_Tx=pformat(ModemParam.data_queue_transmit.queue))
# log.info("S2 DQR: ", DQ_Rx=pformat(ModemParam.data_queue_received.queue))
log.debug("S2 Socket: ", socket_queue=pformat(sock.SOCKET_QUEUE.queue))
for item in final_rx_check:
assert item in str(
sock.SOCKET_QUEUE.queue
), f"{item} not found in {str(sock.SOCKET_QUEUE.queue)}"
# TODO Not sure why we need this for every test run
# assert '"arq":"session","status":"close"' in str(sock.SOCKET_QUEUE.queue)
log.warning("station2: Exiting!")

311
modem/data cemetery/test/util_datac13_negative.py
View file

@ -1,311 +0,0 @@
#!/usr/bin/env python3
# -*- coding: utf-8 -*-
"""
Negative test utilities for datac13 frames.
@author: kronenpj
"""
import json
import time
from pprint import pformat
from typing import Callable, Tuple
import data_handler
import helpers
import modem
import sock
from global_instances import ARQ, AudioParam, Beacon, Channel, Daemon, HamlibParam, ModemParam, Station, Statistics, TCIParam, Modem
import structlog
#from static import FRAME_TYPE as FR_TYPE
def t_setup(
station: int,
mycall: str,
dxcall: str,
rx_channel: str,
tx_channel: str,
lowbwmode: bool,
t_transmit,
t_process_data,
tmp_path,
):
# Disable data_handler testmode - This is required to test a conversation.
data_handler.TESTMODE = False
# Enable socket testmode for overriding socket class
sock.TESTMODE = True
modem.RXCHANNEL = tmp_path / rx_channel
modem.TESTMODE = True
modem.TXCHANNEL = tmp_path / tx_channel
HamlibParam.hamlib_radiocontrol = "disabled"
Modem.low_bandwidth_mode = lowbwmode or True
Station.mygrid = bytes("AA12aa", "utf-8")
Modem.respond_to_cq = True
Station.ssid_list = [0, 1, 2, 3, 4, 5, 6, 7, 8, 9]
mycallsign_bytes = helpers.callsign_to_bytes(mycall)
mycallsign = helpers.bytes_to_callsign(mycallsign_bytes)
Station.mycallsign = mycallsign
Station.mycallsign_crc = helpers.get_crc_24(mycallsign)
dxcallsign_bytes = helpers.callsign_to_bytes(dxcall)
dxcallsign = helpers.bytes_to_callsign(dxcallsign_bytes)
Station.dxcallsign = dxcallsign
Station.dxcallsign_crc = helpers.get_crc_24(dxcallsign)
# Create the Modem
modem_data_handler = data_handler.DATA()
orig_rx_func = data_handler.DATA.process_data
data_handler.DATA.process_data = t_process_data
modem_data_handler.log = structlog.get_logger(f"station{station}_DATA")
# Limit the frame-ack timeout
modem_data_handler.time_list_low_bw = [8, 8, 8]
modem_data_handler.time_list_high_bw = [8, 8, 8]
modem_data_handler.time_list = [8, 8, 8]
# Limit number of retries
modem_data_handler.rx_n_max_retries_per_burst = 4
ModemParam.tx_delay = 50 # add additional delay time for passing test
# Create the modem
t_modem = modem.RF()
orig_tx_func = modem.RF.transmit
modem.RF.transmit = t_transmit
t_modem.log = structlog.get_logger(f"station{station}_RF")
return modem_data_handler, orig_rx_func, orig_tx_func
def t_datac13_1(
parent_pipe,
mycall: str,
dxcall: str,
config: Tuple,
tmp_path,
):
log = structlog.get_logger("station1")
orig_tx_func: Callable
orig_rx_func: Callable
log.debug("t_datac13_1:", TMP_PATH=tmp_path)
# Unpack tuple
data, timeout_duration, tx_check, _, final_tx_check, _ = config
def t_transmit(self, mode, repeats: int, repeat_delay: int, frames: bytearray):
"""'Wrap' RF.transmit function to extract the arguments."""
nonlocal orig_tx_func, parent_pipe
t_frames = frames
parent_pipe.send(t_frames)
# log.info("S1 TX: ", frames=t_frames)
for item in t_frames:
frametype = int.from_bytes(item[:1], "big") # type: ignore
log.info("S1 TX: ", TX=FR_TYPE(frametype).name)
# Apologies for the Python "magic." "orig_func" is a pointer to the
# original function captured before this one was put in place.
orig_tx_func(self, mode, repeats, repeat_delay, frames) # type: ignore
def t_process_data(self, bytes_out, freedv, bytes_per_frame: int):
"""'Wrap' DATA.process_data function to extract the arguments."""
nonlocal orig_rx_func, parent_pipe
t_bytes_out = bytes(bytes_out)
parent_pipe.send(t_bytes_out)
log.debug(
"S1 RX: ",
bytes_out=t_bytes_out,
bytes_per_frame=bytes_per_frame,
)
frametype = int.from_bytes(t_bytes_out[:1], "big")
log.info("S1 RX: ", RX=FR_TYPE(frametype).name)
# Apologies for the Python "magic." "orig_func" is a pointer to the
# original function captured before this one was put in place.
orig_rx_func(self, bytes_out, freedv, bytes_per_frame) # type: ignore
modem_data_handler, orig_rx_func, orig_tx_func = t_setup(
1,
mycall,
dxcall,
"hfchannel1",
"hfchannel2",
True,
t_transmit,
t_process_data,
tmp_path,
)
log.info("t_datac13_1:", RXCHANNEL=modem.RXCHANNEL)
log.info("t_datac13_1:", TXCHANNEL=modem.TXCHANNEL)
orig_dxcall = Station.dxcallsign
if "stop" in data["command"]:
time.sleep(0.5)
log.debug(
"t_datac13_1: STOP test, setting Modem state",
mycall=Station.mycallsign,
dxcall=Station.dxcallsign,
)
Station.dxcallsign = helpers.callsign_to_bytes(data["dxcallsign"])
Station.dxcallsign_CRC = helpers.get_crc_24(Station.dxcallsign)
Modem.modem_state = "BUSY"
ARQ.arq_state = True
sock.ThreadedTCPRequestHandler.process_modem_commands(None,json.dumps(data, indent=None))
sock.ThreadedTCPRequestHandler.process_modem_commands(None,json.dumps(data, indent=None))
# Assure the test completes.
timeout = time.time() + timeout_duration
while tx_check not in str(sock.SOCKET_QUEUE.queue):
if time.time() > timeout:
log.warning(
"station1 TIMEOUT",
first=True,
queue=str(sock.SOCKET_QUEUE.queue),
tx_check=tx_check,
)
break
time.sleep(0.1)
log.info("station1, first")
if "stop" in data["command"]:
time.sleep(0.5)
log.debug("STOP test, resetting DX callsign")
Station.dxcallsign = orig_dxcall
Station.dxcallsign_CRC = helpers.get_crc_24(Station.dxcallsign)
# override ARQ SESSION STATE for allowing disconnect command
ARQ.arq_session_state = "connected"
data = {"type": "arq", "command": "disconnect", "dxcallsign": dxcall}
sock.ThreadedTCPRequestHandler.process_modem_commands(None,json.dumps(data, indent=None))
time.sleep(0.5)
# Allow enough time for this side to process the disconnect frame.
timeout = time.time() + timeout_duration
while modem_data_handler.data_queue_transmit.queue:
if time.time() > timeout:
log.warning("station1", TIMEOUT=True, dq_tx=modem_data_handler.data_queue_transmit.queue)
break
time.sleep(0.5)
log.info("station1, final")
# log.info("S1 DQT: ", DQ_Tx=pformat(modem.data_queue_transmit.queue))
# log.info("S1 DQR: ", DQ_Rx=pformat(modem.data_queue_received.queue))
log.debug("S1 Socket: ", socket_queue=pformat(sock.SOCKET_QUEUE.queue))
for item in final_tx_check:
assert item in str(
sock.SOCKET_QUEUE.queue
), f"{item} not found in {str(sock.SOCKET_QUEUE.queue)}"
assert '"command_response":"disconnect","status":"OK"' in str(
sock.SOCKET_QUEUE.queue
)
log.warning("station1: Exiting!")
def t_datac13_2(
parent_pipe,
mycall: str,
dxcall: str,
config: Tuple,
tmp_path,
):
log = structlog.get_logger("station2")
orig_tx_func: Callable
orig_rx_func: Callable
log.debug("t_datac13_2:", TMP_PATH=tmp_path)
# Unpack tuple
data, timeout_duration, _, rx_check, _, final_rx_check = config
def t_transmit(self, mode, repeats: int, repeat_delay: int, frames: bytearray):
"""'Wrap' RF.transmit function to extract the arguments."""
nonlocal orig_tx_func, parent_pipe
t_frames = frames
parent_pipe.send(t_frames)
# log.info("S2 TX: ", frames=t_frames)
for item in t_frames:
frametype = int.from_bytes(item[:1], "big") # type: ignore
log.info("S2 TX: ", TX=FR_TYPE(frametype).name)
# Apologies for the Python "magic." "orig_func" is a pointer to the
# original function captured before this one was put in place.
orig_tx_func(self, mode, repeats, repeat_delay, frames) # type: ignore
def t_process_data(self, bytes_out, freedv, bytes_per_frame: int):
"""'Wrap' DATA.process_data function to extract the arguments."""
nonlocal orig_rx_func, parent_pipe
t_bytes_out = bytes(bytes_out)
parent_pipe.send(t_bytes_out)
log.debug(
"S2 RX: ",
bytes_out=t_bytes_out,
bytes_per_frame=bytes_per_frame,
)
frametype = int.from_bytes(t_bytes_out[:1], "big")
log.info("S2 RX: ", RX=FR_TYPE(frametype).name)
# Apologies for the Python "magic." "orig_func" is a pointer to the
# original function captured before this one was put in place.
orig_rx_func(self, bytes_out, freedv, bytes_per_frame) # type: ignore
_, orig_rx_func, orig_tx_func = t_setup(
2,
mycall,
dxcall,
"hfchannel2",
"hfchannel1",
True,
t_transmit,
t_process_data,
tmp_path,
)
log.info("t_datac13_2:", RXCHANNEL=modem.RXCHANNEL)
log.info("t_datac13_2:", TXCHANNEL=modem.TXCHANNEL)
log.info("t_datac13_2:", mycall=Station.mycallsign)
if "cq" in data:
t_data = {"type": "arq", "command": "stop_transmission"}
sock.ThreadedTCPRequestHandler.process_modem_commands(None,json.dumps(t_data, indent=None))
sock.ThreadedTCPRequestHandler.process_modem_commands(None,json.dumps(t_data, indent=None))
# Assure the test completes.
timeout = time.time() + timeout_duration
# Compare with the string conversion instead of repeatedly dumping
# the queue to an object for comparisons.
while rx_check not in str(sock.SOCKET_QUEUE.queue):
if time.time() > timeout:
log.warning(
"station2 TIMEOUT",
first=True,
queue=str(sock.SOCKET_QUEUE.queue),
rx_check=rx_check,
)
break
time.sleep(0.5)
log.info("station2, first")
# Allow enough time for this side to receive the disconnect frame.
timeout = time.time() + timeout_duration
while '"arq":"session", "status":"close"' not in str(sock.SOCKET_QUEUE.queue):
if time.time() > timeout:
log.warning("station2", TIMEOUT=True, queue=str(sock.SOCKET_QUEUE.queue))
break
time.sleep(0.5)
log.info("station2, final")
# log.info("S2 DQT: ", DQ_Tx=pformat(modem.data_queue_transmit.queue))
# log.info("S2 DQR: ", DQ_Rx=pformat(modem.data_queue_received.queue))
log.debug("S2 Socket: ", socket_queue=pformat(sock.SOCKET_QUEUE.queue))
for item in final_rx_check:
assert item not in str(
sock.SOCKET_QUEUE.queue
), f"{item} found in {str(sock.SOCKET_QUEUE.queue)}"
# TODO Not sure why we need this for every test run
# assert '"arq":"session","status":"close"' in str(sock.SOCKET_QUEUE.queue)
log.warning("station2: Exiting!")

108
modem/data cemetery/test/util_modem_IRS.py
View file

@ -1,108 +0,0 @@
# -*- coding: utf-8 -*-
"""
Receive-side station emulator for connect frame tests over a high quality simulated audio channel.
Near end-to-end test for sending / receiving connection control frames through the
TNC and modem and back through on the other station. Data injection initiates from the
queue used by the daemon process into and out of the TNC.
Invoked from test_modem.py.
@author: N2KIQ
"""
import signal
import sys
import time
from typing import Callable
import structlog
sys.path.insert(0, "../modem")
import data_handler
import helpers
import modem
import sock
from global_instances import ARQ, AudioParam, Beacon, Channel, Daemon, HamlibParam, ModemParam, Station, Statistics, TCIParam, Modem
IRS_original_arq_cleanup: Callable
MESSAGE: str
log = structlog.get_logger("util_modem_IRS")
def irs_arq_cleanup():
"""Replacement for modem.arq_cleanup to detect when to exit process."""
log.info(
"irs_arq_cleanup", socket_queue=sock.SOCKET_QUEUE.queue, message=MESSAGE.lower()
)
if '"arq":"transmission","status":"stopped"' in str(sock.SOCKET_QUEUE.queue):
# log.info("irs_arq_cleanup", socket_queue=sock.SOCKET_QUEUE.queue)
time.sleep(2)
if f'"{MESSAGE.lower()}":"receiving"' not in str(
sock.SOCKET_QUEUE.queue
) and f'"{MESSAGE.lower()}":"received"' not in str(sock.SOCKET_QUEUE.queue):
print(f"{MESSAGE} was not received.")
log.info("irs_arq_cleanup", socket_queue=sock.SOCKET_QUEUE.queue)
# sys.exit does not terminate threads, and os_exit doesn't allow coverage collection.
signal.raise_signal(signal.SIGKILL)
signal.raise_signal(signal.SIGTERM)
IRS_original_arq_cleanup()
def t_arq_irs(*args):
# not sure why importing at top level isn't working
import modem
import data_handler
# pylint: disable=global-statement
global IRS_original_arq_cleanup, MESSAGE
MESSAGE = args[0]
tmp_path = args[1]
sock.log = structlog.get_logger("util_modem_IRS_sock")
# enable testmode
data_handler.TESTMODE = True
modem.RXCHANNEL = tmp_path / "hfchannel2"
modem.TESTMODE = True
modem.TXCHANNEL = tmp_path / "hfchannel1"
HamlibParam.hamlib_radiocontrol = "disabled"
Modem.respond_to_cq = True
log.info("t_arq_irs:", RXCHANNEL=modem.RXCHANNEL)
log.info("t_arq_irs:", TXCHANNEL=modem.TXCHANNEL)
mycallsign = bytes("DN2LS-2", "utf-8")
mycallsign = helpers.callsign_to_bytes(mycallsign)
Station.mycallsign = helpers.bytes_to_callsign(mycallsign)
Station.mycallsign_CRC = helpers.get_crc_24(Station.mycallsign)
Station.mygrid = bytes("AA12aa", "utf-8")
Station.ssid_list = [0, 1, 2, 3, 4, 5, 6, 7, 8, 9]
# start data handler
data_handler = data_handler.DATA()
data_handler.log = structlog.get_logger("util_modem_IRS_DATA")
# Inject a way to exit the TNC infinite loop
IRS_original_arq_cleanup = data_handler.arq_cleanup
data_handler.arq_cleanup = irs_arq_cleanup
# start modem
t_modem = modem.RF()
t_modem.log = structlog.get_logger("util_modem_IRS_RF")
# Set timeout
timeout = time.time() + 15
while time.time() < timeout:
time.sleep(0.1)
log.warning("queue:", queue=sock.SOCKET_QUEUE.queue)
assert not "TIMEOUT!"
if __name__ == "__main__":
print("This cannot be run as an application.")
sys.exit(1)

156
modem/data cemetery/test/util_modem_ISS.py
View file

@ -1,156 +0,0 @@
# -*- coding: utf-8 -*-
"""
Send-side station emulator for connect frame tests over a high quality simulated audio channel.
Near end-to-end test for sending / receiving connection control frames through the
TNC and modem and back through on the other station. Data injection initiates from the
queue used by the daemon process into and out of the TNC.
Invoked from test_modem.py.
@author: DJ2LS, N2KIQ
"""
import json
import signal
import sys
import time
from typing import Callable
import structlog
sys.path.insert(0, "../modem")
import data_handler
import helpers
import modem
import sock
import static
from global_instances import ARQ, AudioParam, Beacon, Channel, Daemon, HamlibParam, ModemParam, Station, Statistics, TCIParam, Modem
ISS_original_arq_cleanup: Callable
MESSAGE: str
log = structlog.get_logger("util_modem_ISS")
def iss_arq_cleanup():
"""Replacement for modem.arq_cleanup to detect when to exit process."""
log.info(
"iss_arq_cleanup", socket_queue=sock.SOCKET_QUEUE.queue, message=MESSAGE.lower()
)
if '"arq":"transmission","status":"stopped"' in str(sock.SOCKET_QUEUE.queue):
# log.info("iss_arq_cleanup", socket_queue=sock.SOCKET_QUEUE.queue)
time.sleep(1)
if f'"{MESSAGE.lower()}":"transmitting"' not in str(
sock.SOCKET_QUEUE.queue
) and f'"{MESSAGE.lower()}":"sending"' not in str(sock.SOCKET_QUEUE.queue):
print(f"{MESSAGE} was not sent.")
log.info("iss_arq_cleanup", socket_queue=sock.SOCKET_QUEUE.queue)
# sys.exit does not terminate threads, and os_exit doesn't allow coverage collection.
signal.raise_signal(signal.SIGKILL)
signal.raise_signal(signal.SIGTERM)
ISS_original_arq_cleanup()
def t_arq_iss(*args):
# not sure why importing at top level isn't working
import modem
import data_handler
# pylint: disable=global-statement
global ISS_original_arq_cleanup, MESSAGE
MESSAGE = args[0]
tmp_path = args[1]
sock.log = structlog.get_logger("util_modem_ISS_sock")
# enable testmode
data_handler.TESTMODE = True
modem.RXCHANNEL = tmp_path / "hfchannel1"
modem.TESTMODE = True
modem.TXCHANNEL = tmp_path / "hfchannel2"
HamlibParam.hamlib_radiocontrol = "disabled"
log.info("t_arq_iss:", RXCHANNEL=modem.RXCHANNEL)
log.info("t_arq_iss:", TXCHANNEL=modem.TXCHANNEL)
mycallsign = bytes("DJ2LS-2", "utf-8")
mycallsign = helpers.callsign_to_bytes(mycallsign)
Station.mycallsign = helpers.bytes_to_callsign(mycallsign)
Station.mycallsign_CRC = helpers.get_crc_24(Station.mycallsign)
Station.mygrid = bytes("AA12aa", "utf-8")
Station.ssid_list = [0, 1, 2, 3, 4, 5, 6, 7, 8, 9]
dxcallsign = b"DN2LS-0"
dxcallsign = helpers.callsign_to_bytes(dxcallsign)
dxcallsign = helpers.bytes_to_callsign(dxcallsign)
Station.dxcallsign = dxcallsign
Station.dxcallsign_CRC = helpers.get_crc_24(Station.dxcallsign)
bytes_out = b'{"dt":"f","fn":"zeit.txt","ft":"text\\/plain","d":"data:text\\/plain;base64,MyBtb2Rlcywgb2huZSBjbGFzcwowLjAwMDk2OTQ4MTE4MDk5MTg0MTcKCjIgbW9kZXMsIG9obmUgY2xhc3MKMC4wMDA5NjY1NDUxODkxMjI1Mzk0CgoxIG1vZGUsIG9obmUgY2xhc3MKMC4wMDA5NjY5NzY1NTU4Nzc4MjA5CgMyBtb2Rlcywgb2huZSBjbGFzcwowLjAwMDk2OTQ4MTE4MDk5MTg0MTcKCjIgbW9kZXMsIG9obmUgY2xhc3MKMC4wMDA5NjY1NDUxODkxMjI1Mzk0CgoxIG1vZGUsIG9obmUgY2xhc3MKMC4wMDA5NjY5NzY1NTU4Nzc4MjA5Cg=MyBtb2Rlcywgb2huZSBjbGFzcwowLjAwMDk2OTQ4MTE4MDk5MTg0MTcKCjIgbW9kZXMsIG9obmUgY2xhc3MKMC4wMDA5NjY1NDUxODkxMjI1Mzk0CgoxIG1vZGUsIG9obmUgY2xhc3MKMC4wMDA5NjY5NzY1NTU4Nzc4MjA5CgMyBtb2Rlcywgb2huZSBjbGFzcwowLjAwMDk2OTQ4MTE4MDk5MTg0MTcKCjIgbW9kZXMsIG9obmUgY2xhc3MKMC4wMDA5NjY1NDUxODkxMjI1Mzk0CgoxIG1vZGUsIG9obmUgY2xhc3MKMC4wMDA5NjY5NzY1NTU4Nzc4MjA5CgMyBtb2Rlcywgb2huZSBjbGFzcwowLjAwMDk2OTQ4MTE4MDk5MTg0MTcKCjIgbW9kZXMsIG9obmUgY2xhc3MKMC4wMDA5NjY1NDUxODkxMjI1Mzk0CgoxIG1vZGUsIG9obmUgY2xhc3MKMC4wMDA5NjY5NzY1NTU4Nzc4MjA5Cg=","crc":"123123123"}'
# start data handler
data_handler = data_handler.DATA()
data_handler.log = structlog.get_logger("util_modem_ISS_DATA")
# Inject a way to exit the TNC infinite loop
ISS_original_arq_cleanup = data_handler.arq_cleanup
data_handler.arq_cleanup = iss_arq_cleanup
# start modem
t_modem = modem.RF()
t_modem.log = structlog.get_logger("util_modem_ISS_RF")
# mode = codec2.freedv_get_mode_value_by_name(FREEDV_MODE)
# n_frames_per_burst = N_FRAMES_PER_BURST
# add command to data qeue
"""
elif data[0] == 'ARQ_RAW':
# [0] ARQ_RAW
# [1] DATA_OUT bytes
# [2] MODE int
# [3] N_FRAMES_PER_BURST int
# [4] self.transmission_uuid str
# [5] mycallsign with ssid
"""
# data_handler.DATA_QUEUE_TRANSMIT.put(['ARQ_RAW', bytes_out, 255, n_frames_per_burst, '123', b'DJ2LS-0'])
data = {}
if MESSAGE in ["CONNECT"]:
data = {
"type": "arq",
"command": "connect",
"dxcallsign": str(dxcallsign, encoding="UTF-8"),
}
else:
assert not MESSAGE, f"{MESSAGE} not known to test."
time.sleep(2.5)
sock.ThreadedTCPRequestHandler.process_modem_commands(None,json.dumps(data, indent=None))
sock.ThreadedTCPRequestHandler.process_modem_commands(None,json.dumps(data, indent=None))
sock.ThreadedTCPRequestHandler.process_modem_commands(None,json.dumps(data, indent=None))
time.sleep(7.5)
data = {"type": "arq", "command": "stop_transmission"}
sock.ThreadedTCPRequestHandler.process_modem_commands(None,json.dumps(data, indent=None))
time.sleep(2.5)
sock.ThreadedTCPRequestHandler.process_modem_commands(None,json.dumps(data, indent=None))
# Set timeout
timeout = time.time() + 15
while time.time() < timeout:
time.sleep(0.1)
log.warning("queue:", queue=sock.SOCKET_QUEUE.queue)
assert not "TIMEOUT!"
if __name__ == "__main__":
print("This cannot be run as an application.")
sys.exit(-1)

254
modem/data cemetery/test/util_multimode_rx.py
View file

@ -1,254 +0,0 @@
#!/usr/bin/env python3
# -*- coding: utf-8 -*-
"""
Receive-side station emulator for test frame tests over a high quality audio channel
using a physical sound card or STDIO.
Legacy test for sending / receiving connection test frames through the codec2 and
back through on the other station. Data injection initiates directly through
the codec2 API. Tests all three codec2 data frames simultaneously.
Invoked from CMake, test_highsnr_stdio_P_P_multi.py, and many test_virtual[1-3]*.sh.
@author: DJ2LS
"""
import argparse
import ctypes
import sys
import time
from typing import List
import numpy as np
import pyaudio
sys.path.insert(0, "..")
from modem import codec2
def test_mm_rx():
# AUDIO PARAMETERS
AUDIO_FRAMES_PER_BUFFER = 2400 * 2
MODEM_SAMPLE_RATE = codec2.api.FREEDV_FS_8000
AUDIO_SAMPLE_RATE_RX = 48000
# make sure our resampler will work
assert (AUDIO_SAMPLE_RATE_RX / MODEM_SAMPLE_RATE) == codec2.api.FDMDV_OS_48
# SET COUNTERS
rx_bursts_datac = [0, 0, 0]
rx_frames_datac = [0, 0, 0]
rx_total_frames_datac = [0, 0, 0]
# time meassurement
time_end_datac = [0.0, 0.0, 0.0]
time_needed_datac = [0.0, 0.0, 0.0]
time_start_datac = [0.0, 0.0, 0.0]
datac_buffer: List[codec2.audio_buffer] = []
datac_bytes_out: List[ctypes.Array] = []
datac_bytes_per_frame = []
datac_freedv: List[ctypes.c_void_p] = []
args = parse_arguments()
if args.LIST:
p_audio = pyaudio.PyAudio()
for dev in range(p_audio.get_device_count()):
print("audiodev: ", dev, p_audio.get_device_info_by_index(dev)["name"])
sys.exit()
N_BURSTS = args.N_BURSTS
N_FRAMES_PER_BURST = args.N_FRAMES_PER_BURST
AUDIO_INPUT_DEVICE = args.AUDIO_INPUT_DEVICE
DEBUGGING_MODE = args.DEBUGGING_MODE
MAX_TIME = args.TIMEOUT
# open codec2 instances
for idx in range(3):
datac_freedv.append(
ctypes.cast(
codec2.api.freedv_open(codec2.FREEDV_MODE.datac13.value), ctypes.c_void_p
)
)
datac_bytes_per_frame.append(
int(codec2.api.freedv_get_bits_per_modem_frame(datac_freedv[idx]) / 8)
)
datac_bytes_out.append(ctypes.create_string_buffer(datac_bytes_per_frame[idx]))
codec2.api.freedv_set_frames_per_burst(datac_freedv[idx], N_FRAMES_PER_BURST)
datac_buffer.append(codec2.audio_buffer(2 * AUDIO_FRAMES_PER_BUFFER))
resampler = codec2.resampler()
# check if we want to use an audio device then do a pyaudio init
if AUDIO_INPUT_DEVICE != -1:
p_audio = pyaudio.PyAudio()
# auto search for loopback devices
if AUDIO_INPUT_DEVICE == -2:
loopback_list = [
dev
for dev in range(p_audio.get_device_count())
if "Loopback: PCM" in p_audio.get_device_info_by_index(dev)["name"]
]
if len(loopback_list) >= 2:
AUDIO_INPUT_DEVICE = loopback_list[0] # 0 = RX 1 = TX
print(f"loopback_list rx: {loopback_list}", file=sys.stderr)
else:
sys.exit()
print(
f"AUDIO INPUT DEVICE: {AUDIO_INPUT_DEVICE} "
f"DEVICE: {p_audio.get_device_info_by_index(AUDIO_INPUT_DEVICE)['name']} "
f"AUDIO SAMPLE RATE: {AUDIO_SAMPLE_RATE_RX}",
file=sys.stderr,
)
stream_rx = p_audio.open(
format=pyaudio.paInt16,
channels=1,
rate=AUDIO_SAMPLE_RATE_RX,
frames_per_buffer=AUDIO_FRAMES_PER_BUFFER,
input=True,
input_device_index=AUDIO_INPUT_DEVICE,
)
timeout = time.time() + MAX_TIME
print(time.time(), MAX_TIME, timeout)
receive = True
nread_exceptions = 0
# initial nin values
datac_nin = [0, 0, 0]
for idx in range(3):
datac_nin[idx] = codec2.api.freedv_nin(datac_freedv[idx])
def print_stats(time_datac13, time_datac1, time_datac3):
if not DEBUGGING_MODE:
return
time_datac = [time_datac13, time_datac1, time_datac3]
datac_rxstatus = ["", "", ""]
for idx in range(3):
datac_rxstatus[idx] = codec2.api.rx_sync_flags_to_text[
codec2.api.freedv_get_rx_status(datac_freedv[idx])
]
text_out = ""
for idx in range(3):
text_out += f"NIN{idx}: {datac_nin[idx]:5d} "
text_out += f"RX_STATUS{idx}: {datac_rxstatus[idx]:4s} "
text_out += f"TIME: {round(time_datac[idx], 4):.4f} | "
text_out = text_out.rstrip(" ").rstrip("|").rstrip(" ")
print(text_out, file=sys.stderr)
while receive and time.time() < timeout:
if AUDIO_INPUT_DEVICE != -1:
try:
data_in48k = stream_rx.read(
AUDIO_FRAMES_PER_BUFFER, exception_on_overflow=True
)
except OSError as err:
print(err, file=sys.stderr)
if "Input overflowed" in str(err):
nread_exceptions += 1
if "Stream closed" in str(err):
print("Ending....")
receive = False
else:
data_in48k = sys.stdin.buffer.read(AUDIO_FRAMES_PER_BUFFER * 2)
# insert samples in buffer
audio_buffer = np.frombuffer(data_in48k, dtype=np.int16)
if len(audio_buffer) != AUDIO_FRAMES_PER_BUFFER:
print("len(x)", len(audio_buffer))
receive = False
audio_buffer = resampler.resample48_to_8(audio_buffer)
for idx in range(3):
datac_buffer[idx].push(audio_buffer)
while datac_buffer[idx].nbuffer >= datac_nin[idx]:
# demodulate audio
time_start_datac[idx] = time.time()
nbytes = codec2.api.freedv_rawdatarx(
datac_freedv[idx],
datac_bytes_out[idx],
datac_buffer[idx].buffer.ctypes,
)
time_end_datac[idx] = time.time()
datac_buffer[idx].pop(datac_nin[idx])
datac_nin[idx] = codec2.api.freedv_nin(datac_freedv[idx])
if nbytes == datac_bytes_per_frame[idx]:
rx_total_frames_datac[idx] += 1
rx_frames_datac[idx] += 1
if rx_frames_datac[idx] == N_FRAMES_PER_BURST:
rx_frames_datac[idx] = 0
rx_bursts_datac[idx] += 1
time_needed_datac[idx] = time_end_datac[idx] - time_start_datac[idx]
print_stats(
time_needed_datac[0], time_needed_datac[1], time_needed_datac[2]
)
if (
rx_bursts_datac[0] == N_BURSTS
and rx_bursts_datac[1] == N_BURSTS
and rx_bursts_datac[2] == N_BURSTS
):
receive = False
if nread_exceptions:
print(
f"nread_exceptions {nread_exceptions:d} - receive audio lost! "
"Consider increasing Pyaudio frames_per_buffer...",
file=sys.stderr,
)
# INFO IF WE REACHED TIMEOUT
if time.time() > timeout:
print("TIMEOUT REACHED", file=sys.stderr)
print(
f"DATAC13: {rx_bursts_datac[0]}/{rx_total_frames_datac[0]} "
f"DATAC1: {rx_bursts_datac[1]}/{rx_total_frames_datac[1]} "
f"DATAC3: {rx_bursts_datac[2]}/{rx_total_frames_datac[2]}",
file=sys.stderr,
)
if AUDIO_INPUT_DEVICE != -1:
stream_rx.close()
p_audio.terminate()
def parse_arguments():
# --------------------------------------------GET PARAMETER INPUTS
parser = argparse.ArgumentParser(description="Simons TEST TNC")
parser.add_argument("--bursts", dest="N_BURSTS", default=1, type=int)
parser.add_argument(
"--framesperburst", dest="N_FRAMES_PER_BURST", default=1, type=int
)
parser.add_argument(
"--audiodev",
dest="AUDIO_INPUT_DEVICE",
default=-1,
type=int,
help="audio device number to use",
)
parser.add_argument("--debug", dest="DEBUGGING_MODE", action="store_true")
parser.add_argument(
"--list",
dest="LIST",
action="store_true",
help="list audio devices by number and exit",
)
parser.add_argument(
"--timeout",
dest="TIMEOUT",
default=60,
type=int,
help="Timeout (seconds) before test ends",
)
args, _ = parser.parse_known_args()
return args
if __name__ == "__main__":
test_mm_rx()

199
modem/data cemetery/test/util_multimode_tx.py
View file

@ -1,199 +0,0 @@
#!/usr/bin/env python3
# -*- coding: utf-8 -*-
"""
Send-side station emulator for test frame tests over a high quality audio channel
using a physical sound card or STDIO.
Legacy test for sending / receiving connection test frames through the codec2 and
back through on the other station. Data injection initiates directly through
the codec2 API. Tests all three codec2 data frames simultaneously.
Invoked from CMake, test_highsnr_stdio_P_P_multi.py, and many test_virtual[1-3]*.sh.
@author: DJ2LS
"""
import argparse
import ctypes
import sys
import time
import numpy as np
import pyaudio
sys.path.insert(0, "..")
from modem import codec2
def test_mm_tx():
MODEM_SAMPLE_RATE = codec2.api.FREEDV_FS_8000
AUDIO_SAMPLE_RATE_TX = 48000
assert (AUDIO_SAMPLE_RATE_TX % MODEM_SAMPLE_RATE) == 0
args = parse_arguments()
if args.LIST:
p_audio = pyaudio.PyAudio()
for dev in range(p_audio.get_device_count()):
print("audiodev: ", dev, p_audio.get_device_info_by_index(dev)["name"])
sys.exit()
N_BURSTS = args.N_BURSTS
N_FRAMES_PER_BURST = args.N_FRAMES_PER_BURST
DELAY_BETWEEN_BURSTS = args.DELAY_BETWEEN_BURSTS / 1000
AUDIO_OUTPUT_DEVICE = args.AUDIO_OUTPUT_DEVICE
resampler = codec2.resampler()
# Data binary string
data_out = b"HELLO WORLD!"
modes = [
codec2.FREEDV_MODE.datac13.value,
codec2.FREEDV_MODE.datac1.value,
codec2.FREEDV_MODE.datac3.value,
]
if AUDIO_OUTPUT_DEVICE != -1:
p_audio = pyaudio.PyAudio()
# Auto search for loopback devices
if AUDIO_OUTPUT_DEVICE == -2:
loopback_list = [
dev
for dev in range(p_audio.get_device_count())
if "Loopback: PCM" in p_audio.get_device_info_by_index(dev)["name"]
]
if len(loopback_list) >= 2:
AUDIO_OUTPUT_DEVICE = loopback_list[1] # 0 = RX 1 = TX
print(f"loopback_list tx: {loopback_list}", file=sys.stderr)
else:
sys.exit()
# AUDIO PARAMETERS
AUDIO_FRAMES_PER_BUFFER = 2400
# pyaudio init
stream_tx = p_audio.open(
format=pyaudio.paInt16,
channels=1,
rate=AUDIO_SAMPLE_RATE_TX,
frames_per_buffer=AUDIO_FRAMES_PER_BUFFER, # n_nom_modem_samples
output=True,
output_device_index=AUDIO_OUTPUT_DEVICE,
)
for mode in modes:
freedv = ctypes.cast(codec2.api.freedv_open(mode), ctypes.c_void_p)
n_tx_modem_samples = codec2.api.freedv_get_n_tx_modem_samples(freedv)
mod_out = ctypes.create_string_buffer(2 * n_tx_modem_samples)
n_tx_preamble_modem_samples = codec2.api.freedv_get_n_tx_preamble_modem_samples(
freedv
)
mod_out_preamble = ctypes.create_string_buffer(2 * n_tx_preamble_modem_samples)
n_tx_postamble_modem_samples = (
codec2.api.freedv_get_n_tx_postamble_modem_samples(freedv)
)
mod_out_postamble = ctypes.create_string_buffer(
2 * n_tx_postamble_modem_samples
)
bytes_per_frame = int(codec2.api.freedv_get_bits_per_modem_frame(freedv) / 8)
payload_per_frame = bytes_per_frame - 2
buffer = bytearray(payload_per_frame)
# Set buffer size to length of data which will be sent
buffer[: len(data_out)] = data_out
# Generate CRC16
crc = ctypes.c_ushort(
codec2.api.freedv_gen_crc16(bytes(buffer), payload_per_frame)
)
# Convert CRC to 2 byte hex string
crc = crc.value.to_bytes(2, byteorder="big")
buffer += crc # Append crc16 to buffer
data = (ctypes.c_ubyte * bytes_per_frame).from_buffer_copy(buffer)
for brst in range(1, N_BURSTS + 1):
# Write preamble to txbuffer
codec2.api.freedv_rawdatapreambletx(freedv, mod_out_preamble)
txbuffer = bytes(mod_out_preamble)
# Create modulaton for N = FRAMESPERBURST and append it to txbuffer
for frm in range(1, N_FRAMES_PER_BURST + 1):
data = (ctypes.c_ubyte * bytes_per_frame).from_buffer_copy(buffer)
# Modulate DATA and save it into mod_out pointer
codec2.api.freedv_rawdatatx(freedv, mod_out, data)
txbuffer += bytes(mod_out)
print(
f"TX BURST: {brst}/{N_BURSTS} FRAME: {frm}/{N_FRAMES_PER_BURST}",
file=sys.stderr,
)
# Append postamble to txbuffer
codec2.api.freedv_rawdatapostambletx(freedv, mod_out_postamble)
txbuffer += bytes(mod_out_postamble)
# Append a delay between bursts as audio silence
samples_delay = int(MODEM_SAMPLE_RATE * DELAY_BETWEEN_BURSTS)
mod_out_silence = ctypes.create_string_buffer(samples_delay * 2)
txbuffer += bytes(mod_out_silence)
# Resample up to 48k (resampler works on np.int16)
audio_buffer = np.frombuffer(txbuffer, dtype=np.int16)
txbuffer_48k = resampler.resample8_to_48(audio_buffer)
# Check if we want to use an audio device or stdout
if AUDIO_OUTPUT_DEVICE != -1:
stream_tx.write(txbuffer_48k.tobytes())
else:
# This test needs a lot of time, so we are having a look at times...
starttime = time.time()
# Print data to terminal for piping the output to other programs
sys.stdout.buffer.write(txbuffer_48k)
sys.stdout.flush()
# and at least print the needed time to see which time we needed
timeneeded = time.time() - starttime
# print(f"time: {timeneeded} buffer: {len(txbuffer)}", file=sys.stderr)
# and at last check if we had an opened pyaudio instance and close it
if AUDIO_OUTPUT_DEVICE != -1:
time.sleep(stream_tx.get_output_latency())
stream_tx.stop_stream()
stream_tx.close()
p_audio.terminate()
def parse_arguments():
# GET PARAMETER INPUTS
parser = argparse.ArgumentParser(description="FreeDATA TEST")
parser.add_argument("--bursts", dest="N_BURSTS", default=1, type=int)
parser.add_argument(
"--framesperburst", dest="N_FRAMES_PER_BURST", default=1, type=int
)
parser.add_argument("--delay", dest="DELAY_BETWEEN_BURSTS", default=500, type=int)
parser.add_argument(
"--audiodev",
dest="AUDIO_OUTPUT_DEVICE",
default=-1,
type=int,
help="audio output device number to use",
)
parser.add_argument(
"--list",
dest="LIST",
action="store_true",
help="list audio devices by number and exit",
)
args, _ = parser.parse_known_args()
return args
if __name__ == "__main__":
test_mm_tx()

253
modem/data cemetery/test/util_rx.py
View file

@ -1,253 +0,0 @@
#!/usr/bin/env python3
# -*- coding: utf-8 -*-
"""
Receive-side station emulator for test frame tests over a high quality audio channel
using a physical sound card or STDIO.
Legacy test for sending / receiving connection test frames through the codec2 and
back through on the other station. Data injection initiates directly through
the codec2 API.
Invoked from CMake, test_highsnr_stdio_{P_C, P_P}_datacx.py, and many test_virtual[1-3]*.sh.
@author: DJ2LS
"""
import argparse
import ctypes
import sys
import time
import numpy as np
import sounddevice as sd
# pylint: disable=wrong-import-position
sys.path.insert(0, "..")
sys.path.insert(0, "../modem")
from modem import codec2
def util_rx():
args = parse_arguments()
if args.LIST:
devices = sd.query_devices(device=None, kind=None)
for index, device in enumerate(devices):
print(f"{index} {device['name']}")
index += 1
# pylint: disable=protected-access
sd._terminate()
sys.exit()
N_BURSTS = args.N_BURSTS
N_FRAMES_PER_BURST = args.N_FRAMES_PER_BURST
AUDIO_INPUT_DEVICE = args.AUDIO_INPUT_DEVICE
MODE = codec2.FREEDV_MODE[args.FREEDV_MODE].value
DEBUGGING_MODE = args.DEBUGGING_MODE
MAX_TIME = args.TIMEOUT
# AUDIO PARAMETERS
# v-- consider increasing if you get nread_exceptions > 0
AUDIO_FRAMES_PER_BUFFER = 2400 * 2
MODEM_SAMPLE_RATE = codec2.api.FREEDV_FS_8000
AUDIO_SAMPLE_RATE_RX = 48000
# make sure our resampler will work
assert (AUDIO_SAMPLE_RATE_RX / MODEM_SAMPLE_RATE) == codec2.api.FDMDV_OS_48 # type: ignore
# check if we want to use an audio device then do a pyaudio init
if AUDIO_INPUT_DEVICE != -1:
# auto search for loopback devices
if AUDIO_INPUT_DEVICE == -2:
loopback_list = []
devices = sd.query_devices(device=None, kind=None)
for index, device in enumerate(devices):
if "Loopback: PCM" in device["name"]:
print(index)
loopback_list.append(index)
if loopback_list:
# 0 = RX 1 = TX
AUDIO_INPUT_DEVICE = loopback_list[0]
print(f"loopback_list tx: {loopback_list}", file=sys.stderr)
else:
print("not enough audio loopback devices ready...")
print("you should wait about 30 seconds...")
sd._terminate()
sys.exit()
print(f"AUDIO INPUT DEVICE: {AUDIO_INPUT_DEVICE}", file=sys.stderr)
# audio stream init
stream_rx = sd.RawStream(
channels=1,
dtype="int16",
device=AUDIO_INPUT_DEVICE,
samplerate=AUDIO_SAMPLE_RATE_RX,
blocksize=4800,
)
stream_rx.start()
# ----------------------------------------------------------------
# DATA CHANNEL INITIALISATION
# open codec2 instance
freedv = ctypes.cast(codec2.api.freedv_open(MODE), ctypes.c_void_p)
# get number of bytes per frame for mode
bytes_per_frame = int(codec2.api.freedv_get_bits_per_modem_frame(freedv) / 8)
payload_bytes_per_frame = bytes_per_frame - 2
n_max_modem_samples = codec2.api.freedv_get_n_max_modem_samples(freedv)
bytes_out = ctypes.create_string_buffer(bytes_per_frame)
codec2.api.freedv_set_frames_per_burst(freedv, N_FRAMES_PER_BURST)
total_n_bytes = 0
rx_total_frames = 0
rx_frames = 0
rx_bursts = 0
rx_errors = 0
nread_exceptions = 0
timeout = time.time() + MAX_TIME
receive = True
audio_buffer = codec2.audio_buffer(AUDIO_FRAMES_PER_BUFFER * 2)
resampler = codec2.resampler()
# time meassurement
time_start = 0
time_end = 0
with open("rx48.raw", mode="wb") as frx:
# initial number of samples we need
nin = codec2.api.freedv_nin(freedv)
while receive and time.time() < timeout:
if AUDIO_INPUT_DEVICE != -1:
try:
# data_in48k = stream_rx.read(AUDIO_FRAMES_PER_BUFFER, exception_on_overflow = True)
data_in48k, overflowed = stream_rx.read(AUDIO_FRAMES_PER_BUFFER) # type: ignore
except OSError as err:
print(err, file=sys.stderr)
# if str(err).find("Input overflowed") != -1:
# nread_exceptions += 1
# if str(err).find("Stream closed") != -1:
# print("Ending...")
# receive = False
else:
data_in48k = sys.stdin.buffer.read(AUDIO_FRAMES_PER_BUFFER * 2)
# insert samples in buffer
x = np.frombuffer(data_in48k, dtype=np.int16) # type: ignore
# print(x)
# x = data_in48k
x.tofile(frx)
if len(x) != AUDIO_FRAMES_PER_BUFFER:
receive = False
x = resampler.resample48_to_8(x)
audio_buffer.push(x)
# when we have enough samples call FreeDV Rx
while audio_buffer.nbuffer >= nin:
# start time measurement
time_start = time.time()
# demodulate audio
nbytes = codec2.api.freedv_rawdatarx(
freedv, bytes_out, audio_buffer.buffer.ctypes
)
time_end = time.time()
audio_buffer.pop(nin)
# call me on every loop!
nin = codec2.api.freedv_nin(freedv)
rx_status = codec2.api.freedv_get_rx_status(freedv)
if rx_status & codec2.api.FREEDV_RX_BIT_ERRORS:
rx_errors = rx_errors + 1
if DEBUGGING_MODE:
rx_status = codec2.api.rx_sync_flags_to_text[rx_status] # type: ignore
time_needed = time_end - time_start
print(
f"nin: {nin:5d} rx_status: {rx_status:4s} "
f"naudio_buffer: {audio_buffer.nbuffer:4d} time: {time_needed:4f}",
file=sys.stderr,
)
if nbytes:
total_n_bytes += nbytes
if nbytes == bytes_per_frame:
rx_total_frames += 1
rx_frames += 1
if rx_frames == N_FRAMES_PER_BURST:
rx_frames = 0
rx_bursts += 1
if rx_bursts == N_BURSTS:
receive = False
if time.time() >= timeout:
print("TIMEOUT REACHED")
time.sleep(0.01)
if nread_exceptions:
print(
f"nread_exceptions {nread_exceptions:d} - receive audio lost! "
"Consider increasing Pyaudio frames_per_buffer...",
file=sys.stderr,
)
print(
f"RECEIVED BURSTS: {rx_bursts} "
f"RECEIVED FRAMES: {rx_total_frames} "
f"RX_ERRORS: {rx_errors}",
file=sys.stderr,
)
# and at last check if we had an opened audio instance and close it
if AUDIO_INPUT_DEVICE != -1:
sd._terminate()
def parse_arguments():
# --------------------------------------------GET PARAMETER INPUTS
parser = argparse.ArgumentParser(description="Simons TEST TNC")
parser.add_argument("--bursts", dest="N_BURSTS", default=1, type=int)
parser.add_argument(
"--framesperburst", dest="N_FRAMES_PER_BURST", default=1, type=int
)
parser.add_argument(
"--mode", dest="FREEDV_MODE", type=str, choices=["datac13", "datac1", "datac3"]
)
parser.add_argument(
"--audiodev",
dest="AUDIO_INPUT_DEVICE",
default=-1,
type=int,
help="audio device number to use, use -2 to automatically select a loopback device",
)
parser.add_argument("--debug", dest="DEBUGGING_MODE", action="store_true")
parser.add_argument(
"--timeout",
dest="TIMEOUT",
default=60,
type=int,
help="Timeout (seconds) before test ends",
)
parser.add_argument(
"--list",
dest="LIST",
action="store_true",
help="list audio devices by number and exit",
)
args, _ = parser.parse_known_args()
return args
if __name__ == "__main__":
util_rx()

229
modem/data cemetery/test/util_tx.py
View file

@ -1,229 +0,0 @@
#!/usr/bin/env python3
# -*- coding: utf-8 -*-
"""
Send-side station emulator for test frame tests over a high quality audio channel
using a physical sound card or STDIO.
Legacy test for sending / receiving connection test frames through the codec2 and
back through on the other station. Data injection initiates directly through
the codec2 API.
Invoked from CMake, test_highsnr_stdio_{P_C, P_P}_datacx.py, and many test_virtual[1-3]*.sh.
@author: DJ2LS
"""
import argparse
import ctypes
import sys
import numpy as np
import sounddevice as sd
sys.path.insert(0, "..")
from modem import codec2
def util_tx():
args = parse_arguments()
if args.LIST:
devices = sd.query_devices(device=None, kind=None)
for index, device in enumerate(devices):
print(f"{index} {device['name']}")
sd._terminate()
sys.exit()
N_BURSTS = args.N_BURSTS
N_FRAMES_PER_BURST = args.N_FRAMES_PER_BURST
DELAY_BETWEEN_BURSTS = args.DELAY_BETWEEN_BURSTS / 1000
AUDIO_OUTPUT_DEVICE = args.AUDIO_OUTPUT_DEVICE
MODE = codec2.FREEDV_MODE[args.FREEDV_MODE].value
# AUDIO PARAMETERS
AUDIO_FRAMES_PER_BUFFER = 2400
MODEM_SAMPLE_RATE = codec2.api.FREEDV_FS_8000
AUDIO_SAMPLE_RATE_TX = 48000
assert (AUDIO_SAMPLE_RATE_TX % MODEM_SAMPLE_RATE) == 0 # type: ignore
# check if we want to use an audio device then do a pyaudio init
if AUDIO_OUTPUT_DEVICE != -1:
# auto search for loopback devices
if AUDIO_OUTPUT_DEVICE == -2:
loopback_list = []
devices = sd.query_devices(device=None, kind=None)
for index, device in enumerate(devices):
if "Loopback: PCM" in device["name"]:
print(index)
loopback_list.append(index)
if loopback_list:
# 0 = RX 1 = TX
AUDIO_OUTPUT_DEVICE = loopback_list[-1]
print(f"loopback_list tx: {loopback_list}", file=sys.stderr)
else:
print("not enough audio loopback devices ready...")
print("you should wait about 30 seconds...")
sd._terminate()
sys.exit()
print(f"AUDIO OUTPUT DEVICE: {AUDIO_OUTPUT_DEVICE}", file=sys.stderr)
# audio stream init
stream_tx = sd.RawStream(
channels=1,
dtype="int16",
device=(0, AUDIO_OUTPUT_DEVICE),
samplerate=AUDIO_SAMPLE_RATE_TX,
blocksize=4800,
)
resampler = codec2.resampler()
# data binary string
if args.TESTFRAMES:
data_out = bytearray(14)
data_out[:1] = bytes([255])
data_out[1:2] = bytes([1])
data_out[2:] = b"HELLO WORLD"
else:
data_out = b"HELLO WORLD!"
# ----------------------------------------------------------------
# Open codec2 instance
freedv = ctypes.cast(codec2.api.freedv_open(MODE), ctypes.c_void_p)
# Get number of bytes per frame for mode
bytes_per_frame = int(codec2.api.freedv_get_bits_per_modem_frame(freedv) / 8)
payload_bytes_per_frame = bytes_per_frame - 2
# Init buffer for data
n_tx_modem_samples = codec2.api.freedv_get_n_tx_modem_samples(freedv)
mod_out = ctypes.create_string_buffer(n_tx_modem_samples * 2)
# Init buffer for preample
n_tx_preamble_modem_samples = codec2.api.freedv_get_n_tx_preamble_modem_samples(
freedv
)
mod_out_preamble = ctypes.create_string_buffer(n_tx_preamble_modem_samples * 2)
# Init buffer for postamble
n_tx_postamble_modem_samples = codec2.api.freedv_get_n_tx_postamble_modem_samples(
freedv
)
mod_out_postamble = ctypes.create_string_buffer(n_tx_postamble_modem_samples * 2)
# Create buffer for data
# Use this if CRC16 checksum is required (DATA1-3)
buffer = bytearray(payload_bytes_per_frame)
# set buffer size to length of data which will be sent
buffer[: len(data_out)] = data_out
# Create CRC for data frame - we are using the CRC function shipped with codec2 to avoid
# CRC algorithm incompatibilities
# generate CRC16
crc = ctypes.c_ushort(
codec2.api.freedv_gen_crc16(bytes(buffer), payload_bytes_per_frame)
)
crc = crc.value.to_bytes(2, byteorder="big") # convert crc to 2 byte hex string
buffer += crc # append crc16 to buffer
print(
f"TOTAL BURSTS: {N_BURSTS} TOTAL FRAMES_PER_BURST: {N_FRAMES_PER_BURST}",
file=sys.stderr,
)
for brst in range(1, N_BURSTS + 1):
# Write preamble to txbuffer
codec2.api.freedv_rawdatapreambletx(freedv, mod_out_preamble)
txbuffer = bytes(mod_out_preamble)
# Create modulaton for N = FRAMESPERBURST and append it to txbuffer
for frm in range(1, N_FRAMES_PER_BURST + 1):
data = (ctypes.c_ubyte * bytes_per_frame).from_buffer_copy(buffer)
# Modulate DATA and save it into mod_out pointer
codec2.api.freedv_rawdatatx(freedv, mod_out, data)
txbuffer += bytes(mod_out)
print(
f"TX BURST: {brst}/{N_BURSTS} FRAME: {frm}/{N_FRAMES_PER_BURST}",
file=sys.stderr,
)
# Append postamble to txbuffer
codec2.api.freedv_rawdatapostambletx(freedv, mod_out_postamble)
txbuffer += bytes(mod_out_postamble)
# Append a delay between bursts as audio silence
samples_delay = int(MODEM_SAMPLE_RATE * DELAY_BETWEEN_BURSTS)
mod_out_silence = ctypes.create_string_buffer(samples_delay * 2)
txbuffer += bytes(mod_out_silence)
# print(f"samples_delay: {samples_delay} DELAY_BETWEEN_BURSTS: {DELAY_BETWEEN_BURSTS}", file=sys.stderr)
# Resample up to 48k (resampler works on np.int16)
np_buffer = np.frombuffer(txbuffer, dtype=np.int16)
txbuffer_48k = resampler.resample8_to_48(np_buffer)
# Check if we want to use an audio device or stdout
if AUDIO_OUTPUT_DEVICE != -1:
stream_tx.start() # type: ignore
stream_tx.write(txbuffer_48k) # type: ignore
else:
# Print data to terminal for piping the output to other programs
sys.stdout.buffer.write(txbuffer_48k) # type: ignore
sys.stdout.flush()
# and at last check if we had an opened audio instance and close it
if AUDIO_OUTPUT_DEVICE != -1:
sd._terminate()
def parse_arguments():
# GET PARAMETER INPUTS
parser = argparse.ArgumentParser(description="Simons TEST TNC")
parser.add_argument("--bursts", dest="N_BURSTS", default=1, type=int)
parser.add_argument(
"--framesperburst", dest="N_FRAMES_PER_BURST", default=1, type=int
)
parser.add_argument(
"--delay",
dest="DELAY_BETWEEN_BURSTS",
default=500,
type=int,
help="delay between bursts in ms",
)
parser.add_argument(
"--mode", dest="FREEDV_MODE", type=str, choices=["datac13", "datac1", "datac3"]
)
parser.add_argument(
"--audiodev",
dest="AUDIO_OUTPUT_DEVICE",
default=-1,
type=int,
help="audio output device number to use, use -2 to automatically select a loopback device",
)
parser.add_argument(
"--list",
dest="LIST",
action="store_true",
help="list audio devices by number and exit",
)
parser.add_argument(
"--testframes",
dest="TESTFRAMES",
action="store_true",
default=False,
help="list audio devices by number and exit",
)
args, _ = parser.parse_known_args()
return args
if __name__ == "__main__":
util_tx()

95
modem/data_frame_factory.py
View file

@ -7,14 +7,15 @@ class DataFrameFactory:
LENGTH_SIG0_FRAME = 14
LENGTH_SIG1_FRAME = 14
"""
helpers.set_flag(byte, 'DATA-ACK-NACK', True, FLAG_POSITIONS)
helpers.get_flag(byte, 'DATA-ACK-NACK', FLAG_POSITIONS)
"""
ARQ_FLAGS = {
'FINAL': 0, # Bit position for indicating the FINAL state
'CHECKSUM': 1, # Bit position for indicating the CHECKSUM is correct or not
'FINAL': 0, # Bit-position for indicating the FINAL state
'ABORT': 1, # Bit-position for indicating the ABORT request
'CHECKSUM': 2, # Bit-position for indicating the CHECKSUM is correct or not
'ENABLE_COMPRESSION': 3 # Bit-position for indicating compression is enabled
}
def __init__(self, config):
@ -115,6 +116,7 @@ class DataFrameFactory:
"total_length": 4,
"total_crc": 4,
"snr": 1,
"flag": 1,
}
self.template_list[FR_TYPE.ARQ_SESSION_INFO_ACK.value] = {
@ -124,6 +126,17 @@ class DataFrameFactory:
"snr": 1,
"speed_level": 1,
"frames_per_burst": 1,
"flag": 1,
}
self.template_list[FR_TYPE.ARQ_STOP.value] = {
"frame_length": self.LENGTH_SIG0_FRAME,
"session_id": 1,
}
self.template_list[FR_TYPE.ARQ_STOP_ACK.value] = {
"frame_length": self.LENGTH_SIG0_FRAME,
"session_id": 1,
}
# arq burst frame
@ -145,23 +158,6 @@ class DataFrameFactory:
"flag": 1,
}
# arq burst nack
self.template_list[FR_TYPE.ARQ_BURST_NACK.value] = {
"frame_length": self.LENGTH_SIG1_FRAME,
"session_id": 1,
"offset":4,
"speed_level": 1,
"frames_per_burst": 1,
"snr": 1,
}
# arq data ack nack
self.template_list[FR_TYPE.ARQ_DATA_ACK_NACK.value] = {
"frame_length": self.LENGTH_SIG1_FRAME,
"session_id": 1,
"state": 1,
"snr": 1,
}
def construct(self, frametype, content, frame_length = LENGTH_SIG1_FRAME):
frame_template = self.template_list[frametype.value]
@ -221,19 +217,22 @@ class DataFrameFactory:
elif key in ["session_id", "speed_level",
"frames_per_burst", "version",
"snr", "offset", "total_length", "state"]:
"offset", "total_length", "state"]:
extracted_data[key] = int.from_bytes(data, 'big')
elif key in ["snr"]:
extracted_data[key] = helpers.snr_from_bytes(data)
elif key == "flag":
data = int.from_bytes(data, "big")
extracted_data[key] = {}
if frametype == FR_TYPE.ARQ_BURST_ACK.value:
if frametype in [FR_TYPE.ARQ_BURST_ACK.value, FR_TYPE.ARQ_SESSION_INFO_ACK.value]:
flag_dict = self.ARQ_FLAGS
for flag in flag_dict:
# Update extracted_data with the status of each flag
# get_flag returns True or False based on the bit value at the flag's position
extracted_data[key][flag] = helpers.get_flag(data, flag, flag_dict)
for flag in flag_dict:
# Update extracted_data with the status of each flag
# get_flag returns True or False based on the bit value at the flag's position
extracted_data[key][flag] = helpers.get_flag(data, flag, flag_dict)
else:
extracted_data[key] = data
@ -241,7 +240,6 @@ class DataFrameFactory:
return extracted_data
def get_bytes_per_frame(self, mode: codec2.FREEDV_MODE) -> int:
freedv = codec2.open_instance(mode.value)
bytes_per_frame = int(codec2.api.freedv_get_bits_per_modem_frame(freedv) / 8)
@ -341,26 +339,53 @@ class DataFrameFactory:
"origin": helpers.callsign_to_bytes(self.myfullcall),
"destination_crc": helpers.get_crc_24(destination),
"version": bytes([version]),
"snr": snr.to_bytes(1, 'big'), }
"snr": helpers.snr_to_bytes(1),
}
return self.construct(FR_TYPE.ARQ_SESSION_OPEN_ACK, payload)
def build_arq_session_info(self, session_id: int, total_length: int, total_crc: bytes, snr):
def build_arq_session_info(self, session_id: int, total_length: int, total_crc: bytes, snr, flag_compression=False):
flag = 0b00000000
if flag_compression:
flag = helpers.set_flag(flag, 'ENABLE_COMPRESSION', True, self.ARQ_FLAGS)
payload = {
"session_id": session_id.to_bytes(1, 'big'),
"total_length": total_length.to_bytes(4, 'big'),
"total_crc": total_crc,
"snr": snr.to_bytes(1, 'big'),
"snr": helpers.snr_to_bytes(1),
"flag": flag.to_bytes(1, 'big'),
}
return self.construct(FR_TYPE.ARQ_SESSION_INFO, payload)
def build_arq_session_info_ack(self, session_id, total_crc, snr, speed_level, frames_per_burst):
def build_arq_stop(self, session_id: int):
payload = {
"session_id": session_id.to_bytes(1, 'big'),
}
return self.construct(FR_TYPE.ARQ_STOP, payload)
def build_arq_stop_ack(self, session_id: int):
payload = {
"session_id": session_id.to_bytes(1, 'big'),
}
return self.construct(FR_TYPE.ARQ_STOP_ACK, payload)
def build_arq_session_info_ack(self, session_id, total_crc, snr, speed_level, frames_per_burst, flag_final=False, flag_abort=False):
flag = 0b00000000
if flag_final:
flag = helpers.set_flag(flag, 'FINAL', True, self.ARQ_FLAGS)
if flag_abort:
flag = helpers.set_flag(flag, 'ABORT', True, self.ARQ_FLAGS)
payload = {
"frame_length": self.LENGTH_SIG0_FRAME,
"session_id": session_id.to_bytes(1, 'big'),
"total_crc": bytes.fromhex(total_crc),
"snr": snr.to_bytes(1, 'big'),
"snr": helpers.snr_to_bytes(1),
"speed_level": speed_level.to_bytes(1, 'big'),
"frames_per_burst": frames_per_burst.to_bytes(1, 'big'),
"flag": flag.to_bytes(1, 'big'),
}
return self.construct(FR_TYPE.ARQ_SESSION_INFO_ACK, payload)
@ -374,7 +399,7 @@ class DataFrameFactory:
return frame
def build_arq_burst_ack(self, session_id: bytes, offset, speed_level: int,
frames_per_burst: int, snr: int, flag_final=False, flag_checksum=False):
frames_per_burst: int, snr: int, flag_final=False, flag_checksum=False, flag_abort=False):
flag = 0b00000000
if flag_final:
flag = helpers.set_flag(flag, 'FINAL', True, self.ARQ_FLAGS)
@ -382,6 +407,10 @@ class DataFrameFactory:
if flag_checksum:
flag = helpers.set_flag(flag, 'CHECKSUM', True, self.ARQ_FLAGS)
if flag_abort:
flag = helpers.set_flag(flag, 'ABORT', True, self.ARQ_FLAGS)
payload = {
"session_id": session_id.to_bytes(1, 'big'),
"offset": offset.to_bytes(4, 'big'),

21
modem/demodulator.py
View file

@ -37,7 +37,7 @@ class Demodulator():
self.AUDIO_FRAMES_PER_BUFFER_RX = 4800
self.buffer_overflow_counter = [0, 0, 0, 0, 0, 0, 0, 0]
self.is_codec2_traffic_counter = 0
self.is_codec2_traffic_cooldown = 20
self.is_codec2_traffic_cooldown = 5
self.audio_received_queue = audio_rx_q
self.modem_received_queue = modem_rx_q
@ -56,6 +56,12 @@ class Demodulator():
# enable decoding of signalling modes
self.MODE_DICT[codec2.FREEDV_MODE.signalling.value]["decode"] = True
tci_rx_callback_thread = threading.Thread(
target=self.tci_rx_callback,
name="TCI RX CALLBACK THREAD",
daemon=True,
)
tci_rx_callback_thread.start()
def init_codec2(self):
# Open codec2 instances
@ -217,27 +223,18 @@ class Demodulator():
rx_status = codec2.api.freedv_get_rx_status(freedv)
if rx_status not in [0]:
# we need to disable this if in testmode as its causing problems with FIFO it seems
self.states.set("is_codec2_traffic", True)
self.is_codec2_traffic_counter = self.is_codec2_traffic_cooldown
if not self.states.channel_busy:
self.log.debug("[MDM] Setting channel_busy since codec2 data detected")
self.states.set("channel_busy", True)
#self.channel_busy_delay += 10
self.log.debug(
"[MDM] [demod_audio] modem state", mode=mode_name, rx_status=rx_status,
sync_flag=codec2.api.rx_sync_flags_to_text[rx_status]
)
else:
self.states.set("is_codec2_traffic", False)
# decrement codec traffic counter for making state smoother
if self.is_codec2_traffic_counter > 0:
self.is_codec2_traffic_counter -= 1
self.states.set("is_codec2_traffic", True)
self.states.set_channel_busy_condition_codec2(True)
else:
self.states.set("is_codec2_traffic", False)
self.states.set_channel_busy_condition_codec2(False)
if rx_status == 10:
state_buffer.append(rx_status)

13
modem/event_manager.py
View file

@ -1,3 +1,4 @@
import base64
import json
import structlog
@ -28,18 +29,19 @@ class EventManager:
def send_custom_event(self, **event_data):
self.broadcast(event_data)
def send_arq_session_new(self, outbound: bool, session_id, dxcall, total_bytes):
def send_arq_session_new(self, outbound: bool, session_id, dxcall, total_bytes, state):
direction = 'outbound' if outbound else 'inbound'
event = {
f"arq-transfer-{direction}": {
'session_id': session_id,
'dxcall': dxcall,
'total_bytes': total_bytes,
'state': state,
}
}
self.broadcast(event)
def send_arq_session_progress(self, outbound: bool, session_id, dxcall, received_bytes, total_bytes):
def send_arq_session_progress(self, outbound: bool, session_id, dxcall, received_bytes, total_bytes, state):
direction = 'outbound' if outbound else 'inbound'
event = {
f"arq-transfer-{direction}": {
@ -47,11 +49,14 @@ class EventManager:
'dxcall': dxcall,
'received_bytes': received_bytes,
'total_bytes': total_bytes,
'state': state,
}
}
self.broadcast(event)
def send_arq_session_finished(self, outbound: bool, session_id, dxcall, total_bytes, success: bool):
def send_arq_session_finished(self, outbound: bool, session_id, dxcall, total_bytes, success: bool, state, data=False):
if data:
data = base64.b64encode(data).decode("UTF-8")
direction = 'outbound' if outbound else 'inbound'
event = {
f"arq-transfer-{direction}": {
@ -59,6 +64,8 @@ class EventManager:
'dxcall': dxcall,
'total_bytes': total_bytes,
'success': success,
'state': state,
'data': data
}
}
self.broadcast(event)

7
modem/frame_dispatcher.py
View file

@ -24,12 +24,11 @@ class DISPATCHER():
FR_TYPE.ARQ_CONNECTION_CLOSE.value: {"class": ARQFrameHandler, "name": "ARQ CLOSE SESSION"},
FR_TYPE.ARQ_CONNECTION_HB.value: {"class": ARQFrameHandler, "name": "ARQ HEARTBEAT"},
FR_TYPE.ARQ_CONNECTION_OPEN.value: {"class": ARQFrameHandler, "name": "ARQ OPEN SESSION"},
FR_TYPE.ARQ_STOP.value: {"class": ARQFrameHandler, "name": "ARQ STOP TX"},
FR_TYPE.ARQ_STOP.value: {"class": ARQFrameHandler, "name": "ARQ STOP"},
FR_TYPE.ARQ_STOP_ACK.value: {"class": ARQFrameHandler, "name": "ARQ STOP ACK"},
FR_TYPE.BEACON.value: {"class": FrameHandler, "name": "BEACON"},
FR_TYPE.ARQ_BURST_FRAME.value:{"class": ARQFrameHandler, "name": "BURST FRAME"},
FR_TYPE.ARQ_BURST_ACK.value: {"class": ARQFrameHandler, "name": "BURST ACK"},
FR_TYPE.ARQ_BURST_NACK.value: {"class": ARQFrameHandler, "name": "BURST NACK"},
FR_TYPE.ARQ_DATA_ACK_NACK.value: {"class": ARQFrameHandler, "name": "DATA ACK NACK"},
FR_TYPE.CQ.value: {"class": CQFrameHandler, "name": "CQ"},
FR_TYPE.PING_ACK.value: {"class": FrameHandler, "name": "PING ACK"},
FR_TYPE.PING.value: {"class": PingFrameHandler, "name": "PING"},
@ -88,7 +87,7 @@ class DISPATCHER():
# instantiate handler
handler_class = self.FRAME_HANDLER[frametype]['class']
handler = handler_class(self.FRAME_HANDLER[frametype]['name'],
handler: FrameHandler = handler_class(self.FRAME_HANDLER[frametype]['name'],
self.config,
self.states,
self.event_manager,

2
modem/frame_handler.py
View file

@ -89,7 +89,7 @@ class FrameHandler():
self.event_manager.broadcast(event_data)
def get_tx_mode(self):
return FREEDV_MODE.signalling.value
return FREEDV_MODE.signalling
def transmit(self, frame):
if not TESTMODE:

5
modem/frame_handler_arq_session.py
View file

@ -31,14 +31,15 @@ class ARQFrameHandler(frame_handler.FrameHandler):
elif frame['frame_type_int'] in [
FR.ARQ_SESSION_INFO.value,
FR.ARQ_BURST_FRAME.value,
]:
FR.ARQ_STOP.value,
]:
session = self.states.get_arq_irs_session(session_id)
elif frame['frame_type_int'] in [
FR.ARQ_SESSION_OPEN_ACK.value,
FR.ARQ_SESSION_INFO_ACK.value,
FR.ARQ_BURST_ACK.value,
FR.ARQ_DATA_ACK_NACK.value
FR.ARQ_STOP_ACK.value
]:
session = self.states.get_arq_iss_session(session_id)

BIN
modem/lib/codec2/libcodec2_Windows_i686-w64-mingw32.dll Normal file
View file

Binary file not shown.

BIN
modem/lib/codec2/libcodec2_Windows_x86_64-w64-mingw32.dll Normal file
View file

Binary file not shown.

BIN
modem/lib/codec2/libcodec2_bullseye_armv7.so Normal file
View file

Binary file not shown.

BIN
modem/lib/codec2/libcodec2_macos-12_native.dylib Normal file
View file

Binary file not shown.

BIN
modem/lib/codec2/libcodec2_macos-latest_native.dylib Normal file
View file

Binary file not shown.

BIN
modem/lib/codec2/libcodec2_ubuntu-20.04_native.so Normal file
View file

Binary file not shown.

BIN
modem/lib/codec2/libcodec2_ubuntu-22.04_native.so Normal file
View file

Binary file not shown.

BIN
modem/lib/codec2/libcodec2_ubuntu_latest_armv7.so Normal file
View file

Binary file not shown.

295
modem/modem.py
View file

@ -23,7 +23,6 @@ import cw
from queues import RIGCTLD_COMMAND_QUEUE
import audio
import event_manager
import beacon
import demodulator
TESTMODE = False
@ -71,25 +70,15 @@ class RF:
# 8 * (self.AUDIO_SAMPLE_RATE/self.MODEM_SAMPLE_RATE) == 48
self.AUDIO_CHANNELS = 1
self.MODE = 0
# Locking state for mod out so buffer will be filled before we can use it
# https://github.com/DJ2LS/FreeDATA/issues/127
# https://github.com/DJ2LS/FreeDATA/issues/99
self.mod_out_locked = True
self.rms_counter = 0
# Make sure our resampler will work
assert (self.AUDIO_SAMPLE_RATE / self.MODEM_SAMPLE_RATE) == codec2.api.FDMDV_OS_48 # type: ignore
self.modem_transmit_queue = queue.Queue()
self.modem_received_queue = queue.Queue()
self.audio_received_queue = queue.Queue()
self.data_queue_received = queue.Queue()
self.event_manager = event_manager.EventManager([event_queue])
self.fft_queue = fft_queue
self.demodulator = demodulator.Demodulator(self.config,
@ -101,8 +90,7 @@ class RF:
self.fft_queue
)
self.beacon = beacon.Beacon(self.config, self.states, event_queue,
self.log, self.modem_transmit_queue)
def tci_tx_callback(self, audio_48k) -> None:
self.radio.set_ptt(True)
@ -110,34 +98,23 @@ class RF:
self.tci_module.push_audio(audio_48k)
def start_modem(self):
# testmode: We need to call the modem without audio parts for running protocol tests
if self.radiocontrol not in ["tci"]:
result = self.init_audio() if not TESTMODE else True
if not result:
if TESTMODE:
return True
elif self.radiocontrol.lower() == "tci":
if not self.init_tci():
return False
else:
if not self.init_audio():
raise RuntimeError("Unable to init audio devices")
if not TESTMODE:
self.demodulator.start(self.sd_input_stream)
self.demodulator.start(self.sd_input_stream)
atexit.register(self.sd_input_stream.stop)
else:
result = self.init_tci()
# Initialize codec2, rig control, and data threads
self.init_codec2()
self.init_rig_control()
self.init_data_threads()
if result not in [False]:
# init codec2 instances
self.init_codec2()
# init rig control
self.init_rig_control()
# init data thread
self.init_data_threads()
if not TESTMODE:
atexit.register(self.sd_input_stream.stop)
# init beacon
self.beacon.start()
else:
return False
return True
def stop_modem(self):
try:
@ -148,8 +125,6 @@ class RF:
# self.stream.active = False
# self.stream.stop
self.beacon.stop()
except Exception:
self.log.error("[MDM] Error stopping modem")
@ -214,21 +189,6 @@ class RF:
# lets init TCI module
self.tci_module = tci.TCICtrl(self.audio_received_queue)
tci_rx_callback_thread = threading.Thread(
target=self.tci_rx_callback,
name="TCI RX CALLBACK THREAD",
daemon=True,
)
tci_rx_callback_thread.start()
# let's start the audio tx callback
self.log.debug("[MDM] Starting tci tx callback thread")
tci_tx_callback_thread = threading.Thread(
target=self.tci_tx_callback,
name="TCI TX CALLBACK THREAD",
daemon=True,
)
tci_tx_callback_thread.start()
return True
def audio_auto_tune(self):
@ -296,8 +256,8 @@ class RF:
# Wait for some other thread that might be transmitting
self.states.waitForTransmission()
self.states.setTransmitting(True)
# if we're transmitting FreeDATA signals, reset channel busy state
self.states.set("channel_busy", False)
#self.states.waitForChannelBusy()
start_of_transmission = time.time()
# TODO Moved ptt toggle some steps before audio is ready for testing
@ -310,38 +270,14 @@ class RF:
# Open codec2 instance
self.MODE = mode
# Get number of bytes per frame for mode
bytes_per_frame = int(codec2.api.freedv_get_bits_per_modem_frame(freedv) / 8)
payload_bytes_per_frame = bytes_per_frame - 2
# Init buffer for data
n_tx_modem_samples = codec2.api.freedv_get_n_tx_modem_samples(freedv)
mod_out = ctypes.create_string_buffer(n_tx_modem_samples * 2)
# Init buffer for preample
n_tx_preamble_modem_samples = codec2.api.freedv_get_n_tx_preamble_modem_samples(
freedv
)
mod_out_preamble = ctypes.create_string_buffer(n_tx_preamble_modem_samples * 2)
# Init buffer for postamble
n_tx_postamble_modem_samples = (
codec2.api.freedv_get_n_tx_postamble_modem_samples(freedv)
)
mod_out_postamble = ctypes.create_string_buffer(
n_tx_postamble_modem_samples * 2
)
txbuffer = bytes()
# Add empty data to handle ptt toggle time
if self.tx_delay > 0:
data_delay = int(self.MODEM_SAMPLE_RATE * (self.tx_delay / 1000)) # type: ignore
mod_out_silence = ctypes.create_string_buffer(data_delay * 2)
txbuffer = bytes(mod_out_silence)
else:
txbuffer = bytes()
self.transmit_add_silence(txbuffer, self.tx_delay)
self.log.debug(
"[MDM] TRANSMIT", mode=self.MODE, payload=payload_bytes_per_frame, delay=self.tx_delay
"[MDM] TRANSMIT", mode=self.MODE.name, delay=self.tx_delay
)
if not isinstance(frames, list): frames = [frames]
@ -350,43 +286,12 @@ class RF:
# Create modulation for all frames in the list
for frame in frames:
# Write preamble to txbuffer
codec2.api.freedv_rawdatapreambletx(freedv, mod_out_preamble)
txbuffer += bytes(mod_out_preamble)
# Create buffer for data
# Use this if CRC16 checksum is required (DATAc1-3)
buffer = bytearray(payload_bytes_per_frame)
# Set buffersize to length of data which will be send
buffer[: len(frame)] = frame # type: ignore
# Create crc for data frame -
# Use the crc function shipped with codec2
# to avoid CRC algorithm incompatibilities
# Generate CRC16
crc = ctypes.c_ushort(
codec2.api.freedv_gen_crc16(bytes(buffer), payload_bytes_per_frame)
)
# Convert crc to 2-byte (16-bit) hex string
crc = crc.value.to_bytes(2, byteorder="big")
# Append CRC to data buffer
buffer += crc
assert(bytes_per_frame == len(buffer))
data = (ctypes.c_ubyte * bytes_per_frame).from_buffer_copy(buffer)
# modulate DATA and save it into mod_out pointer
codec2.api.freedv_rawdatatx(freedv, mod_out, data)
txbuffer += bytes(mod_out)
# Write postamble to txbuffer
codec2.api.freedv_rawdatapostambletx(freedv, mod_out_postamble)
# Append postamble to txbuffer
txbuffer += bytes(mod_out_postamble)
txbuffer = self.transmit_add_preamble(txbuffer, freedv)
txbuffer = self.transmit_create_frame(txbuffer, freedv, frame)
txbuffer = self.transmit_add_postamble(txbuffer, freedv)
# Add delay to end of frames
samples_delay = int(self.MODEM_SAMPLE_RATE * (repeat_delay / 1000)) # type: ignore
mod_out_silence = ctypes.create_string_buffer(samples_delay * 2)
txbuffer += bytes(mod_out_silence)
self.transmit_add_silence(txbuffer, repeat_delay)
# Re-sample back up to 48k (resampler works on np.int16)
x = np.frombuffer(txbuffer, dtype=np.int16)
@ -394,55 +299,16 @@ class RF:
self.audio_auto_tune()
x = audio.set_audio_volume(x, self.tx_audio_level)
if not self.radiocontrol in ["tci"]:
if self.radiocontrol not in ["tci"]:
txbuffer_out = self.resampler.resample8_to_48(x)
else:
txbuffer_out = x
# Explicitly lock our usage of mod_out_queue if needed
# This could avoid audio problems on slower CPU
# we will fill our modout list with all data, then start
# processing it in audio callback
self.mod_out_locked = True
# -------------------------------
# add modulation to modout_queue
# transmit audio
self.transmit_audio(txbuffer_out)
# Release our mod_out_lock, so we can use the queue
self.mod_out_locked = False
# we need to wait manually for tci processing
if self.radiocontrol in ["tci"]:
duration = len(txbuffer_out) / 8000
timestamp_to_sleep = time.time() + duration
self.log.debug("[MDM] TCI calculated duration", duration=duration)
tci_timeout_reached = False
#while time.time() < timestamp_to_sleep:
# threading.Event().wait(0.01)
else:
timestamp_to_sleep = time.time()
# set tci timeout reached to True for overriding if not used
tci_timeout_reached = True
while not tci_timeout_reached:
if self.radiocontrol in ["tci"]:
if time.time() < timestamp_to_sleep:
tci_timeout_reached = False
else:
tci_timeout_reached = True
threading.Event().wait(0.01)
# if we're transmitting FreeDATA signals, reset channel busy state
self.states.set("channel_busy", False)
self.radio.set_ptt(False)
# Push ptt state to socket stream
self.event_manager.send_ptt_change(False)
# After processing, set the locking state back to true to be prepared for next transmission
self.mod_out_locked = True
self.states.setTransmitting(False)
end_of_transmission = time.time()
@ -450,11 +316,77 @@ class RF:
self.log.debug("[MDM] ON AIR TIME", time=transmission_time)
return True
def transmit_add_preamble(self, buffer, freedv):
# Init buffer for preample
n_tx_preamble_modem_samples = codec2.api.freedv_get_n_tx_preamble_modem_samples(
freedv
)
mod_out_preamble = ctypes.create_string_buffer(n_tx_preamble_modem_samples * 2)
# Write preamble to txbuffer
codec2.api.freedv_rawdatapreambletx(freedv, mod_out_preamble)
buffer += bytes(mod_out_preamble)
return buffer
def transmit_add_postamble(self, buffer, freedv):
# Init buffer for postamble
n_tx_postamble_modem_samples = (
codec2.api.freedv_get_n_tx_postamble_modem_samples(freedv)
)
mod_out_postamble = ctypes.create_string_buffer(
n_tx_postamble_modem_samples * 2
)
# Write postamble to txbuffer
codec2.api.freedv_rawdatapostambletx(freedv, mod_out_postamble)
# Append postamble to txbuffer
buffer += bytes(mod_out_postamble)
return buffer
def transmit_add_silence(self, buffer, duration):
data_delay = int(self.MODEM_SAMPLE_RATE * (duration / 1000)) # type: ignore
mod_out_silence = ctypes.create_string_buffer(data_delay * 2)
buffer += bytes(mod_out_silence)
return buffer
def transmit_create_frame(self, txbuffer, freedv, frame):
# Get number of bytes per frame for mode
bytes_per_frame = int(codec2.api.freedv_get_bits_per_modem_frame(freedv) / 8)
payload_bytes_per_frame = bytes_per_frame - 2
# Init buffer for data
n_tx_modem_samples = codec2.api.freedv_get_n_tx_modem_samples(freedv)
mod_out = ctypes.create_string_buffer(n_tx_modem_samples * 2)
# Create buffer for data
# Use this if CRC16 checksum is required (DATAc1-3)
buffer = bytearray(payload_bytes_per_frame)
# Set buffersize to length of data which will be send
buffer[: len(frame)] = frame # type: ignore
# Create crc for data frame -
# Use the crc function shipped with codec2
# to avoid CRC algorithm incompatibilities
# Generate CRC16
crc = ctypes.c_ushort(
codec2.api.freedv_gen_crc16(bytes(buffer), payload_bytes_per_frame)
)
# Convert crc to 2-byte (16-bit) hex string
crc = crc.value.to_bytes(2, byteorder="big")
# Append CRC to data buffer
buffer += crc
assert (bytes_per_frame == len(buffer))
data = (ctypes.c_ubyte * bytes_per_frame).from_buffer_copy(buffer)
# modulate DATA and save it into mod_out pointer
codec2.api.freedv_rawdatatx(freedv, mod_out, data)
txbuffer += bytes(mod_out)
return txbuffer
def transmit_morse(self, repeats, repeat_delay, frames):
self.states.waitForTransmission()
self.states.setTransmitting(True)
# if we're transmitting FreeDATA signals, reset channel busy state
self.states.set("channel_busy", False)
self.log.debug(
"[MDM] TRANSMIT", mode="MORSE"
)
@ -462,43 +394,10 @@ class RF:
txbuffer_out = cw.MorseCodePlayer().text_to_signal("DJ2LS-1")
self.mod_out_locked = True
self.transmit_audio(txbuffer_out)
self.mod_out_locked = False
# we need to wait manually for tci processing
if self.radiocontrol in ["tci"]:
duration = len(txbuffer_out) / 8000
timestamp_to_sleep = time.time() + duration
self.log.debug("[MDM] TCI calculated duration", duration=duration)
tci_timeout_reached = False
#while time.time() < timestamp_to_sleep:
# threading.Event().wait(0.01)
else:
timestamp_to_sleep = time.time()
# set tci timeout reached to True for overriding if not used
tci_timeout_reached = True
while not tci_timeout_reached:
if self.radiocontrol in ["tci"]:
if time.time() < timestamp_to_sleep:
tci_timeout_reached = False
else:
tci_timeout_reached = True
threading.Event().wait(0.01)
# if we're transmitting FreeDATA signals, reset channel busy state
self.states.set("channel_busy", False)
self.radio.set_ptt(False)
# Push ptt state to socket stream
self.event_manager.send_ptt_change(False)
# After processing, set the locking state back to true to be prepared for next transmission
self.mod_out_locked = True
self.modem_transmit_queue.task_done()
self.states.setTransmitting(False)
end_of_transmission = time.time()
@ -528,6 +427,8 @@ class RF:
if self.radiocontrol in ["tci"]:
self.tci_tx_callback(audio_48k)
# we need to wait manually for tci processing
self.tci_module.wait_until_transmitted(audio_48k)
else:
sd.play(audio_48k, blocking=True)
return
@ -547,9 +448,9 @@ class RF:
rigctld_ip=self.rigctld_ip,
rigctld_port=self.rigctld_port,
)
hamlib_thread = threading.Thread(
target=self.update_rig_data, name="HAMLIB_THREAD", daemon=True
)
hamlib_thread = threading.Thread(
target=self.update_rig_data, name="HAMLIB_THREAD", daemon=True
)
hamlib_thread.start()
hamlib_set_thread = threading.Thread(

11
modem/modem_frametypes.py
View file

@ -10,14 +10,13 @@ class FRAME_TYPE(Enum):
ARQ_CONNECTION_HB = 2
ARQ_CONNECTION_CLOSE = 3
ARQ_STOP = 10
ARQ_SESSION_OPEN = 11
ARQ_SESSION_OPEN_ACK = 12
ARQ_SESSION_INFO = 13
ARQ_SESSION_INFO_ACK = 14
ARQ_STOP_ACK = 11
ARQ_SESSION_OPEN = 12
ARQ_SESSION_OPEN_ACK = 13
ARQ_SESSION_INFO = 14
ARQ_SESSION_INFO_ACK = 15
ARQ_BURST_FRAME = 20
ARQ_BURST_ACK = 21
ARQ_BURST_NACK = 22
ARQ_DATA_ACK_NACK = 23
MESH_BROADCAST = 100
MESH_SIGNALLING_PING = 101
MESH_SIGNALLING_PING_ACK = 102

27
modem/server.py
View file

@ -143,7 +143,14 @@ def post_beacon():
api_abort(f"Incorrect value for 'enabled'. Shoud be bool.")
if not app.state_manager.is_modem_running:
api_abort('Modem not running', 503)
app.state_manager.set('is_beacon_running', request.json['enabled'])
if not app.state_manager.is_beacon_running:
app.state_manager.set('is_beacon_running', request.json['enabled'])
app.modem_service.put("start_beacon")
else:
app.state_manager.set('is_beacon_running', request.json['enabled'])
app.modem_service.put("stop_beacon")
return api_response(request.json)
@app.route('/modem/ping_ping', methods=['POST'])
@ -211,9 +218,23 @@ def post_modem_send_raw():
if not app.state_manager.is_modem_running:
api_abort('Modem not running', 503)
enqueue_tx_command(command_arq_raw.ARQRawCommand, request.json)
return api_response(request.json)
@app.route('/modem/stop_transmission', methods=['POST'])
def post_modem_send_raw_stop():
if request.method not in ['POST']:
return api_response({"info": "endpoint for SENDING a STOP command via POST"})
if not app.state_manager.is_modem_running:
api_abort('Modem not running', 503)
for id in app.state_manager.arq_irs_sessions:
app.state_manager.arq_irs_sessions[id].abort_transmission()
for id in app.state_manager.arq_iss_sessions:
app.state_manager.arq_iss_sessions[id].abort_transmission()
return api_response(request.json)
# server_commands.modem_arq_send_raw(request.json)
return "Not implemented yet"
# @app.route('/modem/arq_connect', methods=['POST'])
# @app.route('/modem/arq_disconnect', methods=['POST'])

17
modem/service_manager.py
View file

@ -5,6 +5,7 @@ import structlog
import audio
import ujson as json
import explorer
import beacon
class SM:
@ -12,6 +13,7 @@ class SM:
self.log = structlog.get_logger("service")
self.modem = False
self.beacon = False
self.data_handler = False
self.app = app
self.config = self.app.config_manager.read()
@ -48,6 +50,14 @@ class SM:
threading.Event().wait(0.5)
if self.start_modem():
self.modem_events.put(json.dumps({"freedata": "modem-event", "event": "restart"}))
elif cmd in ['start_beacon']:
self.start_beacon()
elif cmd in ['stop_beacon']:
self.stop_beacon()
else:
self.log.warning("[SVC] modem command processing failed", cmd=cmd, state=self.states.is_modem_running)
@ -98,3 +108,10 @@ class SM:
return audio_test
def start_beacon(self):
self.beacon = beacon.Beacon(self.config, self.states, self.modem_events, self.log, self.modem)
self.beacon.start()
def stop_beacon(self):
del self.beacon

49
modem/state_manager.py
View file

@ -13,9 +13,11 @@ class StateManager:
# modem related states
# not every state is needed to publish, yet
# TODO can we reduce them?
self.channel_busy = False
self.channel_busy_slot = [False, False, False, False, False]
self.is_codec2_traffic = False
self.channel_busy_event = threading.Event()
self.channel_busy_condition_traffic = threading.Event()
self.channel_busy_condition_codec2 = threading.Event()
self.is_modem_running = False
self.is_modem_busy = False
self.is_beacon_running = False
@ -36,16 +38,6 @@ class StateManager:
self.arq_iss_sessions = {}
self.arq_irs_sessions = {}
self.arq_session_state = 'disconnected'
self.arq_speed_level = 0
self.arq_total_bytes = 0
self.arq_bits_per_second = 0
self.arq_bytes_per_minute = 0
self.arq_transmission_percent = 0
self.arq_compression_factor = 0
self.arq_speed_list = []
self.arq_seconds_until_timeout = 0
self.mesh_routing_table = []
self.radio_frequency = 0
@ -88,8 +80,6 @@ class StateManager:
return {
"freedata-message": msgtype,
"channel_busy": self.channel_busy,
"is_codec2_traffic": self.is_codec2_traffic,
"is_modem_running": self.is_modem_running,
"is_beacon_running": self.is_beacon_running,
"radio_status": self.radio_status,
@ -114,6 +104,9 @@ class StateManager:
def waitForTransmission(self):
self.transmitting_event.wait()
def waitForChannelBusy(self):
self.channel_busy_event.wait(2)
def register_arq_iss_session(self, session):
if session.id in self.arq_iss_sessions:
raise RuntimeError(f"ARQ ISS Session '{session.id}' already exists!")
@ -126,12 +119,16 @@ class StateManager:
def get_arq_iss_session(self, id):
if id not in self.arq_iss_sessions:
raise RuntimeError(f"ARQ ISS Session '{id}' not found!")
#raise RuntimeError(f"ARQ ISS Session '{id}' not found!")
# DJ2LS: WIP We need to find a better way of handling this
pass
return self.arq_iss_sessions[id]
def get_arq_irs_session(self, id):
if id not in self.arq_irs_sessions:
raise RuntimeError(f"ARQ IRS Session '{id}' not found!")
#raise RuntimeError(f"ARQ IRS Session '{id}' not found!")
# DJ2LS: WIP We need to find a better way of handling this
pass
return self.arq_irs_sessions[id]
def remove_arq_iss_session(self, id):
@ -158,3 +155,23 @@ class StateManager:
self.activities_list[activity_id] = activity_data
self.sendStateUpdate()
def calculate_channel_busy_state(self):
if self.channel_busy_condition_traffic.is_set() and self.channel_busy_condition_codec2.is_set():
self.channel_busy_event.set()
else:
self.channel_busy_event = threading.Event()
def set_channel_busy_condition_traffic(self, busy):
if not busy:
self.channel_busy_condition_traffic.set()
else:
self.channel_busy_condition_traffic = threading.Event()
self.calculate_channel_busy_state()
def set_channel_busy_condition_codec2(self, traffic):
if not traffic:
self.channel_busy_condition_codec2.set()
else:
self.channel_busy_condition_codec2 = threading.Event()
self.calculate_channel_busy_state()

14
modem/tci.py
View file

@ -315,3 +315,17 @@ class TCICtrl:
def close_rig(self):
""" """
return
def wait_until_transmitted(self, txbuffer_out):
duration = len(txbuffer_out) / 8000
timestamp_to_sleep = time.time() + duration
self.log.debug("[MDM] TCI calculated duration", duration=duration)
tci_timeout_reached = False
while not tci_timeout_reached:
if self.radiocontrol in ["tci"]:
if time.time() < timestamp_to_sleep:
tci_timeout_reached = False
else:
tci_timeout_reached = True
threading.Event().wait(0.01)
# if we're transmitting FreeDATA signals, reset channel busy state

73
tests/test_arq_session.py
View file

@ -15,15 +15,37 @@ import random
import structlog
import numpy as np
from event_manager import EventManager
from data_frame_factory import DataFrameFactory
import codec2
class TestModem:
def __init__(self, event_q):
self.data_queue_received = queue.Queue()
self.demodulator = unittest.mock.Mock()
self.event_manager = EventManager([event_q])
self.logger = structlog.get_logger('Modem')
def getFrameTransmissionTime(self, mode):
samples = 0
c2instance = codec2.open_instance(mode.value)
samples += codec2.api.freedv_get_n_tx_preamble_modem_samples(c2instance)
samples += codec2.api.freedv_get_n_tx_modem_samples(c2instance)
samples += codec2.api.freedv_get_n_tx_postamble_modem_samples(c2instance)
time = samples / 8000
return time
def transmit(self, mode, repeats: int, repeat_delay: int, frames: bytearray) -> bool:
self.data_queue_received.put(frames)
# Simulate transmission time
tx_time = self.getFrameTransmissionTime(mode) + 0.1 # PTT
self.logger.info(f"TX {tx_time} seconds...")
threading.Event().wait(tx_time)
transmission = {
'mode': mode,
'bytes': frames,
}
self.data_queue_received.put(transmission)
class TestARQSession(unittest.TestCase):
@ -32,6 +54,7 @@ class TestARQSession(unittest.TestCase):
config_manager = CONFIG('modem/config.ini.example')
cls.config = config_manager.read()
cls.logger = structlog.get_logger("TESTS")
cls.frame_factory = DataFrameFactory(cls.config)
# ISS
cls.iss_state_manager = StateManager(queue.Queue())
@ -60,10 +83,12 @@ class TestARQSession(unittest.TestCase):
while self.channels_running:
# Transfer data between both parties
try:
frame_bytes = modem_transmit_queue.get(timeout=1)
transmission = modem_transmit_queue.get(timeout=1)
if random.randint(0, 100) < self.loss_probability:
self.logger.info(f"[{threading.current_thread().name}] Frame lost...")
continue
frame_bytes = transmission['bytes']
frame_dispatcher.new_process_data(frame_bytes, None, len(frame_bytes), 0, 0)
except queue.Empty:
continue
@ -73,7 +98,7 @@ class TestARQSession(unittest.TestCase):
key = 'arq-transfer-outbound' if outbound else 'arq-transfer-inbound'
while True:
ev = q.get()
if key in ev and 'success' in ev[key]:
if key in ev and ('success' in ev[key] or 'ABORTED' in ev[key]):
self.logger.info(f"[{threading.current_thread().name}] {key} session ended.")
break
@ -98,7 +123,7 @@ class TestARQSession(unittest.TestCase):
def testARQSessionSmallPayload(self):
# set Packet Error Rate (PER) / frame loss probability
self.loss_probability = 30
self.loss_probability = 50
self.establishChannels()
params = {
@ -109,9 +134,9 @@ class TestARQSession(unittest.TestCase):
cmd.run(self.iss_event_queue, self.iss_modem)
self.waitAndCloseChannels()
def testARQSessionLargePayload(self):
def DisabledtestARQSessionLargePayload(self):
# set Packet Error Rate (PER) / frame loss probability
self.loss_probability = 50
self.loss_probability = 0
self.establishChannels()
params = {
@ -123,5 +148,41 @@ class TestARQSession(unittest.TestCase):
self.waitAndCloseChannels()
def testARQSessionAbortTransmissionISS(self):
# set Packet Error Rate (PER) / frame loss probability
self.loss_probability = 0
self.establishChannels()
params = {
'dxcall': "DJ2LS-3",
'data': base64.b64encode(np.random.bytes(100)),
}
cmd = ARQRawCommand(self.config, self.iss_state_manager, self.iss_event_queue, params)
cmd.run(self.iss_event_queue, self.iss_modem)
threading.Event().wait(np.random.randint(1,10))
for id in self.iss_state_manager.arq_iss_sessions:
self.iss_state_manager.arq_iss_sessions[id].abort_transmission()
self.waitAndCloseChannels()
def testARQSessionAbortTransmissionIRS(self):
# set Packet Error Rate (PER) / frame loss probability
self.loss_probability = 0
self.establishChannels()
params = {
'dxcall': "DJ2LS-3",
'data': base64.b64encode(np.random.bytes(100)),
}
cmd = ARQRawCommand(self.config, self.iss_state_manager, self.iss_event_queue, params)
cmd.run(self.iss_event_queue, self.iss_modem)
threading.Event().wait(np.random.randint(1,10))
for id in self.irs_state_manager.arq_irs_sessions:
self.irs_state_manager.arq_irs_sessions[id].abort_transmission()
self.waitAndCloseChannels()
if __name__ == '__main__':
unittest.main()