Merge pull request 'Add background KISS connection, log frames to database' (#20) from add-kiss into main

Reviewed-on: #20

api_app.py

@@ -6,6 +6,7 @@ import csv
 import ast
 import glob
 import json
+import sqlite3
 api_app = Flask(__name__)
 api = Api(api_app)
 
@@ -42,16 +43,45 @@ def read_logs(log_folder):
 
     return(json_array)
 
+def dict_factory(cursor, row):
+    d = {}
+    for idx, col in enumerate(cursor.description):
+        d[col[0]] = row[idx]
+    return d
+
+def get_db_connection():
+    conn = sqlite3.connect('database.db')
+    #conn.row_factory = sqlite3.Row
+    conn.row_factory = dict_factory
+    return conn
+
+def select_all_frames(conn):
+    """
+    Query all rows in the frames table
+    :param conn: the Connection object
+    :return:
+    """
+    cur = conn.cursor()
+    cur.execute("SELECT * FROM frames")
+    rows = cur.fetchall()
+    return rows
+
 class Packets(Resource):
     def get(self):
-        data = read_logs(log_folder)
+        #data = read_logs(log_folder)
+        conn = get_db_connection()
+        data = select_all_frames(conn)
         return {'data': data}, 200  # return data and 200 OK code
 
 # Read config
 config = read_config()
 log_folder = config['Settings']['log_folder']
 # Load logs first (just to check for errors before page loads)
-data = read_logs(log_folder)
+#data = read_logs(log_folder)
+
+# Start subprocess to watch KISS connection
+import subprocess
+subprocess.Popen(["python3","kiss_and_db.py"])
 
 api.add_resource(Packets, '/packets')  # and '/locations' is our entry point for Locations
 

api_waitress.py

@@ -9,4 +9,4 @@ os.chdir(this_files_dir)
 
 # `url_prefix` is optional, but useful if you are serving app on a sub-dir
 # behind a reverse-proxy.
-serve(api_app, host='127.0.0.1', port=5001)
+serve(api_app, host='0.0.0.0', port=5001)

config.ini

@@ -7,3 +7,13 @@ log_folder = logs/
 station_call = W1CDN-1
 station_lat = 47.941500
 station_lon = -97.027000
+
+# How long to keep packets (frames) e.g., "2 days", "5 minutes"
+keep_time = "2 days"
+
+# KISS settings
+kiss_host = 192.168.0.30
+kiss_port = 8001
+
+# Development settings (not operational yet)
+mycall = W1CDN-15

init_db.py

@@ -0,0 +1,19 @@
+import sqlite3
+
+connection = sqlite3.connect('database.db')
+
+with open('schema.sql') as f:
+    connection.executescript(f.read())
+
+cur = connection.cursor()
+
+# cur.execute("INSERT INTO posts (title, content) VALUES (?, ?)",
+#             ('First Post', 'Content for the first post')
+#             )
+#
+# cur.execute("INSERT INTO posts (title, content) VALUES (?, ?)",
+#             ('Second Post', 'Content for the second post')
+#             )
+
+connection.commit()
+connection.close()

kiss_and_db.py

@@ -0,0 +1,125 @@
+#!/usr/bin/env python3
+import os
+import sqlite3
+import aprs
+import json
+import aprslib
+import configparser
+
+db_fields = ("id",
+"addresse",
+"alive",
+"altitude",
+"comment",
+"course",
+"created",
+"format",
+"frame",
+"from",
+"gpsfixstatus",
+"latitude",
+"longitude",
+"mbits",
+"messagecapable",
+"message_text",
+"mtype",
+"object_format",
+"object_name",
+"path",
+"phg",
+"phg_dir",
+"phg_gain",
+"phg_height",
+"phg_power",
+"phg_range",
+"posambiguity",
+"raw",
+"raw_timestamp",
+"speed",
+"station_call",
+"station_lat",
+"station_lon",
+"status",
+"subpacket",
+"symbol",
+"symbol_table",
+"telemetry",
+"timestamp",
+"to",
+"tEQNS",
+"tPARM",
+"tUNIT",
+"via",
+"weather",
+"wx_raw_timestamp")
+
+def read_config():
+    config = configparser.ConfigParser()
+    config.read('config.ini')
+    return config
+
+def get_db_connection():
+    conn = sqlite3.connect('database.db')
+    conn.row_factory = sqlite3.Row
+    return conn
+
+def main():
+
+    # Add the call and location of this station to the packet info
+    config = read_config()
+    # MYCALL = os.environ.get("MYCALL", "W1CDN")
+    # KISS_HOST = os.environ.get("KISS_HOST", "192.168.0.30")
+    # KISS_PORT = os.environ.get("KISS_PORT", "8001")
+
+
+    ki = aprs.TCPKISS(host=config['Settings']['kiss_host'], port=int(config['Settings']['kiss_port']))
+    ki.start()
+
+    # Make a simple frame and send it
+    frame = aprs.APRSFrame.ui(
+        destination="APZ001",
+        source=config['Settings']['mycall'],
+        path=["WIDE1-1"],
+        info=b">Hello World!",
+    )
+    ki.write(frame)
+
+    # Watch for new packets to come in
+    while True:
+        conn = get_db_connection()
+        for frame in ki.read(min_frames=1):
+            a = aprslib.parse(str(frame))
+            a['station_call'] = config['Settings']['station_call']
+            a['station_lat'] = config['Settings']['station_lat']
+            a['station_lon'] = config['Settings']['station_lon']
+            print(a)
+            # Make this a string and deal with it later (probably a mistake)
+            a['path'] = str(a['path'])
+            # Store true/false as 1/0
+            if 'alive' in a:
+                if a['alive'] == True:
+                    a['alive'] = 1
+                else:
+                    a['alive'] = 0
+            # Build an INSERT statement based on the fields we have from the frame
+            attrib_names = ', '.join('"%s"' % w for w in a.keys())
+            attrib_values = ", ".join("?" * len(a.keys()))
+            sql = "INSERT INTO frames ("+attrib_names+") VALUES ("+attrib_values+")"
+            try:
+                # Insert data
+                conn.execute(sql, list(a.values()))
+                conn.commit()
+
+                # TODO remove packets that are older ('created') than a limit set in config.ini
+                # "5 minutes" also works
+                conn.execute("DELETE FROM frames WHERE created < DATETIME('now', '"+config['Settings']['keep_time']+"')")
+                conn.commit()
+            except:
+                print("Error with SQLite!")
+
+        conn.close()
+
+
+
+if __name__ == "__main__":
+    main()

requirements.txt

@@ -1,4 +1,4 @@
 flask
 flask_restful
-pandas
+aprs
-numpy
+aprslib

schema.sql

@@ -0,0 +1,50 @@
+DROP TABLE IF EXISTS frames;
+
+CREATE TABLE frames (
+    id INTEGER PRIMARY KEY AUTOINCREMENT,
+    addresse TEXT,
+    alive INT,
+    altitude REAL,
+    comment TEXT,
+    course REAL,
+    created TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP,
+    format TEXT,
+    frame TEXT,
+    "from" TEXT,
+    gpsfixstatus TEXT,
+    latitude REAL,
+    longitude REAL,
+    mbits INT,
+    messagecapable INT,
+    message_text TEXT,
+    mtype TEXT,
+    object_format TEXT,
+    object_name TEXT,
+    path TEXT,
+    phg REAL,
+    phg_dir TEXT,
+    phg_gain REAL,
+    phg_height REAL,
+    phg_power REAL,
+    phg_range REAL,
+    posambiguity INT,
+    raw TEXT,
+    raw_timestamp TEXT,
+    speed REAL,
+    station_call TEXT,
+    station_lat REAL,
+    station_lon REAL,
+    status TEXT,
+    subpacket TEXT,
+    symbol TEXT,
+    symbol_table TEXT,
+    telemetry TEXT,
+    timestamp INT,
+    "to" TEXT,
+    tEQNS TEXT,
+    tPARM TEXT,
+    tUNIT TEXT,
+    via TEXT,
+    weather TEXT,
+    wx_raw_timestamp TIMESTAMP
+);

tcp_kiss_send_recv.py

@@ -0,0 +1,109 @@
+#!/usr/bin/env python3
+import os
+import sqlite3
+import aprs
+import json
+import aprslib
+import configparser
+
+MYCALL = os.environ.get("MYCALL", "W1CDN")
+KISS_HOST = os.environ.get("KISS_HOST", "192.168.0.30")
+KISS_PORT = os.environ.get("KISS_PORT", "8001")
+
+db_fields = ("id",
+"addresse",
+"alive",
+"altitude",
+"comment",
+"course",
+"created",
+"format",
+"frame",
+"from",
+"gpsfixstatus",
+"latitude",
+"longitude",
+"mbits",
+"messagecapable",
+"message_text",
+"mtype",
+"object_format",
+"object_name",
+"path",
+"posambiguity",
+"raw",
+"raw_timestamp",
+"speed",
+"station_call",
+"station_lat",
+"station_lon",
+"status",
+"symbol",
+"symbol_table",
+"telemetry",
+"timestamp",
+"to",
+"tEQNS",
+"tPARM",
+"tUNIT",
+"via",
+"weather",
+"wx_raw_timestamp")
+
+def read_config():
+    config = configparser.ConfigParser()
+    config.read('config.ini')
+    return config
+
+def get_db_connection():
+    conn = sqlite3.connect('database.db')
+    conn.row_factory = sqlite3.Row
+    return conn
+
+def main():
+
+    # Add the call and location of this station to the packet info
+    config = read_config()
+
+    ki = aprs.TCPKISS(host=KISS_HOST, port=int(KISS_PORT))
+    ki.start()
+
+    # Make a simple frame and send it
+    frame = aprs.APRSFrame.ui(
+        destination="APZ001",
+        source=MYCALL,
+        path=["WIDE1-1"],
+        info=b">Hello World!",
+    )
+    #ki.write(frame)
+
+    # Watch for new packets to come in
+    while True:
+        conn = get_db_connection()
+        for frame in ki.read(min_frames=1):
+            a = aprslib.parse(str(frame))
+            a['station_call'] = config['Settings']['station_call']
+            a['station_lat'] = config['Settings']['station_lat']
+            a['station_lon'] = config['Settings']['station_lon']
+            print(a)
+            # Make this a string and deal with it later (probably a mistake)
+            a['path'] = str(a['path'])
+            # Build an INSERT statement based on the fields we have from the frame
+            attrib_names = ', '.join(f'"{w}"' for w in a.keys())
+            attrib_values = ", ".join("?" * len(a.keys()))
+            sql = f"INSERT INTO frames ({attrib_names}) VALUES ({attrib_values})"
+            # Insert data
+            conn.execute(sql, list(a.values()))
+            conn.commit()
+
+            # TODO remove packets that are older ('created') than a limit set in config.ini
+            # "5 minutes" also works
+            conn.execute("DELETE FROM frames WHERE created < DATETIME('now', '"+config['Settings']['keep_time']+"')")
+            conn.commit()
+
+        conn.close()
+
+
+
+if __name__ == "__main__":
+    main()