From e2c3adf5c0a364e8a39c7ab265a3f067cb9b7520 Mon Sep 17 00:00:00 2001 From: W1CDN Date: Sat, 24 Jun 2023 11:17:53 -0500 Subject: [PATCH 01/36] Stub out index/status page. --- api_app.py | 19 ++++++++++++++++--- templates/index.html | 39 +++++++++++++++++++++++++++++++++++++++ 2 files changed, 55 insertions(+), 3 deletions(-) create mode 100644 templates/index.html diff --git a/api_app.py b/api_app.py index a1c2dc2..fa3284f 100644 --- a/api_app.py +++ b/api_app.py @@ -1,4 +1,4 @@ -from flask import Flask, request +from flask import Flask, request, render_template from flask_restful import Resource, Api, reqparse from datetime import date, timedelta import configparser @@ -6,6 +6,7 @@ import csv import ast import glob import json, operator +import requests import sqlite3 api_app = Flask(__name__) api = Api(api_app) @@ -89,6 +90,18 @@ def select_frames(conn, n, from_, url_params): rows = cur.fetchall() return rows +@api_app.route('/') +def index(): + + # Get list of recent packets using API + # TODO use relative path + response = json.loads(requests.get("https://digi.w1cdn.net/aprs_api/packets").text)['data'] + return render_template('index.html', + station_call = config['Settings']['station_call'], + station_lat = config['Settings']['station_lat'], + station_lon = config['Settings']['station_lon'], + d = response) + class Packets(Resource): def get(self): # Handle arguments that may or may not exist @@ -103,7 +116,7 @@ class Packets(Resource): data = select_frames(conn, n = n, from_ = from_, url_params = request.args.to_dict()) # Sort by created date, descending (https://stackoverflow.com/a/45266808) #data.sort(key=operator.itemgetter('created'), reverse=True) - return {'data': data}, 200 # return data and 200 OK code + return {data}, 200 # return data and 200 OK code # Read config config = read_config() @@ -113,7 +126,7 @@ log_folder = config['Settings']['log_folder'] import subprocess subprocess.Popen(["python3","kiss_and_db.py"]) -api.add_resource(Packets, '/packets') # and '/locations' is our entry point for Locations +api.add_resource(Packets, '/packets') if __name__ == '__main__': api_app.run(debug=True, host='0.0.0.0', port=5001) # run our Flask app diff --git a/templates/index.html b/templates/index.html new file mode 100644 index 0000000..a0eb753 --- /dev/null +++ b/templates/index.html @@ -0,0 +1,39 @@ + + + + + {{station_call}} Status + + + + +

{{station_call}} Status

+

{{station_lat}}, {{station_lon}}

+ +

Recent RF Packets

+ + + + + + + + {% for i in d %} + + + + + + {% endfor %} +
from object_name created (utc) more
{{ i['from'] }} {{ i['object_name'] }} {{ i['created'] }} more +
+ +

Recent Stations

+Coming soon, see https://amiok.net/gitea/W1CDN/aprs_tool/issues/16. + + + From 9deb16098991b9de76e5bf4b4d740998d30fbdba Mon Sep 17 00:00:00 2001 From: W1CDN Date: Sat, 24 Jun 2023 11:20:58 -0500 Subject: [PATCH 02/36] Add help link. --- templates/index.html | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/templates/index.html b/templates/index.html index a0eb753..67965fd 100644 --- a/templates/index.html +++ b/templates/index.html @@ -12,7 +12,7 @@

{{station_call}} Status

-

{{station_lat}}, {{station_lon}}

+{{station_lat}}, {{station_lon}}

Recent RF Packets

@@ -32,8 +32,10 @@ {% endfor %}
-

Recent Stations

+

Recent Stations

Coming soon, see https://amiok.net/gitea/W1CDN/aprs_tool/issues/16. +

Help

+This is a work in progress. See https://amiok.net/gitea/W1CDN/aprs_tool for usage. From 1ad8c848c459ca0a9e26aa186195e2fdc15c7b4e Mon Sep 17 00:00:00 2001 From: W1CDN Date: Sat, 24 Jun 2023 11:43:06 -0500 Subject: [PATCH 03/36] Fix bug on production. --- api_app.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/api_app.py b/api_app.py index fa3284f..de4f9f8 100644 --- a/api_app.py +++ b/api_app.py @@ -116,7 +116,7 @@ class Packets(Resource): data = select_frames(conn, n = n, from_ = from_, url_params = request.args.to_dict()) # Sort by created date, descending (https://stackoverflow.com/a/45266808) #data.sort(key=operator.itemgetter('created'), reverse=True) - return {data}, 200 # return data and 200 OK code + return {'data':data}, 200 # return data and 200 OK code # Read config config = read_config() From 1d8699df940a5727361133c5ada9b92ceac47172 Mon Sep 17 00:00:00 2001 From: W1CDN Date: Sat, 24 Jun 2023 18:44:08 -0500 Subject: [PATCH 04/36] Remove extra argument. --- api_app.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/api_app.py b/api_app.py index de4f9f8..548dbaa 100644 --- a/api_app.py +++ b/api_app.py @@ -69,7 +69,7 @@ def select_all_frames(conn): rows = cur.fetchall() return rows -def select_frames(conn, n, from_, url_params): +def select_frames(conn, n, url_params): # Should pass this a dict of fields and values (request.args) # TODO clean data before sending to DB @@ -109,11 +109,10 @@ class Packets(Resource): n = int(request.args.get('n')) except: n = 10 - from_ = None if request.args.get('from') == None else request.args.get('from') conn = get_db_connection() # Limit to number of records requested - data = select_frames(conn, n = n, from_ = from_, url_params = request.args.to_dict()) + data = select_frames(conn, n = n, url_params = request.args.to_dict()) # Sort by created date, descending (https://stackoverflow.com/a/45266808) #data.sort(key=operator.itemgetter('created'), reverse=True) return {'data':data}, 200 # return data and 200 OK code From f6a71e4851fd45c6f66a4797c349b32041b70653 Mon Sep 17 00:00:00 2001 From: W1CDN Date: Sat, 24 Jun 2023 18:44:18 -0500 Subject: [PATCH 05/36] Add raw column. --- templates/index.html | 2 ++ 1 file changed, 2 insertions(+) diff --git a/templates/index.html b/templates/index.html index 67965fd..ae8d3ac 100644 --- a/templates/index.html +++ b/templates/index.html @@ -19,6 +19,7 @@ from object_name + raw created (utc) more @@ -26,6 +27,7 @@ {{ i['from'] }} {{ i['object_name'] }} + {{ i['raw'] }} {{ i['created'] }} more From 1362558deba5142a514e961726759ada0a3e4491 Mon Sep 17 00:00:00 2001 From: W1CDN Date: Sat, 24 Jun 2023 19:06:37 -0500 Subject: [PATCH 06/36] Stub out station query. --- api_app.py | 20 +++++++++++++++++++- templates/index.html | 13 ++++++++++++- 2 files changed, 31 insertions(+), 2 deletions(-) diff --git a/api_app.py b/api_app.py index 548dbaa..4840fca 100644 --- a/api_app.py +++ b/api_app.py @@ -69,6 +69,17 @@ def select_all_frames(conn): rows = cur.fetchall() return rows +def unique_stations(conn): + """ + Query all rows in the frames table + :param conn: the Connection object + :return: + """ + cur = conn.cursor() + cur.execute('SELECT *, MAX(id) FROM frames GROUP BY "from" ORDER BY MAX(id) DESC') + rows = cur.fetchall() + return rows + def select_frames(conn, n, url_params): # Should pass this a dict of fields and values (request.args) @@ -96,11 +107,18 @@ def index(): # Get list of recent packets using API # TODO use relative path response = json.loads(requests.get("https://digi.w1cdn.net/aprs_api/packets").text)['data'] + + # Play with function to create station list + stations = unique_stations(get_db_connection()) + print(stations) + + return render_template('index.html', station_call = config['Settings']['station_call'], station_lat = config['Settings']['station_lat'], station_lon = config['Settings']['station_lon'], - d = response) + frames = response, + stations = stations) class Packets(Resource): def get(self): diff --git a/templates/index.html b/templates/index.html index ae8d3ac..5678d2e 100644 --- a/templates/index.html +++ b/templates/index.html @@ -23,7 +23,7 @@ created (utc) more - {% for i in d %} + {% for i in frames %} {{ i['from'] }} {{ i['object_name'] }} @@ -37,6 +37,17 @@

Recent Stations

Coming soon, see https://amiok.net/gitea/W1CDN/aprs_tool/issues/16. + + + + + {% for i in stations %} + + + + {% endfor %} +
raw
{{ i['raw'] }}
+

Help

This is a work in progress. See https://amiok.net/gitea/W1CDN/aprs_tool for usage. From 8972c8d4477828dc85a9aa32c450ebcc7583d2ee Mon Sep 17 00:00:00 2001 From: W1CDN Date: Sat, 24 Jun 2023 21:30:05 -0500 Subject: [PATCH 07/36] Snapshot. Get closer to new `stations` table. --- api_app.py | 2 +- tcp_kiss_send_recv.py | 5 +++++ templates/index.html | 8 ++++++-- 3 files changed, 12 insertions(+), 3 deletions(-) diff --git a/api_app.py b/api_app.py index 4840fca..bf55831 100644 --- a/api_app.py +++ b/api_app.py @@ -76,7 +76,7 @@ def unique_stations(conn): :return: """ cur = conn.cursor() - cur.execute('SELECT *, MAX(id) FROM frames GROUP BY "from" ORDER BY MAX(id) DESC') + cur.execute('SELECT *, MAX(id), COUNT(id) FROM frames GROUP BY "from" ORDER BY MAX(id) DESC') rows = cur.fetchall() return rows diff --git a/tcp_kiss_send_recv.py b/tcp_kiss_send_recv.py index 4e6fe5a..c168b8f 100644 --- a/tcp_kiss_send_recv.py +++ b/tcp_kiss_send_recv.py @@ -82,6 +82,8 @@ def main(): conn = get_db_connection() for frame in ki.read(min_frames=1): a = aprslib.parse(str(frame)) + # Add information about *this* station - might be useful for + # combining data across stations in the future. a['station_call'] = config['Settings']['station_call'] a['station_lat'] = config['Settings']['station_lat'] a['station_lon'] = config['Settings']['station_lon'] @@ -94,6 +96,9 @@ def main(): sql = f"INSERT INTO frames ({attrib_names}) VALUES ({attrib_values})" # Insert data conn.execute(sql, list(a.values())) + + # TODO update stations table here + conn.commit() # TODO remove packets that are older ('created') than a limit set in config.ini diff --git a/templates/index.html b/templates/index.html index 5678d2e..22bf7eb 100644 --- a/templates/index.html +++ b/templates/index.html @@ -39,11 +39,15 @@ Coming soon, see htt - + + + {% for i in stations %} - + + + {% endfor %}
raw from last heard (utc) count
{{ i['raw'] }} {{ i['from'] }} {{ i['created'] }} {{ i['COUNT(id)']}}
From fb4d89cd9b7f1471e164aa3725fc9fb3b2f3bf4d Mon Sep 17 00:00:00 2001 From: W1CDN Date: Tue, 27 Jun 2023 21:37:05 -0500 Subject: [PATCH 08/36] Remove old dev file. --- tcp_kiss_send_recv.py | 114 ------------------------------------------ 1 file changed, 114 deletions(-) delete mode 100644 tcp_kiss_send_recv.py diff --git a/tcp_kiss_send_recv.py b/tcp_kiss_send_recv.py deleted file mode 100644 index c168b8f..0000000 --- a/tcp_kiss_send_recv.py +++ /dev/null @@ -1,114 +0,0 @@ -#!/usr/bin/env python3 -import os -import sqlite3 -import aprs -import json -import aprslib -import configparser - -MYCALL = os.environ.get("MYCALL", "W1CDN") -KISS_HOST = os.environ.get("KISS_HOST", "192.168.0.30") -KISS_PORT = os.environ.get("KISS_PORT", "8001") - -db_fields = ("id", -"addresse", -"alive", -"altitude", -"comment", -"course", -"created", -"format", -"frame", -"from", -"gpsfixstatus", -"latitude", -"longitude", -"mbits", -"messagecapable", -"message_text", -"mtype", -"object_format", -"object_name", -"path", -"posambiguity", -"raw", -"raw_timestamp", -"speed", -"station_call", -"station_lat", -"station_lon", -"status", -"symbol", -"symbol_table", -"telemetry", -"timestamp", -"to", -"tEQNS", -"tPARM", -"tUNIT", -"via", -"weather", -"wx_raw_timestamp") - -def read_config(): - config = configparser.ConfigParser() - config.read('config.ini') - return config - -def get_db_connection(): - conn = sqlite3.connect('database.db') - conn.row_factory = sqlite3.Row - return conn - -def main(): - - # Add the call and location of this station to the packet info - config = read_config() - - ki = aprs.TCPKISS(host=KISS_HOST, port=int(KISS_PORT)) - ki.start() - - # Make a simple frame and send it - frame = aprs.APRSFrame.ui( - destination="APZ001", - source=MYCALL, - path=["WIDE1-1"], - info=b">Hello World!", - ) - #ki.write(frame) - - # Watch for new packets to come in - while True: - conn = get_db_connection() - for frame in ki.read(min_frames=1): - a = aprslib.parse(str(frame)) - # Add information about *this* station - might be useful for - # combining data across stations in the future. - a['station_call'] = config['Settings']['station_call'] - a['station_lat'] = config['Settings']['station_lat'] - a['station_lon'] = config['Settings']['station_lon'] - print(a) - # Make this a string and deal with it later (probably a mistake) - a['path'] = str(a['path']) - # Build an INSERT statement based on the fields we have from the frame - attrib_names = ', '.join(f'"{w}"' for w in a.keys()) - attrib_values = ", ".join("?" * len(a.keys())) - sql = f"INSERT INTO frames ({attrib_names}) VALUES ({attrib_values})" - # Insert data - conn.execute(sql, list(a.values())) - - # TODO update stations table here - - conn.commit() - - # TODO remove packets that are older ('created') than a limit set in config.ini - # "5 minutes" also works - conn.execute("DELETE FROM frames WHERE created < DATETIME('now', '"+config['Settings']['keep_time']+"')") - conn.commit() - - conn.close() - - - -if __name__ == "__main__": - main() From e05a3790d6fd4cc3bf26bdba3171552fa73966c0 Mon Sep 17 00:00:00 2001 From: W1CDN Date: Tue, 27 Jun 2023 21:43:14 -0500 Subject: [PATCH 09/36] Add links to aprs.fi. --- templates/index.html | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/templates/index.html b/templates/index.html index 22bf7eb..daf22ad 100644 --- a/templates/index.html +++ b/templates/index.html @@ -29,7 +29,8 @@ {{ i['object_name'] }} {{ i['raw'] }} {{ i['created'] }} - more + query, + aprs.fi {% endfor %} @@ -42,12 +43,14 @@ Coming soon, see htt from last heard (utc) count + more {% for i in stations %} {{ i['from'] }} {{ i['created'] }} {{ i['COUNT(id)']}} + aprs.fi {% endfor %} From 8f3b2ae7075cc5c2df22ed9e1293e2690e31f4d8 Mon Sep 17 00:00:00 2001 From: W1CDN Date: Tue, 27 Jun 2023 21:44:12 -0500 Subject: [PATCH 10/36] Don't autodelete yet. --- api_app.py | 2 -- kiss_and_db.py | 8 ++++++-- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/api_app.py b/api_app.py index bf55831..33a6d6e 100644 --- a/api_app.py +++ b/api_app.py @@ -110,8 +110,6 @@ def index(): # Play with function to create station list stations = unique_stations(get_db_connection()) - print(stations) - return render_template('index.html', station_call = config['Settings']['station_call'], diff --git a/kiss_and_db.py b/kiss_and_db.py index bfb5d9b..1bfae30 100644 --- a/kiss_and_db.py +++ b/kiss_and_db.py @@ -64,12 +64,16 @@ def main(): try: # Insert data conn.execute(sql, list(a.values())) + + # TODO update stations table here + + conn.commit() # TODO remove packets that are older ('created') than a limit set in config.ini # "5 minutes" also works - conn.execute("DELETE FROM frames WHERE created < DATETIME('now', '"+config['Settings']['keep_time']+"')") - conn.commit() + #conn.execute("DELETE FROM frames WHERE created < DATETIME('now', '"+config['Settings']['keep_time']+"')") + #conn.commit() except: print("Error with SQLite!") except: From 00ede8860f68662143e98fd7d05884e23041269a Mon Sep 17 00:00:00 2001 From: W1CDN Date: Fri, 7 Jul 2023 18:16:00 -0500 Subject: [PATCH 11/36] Add default config as example, gitignore real config. --- .gitignore | 2 ++ config_default.ini | 19 +++++++++++++++++++ 2 files changed, 21 insertions(+) create mode 100644 config_default.ini diff --git a/.gitignore b/.gitignore index f8018b8..8bfbf57 100644 --- a/.gitignore +++ b/.gitignore @@ -1 +1,3 @@ /logs/* +config.ini +*.db diff --git a/config_default.ini b/config_default.ini new file mode 100644 index 0000000..647b245 --- /dev/null +++ b/config_default.ini @@ -0,0 +1,19 @@ +[Settings] +# Path to direwolf log folder, include trailing slash +log_folder = logs/ +#log_folder = /home/pi/logs/direwolf/ + +# Name and location of this station, for inclusion in the API +station_call = W1CDN-1 +station_lat = 47.941500 +station_lon = -97.027000 + +# How long to keep packets (frames) e.g., "2 days", "5 minutes" +keep_time = "2 days" + +# KISS settings +kiss_host = 192.168.0.30 +kiss_port = 8001 + +# Development settings (not operational yet) +mycall = W1CDN-15 From e1cd7ccaaed30e2310f6a0bec696385dd79849cc Mon Sep 17 00:00:00 2001 From: W1CDN Date: Fri, 7 Jul 2023 18:17:06 -0500 Subject: [PATCH 12/36] Remove config.ini. --- config.ini | 19 ------------------- 1 file changed, 19 deletions(-) delete mode 100644 config.ini diff --git a/config.ini b/config.ini deleted file mode 100644 index 647b245..0000000 --- a/config.ini +++ /dev/null @@ -1,19 +0,0 @@ -[Settings] -# Path to direwolf log folder, include trailing slash -log_folder = logs/ -#log_folder = /home/pi/logs/direwolf/ - -# Name and location of this station, for inclusion in the API -station_call = W1CDN-1 -station_lat = 47.941500 -station_lon = -97.027000 - -# How long to keep packets (frames) e.g., "2 days", "5 minutes" -keep_time = "2 days" - -# KISS settings -kiss_host = 192.168.0.30 -kiss_port = 8001 - -# Development settings (not operational yet) -mycall = W1CDN-15 From c25a10ae77ee0fb3f5ad8bf2b2897ec2fb62f888 Mon Sep 17 00:00:00 2001 From: W1CDN Date: Fri, 7 Jul 2023 18:25:20 -0500 Subject: [PATCH 13/36] Fix path. --- start-aprs_api.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/start-aprs_api.sh b/start-aprs_api.sh index 4199c52..aa828a3 100755 --- a/start-aprs_api.sh +++ b/start-aprs_api.sh @@ -1,4 +1,4 @@ #!/bin/bash # Run `chmod +x start-aprs_api.sh` so this can be run -screen -dmS aprs_api python3 /home/pi/aprs_tools/api_waitress.py +screen -dmS aprs_api python3 /home/pi/aprs_tool/api_waitress.py From 1c057a5555d21b22971274648be613b83c962b9a Mon Sep 17 00:00:00 2001 From: W1CDN Date: Sat, 8 Jul 2023 21:54:37 -0500 Subject: [PATCH 14/36] Stub out db upsert code for stations table in test_db.py. See table definition at https://amiok.net/gitea/W1CDN/aprs_tool/pulls/30#issuecomment-947. --- test_db.py | 29 +++++++++++++++++++++++++++++ 1 file changed, 29 insertions(+) create mode 100644 test_db.py diff --git a/test_db.py b/test_db.py new file mode 100644 index 0000000..1dcdf8b --- /dev/null +++ b/test_db.py @@ -0,0 +1,29 @@ + +# Learn how to update database + +import sqlite3 + +def get_db_connection(): + conn = sqlite3.connect('database.db') + conn.row_factory = sqlite3.Row + return conn + +conn = get_db_connection() + +# example https://stackoverflow.com/a/50718957/2152245 +query1 = "INSERT INTO players (user_name, age) \ +VALUES('steven', 32) \ +ON CONFLICT(user_name) \ +DO UPDATE SET age=excluded.age;" + +query2 = "INSERT INTO stations (id, 'from', frames_id, last_heard_unix, count) \ +VALUES(1, 'KC9TZN-9', 4068, 1687623864, 1) \ +ON CONFLICT(id) \ +DO UPDATE SET count = count + 1;" + +query3 = "INSERT INTO stations ('from', frames_id, last_heard_unix, count) \ +VALUES('KC9TZN-9', 4068, 1687623864, 1)" + +conn.execute(query2) +conn.commit() +conn.close() From ee75cccc6896d22732d089756cc8ea633c33241a Mon Sep 17 00:00:00 2001 From: W1CDN Date: Sat, 8 Jul 2023 21:56:15 -0500 Subject: [PATCH 15/36] Add stations table definiton. --- schema.sql | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/schema.sql b/schema.sql index 4a64402..a4a1cb9 100644 --- a/schema.sql +++ b/schema.sql @@ -50,3 +50,12 @@ CREATE TABLE frames ( weather TEXT, wx_raw_timestamp TIMESTAMP ); + +CREATE TABLE "stations" ( + "id" INTEGER NOT NULL UNIQUE, + "from" TEXT UNIQUE, + "frames_id" INTEGER, + "last_heard_unix" INTEGER, + "count" INTEGER, + PRIMARY KEY("id" AUTOINCREMENT) +); From 8d94794c90557b2fb6ed607eeb2401e95bd51eac Mon Sep 17 00:00:00 2001 From: W1CDN Date: Sun, 9 Jul 2023 09:14:09 -0500 Subject: [PATCH 16/36] Snapshot after adding station table update code, but hasn't been tested on real frames yet. --- kiss_and_db.py | 13 ++++++++++++- test_db.py | 29 ++++++++++++++++++----------- 2 files changed, 30 insertions(+), 12 deletions(-) diff --git a/kiss_and_db.py b/kiss_and_db.py index 1bfae30..3cd25b1 100644 --- a/kiss_and_db.py +++ b/kiss_and_db.py @@ -66,7 +66,18 @@ def main(): conn.execute(sql, list(a.values())) # TODO update stations table here - + # Original intent was to include the id from the frames table, + # but that would mean making another conn.commit() (I think). + # It's not immediately needed, so I'm skipping it. + # Build query + # "from" is wrappedin [] because it is a reserved word and using '' doesn't work. + station_update = ", ".join((a['from'], a['created_unix'], "1")) + query3 = "INSERT INTO stations ([from], last_heard_unix, count) \ + VALUES("+station_update+", 1) \ + ON CONFLICT([from]) \ + DO UPDATE SET count = count + 1;" + # Insert/update data + conn.execute(query3) conn.commit() diff --git a/test_db.py b/test_db.py index 1dcdf8b..e0959e5 100644 --- a/test_db.py +++ b/test_db.py @@ -10,20 +10,27 @@ def get_db_connection(): conn = get_db_connection() -# example https://stackoverflow.com/a/50718957/2152245 -query1 = "INSERT INTO players (user_name, age) \ -VALUES('steven', 32) \ -ON CONFLICT(user_name) \ -DO UPDATE SET age=excluded.age;" +# Grab a random row from frames table and pretend it is new +cur = conn.cursor() +cur.execute("SELECT [from], id, created_unix FROM frames ORDER BY RANDOM() LIMIT 1;") +rows = cur.fetchall() +results = dict(rows[0]) +values = ', '.join('"%s"' % w for w in results.values()) -query2 = "INSERT INTO stations (id, 'from', frames_id, last_heard_unix, count) \ -VALUES(1, 'KC9TZN-9', 4068, 1687623864, 1) \ -ON CONFLICT(id) \ + +# Build query +# "from" is wrappedin [] because it is a reserved word and using '' doesn't work. +query3 = "INSERT INTO stations ([from], frames_id, last_heard_unix, count) \ +VALUES("+values+", 1) \ +ON CONFLICT([from]) \ DO UPDATE SET count = count + 1;" -query3 = "INSERT INTO stations ('from', frames_id, last_heard_unix, count) \ -VALUES('KC9TZN-9', 4068, 1687623864, 1)" +# example https://stackoverflow.com/a/50718957/2152245 +# query2 = "INSERT INTO stations ([from], frames_id, last_heard_unix, count) \ +# VALUES('KC9TZN-8', 4068, 1687623864, 1) \ +# ON CONFLICT([from]) \ +# DO UPDATE SET count = count + 1;" -conn.execute(query2) +conn.execute(query3) conn.commit() conn.close() From b0f0a4f8dcd25df2ccde9227462a7d1821ebd79f Mon Sep 17 00:00:00 2001 From: W1CDN Date: Sun, 9 Jul 2023 09:37:03 -0500 Subject: [PATCH 17/36] Try to combine all the processes to there aren't orphans. --- api_app.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/api_app.py b/api_app.py index 33a6d6e..bdb7504 100644 --- a/api_app.py +++ b/api_app.py @@ -139,7 +139,10 @@ log_folder = config['Settings']['log_folder'] # Start subprocess to watch KISS connection import subprocess -subprocess.Popen(["python3","kiss_and_db.py"]) +#proc = subprocess.Popen(["python3","kiss_and_db.py"]) +# Combine under one process https://stackoverflow.com/a/13143013/2152245 +proc = subprocess.Popen("exec " + "python3 kiss_and_db.py", stdout=subprocess.PIPE, shell=True) +print("kiss_and_db.py as subprocess pid "+str(proc.pid)) api.add_resource(Packets, '/packets') From c0ff61063f485e4ce118e1d3d7a7f73da448b69e Mon Sep 17 00:00:00 2001 From: W1CDN Date: Sun, 9 Jul 2023 10:15:54 -0500 Subject: [PATCH 18/36] Fix query for updating the stations table. I am still not good at knowing when to use quotes for values. --- kiss_and_db.py | 19 ++++++++++++------- 1 file changed, 12 insertions(+), 7 deletions(-) diff --git a/kiss_and_db.py b/kiss_and_db.py index 3cd25b1..d8e5320 100644 --- a/kiss_and_db.py +++ b/kiss_and_db.py @@ -61,6 +61,7 @@ def main(): attrib_names = ', '.join('"%s"' % w for w in a.keys()) attrib_values = ", ".join("?" * len(a.keys())) sql = "INSERT INTO frames ("+attrib_names+") VALUES ("+attrib_values+")" + try: # Insert data conn.execute(sql, list(a.values())) @@ -71,13 +72,17 @@ def main(): # It's not immediately needed, so I'm skipping it. # Build query # "from" is wrappedin [] because it is a reserved word and using '' doesn't work. - station_update = ", ".join((a['from'], a['created_unix'], "1")) - query3 = "INSERT INTO stations ([from], last_heard_unix, count) \ - VALUES("+station_update+", 1) \ - ON CONFLICT([from]) \ - DO UPDATE SET count = count + 1;" - # Insert/update data - conn.execute(query3) + try: + station_update = "'"+a['from'] +"', '"+ str(a['created_unix']) +"', '1'" + query3 = "INSERT INTO stations ([from], last_heard_unix, count) \ + VALUES("+station_update+") \ + ON CONFLICT([from]) \ + DO UPDATE SET count = count + 1;" + print(query3) + # Insert/update data + conn.execute(query3) + except: + print("Stations table couldn't be updated.") conn.commit() From 40513cc4883ea479e26e796eb6bc32d81ab69094 Mon Sep 17 00:00:00 2001 From: W1CDN Date: Sun, 9 Jul 2023 11:22:23 -0500 Subject: [PATCH 19/36] Use stations table on index.html. --- api_app.py | 13 ++++++++++++- kiss_and_db.py | 2 ++ templates/index.html | 10 ++++------ 3 files changed, 18 insertions(+), 7 deletions(-) diff --git a/api_app.py b/api_app.py index bdb7504..26d899f 100644 --- a/api_app.py +++ b/api_app.py @@ -69,6 +69,17 @@ def select_all_frames(conn): rows = cur.fetchall() return rows +def select_all_stations(conn): + """ + Query all rows in the stations table + :param conn: the Connection object + :return: + """ + cur = conn.cursor() + cur.execute("SELECT * FROM stations ORDER BY last_heard_unix DESC") + rows = cur.fetchall() + return rows + def unique_stations(conn): """ Query all rows in the frames table @@ -109,7 +120,7 @@ def index(): response = json.loads(requests.get("https://digi.w1cdn.net/aprs_api/packets").text)['data'] # Play with function to create station list - stations = unique_stations(get_db_connection()) + stations = select_all_stations(get_db_connection()) return render_template('index.html', station_call = config['Settings']['station_call'], diff --git a/kiss_and_db.py b/kiss_and_db.py index d8e5320..b85fb8b 100644 --- a/kiss_and_db.py +++ b/kiss_and_db.py @@ -29,6 +29,7 @@ def main(): ki = aprs.TCPKISS(host=config['Settings']['kiss_host'], port=int(config['Settings']['kiss_port'])) ki.start() + # Make a simple frame and send it frame = aprs.APRSFrame.ui( destination="APZ001", @@ -72,6 +73,7 @@ def main(): # It's not immediately needed, so I'm skipping it. # Build query # "from" is wrappedin [] because it is a reserved word and using '' doesn't work. + # https://www.sqlite.org/lang_keywords.html try: station_update = "'"+a['from'] +"', '"+ str(a['created_unix']) +"', '1'" query3 = "INSERT INTO stations ([from], last_heard_unix, count) \ diff --git a/templates/index.html b/templates/index.html index daf22ad..7ba1ae7 100644 --- a/templates/index.html +++ b/templates/index.html @@ -11,10 +11,10 @@ -

{{station_call}} Status

+

{{station_call}} Status

{{station_lat}}, {{station_lon}} -

Recent RF Packets

+

Recent RF Packets

@@ -36,8 +36,6 @@
from

Recent Stations

-Coming soon, see https://amiok.net/gitea/W1CDN/aprs_tool/issues/16. - @@ -48,8 +46,8 @@ Coming soon, see htt {% for i in stations %} - - + + {% endfor %} From d382a2b8f730061bbe0643a32630afe577d02145 Mon Sep 17 00:00:00 2001 From: W1CDN Date: Sun, 9 Jul 2023 22:06:57 -0500 Subject: [PATCH 20/36] Better station list. --- api_app.py | 28 ++++++++++++++++++++++++++-- templates/index.html | 2 +- 2 files changed, 27 insertions(+), 3 deletions(-) diff --git a/api_app.py b/api_app.py index 26d899f..b35ee05 100644 --- a/api_app.py +++ b/api_app.py @@ -1,8 +1,9 @@ from flask import Flask, request, render_template -from flask_restful import Resource, Api, reqparse +from flask_restful import Resource, Api, reqparse, url_for from datetime import date, timedelta import configparser import csv +import datetime import ast import glob import json, operator @@ -120,7 +121,12 @@ def index(): response = json.loads(requests.get("https://digi.w1cdn.net/aprs_api/packets").text)['data'] # Play with function to create station list - stations = select_all_stations(get_db_connection()) + #stations = select_all_stations(get_db_connection()) + #print(url_for("static", filename="test.txt", _external=True)) + stations = json.loads(requests.get(url_for("stations", _external=True)).text)['data'] + # Convert unix time to datetime on the fly because I'm lazy right now + for station in stations: + station['last_heard'] = datetime.datetime.fromtimestamp(station['last_heard_unix']) return render_template('index.html', station_call = config['Settings']['station_call'], @@ -144,6 +150,21 @@ class Packets(Resource): #data.sort(key=operator.itemgetter('created'), reverse=True) return {'data':data}, 200 # return data and 200 OK code +class Stations(Resource): + def get(self): + # Handle arguments that may or may not exist + try: + n = int(request.args.get('n')) + except: + n = 10 + + conn = get_db_connection() + # Limit to number of records requested + data = select_all_stations(conn) + # Sort by created date, descending (https://stackoverflow.com/a/45266808) + #data.sort(key=operator.itemgetter('created'), reverse=True) + return {'data':data}, 200 # return data and 200 OK code + # Read config config = read_config() log_folder = config['Settings']['log_folder'] @@ -155,7 +176,10 @@ import subprocess proc = subprocess.Popen("exec " + "python3 kiss_and_db.py", stdout=subprocess.PIPE, shell=True) print("kiss_and_db.py as subprocess pid "+str(proc.pid)) +# The packets endpoint api.add_resource(Packets, '/packets') +# The stations endpoint +api.add_resource(Stations, '/stations') if __name__ == '__main__': api_app.run(debug=True, host='0.0.0.0', port=5001) # run our Flask app diff --git a/templates/index.html b/templates/index.html index 7ba1ae7..9f3d900 100644 --- a/templates/index.html +++ b/templates/index.html @@ -46,7 +46,7 @@ {% for i in stations %} - + From 93156311e2441f36c90072b06ed794e90e0ccbaa Mon Sep 17 00:00:00 2001 From: W1CDN Date: Sun, 9 Jul 2023 22:15:03 -0500 Subject: [PATCH 21/36] Fix bug. --- api_app.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/api_app.py b/api_app.py index b35ee05..7f64f08 100644 --- a/api_app.py +++ b/api_app.py @@ -124,9 +124,11 @@ def index(): #stations = select_all_stations(get_db_connection()) #print(url_for("static", filename="test.txt", _external=True)) stations = json.loads(requests.get(url_for("stations", _external=True)).text)['data'] + #stations = json.loads(requests.get("https://digi.w1cdn.net/aprs_api/stations").text)['data'] # Convert unix time to datetime on the fly because I'm lazy right now for station in stations: - station['last_heard'] = datetime.datetime.fromtimestamp(station['last_heard_unix']) + if station['last_heard_unix'] != None: + station['last_heard'] = datetime.datetime.fromtimestamp(station['last_heard_unix']) return render_template('index.html', station_call = config['Settings']['station_call'], From d2cdaa820ab1d5779fb83e713ec62ded7a37ca27 Mon Sep 17 00:00:00 2001 From: W1CDN Date: Sun, 9 Jul 2023 22:17:50 -0500 Subject: [PATCH 22/36] Undo relative URL for now. --- api_app.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/api_app.py b/api_app.py index 7f64f08..bae32f5 100644 --- a/api_app.py +++ b/api_app.py @@ -123,8 +123,8 @@ def index(): # Play with function to create station list #stations = select_all_stations(get_db_connection()) #print(url_for("static", filename="test.txt", _external=True)) - stations = json.loads(requests.get(url_for("stations", _external=True)).text)['data'] - #stations = json.loads(requests.get("https://digi.w1cdn.net/aprs_api/stations").text)['data'] + # this should work: stations = json.loads(requests.get(url_for("stations", _external=True)).text)['data'] + stations = json.loads(requests.get("https://digi.w1cdn.net/aprs_api/stations").text)['data'] # Convert unix time to datetime on the fly because I'm lazy right now for station in stations: if station['last_heard_unix'] != None: From e3cb68551bcab839773732070cca028c53926cca Mon Sep 17 00:00:00 2001 From: Matt Date: Wed, 12 Jul 2023 09:34:39 -0500 Subject: [PATCH 23/36] Try to fix race condition in frames and stations tables. --- kiss_and_db.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/kiss_and_db.py b/kiss_and_db.py index b85fb8b..e9b9723 100644 --- a/kiss_and_db.py +++ b/kiss_and_db.py @@ -62,14 +62,15 @@ def main(): attrib_names = ', '.join('"%s"' % w for w in a.keys()) attrib_values = ", ".join("?" * len(a.keys())) sql = "INSERT INTO frames ("+attrib_names+") VALUES ("+attrib_values+")" - + conn.commit() + try: # Insert data conn.execute(sql, list(a.values())) # TODO update stations table here # Original intent was to include the id from the frames table, - # but that would mean making another conn.commit() (I think). + # but that would mean making another query. # It's not immediately needed, so I'm skipping it. # Build query # "from" is wrappedin [] because it is a reserved word and using '' doesn't work. @@ -83,11 +84,10 @@ def main(): print(query3) # Insert/update data conn.execute(query3) + conn.commit() except: print("Stations table couldn't be updated.") - conn.commit() - # TODO remove packets that are older ('created') than a limit set in config.ini # "5 minutes" also works #conn.execute("DELETE FROM frames WHERE created < DATETIME('now', '"+config['Settings']['keep_time']+"')") From 9c11d8d4949e4fe51f94fd403f4b43d20a2baa68 Mon Sep 17 00:00:00 2001 From: Matt Date: Wed, 12 Jul 2023 09:57:54 -0500 Subject: [PATCH 24/36] Try something else --- kiss_and_db.py | 29 +++++++++++++++-------------- 1 file changed, 15 insertions(+), 14 deletions(-) diff --git a/kiss_and_db.py b/kiss_and_db.py index e9b9723..3b63a78 100644 --- a/kiss_and_db.py +++ b/kiss_and_db.py @@ -61,11 +61,11 @@ def main(): # Build an INSERT statement based on the fields we have from the frame attrib_names = ', '.join('"%s"' % w for w in a.keys()) attrib_values = ", ".join("?" * len(a.keys())) - sql = "INSERT INTO frames ("+attrib_names+") VALUES ("+attrib_values+")" - conn.commit() + try: # Insert data + sql = "INSERT INTO frames ("+attrib_names+") VALUES ("+attrib_values+")" conn.execute(sql, list(a.values())) # TODO update stations table here @@ -75,18 +75,19 @@ def main(): # Build query # "from" is wrappedin [] because it is a reserved word and using '' doesn't work. # https://www.sqlite.org/lang_keywords.html - try: - station_update = "'"+a['from'] +"', '"+ str(a['created_unix']) +"', '1'" - query3 = "INSERT INTO stations ([from], last_heard_unix, count) \ - VALUES("+station_update+") \ - ON CONFLICT([from]) \ - DO UPDATE SET count = count + 1;" - print(query3) - # Insert/update data - conn.execute(query3) - conn.commit() - except: - print("Stations table couldn't be updated.") + #try: + station_update = "'"+a['from'] +"', '"+ str(a['created_unix']) +"', '1'" + query3 = "INSERT INTO stations ([from], last_heard_unix, count) \ + VALUES("+station_update+") \ + ON CONFLICT([from]) \ + DO UPDATE SET count = count + 1;" + print(query3) + # Insert/update data + conn.execute(query3) + + conn.commit() + #except: + # print("Stations table couldn't be updated.") # TODO remove packets that are older ('created') than a limit set in config.ini # "5 minutes" also works From ebd237d9d36dac2ffb83a30c29964a272e8e30d2 Mon Sep 17 00:00:00 2001 From: W1CDN Date: Wed, 12 Jul 2023 12:34:50 -0500 Subject: [PATCH 25/36] Get UTC time from stations table. --- api_app.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/api_app.py b/api_app.py index bae32f5..d0b0a2c 100644 --- a/api_app.py +++ b/api_app.py @@ -128,7 +128,7 @@ def index(): # Convert unix time to datetime on the fly because I'm lazy right now for station in stations: if station['last_heard_unix'] != None: - station['last_heard'] = datetime.datetime.fromtimestamp(station['last_heard_unix']) + station['last_heard'] = datetime.datetime.utcfromtimestamp(station['last_heard_unix']) return render_template('index.html', station_call = config['Settings']['station_call'], From 1a5df46ecaa7d888ad021b47e037e5dfe47c7e6d Mon Sep 17 00:00:00 2001 From: W1CDN Date: Wed, 12 Jul 2023 12:43:24 -0500 Subject: [PATCH 26/36] Add basic logging. --- .gitignore | 1 + config_default.ini | 1 + kiss_and_db.py | 19 +++++++++++++------ 3 files changed, 15 insertions(+), 6 deletions(-) diff --git a/.gitignore b/.gitignore index 8bfbf57..ab303c9 100644 --- a/.gitignore +++ b/.gitignore @@ -1,3 +1,4 @@ /logs/* config.ini *.db +*.log diff --git a/config_default.ini b/config_default.ini index 647b245..db3bf6f 100644 --- a/config_default.ini +++ b/config_default.ini @@ -17,3 +17,4 @@ kiss_port = 8001 # Development settings (not operational yet) mycall = W1CDN-15 +log_path = aprs_api.log diff --git a/kiss_and_db.py b/kiss_and_db.py index 3b63a78..7174038 100644 --- a/kiss_and_db.py +++ b/kiss_and_db.py @@ -6,6 +6,7 @@ import json import aprslib import configparser import time +import logging def read_config(): config = configparser.ConfigParser() @@ -25,6 +26,8 @@ def main(): # KISS_HOST = os.environ.get("KISS_HOST", "192.168.0.30") # KISS_PORT = os.environ.get("KISS_PORT", "8001") + logging.basicConfig(filename=config['Settings']['log_path'], encoding='utf-8', level=logging.DEBUG) + logging.debug('kiss_and_db.py running') ki = aprs.TCPKISS(host=config['Settings']['kiss_host'], port=int(config['Settings']['kiss_port'])) ki.start() @@ -61,11 +64,12 @@ def main(): # Build an INSERT statement based on the fields we have from the frame attrib_names = ', '.join('"%s"' % w for w in a.keys()) attrib_values = ", ".join("?" * len(a.keys())) - - + + try: # Insert data sql = "INSERT INTO frames ("+attrib_names+") VALUES ("+attrib_values+")" + logging.debug(sql) conn.execute(sql, list(a.values())) # TODO update stations table here @@ -81,10 +85,11 @@ def main(): VALUES("+station_update+") \ ON CONFLICT([from]) \ DO UPDATE SET count = count + 1;" - print(query3) + #print(query3) + logging.debug(query3) # Insert/update data conn.execute(query3) - + conn.commit() #except: # print("Stations table couldn't be updated.") @@ -94,9 +99,11 @@ def main(): #conn.execute("DELETE FROM frames WHERE created < DATETIME('now', '"+config['Settings']['keep_time']+"')") #conn.commit() except: - print("Error with SQLite!") + #print("Error with SQLite!") + logging.error("Error with SQLite!") except: - print("Frame could not be parsed.") + #print("Frame could not be parsed.") + logging.error("Frame could not be parsed.") conn.close() From 875546040fedd1d7c0d02823668b63d8a6ffa17c Mon Sep 17 00:00:00 2001 From: W1CDN Date: Wed, 12 Jul 2023 12:47:29 -0500 Subject: [PATCH 27/36] Logging encoding. --- kiss_and_db.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/kiss_and_db.py b/kiss_and_db.py index 7174038..81c43f7 100644 --- a/kiss_and_db.py +++ b/kiss_and_db.py @@ -26,7 +26,7 @@ def main(): # KISS_HOST = os.environ.get("KISS_HOST", "192.168.0.30") # KISS_PORT = os.environ.get("KISS_PORT", "8001") - logging.basicConfig(filename=config['Settings']['log_path'], encoding='utf-8', level=logging.DEBUG) + logging.basicConfig(filename=config['Settings']['log_path'], level=logging.DEBUG) logging.debug('kiss_and_db.py running') ki = aprs.TCPKISS(host=config['Settings']['kiss_host'], port=int(config['Settings']['kiss_port'])) From f447a807b255b6e4983c4b24bffd75542a10b104 Mon Sep 17 00:00:00 2001 From: W1CDN Date: Wed, 12 Jul 2023 13:36:27 -0500 Subject: [PATCH 28/36] Gotta update the datestamp too. --- kiss_and_db.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/kiss_and_db.py b/kiss_and_db.py index 81c43f7..4999e99 100644 --- a/kiss_and_db.py +++ b/kiss_and_db.py @@ -84,7 +84,8 @@ def main(): query3 = "INSERT INTO stations ([from], last_heard_unix, count) \ VALUES("+station_update+") \ ON CONFLICT([from]) \ - DO UPDATE SET count = count + 1;" + DO UPDATE SET count = count + 1,\ + last_heard_unix = excluded.last_heard_unix;" #print(query3) logging.debug(query3) # Insert/update data From 9fb3d28cdc84317cf2d697c8806c9d307013e13c Mon Sep 17 00:00:00 2001 From: W1CDN Date: Wed, 12 Jul 2023 14:48:26 -0500 Subject: [PATCH 29/36] API calls should use relative paths. --- api_app.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/api_app.py b/api_app.py index d0b0a2c..6e45558 100644 --- a/api_app.py +++ b/api_app.py @@ -118,13 +118,13 @@ def index(): # Get list of recent packets using API # TODO use relative path - response = json.loads(requests.get("https://digi.w1cdn.net/aprs_api/packets").text)['data'] + response = json.loads(requests.get(url_for("packets", _external=True)).text)['data'] # Play with function to create station list #stations = select_all_stations(get_db_connection()) #print(url_for("static", filename="test.txt", _external=True)) # this should work: stations = json.loads(requests.get(url_for("stations", _external=True)).text)['data'] - stations = json.loads(requests.get("https://digi.w1cdn.net/aprs_api/stations").text)['data'] + stations = json.loads(requests.get(url_for("stations", _external=True)).text)['data'] # Convert unix time to datetime on the fly because I'm lazy right now for station in stations: if station['last_heard_unix'] != None: From ef539e2aa94c489e3a23e14bab39ab74938ab575 Mon Sep 17 00:00:00 2001 From: W1CDN Date: Wed, 12 Jul 2023 14:54:43 -0500 Subject: [PATCH 30/36] Revert relative paths, which don't work on production. --- api_app.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/api_app.py b/api_app.py index 6e45558..d0b0a2c 100644 --- a/api_app.py +++ b/api_app.py @@ -118,13 +118,13 @@ def index(): # Get list of recent packets using API # TODO use relative path - response = json.loads(requests.get(url_for("packets", _external=True)).text)['data'] + response = json.loads(requests.get("https://digi.w1cdn.net/aprs_api/packets").text)['data'] # Play with function to create station list #stations = select_all_stations(get_db_connection()) #print(url_for("static", filename="test.txt", _external=True)) # this should work: stations = json.loads(requests.get(url_for("stations", _external=True)).text)['data'] - stations = json.loads(requests.get(url_for("stations", _external=True)).text)['data'] + stations = json.loads(requests.get("https://digi.w1cdn.net/aprs_api/stations").text)['data'] # Convert unix time to datetime on the fly because I'm lazy right now for station in stations: if station['last_heard_unix'] != None: From fcd776174ca2da2cd1580f712c20692df2e0c041 Mon Sep 17 00:00:00 2001 From: W1CDN Date: Fri, 25 Aug 2023 21:38:17 -0500 Subject: [PATCH 31/36] Log with timestamp. --- kiss_and_db.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/kiss_and_db.py b/kiss_and_db.py index 4999e99..f463b23 100644 --- a/kiss_and_db.py +++ b/kiss_and_db.py @@ -26,7 +26,8 @@ def main(): # KISS_HOST = os.environ.get("KISS_HOST", "192.168.0.30") # KISS_PORT = os.environ.get("KISS_PORT", "8001") - logging.basicConfig(filename=config['Settings']['log_path'], level=logging.DEBUG) + logging.basicConfig(filename=config['Settings']['log_path'], level=logging.DEBUG, \ + format='%(asctime)s - %(message)s') logging.debug('kiss_and_db.py running') ki = aprs.TCPKISS(host=config['Settings']['kiss_host'], port=int(config['Settings']['kiss_port'])) From acdee84d3ebad6f26a503ea89e9caa373bdcb6bb Mon Sep 17 00:00:00 2001 From: W1CDN Date: Sat, 26 Aug 2023 15:26:44 -0500 Subject: [PATCH 32/36] Update requirements. --- requirements.txt | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 13524b3..567e8db 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,6 +1,8 @@ flask flask_restful -aprs +aprs3 +kiss3 +kiss aprslib sqlite3 json From 50e8324786737b44332b614bfda18b11bb0084c5 Mon Sep 17 00:00:00 2001 From: W1CDN Date: Sat, 26 Aug 2023 16:05:09 -0500 Subject: [PATCH 33/36] Show relative time. --- api_app.py | 10 ++++++++-- templates/index.html | 4 ++++ 2 files changed, 12 insertions(+), 2 deletions(-) diff --git a/api_app.py b/api_app.py index d0b0a2c..e7d0fd6 100644 --- a/api_app.py +++ b/api_app.py @@ -4,6 +4,7 @@ from datetime import date, timedelta import configparser import csv import datetime +import timeago import ast import glob import json, operator @@ -118,7 +119,11 @@ def index(): # Get list of recent packets using API # TODO use relative path - response = json.loads(requests.get("https://digi.w1cdn.net/aprs_api/packets").text)['data'] + frames = json.loads(requests.get("https://digi.w1cdn.net/aprs_api/packets").text)['data'] + for frame in frames: + if frame['created'] != None: + frame['time_ago'] = timeago.format(frame['created_unix'], datetime.datetime.now()) + # Play with function to create station list #stations = select_all_stations(get_db_connection()) @@ -129,12 +134,13 @@ def index(): for station in stations: if station['last_heard_unix'] != None: station['last_heard'] = datetime.datetime.utcfromtimestamp(station['last_heard_unix']) + station['time_ago'] = timeago.format(station['last_heard_unix'], datetime.datetime.now()) return render_template('index.html', station_call = config['Settings']['station_call'], station_lat = config['Settings']['station_lat'], station_lon = config['Settings']['station_lon'], - frames = response, + frames = frames, stations = stations) class Packets(Resource): diff --git a/templates/index.html b/templates/index.html index 9f3d900..6cfa01b 100644 --- a/templates/index.html +++ b/templates/index.html @@ -21,6 +21,7 @@ + {% for i in frames %} @@ -29,6 +30,7 @@ + @@ -40,6 +42,7 @@ + @@ -47,6 +50,7 @@ + From 78641d0eef70238b849d24b85012ffdc9e682c24 Mon Sep 17 00:00:00 2001 From: W1CDN Date: Sat, 26 Aug 2023 16:23:43 -0500 Subject: [PATCH 34/36] Drop function for reading dw logs directly. --- api_app.py | 29 +---------------------------- config_default.ini | 4 ---- 2 files changed, 1 insertion(+), 32 deletions(-) diff --git a/api_app.py b/api_app.py index e7d0fd6..8329603 100644 --- a/api_app.py +++ b/api_app.py @@ -21,34 +21,6 @@ def read_config(): config.read('config.ini') return config -def read_logs(log_folder): - # Read some log files - # UTC time, so let's look at tomorrow, today, and yesterday. - today = date.today() - yesterday = today - timedelta(days = 1) - tomorrow = today + timedelta(days = 1) - file_list = glob.glob(log_folder+str(yesterday)+"*") + \ - glob.glob(log_folder+str(today)+"*") + \ - glob.glob(log_folder+str(tomorrow)+"*") - - # https://stackoverflow.com/a/66071962 - json_array = [] - for file in file_list: - with open(file, encoding='utf-8') as csvf: - csvReader = csv.DictReader(csvf) - for row in csvReader: - #add this python dict to json array - json_array.append(row) - - # Add the call and location of this station to the packet info - config = read_config() - for item in json_array: - item['station_name'] = config['Settings']['station_call'] - item['station_lat'] = config['Settings']['station_lat'] - item['station_lon'] = config['Settings']['station_lon'] - - return(json_array) - def dict_factory(cursor, row): d = {} for idx, col in enumerate(cursor.description): @@ -136,6 +108,7 @@ def index(): station['last_heard'] = datetime.datetime.utcfromtimestamp(station['last_heard_unix']) station['time_ago'] = timeago.format(station['last_heard_unix'], datetime.datetime.now()) + return render_template('index.html', station_call = config['Settings']['station_call'], station_lat = config['Settings']['station_lat'], diff --git a/config_default.ini b/config_default.ini index db3bf6f..77cba3d 100644 --- a/config_default.ini +++ b/config_default.ini @@ -1,8 +1,4 @@ [Settings] -# Path to direwolf log folder, include trailing slash -log_folder = logs/ -#log_folder = /home/pi/logs/direwolf/ - # Name and location of this station, for inclusion in the API station_call = W1CDN-1 station_lat = 47.941500 From f694e65c2a7abdce44b9bca7363660d5ee9fc824 Mon Sep 17 00:00:00 2001 From: W1CDN Date: Sat, 26 Aug 2023 16:23:55 -0500 Subject: [PATCH 35/36] Clean up a bit. --- kiss_and_db.py | 7 ++----- templates/index.html | 7 ++++--- 2 files changed, 6 insertions(+), 8 deletions(-) diff --git a/kiss_and_db.py b/kiss_and_db.py index f463b23..c799a70 100644 --- a/kiss_and_db.py +++ b/kiss_and_db.py @@ -70,9 +70,8 @@ def main(): try: # Insert data sql = "INSERT INTO frames ("+attrib_names+") VALUES ("+attrib_values+")" - logging.debug(sql) conn.execute(sql, list(a.values())) - + logging.debug("Frames table updated") # TODO update stations table here # Original intent was to include the id from the frames table, # but that would mean making another query. @@ -87,11 +86,9 @@ def main(): ON CONFLICT([from]) \ DO UPDATE SET count = count + 1,\ last_heard_unix = excluded.last_heard_unix;" - #print(query3) - logging.debug(query3) # Insert/update data conn.execute(query3) - + logging.debug("Station table updated") conn.commit() #except: # print("Stations table couldn't be updated.") diff --git a/templates/index.html b/templates/index.html index 6cfa01b..3b864fd 100644 --- a/templates/index.html +++ b/templates/index.html @@ -12,7 +12,10 @@

{{station_call}} Status

-{{station_lat}}, {{station_lon}} +Station location: {{station_lat}}, {{station_lon}} + +

About

+This is a work in progress. See https://amiok.net/gitea/W1CDN/aprs_tool for usage.

Recent RF Packets

from
{{ i['from'] }} {{ i['created'] }} {{ i['COUNT(id)']}} {{ i['last_heard_unix'] }} {{ i['count']}} aprs.fi
{{ i['from'] }} {{ i['last_heard_unix'] }} {{ i['last_heard'] }} {{ i['count']}} aprs.fi
object_name raw created (utc) relative more
{{ i['object_name'] }} {{ i['raw'] }} {{ i['created'] }} {{ i['time_ago'] }} query, aprs.fi
from last heard (utc) relative count more
{{ i['from'] }} {{ i['last_heard'] }} {{ i['time_ago'] }} {{ i['count']}} aprs.fi
@@ -57,7 +60,5 @@ {% endfor %}
-

Help

-This is a work in progress. See https://amiok.net/gitea/W1CDN/aprs_tool for usage. From 1b0494c45a5b8ac8bbea621f14688c95ecadf8ee Mon Sep 17 00:00:00 2001 From: W1CDN Date: Sat, 26 Aug 2023 16:47:23 -0500 Subject: [PATCH 36/36] Very rough workaround for relative API urls. --- api_app.py | 9 ++++++--- config_default.ini | 3 +++ requirements.txt | 1 + 3 files changed, 10 insertions(+), 3 deletions(-) diff --git a/api_app.py b/api_app.py index 8329603..f9adff1 100644 --- a/api_app.py +++ b/api_app.py @@ -91,7 +91,9 @@ def index(): # Get list of recent packets using API # TODO use relative path - frames = json.loads(requests.get("https://digi.w1cdn.net/aprs_api/packets").text)['data'] + #frames = json.loads(requests.get(url_for("packets", _external=True)).text)['data'] + #frames = json.loads(requests.get("https://digi.w1cdn.net/aprs_api/packets").text)['data'] + frames = json.loads(requests.get(config['Settings']['base_url']+"/packets").text)['data'] for frame in frames: if frame['created'] != None: frame['time_ago'] = timeago.format(frame['created_unix'], datetime.datetime.now()) @@ -101,7 +103,9 @@ def index(): #stations = select_all_stations(get_db_connection()) #print(url_for("static", filename="test.txt", _external=True)) # this should work: stations = json.loads(requests.get(url_for("stations", _external=True)).text)['data'] - stations = json.loads(requests.get("https://digi.w1cdn.net/aprs_api/stations").text)['data'] + #stations = json.loads(requests.get(url_for("stations", _external=True)).text)['data'] + #stations = json.loads(requests.get("https://digi.w1cdn.net/aprs_api/stations").text)['data'] + stations = json.loads(requests.get(config['Settings']['base_url']+"/stations").text)['data'] # Convert unix time to datetime on the fly because I'm lazy right now for station in stations: if station['last_heard_unix'] != None: @@ -148,7 +152,6 @@ class Stations(Resource): # Read config config = read_config() -log_folder = config['Settings']['log_folder'] # Start subprocess to watch KISS connection import subprocess diff --git a/config_default.ini b/config_default.ini index 77cba3d..b729dce 100644 --- a/config_default.ini +++ b/config_default.ini @@ -4,6 +4,9 @@ station_call = W1CDN-1 station_lat = 47.941500 station_lon = -97.027000 +# Base URL for application (no trailing slash) +base_url = https://digi.w1cdn.net/aprs_api + # How long to keep packets (frames) e.g., "2 days", "5 minutes" keep_time = "2 days" diff --git a/requirements.txt b/requirements.txt index 567e8db..4ef0f1d 100644 --- a/requirements.txt +++ b/requirements.txt @@ -6,3 +6,4 @@ kiss aprslib sqlite3 json +timeago \ No newline at end of file