From e1211fe1081b24eef9cef62117faa4f26c5b812b Mon Sep 17 00:00:00 2001 From: W1CDN Date: Sat, 26 Aug 2023 19:24:00 -0500 Subject: [PATCH 1/9] Stub out fix-kiss. --- test_async.py | 11 +++++++++++ 1 file changed, 11 insertions(+) create mode 100644 test_async.py diff --git a/test_async.py b/test_async.py new file mode 100644 index 0000000..62a9a6a --- /dev/null +++ b/test_async.py @@ -0,0 +1,11 @@ +import asyncio +import aprs + +async def main(): + transport, protocol = await aprs.create_tcp_connection("192.168.0.30", 8001) + + async for frame in protocol.read(): + print(frame) + +if __name__ == "__main__": + asyncio.run(main()) \ No newline at end of file From dc7d4ed8a39a870441ee598ee8ec9d7bca0798d5 Mon Sep 17 00:00:00 2001 From: W1CDN Date: Sun, 10 Dec 2023 13:30:51 -0600 Subject: [PATCH 2/9] Commit deployed changes from a while back. --- kiss_and_db.py | 17 ++++++++++++++++- 1 file changed, 16 insertions(+), 1 deletion(-) diff --git a/kiss_and_db.py b/kiss_and_db.py index c799a70..ae43f14 100644 --- a/kiss_and_db.py +++ b/kiss_and_db.py @@ -7,6 +7,8 @@ import aprslib import configparser import time import logging +from apscheduler.schedulers.asyncio import AsyncIOScheduler +import time def read_config(): config = configparser.ConfigParser() @@ -18,6 +20,15 @@ def get_db_connection(): conn.row_factory = sqlite3.Row return conn +def refresh_kiss_connection(kiss_conn): + logging.debug("Restarting KISS connection on schedule") + logging.debug("Stopping current connection") + kiss_conn.stop() + logging.debug("Waiting 5 seconds") + time.sleep(5) + logging.debug("Starting new connection") + kiss_conn.start() + def main(): # Add the call and location of this station to the packet info @@ -33,6 +44,9 @@ def main(): ki = aprs.TCPKISS(host=config['Settings']['kiss_host'], port=int(config['Settings']['kiss_port'])) ki.start() + scheduler = AsyncIOScheduler() + scheduler.add_job(refresh_kiss_connection, 'interval', hours = 1, args = [ki]) + scheduler.start() # Make a simple frame and send it frame = aprs.APRSFrame.ui( @@ -47,6 +61,7 @@ def main(): while True: conn = get_db_connection() for frame in ki.read(min_frames=1): + logging.debug("New packet, trying to parse") try: a = aprslib.parse(str(frame)) a['station_call'] = config['Settings']['station_call'] @@ -66,7 +81,7 @@ def main(): attrib_names = ', '.join('"%s"' % w for w in a.keys()) attrib_values = ", ".join("?" * len(a.keys())) - + logging.debug("Inserting into database") try: # Insert data sql = "INSERT INTO frames ("+attrib_names+") VALUES ("+attrib_values+")" From 307c1fcd860b9011b606b825c4d27c3bce50dca6 Mon Sep 17 00:00:00 2001 From: W1CDN Date: Sun, 10 Dec 2023 21:04:31 -0600 Subject: [PATCH 3/9] Flail around. --- kiss_and_db.py | 26 ++++++++++++++++++-------- 1 file changed, 18 insertions(+), 8 deletions(-) diff --git a/kiss_and_db.py b/kiss_and_db.py index ae43f14..e3141a4 100644 --- a/kiss_and_db.py +++ b/kiss_and_db.py @@ -28,6 +28,14 @@ def refresh_kiss_connection(kiss_conn): time.sleep(5) logging.debug("Starting new connection") kiss_conn.start() + # alive_frame = aprs.APRSFrame.ui( + # destination="TEST", + # source=config['Settings']['mycall'], + # path=None, + # info=b">None", + # ) + # logging.debug("Sending empty frame to keep connection alive") + # kiss_conn.write(alive_frame) def main(): @@ -43,19 +51,21 @@ def main(): ki = aprs.TCPKISS(host=config['Settings']['kiss_host'], port=int(config['Settings']['kiss_port'])) ki.start() + print(str(ki)) + ki.stop() + print(str(ki)) scheduler = AsyncIOScheduler() - scheduler.add_job(refresh_kiss_connection, 'interval', hours = 1, args = [ki]) + scheduler.add_job(refresh_kiss_connection, 'interval', minutes = 1, args = [ki]) scheduler.start() # Make a simple frame and send it - frame = aprs.APRSFrame.ui( - destination="APZ001", - source=config['Settings']['mycall'], - path=["WIDE1-1"], - info=b">Hello World!", - ) - #ki.write(frame) + # frame = aprs.APRSFrame.ui( + # destination="APZ001", + # source=config['Settings']['mycall'], + # path=['WIDE1-1'], + # info=b">Hello World!", + # ) # Watch for new packets to come in while True: From e7002c712c4417cb03e1017814f31e7c8a8b85e9 Mon Sep 17 00:00:00 2001 From: W1CDN Date: Mon, 11 Dec 2023 21:41:57 -0600 Subject: [PATCH 4/9] Flail. --- constants.py | 3 +++ kiss_and_db.py | 44 ++++++++++++++++++++------------------------ 2 files changed, 23 insertions(+), 24 deletions(-) diff --git a/constants.py b/constants.py index 8689d11..aea9984 100644 --- a/constants.py +++ b/constants.py @@ -3,6 +3,7 @@ db_frames_fields = ("id", "addresse", "alive", "altitude", +"body", "comment", "course", "created", @@ -11,6 +12,7 @@ db_frames_fields = ("id", "frame", "from", "gpsfixstatus", +"id", "latitude", "longitude", "mbits", @@ -44,6 +46,7 @@ db_frames_fields = ("id", "tEQNS", "tPARM", "tUNIT", +"type", "via", "weather", "wx_raw_timestamp") diff --git a/kiss_and_db.py b/kiss_and_db.py index e3141a4..9d4739f 100644 --- a/kiss_and_db.py +++ b/kiss_and_db.py @@ -24,18 +24,10 @@ def refresh_kiss_connection(kiss_conn): logging.debug("Restarting KISS connection on schedule") logging.debug("Stopping current connection") kiss_conn.stop() - logging.debug("Waiting 5 seconds") - time.sleep(5) + #logging.debug("Waiting 5 seconds") + #time.sleep(5) logging.debug("Starting new connection") kiss_conn.start() - # alive_frame = aprs.APRSFrame.ui( - # destination="TEST", - # source=config['Settings']['mycall'], - # path=None, - # info=b">None", - # ) - # logging.debug("Sending empty frame to keep connection alive") - # kiss_conn.write(alive_frame) def main(): @@ -47,33 +39,35 @@ def main(): logging.basicConfig(filename=config['Settings']['log_path'], level=logging.DEBUG, \ format='%(asctime)s - %(message)s') - logging.debug('kiss_and_db.py running') + logging.debug('============= kiss_and_db.py running =============') ki = aprs.TCPKISS(host=config['Settings']['kiss_host'], port=int(config['Settings']['kiss_port'])) ki.start() - print(str(ki)) - ki.stop() - print(str(ki)) - scheduler = AsyncIOScheduler() - scheduler.add_job(refresh_kiss_connection, 'interval', minutes = 1, args = [ki]) - scheduler.start() + #scheduler = AsyncIOScheduler() + #scheduler.add_job(refresh_kiss_connection, 'interval', hours = 1, args = [ki]) + #scheduler.start() # Make a simple frame and send it - # frame = aprs.APRSFrame.ui( - # destination="APZ001", - # source=config['Settings']['mycall'], - # path=['WIDE1-1'], - # info=b">Hello World!", - # ) +# frame = aprs.APRSFrame.ui( +# destination="APZ001", +# source=config['Settings']['mycall'], +# path=["WIDE1-1"], +# info=b">Hello World!", +# ) + #ki.write(frame) # Watch for new packets to come in while True: conn = get_db_connection() for frame in ki.read(min_frames=1): logging.debug("New packet, trying to parse") + logging.debug(str(frame)) try: - a = aprslib.parse(str(frame)) + try: + a = aprslib.parse(str(frame)) + except: + logging.error("Error with aprslib!") a['station_call'] = config['Settings']['station_call'] a['station_lat'] = config['Settings']['station_lat'] a['station_lon'] = config['Settings']['station_lon'] @@ -90,6 +84,8 @@ def main(): # Build an INSERT statement based on the fields we have from the frame attrib_names = ', '.join('"%s"' % w for w in a.keys()) attrib_values = ", ".join("?" * len(a.keys())) + logging.debug(attrib_names) + logging.debug(a.values()) logging.debug("Inserting into database") try: From 50085de7dba57de380a8798ae091943729aa7c8d Mon Sep 17 00:00:00 2001 From: W1CDN Date: Tue, 12 Dec 2023 20:25:00 -0600 Subject: [PATCH 5/9] Try to catch exceptions. --- kiss_and_db.py | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/kiss_and_db.py b/kiss_and_db.py index 9d4739f..453fa47 100644 --- a/kiss_and_db.py +++ b/kiss_and_db.py @@ -66,15 +66,18 @@ def main(): try: try: a = aprslib.parse(str(frame)) - except: - logging.error("Error with aprslib!") + except Exception as error: + logging.error("Error with aprslib:", error) a['station_call'] = config['Settings']['station_call'] a['station_lat'] = config['Settings']['station_lat'] a['station_lon'] = config['Settings']['station_lon'] a['created_unix'] = int(time.time()) - print(a) + # Make this a string and deal with it later (probably a mistake) a['path'] = str(a['path']) + if 'subpacket' in a: + a['subpacket'] = str(a['subpacket']) + #logging.debug(a['path']) # Store true/false as 1/0 if 'alive' in a: if a['alive'] == True: @@ -121,9 +124,9 @@ def main(): except: #print("Error with SQLite!") logging.error("Error with SQLite!") - except: + except Exception as error: #print("Frame could not be parsed.") - logging.error("Frame could not be parsed.") + logging.error("Frame could not be parsed:", error) conn.close() From 26b40817202c30633d883ea7b84ad8ba87c1b2a5 Mon Sep 17 00:00:00 2001 From: W1CDN Date: Wed, 13 Dec 2023 10:13:25 -0600 Subject: [PATCH 6/9] Working snapshot. --- kiss_and_db.py | 109 +++++++++++++++++++++++++------------------------ 1 file changed, 55 insertions(+), 54 deletions(-) diff --git a/kiss_and_db.py b/kiss_and_db.py index 453fa47..e9d7af2 100644 --- a/kiss_and_db.py +++ b/kiss_and_db.py @@ -67,63 +67,64 @@ def main(): try: a = aprslib.parse(str(frame)) except Exception as error: - logging.error("Error with aprslib:", error) - a['station_call'] = config['Settings']['station_call'] - a['station_lat'] = config['Settings']['station_lat'] - a['station_lon'] = config['Settings']['station_lon'] - a['created_unix'] = int(time.time()) + logging.error("Error with aprslib:", exc_info = error) + else: + a['station_call'] = config['Settings']['station_call'] + a['station_lat'] = config['Settings']['station_lat'] + a['station_lon'] = config['Settings']['station_lon'] + a['created_unix'] = int(time.time()) - # Make this a string and deal with it later (probably a mistake) - a['path'] = str(a['path']) - if 'subpacket' in a: - a['subpacket'] = str(a['subpacket']) - #logging.debug(a['path']) - # Store true/false as 1/0 - if 'alive' in a: - if a['alive'] == True: - a['alive'] = 1 - else: - a['alive'] = 0 - # Build an INSERT statement based on the fields we have from the frame - attrib_names = ', '.join('"%s"' % w for w in a.keys()) - attrib_values = ", ".join("?" * len(a.keys())) - logging.debug(attrib_names) - logging.debug(a.values()) + # Make this a string and deal with it later (probably a mistake) + a['path'] = str(a['path']) + if 'subpacket' in a: + a['subpacket'] = str(a['subpacket']) + #logging.debug(a['path']) + # Store true/false as 1/0 + if 'alive' in a: + if a['alive'] == True: + a['alive'] = 1 + else: + a['alive'] = 0 + # Build an INSERT statement based on the fields we have from the frame + attrib_names = ', '.join('"%s"' % w for w in a.keys()) + attrib_values = ", ".join("?" * len(a.keys())) + logging.debug(attrib_names) + logging.debug(a.values()) - logging.debug("Inserting into database") - try: - # Insert data - sql = "INSERT INTO frames ("+attrib_names+") VALUES ("+attrib_values+")" - conn.execute(sql, list(a.values())) - logging.debug("Frames table updated") - # TODO update stations table here - # Original intent was to include the id from the frames table, - # but that would mean making another query. - # It's not immediately needed, so I'm skipping it. - # Build query - # "from" is wrappedin [] because it is a reserved word and using '' doesn't work. - # https://www.sqlite.org/lang_keywords.html - #try: - station_update = "'"+a['from'] +"', '"+ str(a['created_unix']) +"', '1'" - query3 = "INSERT INTO stations ([from], last_heard_unix, count) \ - VALUES("+station_update+") \ - ON CONFLICT([from]) \ - DO UPDATE SET count = count + 1,\ - last_heard_unix = excluded.last_heard_unix;" - # Insert/update data - conn.execute(query3) - logging.debug("Station table updated") - conn.commit() - #except: - # print("Stations table couldn't be updated.") + logging.debug("Inserting into database") + try: + # Insert data + sql = "INSERT INTO frames ("+attrib_names+") VALUES ("+attrib_values+")" + conn.execute(sql, list(a.values())) + logging.debug("Frames table updated") + # TODO update stations table here + # Original intent was to include the id from the frames table, + # but that would mean making another query. + # It's not immediately needed, so I'm skipping it. + # Build query + # "from" is wrappedin [] because it is a reserved word and using '' doesn't work. + # https://www.sqlite.org/lang_keywords.html + #try: + station_update = "'"+a['from'] +"', '"+ str(a['created_unix']) +"', '1'" + query3 = "INSERT INTO stations ([from], last_heard_unix, count) \ + VALUES("+station_update+") \ + ON CONFLICT([from]) \ + DO UPDATE SET count = count + 1,\ + last_heard_unix = excluded.last_heard_unix;" + # Insert/update data + conn.execute(query3) + logging.debug("Station table updated") + conn.commit() + #except: + # print("Stations table couldn't be updated.") - # TODO remove packets that are older ('created') than a limit set in config.ini - # "5 minutes" also works - #conn.execute("DELETE FROM frames WHERE created < DATETIME('now', '"+config['Settings']['keep_time']+"')") - #conn.commit() - except: - #print("Error with SQLite!") - logging.error("Error with SQLite!") + # TODO remove packets that are older ('created') than a limit set in config.ini + # "5 minutes" also works + #conn.execute("DELETE FROM frames WHERE created < DATETIME('now', '"+config['Settings']['keep_time']+"')") + #conn.commit() + except: + #print("Error with SQLite!") + logging.error("Error with SQLite!") except Exception as error: #print("Frame could not be parsed.") logging.error("Frame could not be parsed:", error) From 79da9de6f7646b7480f6e58e78deaefe13fb5672 Mon Sep 17 00:00:00 2001 From: W1CDN Date: Wed, 13 Dec 2023 10:19:45 -0600 Subject: [PATCH 7/9] Align constants and schema. --- constants.py | 1 - schema.sql | 2 ++ 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/constants.py b/constants.py index aea9984..b950705 100644 --- a/constants.py +++ b/constants.py @@ -12,7 +12,6 @@ db_frames_fields = ("id", "frame", "from", "gpsfixstatus", -"id", "latitude", "longitude", "mbits", diff --git a/schema.sql b/schema.sql index a4a1cb9..943b93a 100644 --- a/schema.sql +++ b/schema.sql @@ -5,6 +5,7 @@ CREATE TABLE frames ( addresse TEXT, alive INT, altitude REAL, + body TEXT, comment TEXT, course REAL, created TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP, @@ -46,6 +47,7 @@ CREATE TABLE frames ( tEQNS TEXT, tPARM TEXT, tUNIT TEXT, + type TEXT, via TEXT, weather TEXT, wx_raw_timestamp TIMESTAMP From 1894de5d866706bcf045420a083148aafc419f7d Mon Sep 17 00:00:00 2001 From: W1CDN Date: Wed, 13 Dec 2023 10:24:17 -0600 Subject: [PATCH 8/9] Add header_raw field. --- constants.py | 1 + schema.sql | 1 + 2 files changed, 2 insertions(+) diff --git a/constants.py b/constants.py index b950705..7ade087 100644 --- a/constants.py +++ b/constants.py @@ -12,6 +12,7 @@ db_frames_fields = ("id", "frame", "from", "gpsfixstatus", +"header_raw", "latitude", "longitude", "mbits", diff --git a/schema.sql b/schema.sql index 943b93a..c6fe23d 100644 --- a/schema.sql +++ b/schema.sql @@ -14,6 +14,7 @@ CREATE TABLE frames ( frame TEXT, "from" TEXT, gpsfixstatus TEXT, + header_raw TEXT, latitude REAL, longitude REAL, mbits INT, From 25bffd9b6129bcdc8098617d3dccf176528a7918 Mon Sep 17 00:00:00 2001 From: W1CDN Date: Wed, 13 Dec 2023 10:40:32 -0600 Subject: [PATCH 9/9] Log errors in more detail. --- kiss_and_db.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/kiss_and_db.py b/kiss_and_db.py index e9d7af2..9d74bfa 100644 --- a/kiss_and_db.py +++ b/kiss_and_db.py @@ -122,12 +122,12 @@ def main(): # "5 minutes" also works #conn.execute("DELETE FROM frames WHERE created < DATETIME('now', '"+config['Settings']['keep_time']+"')") #conn.commit() - except: + except Exception as error: #print("Error with SQLite!") - logging.error("Error with SQLite!") + logging.error("Error with SQLite!", exc_info = error) except Exception as error: #print("Frame could not be parsed.") - logging.error("Frame could not be parsed:", error) + logging.error("Frame could not be parsed:", exc_info = error) conn.close()