Update recent data as available and fix some code to work with newer version of R.

bin/data-plots.R

@@ -9,26 +9,33 @@ library(htmlwidgets)
 library(lubridate)
 library(ggrepel)
 library(zoo)
+library(readr)
 
 #### Total/State/Class ####
 # Read in total/state/class data
-d_raw <- read.csv("data/us ham radio licenses over time.csv")
+d_raw <- read_csv("data/us ham radio licenses over time.csv")
 
 # What dates do we have?
 unique(d_raw$Date)
 
 # Clean up here if needed
 d <- d_raw %>% 
-        mutate(Date = as.Date(Date)) #%>% 
+        mutate(Date = as.Date(Date)) %>%
+        rename("State.Territory" = "State/Territory")
         # Drop Tech alone and leave Tech and Tech Plus
         #select(-c(Tech, Tech.Plus))
         
-d_total <- d %>% filter(State.Territory == "TOTAL")
+d_total <- d %>% filter(State.Territory == "TOTAL") # |> 
+    # Make sure blank cells are actually NAs
+    # mutate(across(everything(), gsub, pattern = "^$", replacement = NA_integer_),
+    #        Date = as.Date(Date))
                 
 d_total_long <- d_total %>% 
                 pivot_longer(cols = !c("Date", "State.Territory", "source_name", "source_detail"),
                              names_to = "lclass",
                              values_to = "count") %>% 
+                mutate(count = as.numeric(count),
+                       Date = as.Date(Date)) %>%
                 # only keep rows with data so plots look right
                 filter(!is.na(count))
 
@@ -59,10 +66,12 @@ qst_raw  <- read.csv("data/qst-somc.csv")
 arrl <- arrl_raw %>% 
             mutate(Date = as.Date(Date))
 qst  <- qst_raw %>% 
+            group_by(form_month, source) %>% 
             mutate(Date = as.Date(form_month),
-                   paid_copies_mean_monthly = c15_year_copies_paid + 
-                                                  a16_year_copies_digital_paid) %>% 
-            pivot_longer(cols = -c(form_month, Date))
+                   paid_copies_mean_monthly = sum(c15_year_copies_paid,
+                                                  a16_year_copies_digital_paid,
+                                                  na.rm = T)) %>% 
+            pivot_longer(cols = -c(form_month, Date, source))
 
 #### Plots ####
 

bin/fcc-query.R

@@ -9,7 +9,8 @@ library(janitor)
 library(USAboundaries)
 
 # Update local database
-system("cd python3 fccULSloader/src/fcc_tool.py --update")
+#system("python3 fccULSloader/src/fcc_tool.py --update")
+system("python3 fccULSloader/src/fcc_tool.py --force-download")
 
 # Get current date
 now <- Sys.Date()
@@ -106,6 +107,6 @@ write.table(active_state_class, file = "out/fccULSloader-fcc-licenses-scraped.cs
             col.names = F, row.names = FALSE,
             na = "") 
 
-
+#### 
 
 

bin/scrape-license-counts.R

@@ -8,6 +8,7 @@ library(tidyr)
 
 ###### ARRL ###### 
 arrl_url <- "https://www.arrl.org/fcc-license-counts"
+cat("Checking ARRL...")
 
 # Read the page
 d_raw <- read_html(arrl_url)
@@ -51,6 +52,7 @@ write.table(tab, file = "out/arrl-fcc-licenses-scraped.csv", sep = ",",
 
 ###### HamCall ###### 
 hamcall_url <- "https://hamcall.net/hamcallcounts.html"
+cat("Checking HamCall...")
 
 # Read the page
 hamcall_raw <- read_html(hamcall_url)
@@ -127,9 +129,26 @@ hamcall_table_state <- left_join(hamcall_table_state, state_codes, by = join_by(
     relocate(Count, .after = f) %>% 
     relocate(source_name:source_detail, .after = m)
 
+##### Append tables #####
+write.table(hamcall_table_class_pivot, file = "out/hamcall-licenses-scraped.csv", sep = ",", 
+            append = TRUE, quote = FALSE, 
+            col.names = F, row.names = FALSE,
+            na = "") 
+
+write.table(hamcall_table_city, file = "out/hamcall-cities-scraped.csv", sep = ",", 
+            append = TRUE, quote = FALSE, 
+            col.names = F, row.names = FALSE,
+            na = "") 
+
+write.table(hamcall_table_state, file = "out/hamcall-states-scraped.csv", sep = ",", 
+            append = TRUE, quote = FALSE, 
+            col.names = F, row.names = FALSE,
+            na = "") 
+
 
 ###### AE7Q States ###### 
 ae7q_url <- "https://www.ae7q.com/query/stat/LicenseUSA.php"
+cat("Checking AE7Q...")
 
 # Read the page
 ae7q_raw <- read_html(ae7q_url)
@@ -168,6 +187,7 @@ ae7q_table_state <- ae7q_table_state_raw %>%
 ###### AE7Q License Actions ###### 
 ae7q_new_url <- paste0("https://www.ae7q.com/query/list/ProcessDate.php?DATE=", Sys.Date()-1)
 #ae7q_new_url <- paste0("https://www.ae7q.com/query/list/ProcessDate.php?DATE=2024-11-01")
+cat("Checking AE7Q actions...")
 
     # Read the page
     ae7q_new_raw <- read_html(ae7q_new_url)
@@ -211,21 +231,6 @@ ae7q_new_url <- paste0("https://www.ae7q.com/query/list/ProcessDate.php?DATE=",
 
 
 ##### Append tables #####
-write.table(hamcall_table_class_pivot, file = "out/hamcall-licenses-scraped.csv", sep = ",", 
-            append = TRUE, quote = FALSE, 
-            col.names = F, row.names = FALSE,
-            na = "") 
-
-write.table(hamcall_table_city, file = "out/hamcall-cities-scraped.csv", sep = ",", 
-            append = TRUE, quote = FALSE, 
-            col.names = F, row.names = FALSE,
-            na = "") 
-
-write.table(hamcall_table_state, file = "out/hamcall-states-scraped.csv", sep = ",", 
-            append = TRUE, quote = FALSE, 
-            col.names = F, row.names = FALSE,
-            na = "") 
-
 write.table(ae7q_table_state, file = "out/ae7q-states-scraped.csv", sep = ",", 
             append = TRUE, quote = FALSE, 
             col.names = F, row.names = FALSE,

data/arrl membership over time.csv

@@ -23,3 +23,4 @@ Date,Members,source_name,source_detail
 1993-12-31,170000,W5YI Report,FCC numbers in W5YI Report 1994-04-01 p. 4
 2023-12-31,149501,ARRL Annual Report,https://www.arrl.org/files/file/About%20ARRL/Annual%20Reports/2023%20ARRL%20Annual%20Report.pdf
 2024-12-31,137114,ARRL Annual Report,https://www.arrl.org/files/file/About%20ARRL/Annual%20Reports/2024%20ARRL%20Annual%20Report.pdf
+1995-03-31,172752,W5YI Report,1995-05-15 p. 1

data/qst-somc.csv

@@ -1,6 +1,8 @@
-form_month,a15_month_copies,a15_year_copies,c15_month_copies_paid,c15_year_copies_paid,a16_month_copies_digital_paid,a16_year_copies_digital_paid
-2022-09-01,116950,117409,91984,92933,30298,28207
-2023-09-01,114500,115087,91352,92815,23251,19963
-1997-09-01,192724,191141,178311,167992,0,0
-2020-09-01,127000,134497,91503,95536,22623,14908
-2021-09-01,117099,121600,94634,94710,29269,26839
+form_month,a15_month_copies,a15_year_copies,c15_month_copies_paid,c15_year_copies_paid,a16_month_copies_digital_paid,a16_year_copies_digital_paid,source
+2022-09-01,116950,117409,91984,92933,30298,28207,SOMC
+2023-09-01,114500,115087,91352,92815,23251,19963,SOMC
+1997-09-01,192724,191141,178311,167992,0,0,SOMC
+2020-09-01,127000,134497,91503,95536,22623,14908,SOMC
+2021-09-01,117099,121600,94634,94710,29269,26839,SOMC
+1987-12-01,,,,146176,,,W5YI Report 1988-08-01 p. 4
+1988-06-01,,,152668,150184,,,W5YI Report 1988-08-01 p. 4