Update recent data as available and fix some code to work with newer version of R.

bin/data-plots.R

@@ -9,26 +9,33 @@ library(htmlwidgets)
 library(lubridate)
 library(ggrepel)
 library(zoo)
+library(readr)
 
 #### Total/State/Class ####
 # Read in total/state/class data
-d_raw <- read.csv("data/us ham radio licenses over time.csv")
+d_raw <- read_csv("data/us ham radio licenses over time.csv")
 
 # What dates do we have?
 unique(d_raw$Date)
 
 # Clean up here if needed
 d <- d_raw %>% 
-        mutate(Date = as.Date(Date)) #%>% 
+        mutate(Date = as.Date(Date)) %>%
+        rename("State.Territory" = "State/Territory")
         # Drop Tech alone and leave Tech and Tech Plus
         #select(-c(Tech, Tech.Plus))
         
-d_total <- d %>% filter(State.Territory == "TOTAL")
+d_total <- d %>% filter(State.Territory == "TOTAL") # |> 
+    # Make sure blank cells are actually NAs
+    # mutate(across(everything(), gsub, pattern = "^$", replacement = NA_integer_),
+    #        Date = as.Date(Date))
                 
 d_total_long <- d_total %>% 
                 pivot_longer(cols = !c("Date", "State.Territory", "source_name", "source_detail"),
                              names_to = "lclass",
                              values_to = "count") %>% 
+                mutate(count = as.numeric(count),
+                       Date = as.Date(Date)) %>%
                 # only keep rows with data so plots look right
                 filter(!is.na(count))
 
@@ -59,10 +66,12 @@ qst_raw  <- read.csv("data/qst-somc.csv")
 arrl <- arrl_raw %>% 
             mutate(Date = as.Date(Date))
 qst  <- qst_raw %>% 
+            group_by(form_month, source) %>% 
             mutate(Date = as.Date(form_month),
-                   paid_copies_mean_monthly = c15_year_copies_paid + 
+                   paid_copies_mean_monthly = sum(c15_year_copies_paid,
-                                                  a16_year_copies_digital_paid) %>% 
+                                                  a16_year_copies_digital_paid,
-            pivot_longer(cols = -c(form_month, Date))
+                                                  na.rm = T)) %>% 
+            pivot_longer(cols = -c(form_month, Date, source))
 
 #### Plots ####
 

bin/fcc-query.R

@@ -8,27 +8,105 @@ library(tidyr)
 library(janitor)
 library(USAboundaries)
 
+# Update local database
+#system("python3 fccULSloader/src/fcc_tool.py --update")
+system("python3 fccULSloader/src/fcc_tool.py --force-download")
+
+# Get current date
+now <- Sys.Date()
+
 con <- dbConnect(SQLite(), "fccULSloader/src/data/fcc_data.db")
 
 am <- tbl(con, "AM")
 en <- tbl(con, "EN")
 
-a <- dbGetQuery(con, "SELECT COUNT(*) FROM EN LEFT JOIN HD ON 
+a_raw <- dbGetQuery(con, "SELECT COUNT(*) FROM EN LEFT JOIN HD ON 
                     EN.unique_system_identifier = HD.unique_system_identifier 
-                    WHERE license_status = 'A';") #%>% 
+                    WHERE license_status = 'A';")
-#janitor::clean_names() %>%
-#head(2)
-#filter(status_code == "A") %>%
-#summarize(count = n()) #%>%
-#collect()
 
-print(a)
+total_active <- a_raw %>% rename(count = `COUNT(*)`) %>% 
+                        mutate(state_name = "TOTAL")
 
-b <- dbGetQuery(con, "SELECT UPPER(state), COUNT(*) FROM EN LEFT JOIN HD ON 
+print(total_active)
+
+# Query active licenes by state 
+b_raw <- dbGetQuery(con, "SELECT UPPER(state), COUNT(*) FROM EN LEFT JOIN HD ON 
                     EN.unique_system_identifier = HD.unique_system_identifier 
                     WHERE license_status = 'A'
-                    GROUP BY UPPER(state);") %>%
+                    GROUP BY UPPER(state);") 
-    left_join(state_codes %>% filter(state_abbr != ""), 
+
-              by = c("UPPER(state)" = "state_abbr"))
+active_by_state <- b_raw %>%
+    left_join(state_codes %>% filter(state_abbr != ""), 
+              by = c("UPPER(state)" = "state_abbr")) %>% 
+    rename(abbrev = `UPPER(state)`,
+                      count = `COUNT(*)`) %>% 
+        mutate(state_name = case_when(abbrev == "AA" ~ "Armed Forces America",
+                                      abbrev == "AE" ~ "Armed Forces Europe",
+                                      abbrev == "AP" ~ "Armed Forces Pacific",
+                                      abbrev == "VI" ~ "Virgin Islands",
+                                      abbrev == "UM" ~ "US Minor Islands",
+                                      abbrev == ""   ~ "Other*",
+                                      TRUE ~ state_name)) %>% 
+    bind_rows(total_active)
+
+print(active_by_state)
+
+# Query active licenes by state, by class
+d_raw <- dbGetQuery(con, "SELECT UPPER(state), operator_class, COUNT(*) FROM EN LEFT JOIN HD ON 
+                    EN.unique_system_identifier = HD.unique_system_identifier LEFT JOIN AM ON
+                    EN.unique_system_identifier = AM.unique_system_identifier 
+                    WHERE license_status = 'A'
+                    GROUP BY UPPER(state), operator_class;") 
+
+active_state_class <- d_raw %>%
+    left_join(state_codes %>% filter(state_abbr != ""), 
+              by = c("UPPER(state)" = "state_abbr")) %>% 
+    rename(abbrev = `UPPER(state)`,
+           count = `COUNT(*)`) %>% 
+    mutate(state_name = case_when(abbrev == "AA" ~ "Armed Forces America",
+                                  abbrev == "AE" ~ "Armed Forces Europe",
+                                  abbrev == "AP" ~ "Armed Forces Pacific",
+                                  abbrev == "VI" ~ "Virgin Islands",
+                                  abbrev == "UM" ~ "US Minor Islands",
+                                  abbrev == ""   ~ "Other*",
+                                  TRUE ~ state_name)) %>% 
+    #bind_rows(total_active) %>% 
+    select(-state_code, -abbrev, -jurisdiction_type) %>% 
+    mutate(operator_class = case_when(operator_class == "A" ~ "Advanced",
+                                      operator_class == "E" ~ "Extra",
+                                      operator_class == "G" ~ "General",
+                                      operator_class == "N" ~ "Novice",
+                                      operator_class %in% c("T","P") ~ "Tech")) %>% 
+    pivot_wider(names_from = operator_class,
+                values_from = count) %>% 
+    mutate(Date = now) %>% 
+    rename("State/Territory" = state_name) %>% 
+    select(Date, `State/Territory`, Novice, Tech, General, Advanced, Extra, `NA`) %>% 
+    mutate("Tech-Plus" = NA,
+           .before = General) %>% 
+    rowwise() %>% mutate("Total" = sum(c_across(Novice:`NA`),
+                                       na.rm = T)) %>% 
+    select(-`NA`) %>% 
+    group_by(Date) %>% 
+    bind_rows(summarise(., across(where(is.numeric), sum, na.rm = T),
+                        across(where(is.character), ~'TOTAL'))) %>% 
+    mutate(Tech_and_TechPlus = NA,
+           Conditional = NA,
+           Club	= NA,
+           Military = NA,
+           Multiple	= NA,
+           Repeater	= NA,
+           GMRS	= NA,
+           source_name	= "fccULSloader",
+           source_detail = "w1cdn.net"
+)
+
+# Append to table
+write.table(active_state_class, file = "out/fccULSloader-fcc-licenses-scraped.csv", sep = ",", 
+            append = TRUE, quote = FALSE, 
+            col.names = F, row.names = FALSE,
+            na = "") 
+
+#### 
+
 
-print(b)

bin/scrape-license-counts.R

@@ -8,6 +8,7 @@ library(tidyr)
 
 ###### ARRL ###### 
 arrl_url <- "https://www.arrl.org/fcc-license-counts"
+cat("Checking ARRL...")
 
 # Read the page
 d_raw <- read_html(arrl_url)
@@ -51,6 +52,7 @@ write.table(tab, file = "out/arrl-fcc-licenses-scraped.csv", sep = ",",
 
 ###### HamCall ###### 
 hamcall_url <- "https://hamcall.net/hamcallcounts.html"
+cat("Checking HamCall...")
 
 # Read the page
 hamcall_raw <- read_html(hamcall_url)
@@ -127,9 +129,26 @@ hamcall_table_state <- left_join(hamcall_table_state, state_codes, by = join_by(
     relocate(Count, .after = f) %>% 
     relocate(source_name:source_detail, .after = m)
 
+##### Append tables #####
+write.table(hamcall_table_class_pivot, file = "out/hamcall-licenses-scraped.csv", sep = ",", 
+            append = TRUE, quote = FALSE, 
+            col.names = F, row.names = FALSE,
+            na = "") 
+
+write.table(hamcall_table_city, file = "out/hamcall-cities-scraped.csv", sep = ",", 
+            append = TRUE, quote = FALSE, 
+            col.names = F, row.names = FALSE,
+            na = "") 
+
+write.table(hamcall_table_state, file = "out/hamcall-states-scraped.csv", sep = ",", 
+            append = TRUE, quote = FALSE, 
+            col.names = F, row.names = FALSE,
+            na = "") 
+
 
 ###### AE7Q States ###### 
 ae7q_url <- "https://www.ae7q.com/query/stat/LicenseUSA.php"
+cat("Checking AE7Q...")
 
 # Read the page
 ae7q_raw <- read_html(ae7q_url)
@@ -168,6 +187,7 @@ ae7q_table_state <- ae7q_table_state_raw %>%
 ###### AE7Q License Actions ###### 
 ae7q_new_url <- paste0("https://www.ae7q.com/query/list/ProcessDate.php?DATE=", Sys.Date()-1)
 #ae7q_new_url <- paste0("https://www.ae7q.com/query/list/ProcessDate.php?DATE=2024-11-01")
+cat("Checking AE7Q actions...")
 
     # Read the page
     ae7q_new_raw <- read_html(ae7q_new_url)
@@ -211,21 +231,6 @@ ae7q_new_url <- paste0("https://www.ae7q.com/query/list/ProcessDate.php?DATE=",
 
 
 ##### Append tables #####
-write.table(hamcall_table_class_pivot, file = "out/hamcall-licenses-scraped.csv", sep = ",", 
-            append = TRUE, quote = FALSE, 
-            col.names = F, row.names = FALSE,
-            na = "") 
-
-write.table(hamcall_table_city, file = "out/hamcall-cities-scraped.csv", sep = ",", 
-            append = TRUE, quote = FALSE, 
-            col.names = F, row.names = FALSE,
-            na = "") 
-
-write.table(hamcall_table_state, file = "out/hamcall-states-scraped.csv", sep = ",", 
-            append = TRUE, quote = FALSE, 
-            col.names = F, row.names = FALSE,
-            na = "") 
-
 write.table(ae7q_table_state, file = "out/ae7q-states-scraped.csv", sep = ",", 
             append = TRUE, quote = FALSE, 
             col.names = F, row.names = FALSE,

data/arrl membership over time.csv

@@ -23,3 +23,4 @@ Date,Members,source_name,source_detail
 1993-12-31,170000,W5YI Report,FCC numbers in W5YI Report 1994-04-01 p. 4
 2023-12-31,149501,ARRL Annual Report,https://www.arrl.org/files/file/About%20ARRL/Annual%20Reports/2023%20ARRL%20Annual%20Report.pdf
 2024-12-31,137114,ARRL Annual Report,https://www.arrl.org/files/file/About%20ARRL/Annual%20Reports/2024%20ARRL%20Annual%20Report.pdf
+1995-03-31,172752,W5YI Report,1995-05-15 p. 1

data/qst-somc.csv

@@ -1,6 +1,8 @@
-form_month,a15_month_copies,a15_year_copies,c15_month_copies_paid,c15_year_copies_paid,a16_month_copies_digital_paid,a16_year_copies_digital_paid
+form_month,a15_month_copies,a15_year_copies,c15_month_copies_paid,c15_year_copies_paid,a16_month_copies_digital_paid,a16_year_copies_digital_paid,source
-2022-09-01,116950,117409,91984,92933,30298,28207
+2022-09-01,116950,117409,91984,92933,30298,28207,SOMC
-2023-09-01,114500,115087,91352,92815,23251,19963
+2023-09-01,114500,115087,91352,92815,23251,19963,SOMC
-1997-09-01,192724,191141,178311,167992,0,0
+1997-09-01,192724,191141,178311,167992,0,0,SOMC
-2020-09-01,127000,134497,91503,95536,22623,14908
+2020-09-01,127000,134497,91503,95536,22623,14908,SOMC
-2021-09-01,117099,121600,94634,94710,29269,26839
+2021-09-01,117099,121600,94634,94710,29269,26839,SOMC
+1987-12-01,,,,146176,,,W5YI Report 1988-08-01 p. 4
+1988-06-01,,,152668,150184,,,W5YI Report 1988-08-01 p. 4