Stub out AE7Q actions work.

bin/data-plots.R

@@ -39,6 +39,10 @@ d_state_total_long <- d %>% filter(State.Territory != "TOTAL") %>%
 city_raw <- read.csv("data/us cities ham radio licenses over time.csv")
 city <- city_raw %>% mutate(Date = as.Date(Date),
                             city_label = paste0(City, ", ", State))
+
+#### License Actions ####
+ae7q_actions <- read.csv("data/ae7q-actions-scraped.csv") %>% 
+                mutate(date = as.Date(date))
                 
 #### Plots ####
 
@@ -283,3 +287,18 @@ ggplot(data = city,
     theme(legend.position="bottom")
 
 ggsave("plots/cities-over-time-freey.png", width = 15, height = 9)
+
+##### Actions Over Time #####
+ggplot(data = ae7q_actions,
+       aes(x = date,
+           y = count,
+           color = action)) +
+    geom_line() +
+    scale_x_date(date_breaks = "7 days",
+                 date_minor_breaks = "1 days",
+                 date_labels = "%Y-%m-%d") +
+    theme_bw() +
+    labs(title = "US Amateur License Actions",
+         caption = "w1cdn.net",
+         color = "Action") 
+

bin/scrape-ae7q-mass.R

@@ -0,0 +1,55 @@
+
+# Counts of license actions by date
+# Use this file to scrape a series of dates from AE7Q
+
+date_vec = seq(as.Date("2024-08-01"), as.Date("2024-09-21"), by="days")
+
+ae7q_list <- list()
+for(i in 1:length(date_vec)){
+    ae7q_new_url <- paste0("https://www.ae7q.com/query/list/ProcessDate.php?DATE=", date_vec[i])
+    print(ae7q_new_url)
+    
+    # Read the page
+    ae7q_new_raw <- read_html(ae7q_new_url)
+    
+    # Make sure the new license table exists first
+    if(!grepl("No license grants found issued on",  ae7q_new_raw %>% html_text())){
+        # Get tables and clean up
+        ae7q_new_tables <- ae7q_new_raw %>% 
+            html_elements(xpath = "//table") %>% 
+            html_table()
+        
+        # Find the right table by the column names
+        right_table_id <- grep(c("Callsign",
+                                 "Region/ State",
+                                 "Entity Name",
+                                 "Applicant Type",
+                                 "Licensee Class",
+                                 "License Status",
+                                 "Action Type"),
+                               lapply(ae7q_new_tables, names))
+        
+        ae7q_table_new <- ae7q_new_tables[[right_table_id]]
+        
+        ae7q_list[[i]] <- ae7q_table_new %>% mutate(across(everything(), ~na_if(., "\""))) %>% 
+            fill(everything()) %>% 
+            group_by(`Action Type`) %>% 
+            summarize(count = n(), .groups = "keep") %>% 
+            mutate(date = date_vec[i],
+                   source = "AE7Q", source_detail = ae7q_new_url) %>% 
+            relocate(date)
+    } else {
+        ae7q_list[[i]]<- data.frame("date" = date_vec[i],
+                                 "Action Type" = NA,
+                                 "count" = NA,
+                                 "source" = "AE7Q",
+                                 "source_detail" = ae7q_new_url)
+    }
+}
+
+a <- bind_rows(ae7q_list)
+
+write.csv(a, "out/ae7q-actions-scraped.csv", row.names = F)
+
+
+

bin/scrape-license-counts.R

@@ -165,7 +165,7 @@ ae7q_table_state <- ae7q_table_state_raw %>%
                  General, Advanced, AmateurExtra, Total, ttp, conditional,
                  Club)
 
-###### AE7Q New Licenses ###### 
+###### AE7Q License Actions ###### 
 ae7q_new_url <- paste0("https://www.ae7q.com/query/list/ProcessDate.php?DATE=", Sys.Date())
 
 # Read the page
@@ -178,15 +178,31 @@ if(!grepl("No license grants found issued on",  ae7q_new_raw %>% html_text())){
         html_elements(xpath = "//table") %>% 
         html_table()
     
-    ae7q_table_new <- ae7q_new_tables[[14]]
+    # Find the right table by the column names
+    right_table_id <- grep(c("Callsign",
+                 "Region/ State",
+                 "Entity Name",
+                 "Applicant Type",
+                 "Licensee Class",
+                 "License Status",
+                 "Action Type"),
+               lapply(ae7q_new_tables, names))
+    
+    ae7q_table_new <- ae7q_new_tables[[right_table_id]]
     
     ae7q_sum01 <- ae7q_table_new %>% mutate(across(everything(), ~na_if(., "\""))) %>% 
                             fill(everything()) %>% 
-                            group_by(`Applicant Type`,
-                                     `Licensee Class`,
-                                     `License Status`,
-                                     `Action Type`) %>% 
-                        summarize(count = n(), .groups = "keep")
+                            group_by(`Action Type`) %>% 
+                        summarize(count = n(), .groups = "keep") %>% 
+                    mutate(date = Sys.Date(),
+                           source = "AE7Q", source_detail = ae7q_new_url) %>% 
+                    relocate(date)
+} else {
+    ae7q_sum01 <- data.frame("date" = Sys.Date(),
+                             "Action Type" = NA,
+                             "count" = NA,
+                             "source" = "AE7Q",
+                             "source_detail" = ae7q_new_url)
 }
 
 
@@ -210,3 +226,8 @@ write.table(ae7q_table_state, file = "out/ae7q-states-scraped.csv", sep = ",",
             append = TRUE, quote = FALSE, 
             col.names = F, row.names = FALSE,
             na = "") 
+
+write.table(ae7q_sum01, file = "out/ae7q-actions-scraped.csv", sep = ",", 
+            append = TRUE, quote = FALSE, 
+            col.names = F, row.names = FALSE,
+            na = "")