More work on AE7Q.

bin/data-plots.R

@@ -43,7 +43,10 @@ city <- city_raw %>% mutate(Date = as.Date(Date),
 #### License Actions ####
 ae7q_actions <- read.csv("data/ae7q-actions-scraped.csv") %>% 
                 mutate(date = as.Date(date)) %>% 
-                filter(!is.na(action))
+                filter(!is.na(action)) %>% 
+                arrange(date) %>% 
+                group_by(action) %>% 
+                mutate(mean_30 = rollmean(count, k=30, fill=NA, align='right'))
                 
 #### Plots ####
 
@@ -295,16 +298,21 @@ ggplot(data = ae7q_actions,
            y = count,
            color = action)) +
     geom_line() +
-    scale_x_date(date_breaks = "1 month",
-                 date_minor_breaks = "1 week",
-                 date_labels = "%Y-%m-%d") +
+    geom_line(data = ae7q_actions,
+              aes(x = date,
+                  y = mean_30),
+              color = "black") +
+    scale_x_date(date_breaks = "5 years",
+                 date_minor_breaks = "1 year",
+                 date_labels = "%Y") +
     facet_wrap(~action,
               scales = "free_y") +
     theme_bw() +
     labs(title = "US Amateur License Actions",
+         subtitle = "with 30-day rolling mean",
          y = "Count",
          x = "Date",
-         caption = "w1cdn.net",
+         caption = "w1cdn.net; source: ae7q.com",
          color = "Action") +
     guides(color = "none")
 

bin/scrape-ae7q-mass.R

@@ -3,10 +3,13 @@
 # Use this file to scrape a series of dates from AE7Q
 
 # Set start and end date
-date_vec <- seq(as.Date("2024-07-23"), as.Date("2024-07-31"), by="days")
+date_vec <- seq(as.Date("2004-11-29"), as.Date("2024-07-22"), by="days")
 # Randomize dates we are querying
 date_vec <- sample(date_vec)
 
+dvbackup <- date_vec
+date_vec <- date_vec[1177:7176]
+
 ae7q_list <- list()
 for(i in 1:length(date_vec)){
     ae7q_new_url <- paste0("https://www.ae7q.com/query/list/ProcessDate.php?DATE=", date_vec[i])
@@ -23,14 +26,15 @@ for(i in 1:length(date_vec)){
             html_table()
         
         # Find the right table by the column names
-        right_table_id <- grep(c("Callsign",
-                                 "Region/ State",
-                                 "Entity Name",
-                                 "Applicant Type",
-                                 "Licensee Class",
-                                 "License Status",
-                                 "Action Type"),
-                               lapply(ae7q_new_tables, names))
+        right_table_id <- grep(paste(c("Callsign",
+                                       "Region/ State",
+                                       "Entity Name",
+                                       "Applicant Type",
+                                       "Licensee Class",
+                                       "License Status",
+                                       "Action Type"), collapse = " "),
+                               lapply(ae7q_new_tables, function(x) paste(names(x), collapse = " ")))
+        
         
         ae7q_table_new <- ae7q_new_tables[[right_table_id]]
         
@@ -43,18 +47,18 @@ for(i in 1:length(date_vec)){
             relocate(date)
     } else {
         ae7q_list[[i]]<- data.frame("date" = date_vec[i],
-                                 "Action Type" = NA,
-                                 "count" = NA,
-                                 "source" = "AE7Q",
-                                 "source_detail" = ae7q_new_url)
+                                    "Action Type" = NA,
+                                    "count" = NA,
+                                    "source" = "AE7Q",
+                                    "source_detail" = ae7q_new_url)
     }
     # Wait for random time up to 10 seconds
-    Sys.sleep(sample(1:10, 1))
+    Sys.sleep(sample(1:5, 1))
 }
 
 # Combine all the data and sort by date
 a <- bind_rows(ae7q_list) %>% 
-        arrange(date)
+    arrange(date)
 
 write.csv(a, "out/ae7q-actions-scraped.csv", row.names = F)