Fix up actions plot, add to readme, add a little more data.

bin/data-plots.R

@@ -42,7 +42,8 @@ city <- city_raw %>% mutate(Date = as.Date(Date),
 
 #### License Actions ####
 ae7q_actions <- read.csv("data/ae7q-actions-scraped.csv") %>% 
-                mutate(date = as.Date(date))
+                mutate(date = as.Date(date)) %>% 
+                filter(!is.na(action))
                 
 #### Plots ####
 
@@ -294,11 +295,17 @@ ggplot(data = ae7q_actions,
            y = count,
            color = action)) +
     geom_line() +
-    scale_x_date(date_breaks = "7 days",
-                 date_minor_breaks = "1 days",
+    scale_x_date(date_breaks = "1 month",
+                 date_minor_breaks = "1 week",
                  date_labels = "%Y-%m-%d") +
+    facet_wrap(~action,
+              scales = "free_y") +
     theme_bw() +
     labs(title = "US Amateur License Actions",
+         y = "Count",
+         x = "Date",
          caption = "w1cdn.net",
-         color = "Action") 
+         color = "Action") +
+    guides(color = "none")
 
+ggsave("plots/actions-over-time.png", width = 6, height = 4)

bin/scrape-ae7q-mass.R

@@ -2,7 +2,10 @@
 # Counts of license actions by date
 # Use this file to scrape a series of dates from AE7Q
 
-date_vec = seq(as.Date("2024-08-01"), as.Date("2024-09-21"), by="days")
+# Set start and end date
+date_vec <- seq(as.Date("2024-07-23"), as.Date("2024-07-31"), by="days")
+# Randomize dates we are querying
+date_vec <- sample(date_vec)
 
 ae7q_list <- list()
 for(i in 1:length(date_vec)){
@@ -45,9 +48,13 @@ for(i in 1:length(date_vec)){
                                  "source" = "AE7Q",
                                  "source_detail" = ae7q_new_url)
     }
+    # Wait for random time up to 10 seconds
+    Sys.sleep(sample(1:10, 1))
 }
 
-a <- bind_rows(ae7q_list)
+# Combine all the data and sort by date
+a <- bind_rows(ae7q_list) %>% 
+        arrange(date)
 
 write.csv(a, "out/ae7q-actions-scraped.csv", row.names = F)