W1CDN/ham-radio-licenses
1
0
Fork 0
Code Issues 132 Pull Requests Packages Projects Releases Wiki Activity

More work on AE7Q.

Browse Source
This commit is contained in:
mattbk
2024-09-22 10:17:37 -05:00
parent 419d39c569
commit 618ac4eff7
3 changed files with 3111 additions and 20 deletions
Download Patch File Download Diff File Expand all files Collapse all files

34
bin/scrape-ae7q-mass.R
View File

@ -3,10 +3,13 @@
# Use this file to scrape a series of dates from AE7Q
# Set start and end date
date_vec <- seq(as.Date("2024-07-23"), as.Date("2024-07-31"), by="days")
date_vec <- seq(as.Date("2004-11-29"), as.Date("2024-07-22"), by="days")
# Randomize dates we are querying
date_vec <- sample(date_vec)
dvbackup <- date_vec
date_vec <- date_vec[1177:7176]
ae7q_list <- list()
for(i in 1:length(date_vec)){
ae7q_new_url <- paste0("https://www.ae7q.com/query/list/ProcessDate.php?DATE=", date_vec[i])
@ -23,14 +26,15 @@ for(i in 1:length(date_vec)){
html_table()
# Find the right table by the column names
right_table_id <- grep(c("Callsign",
"Region/ State",
"Entity Name",
"Applicant Type",
"Licensee Class",
"License Status",
"Action Type"),
lapply(ae7q_new_tables, names))
right_table_id <- grep(paste(c("Callsign",
"Region/ State",
"Entity Name",
"Applicant Type",
"Licensee Class",
"License Status",
"Action Type"), collapse = " "),
lapply(ae7q_new_tables, function(x) paste(names(x), collapse = " ")))
ae7q_table_new <- ae7q_new_tables[[right_table_id]]
@ -43,18 +47,18 @@ for(i in 1:length(date_vec)){
relocate(date)
} else {
ae7q_list[[i]]<- data.frame("date" = date_vec[i],
"Action Type" = NA,
"count" = NA,
"source" = "AE7Q",
"source_detail" = ae7q_new_url)
"Action Type" = NA,
"count" = NA,
"source" = "AE7Q",
"source_detail" = ae7q_new_url)
}
# Wait for random time up to 10 seconds
Sys.sleep(sample(1:10, 1))
Sys.sleep(sample(1:5, 1))
}
# Combine all the data and sort by date
a <- bind_rows(ae7q_list) %>%
arrange(date)
arrange(date)
write.csv(a, "out/ae7q-actions-scraped.csv", row.names = F)