W1CDN/ham-radio-licenses
1
0
Fork 0
Code Issues 132 Pull Requests Packages Projects Releases Wiki Activity

Try to fix AE7Q action counts.

Browse Source
This commit is contained in:
mattbk 2024-09-29 18:10:44 -05:00
parent 312ab95578
commit 9127b43f27
2 changed files with 39 additions and 36 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
bin/scrape-ae7q-mass.R
View File

@ -35,7 +35,6 @@ for(i in 1:length(date_vec)){
"Action Type"), collapse = " "),
lapply(ae7q_new_tables, function(x) paste(names(x), collapse = " ")))
ae7q_table_new <- ae7q_new_tables[[right_table_id]]
ae7q_list[[i]] <- ae7q_table_new %>%

74
bin/scrape-license-counts.R
View File

@ -168,42 +168,46 @@ ae7q_table_state <- ae7q_table_state_raw %>%
###### AE7Q License Actions ######
ae7q_new_url <- paste0("https://www.ae7q.com/query/list/ProcessDate.php?DATE=", Sys.Date())
# Read the page
ae7q_new_raw <- read_html(ae7q_new_url)
# Make sure the new license table exists first
if(!grepl("No license grants found issued on", ae7q_new_raw %>% html_text())){
# Get tables and clean up
ae7q_new_tables <- ae7q_new_raw %>%
html_elements(xpath = "//table") %>%
html_table()
# Read the page
ae7q_new_raw <- read_html(ae7q_new_url)
# Find the right table by the column names
right_table_id <- grep(c("Callsign",
"Region/ State",
"Entity Name",
"Applicant Type",
"Licensee Class",
"License Status",
"Action Type"),
lapply(ae7q_new_tables, names))
ae7q_table_new <- ae7q_new_tables[[right_table_id]]
ae7q_sum01 <- ae7q_table_new %>% mutate(across(everything(), ~na_if(., "\""))) %>%
fill(everything()) %>%
group_by(`Action Type`) %>%
summarize(count = n(), .groups = "keep") %>%
mutate(date = Sys.Date(),
source = "AE7Q", source_detail = ae7q_new_url) %>%
relocate(date)
} else {
ae7q_sum01 <- data.frame("date" = Sys.Date(),
"Action Type" = NA,
"count" = NA,
"source" = "AE7Q",
"source_detail" = ae7q_new_url)
}
# Make sure the new license table exists first
if(!grepl("No license grants found issued on", ae7q_new_raw %>% html_text())){
# Get tables and clean up
ae7q_new_tables <- ae7q_new_raw %>%
html_elements(xpath = "//table") %>%
html_table()
# Find the right table by the column names
right_table_id <- grep(paste(c("Callsign",
"Region/ State",
"Entity Name",
"Applicant Type",
"Licensee Class",
"License Status",
"Action Type"), collapse = " "),
lapply(ae7q_new_tables, function(x) paste(names(x), collapse = " ")))
ae7q_table_new <- ae7q_new_tables[[right_table_id]]
ae7q_sum01 <- ae7q_table_new %>%
#mutate(across(everything(), ~na_if(., "\""))) %>%
mutate(across(everything(),
~ case_when(. == "\"" ~ NA,
TRUE ~ .))) %>%
fill(everything()) %>%
group_by(`Action Type`) %>%
summarize(count = n(), .groups = "keep") %>%
mutate(date = Sys.Date(),
source = "AE7Q", source_detail = ae7q_new_url) %>%
relocate(date)
} else {
ae7q_sum01<- data.frame("date" = Sys.Date(),
"Action Type" = NA,
"count" = NA,
"source" = "AE7Q",
"source_detail" = ae7q_new_url)
}
##### Append tables #####