W1CDN/ham-radio-licenses
1
0
Fork 0
Code Issues 132 Pull Requests Packages Projects Releases Wiki Activity

Try to fix AE7Q action counts.

Browse Source
This commit is contained in:
mattbk 2024-09-29 18:10:44 -05:00
parent 312ab95578
commit 9127b43f27
2 changed files with 39 additions and 36 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
bin/scrape-ae7q-mass.R
View File

@ -35,7 +35,6 @@ for(i in 1:length(date_vec)){
"Action Type"), collapse = " "), "Action Type"), collapse = " "),
lapply(ae7q_new_tables, function(x) paste(names(x), collapse = " "))) lapply(ae7q_new_tables, function(x) paste(names(x), collapse = " ")))
ae7q_table_new <- ae7q_new_tables[[right_table_id]] ae7q_table_new <- ae7q_new_tables[[right_table_id]]
ae7q_list[[i]] <- ae7q_table_new %>% ae7q_list[[i]] <- ae7q_table_new %>%

26
bin/scrape-license-counts.R
View File

@ -168,42 +168,46 @@ ae7q_table_state <- ae7q_table_state_raw %>%
###### AE7Q License Actions ###### ###### AE7Q License Actions ######
ae7q_new_url <- paste0("https://www.ae7q.com/query/list/ProcessDate.php?DATE=", Sys.Date()) ae7q_new_url <- paste0("https://www.ae7q.com/query/list/ProcessDate.php?DATE=", Sys.Date())
# Read the page # Read the page
ae7q_new_raw <- read_html(ae7q_new_url) ae7q_new_raw <- read_html(ae7q_new_url)
# Make sure the new license table exists first # Make sure the new license table exists first
if(!grepl("No license grants found issued on", ae7q_new_raw %>% html_text())){ if(!grepl("No license grants found issued on", ae7q_new_raw %>% html_text())){
# Get tables and clean up # Get tables and clean up
ae7q_new_tables <- ae7q_new_raw %>% ae7q_new_tables <- ae7q_new_raw %>%
html_elements(xpath = "//table") %>% html_elements(xpath = "//table") %>%
html_table() html_table()
# Find the right table by the column names # Find the right table by the column names
right_table_id <- grep(c("Callsign", right_table_id <- grep(paste(c("Callsign",
"Region/ State", "Region/ State",
"Entity Name", "Entity Name",
"Applicant Type", "Applicant Type",
"Licensee Class", "Licensee Class",
"License Status", "License Status",
"Action Type"), "Action Type"), collapse = " "),
lapply(ae7q_new_tables, names)) lapply(ae7q_new_tables, function(x) paste(names(x), collapse = " ")))
ae7q_table_new <- ae7q_new_tables[[right_table_id]] ae7q_table_new <- ae7q_new_tables[[right_table_id]]
ae7q_sum01 <- ae7q_table_new %>% mutate(across(everything(), ~na_if(., "\""))) %>% ae7q_sum01 <- ae7q_table_new %>%
#mutate(across(everything(), ~na_if(., "\""))) %>%
mutate(across(everything(),
~ case_when(. == "\"" ~ NA,
TRUE ~ .))) %>%
fill(everything()) %>% fill(everything()) %>%
group_by(`Action Type`) %>% group_by(`Action Type`) %>%
summarize(count = n(), .groups = "keep") %>% summarize(count = n(), .groups = "keep") %>%
mutate(date = Sys.Date(), mutate(date = Sys.Date(),
source = "AE7Q", source_detail = ae7q_new_url) %>% source = "AE7Q", source_detail = ae7q_new_url) %>%
relocate(date) relocate(date)
} else { } else {
ae7q_sum01 <- data.frame("date" = Sys.Date(), ae7q_sum01<- data.frame("date" = Sys.Date(),
"Action Type" = NA, "Action Type" = NA,
"count" = NA, "count" = NA,
"source" = "AE7Q", "source" = "AE7Q",
"source_detail" = ae7q_new_url) "source_detail" = ae7q_new_url)
} }
##### Append tables ##### ##### Append tables #####