W1CDN/ham-radio-licenses
1
0
Fork 0
Code Issues 132 Pull Requests Packages Projects Releases Wiki Activity

Try to fix AE7Q action counts.

Browse Source
This commit is contained in:
mattbk 2024-09-29 18:10:44 -05:00
parent 312ab95578
commit 9127b43f27
2 changed files with 39 additions and 36 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
bin/scrape-ae7q-mass.R
View File

@ -35,7 +35,6 @@ for(i in 1:length(date_vec)){
"Action Type"), collapse = " "), "Action Type"), collapse = " "),
lapply(ae7q_new_tables, function(x) paste(names(x), collapse = " "))) lapply(ae7q_new_tables, function(x) paste(names(x), collapse = " ")))
ae7q_table_new <- ae7q_new_tables[[right_table_id]] ae7q_table_new <- ae7q_new_tables[[right_table_id]]
ae7q_list[[i]] <- ae7q_table_new %>% ae7q_list[[i]] <- ae7q_table_new %>%

74
bin/scrape-license-counts.R
View File

@ -168,42 +168,46 @@ ae7q_table_state <- ae7q_table_state_raw %>%
###### AE7Q License Actions ###### ###### AE7Q License Actions ######
ae7q_new_url <- paste0("https://www.ae7q.com/query/list/ProcessDate.php?DATE=", Sys.Date()) ae7q_new_url <- paste0("https://www.ae7q.com/query/list/ProcessDate.php?DATE=", Sys.Date())
# Read the page # Read the page
ae7q_new_raw <- read_html(ae7q_new_url) ae7q_new_raw <- read_html(ae7q_new_url)
# Make sure the new license table exists first
if(!grepl("No license grants found issued on", ae7q_new_raw %>% html_text())){
# Get tables and clean up
ae7q_new_tables <- ae7q_new_raw %>%
html_elements(xpath = "//table") %>%
html_table()
# Find the right table by the column names # Make sure the new license table exists first
right_table_id <- grep(c("Callsign", if(!grepl("No license grants found issued on", ae7q_new_raw %>% html_text())){
"Region/ State", # Get tables and clean up
"Entity Name", ae7q_new_tables <- ae7q_new_raw %>%
"Applicant Type", html_elements(xpath = "//table") %>%
"Licensee Class", html_table()
"License Status",
"Action Type"), # Find the right table by the column names
lapply(ae7q_new_tables, names)) right_table_id <- grep(paste(c("Callsign",
"Region/ State",
ae7q_table_new <- ae7q_new_tables[[right_table_id]] "Entity Name",
"Applicant Type",
ae7q_sum01 <- ae7q_table_new %>% mutate(across(everything(), ~na_if(., "\""))) %>% "Licensee Class",
fill(everything()) %>% "License Status",
group_by(`Action Type`) %>% "Action Type"), collapse = " "),
summarize(count = n(), .groups = "keep") %>% lapply(ae7q_new_tables, function(x) paste(names(x), collapse = " ")))
mutate(date = Sys.Date(),
source = "AE7Q", source_detail = ae7q_new_url) %>% ae7q_table_new <- ae7q_new_tables[[right_table_id]]
relocate(date)
} else { ae7q_sum01 <- ae7q_table_new %>%
ae7q_sum01 <- data.frame("date" = Sys.Date(), #mutate(across(everything(), ~na_if(., "\""))) %>%
"Action Type" = NA, mutate(across(everything(),
"count" = NA, ~ case_when(. == "\"" ~ NA,
"source" = "AE7Q", TRUE ~ .))) %>%
"source_detail" = ae7q_new_url) fill(everything()) %>%
} group_by(`Action Type`) %>%
summarize(count = n(), .groups = "keep") %>%
mutate(date = Sys.Date(),
source = "AE7Q", source_detail = ae7q_new_url) %>%
relocate(date)
} else {
ae7q_sum01<- data.frame("date" = Sys.Date(),
"Action Type" = NA,
"count" = NA,
"source" = "AE7Q",
"source_detail" = ae7q_new_url)
}
##### Append tables ##### ##### Append tables #####