W1CDN/ham-radio-licenses
1
0
Fork 0
Code Issues 132 Pull Requests Packages Projects Releases Wiki Activity

Fix error handling in scrape.

Browse Source
This commit is contained in:
Matt 2024-09-23 02:47:30 +00:00
parent 618ac4eff7
commit 072af64293
1 changed files with 9 additions and 4 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

13
bin/scrape-ae7q-mass.R
View File

@ -8,7 +8,7 @@ date_vec <- seq(as.Date("2004-11-29"), as.Date("2024-07-22"), by="days")
date_vec <- sample(date_vec) date_vec <- sample(date_vec)
dvbackup <- date_vec dvbackup <- date_vec
date_vec <- date_vec[1177:7176] date_vec <- date_vec[1687:7176]
ae7q_list <- list() ae7q_list <- list()
for(i in 1:length(date_vec)){ for(i in 1:length(date_vec)){
@ -38,7 +38,11 @@ for(i in 1:length(date_vec)){
ae7q_table_new <- ae7q_new_tables[[right_table_id]] ae7q_table_new <- ae7q_new_tables[[right_table_id]]
ae7q_list[[i]] <- ae7q_table_new %>% mutate(across(everything(), ~na_if(., "\""))) %>% ae7q_list[[i]] <- ae7q_table_new %>%
#mutate(across(everything(), ~na_if(., "\""))) %>%
mutate(across(everything(),
~ case_when(. == "\"" ~ NA,
TRUE ~ .))) %>%
fill(everything()) %>% fill(everything()) %>%
group_by(`Action Type`) %>% group_by(`Action Type`) %>%
summarize(count = n(), .groups = "keep") %>% summarize(count = n(), .groups = "keep") %>%
@ -58,9 +62,10 @@ for(i in 1:length(date_vec)){
# Combine all the data and sort by date # Combine all the data and sort by date
a <- bind_rows(ae7q_list) %>% a <- bind_rows(ae7q_list) %>%
arrange(date) arrange(date) %>%
filter(!is.na(date))
write.csv(a, "out/ae7q-actions-scraped.csv", row.names = F) write.csv(a, "out/ae7q-actions-scraped03.csv", row.names = F)