From cd7b11c16b54621beb23dcfd0d1b515bc3a2bbc5 Mon Sep 17 00:00:00 2001
From: mattbk <mattbk@users.noreply.github.com>
Date: Sun, 21 Apr 2024 16:14:02 -0500
Subject: [PATCH] Stub out scraper for HamCall.

---
 bin/scrape-arrl-fcc.R | 51 ++++++++++++++++++++++++++++++++++++++++++-
 1 file changed, 50 insertions(+), 1 deletion(-)

diff --git a/bin/scrape-arrl-fcc.R b/bin/scrape-arrl-fcc.R
index e4cae9f..9267fa4 100644
--- a/bin/scrape-arrl-fcc.R
+++ b/bin/scrape-arrl-fcc.R
@@ -36,7 +36,7 @@ tab <- d_raw %>%
 # Append table
 write.table(tab, file = "out/arrl-fcc-licenses-scraped.csv", sep = ",", 
             append = TRUE, quote = FALSE, 
-            col.names = TRUE, row.names = FALSE,
+            col.names = F, row.names = FALSE,
             na = "") 
 
 # Clean up table to remove any duplicates (sometimes the page isn't updated regularly)
@@ -48,3 +48,52 @@ write.table(tab, file = "out/arrl-fcc-licenses-scraped.csv", sep = ",",
 #           row.names = F,
 #           na = "")
 
+###### HamCall
+
+hamcall_url <- "https://hamcall.net/hamcallcounts.html"
+
+# Read the page
+hamcall_raw <- read_html(hamcall_url)
+
+# Get date (weird for HamCall because no xpath)
+hamcall_date <- hamcall_raw %>% 
+                    html_text() %>% 
+                    as.character() %>% 
+                    gsub(".*All counts current as of ", "", .) %>% 
+                    gsub("\n\r\n\r\nAll Current US Hams.*", "", .) %>% 
+                    as.Date()
+
+# Get tables and clean up
+hamcall_tables <- hamcall_raw %>%
+    html_elements(xpath = "//table") %>% 
+    html_table()
+
+hamcall_table_all_hams_raw <- hamcall_tables[[2]]
+hamcall_table_class_raw <- hamcall_tables[[3]]
+hamcall_table_city_raw <- hamcall_tables[[4]]
+hamcall_table_state_raw <- hamcall_tables[[5]]
+
+# Total licenses and class counts
+hamcall_table_class_pivot <- hamcall_table_class_raw %>% 
+                            pivot_wider(names_from = "Class",
+                                        values_from = "Count") %>% 
+                            # Grab total and date
+                            mutate(date = hamcall_date,
+                                   state = "TOTAL",
+                                   techplus = NA,
+                                   total = hamcall_table_all_hams_raw[1,2] %>% pull()) %>% 
+                            # Arrange columns
+                            relocate(date, state, N, T, techplus, G, A, E, total) %>% 
+                            select(1:9) %>% 
+                            mutate(a = NA, b = NA,
+                                   club = hamcall_table_all_hams_raw[2,2] %>% pull(),
+                                   military = hamcall_table_all_hams_raw[3,2] %>% pull(),
+                                   c = NA, d = NA, e = NA,
+                                   source_name = "HamCall",
+                                   source_detail = hamcall_url)
+
+# Append table
+write.table(hamcall_table_class_pivot, file = "out/hamcall-licenses-scraped.csv", sep = ",", 
+            append = TRUE, quote = FALSE, 
+            col.names = F, row.names = FALSE,
+            na = "")