Fix merge.

Merge branch 'master' of https://github.com/mattbk/hack-grand-forks

# Conflicts:
#	bin/gfk-publicstuff.R

bin/gfk-publicstuff.R

@@ -3,52 +3,58 @@
 # Note that the API version at https://www.publicstuff.com/developers#!/API is v2.0,
 # but this only includes requests up to a certain date. Use v2.1 for recent requests.
 
+# Run from one machine, otherwise you'll get duplicate databases that have differing posted items.
+# Run with `Rscript bin/gfk-publicstuff.R`
+
 library(jsonlite)
 library(rjson)
 library(dplyr)
 library(twitteR)
 library(ini)
-library(mastodon)
+library(mastodon) #devtools::install_github('ThomasChln/mastodon')
 library(RSQLite)
 
-# TODO Generalize to remove gfk_ from these variable names
+### Config
-
+# Authentication variables
+auth <- read.ini("auth.ini")
 # Grab city view for Grand Forks
-gfk <- rjson::fromJSON(file="https://www.publicstuff.com/api/2.1/city_view?space_id=15174")
+space_id <- 15174
+client_id <- 1353 #needed later
+
+
+### Get going
+city <- rjson::fromJSON(file=paste0("https://www.publicstuff.com/api/2.1/city_view?space_id=",space_id))
 ## Make a data frame of request_type IDs and names
-gfk_request_types <- as.data.frame(t(sapply(gfk$response$request_types$request_types,
+city_request_types <- as.data.frame(t(sapply(city$response$request_types$request_types,
                                             function(x) c(x$request_type$id, x$request_type$name))))
 # Add column names
-names(gfk_request_types) <- c("request_type_id","request_type_name")
+names(city_request_types) <- c("request_type_id","request_type_name")
 # Loop through request types and get n most recent in each category
 # Unix timestamp from a week ago
 today <- as.numeric(as.POSIXct(Sys.time()))
 week_ago <- today-604800
-# For all request types, get (at most) 10 requests from the last week from the PublicStuff API.
+# For all request types, get requests from the last week from the PublicStuff API.
-gfk_requests <- lapply(gfk_request_types$request_type_id,
+recent_requests <- lapply(city_request_types$request_type_id,
                        function(x) jsonlite::fromJSON(paste0("https://www.publicstuff.com/api/2.1/requests_list?request_type_id=",
-                                                        x,"&after_timestamp=",week_ago,"&limit=10")))
+                                                        x,"&after_timestamp=",week_ago,"&limit=100")))
 # Pull out exactly the data we need
-gfk_requests <- lapply(gfk_requests, function(x) x$response$requests$request)
+recent_requests <- lapply(recent_requests, function(x) x$response$requests$request)
 # Drop null list items
-gfk_requests <- Filter(Negate(is.null), gfk_requests)
+recent_requests <- Filter(Negate(is.null), recent_requests)
-# Drop images (in fact, there is image_thumbnail in the data we want,
+# Image data is in a sub-dataframe, which we don't need
-# and we can just replace small_ with large_ to get a bigger image later!)
 drop_image <- function(x){
     if(class(x$primary_attachment) == "data.frame") {
         x$primary_attachment <- NULL
         }
     return(x)
 }
-gfk_requests <- lapply(gfk_requests, drop_image)
+recent_requests <- lapply(recent_requests, drop_image)
 # Put the requests together in a data frame
-requests <- bind_rows(gfk_requests)
+recent_requests <- bind_rows(recent_requests)
-# Add URL column
+# Add URL
-requests$url <- paste0("https://iframe.publicstuff.com/#?client_id=1353&request_id=",requests$id)
+recent_requests$url <- paste0("https://iframe.publicstuff.com/#?client_id=",client_id,"&request_id=",recent_requests$id)
-# Add posted column
+# Add posted column (to include in database table)
-requests$posted <- NA
+recent_requests$posted <- NA
-# Sort by date
-#gfk_requests <- gfk_requests[order(gfk_requests$date_created),]
 
 ## Store requests in a database
 # Create DB if it doesn't exist, otherwise connect
@@ -58,9 +64,46 @@ if(nrow(dbGetQuery(mydb, "SELECT name FROM sqlite_master WHERE type='table' AND
     rows.exist <- dbGetQuery(mydb, 'SELECT id FROM requests')$id
 } else rows.exist <- NA
 # Only add rows that don't exist, by request ID
-rows.add <- requests[!requests$id %in% rows.exist,]
+rows.add <- recent_requests[!recent_requests$id %in% rows.exist,]
 # Add the rows
 dbWriteTable(mydb, "requests", rows.add,append=T)
+# Get out of the database
+dbDisconnect(mydb)
+
+#### Tooting
+# https://shkspr.mobi/blog/2018/08/easy-guide-to-building-mastodon-bots/
+# https://github.com/ThomasChln/mastodon
+mastodon_token <- login(auth$mastodon$server, auth$mastodon$email, auth$mastodon$password)
+
+# Each time this script runs, take the oldest n requests, post them, and mark them in the db.
+mydb <- dbConnect(RSQLite::SQLite(), "requests.sqlite")
+new_requests <- dbGetQuery(mydb, 'SELECT * FROM requests WHERE posted IS NULL ORDER BY date_created')
+
+# Set number of posts allowed at once. Will need to adjust according to cron
+# schedule and number of posts coming in daily so you don't get behind.
+posts_at_once <- 3
+for(i in 1:posts_at_once){
+    request <- new_requests[i,]
+    # Post one selected request
+    post_text <- paste0(request$title, " at ", request$address, " (",request$url,"): ", request$description)
+    if(nchar(request$image_thumbnail) > 1){
+        download.file(gsub("small","large",request$image_thumbnail), 'temp.jpg', mode="wb")
+        post_media(mastodon_token, post_text, file = "temp.jpg")
+    } else {
+        post_status(mastodon_token, post_text)
+        }
+
+    # After tweeting or tooting, mark what has been posted.
+    # https://cran.r-project.org/web/packages/RSQLite/vignettes/RSQLite.html
+    # https://stackoverflow.com/a/43978368/2152245
+
+    # Update posted column as needed
+    dbExecute(mydb, "UPDATE requests SET posted = :posted where id = :id",
+                       params=data.frame(posted=TRUE,
+                                            id=request$id))
+}
+# Get out of the database
+dbDisconnect(mydb)
 
 #### Tweeting
 # You now need a developer account to set up an app, which takes some time.
@@ -79,45 +122,3 @@ auth <- read.ini("auth.ini")
 
 # https://rcrastinate.blogspot.com/2018/05/send-tweets-from-r-very-short.html
 
-
-
-
-#### Tooting
-# https://shkspr.mobi/blog/2018/08/easy-guide-to-building-mastodon-bots/
-# Might be able to use this natively: https://github.com/ThomasChln/mastodon
-
-auth <- read.ini("auth.ini")
-mastodon_token <- login(auth$mastodon$server, auth$mastodon$email, auth$mastodon$password)
-
-# Each time this script runs, take the oldest n requests, post them, and mark them in the db.
-requests.new <- dbGetQuery(mydb, 'SELECT * FROM requests WHERE posted IS NULL ORDER BY date_created')
-
-# Set number of posts allowed at once. Will need to adjust according to cron
-# schedule and number of posts coming in daily so you don't get behind.
-posts_at_once <- 3
-for(i in 1:posts_at_once){
-    request <- requests.new[i,]
-    text_to_post <- paste0(request$title, " at ", request$address, " (",request$url,"): ", request$description)
-    # Post one selected request
-    if(nchar(request$image_thumbnail) > 1){
-        download.file(gsub("small","large",request$image_thumbnail), 'temp.jpg', mode="wb")
-        post_media(mastodon_token, text_to_post, file = "temp.jpg")
-    } else {
-        post_status(mastodon_token, text_to_post)
-        }
-
-    # After tweeting or tooting, mark what has been posted.
-    # https://cran.r-project.org/web/packages/RSQLite/vignettes/RSQLite.html
-    # https://stackoverflow.com/a/43978368/2152245
-
-    # Update posted column as needed
-    dbExecute(mydb, "UPDATE requests SET posted = :posted where id = :id",
-                       params=data.frame(posted=TRUE,
-                                            id=request$id))
-
-}
-
-# Get out of the database
-dbDisconnect(mydb)
-unlink("requests.sqlite")
-