diff --git a/bin/gfk-publicstuff.R b/bin/gfk-publicstuff.R index 9790529..0d37b49 100644 --- a/bin/gfk-publicstuff.R +++ b/bin/gfk-publicstuff.R @@ -3,52 +3,58 @@ # Note that the API version at https://www.publicstuff.com/developers#!/API is v2.0, # but this only includes requests up to a certain date. Use v2.1 for recent requests. +# Run from one machine, otherwise you'll get duplicate databases that have differing posted items. +# Run with `Rscript bin/gfk-publicstuff.R` + library(jsonlite) library(rjson) library(dplyr) library(twitteR) library(ini) -library(mastodon) +library(mastodon) #devtools::install_github('ThomasChln/mastodon') library(RSQLite) -# TODO Generalize to remove gfk_ from these variable names - +### Config +# Authentication variables +auth <- read.ini("auth.ini") # Grab city view for Grand Forks -gfk <- rjson::fromJSON(file="https://www.publicstuff.com/api/2.1/city_view?space_id=15174") +space_id <- 15174 +client_id <- 1353 #needed later + + +### Get going +city <- rjson::fromJSON(file=paste0("https://www.publicstuff.com/api/2.1/city_view?space_id=",space_id)) ## Make a data frame of request_type IDs and names -gfk_request_types <- as.data.frame(t(sapply(gfk$response$request_types$request_types, +city_request_types <- as.data.frame(t(sapply(city$response$request_types$request_types, function(x) c(x$request_type$id, x$request_type$name)))) # Add column names -names(gfk_request_types) <- c("request_type_id","request_type_name") +names(city_request_types) <- c("request_type_id","request_type_name") # Loop through request types and get n most recent in each category # Unix timestamp from a week ago today <- as.numeric(as.POSIXct(Sys.time())) week_ago <- today-604800 -# For all request types, get (at most) 10 requests from the last week from the PublicStuff API. -gfk_requests <- lapply(gfk_request_types$request_type_id, +# For all request types, get requests from the last week from the PublicStuff API. +recent_requests <- lapply(city_request_types$request_type_id, function(x) jsonlite::fromJSON(paste0("https://www.publicstuff.com/api/2.1/requests_list?request_type_id=", - x,"&after_timestamp=",week_ago,"&limit=10"))) + x,"&after_timestamp=",week_ago,"&limit=100"))) # Pull out exactly the data we need -gfk_requests <- lapply(gfk_requests, function(x) x$response$requests$request) +recent_requests <- lapply(recent_requests, function(x) x$response$requests$request) # Drop null list items -gfk_requests <- Filter(Negate(is.null), gfk_requests) -# Drop images (in fact, there is image_thumbnail in the data we want, -# and we can just replace small_ with large_ to get a bigger image later!) +recent_requests <- Filter(Negate(is.null), recent_requests) +# Image data is in a sub-dataframe, which we don't need drop_image <- function(x){ if(class(x$primary_attachment) == "data.frame") { x$primary_attachment <- NULL } return(x) } -gfk_requests <- lapply(gfk_requests, drop_image) +recent_requests <- lapply(recent_requests, drop_image) # Put the requests together in a data frame -requests <- bind_rows(gfk_requests) -# Add URL column -requests$url <- paste0("https://iframe.publicstuff.com/#?client_id=1353&request_id=",requests$id) -# Add posted column -requests$posted <- NA -# Sort by date -#gfk_requests <- gfk_requests[order(gfk_requests$date_created),] +recent_requests <- bind_rows(recent_requests) +# Add URL +recent_requests$url <- paste0("https://iframe.publicstuff.com/#?client_id=",client_id,"&request_id=",recent_requests$id) +# Add posted column (to include in database table) +recent_requests$posted <- NA ## Store requests in a database # Create DB if it doesn't exist, otherwise connect @@ -58,9 +64,46 @@ if(nrow(dbGetQuery(mydb, "SELECT name FROM sqlite_master WHERE type='table' AND rows.exist <- dbGetQuery(mydb, 'SELECT id FROM requests')$id } else rows.exist <- NA # Only add rows that don't exist, by request ID -rows.add <- requests[!requests$id %in% rows.exist,] +rows.add <- recent_requests[!recent_requests$id %in% rows.exist,] # Add the rows dbWriteTable(mydb, "requests", rows.add,append=T) +# Get out of the database +dbDisconnect(mydb) + +#### Tooting +# https://shkspr.mobi/blog/2018/08/easy-guide-to-building-mastodon-bots/ +# https://github.com/ThomasChln/mastodon +mastodon_token <- login(auth$mastodon$server, auth$mastodon$email, auth$mastodon$password) + +# Each time this script runs, take the oldest n requests, post them, and mark them in the db. +mydb <- dbConnect(RSQLite::SQLite(), "requests.sqlite") +new_requests <- dbGetQuery(mydb, 'SELECT * FROM requests WHERE posted IS NULL ORDER BY date_created') + +# Set number of posts allowed at once. Will need to adjust according to cron +# schedule and number of posts coming in daily so you don't get behind. +posts_at_once <- 3 +for(i in 1:posts_at_once){ + request <- new_requests[i,] + # Post one selected request + post_text <- paste0(request$title, " at ", request$address, " (",request$url,"): ", request$description) + if(nchar(request$image_thumbnail) > 1){ + download.file(gsub("small","large",request$image_thumbnail), 'temp.jpg', mode="wb") + post_media(mastodon_token, post_text, file = "temp.jpg") + } else { + post_status(mastodon_token, post_text) + } + + # After tweeting or tooting, mark what has been posted. + # https://cran.r-project.org/web/packages/RSQLite/vignettes/RSQLite.html + # https://stackoverflow.com/a/43978368/2152245 + + # Update posted column as needed + dbExecute(mydb, "UPDATE requests SET posted = :posted where id = :id", + params=data.frame(posted=TRUE, + id=request$id)) +} +# Get out of the database +dbDisconnect(mydb) #### Tweeting # You now need a developer account to set up an app, which takes some time. @@ -79,45 +122,3 @@ auth <- read.ini("auth.ini") # https://rcrastinate.blogspot.com/2018/05/send-tweets-from-r-very-short.html - - - -#### Tooting -# https://shkspr.mobi/blog/2018/08/easy-guide-to-building-mastodon-bots/ -# Might be able to use this natively: https://github.com/ThomasChln/mastodon - -auth <- read.ini("auth.ini") -mastodon_token <- login(auth$mastodon$server, auth$mastodon$email, auth$mastodon$password) - -# Each time this script runs, take the oldest n requests, post them, and mark them in the db. -requests.new <- dbGetQuery(mydb, 'SELECT * FROM requests WHERE posted IS NULL ORDER BY date_created') - -# Set number of posts allowed at once. Will need to adjust according to cron -# schedule and number of posts coming in daily so you don't get behind. -posts_at_once <- 3 -for(i in 1:posts_at_once){ - request <- requests.new[i,] - text_to_post <- paste0(request$title, " at ", request$address, " (",request$url,"): ", request$description) - # Post one selected request - if(nchar(request$image_thumbnail) > 1){ - download.file(gsub("small","large",request$image_thumbnail), 'temp.jpg', mode="wb") - post_media(mastodon_token, text_to_post, file = "temp.jpg") - } else { - post_status(mastodon_token, text_to_post) - } - - # After tweeting or tooting, mark what has been posted. - # https://cran.r-project.org/web/packages/RSQLite/vignettes/RSQLite.html - # https://stackoverflow.com/a/43978368/2152245 - - # Update posted column as needed - dbExecute(mydb, "UPDATE requests SET posted = :posted where id = :id", - params=data.frame(posted=TRUE, - id=request$id)) - -} - -# Get out of the database -dbDisconnect(mydb) -unlink("requests.sqlite") -