From 1fc3684a18d819d5861319dc0c04aedaad4a9de5 Mon Sep 17 00:00:00 2001 From: mburtonkelly Date: Wed, 31 Oct 2018 16:36:54 -0500 Subject: [PATCH 1/8] Update API version and grab all the requests from the last week. --- bin/gfk-publicstuff.R | 22 +++++++++++++++++++--- 1 file changed, 19 insertions(+), 3 deletions(-) diff --git a/bin/gfk-publicstuff.R b/bin/gfk-publicstuff.R index 9c69925..6be2abf 100644 --- a/bin/gfk-publicstuff.R +++ b/bin/gfk-publicstuff.R @@ -1,7 +1,23 @@ +## +# R script to get data from PublicStuff +# Note that the API version at https://www.publicstuff.com/developers#!/API is v2.0, +# but this only includes requests up to a certain date. Use v2.1 for recent requests. + library(rjson) # Grab city view for Grand Forks -gfk <- fromJSON(file="https://www.publicstuff.com/api/2.0/city_view?space_id=15174") - +gfk <- fromJSON(file="https://www.publicstuff.com/api/2.1/city_view?space_id=15174") ## Make a data frame of request_type IDs and names -gfk_request_types <- as.data.frame(t(sapply(gfk$response$request_types$request_types, function(x) c(x$request_type$id, x$request_type$name)))) +gfk_request_types <- as.data.frame(t(sapply(gfk$response$request_types$request_types, + function(x) c(x$request_type$id, x$request_type$name)))) +# Add column names +names(gfk_request_types) <- c("request_type_id","request_type_name") +# Loop through request types and get n most recent in each category +# Unix timestamp from a week ago +today <- as.numeric(as.POSIXct(Sys.time())) +week_ago <- today-604800 +gfk_requests <- lapply(gfk_request_types$request_type_id, + function(x) fromJSON(file=paste0("https://www.publicstuff.com/api/2.1/requests_list?request_type_id=", + x,"&after_timestamp=",week_ago,"&limit=10"))) + + From 5d1ca6d3a44e85c43364546c73ff2ba36689068c Mon Sep 17 00:00:00 2001 From: mburtonkelly Date: Wed, 31 Oct 2018 17:12:02 -0500 Subject: [PATCH 2/8] Get closer to parsing JSON right. --- bin/gfk-publicstuff.R | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/bin/gfk-publicstuff.R b/bin/gfk-publicstuff.R index 6be2abf..bf9d4b4 100644 --- a/bin/gfk-publicstuff.R +++ b/bin/gfk-publicstuff.R @@ -3,10 +3,11 @@ # Note that the API version at https://www.publicstuff.com/developers#!/API is v2.0, # but this only includes requests up to a certain date. Use v2.1 for recent requests. +library(jsonlite) library(rjson) # Grab city view for Grand Forks -gfk <- fromJSON(file="https://www.publicstuff.com/api/2.1/city_view?space_id=15174") +gfk <- rjson::fromJSON(file="https://www.publicstuff.com/api/2.1/city_view?space_id=15174") ## Make a data frame of request_type IDs and names gfk_request_types <- as.data.frame(t(sapply(gfk$response$request_types$request_types, function(x) c(x$request_type$id, x$request_type$name)))) @@ -16,8 +17,17 @@ names(gfk_request_types) <- c("request_type_id","request_type_name") # Unix timestamp from a week ago today <- as.numeric(as.POSIXct(Sys.time())) week_ago <- today-604800 + gfk_requests <- lapply(gfk_request_types$request_type_id, - function(x) fromJSON(file=paste0("https://www.publicstuff.com/api/2.1/requests_list?request_type_id=", + function(x) jsonlite::fromJSON(paste0("https://www.publicstuff.com/api/2.1/requests_list?request_type_id=", x,"&after_timestamp=",week_ago,"&limit=10"))) +#gfk_requests1[sapply(gfk_requests1 , is.null)] <- NULL + +gfk_requests <- lapply(gfk_requests, function(x) x$response$requests$request) +## Need to clear out attachments in some way here + +# Drop nulls +b<-Filter(Negate(is.null), gfk_requests) +gfk_requests <- dplyr::bind_rows(b) From d723990b7731302ed118f04e84fc058f5853972d Mon Sep 17 00:00:00 2001 From: Matt Date: Wed, 31 Oct 2018 23:27:18 -0500 Subject: [PATCH 3/8] Finish cleaning up requests into a dataframe. --- bin/gfk-publicstuff.R | 33 ++++++++++++++++++++++++++------- 1 file changed, 26 insertions(+), 7 deletions(-) diff --git a/bin/gfk-publicstuff.R b/bin/gfk-publicstuff.R index bf9d4b4..0a3a5c3 100644 --- a/bin/gfk-publicstuff.R +++ b/bin/gfk-publicstuff.R @@ -5,6 +5,7 @@ library(jsonlite) library(rjson) +library(dplyr) # Grab city view for Grand Forks gfk <- rjson::fromJSON(file="https://www.publicstuff.com/api/2.1/city_view?space_id=15174") @@ -17,17 +18,35 @@ names(gfk_request_types) <- c("request_type_id","request_type_name") # Unix timestamp from a week ago today <- as.numeric(as.POSIXct(Sys.time())) week_ago <- today-604800 - +# For all request types, get (at most) 10 requests from the last week from the PublicStuff API. gfk_requests <- lapply(gfk_request_types$request_type_id, function(x) jsonlite::fromJSON(paste0("https://www.publicstuff.com/api/2.1/requests_list?request_type_id=", x,"&after_timestamp=",week_ago,"&limit=10"))) -#gfk_requests1[sapply(gfk_requests1 , is.null)] <- NULL - +# Pull out exactly the data we need gfk_requests <- lapply(gfk_requests, function(x) x$response$requests$request) +# Drop null list items +gfk_requests <- Filter(Negate(is.null), gfk_requests) +# Drop images (in fact, there is image_thumbnail in the data we want, +# and we can just replace small_ with large_ to get a bigger image later!) +drop_image <- function(x){ + if(class(x$primary_attachment) == "data.frame") { + x$primary_attachment <- NULL + } + return(x) +} +gfk_requests <- lapply(gfk_requests, drop_image) +# Put the requests together in a data frame +gfk_requests <- bind_rows(gfk_requests) + + +#### Tweeting + +# After tweeting, write a small text file that has the last timestamp that was tweeted. Use that for grabbing future requests. + + + + + -## Need to clear out attachments in some way here -# Drop nulls -b<-Filter(Negate(is.null), gfk_requests) -gfk_requests <- dplyr::bind_rows(b) From 7ae6d2c4d4724e70e0beff3b8f20040fde03a5b2 Mon Sep 17 00:00:00 2001 From: Matt Date: Wed, 31 Oct 2018 23:31:56 -0500 Subject: [PATCH 4/8] Get rolling on Twitter... --- bin/gfk-publicstuff.R | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/bin/gfk-publicstuff.R b/bin/gfk-publicstuff.R index 0a3a5c3..2934d22 100644 --- a/bin/gfk-publicstuff.R +++ b/bin/gfk-publicstuff.R @@ -6,6 +6,7 @@ library(jsonlite) library(rjson) library(dplyr) +library(twitteR) # Grab city view for Grand Forks gfk <- rjson::fromJSON(file="https://www.publicstuff.com/api/2.1/city_view?space_id=15174") @@ -40,6 +41,13 @@ gfk_requests <- bind_rows(gfk_requests) #### Tweeting +# https://rcrastinate.blogspot.com/2018/05/send-tweets-from-r-very-short.html + +# Need to create an external file with keys so they don't end up on GitHub +setup_twitter_oauth(consumer_key = "", + access_token = "", + consumer_secret = "", + access_secret = "") # After tweeting, write a small text file that has the last timestamp that was tweeted. Use that for grabbing future requests. From c8dc23e462fd55707c5a6ecc67e7f9b7020407a6 Mon Sep 17 00:00:00 2001 From: mburtonkelly Date: Thu, 1 Nov 2018 11:16:01 -0500 Subject: [PATCH 5/8] Set up .ini file for auth keys. --- auth.ini | 5 +++++ bin/gfk-publicstuff.R | 14 +++++++++----- 2 files changed, 14 insertions(+), 5 deletions(-) create mode 100644 auth.ini diff --git a/auth.ini b/auth.ini new file mode 100644 index 0000000..5e317c3 --- /dev/null +++ b/auth.ini @@ -0,0 +1,5 @@ +[twitter] +consumer_key = +access_token = +consumer_secret = +access_secret = \ No newline at end of file diff --git a/bin/gfk-publicstuff.R b/bin/gfk-publicstuff.R index 2934d22..605179d 100644 --- a/bin/gfk-publicstuff.R +++ b/bin/gfk-publicstuff.R @@ -7,6 +7,7 @@ library(jsonlite) library(rjson) library(dplyr) library(twitteR) +library(ini) # Grab city view for Grand Forks gfk <- rjson::fromJSON(file="https://www.publicstuff.com/api/2.1/city_view?space_id=15174") @@ -43,11 +44,14 @@ gfk_requests <- bind_rows(gfk_requests) #### Tweeting # https://rcrastinate.blogspot.com/2018/05/send-tweets-from-r-very-short.html -# Need to create an external file with keys so they don't end up on GitHub -setup_twitter_oauth(consumer_key = "", - access_token = "", - consumer_secret = "", - access_secret = "") +# Read authentication values from ini file +# Don't commit real values to git! +auth <- read.ini("auth.ini") + +setup_twitter_oauth(consumer_key = auth$twitter$consumer_key, + access_token = auth$twitter$access_token, + consumer_secret = auth$twitter$consumer_secret, + access_secret = auth$twitter$access_secret) # After tweeting, write a small text file that has the last timestamp that was tweeted. Use that for grabbing future requests. From 40f19c9dd9025dc14895a0bafc2971ab9cc1edcc Mon Sep 17 00:00:00 2001 From: mburtonkelly Date: Thu, 1 Nov 2018 11:16:45 -0500 Subject: [PATCH 6/8] Add auth.ini to .gitignore so keys don't get shared. --- .gitignore | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitignore b/.gitignore index 5b6a065..4566cd8 100644 --- a/.gitignore +++ b/.gitignore @@ -2,3 +2,4 @@ .Rhistory .RData .Ruserdata +auth.ini From eb1bd43d405cb8d75535497d680a568688b9f4c7 Mon Sep 17 00:00:00 2001 From: mburtonkelly Date: Thu, 1 Nov 2018 12:14:24 -0500 Subject: [PATCH 7/8] Stub out Mastodon tooting. --- auth.ini | 6 +++++- bin/gfk-publicstuff.R | 19 +++++++++++++------ 2 files changed, 18 insertions(+), 7 deletions(-) diff --git a/auth.ini b/auth.ini index 5e317c3..ee5bac2 100644 --- a/auth.ini +++ b/auth.ini @@ -2,4 +2,8 @@ consumer_key = access_token = consumer_secret = -access_secret = \ No newline at end of file +access_secret = +[mastodon] +server = +email = +password = \ No newline at end of file diff --git a/bin/gfk-publicstuff.R b/bin/gfk-publicstuff.R index 605179d..10d30da 100644 --- a/bin/gfk-publicstuff.R +++ b/bin/gfk-publicstuff.R @@ -42,7 +42,10 @@ gfk_requests <- bind_rows(gfk_requests) #### Tweeting -# https://rcrastinate.blogspot.com/2018/05/send-tweets-from-r-very-short.html +# You now need a developer account to set up an app, which takes some time. +# Do that here: https://developer.twitter.com/en/apply-for-access.html +# A workaround could be to set up a Mastodon account and then auto-tweet +# using the fantastic https://crossposter.masto.donte.com.br/. # Read authentication values from ini file # Don't commit real values to git! @@ -53,12 +56,16 @@ setup_twitter_oauth(consumer_key = auth$twitter$consumer_key, consumer_secret = auth$twitter$consumer_secret, access_secret = auth$twitter$access_secret) +# https://rcrastinate.blogspot.com/2018/05/send-tweets-from-r-very-short.html + # After tweeting, write a small text file that has the last timestamp that was tweeted. Use that for grabbing future requests. - - - - - +#### Tooting +# https://shkspr.mobi/blog/2018/08/easy-guide-to-building-mastodon-bots/ +# Might be able to use this natively: https://github.com/ThomasChln/mastodon +library(mastodon) +auth <- read.ini("auth.ini") +mastodon_token <- login(auth$mastodon$server, auth$mastodon$email, auth$mastodon$password) +post_status(mastodon_token, 'I posted this status with the wonderful https://github.com/ThomasChln/mastodon #rstats package.') From 902fae8c0106252cabe2e2060f6754b593cf806f Mon Sep 17 00:00:00 2001 From: mburtonkelly Date: Thu, 1 Nov 2018 15:00:55 -0500 Subject: [PATCH 8/8] Complete storage in database and checks to make sure everything gets posted once and only once. --- bin/gfk-publicstuff.R | 65 +++++++++++++++++++++++++++++++++++++------ 1 file changed, 57 insertions(+), 8 deletions(-) diff --git a/bin/gfk-publicstuff.R b/bin/gfk-publicstuff.R index 10d30da..3dca71d 100644 --- a/bin/gfk-publicstuff.R +++ b/bin/gfk-publicstuff.R @@ -8,6 +8,10 @@ library(rjson) library(dplyr) library(twitteR) library(ini) +library(mastodon) +library(RSQLite) + +# TODO Generalize to remove gfk_ from these variable names # Grab city view for Grand Forks gfk <- rjson::fromJSON(file="https://www.publicstuff.com/api/2.1/city_view?space_id=15174") @@ -38,8 +42,23 @@ drop_image <- function(x){ } gfk_requests <- lapply(gfk_requests, drop_image) # Put the requests together in a data frame -gfk_requests <- bind_rows(gfk_requests) +requests <- bind_rows(gfk_requests) +# Add posted column +requests$posted <- NA +# Sort by date +#gfk_requests <- gfk_requests[order(gfk_requests$date_created),] +## Store requests in a database +# Create DB if it doesn't exist, otherwise connect +mydb <- dbConnect(RSQLite::SQLite(), "requests.sqlite") +# See if table exists, then get existing rows back +if(nrow(dbGetQuery(mydb, "SELECT name FROM sqlite_master WHERE type='table' AND name='requests'")) > 0){ + rows.exist <- dbGetQuery(mydb, 'SELECT id FROM requests')$id +} else rows.exist <- NA +# Only add rows that don't exist, by request ID +rows.add <- requests[!requests$id %in% rows.exist,] +# Add the rows +dbWriteTable(mydb, "requests", rows.add,append=T) #### Tweeting # You now need a developer account to set up an app, which takes some time. @@ -51,21 +70,51 @@ gfk_requests <- bind_rows(gfk_requests) # Don't commit real values to git! auth <- read.ini("auth.ini") -setup_twitter_oauth(consumer_key = auth$twitter$consumer_key, - access_token = auth$twitter$access_token, - consumer_secret = auth$twitter$consumer_secret, - access_secret = auth$twitter$access_secret) +# setup_twitter_oauth(consumer_key = auth$twitter$consumer_key, +# access_token = auth$twitter$access_token, +# consumer_secret = auth$twitter$consumer_secret, +# access_secret = auth$twitter$access_secret) # https://rcrastinate.blogspot.com/2018/05/send-tweets-from-r-very-short.html -# After tweeting, write a small text file that has the last timestamp that was tweeted. Use that for grabbing future requests. + #### Tooting # https://shkspr.mobi/blog/2018/08/easy-guide-to-building-mastodon-bots/ # Might be able to use this natively: https://github.com/ThomasChln/mastodon -library(mastodon) + auth <- read.ini("auth.ini") mastodon_token <- login(auth$mastodon$server, auth$mastodon$email, auth$mastodon$password) -post_status(mastodon_token, 'I posted this status with the wonderful https://github.com/ThomasChln/mastodon #rstats package.') + +# Each time this script runs, take the oldest n requests, post them, and mark them in the db. +requests.new <- dbGetQuery(mydb, 'SELECT * FROM requests WHERE posted IS NULL ORDER BY date_created') + +# Set number of posts allowed at once. Will need to adjust according to cron +# schedule and number of posts coming in daily so you don't get behind. +posts_at_once <- 3 +for(i in 1:posts_at_once){ + request <- requests.new[i,] + # Post one selected request + if(nchar(request$image_thumbnail) > 1){ + download.file(gsub("small","large",request$image_thumbnail), 'temp.jpg', mode="wb") + post_media(mastodon_token, paste0(request$title, " at ", request$address, ": ", request$description), file = "temp.jpg") + } else { + post_status(mastodon_token, paste0(request$title, " at ", request$address, ": ", request$description)) + } + + # After tweeting or tooting, mark what has been posted. + # https://cran.r-project.org/web/packages/RSQLite/vignettes/RSQLite.html + # https://stackoverflow.com/a/43978368/2152245 + + # Update posted column as needed + dbExecute(mydb, "UPDATE requests SET posted = :posted where id = :id", + params=data.frame(posted=TRUE, + id=request$id)) + +} + +# Get out of the database +dbDisconnect(mydb) +unlink("requests.sqlite")