Add attribution of tweets, scraping cmds and dry run
This commit is contained in:
parent
b8c794f9d7
commit
3df33fc522
File diff suppressed because it is too large
Load Diff
12
Cargo.toml
12
Cargo.toml
|
|
@ -1,22 +1,28 @@
|
||||||
[package]
|
[package]
|
||||||
name = "iso7010_a_day"
|
name = "iso7010_a_day"
|
||||||
|
description = "A silly twitter bot"
|
||||||
version = "0.1.0"
|
version = "0.1.0"
|
||||||
edition = "2018"
|
edition = "2018"
|
||||||
|
|
||||||
[dependencies]
|
[dependencies]
|
||||||
reqwest = { version = "0.11", features = ["blocking", "json", "multipart"]}
|
reqwest = { version = "0.11", features = ["blocking", "json", "multipart"]}
|
||||||
serde_json = "*"
|
serde_json = "*"
|
||||||
serde = "*"
|
serde = { version = "1", features = ["derive"] }
|
||||||
scraper = "*"
|
scraper = "*"
|
||||||
rand = "*"
|
rand = "*"
|
||||||
resvg = "*"
|
resvg = "*"
|
||||||
tiny-skia = "*"
|
tiny-skia = "*"
|
||||||
usvg = "*"
|
usvg = "*"
|
||||||
oauth1 = "*"
|
oauth1 = "*"
|
||||||
clap = "*"
|
clap = { version = "*", features = ["derive"] }
|
||||||
webbrowser = "*"
|
webbrowser = "*"
|
||||||
itertools = "*"
|
itertools = "*"
|
||||||
url = "*"
|
tracing = "0.1.36"
|
||||||
|
tracing-subscriber = "0.3.15"
|
||||||
|
regex = "1.6.0"
|
||||||
|
image = "0.24.3"
|
||||||
|
viuer = "0.6.1"
|
||||||
|
url = { version = "2.3.1", features = ["serde"] }
|
||||||
|
|
||||||
[build-dependencies]
|
[build-dependencies]
|
||||||
toml = "*"
|
toml = "*"
|
||||||
497
src/main.rs
497
src/main.rs
|
|
@ -1,15 +1,14 @@
|
||||||
use itertools::Itertools;
|
|
||||||
use rand::seq::SliceRandom;
|
use rand::seq::SliceRandom;
|
||||||
use std::borrow::Cow;
|
|
||||||
use std::convert::TryInto;
|
use std::convert::TryInto;
|
||||||
use std::io::prelude::*;
|
use std::io::Cursor;
|
||||||
use std::{collections::HashMap, io::Write};
|
use tracing::{event, Level};
|
||||||
|
mod twitter;
|
||||||
const APP_TOKEN_ENV_VAR: &str = "TWITTER_APP_TOKEN";
|
mod wiki;
|
||||||
const APP_SECRET_ENV_VAR: &str = "TWITTER_APP_SECRET";
|
use clap::{Parser, Subcommand};
|
||||||
const USER_TOKEN_ENV_VAR: &str = "TWITTER_USER_TOKEN";
|
use image::{DynamicImage, RgbaImage};
|
||||||
const USER_SECRET_ENV_VAR: &str = "TWITTER_USER_SECRET";
|
use tiny_skia::{Paint, PathBuilder, Pixmap, PixmapPaint, Stroke, Transform};
|
||||||
const IMG_HEIGHT: u32 = 1000;
|
use twitter::*;
|
||||||
|
use wiki::*;
|
||||||
|
|
||||||
static APP_USER_AGENT: &str = concat!(
|
static APP_USER_AGENT: &str = concat!(
|
||||||
"bot_",
|
"bot_",
|
||||||
|
|
@ -20,316 +19,224 @@ static APP_USER_AGENT: &str = concat!(
|
||||||
"reqwest",
|
"reqwest",
|
||||||
);
|
);
|
||||||
|
|
||||||
static CB_URL: &str = "http://localhost:6969/cb";
|
// Render the raw SVG data to an image
|
||||||
|
fn render_svg(data: &[u8], height: u32, with_border: bool) -> StdError<DynamicImage> {
|
||||||
fn render_svg(data: &[u8]) -> Result<Vec<u8>, Box<dyn std::error::Error>> {
|
|
||||||
let opt = usvg::Options::default();
|
let opt = usvg::Options::default();
|
||||||
let rtree = usvg::Tree::from_data(&data, &opt.to_ref()).expect("couldn't parse");
|
let rtree = usvg::Tree::from_data(data, &opt.to_ref()).expect("couldn't parse");
|
||||||
let mut pixmap =
|
let svg_size = rtree.svg_node().size;
|
||||||
tiny_skia::Pixmap::new(IMG_HEIGHT, IMG_HEIGHT).ok_or("Error creating pixmap")?;
|
// Work out how wide the pixmap of height `height` needs to be to entirely fit the SVG.
|
||||||
resvg::render(
|
let pm_width = ((height as f64 / svg_size.height()) * svg_size.width()).ceil() as u32;
|
||||||
&rtree,
|
let mut pixmap = Pixmap::new(pm_width, height).ok_or("Error creating pixmap")?;
|
||||||
usvg::FitTo::Size(IMG_HEIGHT, IMG_HEIGHT),
|
// Render the svg into a pixmap.
|
||||||
pixmap.as_mut(),
|
resvg::render(&rtree, usvg::FitTo::Height(height), pixmap.as_mut())
|
||||||
)
|
.ok_or("Error rendering svg")?;
|
||||||
.ok_or_else(|| "Error rendering svg")?;
|
// Make a wider pixmap with a 16:9 AR and the same height. This is a blesséd ratio by twitter
|
||||||
let mut bigger_pixmap = tiny_skia::Pixmap::new(IMG_HEIGHT / 9 * 16, IMG_HEIGHT)
|
// and means we see the whole image nicely in the timeline with no truncation.
|
||||||
.ok_or("Error creating bigger pixmap")?;
|
let mut bigger_pixmap =
|
||||||
|
Pixmap::new(height / 9 * 16, height).ok_or("Error creating bigger pixmap")?;
|
||||||
|
// Then draw our freshly rendered SVG into the middle of the bigger pixmap.
|
||||||
bigger_pixmap
|
bigger_pixmap
|
||||||
.draw_pixmap(
|
.draw_pixmap(
|
||||||
(bigger_pixmap.width() / 2 - IMG_HEIGHT / 2)
|
((bigger_pixmap.width() - pm_width) / 2).try_into().unwrap(),
|
||||||
.try_into()
|
|
||||||
.unwrap(),
|
|
||||||
0,
|
0,
|
||||||
pixmap.as_ref(),
|
pixmap.as_ref(),
|
||||||
&tiny_skia::PixmapPaint::default(),
|
&PixmapPaint::default(),
|
||||||
tiny_skia::Transform::identity(),
|
Transform::identity(),
|
||||||
None,
|
None,
|
||||||
)
|
)
|
||||||
.ok_or("Error drawing onto bigger pixmap")?;
|
.ok_or("Error drawing onto bigger pixmap")?;
|
||||||
let png_data = bigger_pixmap.encode_png()?;
|
let (w, h) = (bigger_pixmap.width(), bigger_pixmap.height());
|
||||||
Ok(png_data)
|
// Render a red border for debug purposes
|
||||||
}
|
if with_border {
|
||||||
|
let mut paint = Paint::default();
|
||||||
enum PostData<'a> {
|
paint.set_color_rgba8(255, 0, 0, 255);
|
||||||
Empty,
|
let stroke = Stroke {
|
||||||
Multipart(reqwest::blocking::multipart::Form),
|
width: 1.0,
|
||||||
Data(&'a [(&'a str, &'a str)]),
|
..Default::default()
|
||||||
}
|
|
||||||
|
|
||||||
enum APIAction<'a> {
|
|
||||||
Get,
|
|
||||||
Post(PostData<'a>),
|
|
||||||
}
|
|
||||||
|
|
||||||
struct TwitterEndpoint<'a>(&'a str);
|
|
||||||
|
|
||||||
impl TryInto<reqwest::Url> for TwitterEndpoint<'_> {
|
|
||||||
type Error = url::ParseError;
|
|
||||||
fn try_into(self) -> Result<reqwest::Url, Self::Error> {
|
|
||||||
reqwest::Url::parse(&format!("https://api.twitter.com/{}", self.0))
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// Make an authed twitter API request
|
|
||||||
fn twitter_api<'a>(
|
|
||||||
url: reqwest::Url,
|
|
||||||
user_token: Option<&oauth1::Token>,
|
|
||||||
action: APIAction,
|
|
||||||
extra_oauth_params: &[(&str, &str)],
|
|
||||||
) -> StdError<reqwest::blocking::Response> {
|
|
||||||
let consumer_token = oauth1::Token::new(
|
|
||||||
std::env::var(APP_TOKEN_ENV_VAR)?,
|
|
||||||
std::env::var(APP_SECRET_ENV_VAR)?,
|
|
||||||
);
|
|
||||||
let mut headers = reqwest::header::HeaderMap::new();
|
|
||||||
let mut oauth_params: HashMap<&str, Cow<str>> = extra_oauth_params
|
|
||||||
.iter()
|
|
||||||
.cloned()
|
|
||||||
.map(|(x, y)| (x, y.into()))
|
|
||||||
.collect();
|
|
||||||
|
|
||||||
// If the request is a key/value form post, we need to include those parameters when
|
|
||||||
// generating the signature.
|
|
||||||
match action {
|
|
||||||
APIAction::Post(PostData::Data(d)) => {
|
|
||||||
oauth_params.extend(d.iter().cloned().map(|(x, y)| (x, y.into())))
|
|
||||||
}
|
|
||||||
_ => {}
|
|
||||||
}
|
|
||||||
|
|
||||||
headers.insert(
|
|
||||||
reqwest::header::AUTHORIZATION,
|
|
||||||
reqwest::header::HeaderValue::from_str(&oauth1::authorize(
|
|
||||||
if matches!(action, APIAction::Post(_)) {
|
|
||||||
"POST"
|
|
||||||
} else {
|
|
||||||
"GET"
|
|
||||||
},
|
|
||||||
url.as_str(),
|
|
||||||
&consumer_token,
|
|
||||||
user_token,
|
|
||||||
Some(oauth_params),
|
|
||||||
))?,
|
|
||||||
);
|
|
||||||
|
|
||||||
let client = reqwest::blocking::Client::builder()
|
|
||||||
.user_agent(APP_USER_AGENT)
|
|
||||||
.default_headers(headers)
|
|
||||||
.build()?;
|
|
||||||
let req = match action {
|
|
||||||
APIAction::Get => client.get(url),
|
|
||||||
APIAction::Post(PostData::Empty) => client.post(url),
|
|
||||||
APIAction::Post(PostData::Data(data)) => client.post(url).form(data),
|
|
||||||
APIAction::Post(PostData::Multipart(form)) => client.post(url).multipart(form),
|
|
||||||
};
|
};
|
||||||
|
|
||||||
let res = req.send()?;
|
let path = {
|
||||||
if !res.status().is_success() {
|
let mut pb = PathBuilder::new();
|
||||||
return Err(format!(
|
pb.move_to(0.0, 0.0);
|
||||||
"Got non-200 response: status {}, {}",
|
pb.line_to(0.0, h as f32 - stroke.width);
|
||||||
res.status(),
|
pb.line_to(w as f32, h as f32 - stroke.width);
|
||||||
res.text()?
|
pb.line_to(w as f32 - stroke.width, 0.0);
|
||||||
)
|
pb.line_to(0.0, 0.0);
|
||||||
.into());
|
pb.finish().unwrap()
|
||||||
|
};
|
||||||
|
bigger_pixmap.stroke_path(&path, &paint, &stroke, Transform::identity(), None);
|
||||||
}
|
}
|
||||||
Ok(res)
|
let img = RgbaImage::from_raw(
|
||||||
|
bigger_pixmap.width(),
|
||||||
|
bigger_pixmap.height(),
|
||||||
|
bigger_pixmap.data().to_vec(),
|
||||||
|
)
|
||||||
|
.ok_or("Error creating image from pixmap")?;
|
||||||
|
Ok(DynamicImage::ImageRgba8(img))
|
||||||
}
|
}
|
||||||
|
|
||||||
type StdError<T> = Result<T, Box<dyn std::error::Error>>;
|
type StdError<T> = Result<T, Box<dyn std::error::Error>>;
|
||||||
|
|
||||||
|
#[derive(Parser)]
|
||||||
|
#[clap(author, version, about)]
|
||||||
|
struct Cli {
|
||||||
|
#[clap(subcommand)]
|
||||||
|
command: Commands,
|
||||||
|
}
|
||||||
|
|
||||||
|
#[derive(Subcommand)]
|
||||||
|
enum Commands {
|
||||||
|
/// Authorize the twitter application to acccess a user's account
|
||||||
|
Authorize,
|
||||||
|
/// Scrape images from the category on wikimedia commons
|
||||||
|
ScrapeCategory,
|
||||||
|
/// Scrape images from the iso7010 wikipedia page
|
||||||
|
ScrapeWeb,
|
||||||
|
/// List tweets from the authed user's timeline
|
||||||
|
ListTweets,
|
||||||
|
/// Run the bot - scrape, pick a random entry and tweet it
|
||||||
|
RunBot {
|
||||||
|
#[clap(short, long, action)]
|
||||||
|
dry_run: bool,
|
||||||
|
},
|
||||||
|
/// Print details about the currently authed user
|
||||||
|
Whoami,
|
||||||
|
}
|
||||||
|
|
||||||
fn main() -> StdError<()> {
|
fn main() -> StdError<()> {
|
||||||
let matches = clap::App::new(env!("CARGO_PKG_NAME"))
|
tracing_subscriber::fmt::init();
|
||||||
.version(env!("CARGO_PKG_VERSION"))
|
let cli = Cli::parse();
|
||||||
.subcommand(clap::SubCommand::with_name("authorize").about("Authorize the twitter application to access a user's account by popping open a web browser and returning the credentials once authorized.")).get_matches();
|
match &cli.command {
|
||||||
match matches.subcommand() {
|
Commands::Authorize => do_authorize(),
|
||||||
("authorize", _) => do_authorize(),
|
Commands::ScrapeCategory => do_scrape_category(),
|
||||||
_ => run_bot(),
|
Commands::ScrapeWeb => do_scrape_web(),
|
||||||
|
Commands::ListTweets => do_list_tweets(),
|
||||||
|
Commands::Whoami => do_whoami(),
|
||||||
|
Commands::RunBot { dry_run } => run_bot(*dry_run),
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
fn do_authorize() -> StdError<()> {
|
fn do_whoami() -> StdError<()> {
|
||||||
println!("Authorizing you lol!");
|
let user_token = user_token_from_env();
|
||||||
|
|
||||||
// Oauth1 leg 1
|
|
||||||
let res = twitter_api(
|
|
||||||
TwitterEndpoint("oauth/request_token").try_into()?,
|
|
||||||
None,
|
|
||||||
APIAction::Post(PostData::Empty),
|
|
||||||
&[("oauth_callback", CB_URL)],
|
|
||||||
)?
|
|
||||||
.text()?;
|
|
||||||
|
|
||||||
let returned_params: HashMap<&str, &str> = res
|
|
||||||
.split("&")
|
|
||||||
.map(|s| s.split("=").collect_tuple())
|
|
||||||
.collect::<Option<_>>()
|
|
||||||
.ok_or("Unexpected oauth step 1 response")?;
|
|
||||||
|
|
||||||
// Oauth1 leg 2
|
|
||||||
let user_url = reqwest::Url::parse_with_params(
|
|
||||||
"https://api.twitter.com/oauth/authenticate",
|
|
||||||
[("oauth_token", returned_params["oauth_token"])],
|
|
||||||
)?;
|
|
||||||
println!("Plz do the thing in the browser");
|
|
||||||
webbrowser::open(user_url.as_str())?;
|
|
||||||
|
|
||||||
let listener = std::net::TcpListener::bind("127.0.0.1:6969")?;
|
|
||||||
let mut stream = listener.incoming().next().ok_or("Error getting stream")??;
|
|
||||||
let mut buf = [0u8; 4096];
|
|
||||||
stream.read(&mut buf[..])?;
|
|
||||||
|
|
||||||
let target = std::str::from_utf8(
|
|
||||||
buf.split(|c| *c == b' ')
|
|
||||||
.skip(1)
|
|
||||||
.next()
|
|
||||||
.ok_or("No target found")?,
|
|
||||||
)?;
|
|
||||||
let oauth_verifier = reqwest::Url::parse("https://example.net/")?
|
|
||||||
.join(target.into())?
|
|
||||||
.query_pairs()
|
|
||||||
.find_map(|(k, v)| {
|
|
||||||
if k == "oauth_verifier" {
|
|
||||||
Some(v.into_owned())
|
|
||||||
} else {
|
|
||||||
None
|
|
||||||
}
|
|
||||||
})
|
|
||||||
.ok_or("no oauth_verifier in response")?;
|
|
||||||
stream.write(b"HTTP/1.1 200 OK\r\n\r\nThanks lmao\r\n")?;
|
|
||||||
stream.shutdown(std::net::Shutdown::Read)?;
|
|
||||||
|
|
||||||
// Oauth1 leg 3
|
|
||||||
let res = twitter_api(
|
|
||||||
TwitterEndpoint("oauth/access_token").try_into()?,
|
|
||||||
None,
|
|
||||||
APIAction::Post(PostData::Data(&[("oauth_verifier", &oauth_verifier)])),
|
|
||||||
&[("oauth_token", returned_params["oauth_token"])],
|
|
||||||
)?
|
|
||||||
.text()?;
|
|
||||||
let returned_params: HashMap<&str, &str> = res
|
|
||||||
.split("&")
|
|
||||||
.map(|s| s.split("=").collect_tuple())
|
|
||||||
.collect::<Option<_>>()
|
|
||||||
.ok_or("Unexpected oauth step 3 response")?;
|
|
||||||
|
|
||||||
println!(
|
|
||||||
"Authorized for {}.\nRun with {}={} {}={}",
|
|
||||||
returned_params["screen_name"],
|
|
||||||
USER_TOKEN_ENV_VAR,
|
|
||||||
returned_params["oauth_token"],
|
|
||||||
USER_SECRET_ENV_VAR,
|
|
||||||
returned_params["oauth_token_secret"]
|
|
||||||
);
|
|
||||||
|
|
||||||
Ok(())
|
|
||||||
}
|
|
||||||
|
|
||||||
fn upload_image(user_token: &oauth1::Token, img: Cow<'static, [u8]>) -> StdError<u64> {
|
|
||||||
let form = reqwest::blocking::multipart::Form::new()
|
|
||||||
.part("media", reqwest::blocking::multipart::Part::bytes(img));
|
|
||||||
let res: serde_json::Value = twitter_api(
|
|
||||||
"https://upload.twitter.com/1.1/media/upload.json".try_into()?,
|
|
||||||
Some(&user_token),
|
|
||||||
APIAction::Post(PostData::Multipart(form)),
|
|
||||||
&[],
|
|
||||||
)?
|
|
||||||
.json()?;
|
|
||||||
Ok(res["media_id"].as_u64().ok_or("media_id not u64!")?)
|
|
||||||
}
|
|
||||||
|
|
||||||
fn run_bot() -> StdError<()> {
|
|
||||||
let user_token = oauth1::Token::new(
|
|
||||||
std::env::var(USER_TOKEN_ENV_VAR)?,
|
|
||||||
std::env::var(USER_SECRET_ENV_VAR)?,
|
|
||||||
);
|
|
||||||
let args: Vec<String> = std::env::args().collect();
|
|
||||||
if args.len() < 2 {
|
|
||||||
println!("usage: ./thing out.png");
|
|
||||||
}
|
|
||||||
// Parse CSS selectors to scrape elements
|
|
||||||
let gallerybox_sel = scraper::Selector::parse(".mw-body-content li.gallerybox")
|
|
||||||
.map_err(|e| format!("{:?}", e))?;
|
|
||||||
let link_sel = scraper::Selector::parse("a.image").map_err(|e| format!("{:?}", e))?;
|
|
||||||
let title_sel = scraper::Selector::parse(".gallerytext p").map_err(|e| format!("{:?}", e))?;
|
|
||||||
let original_sel = scraper::Selector::parse(".fullMedia a").map_err(|e| format!("{:?}", e))?;
|
|
||||||
|
|
||||||
// Fetch stuff!
|
|
||||||
let client = reqwest::blocking::Client::builder()
|
|
||||||
.user_agent(APP_USER_AGENT)
|
|
||||||
.build()?;
|
|
||||||
println!("Fetching main page");
|
|
||||||
let txt = client
|
|
||||||
.get("https://en.wikipedia.org/wiki/ISO_7010")
|
|
||||||
.send()?
|
|
||||||
.text()?;
|
|
||||||
let page = scraper::Html::parse_document(txt.as_str());
|
|
||||||
let things = page
|
|
||||||
.select(&gallerybox_sel)
|
|
||||||
.map(|a| {
|
|
||||||
let link = a
|
|
||||||
.select(&link_sel)
|
|
||||||
.next()
|
|
||||||
.unwrap()
|
|
||||||
.value()
|
|
||||||
.attr("href")
|
|
||||||
.unwrap();
|
|
||||||
let title = a
|
|
||||||
.select(&title_sel)
|
|
||||||
.next()
|
|
||||||
.unwrap()
|
|
||||||
.text()
|
|
||||||
.collect::<String>()
|
|
||||||
.trim()
|
|
||||||
.to_owned();
|
|
||||||
(title, link)
|
|
||||||
})
|
|
||||||
.collect::<Vec<(String, &str)>>();
|
|
||||||
// Pick a random entry and fetch the original file
|
|
||||||
|
|
||||||
let (title, link) = things
|
|
||||||
.choose(&mut rand::thread_rng())
|
|
||||||
.ok_or_else(|| "got no images m8")?;
|
|
||||||
println!("Fetching image page");
|
|
||||||
let media_page = client
|
|
||||||
.get(format!("https://en.wikipedia.org{}", link))
|
|
||||||
.send()?
|
|
||||||
.text()?;
|
|
||||||
let page = scraper::Html::parse_document(media_page.as_str());
|
|
||||||
let link = page
|
|
||||||
.select(&original_sel)
|
|
||||||
.next()
|
|
||||||
.unwrap()
|
|
||||||
.value()
|
|
||||||
.attr("href")
|
|
||||||
.unwrap();
|
|
||||||
let svg = client.get(format!("https:{}", link)).send()?.bytes()?;
|
|
||||||
let png_data = render_svg(&svg)?;
|
|
||||||
|
|
||||||
let user: serde_json::Value = twitter_api(
|
let user: serde_json::Value = twitter_api(
|
||||||
TwitterEndpoint("1.1/account/verify_credentials.json").try_into()?,
|
TwitterEndpoint::VerifyCredentials.try_into()?,
|
||||||
Some(&user_token),
|
Some(&user_token),
|
||||||
APIAction::Get,
|
APIAction::Get,
|
||||||
&[],
|
&[],
|
||||||
)?
|
)?
|
||||||
.json()?;
|
.json()?;
|
||||||
println!(
|
println!("User @{}, (id: {})", user["screen_name"], user["id"]);
|
||||||
"Tweeting for user @{}, (id: {})",
|
Ok(())
|
||||||
user["screen_name"], user["id"]
|
}
|
||||||
);
|
|
||||||
|
fn do_list_tweets() -> StdError<()> {
|
||||||
println!("Uploading image...");
|
let user_token = user_token_from_env();
|
||||||
let img_id = upload_image(&user_token, Cow::from(png_data))?;
|
|
||||||
let tweet = title;
|
let user = twitter_api(
|
||||||
println!("Sending tweet...");
|
TwitterEndpoint::VerifyCredentials.try_into()?,
|
||||||
twitter_api(
|
Some(&user_token),
|
||||||
TwitterEndpoint("1.1/statuses/update.json").try_into()?,
|
APIAction::Get,
|
||||||
Some(&user_token),
|
&[],
|
||||||
APIAction::Post(PostData::Data(&[
|
)?
|
||||||
("media_ids", &img_id.to_string()),
|
.json::<serde_json::Value>()?;
|
||||||
("status", tweet),
|
|
||||||
])),
|
let id = user["id"].as_u64().unwrap();
|
||||||
&[],
|
|
||||||
)?;
|
let timeline: serde_json::Value = twitter_api(
|
||||||
|
reqwest::Url::parse_with_params(
|
||||||
|
&TwitterEndpoint::UserTimeline.to_string(),
|
||||||
|
[
|
||||||
|
("count", "200"),
|
||||||
|
("exclude_replies", "true"),
|
||||||
|
("include_retweets", "false"),
|
||||||
|
("trim_user", "true"),
|
||||||
|
("user_id", id.to_string().as_ref()),
|
||||||
|
],
|
||||||
|
)?,
|
||||||
|
Some(&user_token),
|
||||||
|
APIAction::Get,
|
||||||
|
&[],
|
||||||
|
)?
|
||||||
|
.json()?;
|
||||||
|
for tweet in timeline.as_array().unwrap() {
|
||||||
|
let tweet = tweet.as_object().unwrap();
|
||||||
|
println!("{}, \"{}\"", tweet["id"], tweet["text"]);
|
||||||
|
}
|
||||||
|
Ok(())
|
||||||
|
}
|
||||||
|
|
||||||
|
fn do_scrape_category() -> StdError<()> {
|
||||||
|
let mut files = get_files_in_category("Category:ISO_7010_safety_signs_(vector_drawings)")?;
|
||||||
|
files.sort();
|
||||||
|
for f in files {
|
||||||
|
println!("{}", f);
|
||||||
|
}
|
||||||
|
|
||||||
|
Ok(())
|
||||||
|
}
|
||||||
|
|
||||||
|
fn do_scrape_web() -> StdError<()> {
|
||||||
|
let mut files: Vec<_> = scrape_web()?.into_iter().map(|(_, file)| file).collect();
|
||||||
|
files.sort();
|
||||||
|
for f in files {
|
||||||
|
println!("{}", f);
|
||||||
|
}
|
||||||
|
|
||||||
|
Ok(())
|
||||||
|
}
|
||||||
|
|
||||||
|
fn get_client(headers: Option<reqwest::header::HeaderMap>) -> StdError<reqwest::blocking::Client> {
|
||||||
|
let mut c = reqwest::blocking::Client::builder().user_agent(APP_USER_AGENT);
|
||||||
|
if let Some(headers) = headers {
|
||||||
|
c = c.default_headers(headers);
|
||||||
|
}
|
||||||
|
Ok(c.build()?)
|
||||||
|
}
|
||||||
|
|
||||||
|
fn run_bot(dry_run: bool) -> StdError<()> {
|
||||||
|
let all = scrape_web()?;
|
||||||
|
let (title, filename) = all
|
||||||
|
.choose(&mut rand::thread_rng())
|
||||||
|
.ok_or("got no images m8")?;
|
||||||
|
event!(Level::INFO, title, filename, "Picked random thing");
|
||||||
|
let client = get_client(None)?;
|
||||||
|
event!(Level::INFO, "Fetching metadata...");
|
||||||
|
// TODO: could crash, probably doesn't matter
|
||||||
|
let meta = get_file_metadata(vec![filename])?.remove(0);
|
||||||
|
event!(Level::INFO, %meta, "Got metadata");
|
||||||
|
event!(Level::INFO, url = meta.url.to_string(), "Fetching image");
|
||||||
|
let svg = client.get(meta.url).send()?.bytes()?;
|
||||||
|
|
||||||
|
let text = format!(
|
||||||
|
"{}\n\nImage source: {}\nAuthor: Wikimedia Commons user {}\n{}{}",
|
||||||
|
title,
|
||||||
|
meta.html_url,
|
||||||
|
meta.author,
|
||||||
|
meta.license_short_name,
|
||||||
|
meta.license_url
|
||||||
|
.map_or("".to_owned(), |u| format!(" ({})", u))
|
||||||
|
);
|
||||||
|
|
||||||
|
if !dry_run {
|
||||||
|
// Render the image nice and big for twitter
|
||||||
|
let img = render_svg(&svg, 1000, false)?;
|
||||||
|
let mut buf = Cursor::new(Vec::new());
|
||||||
|
img.write_to(&mut buf, image::ImageFormat::Png)?;
|
||||||
|
tweet(&text, Some(buf.into_inner().into()))?;
|
||||||
|
} else {
|
||||||
|
// Render the image smaller for output to terminal
|
||||||
|
let img = render_svg(&svg, 128, true)?;
|
||||||
|
println!("Dry run - would tweet:\n \"{}\"", text);
|
||||||
|
viuer::print(
|
||||||
|
&img,
|
||||||
|
&viuer::Config {
|
||||||
|
absolute_offset: false,
|
||||||
|
width: Some(32),
|
||||||
|
..Default::default()
|
||||||
|
},
|
||||||
|
)?;
|
||||||
|
}
|
||||||
|
|
||||||
Ok(())
|
Ok(())
|
||||||
}
|
}
|
||||||
|
|
|
||||||
|
|
@ -0,0 +1,276 @@
|
||||||
|
use crate::{get_client, StdError};
|
||||||
|
use itertools::Itertools;
|
||||||
|
use std::borrow::Cow;
|
||||||
|
use std::convert::TryInto;
|
||||||
|
use std::fmt;
|
||||||
|
use std::io::prelude::*;
|
||||||
|
use std::{collections::HashMap, io::Write};
|
||||||
|
use tracing::{event, instrument, Level};
|
||||||
|
|
||||||
|
const APP_TOKEN_ENV_VAR: &str = "TWITTER_APP_TOKEN";
|
||||||
|
const APP_SECRET_ENV_VAR: &str = "TWITTER_APP_SECRET";
|
||||||
|
const USER_TOKEN_ENV_VAR: &str = "TWITTER_USER_TOKEN";
|
||||||
|
const USER_SECRET_ENV_VAR: &str = "TWITTER_USER_SECRET";
|
||||||
|
|
||||||
|
static CB_URL: &str = "http://localhost:6969/cb";
|
||||||
|
|
||||||
|
pub fn user_token_from_env() -> oauth1::Token<'static> {
|
||||||
|
oauth1::Token::new(
|
||||||
|
std::env::var(USER_TOKEN_ENV_VAR).expect("No user token env var"),
|
||||||
|
std::env::var(USER_SECRET_ENV_VAR).expect("No user secret env var"),
|
||||||
|
)
|
||||||
|
}
|
||||||
|
|
||||||
|
pub enum TwitterEndpoint {
|
||||||
|
OauthRequestToken,
|
||||||
|
OauthAccessToken,
|
||||||
|
OauthAuthenticate,
|
||||||
|
UpdateStatus,
|
||||||
|
UserTimeline,
|
||||||
|
VerifyCredentials,
|
||||||
|
}
|
||||||
|
|
||||||
|
impl std::fmt::Display for TwitterEndpoint {
|
||||||
|
fn fmt(&self, f: &mut std::fmt::Formatter) -> Result<(), std::fmt::Error> {
|
||||||
|
const BASE: &str = "https://api.twitter.com";
|
||||||
|
let endpoint = match self {
|
||||||
|
Self::OauthAuthenticate => "oauth/authenticate",
|
||||||
|
Self::OauthRequestToken => "oauth/request_token",
|
||||||
|
Self::OauthAccessToken => "oauth/access_token",
|
||||||
|
Self::UpdateStatus => "1.1/statuses/update.json",
|
||||||
|
Self::UserTimeline => "1.1/statuses/user_timeline.json",
|
||||||
|
Self::VerifyCredentials => "1.1/account/verify_credentials.json",
|
||||||
|
};
|
||||||
|
write!(f, "{}/{}", BASE, endpoint)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
impl TryInto<reqwest::Url> for TwitterEndpoint {
|
||||||
|
type Error = url::ParseError;
|
||||||
|
fn try_into(self) -> Result<reqwest::Url, Self::Error> {
|
||||||
|
reqwest::Url::parse(&self.to_string())
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
pub enum PostData<'a> {
|
||||||
|
Empty,
|
||||||
|
Multipart(reqwest::blocking::multipart::Form),
|
||||||
|
Data(&'a [(&'a str, Cow<'a, str>)]),
|
||||||
|
}
|
||||||
|
|
||||||
|
impl fmt::Debug for PostData<'_> {
|
||||||
|
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
|
||||||
|
write!(
|
||||||
|
f,
|
||||||
|
"{}",
|
||||||
|
match self {
|
||||||
|
Self::Empty => "Empty",
|
||||||
|
Self::Multipart(_) => "Multipart",
|
||||||
|
Self::Data(_) => "Data",
|
||||||
|
}
|
||||||
|
)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
#[derive(Debug)]
|
||||||
|
pub enum APIAction<'a> {
|
||||||
|
Get,
|
||||||
|
Post(PostData<'a>),
|
||||||
|
}
|
||||||
|
|
||||||
|
impl APIAction<'_> {
|
||||||
|
pub fn get_verb(&self) -> &'static str {
|
||||||
|
match self {
|
||||||
|
Self::Get => "GET",
|
||||||
|
Self::Post(_) => "POST",
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
// Make an authed twitter API request
|
||||||
|
#[instrument(skip(user_token), fields(url=url.to_string()))]
|
||||||
|
pub fn twitter_api<'a>(
|
||||||
|
url: reqwest::Url,
|
||||||
|
user_token: Option<&oauth1::Token>,
|
||||||
|
action: APIAction,
|
||||||
|
extra_oauth_params: &[(&str, &str)],
|
||||||
|
) -> StdError<reqwest::blocking::Response> {
|
||||||
|
let consumer_token = oauth1::Token::new(
|
||||||
|
std::env::var(APP_TOKEN_ENV_VAR)?,
|
||||||
|
std::env::var(APP_SECRET_ENV_VAR)?,
|
||||||
|
);
|
||||||
|
let mut headers = reqwest::header::HeaderMap::new();
|
||||||
|
let mut params: HashMap<&str, Cow<str>> = extra_oauth_params
|
||||||
|
.iter()
|
||||||
|
.cloned()
|
||||||
|
.map(|(x, y)| (x, y.into()))
|
||||||
|
.collect();
|
||||||
|
|
||||||
|
// Copy all our query parameters and add them to the list of params for oauth1 signature
|
||||||
|
// generation.
|
||||||
|
// This is a bit awkward, as params is a map from &str to Cow<str> but query_pairs() returns
|
||||||
|
// (Cow<str>, Cow<str>) tuples. So we call into_owned to make (String, String) tuples, and then
|
||||||
|
// borrow from there. If there's a way to do it without copying, I couldn't find it.
|
||||||
|
let pairs: Vec<_> = url.query_pairs().into_owned().collect();
|
||||||
|
for (k, v) in &pairs {
|
||||||
|
params.insert(k, Cow::Borrowed(v));
|
||||||
|
}
|
||||||
|
|
||||||
|
// If the request is a key/value form post, we also need to include those parameters when
|
||||||
|
// generating the signature.
|
||||||
|
if let APIAction::Post(PostData::Data(d)) = action {
|
||||||
|
params.extend(d.to_owned())
|
||||||
|
}
|
||||||
|
|
||||||
|
// The url used to generate the signature must not include the query params
|
||||||
|
let mut url_sans_query = url.clone();
|
||||||
|
url_sans_query.set_query(None);
|
||||||
|
headers.insert(
|
||||||
|
reqwest::header::AUTHORIZATION,
|
||||||
|
reqwest::header::HeaderValue::from_str(&oauth1::authorize(
|
||||||
|
action.get_verb(),
|
||||||
|
url_sans_query.as_str(),
|
||||||
|
&consumer_token,
|
||||||
|
user_token,
|
||||||
|
Some(params),
|
||||||
|
))?,
|
||||||
|
);
|
||||||
|
let client = get_client(Some(headers))?;
|
||||||
|
let req = match action {
|
||||||
|
APIAction::Get => client.get(url),
|
||||||
|
APIAction::Post(PostData::Empty) => client.post(url),
|
||||||
|
APIAction::Post(PostData::Data(data)) => client.post(url).form(data),
|
||||||
|
APIAction::Post(PostData::Multipart(form)) => client.post(url).multipart(form),
|
||||||
|
};
|
||||||
|
event!(Level::INFO, "Sending request");
|
||||||
|
let res = req.send()?;
|
||||||
|
if !res.status().is_success() {
|
||||||
|
return Err(format!(
|
||||||
|
"Got non-200 response: status {}, {}",
|
||||||
|
res.status(),
|
||||||
|
res.text()?
|
||||||
|
)
|
||||||
|
.into());
|
||||||
|
}
|
||||||
|
Ok(res)
|
||||||
|
}
|
||||||
|
|
||||||
|
pub fn do_authorize() -> StdError<()> {
|
||||||
|
println!("Authorizing you lol!");
|
||||||
|
|
||||||
|
// Oauth1 leg 1
|
||||||
|
let res = twitter_api(
|
||||||
|
TwitterEndpoint::OauthRequestToken.try_into()?,
|
||||||
|
None,
|
||||||
|
APIAction::Post(PostData::Empty),
|
||||||
|
&[("oauth_callback", CB_URL)],
|
||||||
|
)?
|
||||||
|
.text()?;
|
||||||
|
|
||||||
|
let returned_params: HashMap<&str, &str> = res
|
||||||
|
.split('&')
|
||||||
|
.map(|s| s.split('=').collect_tuple())
|
||||||
|
.collect::<Option<_>>()
|
||||||
|
.ok_or("Unexpected oauth step 1 response")?;
|
||||||
|
|
||||||
|
// Oauth1 leg 2
|
||||||
|
let user_url = reqwest::Url::parse_with_params(
|
||||||
|
&TwitterEndpoint::OauthAuthenticate.to_string(),
|
||||||
|
[("oauth_token", returned_params["oauth_token"])],
|
||||||
|
)?;
|
||||||
|
println!("Plz do the thing in the browser");
|
||||||
|
webbrowser::open(user_url.as_str())?;
|
||||||
|
|
||||||
|
let listener = std::net::TcpListener::bind("127.0.0.1:6969")?;
|
||||||
|
let mut stream = listener.incoming().next().ok_or("Error getting stream")??;
|
||||||
|
let mut buf = [0u8; 4096];
|
||||||
|
stream.read(&mut buf[..])?;
|
||||||
|
|
||||||
|
let target = std::str::from_utf8(buf.split(|c| *c == b' ').nth(1).ok_or("No target found")?)?;
|
||||||
|
let oauth_verifier = reqwest::Url::parse("https://example.net/")?
|
||||||
|
.join(target)?
|
||||||
|
.query_pairs()
|
||||||
|
.find_map(|(k, v)| {
|
||||||
|
if k == "oauth_verifier" {
|
||||||
|
Some(v.into_owned())
|
||||||
|
} else {
|
||||||
|
None
|
||||||
|
}
|
||||||
|
})
|
||||||
|
.ok_or("no oauth_verifier in response")?;
|
||||||
|
stream.write_all(b"HTTP/1.1 200 OK\r\n\r\nThanks lmao\r\n")?;
|
||||||
|
stream.shutdown(std::net::Shutdown::Read)?;
|
||||||
|
|
||||||
|
// Oauth1 leg 3
|
||||||
|
let res = twitter_api(
|
||||||
|
TwitterEndpoint::OauthAccessToken.try_into()?,
|
||||||
|
None,
|
||||||
|
APIAction::Post(PostData::Data(&[(
|
||||||
|
"oauth_verifier",
|
||||||
|
Cow::Owned(oauth_verifier),
|
||||||
|
)])),
|
||||||
|
&[("oauth_token", returned_params["oauth_token"])],
|
||||||
|
)?
|
||||||
|
.text()?;
|
||||||
|
let returned_params: HashMap<&str, &str> = res
|
||||||
|
.split('&')
|
||||||
|
.map(|s| s.split('=').collect_tuple())
|
||||||
|
.collect::<Option<_>>()
|
||||||
|
.ok_or("Unexpected oauth step 3 response")?;
|
||||||
|
|
||||||
|
println!(
|
||||||
|
"Authorized for {}.\nRun with {}={} {}={}",
|
||||||
|
returned_params["screen_name"],
|
||||||
|
USER_TOKEN_ENV_VAR,
|
||||||
|
returned_params["oauth_token"],
|
||||||
|
USER_SECRET_ENV_VAR,
|
||||||
|
returned_params["oauth_token_secret"]
|
||||||
|
);
|
||||||
|
|
||||||
|
Ok(())
|
||||||
|
}
|
||||||
|
|
||||||
|
fn upload_image(user_token: &oauth1::Token, img: Cow<'static, [u8]>) -> StdError<u64> {
|
||||||
|
let form = reqwest::blocking::multipart::Form::new()
|
||||||
|
.part("media", reqwest::blocking::multipart::Part::bytes(img));
|
||||||
|
let res: serde_json::Value = twitter_api(
|
||||||
|
"https://upload.twitter.com/1.1/media/upload.json".try_into()?,
|
||||||
|
Some(user_token),
|
||||||
|
APIAction::Post(PostData::Multipart(form)),
|
||||||
|
&[],
|
||||||
|
)?
|
||||||
|
.json()?;
|
||||||
|
Ok(res["media_id"].as_u64().ok_or("media_id not u64!")?)
|
||||||
|
}
|
||||||
|
|
||||||
|
pub fn tweet(text: &str, img: Option<Cow<'static, [u8]>>) -> StdError<()> {
|
||||||
|
let user_token = oauth1::Token::new(
|
||||||
|
std::env::var(USER_TOKEN_ENV_VAR)?,
|
||||||
|
std::env::var(USER_SECRET_ENV_VAR)?,
|
||||||
|
);
|
||||||
|
|
||||||
|
let user: serde_json::Value = twitter_api(
|
||||||
|
TwitterEndpoint::VerifyCredentials.try_into()?,
|
||||||
|
Some(&user_token),
|
||||||
|
APIAction::Get,
|
||||||
|
&[],
|
||||||
|
)?
|
||||||
|
.json()?;
|
||||||
|
println!(
|
||||||
|
"Tweeting for user @{}, (id: {})",
|
||||||
|
user["screen_name"], user["id"]
|
||||||
|
);
|
||||||
|
let mut post_data = vec![("status", Cow::Borrowed(text))];
|
||||||
|
if let Some(img) = img {
|
||||||
|
println!("Uploading image...");
|
||||||
|
let img_id = upload_image(&user_token, img)?;
|
||||||
|
post_data.push(("media_ids", Cow::Owned(img_id.to_string())))
|
||||||
|
}
|
||||||
|
event!(Level::INFO, "Sending tweet...");
|
||||||
|
twitter_api(
|
||||||
|
TwitterEndpoint::UpdateStatus.try_into()?,
|
||||||
|
Some(&user_token),
|
||||||
|
APIAction::Post(PostData::Data(&post_data[0..])),
|
||||||
|
&[],
|
||||||
|
)?;
|
||||||
|
Ok(())
|
||||||
|
}
|
||||||
|
|
@ -0,0 +1,204 @@
|
||||||
|
use std::collections::HashMap;
|
||||||
|
|
||||||
|
use crate::{get_client, StdError};
|
||||||
|
use regex::Regex;
|
||||||
|
use serde::Deserialize;
|
||||||
|
use std::fmt::Display;
|
||||||
|
use tracing::{event, instrument, Level};
|
||||||
|
use url::Url;
|
||||||
|
|
||||||
|
// Filter a filename string for filenames
|
||||||
|
fn filter_filename(filename: &str) -> bool {
|
||||||
|
let re = Regex::new("ISO.7010.[EWMPF][0-9]{3}.*").unwrap();
|
||||||
|
re.is_match(filename)
|
||||||
|
}
|
||||||
|
|
||||||
|
// Scrape all images from the wikipedia page, returning a vec of title, filename pairs
|
||||||
|
pub fn scrape_web() -> StdError<Vec<(String, String)>> {
|
||||||
|
event!(Level::INFO, "Scraping the wikipedia page for things");
|
||||||
|
// Parse CSS selectors to scrape elements
|
||||||
|
let gallerybox_sel = scraper::Selector::parse(".mw-body-content li.gallerybox")
|
||||||
|
.map_err(|e| format!("{:?}", e))?;
|
||||||
|
let link_sel = scraper::Selector::parse("a.image").map_err(|e| format!("{:?}", e))?;
|
||||||
|
let title_sel = scraper::Selector::parse(".gallerytext p").map_err(|e| format!("{:?}", e))?;
|
||||||
|
|
||||||
|
// Fetch stuff!
|
||||||
|
let client = get_client(None)?;
|
||||||
|
event!(Level::INFO, "Fetching wiki page");
|
||||||
|
let txt = client
|
||||||
|
.get("https://en.wikipedia.org/wiki/ISO_7010")
|
||||||
|
.send()?
|
||||||
|
.text()?;
|
||||||
|
let page = scraper::Html::parse_document(txt.as_str());
|
||||||
|
return Ok(page
|
||||||
|
.select(&gallerybox_sel)
|
||||||
|
.map(|a| {
|
||||||
|
let link = a
|
||||||
|
.select(&link_sel)
|
||||||
|
.next()
|
||||||
|
.unwrap()
|
||||||
|
.value()
|
||||||
|
.attr("href")
|
||||||
|
.unwrap()
|
||||||
|
.to_owned();
|
||||||
|
let title = a
|
||||||
|
.select(&title_sel)
|
||||||
|
.next()
|
||||||
|
.unwrap()
|
||||||
|
.text()
|
||||||
|
.collect::<String>()
|
||||||
|
.trim()
|
||||||
|
.to_owned();
|
||||||
|
(title, link)
|
||||||
|
})
|
||||||
|
// Filter for filenames that look like ISO diagrams
|
||||||
|
.filter(|tup| filter_filename(&tup.1))
|
||||||
|
// Extract the file name only (.e.g `File:ISO_7010_X000.svg`)
|
||||||
|
.filter_map(|(title, link)| {
|
||||||
|
link.split('/')
|
||||||
|
.next_back()
|
||||||
|
.map(|end| (title, end.to_owned()))
|
||||||
|
})
|
||||||
|
.collect::<Vec<_>>());
|
||||||
|
}
|
||||||
|
|
||||||
|
#[instrument]
|
||||||
|
pub fn wiki_query_url(params: Vec<(&str, &str)>) -> StdError<Url> {
|
||||||
|
let mut url = Url::parse("https://commons.wikimedia.org/w/api.php?action=query&format=json")?;
|
||||||
|
url.query_pairs_mut().extend_pairs(params);
|
||||||
|
Ok(url)
|
||||||
|
}
|
||||||
|
// https://commons.wikimedia.org/w/api.php?action=query&format=json&list=categorymembers&cmtitle=Category:ISO_7010_safety_signs_(vector_drawings)&cmlimit=2
|
||||||
|
|
||||||
|
#[instrument]
|
||||||
|
pub fn get_files_in_category(category: &str) -> StdError<Vec<String>> {
|
||||||
|
let client = get_client(None)?;
|
||||||
|
let url = wiki_query_url(
|
||||||
|
[
|
||||||
|
("list", "categorymembers"),
|
||||||
|
("cmtitle", category),
|
||||||
|
("cmtype", "file"),
|
||||||
|
("cmlimit", "max"),
|
||||||
|
]
|
||||||
|
.into(),
|
||||||
|
)?;
|
||||||
|
let data = client.get(url).send()?.json::<serde_json::Value>()?;
|
||||||
|
if data.get("continue").is_some() {
|
||||||
|
// There are more results than are contained in one response, so now you need to implement
|
||||||
|
// pagination. Have fun!
|
||||||
|
panic!("Wikimedia query result is paginated!");
|
||||||
|
}
|
||||||
|
Ok(data["query"]["categorymembers"]
|
||||||
|
.as_array()
|
||||||
|
.unwrap()
|
||||||
|
.iter()
|
||||||
|
.filter_map(|m| Some(m.as_object().unwrap()["title"].as_str()?.replace(' ', "_")))
|
||||||
|
.collect())
|
||||||
|
}
|
||||||
|
|
||||||
|
#[derive(Debug)]
|
||||||
|
pub struct FileMeta {
|
||||||
|
pub url: url::Url,
|
||||||
|
pub name: String,
|
||||||
|
pub html_url: url::Url,
|
||||||
|
pub author: String,
|
||||||
|
pub attribution_required: String,
|
||||||
|
pub license_short_name: String,
|
||||||
|
pub license_url: Option<url::Url>,
|
||||||
|
}
|
||||||
|
|
||||||
|
impl Display for FileMeta {
|
||||||
|
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
|
||||||
|
write!(
|
||||||
|
f,
|
||||||
|
"FileMeta{{url: {}, name: {}, html_url: {}, author: {}, attribution_required: {}, license_short_name: {}, license_url: {}}}",
|
||||||
|
self.url, self.name, self.html_url, self.author, self.attribution_required, self.license_short_name,
|
||||||
|
self.license_url.clone().map_or("None".to_owned(), |u| u.to_string()) // Ew.
|
||||||
|
)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Partial representation of the data returned from a MediaWiki imageinfo query
|
||||||
|
#[derive(Deserialize)]
|
||||||
|
struct Query {
|
||||||
|
query: QueryInner,
|
||||||
|
}
|
||||||
|
|
||||||
|
#[derive(Deserialize, Debug)]
|
||||||
|
struct QueryInner {
|
||||||
|
pages: HashMap<String, Page>,
|
||||||
|
}
|
||||||
|
|
||||||
|
#[derive(Deserialize, Debug)]
|
||||||
|
struct Page {
|
||||||
|
imageinfo: Vec<ImageInfo>,
|
||||||
|
}
|
||||||
|
|
||||||
|
#[derive(Deserialize, Debug)]
|
||||||
|
struct ImageInfo {
|
||||||
|
user: String,
|
||||||
|
url: url::Url,
|
||||||
|
descriptionurl: url::Url,
|
||||||
|
extmetadata: ExtMeta,
|
||||||
|
}
|
||||||
|
|
||||||
|
#[derive(Deserialize, Debug)]
|
||||||
|
#[serde(rename_all = "PascalCase")]
|
||||||
|
struct ExtMeta {
|
||||||
|
object_name: ExtMetaItem<String>,
|
||||||
|
license_short_name: ExtMetaItem<String>,
|
||||||
|
attribution_required: ExtMetaItem<String>,
|
||||||
|
license_url: Option<ExtMetaItem<url::Url>>,
|
||||||
|
}
|
||||||
|
|
||||||
|
#[derive(Deserialize, Debug, Clone)]
|
||||||
|
struct ExtMetaItem<T> {
|
||||||
|
value: T,
|
||||||
|
}
|
||||||
|
|
||||||
|
pub fn get_file_metadata(files: Vec<&str>) -> StdError<Vec<FileMeta>> {
|
||||||
|
let client = get_client(None)?;
|
||||||
|
// Api only lets us do 50 files in one request
|
||||||
|
Ok(files
|
||||||
|
.chunks(50)
|
||||||
|
.flat_map(|files_chunk| {
|
||||||
|
let url = wiki_query_url(
|
||||||
|
[
|
||||||
|
("titles", files_chunk.join("|").as_ref()),
|
||||||
|
("prop", "imageinfo"),
|
||||||
|
(
|
||||||
|
"iiprop",
|
||||||
|
"timestamp|url|size|mime|mediatype|extmetadata|user",
|
||||||
|
),
|
||||||
|
// Get metadata for as many revisions of the file as we are allowed. We're unlikely to encounter a file with >500 revisions.
|
||||||
|
("iilimit", "500"),
|
||||||
|
(
|
||||||
|
"iiextmetadatafilter",
|
||||||
|
"ObjectName|LicenseShortName|AttributionRequired|LicenseUrl",
|
||||||
|
),
|
||||||
|
]
|
||||||
|
.into(),
|
||||||
|
)
|
||||||
|
.unwrap();
|
||||||
|
let data = client.get(url).send().unwrap().json::<Query>().unwrap();
|
||||||
|
|
||||||
|
data.query
|
||||||
|
.pages
|
||||||
|
.values()
|
||||||
|
.map(|page| {
|
||||||
|
let latest = page.imageinfo.first().unwrap();
|
||||||
|
let oldest = page.imageinfo.last().unwrap();
|
||||||
|
FileMeta {
|
||||||
|
url: latest.url.clone(),
|
||||||
|
name: latest.extmetadata.object_name.value.clone(),
|
||||||
|
html_url: latest.descriptionurl.clone(),
|
||||||
|
author: oldest.user.clone(),
|
||||||
|
license_short_name: latest.extmetadata.license_short_name.value.clone(),
|
||||||
|
license_url: latest.extmetadata.license_url.clone().map(|i| i.value),
|
||||||
|
attribution_required: latest.extmetadata.attribution_required.value.clone(),
|
||||||
|
}
|
||||||
|
})
|
||||||
|
.collect::<Vec<_>>()
|
||||||
|
})
|
||||||
|
.collect())
|
||||||
|
}
|
||||||
Loading…
Reference in New Issue