Add attribution of tweets, scraping cmds and dry run

Cargo.toml

@@ -1,22 +1,28 @@
 [package]
 name = "iso7010_a_day"
+description = "A silly twitter bot"
 version = "0.1.0"
 edition = "2018"
 
 [dependencies]
 reqwest = { version = "0.11", features = ["blocking", "json", "multipart"]}
 serde_json = "*"
-serde = "*"
+serde = { version = "1", features = ["derive"] }
 scraper = "*"
 rand = "*"
 resvg = "*"
 tiny-skia = "*"
 usvg = "*"
 oauth1 = "*"
-clap = "*"
+clap = { version = "*", features = ["derive"] }
 webbrowser = "*"
 itertools = "*"
-url = "*"
+tracing = "0.1.36"
+tracing-subscriber = "0.3.15"
+regex = "1.6.0"
+image = "0.24.3"
+viuer = "0.6.1"
+url = { version = "2.3.1", features = ["serde"] }
 
 [build-dependencies]
 toml = "*"

src/main.rs

@@ -1,15 +1,14 @@
-use itertools::Itertools;
 use rand::seq::SliceRandom;
-use std::borrow::Cow;
 use std::convert::TryInto;
-use std::io::prelude::*;
+use std::io::Cursor;
-use std::{collections::HashMap, io::Write};
+use tracing::{event, Level};
-
+mod twitter;
-const APP_TOKEN_ENV_VAR: &str = "TWITTER_APP_TOKEN";
+mod wiki;
-const APP_SECRET_ENV_VAR: &str = "TWITTER_APP_SECRET";
+use clap::{Parser, Subcommand};
-const USER_TOKEN_ENV_VAR: &str = "TWITTER_USER_TOKEN";
+use image::{DynamicImage, RgbaImage};
-const USER_SECRET_ENV_VAR: &str = "TWITTER_USER_SECRET";
+use tiny_skia::{Paint, PathBuilder, Pixmap, PixmapPaint, Stroke, Transform};
-const IMG_HEIGHT: u32 = 1000;
+use twitter::*;
+use wiki::*;
 
 static APP_USER_AGENT: &str = concat!(
     "bot_",
@@ -20,316 +19,224 @@ static APP_USER_AGENT: &str = concat!(
     "reqwest",
 );
 
-static CB_URL: &str = "http://localhost:6969/cb";
+// Render the raw SVG data to an image
-
+fn render_svg(data: &[u8], height: u32, with_border: bool) -> StdError<DynamicImage> {
-fn render_svg(data: &[u8]) -> Result<Vec<u8>, Box<dyn std::error::Error>> {
     let opt = usvg::Options::default();
-    let rtree = usvg::Tree::from_data(&data, &opt.to_ref()).expect("couldn't parse");
+    let rtree = usvg::Tree::from_data(data, &opt.to_ref()).expect("couldn't parse");
-    let mut pixmap =
+    let svg_size = rtree.svg_node().size;
-        tiny_skia::Pixmap::new(IMG_HEIGHT, IMG_HEIGHT).ok_or("Error creating pixmap")?;
+    // Work out how wide the pixmap of height `height` needs to be to entirely fit the SVG.
-    resvg::render(
+    let pm_width = ((height as f64 / svg_size.height()) * svg_size.width()).ceil() as u32;
-        &rtree,
+    let mut pixmap = Pixmap::new(pm_width, height).ok_or("Error creating pixmap")?;
-        usvg::FitTo::Size(IMG_HEIGHT, IMG_HEIGHT),
+    // Render the svg into a pixmap.
-        pixmap.as_mut(),
+    resvg::render(&rtree, usvg::FitTo::Height(height), pixmap.as_mut())
-    )
+        .ok_or("Error rendering svg")?;
-    .ok_or_else(|| "Error rendering svg")?;
+    // Make a wider pixmap with a 16:9 AR and the same height. This is a blesséd ratio by twitter
-    let mut bigger_pixmap = tiny_skia::Pixmap::new(IMG_HEIGHT / 9 * 16, IMG_HEIGHT)
+    // and means we see the whole image nicely in the timeline with no truncation.
-        .ok_or("Error creating bigger pixmap")?;
+    let mut bigger_pixmap =
+        Pixmap::new(height / 9 * 16, height).ok_or("Error creating bigger pixmap")?;
+    // Then draw our freshly rendered SVG into the middle of the bigger pixmap.
     bigger_pixmap
         .draw_pixmap(
-            (bigger_pixmap.width() / 2 - IMG_HEIGHT / 2)
+            ((bigger_pixmap.width() - pm_width) / 2).try_into().unwrap(),
-                .try_into()
-                .unwrap(),
             0,
             pixmap.as_ref(),
-            &tiny_skia::PixmapPaint::default(),
+            &PixmapPaint::default(),
-            tiny_skia::Transform::identity(),
+            Transform::identity(),
             None,
         )
         .ok_or("Error drawing onto bigger pixmap")?;
-    let png_data = bigger_pixmap.encode_png()?;
+    let (w, h) = (bigger_pixmap.width(), bigger_pixmap.height());
-    Ok(png_data)
+    // Render a red border for debug purposes
-}
+    if with_border {
+        let mut paint = Paint::default();
+        paint.set_color_rgba8(255, 0, 0, 255);
+        let stroke = Stroke {
+            width: 1.0,
+            ..Default::default()
+        };
 
-enum PostData<'a> {
+        let path = {
-    Empty,
+            let mut pb = PathBuilder::new();
-    Multipart(reqwest::blocking::multipart::Form),
+            pb.move_to(0.0, 0.0);
-    Data(&'a [(&'a str, &'a str)]),
+            pb.line_to(0.0, h as f32 - stroke.width);
-}
+            pb.line_to(w as f32, h as f32 - stroke.width);
-
+            pb.line_to(w as f32 - stroke.width, 0.0);
-enum APIAction<'a> {
+            pb.line_to(0.0, 0.0);
-    Get,
+            pb.finish().unwrap()
-    Post(PostData<'a>),
+        };
-}
+        bigger_pixmap.stroke_path(&path, &paint, &stroke, Transform::identity(), None);
-
-struct TwitterEndpoint<'a>(&'a str);
-
-impl TryInto<reqwest::Url> for TwitterEndpoint<'_> {
-    type Error = url::ParseError;
-    fn try_into(self) -> Result<reqwest::Url, Self::Error> {
-        reqwest::Url::parse(&format!("https://api.twitter.com/{}", self.0))
     }
-}
+    let img = RgbaImage::from_raw(
-
+        bigger_pixmap.width(),
-// Make an authed twitter API request
+        bigger_pixmap.height(),
-fn twitter_api<'a>(
+        bigger_pixmap.data().to_vec(),
-    url: reqwest::Url,
+    )
-    user_token: Option<&oauth1::Token>,
+    .ok_or("Error creating image from pixmap")?;
-    action: APIAction,
+    Ok(DynamicImage::ImageRgba8(img))
-    extra_oauth_params: &[(&str, &str)],
-) -> StdError<reqwest::blocking::Response> {
-    let consumer_token = oauth1::Token::new(
-        std::env::var(APP_TOKEN_ENV_VAR)?,
-        std::env::var(APP_SECRET_ENV_VAR)?,
-    );
-    let mut headers = reqwest::header::HeaderMap::new();
-    let mut oauth_params: HashMap<&str, Cow<str>> = extra_oauth_params
-        .iter()
-        .cloned()
-        .map(|(x, y)| (x, y.into()))
-        .collect();
-
-    // If the request is a key/value form post, we need to include those parameters when
-    // generating the signature.
-    match action {
-        APIAction::Post(PostData::Data(d)) => {
-            oauth_params.extend(d.iter().cloned().map(|(x, y)| (x, y.into())))
-        }
-        _ => {}
-    }
-
-    headers.insert(
-        reqwest::header::AUTHORIZATION,
-        reqwest::header::HeaderValue::from_str(&oauth1::authorize(
-            if matches!(action, APIAction::Post(_)) {
-                "POST"
-            } else {
-                "GET"
-            },
-            url.as_str(),
-            &consumer_token,
-            user_token,
-            Some(oauth_params),
-        ))?,
-    );
-
-    let client = reqwest::blocking::Client::builder()
-        .user_agent(APP_USER_AGENT)
-        .default_headers(headers)
-        .build()?;
-    let req = match action {
-        APIAction::Get => client.get(url),
-        APIAction::Post(PostData::Empty) => client.post(url),
-        APIAction::Post(PostData::Data(data)) => client.post(url).form(data),
-        APIAction::Post(PostData::Multipart(form)) => client.post(url).multipart(form),
-    };
-
-    let res = req.send()?;
-    if !res.status().is_success() {
-        return Err(format!(
-            "Got non-200 response: status {}, {}",
-            res.status(),
-            res.text()?
-        )
-        .into());
-    }
-    Ok(res)
 }
 
 type StdError<T> = Result<T, Box<dyn std::error::Error>>;
 
+#[derive(Parser)]
+#[clap(author, version, about)]
+struct Cli {
+    #[clap(subcommand)]
+    command: Commands,
+}
+
+#[derive(Subcommand)]
+enum Commands {
+    /// Authorize the twitter application to acccess a user's account
+    Authorize,
+    /// Scrape images from the category on wikimedia commons
+    ScrapeCategory,
+    /// Scrape images from the iso7010 wikipedia page
+    ScrapeWeb,
+    /// List tweets from the authed user's timeline
+    ListTweets,
+    /// Run the bot - scrape, pick a random entry and tweet it
+    RunBot {
+        #[clap(short, long, action)]
+        dry_run: bool,
+    },
+    /// Print details about the currently authed user
+    Whoami,
+}
+
 fn main() -> StdError<()> {
-    let matches = clap::App::new(env!("CARGO_PKG_NAME"))
+    tracing_subscriber::fmt::init();
-        .version(env!("CARGO_PKG_VERSION"))
+    let cli = Cli::parse();
-        .subcommand(clap::SubCommand::with_name("authorize").about("Authorize the twitter application to access a user's account by popping open a web browser and returning the credentials once authorized.")).get_matches();
+    match &cli.command {
-    match matches.subcommand() {
+        Commands::Authorize => do_authorize(),
-        ("authorize", _) => do_authorize(),
+        Commands::ScrapeCategory => do_scrape_category(),
-        _ => run_bot(),
+        Commands::ScrapeWeb => do_scrape_web(),
+        Commands::ListTweets => do_list_tweets(),
+        Commands::Whoami => do_whoami(),
+        Commands::RunBot { dry_run } => run_bot(*dry_run),
     }
 }
 
-fn do_authorize() -> StdError<()> {
+fn do_whoami() -> StdError<()> {
-    println!("Authorizing you lol!");
+    let user_token = user_token_from_env();
-
-    // Oauth1 leg 1
-    let res = twitter_api(
-        TwitterEndpoint("oauth/request_token").try_into()?,
-        None,
-        APIAction::Post(PostData::Empty),
-        &[("oauth_callback", CB_URL)],
-    )?
-    .text()?;
-
-    let returned_params: HashMap<&str, &str> = res
-        .split("&")
-        .map(|s| s.split("=").collect_tuple())
-        .collect::<Option<_>>()
-        .ok_or("Unexpected oauth step 1 response")?;
-
-    // Oauth1 leg 2
-    let user_url = reqwest::Url::parse_with_params(
-        "https://api.twitter.com/oauth/authenticate",
-        [("oauth_token", returned_params["oauth_token"])],
-    )?;
-    println!("Plz do the thing in the browser");
-    webbrowser::open(user_url.as_str())?;
-
-    let listener = std::net::TcpListener::bind("127.0.0.1:6969")?;
-    let mut stream = listener.incoming().next().ok_or("Error getting stream")??;
-    let mut buf = [0u8; 4096];
-    stream.read(&mut buf[..])?;
-
-    let target = std::str::from_utf8(
-        buf.split(|c| *c == b' ')
-            .skip(1)
-            .next()
-            .ok_or("No target found")?,
-    )?;
-    let oauth_verifier = reqwest::Url::parse("https://example.net/")?
-        .join(target.into())?
-        .query_pairs()
-        .find_map(|(k, v)| {
-            if k == "oauth_verifier" {
-                Some(v.into_owned())
-            } else {
-                None
-            }
-        })
-        .ok_or("no oauth_verifier in response")?;
-    stream.write(b"HTTP/1.1 200 OK\r\n\r\nThanks lmao\r\n")?;
-    stream.shutdown(std::net::Shutdown::Read)?;
-
-    // Oauth1 leg 3
-    let res = twitter_api(
-        TwitterEndpoint("oauth/access_token").try_into()?,
-        None,
-        APIAction::Post(PostData::Data(&[("oauth_verifier", &oauth_verifier)])),
-        &[("oauth_token", returned_params["oauth_token"])],
-    )?
-    .text()?;
-    let returned_params: HashMap<&str, &str> = res
-        .split("&")
-        .map(|s| s.split("=").collect_tuple())
-        .collect::<Option<_>>()
-        .ok_or("Unexpected oauth step 3 response")?;
-
-    println!(
-        "Authorized for {}.\nRun with {}={} {}={}",
-        returned_params["screen_name"],
-        USER_TOKEN_ENV_VAR,
-        returned_params["oauth_token"],
-        USER_SECRET_ENV_VAR,
-        returned_params["oauth_token_secret"]
-    );
-
-    Ok(())
-}
-
-fn upload_image(user_token: &oauth1::Token, img: Cow<'static, [u8]>) -> StdError<u64> {
-    let form = reqwest::blocking::multipart::Form::new()
-        .part("media", reqwest::blocking::multipart::Part::bytes(img));
-    let res: serde_json::Value = twitter_api(
-        "https://upload.twitter.com/1.1/media/upload.json".try_into()?,
-        Some(&user_token),
-        APIAction::Post(PostData::Multipart(form)),
-        &[],
-    )?
-    .json()?;
-    Ok(res["media_id"].as_u64().ok_or("media_id not u64!")?)
-}
-
-fn run_bot() -> StdError<()> {
-    let user_token = oauth1::Token::new(
-        std::env::var(USER_TOKEN_ENV_VAR)?,
-        std::env::var(USER_SECRET_ENV_VAR)?,
-    );
-    let args: Vec<String> = std::env::args().collect();
-    if args.len() < 2 {
-        println!("usage: ./thing out.png");
-    }
-    // Parse CSS selectors to scrape elements
-    let gallerybox_sel = scraper::Selector::parse(".mw-body-content li.gallerybox")
-        .map_err(|e| format!("{:?}", e))?;
-    let link_sel = scraper::Selector::parse("a.image").map_err(|e| format!("{:?}", e))?;
-    let title_sel = scraper::Selector::parse(".gallerytext p").map_err(|e| format!("{:?}", e))?;
-    let original_sel = scraper::Selector::parse(".fullMedia a").map_err(|e| format!("{:?}", e))?;
-
-    // Fetch stuff!
-    let client = reqwest::blocking::Client::builder()
-        .user_agent(APP_USER_AGENT)
-        .build()?;
-    println!("Fetching main page");
-    let txt = client
-        .get("https://en.wikipedia.org/wiki/ISO_7010")
-        .send()?
-        .text()?;
-    let page = scraper::Html::parse_document(txt.as_str());
-    let things = page
-        .select(&gallerybox_sel)
-        .map(|a| {
-            let link = a
-                .select(&link_sel)
-                .next()
-                .unwrap()
-                .value()
-                .attr("href")
-                .unwrap();
-            let title = a
-                .select(&title_sel)
-                .next()
-                .unwrap()
-                .text()
-                .collect::<String>()
-                .trim()
-                .to_owned();
-            (title, link)
-        })
-        .collect::<Vec<(String, &str)>>();
-    // Pick a random entry and fetch the original file
-
-    let (title, link) = things
-        .choose(&mut rand::thread_rng())
-        .ok_or_else(|| "got no images m8")?;
-    println!("Fetching image page");
-    let media_page = client
-        .get(format!("https://en.wikipedia.org{}", link))
-        .send()?
-        .text()?;
-    let page = scraper::Html::parse_document(media_page.as_str());
-    let link = page
-        .select(&original_sel)
-        .next()
-        .unwrap()
-        .value()
-        .attr("href")
-        .unwrap();
-    let svg = client.get(format!("https:{}", link)).send()?.bytes()?;
-    let png_data = render_svg(&svg)?;
 
     let user: serde_json::Value = twitter_api(
-        TwitterEndpoint("1.1/account/verify_credentials.json").try_into()?,
+        TwitterEndpoint::VerifyCredentials.try_into()?,
         Some(&user_token),
         APIAction::Get,
         &[],
     )?
     .json()?;
-    println!(
+    println!("User @{}, (id: {})", user["screen_name"], user["id"]);
-        "Tweeting for user @{}, (id: {})",
+    Ok(())
-        user["screen_name"], user["id"]
+}
-    );
+
-
+fn do_list_tweets() -> StdError<()> {
-    println!("Uploading image...");
+    let user_token = user_token_from_env();
-    let img_id = upload_image(&user_token, Cow::from(png_data))?;
+
-    let tweet = title;
+    let user = twitter_api(
-    println!("Sending tweet...");
+        TwitterEndpoint::VerifyCredentials.try_into()?,
-    twitter_api(
+        Some(&user_token),
-        TwitterEndpoint("1.1/statuses/update.json").try_into()?,
+        APIAction::Get,
-        Some(&user_token),
+        &[],
-        APIAction::Post(PostData::Data(&[
+    )?
-            ("media_ids", &img_id.to_string()),
+    .json::<serde_json::Value>()?;
-            ("status", tweet),
+
-        ])),
+    let id = user["id"].as_u64().unwrap();
-        &[],
+
-    )?;
+    let timeline: serde_json::Value = twitter_api(
+        reqwest::Url::parse_with_params(
+            &TwitterEndpoint::UserTimeline.to_string(),
+            [
+                ("count", "200"),
+                ("exclude_replies", "true"),
+                ("include_retweets", "false"),
+                ("trim_user", "true"),
+                ("user_id", id.to_string().as_ref()),
+            ],
+        )?,
+        Some(&user_token),
+        APIAction::Get,
+        &[],
+    )?
+    .json()?;
+    for tweet in timeline.as_array().unwrap() {
+        let tweet = tweet.as_object().unwrap();
+        println!("{}, \"{}\"", tweet["id"], tweet["text"]);
+    }
+    Ok(())
+}
+
+fn do_scrape_category() -> StdError<()> {
+    let mut files = get_files_in_category("Category:ISO_7010_safety_signs_(vector_drawings)")?;
+    files.sort();
+    for f in files {
+        println!("{}", f);
+    }
+
+    Ok(())
+}
+
+fn do_scrape_web() -> StdError<()> {
+    let mut files: Vec<_> = scrape_web()?.into_iter().map(|(_, file)| file).collect();
+    files.sort();
+    for f in files {
+        println!("{}", f);
+    }
+
+    Ok(())
+}
+
+fn get_client(headers: Option<reqwest::header::HeaderMap>) -> StdError<reqwest::blocking::Client> {
+    let mut c = reqwest::blocking::Client::builder().user_agent(APP_USER_AGENT);
+    if let Some(headers) = headers {
+        c = c.default_headers(headers);
+    }
+    Ok(c.build()?)
+}
+
+fn run_bot(dry_run: bool) -> StdError<()> {
+    let all = scrape_web()?;
+    let (title, filename) = all
+        .choose(&mut rand::thread_rng())
+        .ok_or("got no images m8")?;
+    event!(Level::INFO, title, filename, "Picked random thing");
+    let client = get_client(None)?;
+    event!(Level::INFO, "Fetching metadata...");
+    // TODO: could crash, probably doesn't matter
+    let meta = get_file_metadata(vec![filename])?.remove(0);
+    event!(Level::INFO, %meta, "Got metadata");
+    event!(Level::INFO, url = meta.url.to_string(), "Fetching image");
+    let svg = client.get(meta.url).send()?.bytes()?;
+
+    let text = format!(
+        "{}\n\nImage source: {}\nAuthor: Wikimedia Commons user {}\n{}{}",
+        title,
+        meta.html_url,
+        meta.author,
+        meta.license_short_name,
+        meta.license_url
+            .map_or("".to_owned(), |u| format!(" ({})", u))
+    );
+
+    if !dry_run {
+        // Render the image nice and big for twitter
+        let img = render_svg(&svg, 1000, false)?;
+        let mut buf = Cursor::new(Vec::new());
+        img.write_to(&mut buf, image::ImageFormat::Png)?;
+        tweet(&text, Some(buf.into_inner().into()))?;
+    } else {
+        // Render the image smaller for output to terminal
+        let img = render_svg(&svg, 128, true)?;
+        println!("Dry run - would tweet:\n \"{}\"", text);
+        viuer::print(
+            &img,
+            &viuer::Config {
+                absolute_offset: false,
+                width: Some(32),
+                ..Default::default()
+            },
+        )?;
+    }
+
     Ok(())
 }

src/twitter.rs

@@ -0,0 +1,276 @@
+use crate::{get_client, StdError};
+use itertools::Itertools;
+use std::borrow::Cow;
+use std::convert::TryInto;
+use std::fmt;
+use std::io::prelude::*;
+use std::{collections::HashMap, io::Write};
+use tracing::{event, instrument, Level};
+
+const APP_TOKEN_ENV_VAR: &str = "TWITTER_APP_TOKEN";
+const APP_SECRET_ENV_VAR: &str = "TWITTER_APP_SECRET";
+const USER_TOKEN_ENV_VAR: &str = "TWITTER_USER_TOKEN";
+const USER_SECRET_ENV_VAR: &str = "TWITTER_USER_SECRET";
+
+static CB_URL: &str = "http://localhost:6969/cb";
+
+pub fn user_token_from_env() -> oauth1::Token<'static> {
+    oauth1::Token::new(
+        std::env::var(USER_TOKEN_ENV_VAR).expect("No user token env var"),
+        std::env::var(USER_SECRET_ENV_VAR).expect("No user secret env var"),
+    )
+}
+
+pub enum TwitterEndpoint {
+    OauthRequestToken,
+    OauthAccessToken,
+    OauthAuthenticate,
+    UpdateStatus,
+    UserTimeline,
+    VerifyCredentials,
+}
+
+impl std::fmt::Display for TwitterEndpoint {
+    fn fmt(&self, f: &mut std::fmt::Formatter) -> Result<(), std::fmt::Error> {
+        const BASE: &str = "https://api.twitter.com";
+        let endpoint = match self {
+            Self::OauthAuthenticate => "oauth/authenticate",
+            Self::OauthRequestToken => "oauth/request_token",
+            Self::OauthAccessToken => "oauth/access_token",
+            Self::UpdateStatus => "1.1/statuses/update.json",
+            Self::UserTimeline => "1.1/statuses/user_timeline.json",
+            Self::VerifyCredentials => "1.1/account/verify_credentials.json",
+        };
+        write!(f, "{}/{}", BASE, endpoint)
+    }
+}
+
+impl TryInto<reqwest::Url> for TwitterEndpoint {
+    type Error = url::ParseError;
+    fn try_into(self) -> Result<reqwest::Url, Self::Error> {
+        reqwest::Url::parse(&self.to_string())
+    }
+}
+
+pub enum PostData<'a> {
+    Empty,
+    Multipart(reqwest::blocking::multipart::Form),
+    Data(&'a [(&'a str, Cow<'a, str>)]),
+}
+
+impl fmt::Debug for PostData<'_> {
+    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+        write!(
+            f,
+            "{}",
+            match self {
+                Self::Empty => "Empty",
+                Self::Multipart(_) => "Multipart",
+                Self::Data(_) => "Data",
+            }
+        )
+    }
+}
+
+#[derive(Debug)]
+pub enum APIAction<'a> {
+    Get,
+    Post(PostData<'a>),
+}
+
+impl APIAction<'_> {
+    pub fn get_verb(&self) -> &'static str {
+        match self {
+            Self::Get => "GET",
+            Self::Post(_) => "POST",
+        }
+    }
+}
+// Make an authed twitter API request
+#[instrument(skip(user_token), fields(url=url.to_string()))]
+pub fn twitter_api<'a>(
+    url: reqwest::Url,
+    user_token: Option<&oauth1::Token>,
+    action: APIAction,
+    extra_oauth_params: &[(&str, &str)],
+) -> StdError<reqwest::blocking::Response> {
+    let consumer_token = oauth1::Token::new(
+        std::env::var(APP_TOKEN_ENV_VAR)?,
+        std::env::var(APP_SECRET_ENV_VAR)?,
+    );
+    let mut headers = reqwest::header::HeaderMap::new();
+    let mut params: HashMap<&str, Cow<str>> = extra_oauth_params
+        .iter()
+        .cloned()
+        .map(|(x, y)| (x, y.into()))
+        .collect();
+
+    // Copy all our query parameters and add them to the list of params for oauth1 signature
+    // generation.
+    // This is a bit awkward, as params is a map from &str to Cow<str> but query_pairs() returns
+    // (Cow<str>, Cow<str>) tuples. So we call into_owned to make (String, String) tuples, and then
+    // borrow from there. If there's a way to do it without copying, I couldn't find it.
+    let pairs: Vec<_> = url.query_pairs().into_owned().collect();
+    for (k, v) in &pairs {
+        params.insert(k, Cow::Borrowed(v));
+    }
+
+    // If the request is a key/value form post, we also need to include those parameters when
+    // generating the signature.
+    if let APIAction::Post(PostData::Data(d)) = action {
+        params.extend(d.to_owned())
+    }
+
+    // The url used to generate the signature must not include the query params
+    let mut url_sans_query = url.clone();
+    url_sans_query.set_query(None);
+    headers.insert(
+        reqwest::header::AUTHORIZATION,
+        reqwest::header::HeaderValue::from_str(&oauth1::authorize(
+            action.get_verb(),
+            url_sans_query.as_str(),
+            &consumer_token,
+            user_token,
+            Some(params),
+        ))?,
+    );
+    let client = get_client(Some(headers))?;
+    let req = match action {
+        APIAction::Get => client.get(url),
+        APIAction::Post(PostData::Empty) => client.post(url),
+        APIAction::Post(PostData::Data(data)) => client.post(url).form(data),
+        APIAction::Post(PostData::Multipart(form)) => client.post(url).multipart(form),
+    };
+    event!(Level::INFO, "Sending request");
+    let res = req.send()?;
+    if !res.status().is_success() {
+        return Err(format!(
+            "Got non-200 response: status {}, {}",
+            res.status(),
+            res.text()?
+        )
+        .into());
+    }
+    Ok(res)
+}
+
+pub fn do_authorize() -> StdError<()> {
+    println!("Authorizing you lol!");
+
+    // Oauth1 leg 1
+    let res = twitter_api(
+        TwitterEndpoint::OauthRequestToken.try_into()?,
+        None,
+        APIAction::Post(PostData::Empty),
+        &[("oauth_callback", CB_URL)],
+    )?
+    .text()?;
+
+    let returned_params: HashMap<&str, &str> = res
+        .split('&')
+        .map(|s| s.split('=').collect_tuple())
+        .collect::<Option<_>>()
+        .ok_or("Unexpected oauth step 1 response")?;
+
+    // Oauth1 leg 2
+    let user_url = reqwest::Url::parse_with_params(
+        &TwitterEndpoint::OauthAuthenticate.to_string(),
+        [("oauth_token", returned_params["oauth_token"])],
+    )?;
+    println!("Plz do the thing in the browser");
+    webbrowser::open(user_url.as_str())?;
+
+    let listener = std::net::TcpListener::bind("127.0.0.1:6969")?;
+    let mut stream = listener.incoming().next().ok_or("Error getting stream")??;
+    let mut buf = [0u8; 4096];
+    stream.read(&mut buf[..])?;
+
+    let target = std::str::from_utf8(buf.split(|c| *c == b' ').nth(1).ok_or("No target found")?)?;
+    let oauth_verifier = reqwest::Url::parse("https://example.net/")?
+        .join(target)?
+        .query_pairs()
+        .find_map(|(k, v)| {
+            if k == "oauth_verifier" {
+                Some(v.into_owned())
+            } else {
+                None
+            }
+        })
+        .ok_or("no oauth_verifier in response")?;
+    stream.write_all(b"HTTP/1.1 200 OK\r\n\r\nThanks lmao\r\n")?;
+    stream.shutdown(std::net::Shutdown::Read)?;
+
+    // Oauth1 leg 3
+    let res = twitter_api(
+        TwitterEndpoint::OauthAccessToken.try_into()?,
+        None,
+        APIAction::Post(PostData::Data(&[(
+            "oauth_verifier",
+            Cow::Owned(oauth_verifier),
+        )])),
+        &[("oauth_token", returned_params["oauth_token"])],
+    )?
+    .text()?;
+    let returned_params: HashMap<&str, &str> = res
+        .split('&')
+        .map(|s| s.split('=').collect_tuple())
+        .collect::<Option<_>>()
+        .ok_or("Unexpected oauth step 3 response")?;
+
+    println!(
+        "Authorized for {}.\nRun with {}={} {}={}",
+        returned_params["screen_name"],
+        USER_TOKEN_ENV_VAR,
+        returned_params["oauth_token"],
+        USER_SECRET_ENV_VAR,
+        returned_params["oauth_token_secret"]
+    );
+
+    Ok(())
+}
+
+fn upload_image(user_token: &oauth1::Token, img: Cow<'static, [u8]>) -> StdError<u64> {
+    let form = reqwest::blocking::multipart::Form::new()
+        .part("media", reqwest::blocking::multipart::Part::bytes(img));
+    let res: serde_json::Value = twitter_api(
+        "https://upload.twitter.com/1.1/media/upload.json".try_into()?,
+        Some(user_token),
+        APIAction::Post(PostData::Multipart(form)),
+        &[],
+    )?
+    .json()?;
+    Ok(res["media_id"].as_u64().ok_or("media_id not u64!")?)
+}
+
+pub fn tweet(text: &str, img: Option<Cow<'static, [u8]>>) -> StdError<()> {
+    let user_token = oauth1::Token::new(
+        std::env::var(USER_TOKEN_ENV_VAR)?,
+        std::env::var(USER_SECRET_ENV_VAR)?,
+    );
+
+    let user: serde_json::Value = twitter_api(
+        TwitterEndpoint::VerifyCredentials.try_into()?,
+        Some(&user_token),
+        APIAction::Get,
+        &[],
+    )?
+    .json()?;
+    println!(
+        "Tweeting for user @{}, (id: {})",
+        user["screen_name"], user["id"]
+    );
+    let mut post_data = vec![("status", Cow::Borrowed(text))];
+    if let Some(img) = img {
+        println!("Uploading image...");
+        let img_id = upload_image(&user_token, img)?;
+        post_data.push(("media_ids", Cow::Owned(img_id.to_string())))
+    }
+    event!(Level::INFO, "Sending tweet...");
+    twitter_api(
+        TwitterEndpoint::UpdateStatus.try_into()?,
+        Some(&user_token),
+        APIAction::Post(PostData::Data(&post_data[0..])),
+        &[],
+    )?;
+    Ok(())
+}

src/wiki.rs

@@ -0,0 +1,204 @@
+use std::collections::HashMap;
+
+use crate::{get_client, StdError};
+use regex::Regex;
+use serde::Deserialize;
+use std::fmt::Display;
+use tracing::{event, instrument, Level};
+use url::Url;
+
+// Filter a filename string for filenames
+fn filter_filename(filename: &str) -> bool {
+    let re = Regex::new("ISO.7010.[EWMPF][0-9]{3}.*").unwrap();
+    re.is_match(filename)
+}
+
+// Scrape all images from the wikipedia page, returning a vec of title, filename pairs
+pub fn scrape_web() -> StdError<Vec<(String, String)>> {
+    event!(Level::INFO, "Scraping the wikipedia page for things");
+    // Parse CSS selectors to scrape elements
+    let gallerybox_sel = scraper::Selector::parse(".mw-body-content li.gallerybox")
+        .map_err(|e| format!("{:?}", e))?;
+    let link_sel = scraper::Selector::parse("a.image").map_err(|e| format!("{:?}", e))?;
+    let title_sel = scraper::Selector::parse(".gallerytext p").map_err(|e| format!("{:?}", e))?;
+
+    // Fetch stuff!
+    let client = get_client(None)?;
+    event!(Level::INFO, "Fetching wiki page");
+    let txt = client
+        .get("https://en.wikipedia.org/wiki/ISO_7010")
+        .send()?
+        .text()?;
+    let page = scraper::Html::parse_document(txt.as_str());
+    return Ok(page
+        .select(&gallerybox_sel)
+        .map(|a| {
+            let link = a
+                .select(&link_sel)
+                .next()
+                .unwrap()
+                .value()
+                .attr("href")
+                .unwrap()
+                .to_owned();
+            let title = a
+                .select(&title_sel)
+                .next()
+                .unwrap()
+                .text()
+                .collect::<String>()
+                .trim()
+                .to_owned();
+            (title, link)
+        })
+        // Filter for filenames that look like ISO diagrams
+        .filter(|tup| filter_filename(&tup.1))
+        // Extract the file name only (.e.g `File:ISO_7010_X000.svg`)
+        .filter_map(|(title, link)| {
+            link.split('/')
+                .next_back()
+                .map(|end| (title, end.to_owned()))
+        })
+        .collect::<Vec<_>>());
+}
+
+#[instrument]
+pub fn wiki_query_url(params: Vec<(&str, &str)>) -> StdError<Url> {
+    let mut url = Url::parse("https://commons.wikimedia.org/w/api.php?action=query&format=json")?;
+    url.query_pairs_mut().extend_pairs(params);
+    Ok(url)
+}
+// https://commons.wikimedia.org/w/api.php?action=query&format=json&list=categorymembers&cmtitle=Category:ISO_7010_safety_signs_(vector_drawings)&cmlimit=2
+
+#[instrument]
+pub fn get_files_in_category(category: &str) -> StdError<Vec<String>> {
+    let client = get_client(None)?;
+    let url = wiki_query_url(
+        [
+            ("list", "categorymembers"),
+            ("cmtitle", category),
+            ("cmtype", "file"),
+            ("cmlimit", "max"),
+        ]
+        .into(),
+    )?;
+    let data = client.get(url).send()?.json::<serde_json::Value>()?;
+    if data.get("continue").is_some() {
+        // There are more results than are contained in one response, so now you need to implement
+        // pagination. Have fun!
+        panic!("Wikimedia query result is paginated!");
+    }
+    Ok(data["query"]["categorymembers"]
+        .as_array()
+        .unwrap()
+        .iter()
+        .filter_map(|m| Some(m.as_object().unwrap()["title"].as_str()?.replace(' ', "_")))
+        .collect())
+}
+
+#[derive(Debug)]
+pub struct FileMeta {
+    pub url: url::Url,
+    pub name: String,
+    pub html_url: url::Url,
+    pub author: String,
+    pub attribution_required: String,
+    pub license_short_name: String,
+    pub license_url: Option<url::Url>,
+}
+
+impl Display for FileMeta {
+    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+        write!(
+            f,
+            "FileMeta{{url: {}, name: {}, html_url: {}, author: {}, attribution_required: {}, license_short_name: {}, license_url: {}}}",
+            self.url, self.name, self.html_url, self.author, self.attribution_required, self.license_short_name,
+            self.license_url.clone().map_or("None".to_owned(), |u| u.to_string()) // Ew. 
+        )
+    }
+}
+
+// Partial representation of the data returned from a MediaWiki imageinfo query
+#[derive(Deserialize)]
+struct Query {
+    query: QueryInner,
+}
+
+#[derive(Deserialize, Debug)]
+struct QueryInner {
+    pages: HashMap<String, Page>,
+}
+
+#[derive(Deserialize, Debug)]
+struct Page {
+    imageinfo: Vec<ImageInfo>,
+}
+
+#[derive(Deserialize, Debug)]
+struct ImageInfo {
+    user: String,
+    url: url::Url,
+    descriptionurl: url::Url,
+    extmetadata: ExtMeta,
+}
+
+#[derive(Deserialize, Debug)]
+#[serde(rename_all = "PascalCase")]
+struct ExtMeta {
+    object_name: ExtMetaItem<String>,
+    license_short_name: ExtMetaItem<String>,
+    attribution_required: ExtMetaItem<String>,
+    license_url: Option<ExtMetaItem<url::Url>>,
+}
+
+#[derive(Deserialize, Debug, Clone)]
+struct ExtMetaItem<T> {
+    value: T,
+}
+
+pub fn get_file_metadata(files: Vec<&str>) -> StdError<Vec<FileMeta>> {
+    let client = get_client(None)?;
+    // Api only lets us do 50 files in one request
+    Ok(files
+        .chunks(50)
+        .flat_map(|files_chunk| {
+            let url = wiki_query_url(
+                [
+                    ("titles", files_chunk.join("|").as_ref()),
+                    ("prop", "imageinfo"),
+                    (
+                        "iiprop",
+                        "timestamp|url|size|mime|mediatype|extmetadata|user",
+                    ),
+                    // Get metadata for as many revisions of the file as we are allowed. We're unlikely to encounter a file with >500 revisions.
+                    ("iilimit", "500"),
+                    (
+                        "iiextmetadatafilter",
+                        "ObjectName|LicenseShortName|AttributionRequired|LicenseUrl",
+                    ),
+                ]
+                .into(),
+            )
+            .unwrap();
+            let data = client.get(url).send().unwrap().json::<Query>().unwrap();
+
+            data.query
+                .pages
+                .values()
+                .map(|page| {
+                    let latest = page.imageinfo.first().unwrap();
+                    let oldest = page.imageinfo.last().unwrap();
+                    FileMeta {
+                        url: latest.url.clone(),
+                        name: latest.extmetadata.object_name.value.clone(),
+                        html_url: latest.descriptionurl.clone(),
+                        author: oldest.user.clone(),
+                        license_short_name: latest.extmetadata.license_short_name.value.clone(),
+                        license_url: latest.extmetadata.license_url.clone().map(|i| i.value),
+                        attribution_required: latest.extmetadata.attribution_required.value.clone(),
+                    }
+                })
+                .collect::<Vec<_>>()
+        })
+        .collect())
+}