Add mastodon posting, update deps, go async
This commit is contained in:
parent
fa1c07e059
commit
cff99c2788
File diff suppressed because it is too large
Load Diff
|
|
@ -5,14 +5,12 @@ version = "0.2.0"
|
|||
edition = "2021"
|
||||
|
||||
[dependencies]
|
||||
reqwest = { version = "0.11", features = ["blocking", "json", "multipart"]}
|
||||
reqwest = { version = "0.11", features = ["json", "multipart"]}
|
||||
serde_json = "*"
|
||||
serde = { version = "1", features = ["derive"] }
|
||||
scraper = "*"
|
||||
rand = "*"
|
||||
resvg = "*"
|
||||
tiny-skia = "*"
|
||||
usvg = "*"
|
||||
oauth1 = "*"
|
||||
clap = { version = "*", features = ["derive"] }
|
||||
webbrowser = "*"
|
||||
|
|
@ -23,6 +21,9 @@ regex = "1.6.0"
|
|||
image = "0.24.3"
|
||||
viuer = "0.6.1"
|
||||
url = { version = "2.3.1", features = ["serde"] }
|
||||
megalodon = { git = "https://github.com/wlcx/megalodon-rs.git" }
|
||||
tokio = "*"
|
||||
futures-util = "*"
|
||||
|
||||
[build-dependencies]
|
||||
toml = "*"
|
||||
|
|
|
|||
136
flake.lock
136
flake.lock
|
|
@ -2,15 +2,15 @@
|
|||
"nodes": {
|
||||
"devshell": {
|
||||
"inputs": {
|
||||
"flake-utils": "flake-utils",
|
||||
"nixpkgs": "nixpkgs"
|
||||
"nixpkgs": "nixpkgs",
|
||||
"systems": "systems"
|
||||
},
|
||||
"locked": {
|
||||
"lastModified": 1660811669,
|
||||
"narHash": "sha256-V6lmsaLNFz41myppL0yxglta92ijkSvpZ+XVygAh+bU=",
|
||||
"lastModified": 1686680692,
|
||||
"narHash": "sha256-SsLZz3TDleraAiJq4EkmdyewSyiv5g0LZYc6vaLZOMQ=",
|
||||
"owner": "numtide",
|
||||
"repo": "devshell",
|
||||
"rev": "c2feacb46ee69949124c835419861143c4016fb5",
|
||||
"rev": "fd6223370774dd9c33354e87a007004b5fd36442",
|
||||
"type": "github"
|
||||
},
|
||||
"original": {
|
||||
|
|
@ -20,27 +20,15 @@
|
|||
}
|
||||
},
|
||||
"flake-utils": {
|
||||
"locked": {
|
||||
"lastModified": 1642700792,
|
||||
"narHash": "sha256-XqHrk7hFb+zBvRg6Ghl+AZDq03ov6OshJLiSWOoX5es=",
|
||||
"owner": "numtide",
|
||||
"repo": "flake-utils",
|
||||
"rev": "846b2ae0fc4cc943637d3d1def4454213e203cba",
|
||||
"type": "github"
|
||||
"inputs": {
|
||||
"systems": "systems_2"
|
||||
},
|
||||
"original": {
|
||||
"owner": "numtide",
|
||||
"repo": "flake-utils",
|
||||
"type": "github"
|
||||
}
|
||||
},
|
||||
"flake-utils_2": {
|
||||
"locked": {
|
||||
"lastModified": 1656928814,
|
||||
"narHash": "sha256-RIFfgBuKz6Hp89yRr7+NR5tzIAbn52h8vT6vXkYjZoM=",
|
||||
"lastModified": 1681202837,
|
||||
"narHash": "sha256-H+Rh19JDwRtpVPAWp64F+rlEtxUWBAQW28eAi3SRSzg=",
|
||||
"owner": "numtide",
|
||||
"repo": "flake-utils",
|
||||
"rev": "7e2a3b3dfd9af950a856d66b0a7d01e3c18aa249",
|
||||
"rev": "cfacdce06f30d2b68473a46042957675eebb3401",
|
||||
"type": "github"
|
||||
},
|
||||
"original": {
|
||||
|
|
@ -54,11 +42,11 @@
|
|||
"nixpkgs": "nixpkgs_2"
|
||||
},
|
||||
"locked": {
|
||||
"lastModified": 1659610603,
|
||||
"narHash": "sha256-LYgASYSPYo7O71WfeUOaEUzYfzuXm8c8eavJcel+pfI=",
|
||||
"lastModified": 1686572087,
|
||||
"narHash": "sha256-jXTut7ZSYqLEgm/nTk7TuVL2ExahTip605bLINklAnQ=",
|
||||
"owner": "nix-community",
|
||||
"repo": "naersk",
|
||||
"rev": "c6a45e4277fa58abd524681466d3450f896dc094",
|
||||
"rev": "8507af04eb40c5520bd35d9ce6f9d2342cea5ad1",
|
||||
"type": "github"
|
||||
},
|
||||
"original": {
|
||||
|
|
@ -69,11 +57,11 @@
|
|||
},
|
||||
"nixpkgs": {
|
||||
"locked": {
|
||||
"lastModified": 1643381941,
|
||||
"narHash": "sha256-pHTwvnN4tTsEKkWlXQ8JMY423epos8wUOhthpwJjtpc=",
|
||||
"lastModified": 1677383253,
|
||||
"narHash": "sha256-UfpzWfSxkfXHnb4boXZNaKsAcUrZT9Hw+tao1oZxd08=",
|
||||
"owner": "NixOS",
|
||||
"repo": "nixpkgs",
|
||||
"rev": "5efc8ca954272c4376ac929f4c5ffefcc20551d5",
|
||||
"rev": "9952d6bc395f5841262b006fbace8dd7e143b634",
|
||||
"type": "github"
|
||||
},
|
||||
"original": {
|
||||
|
|
@ -85,12 +73,11 @@
|
|||
},
|
||||
"nixpkgs_2": {
|
||||
"locked": {
|
||||
"lastModified": 1661353537,
|
||||
"narHash": "sha256-1E2IGPajOsrkR49mM5h55OtYnU0dGyre6gl60NXKITE=",
|
||||
"owner": "NixOS",
|
||||
"repo": "nixpkgs",
|
||||
"rev": "0e304ff0d9db453a4b230e9386418fd974d5804a",
|
||||
"type": "github"
|
||||
"lastModified": 1685789966,
|
||||
"narHash": "sha256-pyqctu5Cq1jwymO3Os0/RNj5Nm3q5kmRCT24p7gtG70=",
|
||||
"path": "/nix/store/hnkjxwx9zv2k0gkiznbpkrsvyrzaz6w1-source",
|
||||
"rev": "4eaa9e3eb36386de0c6a268ba5da72cafc959619",
|
||||
"type": "path"
|
||||
},
|
||||
"original": {
|
||||
"id": "nixpkgs",
|
||||
|
|
@ -99,12 +86,11 @@
|
|||
},
|
||||
"nixpkgs_3": {
|
||||
"locked": {
|
||||
"lastModified": 1662907018,
|
||||
"narHash": "sha256-rMPfDmY7zJzv/tJj+LComcGEa1UuwI67kpbz5WC6abE=",
|
||||
"owner": "NixOS",
|
||||
"repo": "nixpkgs",
|
||||
"rev": "17352e8995e1409636b0817a7f38d6314ccd73c4",
|
||||
"type": "github"
|
||||
"lastModified": 1685789966,
|
||||
"narHash": "sha256-pyqctu5Cq1jwymO3Os0/RNj5Nm3q5kmRCT24p7gtG70=",
|
||||
"path": "/nix/store/hnkjxwx9zv2k0gkiznbpkrsvyrzaz6w1-source",
|
||||
"rev": "4eaa9e3eb36386de0c6a268ba5da72cafc959619",
|
||||
"type": "path"
|
||||
},
|
||||
"original": {
|
||||
"id": "nixpkgs",
|
||||
|
|
@ -113,11 +99,11 @@
|
|||
},
|
||||
"nixpkgs_4": {
|
||||
"locked": {
|
||||
"lastModified": 1659102345,
|
||||
"narHash": "sha256-Vbzlz254EMZvn28BhpN8JOi5EuKqnHZ3ujFYgFcSGvk=",
|
||||
"lastModified": 1681358109,
|
||||
"narHash": "sha256-eKyxW4OohHQx9Urxi7TQlFBTDWII+F+x2hklDOQPB50=",
|
||||
"owner": "NixOS",
|
||||
"repo": "nixpkgs",
|
||||
"rev": "11b60e4f80d87794a2a4a8a256391b37c59a1ea7",
|
||||
"rev": "96ba1c52e54e74c3197f4d43026b3f3d92e83ff9",
|
||||
"type": "github"
|
||||
},
|
||||
"original": {
|
||||
|
|
@ -138,15 +124,15 @@
|
|||
},
|
||||
"rust-overlay": {
|
||||
"inputs": {
|
||||
"flake-utils": "flake-utils_2",
|
||||
"flake-utils": "flake-utils",
|
||||
"nixpkgs": "nixpkgs_4"
|
||||
},
|
||||
"locked": {
|
||||
"lastModified": 1662001050,
|
||||
"narHash": "sha256-tduflWLNZ6C3Xz0eUHf5Cnnfl47Vgey2NUY5ZU9f/S4=",
|
||||
"lastModified": 1686795910,
|
||||
"narHash": "sha256-jDa40qRZ0GRQtP9EMZdf+uCbvzuLnJglTUI2JoHfWDc=",
|
||||
"owner": "oxalica",
|
||||
"repo": "rust-overlay",
|
||||
"rev": "6f27501ff78beb62728cb292daca846fcab96c9e",
|
||||
"rev": "5c2b97c0a9bc5217fc3dfb1555aae0fb756d99f9",
|
||||
"type": "github"
|
||||
},
|
||||
"original": {
|
||||
|
|
@ -155,13 +141,61 @@
|
|||
"type": "github"
|
||||
}
|
||||
},
|
||||
"utils": {
|
||||
"systems": {
|
||||
"locked": {
|
||||
"lastModified": 1637014545,
|
||||
"narHash": "sha256-26IZAc5yzlD9FlDT54io1oqG/bBoyka+FJk5guaX4x4=",
|
||||
"lastModified": 1681028828,
|
||||
"narHash": "sha256-Vy1rq5AaRuLzOxct8nz4T6wlgyUR7zLU309k9mBC768=",
|
||||
"owner": "nix-systems",
|
||||
"repo": "default",
|
||||
"rev": "da67096a3b9bf56a91d16901293e51ba5b49a27e",
|
||||
"type": "github"
|
||||
},
|
||||
"original": {
|
||||
"owner": "nix-systems",
|
||||
"repo": "default",
|
||||
"type": "github"
|
||||
}
|
||||
},
|
||||
"systems_2": {
|
||||
"locked": {
|
||||
"lastModified": 1681028828,
|
||||
"narHash": "sha256-Vy1rq5AaRuLzOxct8nz4T6wlgyUR7zLU309k9mBC768=",
|
||||
"owner": "nix-systems",
|
||||
"repo": "default",
|
||||
"rev": "da67096a3b9bf56a91d16901293e51ba5b49a27e",
|
||||
"type": "github"
|
||||
},
|
||||
"original": {
|
||||
"owner": "nix-systems",
|
||||
"repo": "default",
|
||||
"type": "github"
|
||||
}
|
||||
},
|
||||
"systems_3": {
|
||||
"locked": {
|
||||
"lastModified": 1681028828,
|
||||
"narHash": "sha256-Vy1rq5AaRuLzOxct8nz4T6wlgyUR7zLU309k9mBC768=",
|
||||
"owner": "nix-systems",
|
||||
"repo": "default",
|
||||
"rev": "da67096a3b9bf56a91d16901293e51ba5b49a27e",
|
||||
"type": "github"
|
||||
},
|
||||
"original": {
|
||||
"owner": "nix-systems",
|
||||
"repo": "default",
|
||||
"type": "github"
|
||||
}
|
||||
},
|
||||
"utils": {
|
||||
"inputs": {
|
||||
"systems": "systems_3"
|
||||
},
|
||||
"locked": {
|
||||
"lastModified": 1685518550,
|
||||
"narHash": "sha256-o2d0KcvaXzTrPRIo0kOLV0/QXHhDQ5DTi+OxcjO8xqY=",
|
||||
"owner": "numtide",
|
||||
"repo": "flake-utils",
|
||||
"rev": "bba5dcc8e0b20ab664967ad83d24d64cb64ec4f4",
|
||||
"rev": "a1720a10a6cfe8234c0e93907ffe81be440f4cef",
|
||||
"type": "github"
|
||||
},
|
||||
"original": {
|
||||
|
|
|
|||
15
flake.nix
15
flake.nix
|
|
@ -21,28 +21,31 @@
|
|||
inherit system;
|
||||
overlays = [(import rust-overlay)];
|
||||
};
|
||||
rust = pkgs.rust-bin.stable.latest.default;
|
||||
rust = pkgs.rust-bin.stable.latest.default.override {
|
||||
extensions = [ "rust-src" ];
|
||||
};
|
||||
# Override naersk to use our chosen rust version from rust-overlay
|
||||
naersk-lib = naersk.lib.${system}.override {
|
||||
cargo = rust;
|
||||
rustc = rust;
|
||||
};
|
||||
in rec {
|
||||
packages.default = naersk-lib.buildPackage {
|
||||
packig = naersk-lib.buildPackage {
|
||||
pname = "iso7010-a-day";
|
||||
root = ./.;
|
||||
buildInputs = [pkgs.openssl pkgs.pkgconfig];
|
||||
};
|
||||
in {
|
||||
packages.default = packig;
|
||||
|
||||
apps.default = utils.lib.mkApp {drv = packages.default;};
|
||||
apps.default = utils.lib.mkApp {drv = packig;};
|
||||
|
||||
hydraJobs.build = packages.default;
|
||||
hydraJobs.build = packig;
|
||||
|
||||
# Provide a dev env with rust and rls
|
||||
devShells.default = let
|
||||
pkgs = import nixpkgs {
|
||||
inherit system;
|
||||
overlays = [devshell.overlay];
|
||||
overlays = [devshell.overlays.default];
|
||||
};
|
||||
in
|
||||
pkgs.devshell.mkShell {
|
||||
|
|
|
|||
113
src/main.rs
113
src/main.rs
|
|
@ -1,16 +1,23 @@
|
|||
use mastodon::authorize_fedi;
|
||||
use rand::seq::SliceRandom;
|
||||
use std::convert::TryInto;
|
||||
use std::io::Cursor;
|
||||
use tracing::{event, Level};
|
||||
mod mastodon;
|
||||
mod twitter;
|
||||
mod wiki;
|
||||
use clap::{Parser, Subcommand};
|
||||
use image::{DynamicImage, RgbaImage};
|
||||
use resvg::tiny_skia::{Paint, PathBuilder, Pixmap, PixmapPaint, Stroke};
|
||||
use resvg::usvg::TreeParsing;
|
||||
use resvg::usvg::{Options, Transform};
|
||||
use resvg::Tree;
|
||||
use std::borrow::Cow;
|
||||
use tiny_skia::{Paint, PathBuilder, Pixmap, PixmapPaint, Stroke, Transform};
|
||||
use twitter::*;
|
||||
use wiki::*;
|
||||
|
||||
use crate::mastodon::toot;
|
||||
|
||||
static APP_USER_AGENT: &str = concat!(
|
||||
"bot_",
|
||||
env!("CARGO_PKG_NAME"),
|
||||
|
|
@ -22,30 +29,31 @@ static APP_USER_AGENT: &str = concat!(
|
|||
|
||||
// Render the raw SVG data to an image
|
||||
fn render_svg(data: &[u8], height: u32, with_border: bool) -> StdError<DynamicImage> {
|
||||
let opt = usvg::Options::default();
|
||||
let rtree = usvg::Tree::from_data(data, &opt.to_ref()).expect("couldn't parse");
|
||||
let svg_size = rtree.svg_node().size;
|
||||
let opt = Options::default();
|
||||
let rtree = resvg::usvg::Tree::from_data(data, &opt).expect("couldn't parse");
|
||||
let svg_size = rtree.size;
|
||||
// Work out how wide the pixmap of height `height` needs to be to entirely fit the SVG.
|
||||
let pm_width = ((height as f64 / svg_size.height()) * svg_size.width()).ceil() as u32;
|
||||
let scale_factor = height as f32 / svg_size.height();
|
||||
let pm_width = (scale_factor * svg_size.width()).ceil() as u32;
|
||||
let mut pixmap = Pixmap::new(pm_width, height).ok_or("Error creating pixmap")?;
|
||||
// Render the svg into a pixmap.
|
||||
resvg::render(&rtree, usvg::FitTo::Height(height), pixmap.as_mut())
|
||||
.ok_or("Error rendering svg")?;
|
||||
Tree::from_usvg(&rtree).render(
|
||||
Transform::from_scale(scale_factor, scale_factor),
|
||||
&mut pixmap.as_mut(),
|
||||
);
|
||||
// Make a wider pixmap with a 16:9 AR and the same height. This is a blesséd ratio by twitter
|
||||
// and means we see the whole image nicely in the timeline with no truncation.
|
||||
let mut bigger_pixmap =
|
||||
Pixmap::new(height / 9 * 16, height).ok_or("Error creating bigger pixmap")?;
|
||||
// Then draw our freshly rendered SVG into the middle of the bigger pixmap.
|
||||
bigger_pixmap
|
||||
.draw_pixmap(
|
||||
((bigger_pixmap.width() - pm_width) / 2).try_into().unwrap(),
|
||||
0,
|
||||
pixmap.as_ref(),
|
||||
&PixmapPaint::default(),
|
||||
Transform::identity(),
|
||||
None,
|
||||
)
|
||||
.ok_or("Error drawing onto bigger pixmap")?;
|
||||
bigger_pixmap.draw_pixmap(
|
||||
((bigger_pixmap.width() - pm_width) / 2).try_into().unwrap(),
|
||||
0,
|
||||
pixmap.as_ref(),
|
||||
&PixmapPaint::default(),
|
||||
Transform::identity(),
|
||||
None,
|
||||
);
|
||||
let (w, h) = (bigger_pixmap.width(), bigger_pixmap.height());
|
||||
// Render a red border for debug purposes
|
||||
if with_border {
|
||||
|
|
@ -88,14 +96,16 @@ struct Cli {
|
|||
#[derive(Subcommand)]
|
||||
enum Commands {
|
||||
/// Authorize the twitter application to acccess a user's account
|
||||
Authorize,
|
||||
AuthorizeTwitter,
|
||||
/// Scrape images from the category on wikimedia commons
|
||||
ScrapeCategory,
|
||||
/// Scrape images from the iso7010 wikipedia page
|
||||
ScrapeWeb,
|
||||
/// List tweets from the authed user's timeline
|
||||
ListTweets,
|
||||
/// Run the bot - scrape, pick a random entry and tweet it
|
||||
/// Authorize against a pleroma server
|
||||
AuthorizeFedi,
|
||||
/// Run the bot - scrape, pick a random entry and toot it
|
||||
RunBot {
|
||||
#[clap(short, long, action)]
|
||||
dry_run: bool,
|
||||
|
|
@ -106,20 +116,22 @@ enum Commands {
|
|||
Whoami,
|
||||
}
|
||||
|
||||
fn main() -> StdError<()> {
|
||||
#[tokio::main]
|
||||
async fn main() -> StdError<()> {
|
||||
tracing_subscriber::fmt::init();
|
||||
let cli = Cli::parse();
|
||||
match &cli.command {
|
||||
Commands::Authorize => do_authorize(),
|
||||
Commands::ScrapeCategory => do_scrape_category(),
|
||||
Commands::ScrapeWeb => do_scrape_web(),
|
||||
Commands::ListTweets => do_list_tweets(),
|
||||
Commands::Whoami => do_whoami(),
|
||||
Commands::RunBot { dry_run, target } => run_bot(*dry_run, target.to_owned()),
|
||||
Commands::AuthorizeTwitter => do_authorize().await,
|
||||
Commands::ScrapeCategory => do_scrape_category().await,
|
||||
Commands::ScrapeWeb => do_scrape_web().await,
|
||||
Commands::ListTweets => do_list_tweets().await,
|
||||
Commands::Whoami => do_whoami().await,
|
||||
Commands::AuthorizeFedi => authorize_fedi().await,
|
||||
Commands::RunBot { dry_run, target } => run_bot(*dry_run, target.to_owned()).await,
|
||||
}
|
||||
}
|
||||
|
||||
fn do_whoami() -> StdError<()> {
|
||||
async fn do_whoami() -> StdError<()> {
|
||||
let user_token = user_token_from_env();
|
||||
|
||||
let user: serde_json::Value = twitter_api(
|
||||
|
|
@ -127,13 +139,15 @@ fn do_whoami() -> StdError<()> {
|
|||
Some(&user_token),
|
||||
APIAction::Get,
|
||||
&[],
|
||||
)?
|
||||
.json()?;
|
||||
)
|
||||
.await?
|
||||
.json()
|
||||
.await?;
|
||||
println!("User @{}, (id: {})", user["screen_name"], user["id"]);
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn do_list_tweets() -> StdError<()> {
|
||||
async fn do_list_tweets() -> StdError<()> {
|
||||
let user_token = user_token_from_env();
|
||||
|
||||
let user = twitter_api(
|
||||
|
|
@ -141,8 +155,10 @@ fn do_list_tweets() -> StdError<()> {
|
|||
Some(&user_token),
|
||||
APIAction::Get,
|
||||
&[],
|
||||
)?
|
||||
.json::<serde_json::Value>()?;
|
||||
)
|
||||
.await?
|
||||
.json::<serde_json::Value>()
|
||||
.await?;
|
||||
|
||||
let id = user["id"].as_u64().unwrap().to_string();
|
||||
let mut timeline = vec![];
|
||||
|
|
@ -165,8 +181,10 @@ fn do_list_tweets() -> StdError<()> {
|
|||
Some(&user_token),
|
||||
APIAction::Get,
|
||||
&[],
|
||||
)?
|
||||
.json::<serde_json::Value>()?;
|
||||
)
|
||||
.await?
|
||||
.json::<serde_json::Value>()
|
||||
.await?;
|
||||
let chunk = timeline_chunk.as_array().unwrap().to_owned();
|
||||
event!(Level::INFO, count = chunk.len(), "Got tweets.");
|
||||
if chunk.is_empty() {
|
||||
|
|
@ -186,8 +204,9 @@ fn do_list_tweets() -> StdError<()> {
|
|||
Ok(())
|
||||
}
|
||||
|
||||
fn do_scrape_category() -> StdError<()> {
|
||||
let mut files = get_files_in_category("Category:ISO_7010_safety_signs_(vector_drawings)")?;
|
||||
async fn do_scrape_category() -> StdError<()> {
|
||||
let mut files =
|
||||
get_files_in_category("Category:ISO_7010_safety_signs_(vector_drawings)").await?;
|
||||
files.sort();
|
||||
for f in files {
|
||||
println!("{}", f);
|
||||
|
|
@ -196,8 +215,12 @@ fn do_scrape_category() -> StdError<()> {
|
|||
Ok(())
|
||||
}
|
||||
|
||||
fn do_scrape_web() -> StdError<()> {
|
||||
let mut files: Vec<_> = scrape_web()?.into_iter().map(|(_, file)| file).collect();
|
||||
async fn do_scrape_web() -> StdError<()> {
|
||||
let mut files: Vec<_> = scrape_web()
|
||||
.await?
|
||||
.into_iter()
|
||||
.map(|(_, file)| file)
|
||||
.collect();
|
||||
files.sort();
|
||||
for f in files {
|
||||
println!("{}", f);
|
||||
|
|
@ -206,16 +229,16 @@ fn do_scrape_web() -> StdError<()> {
|
|||
Ok(())
|
||||
}
|
||||
|
||||
fn get_client(headers: Option<reqwest::header::HeaderMap>) -> StdError<reqwest::blocking::Client> {
|
||||
let mut c = reqwest::blocking::Client::builder().user_agent(APP_USER_AGENT);
|
||||
fn get_client(headers: Option<reqwest::header::HeaderMap>) -> StdError<reqwest::Client> {
|
||||
let mut c = reqwest::Client::builder().user_agent(APP_USER_AGENT);
|
||||
if let Some(headers) = headers {
|
||||
c = c.default_headers(headers);
|
||||
}
|
||||
Ok(c.build()?)
|
||||
}
|
||||
|
||||
fn run_bot(dry_run: bool, target: Option<String>) -> StdError<()> {
|
||||
let all = scrape_web()?;
|
||||
async fn run_bot(dry_run: bool, target: Option<String>) -> StdError<()> {
|
||||
let all = scrape_web().await?;
|
||||
let (title, filename) = if let Some(target) = target {
|
||||
all.iter()
|
||||
.find(|(title, _)| title.to_lowercase().contains(&target.to_lowercase()))
|
||||
|
|
@ -227,10 +250,10 @@ fn run_bot(dry_run: bool, target: Option<String>) -> StdError<()> {
|
|||
let client = get_client(None)?;
|
||||
event!(Level::INFO, "Fetching metadata...");
|
||||
// TODO: could crash, probably doesn't matter
|
||||
let meta = get_file_metadata(&[filename.as_str()])?.remove(0);
|
||||
let meta = get_file_metadata(&[filename.as_str()]).await?.remove(0);
|
||||
event!(Level::INFO, %meta, "Got metadata");
|
||||
event!(Level::INFO, url = meta.url.to_string(), "Fetching image");
|
||||
let svg = client.get(meta.url).send()?.bytes()?;
|
||||
let svg = client.get(meta.url).send().await?.bytes().await?;
|
||||
|
||||
let text = format!(
|
||||
"{}\n\nImage source: {}\nAuthor: Wikimedia Commons user {}\n{}{}",
|
||||
|
|
@ -247,7 +270,7 @@ fn run_bot(dry_run: bool, target: Option<String>) -> StdError<()> {
|
|||
let img = render_svg(&svg, 1000, false)?;
|
||||
let mut buf = Cursor::new(Vec::new());
|
||||
img.write_to(&mut buf, image::ImageFormat::Png)?;
|
||||
tweet(&text, Some(buf.into_inner().into()))?;
|
||||
toot(&text, Some(buf.into_inner().into())).await?;
|
||||
} else {
|
||||
// Render the image smaller for output to terminal
|
||||
let img = render_svg(&svg, 128, true)?;
|
||||
|
|
|
|||
|
|
@ -0,0 +1,87 @@
|
|||
// Interface to mastodon (etc) instances
|
||||
|
||||
use std::borrow::Cow;
|
||||
|
||||
use megalodon::entities::UploadMedia;
|
||||
use megalodon::generator;
|
||||
use megalodon::megalodon::PostStatusInputOptions;
|
||||
use megalodon::{self, megalodon::UploadMediaInputOptions};
|
||||
|
||||
use crate::{StdError, APP_USER_AGENT};
|
||||
|
||||
const FEDI_ACCESS_TOKEN_ENV_VAR: &str = "FEDI_ACCESS_TOKEN";
|
||||
const FEDI_INSTANCE_ENV_VAR: &str = "FEDI_INSTANCE";
|
||||
|
||||
pub async fn toot(text: &str, img: Option<Cow<'static, [u8]>>) -> StdError<()> {
|
||||
let client = megalodon::generator(
|
||||
megalodon::SNS::Pleroma,
|
||||
std::env::var(FEDI_INSTANCE_ENV_VAR)
|
||||
.unwrap_or_else(|_| panic!("{} env var not present", FEDI_ACCESS_TOKEN_ENV_VAR))
|
||||
.into(),
|
||||
Some(
|
||||
std::env::var(FEDI_ACCESS_TOKEN_ENV_VAR)
|
||||
.unwrap_or_else(|_| panic!("{} env var not present", FEDI_ACCESS_TOKEN_ENV_VAR))
|
||||
.into(),
|
||||
),
|
||||
Some(APP_USER_AGENT.into()),
|
||||
);
|
||||
let mut ops = PostStatusInputOptions::default();
|
||||
if let Some(img) = img {
|
||||
let media = client
|
||||
.upload_media_bytes(
|
||||
// TODO: get better at lifetimes
|
||||
Box::leak(Box::new(img)),
|
||||
Some(&UploadMediaInputOptions::default()),
|
||||
)
|
||||
.await?;
|
||||
ops.media_ids = Some(vec![match media.json {
|
||||
UploadMedia::Attachment(a) => a.id,
|
||||
UploadMedia::AsyncAttachment(a) => a.id,
|
||||
}]);
|
||||
}
|
||||
client.post_status(text.into(), Some(&ops)).await?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
pub async fn authorize_fedi() -> StdError<()> {
|
||||
let url = std::env::var(FEDI_INSTANCE_ENV_VAR)
|
||||
.unwrap_or_else(|_| panic!("{} env var not present", FEDI_ACCESS_TOKEN_ENV_VAR))
|
||||
.into();
|
||||
let client = generator(megalodon::SNS::Pleroma, url, None, None);
|
||||
let options = megalodon::megalodon::AppInputOptions {
|
||||
scopes: Some([String::from("read"), String::from("write")].to_vec()),
|
||||
..Default::default()
|
||||
};
|
||||
|
||||
match client.register_app(String::from("iso7010"), &options).await {
|
||||
Ok(app_data) => {
|
||||
println!("{}", app_data.url.unwrap());
|
||||
println!("Enter code:");
|
||||
let mut code = String::new();
|
||||
std::io::stdin().read_line(&mut code).ok();
|
||||
match client
|
||||
.fetch_access_token(
|
||||
app_data.client_id,
|
||||
app_data.client_secret,
|
||||
code.trim().to_string(),
|
||||
megalodon::default::NO_REDIRECT.to_string(),
|
||||
)
|
||||
.await
|
||||
{
|
||||
Ok(token_data) => {
|
||||
println!("token: {}", token_data.access_token);
|
||||
if let Some(refresh) = token_data.refresh_token {
|
||||
println!("refresh_token: {}", refresh);
|
||||
}
|
||||
}
|
||||
Err(err) => {
|
||||
panic!("{}", err);
|
||||
}
|
||||
}
|
||||
}
|
||||
Err(err) => {
|
||||
panic!("{}", err);
|
||||
}
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
|
@ -54,7 +54,7 @@ impl TryInto<reqwest::Url> for TwitterEndpoint {
|
|||
|
||||
pub enum PostData<'a> {
|
||||
Empty,
|
||||
Multipart(reqwest::blocking::multipart::Form),
|
||||
Multipart(reqwest::multipart::Form),
|
||||
Data(&'a [(&'a str, Cow<'a, str>)]),
|
||||
}
|
||||
|
||||
|
|
@ -88,12 +88,12 @@ impl APIAction<'_> {
|
|||
}
|
||||
// Make an authed twitter API request
|
||||
#[instrument(skip(user_token), fields(url=url.to_string()))]
|
||||
pub fn twitter_api<'a>(
|
||||
pub async fn twitter_api<'a>(
|
||||
url: reqwest::Url,
|
||||
user_token: Option<&oauth1::Token>,
|
||||
action: APIAction,
|
||||
user_token: Option<&oauth1::Token<'a>>,
|
||||
action: APIAction<'a>,
|
||||
extra_oauth_params: &[(&str, &str)],
|
||||
) -> StdError<reqwest::blocking::Response> {
|
||||
) -> StdError<reqwest::Response> {
|
||||
let consumer_token = oauth1::Token::new(
|
||||
std::env::var(APP_TOKEN_ENV_VAR)?,
|
||||
std::env::var(APP_SECRET_ENV_VAR)?,
|
||||
|
|
@ -142,19 +142,19 @@ pub fn twitter_api<'a>(
|
|||
APIAction::Post(PostData::Multipart(form)) => client.post(url).multipart(form),
|
||||
};
|
||||
event!(Level::INFO, "Sending request");
|
||||
let res = req.send()?;
|
||||
let res = req.send().await?;
|
||||
if !res.status().is_success() {
|
||||
return Err(format!(
|
||||
"Got non-200 response: status {}, {}",
|
||||
res.status(),
|
||||
res.text()?
|
||||
res.text().await?
|
||||
)
|
||||
.into());
|
||||
}
|
||||
Ok(res)
|
||||
}
|
||||
|
||||
pub fn do_authorize() -> StdError<()> {
|
||||
pub async fn do_authorize() -> StdError<()> {
|
||||
println!("Authorizing you lol!");
|
||||
|
||||
// Oauth1 leg 1
|
||||
|
|
@ -163,8 +163,10 @@ pub fn do_authorize() -> StdError<()> {
|
|||
None,
|
||||
APIAction::Post(PostData::Empty),
|
||||
&[("oauth_callback", CB_URL)],
|
||||
)?
|
||||
.text()?;
|
||||
)
|
||||
.await?
|
||||
.text()
|
||||
.await?;
|
||||
|
||||
let returned_params: HashMap<&str, &str> = res
|
||||
.split('&')
|
||||
|
|
@ -209,8 +211,10 @@ pub fn do_authorize() -> StdError<()> {
|
|||
Cow::Owned(oauth_verifier),
|
||||
)])),
|
||||
&[("oauth_token", returned_params["oauth_token"])],
|
||||
)?
|
||||
.text()?;
|
||||
)
|
||||
.await?
|
||||
.text()
|
||||
.await?;
|
||||
let returned_params: HashMap<&str, &str> = res
|
||||
.split('&')
|
||||
.map(|s| s.split('=').collect_tuple())
|
||||
|
|
@ -229,20 +233,24 @@ pub fn do_authorize() -> StdError<()> {
|
|||
Ok(())
|
||||
}
|
||||
|
||||
fn upload_image(user_token: &oauth1::Token, img: Cow<'static, [u8]>) -> StdError<u64> {
|
||||
let form = reqwest::blocking::multipart::Form::new()
|
||||
.part("media", reqwest::blocking::multipart::Part::bytes(img));
|
||||
async fn upload_image<'a>(
|
||||
user_token: &oauth1::Token<'a>,
|
||||
img: Cow<'static, [u8]>,
|
||||
) -> StdError<u64> {
|
||||
let form = reqwest::multipart::Form::new().part("media", reqwest::multipart::Part::bytes(img));
|
||||
let res: serde_json::Value = twitter_api(
|
||||
"https://upload.twitter.com/1.1/media/upload.json".try_into()?,
|
||||
Some(user_token),
|
||||
APIAction::Post(PostData::Multipart(form)),
|
||||
&[],
|
||||
)?
|
||||
.json()?;
|
||||
)
|
||||
.await?
|
||||
.json()
|
||||
.await?;
|
||||
Ok(res["media_id"].as_u64().ok_or("media_id not u64!")?)
|
||||
}
|
||||
|
||||
pub fn tweet(text: &str, img: Option<Cow<'static, [u8]>>) -> StdError<()> {
|
||||
pub async fn tweet(text: &str, img: Option<Cow<'static, [u8]>>) -> StdError<()> {
|
||||
let user_token = oauth1::Token::new(
|
||||
std::env::var(USER_TOKEN_ENV_VAR)?,
|
||||
std::env::var(USER_SECRET_ENV_VAR)?,
|
||||
|
|
@ -253,8 +261,10 @@ pub fn tweet(text: &str, img: Option<Cow<'static, [u8]>>) -> StdError<()> {
|
|||
Some(&user_token),
|
||||
APIAction::Get,
|
||||
&[],
|
||||
)?
|
||||
.json()?;
|
||||
)
|
||||
.await?
|
||||
.json()
|
||||
.await?;
|
||||
println!(
|
||||
"Tweeting for user @{}, (id: {})",
|
||||
user["screen_name"], user["id"]
|
||||
|
|
@ -262,7 +272,7 @@ pub fn tweet(text: &str, img: Option<Cow<'static, [u8]>>) -> StdError<()> {
|
|||
let mut post_data = vec![("status", Cow::Borrowed(text))];
|
||||
if let Some(img) = img {
|
||||
println!("Uploading image...");
|
||||
let img_id = upload_image(&user_token, img)?;
|
||||
let img_id = upload_image(&user_token, img).await?;
|
||||
post_data.push(("media_ids", Cow::Owned(img_id.to_string())))
|
||||
}
|
||||
event!(Level::INFO, "Sending tweet...");
|
||||
|
|
@ -271,6 +281,7 @@ pub fn tweet(text: &str, img: Option<Cow<'static, [u8]>>) -> StdError<()> {
|
|||
Some(&user_token),
|
||||
APIAction::Post(PostData::Data(&post_data[0..])),
|
||||
&[],
|
||||
)?;
|
||||
)
|
||||
.await?;
|
||||
Ok(())
|
||||
}
|
||||
|
|
|
|||
106
src/wiki.rs
106
src/wiki.rs
|
|
@ -14,7 +14,7 @@ fn extract_filename(filename: &str) -> Option<&str> {
|
|||
}
|
||||
|
||||
// Scrape all images from the wikipedia page, returning a vec of title, filename pairs
|
||||
pub fn scrape_web() -> StdError<Vec<(String, String)>> {
|
||||
pub async fn scrape_web() -> StdError<Vec<(String, String)>> {
|
||||
event!(Level::INFO, "Scraping the wikipedia page for things");
|
||||
// Parse CSS selectors to scrape elements
|
||||
let gallerybox_sel =
|
||||
|
|
@ -27,8 +27,10 @@ pub fn scrape_web() -> StdError<Vec<(String, String)>> {
|
|||
event!(Level::INFO, "Fetching wiki page");
|
||||
let txt = client
|
||||
.get("https://en.wikipedia.org/wiki/ISO_7010")
|
||||
.send()?
|
||||
.text()?;
|
||||
.send()
|
||||
.await?
|
||||
.text()
|
||||
.await?;
|
||||
let page = scraper::Html::parse_document(txt.as_str());
|
||||
return Ok(page
|
||||
.select(&gallerybox_sel)
|
||||
|
|
@ -62,7 +64,7 @@ pub fn wiki_query_url(params: Vec<(&str, &str)>) -> StdError<Url> {
|
|||
// https://commons.wikimedia.org/w/api.php?action=query&format=json&list=categorymembers&cmtitle=Category:ISO_7010_safety_signs_(vector_drawings)&cmlimit=2
|
||||
|
||||
#[instrument]
|
||||
pub fn get_files_in_category(category: &str) -> StdError<Vec<String>> {
|
||||
pub async fn get_files_in_category(category: &str) -> StdError<Vec<String>> {
|
||||
let client = get_client(None)?;
|
||||
let url = wiki_query_url(
|
||||
[
|
||||
|
|
@ -73,7 +75,12 @@ pub fn get_files_in_category(category: &str) -> StdError<Vec<String>> {
|
|||
]
|
||||
.into(),
|
||||
)?;
|
||||
let data = client.get(url).send()?.json::<serde_json::Value>()?;
|
||||
let data = client
|
||||
.get(url)
|
||||
.send()
|
||||
.await?
|
||||
.json::<serde_json::Value>()
|
||||
.await?;
|
||||
if data.get("continue").is_some() {
|
||||
// There are more results than are contained in one response, so now you need to implement
|
||||
// pagination. Have fun!
|
||||
|
|
@ -147,49 +154,54 @@ struct ExtMetaItem<T> {
|
|||
value: T,
|
||||
}
|
||||
|
||||
pub fn get_file_metadata(files: &[&str]) -> StdError<Vec<FileMeta>> {
|
||||
pub async fn get_file_metadata(files: &[&str]) -> StdError<Vec<FileMeta>> {
|
||||
let client = get_client(None)?;
|
||||
// Api only lets us do 50 files in one request
|
||||
Ok(files
|
||||
.chunks(50)
|
||||
.flat_map(|files_chunk| {
|
||||
let url = wiki_query_url(
|
||||
[
|
||||
("titles", files_chunk.join("|").as_ref()),
|
||||
("prop", "imageinfo"),
|
||||
(
|
||||
"iiprop",
|
||||
"timestamp|url|size|mime|mediatype|extmetadata|user",
|
||||
),
|
||||
// Get metadata for as many revisions of the file as we are allowed. We're unlikely to encounter a file with >500 revisions.
|
||||
("iilimit", "500"),
|
||||
(
|
||||
"iiextmetadatafilter",
|
||||
"ObjectName|LicenseShortName|AttributionRequired|LicenseUrl",
|
||||
),
|
||||
]
|
||||
.into(),
|
||||
)
|
||||
.unwrap();
|
||||
let data = client.get(url).send().unwrap().json::<Query>().unwrap();
|
||||
let urls = files.chunks(50).map(|files_chunk| {
|
||||
wiki_query_url(
|
||||
[
|
||||
("titles", files_chunk.join("|").as_ref()),
|
||||
("prop", "imageinfo"),
|
||||
(
|
||||
"iiprop",
|
||||
"timestamp|url|size|mime|mediatype|extmetadata|user",
|
||||
),
|
||||
// Get metadata for as many revisions of the file as we are allowed. We're unlikely to encounter a file with >500 revisions.
|
||||
("iilimit", "500"),
|
||||
(
|
||||
"iiextmetadatafilter",
|
||||
"ObjectName|LicenseShortName|AttributionRequired|LicenseUrl",
|
||||
),
|
||||
]
|
||||
.into(),
|
||||
)
|
||||
.unwrap()
|
||||
});
|
||||
|
||||
data.query
|
||||
.pages
|
||||
.values()
|
||||
.map(|page| {
|
||||
let latest = page.imageinfo.first().unwrap();
|
||||
let oldest = page.imageinfo.last().unwrap();
|
||||
FileMeta {
|
||||
url: latest.url.clone(),
|
||||
name: latest.extmetadata.object_name.value.clone(),
|
||||
html_url: latest.descriptionurl.clone(),
|
||||
author: oldest.user.clone(),
|
||||
license_short_name: latest.extmetadata.license_short_name.value.clone(),
|
||||
license_url: latest.extmetadata.license_url.clone().map(|i| i.value),
|
||||
attribution_required: latest.extmetadata.attribution_required.value.clone(),
|
||||
}
|
||||
})
|
||||
.collect::<Vec<_>>()
|
||||
})
|
||||
.collect())
|
||||
let mut meta = Vec::new();
|
||||
for u in urls {
|
||||
let data = client
|
||||
.get(u)
|
||||
.send()
|
||||
.await
|
||||
.unwrap()
|
||||
.json::<Query>()
|
||||
.await
|
||||
.unwrap();
|
||||
|
||||
meta.extend(data.query.pages.values().map(|page| {
|
||||
let latest = page.imageinfo.first().unwrap();
|
||||
let oldest = page.imageinfo.last().unwrap();
|
||||
FileMeta {
|
||||
url: latest.url.clone(),
|
||||
name: latest.extmetadata.object_name.value.clone(),
|
||||
html_url: latest.descriptionurl.clone(),
|
||||
author: oldest.user.clone(),
|
||||
license_short_name: latest.extmetadata.license_short_name.value.clone(),
|
||||
license_url: latest.extmetadata.license_url.clone().map(|i| i.value),
|
||||
attribution_required: latest.extmetadata.attribution_required.value.clone(),
|
||||
}
|
||||
}))
|
||||
}
|
||||
Ok(meta)
|
||||
}
|
||||
|
|
|
|||
Loading…
Reference in New Issue