Add dump command, update deps

Cargo.toml

@@ -14,6 +14,7 @@ resvg = "*"
 oauth1 = "*"
 clap = { version = "*", features = ["derive"] }
 webbrowser = "*"
+#webbrowser = { path = "../webbrowser-rs" }
 itertools = "*"
 tracing = "0.1.36"
 tracing-subscriber = "0.3.15"
@@ -21,7 +22,7 @@ regex = "1.6.0"
 image = "0.24.3"
 viuer = "0.6.1"
 url = { version = "2.3.1", features = ["serde"] }
-megalodon = { git = "https://github.com/wlcx/megalodon-rs.git" }
+megalodon = "0.10"
 tokio = "*"
 futures-util = "*"
 

src/main.rs

@@ -1,17 +1,15 @@
 use mastodon::authorize_fedi;
 use rand::seq::SliceRandom;
 use std::convert::TryInto;
-use std::io::Cursor;
+use std::fs::File;
+use std::io::{BufWriter, Cursor, Write};
 use tracing::{event, Level};
 mod mastodon;
+mod svg;
 mod twitter;
 mod wiki;
 use clap::{Parser, Subcommand};
-use image::{DynamicImage, RgbaImage};
-use resvg::tiny_skia::{Paint, PathBuilder, Pixmap, PixmapPaint, Stroke};
-use resvg::usvg::TreeParsing;
-use resvg::usvg::{Options, Transform};
-use resvg::Tree;
+
 use std::borrow::Cow;
 use twitter::*;
 use wiki::*;
@@ -27,63 +25,6 @@ static APP_USER_AGENT: &str = concat!(
     "reqwest",
 );
 
-// Render the raw SVG data to an image
-fn render_svg(data: &[u8], height: u32, with_border: bool) -> StdError<DynamicImage> {
-    let opt = Options::default();
-    let rtree = resvg::usvg::Tree::from_data(data, &opt).expect("couldn't parse");
-    let svg_size = rtree.size;
-    // Work out how wide the pixmap of height `height` needs to be to entirely fit the SVG.
-    let scale_factor = height as f32 / svg_size.height();
-    let pm_width = (scale_factor * svg_size.width()).ceil() as u32;
-    let mut pixmap = Pixmap::new(pm_width, height).ok_or("Error creating pixmap")?;
-    // Render the svg into a pixmap.
-    Tree::from_usvg(&rtree).render(
-        Transform::from_scale(scale_factor, scale_factor),
-        &mut pixmap.as_mut(),
-    );
-    // Make a wider pixmap with a 16:9 AR and the same height. This is a blesséd ratio by twitter
-    // and means we see the whole image nicely in the timeline with no truncation.
-    let mut bigger_pixmap =
-        Pixmap::new(height / 9 * 16, height).ok_or("Error creating bigger pixmap")?;
-    // Then draw our freshly rendered SVG into the middle of the bigger pixmap.
-    bigger_pixmap.draw_pixmap(
-        ((bigger_pixmap.width() - pm_width) / 2).try_into().unwrap(),
-        0,
-        pixmap.as_ref(),
-        &PixmapPaint::default(),
-        Transform::identity(),
-        None,
-    );
-    let (w, h) = (bigger_pixmap.width(), bigger_pixmap.height());
-    // Render a red border for debug purposes
-    if with_border {
-        let mut paint = Paint::default();
-        paint.set_color_rgba8(255, 0, 0, 255);
-        let stroke = Stroke {
-            width: 1.0,
-            ..Default::default()
-        };
-
-        let path = {
-            let mut pb = PathBuilder::new();
-            pb.move_to(0.0, 0.0);
-            pb.line_to(0.0, h as f32 - stroke.width);
-            pb.line_to(w as f32, h as f32 - stroke.width);
-            pb.line_to(w as f32 - stroke.width, 0.0);
-            pb.line_to(0.0, 0.0);
-            pb.finish().unwrap()
-        };
-        bigger_pixmap.stroke_path(&path, &paint, &stroke, Transform::identity(), None);
-    }
-    let img = RgbaImage::from_raw(
-        bigger_pixmap.width(),
-        bigger_pixmap.height(),
-        bigger_pixmap.data().to_vec(),
-    )
-    .ok_or("Error creating image from pixmap")?;
-    Ok(DynamicImage::ImageRgba8(img))
-}
-
 type StdError<T> = Result<T, Box<dyn std::error::Error>>;
 
 #[derive(Parser)]
@@ -114,6 +55,8 @@ enum Commands {
     },
     /// Print details about the currently authed user
     Whoami,
+    /// Download and dump all images to disk
+    Dump,
 }
 
 #[tokio::main]
@@ -128,6 +71,7 @@ async fn main() -> StdError<()> {
         Commands::Whoami => do_whoami().await,
         Commands::AuthorizeFedi => authorize_fedi().await,
         Commands::RunBot { dry_run, target } => run_bot(*dry_run, target.to_owned()).await,
+        Commands::Dump => do_dump().await,
     }
 }
 
@@ -237,6 +181,22 @@ fn get_client(headers: Option<reqwest::header::HeaderMap>) -> StdError<reqwest::
     Ok(c.build()?)
 }
 
+async fn do_dump() -> StdError<()> {
+    let all = scrape_web().await?;
+    let filenames: Vec<_> = all.iter().map(|(_, filename)| filename.as_str()).collect();
+    let metas = get_file_metadata(&filenames).await?;
+    // for m in &metas {
+    //     let bytes = client.get(m.url.to_string()).await?.bytes().await?;
+    // }
+    let file = File::create("dump.json")?;
+    let mut writer = BufWriter::new(file);
+    serde_json::to_writer_pretty(&mut writer, &metas)?;
+    writer.flush()?;
+    let count = metas.len();
+    event!(Level::INFO, count, "Dumped records to json file");
+    Ok(())
+}
+
 async fn run_bot(dry_run: bool, target: Option<String>) -> StdError<()> {
     let all = scrape_web().await?;
     let (title, filename) = if let Some(target) = target {
@@ -267,13 +227,13 @@ async fn run_bot(dry_run: bool, target: Option<String>) -> StdError<()> {
 
     if !dry_run {
         // Render the image nice and big for twitter
-        let img = render_svg(&svg, 1000, false)?;
+        let img = svg::render_svg(&svg, 1000, false)?;
         let mut buf = Cursor::new(Vec::new());
         img.write_to(&mut buf, image::ImageFormat::Png)?;
-        toot(&text, Some(buf.into_inner().into())).await?;
+        toot(&text, Some(Box::new(buf))).await?;
     } else {
         // Render the image smaller for output to terminal
-        let img = render_svg(&svg, 128, true)?;
+        let img = svg::render_svg(&svg, 128, true)?;
         println!("Dry run - would tweet:\n \"{}\"", text);
         viuer::print(
             &img,

src/mastodon.rs

@@ -1,18 +1,17 @@
 // Interface to mastodon (etc) instances
 
-use std::borrow::Cow;
-
 use megalodon::entities::UploadMedia;
 use megalodon::generator;
 use megalodon::megalodon::PostStatusInputOptions;
 use megalodon::{self, megalodon::UploadMediaInputOptions};
+use std::io::Cursor;
 
 use crate::{StdError, APP_USER_AGENT};
 
 const FEDI_ACCESS_TOKEN_ENV_VAR: &str = "FEDI_ACCESS_TOKEN";
 const FEDI_INSTANCE_ENV_VAR: &str = "FEDI_INSTANCE";
 
-pub async fn toot(text: &str, img: Option<Cow<'static, [u8]>>) -> StdError<()> {
+pub async fn toot(text: &str, img: Option<Box<Cursor<Vec<u8>>>>) -> StdError<()> {
     let client = megalodon::generator(
         megalodon::SNS::Pleroma,
         std::env::var(FEDI_INSTANCE_ENV_VAR)
@@ -28,9 +27,9 @@ pub async fn toot(text: &str, img: Option<Cow<'static, [u8]>>) -> StdError<()> {
     let mut ops = PostStatusInputOptions::default();
     if let Some(img) = img {
         let media = client
-            .upload_media_bytes(
+            .upload_media_reader(
                 // TODO: get better at lifetimes
-                Box::leak(Box::new(img)),
+                img,
                 Some(&UploadMediaInputOptions::default()),
             )
             .await?;

src/svg.rs

@@ -0,0 +1,64 @@
+use image::{DynamicImage, RgbaImage};
+use resvg::tiny_skia::{Paint, PathBuilder, Pixmap, PixmapPaint, Stroke};
+use resvg::usvg::TreeParsing;
+use resvg::usvg::{Options, Transform};
+use resvg::Tree;
+
+use crate::StdError;
+
+// Render the raw SVG data to an image
+pub fn render_svg(data: &[u8], height: u32, with_border: bool) -> StdError<DynamicImage> {
+    let opt = Options::default();
+    let rtree = resvg::usvg::Tree::from_data(data, &opt).expect("couldn't parse");
+    let svg_size = rtree.size;
+    // Work out how wide the pixmap of height `height` needs to be to entirely fit the SVG.
+    let scale_factor = height as f32 / svg_size.height();
+    let pm_width = (scale_factor * svg_size.width()).ceil() as u32;
+    let mut pixmap = Pixmap::new(pm_width, height).ok_or("Error creating pixmap")?;
+    // Render the svg into a pixmap.
+    Tree::from_usvg(&rtree).render(
+        Transform::from_scale(scale_factor, scale_factor),
+        &mut pixmap.as_mut(),
+    );
+    // Make a wider pixmap with a 16:9 AR and the same height. This is a blesséd ratio by twitter
+    // and means we see the whole image nicely in the timeline with no truncation.
+    let mut bigger_pixmap =
+        Pixmap::new(height / 9 * 16, height).ok_or("Error creating bigger pixmap")?;
+    // Then draw our freshly rendered SVG into the middle of the bigger pixmap.
+    bigger_pixmap.draw_pixmap(
+        ((bigger_pixmap.width() - pm_width) / 2).try_into().unwrap(),
+        0,
+        pixmap.as_ref(),
+        &PixmapPaint::default(),
+        Transform::identity(),
+        None,
+    );
+    let (w, h) = (bigger_pixmap.width(), bigger_pixmap.height());
+    // Render a red border for debug purposes
+    if with_border {
+        let mut paint = Paint::default();
+        paint.set_color_rgba8(255, 0, 0, 255);
+        let stroke = Stroke {
+            width: 1.0,
+            ..Default::default()
+        };
+
+        let path = {
+            let mut pb = PathBuilder::new();
+            pb.move_to(0.0, 0.0);
+            pb.line_to(0.0, h as f32 - stroke.width);
+            pb.line_to(w as f32, h as f32 - stroke.width);
+            pb.line_to(w as f32 - stroke.width, 0.0);
+            pb.line_to(0.0, 0.0);
+            pb.finish().unwrap()
+        };
+        bigger_pixmap.stroke_path(&path, &paint, &stroke, Transform::identity(), None);
+    }
+    let img = RgbaImage::from_raw(
+        bigger_pixmap.width(),
+        bigger_pixmap.height(),
+        bigger_pixmap.data().to_vec(),
+    )
+    .ok_or("Error creating image from pixmap")?;
+    Ok(DynamicImage::ImageRgba8(img))
+}

src/wiki.rs

@@ -2,7 +2,7 @@ use std::collections::HashMap;
 
 use crate::{get_client, StdError};
 use regex::Regex;
-use serde::Deserialize;
+use serde::{Deserialize, Serialize};
 use std::fmt::Display;
 use tracing::{event, instrument, Level};
 use url::Url;
@@ -19,7 +19,8 @@ pub async fn scrape_web() -> StdError<Vec<(String, String)>> {
     // Parse CSS selectors to scrape elements
     let gallerybox_sel =
         scraper::Selector::parse("li.gallerybox").map_err(|e| format!("{:?}", e))?;
-    let link_sel = scraper::Selector::parse("a.mw-file-description").map_err(|e| format!("{:?}", e))?;
+    let link_sel =
+        scraper::Selector::parse("a.mw-file-description").map_err(|e| format!("{:?}", e))?;
     let title_sel = scraper::Selector::parse(".gallerytext p").map_err(|e| format!("{:?}", e))?;
 
     // Fetch stuff!
@@ -94,7 +95,7 @@ pub async fn get_files_in_category(category: &str) -> StdError<Vec<String>> {
         .collect())
 }
 
-#[derive(Debug)]
+#[derive(Debug, Serialize)]
 pub struct FileMeta {
     pub url: url::Url,
     pub name: String,