use error_stack::{report, Result};
use indicatif::ProgressBar;
use megalodon::generator;
use megalodon::SNS;
use platform_dirs::AppDirs;
use serverconf::ServerData;
use tokio::task::JoinSet;
use tokio::time::sleep;
use std::error::Error;
use std::fmt;
use std::fs::File;
use std::io::Read;
use std::io::Write;
use std::path::Path;
use std::path::PathBuf;
use std::rc::Rc;
use std::time::Duration;
use tokio::fs;
use tokio::sync::mpsc;
use zip::ZipArchive;
use imghdr;
use indicatif::ProgressIterator;
use reqwest;
use reqwest::Url;
use tempdir::TempDir;
use clap::{Parser, Subcommand};
use serde::{Deserialize, Serialize};
use walkdir::DirEntry;
use walkdir::WalkDir;
use zip::write::FileOptions;
use env_logger;
use log::{debug, error, warn};
use chrono::Local;
use crate::serverconf::SaveFile;
mod serverconf;
#[derive(Parser, Debug)]
#[command(author, version, about, long_about = None)]
struct Cli {
#[command(subcommand)]
command: Command,
}
#[derive(Debug, Subcommand)]
enum Command {
Local {
#[arg(short = 'o', long = "output", default_value_t = String::from("generated_emojis"))]
outputFilepath: String,
#[arg(short, long)]
folder: String,
#[arg(short = 'h', long = "host", default_value_t = String::from("https://github.com/cutestnekoaqua/emoji-gen"))]
originHost: String,
#[arg(short, long, default_value_t = ("Custom").to_string())]
group: String,
},
Crawl {
#[arg(short = 'o', long = "output", default_value_t = String::from("generated_emojis"))]
outputFilepath: String,
#[arg(short, long)]
host: String,
},
Upload {
#[arg(short = 'i', long = "input")]
inputFilepath: String,
#[arg(short, long)]
host: String,
#[arg(short, long)]
global: bool,
},
}
#[derive(Serialize, Deserialize)]
struct Meta {
metaVersion: i8,
host: String,
exportedAt: String,
emojis: Vec<Emoji>,
}
#[derive(Serialize, Deserialize)]
struct EmojiResponse {
shortcode: Option<String>,
url: Option<String>,
static_url: String,
category: Option<String>,
}
#[derive(Serialize, Deserialize)]
struct Emoji {
downloaded: bool,
fileName: String,
emoji: EmojiData,
}
#[derive(Serialize, Deserialize)]
struct EmojiData {
name: String,
category: String,
aliases: Vec<String>,
}
fn getTypename(typeEnum: imghdr::Type) -> &'static str {
return match typeEnum {
imghdr::Type::Bgp => "bgp",
imghdr::Type::Bmp => "bmp",
imghdr::Type::Exr => "exr",
imghdr::Type::Flif => "flif",
imghdr::Type::Gif => "gif",
imghdr::Type::Ico => "ico",
imghdr::Type::Jpeg => "jpg",
imghdr::Type::Pbm => "pbm",
imghdr::Type::Pgm => "pgm",
imghdr::Type::Png => "png",
imghdr::Type::Ppm => "ppm",
imghdr::Type::Rast => "rast",
imghdr::Type::Rgb => "rgb",
imghdr::Type::Rgbe => "rgbe",
imghdr::Type::Tiff => "tiff",
imghdr::Type::Webp => "webp",
imghdr::Type::Xbm => "xbm",
};
}
#[derive(Debug)]
enum EmojiGenError {
ZipCreationFailed,
EmojiFetchFailed,
MetadataGenerationFailed,
ImageFetchingFailed,
}
impl fmt::Display for EmojiGenError {
fn fmt(&self, fmt: &mut fmt::Formatter<'_>) -> fmt::Result {
fmt.write_str("Error processing emojis: Could not create the emoji zip bundle.")
}
}
impl Error for EmojiGenError {}
#[derive(Debug)]
enum EmojiRemoteError {
FileOpenFailed,
ZipOpenFailed,
EmojiFetchFailed,
UploadFailed,
OauthError,
}
impl fmt::Display for EmojiRemoteError {
fn fmt(&self, fmt: &mut fmt::Formatter<'_>) -> fmt::Result {
fmt.write_str("Error while interacting with remote.")
}
}
impl Error for EmojiRemoteError {}
#[tokio::main]
async fn main() -> anyhow::Result<()> {
env_logger::init();
let args = Cli::parse();
match process(args.command).await {
Ok(()) => {}
Err(err) => {
error!("The process could not be completed. Quiting.");
error!("\n{err:?}");
}
};
Ok(())
}
async fn process(command: Command) -> anyhow::Result<()> {
let result = match command {
Command::Upload {
inputFilepath,
host,
global,
} => process_remote(Command::Upload {
inputFilepath,
host,
global
})
.await
.unwrap(),
_ => process_creation(command).await.unwrap(),
};
Ok(result)
}
async fn process_remote(command: Command) -> anyhow::Result<()> {
match command {
Command::Upload {
inputFilepath,
host,
global,
} => {
upload_zip(inputFilepath, host, global).await?;
}
_ => unimplemented!(),
}
Ok(())
}
async fn upload_zip(inputFilepath: String, host: String, global: bool) -> anyhow::Result<()> {
let file = File::open(inputFilepath.clone())
.map_err(|_| {
report!(EmojiRemoteError::FileOpenFailed)
.attach_printable(format!("Could not open file '{}'.", inputFilepath.clone()))
})
.unwrap();
let app_dirs = AppDirs::new(Some("emoji-gen"), true).unwrap();
let save_file_path = app_dirs.config_dir.join("save_file");
let zip_worker = tokio::spawn(parse_zip(file, inputFilepath));
println!("[Config] Searching save file directory..");
fs::create_dir_all(&app_dirs.config_dir).await.unwrap();
println!("[Config] Opening config file..");
let mut config = if save_file_path.exists() {
println!("[Config] Save file found. Loading..");
let content = fs::read(save_file_path.clone()).await?;
let config: SaveFile = rmp_serde::from_slice(&content)?;
config
} else {
println!("[Config] No save file found. Creating new one..");
SaveFile {
map: Default::default(),
}
};
let server_data = match config.map.get(&host) {
Some(data) => {
println!("[Config] Server data found. Using existing data..");
data.clone()
}
None => {
println!("[Config] Server data not found. Creating new data..");
let host_data = host_auth(host.clone()).await.unwrap();
config.map.insert(host.clone(), host_data.clone());
println!("[Config] Done! Saving to file..");
let serialised = rmp_serde::to_vec(&config)
.map_err(|_| {
report!(EmojiRemoteError::OauthError)
.attach_printable(format!("Could not serialize save file."))
})
.unwrap();
fs::write(save_file_path, serialised).await.unwrap();
host_data
}
};
let client = generator(
SNS::Gotosocial,
host.to_string(),
Some(server_data.access_token.clone()),
None,
);
let res = client.verify_account_credentials().await?;
println!("{:#?}", res.json());
let (mut zip, meta) = zip_worker.await.unwrap().unwrap();
let (tx, mut rx): (mpsc::Sender<bool>, mpsc::Receiver<bool>) = mpsc::channel(10000);
let len = meta.emojis.len() as u64;
let bar = ProgressBar::new(len);
tokio::spawn(async move {
let mut count = 0;
for emoji in meta.emojis {
let mut buf = Vec::new();
zip.by_name(&emoji.fileName.as_str())
.unwrap()
.read_to_end(buf.as_mut()).unwrap();
let host = server_data.host.clone();
let access_token = server_data.access_token.clone();
let tx = tx.clone();
tokio::spawn(upload_emoji(host, access_token, emoji, buf, tx.clone(), global));
count += 1;
if count > 200 {
sleep(Duration::from_millis(7000)).await;
count = 0;
}
}
});
while let Some(message) = rx.recv().await {
bar.inc(1);
if !message {
bar.println("❌ Error! One file could not be uploaded.");
}
if bar.position() == len {
bar.finish_and_clear();
println!("✅ Done! All files uploaded.");
break;
}
}
Ok(())
}
async fn upload_emoji(
host: String,
access_token: String,
emoji: Emoji,
file: Vec<u8>,
tx: mpsc::Sender<bool>,
global: bool
) -> anyhow::Result<()> {
let client = reqwest::Client::builder()
.user_agent("EmojiGen/".to_string() + env!("CARGO_PKG_VERSION"))
.build()?;
if file.len() == 0 {
println!("❌ Error! File '{}' is empty.", emoji.fileName.clone());
tx.send(false).await?;
return Ok(());
}
let form = reqwest::multipart::Form::new()
.part(
"element",
reqwest::multipart::Part::bytes(file).file_name(emoji.fileName.clone()),
)
.text("shortcode", emoji.emoji.name)
.text("global", global.to_string())
.text("category", emoji.emoji.category);
let request = client
.post(host.to_string() + "/api/v1/emojis")
.bearer_auth(access_token)
.multipart(form)
.build()?;
let response = client.execute(request).await?;
if response.status().is_success() {
tx.send(true).await?;
} else {
tx.send(false).await?;
println!(
"❌ Error! {}, {}",
response.status().as_u16(),
response
.text()
.await
.unwrap_or("Could not get error message.".to_string())
);
}
drop(tx);
Ok(())
}
async fn parse_zip(file: File, inputFilepath: String) -> anyhow::Result<(ZipArchive<File>, Meta)> {
println!("[ZIP] Opening zip file '{}'...", inputFilepath.clone());
let mut zip = zip::ZipArchive::new(file)
.map_err(|_| {
report!(EmojiRemoteError::ZipOpenFailed).attach_printable(format!(
"[ZIP] Could not open zip file '{}'.",
inputFilepath.clone()
))
})
.unwrap();
let mut meta = zip
.by_name("meta.json")
.map_err(|_| {
report!(EmojiRemoteError::ZipOpenFailed).attach_printable(format!(
"[ZIP] Could not find 'meta.json' in zip file '{}'.",
inputFilepath.clone()
))
})
.unwrap();
let mut content = String::new();
println!(
"[ZIP] Reading 'meta.json' in '{}'...",
inputFilepath.clone()
);
meta.read_to_string(&mut content)
.map_err(|_| {
report!(EmojiRemoteError::ZipOpenFailed).attach_printable(format!(
"[ZIP] Could not read 'meta.json' in zip file '{}'.",
inputFilepath.clone()
))
})
.unwrap();
drop(meta);
let meta: Meta = serde_json::from_str(&content)
.map_err(|_| {
report!(EmojiRemoteError::ZipOpenFailed).attach_printable(format!(
"[ZIP] Could not parse 'meta.json' in zip file '{}'.",
inputFilepath.clone()
))
})
.unwrap();
println!("[ZIP] ✅ Done! 'meta.json' read successfully.");
Ok((zip, meta))
}
async fn host_auth(host: String) -> anyhow::Result<ServerData> {
let client = generator(megalodon::SNS::Gotosocial, host.clone(), None, None); let options = megalodon::megalodon::AppInputOptions {
scopes: Some(["read".to_string(), "write".to_string(), "admin".to_string()].to_vec()),
..Default::default()
};
let mut access_token = "".to_string();
let mut refresh_token: Option<String> = None;
let mut client_id = "".to_string();
let mut client_secret = "".to_string();
match client.register_app("EmojiGen".to_string(), &options).await {
Ok(app_data) => {
client_id = app_data.client_id;
client_secret = app_data.client_secret;
println!("Authorization URL is generated");
println!("{}", app_data.url.unwrap());
println!("Enter authorization code from website: ");
let mut code = String::new();
std::io::stdin().read_line(&mut code).ok();
match client
.fetch_access_token(
client_id.clone(),
client_secret.clone(),
code.trim().to_string(),
megalodon::default::NO_REDIRECT.to_string(),
)
.await
{
Ok(token_data) => {
println!("access_token: {}", token_data.access_token);
access_token = token_data.access_token;
if let Some(refresh) = token_data.refresh_token {
println!("refresh_token: {}", refresh);
refresh_token = Some(refresh);
}
}
Err(err) => {
println!("{:#?}", err);
}
}
}
Err(err) => {
println!("{:#?}", err);
}
}
Ok(ServerData {
access_token,
host,
refresh_token,
client_id,
client_secret,
})
}
async fn process_creation(command: Command) -> Result<(), EmojiGenError> {
let tmpDir: TempDir = TempDir::new("emoji-gen").unwrap();
let tmpFolder: &Path = tmpDir.path();
let zipFilepath: PathBuf;
let emojis: Vec<Emoji>;
let hostUrl: String;
match command {
Command::Local {
outputFilepath,
folder,
originHost,
group,
} => {
zipFilepath = prepare_zip_filepath(outputFilepath)?;
emojis = get_local_emojis(Path::new(folder.as_str()), group, tmpFolder)?;
hostUrl = originHost;
}
Command::Crawl {
outputFilepath,
host,
} => {
zipFilepath = prepare_zip_filepath(outputFilepath)?;
let host = host.clone();
hostUrl = host.clone();
let dir = tmpDir.path().to_path_buf();
let task = tokio::task::spawn_blocking(move || {
get_host_emojis(
&Url::parse(&host)
.map_err(|_| {
report!(EmojiGenError::EmojiFetchFailed)
.attach_printable(format!("Url '{}' is invalid.", host))
})
.unwrap(),
dir.as_path(),
)
.unwrap()
});
emojis = task.await.unwrap();
}
_ => unimplemented!(),
}
generate_meta(hostUrl, emojis, tmpFolder)?;
zip(tmpFolder, &zipFilepath)?;
println!(
"✅ Done! Importable ZIP file under '{}'",
zipFilepath.display()
);
Ok(())
}
fn prepare_zip_filepath(outputFilepath: String) -> Result<PathBuf, EmojiGenError> {
let mut zipFilepath = PathBuf::from(outputFilepath.as_str());
zipFilepath.set_extension("zip");
if zipFilepath.exists() {
return Err(
report!(EmojiGenError::ZipCreationFailed).attach_printable(format!(
"File '{}' exists. Please choose another name.",
zipFilepath.display()
)),
);
}
Ok(zipFilepath)
}
fn get_host_emojis(host: &Url, tmpFolder: &Path) -> Result<Vec<Emoji>, EmojiGenError> {
println!(
"Getting all the fine emojis from Url '{}'...",
host.as_str()
);
let hostUrl = &host.join("/api/v1/custom_emojis").unwrap();
let emojis = match reqwest::blocking::get(hostUrl.clone()) {
Ok(response) => match response.json::<Vec<EmojiResponse>>() {
Ok(emojiRes) => {
let emojos: Vec<EmojiResponse> = emojiRes;
let iter = emojos.iter();
Ok(iter
.progress_count(emojos.len() as u64)
.map(|res| {
get_host_emoji_data(
Url::parse(res.url.as_ref().unwrap().as_str()).unwrap(),
res.shortcode.clone().unwrap_or_default(),
res.category.clone().unwrap_or_default(),
tmpFolder,
)
})
.filter_map(|r| r.ok())
.collect::<Vec<Emoji>>())
}
Err(_) => Err(
report!(EmojiGenError::ImageFetchingFailed).attach_printable(format!(
"Could not get emoji list from url '{}'.",
hostUrl.as_str()
)),
),
},
Err(_) => Err(
report!(EmojiGenError::ImageFetchingFailed).attach_printable(format!(
"Could not get response from url '{}'.",
hostUrl.as_str()
)),
),
}?;
Ok(emojis)
}
fn get_host_emoji_data(
fileUrl: Url,
name: String,
category: String,
tmpFolder: &Path,
) -> Result<Emoji, EmojiGenError> {
debug!("{}", fileUrl.to_string());
let newFilename = get_image_from_url(&fileUrl, tmpFolder, name.clone())?;
let data: EmojiData = EmojiData {
name: name,
category: category.to_string(),
aliases: Vec::<String>::new(),
};
Ok(Emoji {
downloaded: true,
fileName: newFilename,
emoji: data,
})
}
fn get_image_from_url(
fileUrl: &Url,
tmpFolder: &Path,
filename: String,
) -> Result<String, EmojiGenError> {
let img_bytes = &reqwest::blocking::get(fileUrl.clone())
.map_err(|_| {
report!(EmojiGenError::ImageFetchingFailed).attach_printable(format!(
"Could not get image file from url '{}'.",
fileUrl.as_str()
))
})?
.bytes()
.unwrap();
let mut tmpFilepath: PathBuf = tmpFolder.join(filename);
match imghdr::from_bytes(img_bytes) {
Some(extension) => tmpFilepath.set_extension(getTypename(extension)),
None => tmpFilepath.set_extension("xxx"),
};
println!("Creating image file at path '{}'...", tmpFilepath.display());
let mut imageFile = File::create(tmpFilepath.as_os_str()).map_err(|_| {
report!(EmojiGenError::ImageFetchingFailed).attach_printable(format!(
"Could not create image file at temporary path '{}'.",
tmpFilepath.display()
))
})?;
imageFile.write_all(img_bytes).map_err(|_| {
report!(EmojiGenError::ImageFetchingFailed).attach_printable(format!(
"Could not write image at temporary path '{}'.",
tmpFilepath.display()
))
})?;
Ok(String::from(
tmpFilepath.file_name().unwrap().to_str().unwrap(),
))
}
fn get_local_emojis(
folder: &Path,
group: String,
tmpFolder: &Path,
) -> Result<Vec<Emoji>, EmojiGenError> {
match folder.canonicalize() {
Ok(f) => {
if !f.is_dir() {
Err(
report!(EmojiGenError::EmojiFetchFailed).attach_printable(format!(
"Folder path '{}' is not a directory.",
folder.display()
)),
)
} else {
Ok(f)
}
}
Err(_) => Err(report!(EmojiGenError::EmojiFetchFailed)
.attach_printable(format!("Folder '{}' does not exist.", folder.display()))),
}?;
println!(
"Getting all the fine emojis from folder '{}'...",
folder.display()
);
let mut emojis = Vec::<Emoji>::new();
let iter = WalkDir::new(folder).into_iter();
let count = WalkDir::new(folder).into_iter().count() as u64;
for result in iter.progress_count(count) {
if let Err(_) = result {
continue;
}
let opt_file = result.ok();
if opt_file.is_none() {
continue;
}
let file = opt_file.unwrap();
if !file.metadata().unwrap().is_file() {
continue;
}
let filename = file.path().file_name().unwrap();
println!("Checking file '{}'...", filename.to_string_lossy());
let image = imghdr::from_file(file.path()).map_err(|_| {
report!(EmojiGenError::ImageFetchingFailed).attach_printable(format!(
"Could not get image at path '{}'.",
file.path().display()
))
})?;
if image.is_none() {
if filename.to_ascii_uppercase() == "LICENSE"
|| filename.to_ascii_uppercase() == "LICENSE.md"
{
get_image_from_path(&file.path(), tmpFolder)
.map_err(|err| warn!("{}", err))
.unwrap();
}
continue;
}
match get_local_emoji_data(file, group.clone().into(), tmpFolder) {
Ok(emoji) => {
emojis.push(emoji);
}
Err(err) => {
warn!("{}", err)
}
}
}
Ok(emojis)
}
fn get_local_emoji_data(
file: DirEntry,
original_category: Rc<String>,
tmpFolder: &Path,
) -> Result<Emoji, EmojiGenError> {
debug!("{}", file.path().display());
let fileName = String::from(file.file_name().to_str().unwrap());
let name = String::from(file.path().file_stem().unwrap().to_str().unwrap())
.replace(&[' ', '-'][..], "_");
get_image_from_path(&file.path(), tmpFolder)?;
let data = EmojiData {
name: name,
category: original_category.to_string(),
aliases: Vec::<String>::new(),
};
Ok(Emoji {
downloaded: true,
fileName,
emoji: data,
})
}
fn get_image_from_path(filePath: &Path, tmpFolder: &Path) -> Result<(), EmojiGenError> {
let filename = filePath.file_name().unwrap();
let img_data = std::fs::read(filePath.as_os_str()).map_err(|_| {
report!(EmojiGenError::ImageFetchingFailed).attach_printable(format!(
"Could not read image file at path '{}'.",
filePath.display()
))
})?;
let img_bytes = img_data.as_slice();
let imageFilePath = &tmpFolder.join(filename);
let mut imageFile = File::create(imageFilePath).map_err(|_| {
report!(EmojiGenError::ImageFetchingFailed).attach_printable(format!(
"Could not create image to temporary path '{}'.",
imageFilePath.display()
))
})?;
imageFile.write_all(&img_bytes).map_err(|_| {
report!(EmojiGenError::ImageFetchingFailed).attach_printable(format!(
"Could not write image to temporary path '{}'.",
imageFilePath.display()
))
})?;
Ok(())
}
fn generate_meta(host: String, emojis: Vec<Emoji>, tmpFolder: &Path) -> Result<(), EmojiGenError> {
let meta = Meta {
metaVersion: 1,
host: host,
exportedAt: Local::now().to_rfc3339(),
emojis: emojis,
};
let json = serde_json::to_string(&meta).map_err(|_| {
report!(EmojiGenError::MetadataGenerationFailed)
.attach_printable(format!("Could not generate metadata for 'meta.json'."))
})?;
let metaFilepath: &PathBuf = &tmpFolder.join("meta.json");
println!(
"Creating file 'meta.json' at path '{}'...",
metaFilepath.to_str().unwrap()
);
let mut file = File::create(metaFilepath).map_err(|_| {
report!(EmojiGenError::MetadataGenerationFailed).attach_printable(format!(
"Could not create file '{}'.",
metaFilepath.display()
))
})?;
write!(file, "{}", json).map_err(|_| {
report!(EmojiGenError::MetadataGenerationFailed).attach_printable(format!(
"Could not write metadata to file '{}'.",
metaFilepath.display()
))
})?;
Ok(())
}
fn zip(src_dir: &Path, dst_file: &Path) -> Result<(), EmojiGenError> {
if !std::path::Path::new(src_dir).is_dir() {
return Err(report!(EmojiGenError::ZipCreationFailed)
.attach_printable(format!("Could not find folder '{}'.", src_dir.display())));
}
println!("Creating zip file at path '{}'...", dst_file.display());
let zipFile = &File::create(dst_file).map_err(|_| {
report!(EmojiGenError::ZipCreationFailed)
.attach_printable(format!("Could not create file '{}'.", dst_file.display()))
})?;
let mut zip = zip::ZipWriter::new(zipFile);
let options = FileOptions::default()
.compression_method(zip::CompressionMethod::Deflated)
.unix_permissions(0o755);
let iter = WalkDir::new(src_dir).into_iter();
let count = WalkDir::new(src_dir).into_iter().count() as u64;
let mut buffer = Vec::new();
for entryRes in iter.progress_count(count) {
let entry = entryRes.map_err(|_| {
report!(EmojiGenError::ZipCreationFailed).attach_printable(format!(
"Could get path to a file in folder '{}'.",
src_dir.display()
))
})?;
let path = entry.path();
let name = path.strip_prefix(src_dir).map_err(|_| {
report!(EmojiGenError::ZipCreationFailed).attach_printable(format!(
"Could not strip prefix on file path '{}'.",
path.display()
))
})?;
if path.is_file() {
debug!("adding file {:?} as {:?} ...", path, name);
#[allow(deprecated)]
zip.start_file_from_path(name, options).map_err(|_| {
report!(EmojiGenError::ZipCreationFailed).attach_printable(format!(
"Could not create file '{}' in zip file '{:?}'.",
path.display(),
zipFile
))
})?;
let mut f = File::open(path).map_err(|_| {
report!(EmojiGenError::ZipCreationFailed)
.attach_printable(format!("Could not read file '{}'.", path.display()))
})?;
f.read_to_end(&mut buffer).map_err(|_| {
report!(EmojiGenError::ZipCreationFailed)
.attach_printable(format!("Could not read file '{}'.", path.display()))
})?;
zip.write_all(&*buffer).map_err(|_| {
report!(EmojiGenError::ZipCreationFailed)
.attach_printable(format!("Could not write data to zip file '{:?}'.", zipFile))
})?;
buffer.clear();
} else if !name.as_os_str().is_empty() {
debug!("adding dir {:?} as {:?} ...", path, name);
#[allow(deprecated)]
zip.add_directory_from_path(name, options).map_err(|_| {
report!(EmojiGenError::ZipCreationFailed).attach_printable(format!(
"Could not create folder '{}' in zip file '{:?}'.",
path.display(),
zipFile
))
})?;
}
}
zip.finish().map_err(|_| {
report!(EmojiGenError::ZipCreationFailed)
.attach_printable(format!("Could not close zip file '{:?}'.", zipFile))
})?;
Ok(())
}