Files
Ganonmaster 9c5212de05 feat: add emote import from legacy source with dry-run mode
Add POST /manage/import (auth-protected) that fetches emotes from a
legacy JSON endpoint, downloads each image, uploads it to S3, and
inserts it into the DB with the original timestamps preserved.

- Skip emotes whose name already exists (best-effort duplicate detection
  across SQLite and PostgreSQL via error code + message fallback)
- Validate source_url against a configurable host allowlist
  ([import] allowed_hosts in config, default ["smutba.se"])
- dry_run: true previews the import without writing to S3 or DB;
  result statuses are "would_import" / "would_skip" instead of
  "imported" / "skipped"
- Add db.name_exists() for efficient per-name existence checks used
  by dry-run
- Add reqwest (rustls-tls + json) and url dependencies
- Integration tests: auth guard, allowlist rejection, mirror +
  skip-duplicates, dry-run no-persist
2026-05-02 02:48:24 +02:00

595 lines
19 KiB
Rust

use std::sync::Arc;
use axum::{
body::Body,
http::{Request, StatusCode},
routing::get,
Json,
Router,
};
use base64::{engine::general_purpose::STANDARD, Engine};
use chrono::{DateTime, Utc};
use serde_json::json;
use sqlx::any::install_default_drivers;
use tower::ServiceExt;
use tokio::net::TcpListener;
use mikebase::{
build_router,
config::{AppConfig, AuthConfig, DatabaseConfig, ImportConfig, S3Config, ServerConfig},
db::Database,
models::new_uuid,
storage::S3Storage,
AppState,
};
// ── Helpers ───────────────────────────────────────────────────────────────────
async fn test_state() -> AppState {
test_state_with_s3(
"http://localhost:19999".to_string(),
"http://localhost:19999/test-bucket".to_string(),
)
.await
}
async fn test_state_with_s3(s3_endpoint: String, s3_public_url: String) -> AppState {
install_default_drivers();
let pool = sqlx::pool::PoolOptions::<sqlx::Any>::new()
.max_connections(1)
.connect("sqlite::memory:")
.await
.unwrap();
let db = Database { pool };
db.migrate().await.unwrap();
let cfg = Arc::new(AppConfig {
s3: S3Config {
endpoint: s3_endpoint,
region: "us-east-1".to_string(),
bucket: "test-bucket".to_string(),
access_key: "test".to_string(),
secret_key: "test".to_string(),
public_url: s3_public_url,
},
database: DatabaseConfig { url: "sqlite::memory:".to_string() },
server: ServerConfig::default(),
auth: Some(AuthConfig {
username: "admin".to_string(),
password: "secret".to_string(),
}),
import: ImportConfig {
allowed_hosts: vec!["smutba.se".to_string(), "localhost".to_string(), "127.0.0.1".to_string()],
},
});
let storage = S3Storage::new(&cfg);
AppState { db, storage, cfg }
}
async fn spawn_mock_s3_server() -> (String, tokio::task::JoinHandle<()>) {
async fn ok() -> StatusCode {
StatusCode::OK
}
let listener = TcpListener::bind("127.0.0.1:0").await.unwrap();
let addr = listener.local_addr().unwrap();
let base = format!("http://{}", addr);
let app = Router::new()
.route("/", get(ok))
.route("/{*path}", axum::routing::any(ok));
let handle = tokio::spawn(async move {
axum::serve(listener, app).await.unwrap();
});
(base, handle)
}
async fn spawn_legacy_source_server() -> (String, tokio::task::JoinHandle<()>) {
async fn image_new() -> ([(&'static str, &'static str); 1], &'static [u8]) {
([ ("content-type", "image/png") ], b"PNGDATA")
}
async fn image_dup() -> ([(&'static str, &'static str); 1], &'static [u8]) {
([ ("content-type", "image/png") ], b"PNGDATA2")
}
let listener = TcpListener::bind("127.0.0.1:0").await.unwrap();
let addr = listener.local_addr().unwrap();
let base = format!("http://{}", addr);
let payload_base = base.clone();
let app = Router::new()
.route(
"/emoji/json/",
get(move || {
let payload_base = payload_base.clone();
async move {
Json(json!({
"emotes": [
{
"name": "legacy_new",
"url": format!("{}/images/new.png", payload_base),
"created": "2020-01-01T00:00:00+00:00",
"modified": "2020-02-02T00:00:00+00:00"
},
{
"name": "legacy_duplicate",
"url": format!("{}/images/duplicate.png", payload_base),
"created": "2021-01-01T00:00:00+00:00",
"modified": "2021-01-02T00:00:00+00:00"
}
]
}))
}
}),
)
.route("/images/new.png", get(image_new))
.route("/images/duplicate.png", get(image_dup));
let handle = tokio::spawn(async move {
axum::serve(listener, app).await.unwrap();
});
(base, handle)
}
fn auth_header() -> String {
format!("Basic {}", STANDARD.encode("admin:secret"))
}
fn wrong_auth_header() -> String {
format!("Basic {}", STANDARD.encode("admin:wrong"))
}
async fn response_json(body: Body) -> serde_json::Value {
let bytes = axum::body::to_bytes(body, usize::MAX).await.unwrap();
serde_json::from_slice(&bytes).unwrap()
}
/// Build a minimal multipart/form-data body.
fn multipart_body(boundary: &str, parts: &[(&str, Option<&str>, &[u8])]) -> Vec<u8> {
let mut body = Vec::new();
for (name, filename, data) in parts {
let disp = match filename {
Some(fname) => format!(
"Content-Disposition: form-data; name=\"{name}\"; filename=\"{fname}\"\r\nContent-Type: application/octet-stream"
),
None => format!("Content-Disposition: form-data; name=\"{name}\""),
};
body.extend_from_slice(format!("--{boundary}\r\n{disp}\r\n\r\n").as_bytes());
body.extend_from_slice(data);
body.extend_from_slice(b"\r\n");
}
body.extend_from_slice(format!("--{boundary}--\r\n").as_bytes());
body
}
// ── Public routes ─────────────────────────────────────────────────────────────
#[tokio::test]
async fn health_returns_200() {
let app = build_router(test_state().await);
let resp = app
.oneshot(Request::builder().uri("/health").body(Body::empty()).unwrap())
.await
.unwrap();
assert_eq!(resp.status(), StatusCode::OK);
let json = response_json(resp.into_body()).await;
assert_eq!(json["status"], "ok");
}
#[tokio::test]
async fn root_returns_html() {
let app = build_router(test_state().await);
let resp = app
.oneshot(Request::builder().uri("/").body(Body::empty()).unwrap())
.await
.unwrap();
assert_eq!(resp.status(), StatusCode::OK);
assert!(resp
.headers()
.get("content-type")
.unwrap()
.to_str()
.unwrap()
.contains("text/html"));
}
#[tokio::test]
async fn list_emotes_empty() {
let app = build_router(test_state().await);
let resp = app
.oneshot(Request::builder().uri("/json").body(Body::empty()).unwrap())
.await
.unwrap();
assert_eq!(resp.status(), StatusCode::OK);
let json = response_json(resp.into_body()).await;
assert_eq!(json["emotes"], serde_json::json!([]));
}
#[tokio::test]
async fn list_emotes_returns_seeded_data() {
let state = test_state().await;
let id = new_uuid();
state
.db
.create_emote(&id, "wave", Some("hello"), "emoji/wave.png")
.await
.unwrap();
let resp = build_router(state)
.oneshot(Request::builder().uri("/json").body(Body::empty()).unwrap())
.await
.unwrap();
assert_eq!(resp.status(), StatusCode::OK);
let json = response_json(resp.into_body()).await;
assert_eq!(json["emotes"].as_array().unwrap().len(), 1);
assert_eq!(json["emotes"][0]["name"], "wave");
}
// ── Auth middleware ───────────────────────────────────────────────────────────
#[tokio::test]
async fn manage_without_credentials_returns_401() {
let app = build_router(test_state().await);
let resp = app
.oneshot(Request::builder().uri("/manage").body(Body::empty()).unwrap())
.await
.unwrap();
assert_eq!(resp.status(), StatusCode::UNAUTHORIZED);
}
#[tokio::test]
async fn manage_with_wrong_credentials_returns_401() {
let app = build_router(test_state().await);
let resp = app
.oneshot(
Request::builder()
.uri("/manage")
.header("authorization", wrong_auth_header())
.body(Body::empty())
.unwrap(),
)
.await
.unwrap();
assert_eq!(resp.status(), StatusCode::UNAUTHORIZED);
}
#[tokio::test]
async fn manage_with_correct_credentials_returns_200() {
let app = build_router(test_state().await);
let resp = app
.oneshot(
Request::builder()
.uri("/manage")
.header("authorization", auth_header())
.body(Body::empty())
.unwrap(),
)
.await
.unwrap();
assert_eq!(resp.status(), StatusCode::OK);
}
#[tokio::test]
async fn manage_emotes_requires_auth() {
let app = build_router(test_state().await);
let resp = app
.oneshot(
Request::builder()
.uri("/manage/emotes")
.body(Body::empty())
.unwrap(),
)
.await
.unwrap();
assert_eq!(resp.status(), StatusCode::UNAUTHORIZED);
}
#[tokio::test]
async fn manage_import_requires_auth() {
let app = build_router(test_state().await);
let resp = app
.oneshot(
Request::builder()
.method("POST")
.uri("/manage/import")
.header("content-type", "application/json")
.body(Body::from(r#"{"source_url":"https://smutba.se/emoji/json/"}"#))
.unwrap(),
)
.await
.unwrap();
assert_eq!(resp.status(), StatusCode::UNAUTHORIZED);
}
#[tokio::test]
async fn manage_import_rejects_non_allowlisted_host() {
let app = build_router(test_state().await);
let resp = app
.oneshot(
Request::builder()
.method("POST")
.uri("/manage/import")
.header("authorization", auth_header())
.header("content-type", "application/json")
.body(Body::from(r#"{"source_url":"https://example.com/emoji/json/"}"#))
.unwrap(),
)
.await
.unwrap();
assert_eq!(resp.status(), StatusCode::BAD_REQUEST);
}
#[tokio::test]
async fn manage_import_mirrors_and_skips_duplicates() {
let (legacy_base, server_handle) = spawn_legacy_source_server().await;
let (s3_base, s3_handle) = spawn_mock_s3_server().await;
let state = test_state_with_s3(s3_base.clone(), format!("{}/test-bucket", s3_base)).await;
// Pre-seed one duplicate emote name.
let existing_id = new_uuid();
state
.db
.create_emote(&existing_id, "legacy_duplicate", None, "emoji/existing.png")
.await
.unwrap();
let app = build_router(state.clone());
let resp = app
.oneshot(
Request::builder()
.method("POST")
.uri("/manage/import")
.header("authorization", auth_header())
.header("content-type", "application/json")
.body(Body::from(format!(
"{{\"source_url\":\"{}/emoji/json/\"}}",
legacy_base
)))
.unwrap(),
)
.await
.unwrap();
assert_eq!(resp.status(), StatusCode::OK);
let json = response_json(resp.into_body()).await;
assert_eq!(json["total"], 2);
assert_eq!(json["imported"], 1);
assert_eq!(json["skipped"], 1);
assert_eq!(json["failed"], 0);
let rows = state.db.list_emotes().await.unwrap();
let imported = rows.iter().find(|r| r.name == "legacy_new").unwrap();
let created = DateTime::parse_from_rfc3339(&imported.created)
.unwrap()
.with_timezone(&Utc)
.to_rfc3339();
let modified = DateTime::parse_from_rfc3339(&imported.modified)
.unwrap()
.with_timezone(&Utc)
.to_rfc3339();
assert_eq!(created, "2020-01-01T00:00:00+00:00");
assert_eq!(modified, "2020-02-02T00:00:00+00:00");
server_handle.abort();
s3_handle.abort();
}
#[tokio::test]
async fn manage_import_dry_run_does_not_persist() {
let (legacy_base, server_handle) = spawn_legacy_source_server().await;
let (s3_base, s3_handle) = spawn_mock_s3_server().await;
let state = test_state_with_s3(s3_base.clone(), format!("{}/test-bucket", s3_base)).await;
// Pre-seed one duplicate so we can verify would_skip detection.
let existing_id = new_uuid();
state
.db
.create_emote(&existing_id, "legacy_duplicate", None, "emoji/existing.png")
.await
.unwrap();
let app = build_router(state.clone());
let body = format!(
"{{\"source_url\":\"{}/emoji/json/\",\"dry_run\":true}}",
legacy_base
);
let resp = app
.oneshot(
Request::builder()
.method("POST")
.uri("/manage/import")
.header("authorization", auth_header())
.header("content-type", "application/json")
.body(Body::from(body))
.unwrap(),
)
.await
.unwrap();
assert_eq!(resp.status(), StatusCode::OK);
let json = response_json(resp.into_body()).await;
assert_eq!(json["dry_run"], true);
assert_eq!(json["total"], 2);
assert_eq!(json["imported"], 1);
assert_eq!(json["skipped"], 1);
assert_eq!(json["failed"], 0);
let results = json["results"].as_array().unwrap();
let new_result = results.iter().find(|r| r["name"] == "legacy_new").unwrap();
assert_eq!(new_result["status"], "would_import");
let dup_result = results.iter().find(|r| r["name"] == "legacy_duplicate").unwrap();
assert_eq!(dup_result["status"], "would_skip");
// Nothing new should have been written to the DB.
let rows = state.db.list_emotes().await.unwrap();
assert_eq!(rows.len(), 1, "dry-run must not insert any rows");
assert_eq!(rows[0].name, "legacy_duplicate");
server_handle.abort();
s3_handle.abort();
}
// ── POST /emotes input validation ─────────────────────────────────────────────
#[tokio::test]
async fn create_emote_without_auth_returns_401() {
let boundary = "testboundary";
let body = multipart_body(boundary, &[("name", None, b"myemote")]);
let app = build_router(test_state().await);
let resp = app
.oneshot(
Request::builder()
.method("POST")
.uri("/emotes")
.header("content-type", format!("multipart/form-data; boundary={boundary}"))
.body(Body::from(body))
.unwrap(),
)
.await
.unwrap();
assert_eq!(resp.status(), StatusCode::UNAUTHORIZED);
}
#[tokio::test]
async fn create_emote_missing_name_returns_400() {
let boundary = "testboundary";
let body = multipart_body(boundary, &[("file", Some("cat.png"), b"\x89PNG")]);
let app = build_router(test_state().await);
let resp = app
.oneshot(
Request::builder()
.method("POST")
.uri("/emotes")
.header("authorization", auth_header())
.header("content-type", format!("multipart/form-data; boundary={boundary}"))
.body(Body::from(body))
.unwrap(),
)
.await
.unwrap();
assert_eq!(resp.status(), StatusCode::BAD_REQUEST);
let json = response_json(resp.into_body()).await;
assert!(json["error"].as_str().unwrap().contains("name"));
}
#[tokio::test]
async fn create_emote_missing_file_returns_400() {
let boundary = "testboundary";
let body = multipart_body(boundary, &[("name", None, b"myemote")]);
let app = build_router(test_state().await);
let resp = app
.oneshot(
Request::builder()
.method("POST")
.uri("/emotes")
.header("authorization", auth_header())
.header("content-type", format!("multipart/form-data; boundary={boundary}"))
.body(Body::from(body))
.unwrap(),
)
.await
.unwrap();
assert_eq!(resp.status(), StatusCode::BAD_REQUEST);
let json = response_json(resp.into_body()).await;
assert!(json["error"].as_str().unwrap().contains("file"));
}
// ── PUT /emotes/{uuid} ────────────────────────────────────────────────────────
#[tokio::test]
async fn update_emote_not_found_returns_404() {
let app = build_router(test_state().await);
let resp = app
.oneshot(
Request::builder()
.method("PUT")
.uri("/emotes/no-such-uuid")
.header("authorization", auth_header())
.header("content-type", "application/json")
.body(Body::from(r#"{"name":"new"}"#))
.unwrap(),
)
.await
.unwrap();
assert_eq!(resp.status(), StatusCode::NOT_FOUND);
}
#[tokio::test]
async fn update_emote_ok() {
let state = test_state().await;
let id = new_uuid();
state
.db
.create_emote(&id, "original", None, "emoji/x.png")
.await
.unwrap();
let resp = build_router(state)
.oneshot(
Request::builder()
.method("PUT")
.uri(format!("/emotes/{id}"))
.header("authorization", auth_header())
.header("content-type", "application/json")
.body(Body::from(r#"{"name":"updated"}"#))
.unwrap(),
)
.await
.unwrap();
assert_eq!(resp.status(), StatusCode::OK);
let json = response_json(resp.into_body()).await;
assert_eq!(json["name"], "updated");
}
// ── DELETE /emotes/{uuid} ─────────────────────────────────────────────────────
#[tokio::test]
async fn delete_emote_not_found_returns_404() {
let app = build_router(test_state().await);
let resp = app
.oneshot(
Request::builder()
.method("DELETE")
.uri("/emotes/no-such-uuid")
.header("authorization", auth_header())
.body(Body::empty())
.unwrap(),
)
.await
.unwrap();
assert_eq!(resp.status(), StatusCode::NOT_FOUND);
}
#[tokio::test]
async fn delete_emote_ok() {
let state = test_state().await;
let id = new_uuid();
state
.db
.create_emote(&id, "todelete", None, "emoji/gone.png")
.await
.unwrap();
// S3 delete will fail against the fake endpoint, but it is best-effort.
let resp = build_router(state)
.oneshot(
Request::builder()
.method("DELETE")
.uri(format!("/emotes/{id}"))
.header("authorization", auth_header())
.body(Body::empty())
.unwrap(),
)
.await
.unwrap();
assert_eq!(resp.status(), StatusCode::NO_CONTENT);
}