9c5212de05
Add POST /manage/import (auth-protected) that fetches emotes from a legacy JSON endpoint, downloads each image, uploads it to S3, and inserts it into the DB with the original timestamps preserved. - Skip emotes whose name already exists (best-effort duplicate detection across SQLite and PostgreSQL via error code + message fallback) - Validate source_url against a configurable host allowlist ([import] allowed_hosts in config, default ["smutba.se"]) - dry_run: true previews the import without writing to S3 or DB; result statuses are "would_import" / "would_skip" instead of "imported" / "skipped" - Add db.name_exists() for efficient per-name existence checks used by dry-run - Add reqwest (rustls-tls + json) and url dependencies - Integration tests: auth guard, allowlist rejection, mirror + skip-duplicates, dry-run no-persist
595 lines
19 KiB
Rust
595 lines
19 KiB
Rust
use std::sync::Arc;
|
|
|
|
use axum::{
|
|
body::Body,
|
|
http::{Request, StatusCode},
|
|
routing::get,
|
|
Json,
|
|
Router,
|
|
};
|
|
use base64::{engine::general_purpose::STANDARD, Engine};
|
|
use chrono::{DateTime, Utc};
|
|
use serde_json::json;
|
|
use sqlx::any::install_default_drivers;
|
|
use tower::ServiceExt;
|
|
use tokio::net::TcpListener;
|
|
|
|
use mikebase::{
|
|
build_router,
|
|
config::{AppConfig, AuthConfig, DatabaseConfig, ImportConfig, S3Config, ServerConfig},
|
|
db::Database,
|
|
models::new_uuid,
|
|
storage::S3Storage,
|
|
AppState,
|
|
};
|
|
|
|
// ── Helpers ───────────────────────────────────────────────────────────────────
|
|
|
|
async fn test_state() -> AppState {
|
|
test_state_with_s3(
|
|
"http://localhost:19999".to_string(),
|
|
"http://localhost:19999/test-bucket".to_string(),
|
|
)
|
|
.await
|
|
}
|
|
|
|
async fn test_state_with_s3(s3_endpoint: String, s3_public_url: String) -> AppState {
|
|
install_default_drivers();
|
|
let pool = sqlx::pool::PoolOptions::<sqlx::Any>::new()
|
|
.max_connections(1)
|
|
.connect("sqlite::memory:")
|
|
.await
|
|
.unwrap();
|
|
let db = Database { pool };
|
|
db.migrate().await.unwrap();
|
|
|
|
let cfg = Arc::new(AppConfig {
|
|
s3: S3Config {
|
|
endpoint: s3_endpoint,
|
|
region: "us-east-1".to_string(),
|
|
bucket: "test-bucket".to_string(),
|
|
access_key: "test".to_string(),
|
|
secret_key: "test".to_string(),
|
|
public_url: s3_public_url,
|
|
},
|
|
database: DatabaseConfig { url: "sqlite::memory:".to_string() },
|
|
server: ServerConfig::default(),
|
|
auth: Some(AuthConfig {
|
|
username: "admin".to_string(),
|
|
password: "secret".to_string(),
|
|
}),
|
|
import: ImportConfig {
|
|
allowed_hosts: vec!["smutba.se".to_string(), "localhost".to_string(), "127.0.0.1".to_string()],
|
|
},
|
|
});
|
|
|
|
let storage = S3Storage::new(&cfg);
|
|
AppState { db, storage, cfg }
|
|
}
|
|
|
|
async fn spawn_mock_s3_server() -> (String, tokio::task::JoinHandle<()>) {
|
|
async fn ok() -> StatusCode {
|
|
StatusCode::OK
|
|
}
|
|
|
|
let listener = TcpListener::bind("127.0.0.1:0").await.unwrap();
|
|
let addr = listener.local_addr().unwrap();
|
|
let base = format!("http://{}", addr);
|
|
|
|
let app = Router::new()
|
|
.route("/", get(ok))
|
|
.route("/{*path}", axum::routing::any(ok));
|
|
|
|
let handle = tokio::spawn(async move {
|
|
axum::serve(listener, app).await.unwrap();
|
|
});
|
|
|
|
(base, handle)
|
|
}
|
|
|
|
async fn spawn_legacy_source_server() -> (String, tokio::task::JoinHandle<()>) {
|
|
async fn image_new() -> ([(&'static str, &'static str); 1], &'static [u8]) {
|
|
([ ("content-type", "image/png") ], b"PNGDATA")
|
|
}
|
|
|
|
async fn image_dup() -> ([(&'static str, &'static str); 1], &'static [u8]) {
|
|
([ ("content-type", "image/png") ], b"PNGDATA2")
|
|
}
|
|
|
|
let listener = TcpListener::bind("127.0.0.1:0").await.unwrap();
|
|
let addr = listener.local_addr().unwrap();
|
|
let base = format!("http://{}", addr);
|
|
|
|
let payload_base = base.clone();
|
|
let app = Router::new()
|
|
.route(
|
|
"/emoji/json/",
|
|
get(move || {
|
|
let payload_base = payload_base.clone();
|
|
async move {
|
|
Json(json!({
|
|
"emotes": [
|
|
{
|
|
"name": "legacy_new",
|
|
"url": format!("{}/images/new.png", payload_base),
|
|
"created": "2020-01-01T00:00:00+00:00",
|
|
"modified": "2020-02-02T00:00:00+00:00"
|
|
},
|
|
{
|
|
"name": "legacy_duplicate",
|
|
"url": format!("{}/images/duplicate.png", payload_base),
|
|
"created": "2021-01-01T00:00:00+00:00",
|
|
"modified": "2021-01-02T00:00:00+00:00"
|
|
}
|
|
]
|
|
}))
|
|
}
|
|
}),
|
|
)
|
|
.route("/images/new.png", get(image_new))
|
|
.route("/images/duplicate.png", get(image_dup));
|
|
|
|
let handle = tokio::spawn(async move {
|
|
axum::serve(listener, app).await.unwrap();
|
|
});
|
|
|
|
(base, handle)
|
|
}
|
|
|
|
fn auth_header() -> String {
|
|
format!("Basic {}", STANDARD.encode("admin:secret"))
|
|
}
|
|
|
|
fn wrong_auth_header() -> String {
|
|
format!("Basic {}", STANDARD.encode("admin:wrong"))
|
|
}
|
|
|
|
async fn response_json(body: Body) -> serde_json::Value {
|
|
let bytes = axum::body::to_bytes(body, usize::MAX).await.unwrap();
|
|
serde_json::from_slice(&bytes).unwrap()
|
|
}
|
|
|
|
/// Build a minimal multipart/form-data body.
|
|
fn multipart_body(boundary: &str, parts: &[(&str, Option<&str>, &[u8])]) -> Vec<u8> {
|
|
let mut body = Vec::new();
|
|
for (name, filename, data) in parts {
|
|
let disp = match filename {
|
|
Some(fname) => format!(
|
|
"Content-Disposition: form-data; name=\"{name}\"; filename=\"{fname}\"\r\nContent-Type: application/octet-stream"
|
|
),
|
|
None => format!("Content-Disposition: form-data; name=\"{name}\""),
|
|
};
|
|
body.extend_from_slice(format!("--{boundary}\r\n{disp}\r\n\r\n").as_bytes());
|
|
body.extend_from_slice(data);
|
|
body.extend_from_slice(b"\r\n");
|
|
}
|
|
body.extend_from_slice(format!("--{boundary}--\r\n").as_bytes());
|
|
body
|
|
}
|
|
|
|
// ── Public routes ─────────────────────────────────────────────────────────────
|
|
|
|
#[tokio::test]
|
|
async fn health_returns_200() {
|
|
let app = build_router(test_state().await);
|
|
let resp = app
|
|
.oneshot(Request::builder().uri("/health").body(Body::empty()).unwrap())
|
|
.await
|
|
.unwrap();
|
|
assert_eq!(resp.status(), StatusCode::OK);
|
|
let json = response_json(resp.into_body()).await;
|
|
assert_eq!(json["status"], "ok");
|
|
}
|
|
|
|
#[tokio::test]
|
|
async fn root_returns_html() {
|
|
let app = build_router(test_state().await);
|
|
let resp = app
|
|
.oneshot(Request::builder().uri("/").body(Body::empty()).unwrap())
|
|
.await
|
|
.unwrap();
|
|
assert_eq!(resp.status(), StatusCode::OK);
|
|
assert!(resp
|
|
.headers()
|
|
.get("content-type")
|
|
.unwrap()
|
|
.to_str()
|
|
.unwrap()
|
|
.contains("text/html"));
|
|
}
|
|
|
|
#[tokio::test]
|
|
async fn list_emotes_empty() {
|
|
let app = build_router(test_state().await);
|
|
let resp = app
|
|
.oneshot(Request::builder().uri("/json").body(Body::empty()).unwrap())
|
|
.await
|
|
.unwrap();
|
|
assert_eq!(resp.status(), StatusCode::OK);
|
|
let json = response_json(resp.into_body()).await;
|
|
assert_eq!(json["emotes"], serde_json::json!([]));
|
|
}
|
|
|
|
#[tokio::test]
|
|
async fn list_emotes_returns_seeded_data() {
|
|
let state = test_state().await;
|
|
let id = new_uuid();
|
|
state
|
|
.db
|
|
.create_emote(&id, "wave", Some("hello"), "emoji/wave.png")
|
|
.await
|
|
.unwrap();
|
|
|
|
let resp = build_router(state)
|
|
.oneshot(Request::builder().uri("/json").body(Body::empty()).unwrap())
|
|
.await
|
|
.unwrap();
|
|
|
|
assert_eq!(resp.status(), StatusCode::OK);
|
|
let json = response_json(resp.into_body()).await;
|
|
assert_eq!(json["emotes"].as_array().unwrap().len(), 1);
|
|
assert_eq!(json["emotes"][0]["name"], "wave");
|
|
}
|
|
|
|
// ── Auth middleware ───────────────────────────────────────────────────────────
|
|
|
|
#[tokio::test]
|
|
async fn manage_without_credentials_returns_401() {
|
|
let app = build_router(test_state().await);
|
|
let resp = app
|
|
.oneshot(Request::builder().uri("/manage").body(Body::empty()).unwrap())
|
|
.await
|
|
.unwrap();
|
|
assert_eq!(resp.status(), StatusCode::UNAUTHORIZED);
|
|
}
|
|
|
|
#[tokio::test]
|
|
async fn manage_with_wrong_credentials_returns_401() {
|
|
let app = build_router(test_state().await);
|
|
let resp = app
|
|
.oneshot(
|
|
Request::builder()
|
|
.uri("/manage")
|
|
.header("authorization", wrong_auth_header())
|
|
.body(Body::empty())
|
|
.unwrap(),
|
|
)
|
|
.await
|
|
.unwrap();
|
|
assert_eq!(resp.status(), StatusCode::UNAUTHORIZED);
|
|
}
|
|
|
|
#[tokio::test]
|
|
async fn manage_with_correct_credentials_returns_200() {
|
|
let app = build_router(test_state().await);
|
|
let resp = app
|
|
.oneshot(
|
|
Request::builder()
|
|
.uri("/manage")
|
|
.header("authorization", auth_header())
|
|
.body(Body::empty())
|
|
.unwrap(),
|
|
)
|
|
.await
|
|
.unwrap();
|
|
assert_eq!(resp.status(), StatusCode::OK);
|
|
}
|
|
|
|
#[tokio::test]
|
|
async fn manage_emotes_requires_auth() {
|
|
let app = build_router(test_state().await);
|
|
let resp = app
|
|
.oneshot(
|
|
Request::builder()
|
|
.uri("/manage/emotes")
|
|
.body(Body::empty())
|
|
.unwrap(),
|
|
)
|
|
.await
|
|
.unwrap();
|
|
assert_eq!(resp.status(), StatusCode::UNAUTHORIZED);
|
|
}
|
|
|
|
#[tokio::test]
|
|
async fn manage_import_requires_auth() {
|
|
let app = build_router(test_state().await);
|
|
let resp = app
|
|
.oneshot(
|
|
Request::builder()
|
|
.method("POST")
|
|
.uri("/manage/import")
|
|
.header("content-type", "application/json")
|
|
.body(Body::from(r#"{"source_url":"https://smutba.se/emoji/json/"}"#))
|
|
.unwrap(),
|
|
)
|
|
.await
|
|
.unwrap();
|
|
assert_eq!(resp.status(), StatusCode::UNAUTHORIZED);
|
|
}
|
|
|
|
#[tokio::test]
|
|
async fn manage_import_rejects_non_allowlisted_host() {
|
|
let app = build_router(test_state().await);
|
|
let resp = app
|
|
.oneshot(
|
|
Request::builder()
|
|
.method("POST")
|
|
.uri("/manage/import")
|
|
.header("authorization", auth_header())
|
|
.header("content-type", "application/json")
|
|
.body(Body::from(r#"{"source_url":"https://example.com/emoji/json/"}"#))
|
|
.unwrap(),
|
|
)
|
|
.await
|
|
.unwrap();
|
|
assert_eq!(resp.status(), StatusCode::BAD_REQUEST);
|
|
}
|
|
|
|
#[tokio::test]
|
|
async fn manage_import_mirrors_and_skips_duplicates() {
|
|
let (legacy_base, server_handle) = spawn_legacy_source_server().await;
|
|
let (s3_base, s3_handle) = spawn_mock_s3_server().await;
|
|
|
|
let state = test_state_with_s3(s3_base.clone(), format!("{}/test-bucket", s3_base)).await;
|
|
// Pre-seed one duplicate emote name.
|
|
let existing_id = new_uuid();
|
|
state
|
|
.db
|
|
.create_emote(&existing_id, "legacy_duplicate", None, "emoji/existing.png")
|
|
.await
|
|
.unwrap();
|
|
|
|
let app = build_router(state.clone());
|
|
let resp = app
|
|
.oneshot(
|
|
Request::builder()
|
|
.method("POST")
|
|
.uri("/manage/import")
|
|
.header("authorization", auth_header())
|
|
.header("content-type", "application/json")
|
|
.body(Body::from(format!(
|
|
"{{\"source_url\":\"{}/emoji/json/\"}}",
|
|
legacy_base
|
|
)))
|
|
.unwrap(),
|
|
)
|
|
.await
|
|
.unwrap();
|
|
|
|
assert_eq!(resp.status(), StatusCode::OK);
|
|
let json = response_json(resp.into_body()).await;
|
|
assert_eq!(json["total"], 2);
|
|
assert_eq!(json["imported"], 1);
|
|
assert_eq!(json["skipped"], 1);
|
|
assert_eq!(json["failed"], 0);
|
|
|
|
let rows = state.db.list_emotes().await.unwrap();
|
|
let imported = rows.iter().find(|r| r.name == "legacy_new").unwrap();
|
|
let created = DateTime::parse_from_rfc3339(&imported.created)
|
|
.unwrap()
|
|
.with_timezone(&Utc)
|
|
.to_rfc3339();
|
|
let modified = DateTime::parse_from_rfc3339(&imported.modified)
|
|
.unwrap()
|
|
.with_timezone(&Utc)
|
|
.to_rfc3339();
|
|
assert_eq!(created, "2020-01-01T00:00:00+00:00");
|
|
assert_eq!(modified, "2020-02-02T00:00:00+00:00");
|
|
|
|
server_handle.abort();
|
|
s3_handle.abort();
|
|
}
|
|
|
|
#[tokio::test]
|
|
async fn manage_import_dry_run_does_not_persist() {
|
|
let (legacy_base, server_handle) = spawn_legacy_source_server().await;
|
|
let (s3_base, s3_handle) = spawn_mock_s3_server().await;
|
|
|
|
let state = test_state_with_s3(s3_base.clone(), format!("{}/test-bucket", s3_base)).await;
|
|
// Pre-seed one duplicate so we can verify would_skip detection.
|
|
let existing_id = new_uuid();
|
|
state
|
|
.db
|
|
.create_emote(&existing_id, "legacy_duplicate", None, "emoji/existing.png")
|
|
.await
|
|
.unwrap();
|
|
|
|
let app = build_router(state.clone());
|
|
let body = format!(
|
|
"{{\"source_url\":\"{}/emoji/json/\",\"dry_run\":true}}",
|
|
legacy_base
|
|
);
|
|
let resp = app
|
|
.oneshot(
|
|
Request::builder()
|
|
.method("POST")
|
|
.uri("/manage/import")
|
|
.header("authorization", auth_header())
|
|
.header("content-type", "application/json")
|
|
.body(Body::from(body))
|
|
.unwrap(),
|
|
)
|
|
.await
|
|
.unwrap();
|
|
|
|
assert_eq!(resp.status(), StatusCode::OK);
|
|
let json = response_json(resp.into_body()).await;
|
|
assert_eq!(json["dry_run"], true);
|
|
assert_eq!(json["total"], 2);
|
|
assert_eq!(json["imported"], 1);
|
|
assert_eq!(json["skipped"], 1);
|
|
assert_eq!(json["failed"], 0);
|
|
|
|
let results = json["results"].as_array().unwrap();
|
|
let new_result = results.iter().find(|r| r["name"] == "legacy_new").unwrap();
|
|
assert_eq!(new_result["status"], "would_import");
|
|
let dup_result = results.iter().find(|r| r["name"] == "legacy_duplicate").unwrap();
|
|
assert_eq!(dup_result["status"], "would_skip");
|
|
|
|
// Nothing new should have been written to the DB.
|
|
let rows = state.db.list_emotes().await.unwrap();
|
|
assert_eq!(rows.len(), 1, "dry-run must not insert any rows");
|
|
assert_eq!(rows[0].name, "legacy_duplicate");
|
|
|
|
server_handle.abort();
|
|
s3_handle.abort();
|
|
}
|
|
|
|
// ── POST /emotes input validation ─────────────────────────────────────────────
|
|
|
|
#[tokio::test]
|
|
async fn create_emote_without_auth_returns_401() {
|
|
let boundary = "testboundary";
|
|
let body = multipart_body(boundary, &[("name", None, b"myemote")]);
|
|
let app = build_router(test_state().await);
|
|
let resp = app
|
|
.oneshot(
|
|
Request::builder()
|
|
.method("POST")
|
|
.uri("/emotes")
|
|
.header("content-type", format!("multipart/form-data; boundary={boundary}"))
|
|
.body(Body::from(body))
|
|
.unwrap(),
|
|
)
|
|
.await
|
|
.unwrap();
|
|
assert_eq!(resp.status(), StatusCode::UNAUTHORIZED);
|
|
}
|
|
|
|
#[tokio::test]
|
|
async fn create_emote_missing_name_returns_400() {
|
|
let boundary = "testboundary";
|
|
let body = multipart_body(boundary, &[("file", Some("cat.png"), b"\x89PNG")]);
|
|
let app = build_router(test_state().await);
|
|
let resp = app
|
|
.oneshot(
|
|
Request::builder()
|
|
.method("POST")
|
|
.uri("/emotes")
|
|
.header("authorization", auth_header())
|
|
.header("content-type", format!("multipart/form-data; boundary={boundary}"))
|
|
.body(Body::from(body))
|
|
.unwrap(),
|
|
)
|
|
.await
|
|
.unwrap();
|
|
assert_eq!(resp.status(), StatusCode::BAD_REQUEST);
|
|
let json = response_json(resp.into_body()).await;
|
|
assert!(json["error"].as_str().unwrap().contains("name"));
|
|
}
|
|
|
|
#[tokio::test]
|
|
async fn create_emote_missing_file_returns_400() {
|
|
let boundary = "testboundary";
|
|
let body = multipart_body(boundary, &[("name", None, b"myemote")]);
|
|
let app = build_router(test_state().await);
|
|
let resp = app
|
|
.oneshot(
|
|
Request::builder()
|
|
.method("POST")
|
|
.uri("/emotes")
|
|
.header("authorization", auth_header())
|
|
.header("content-type", format!("multipart/form-data; boundary={boundary}"))
|
|
.body(Body::from(body))
|
|
.unwrap(),
|
|
)
|
|
.await
|
|
.unwrap();
|
|
assert_eq!(resp.status(), StatusCode::BAD_REQUEST);
|
|
let json = response_json(resp.into_body()).await;
|
|
assert!(json["error"].as_str().unwrap().contains("file"));
|
|
}
|
|
|
|
// ── PUT /emotes/{uuid} ────────────────────────────────────────────────────────
|
|
|
|
#[tokio::test]
|
|
async fn update_emote_not_found_returns_404() {
|
|
let app = build_router(test_state().await);
|
|
let resp = app
|
|
.oneshot(
|
|
Request::builder()
|
|
.method("PUT")
|
|
.uri("/emotes/no-such-uuid")
|
|
.header("authorization", auth_header())
|
|
.header("content-type", "application/json")
|
|
.body(Body::from(r#"{"name":"new"}"#))
|
|
.unwrap(),
|
|
)
|
|
.await
|
|
.unwrap();
|
|
assert_eq!(resp.status(), StatusCode::NOT_FOUND);
|
|
}
|
|
|
|
#[tokio::test]
|
|
async fn update_emote_ok() {
|
|
let state = test_state().await;
|
|
let id = new_uuid();
|
|
state
|
|
.db
|
|
.create_emote(&id, "original", None, "emoji/x.png")
|
|
.await
|
|
.unwrap();
|
|
|
|
let resp = build_router(state)
|
|
.oneshot(
|
|
Request::builder()
|
|
.method("PUT")
|
|
.uri(format!("/emotes/{id}"))
|
|
.header("authorization", auth_header())
|
|
.header("content-type", "application/json")
|
|
.body(Body::from(r#"{"name":"updated"}"#))
|
|
.unwrap(),
|
|
)
|
|
.await
|
|
.unwrap();
|
|
|
|
assert_eq!(resp.status(), StatusCode::OK);
|
|
let json = response_json(resp.into_body()).await;
|
|
assert_eq!(json["name"], "updated");
|
|
}
|
|
|
|
// ── DELETE /emotes/{uuid} ─────────────────────────────────────────────────────
|
|
|
|
#[tokio::test]
|
|
async fn delete_emote_not_found_returns_404() {
|
|
let app = build_router(test_state().await);
|
|
let resp = app
|
|
.oneshot(
|
|
Request::builder()
|
|
.method("DELETE")
|
|
.uri("/emotes/no-such-uuid")
|
|
.header("authorization", auth_header())
|
|
.body(Body::empty())
|
|
.unwrap(),
|
|
)
|
|
.await
|
|
.unwrap();
|
|
assert_eq!(resp.status(), StatusCode::NOT_FOUND);
|
|
}
|
|
|
|
#[tokio::test]
|
|
async fn delete_emote_ok() {
|
|
let state = test_state().await;
|
|
let id = new_uuid();
|
|
state
|
|
.db
|
|
.create_emote(&id, "todelete", None, "emoji/gone.png")
|
|
.await
|
|
.unwrap();
|
|
|
|
// S3 delete will fail against the fake endpoint, but it is best-effort.
|
|
let resp = build_router(state)
|
|
.oneshot(
|
|
Request::builder()
|
|
.method("DELETE")
|
|
.uri(format!("/emotes/{id}"))
|
|
.header("authorization", auth_header())
|
|
.body(Body::empty())
|
|
.unwrap(),
|
|
)
|
|
.await
|
|
.unwrap();
|
|
|
|
assert_eq!(resp.status(), StatusCode::NO_CONTENT);
|
|
}
|