Open3DLab/mikebase
Public Access
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 5 Activity

feat: add emote import from legacy source with dry-run mode

Browse Source 
Add POST /manage/import (auth-protected) that fetches emotes from a
legacy JSON endpoint, downloads each image, uploads it to S3, and
inserts it into the DB with the original timestamps preserved.

- Skip emotes whose name already exists (best-effort duplicate detection
  across SQLite and PostgreSQL via error code + message fallback)
- Validate source_url against a configurable host allowlist
  ([import] allowed_hosts in config, default ["smutba.se"])
- dry_run: true previews the import without writing to S3 or DB;
  result statuses are "would_import" / "would_skip" instead of
  "imported" / "skipped"
- Add db.name_exists() for efficient per-name existence checks used
  by dry-run
- Add reqwest (rustls-tls + json) and url dependencies
- Integration tests: auth guard, allowlist rejection, mirror +
  skip-duplicates, dry-run no-persist
This commit is contained in:
Ganonmaster
2026-05-02 02:48:24 +02:00
parent b7365ef1e9
commit 9c5212de05
10 changed files with 915 additions and 4 deletions
Download Patch File Download Diff File Expand all files Collapse all files
tests/routes.rs
+234 -3
View File
@@ -3,14 +3,20 @@ use std::sync::Arc;
use axum::{
body::Body,
http::{Request, StatusCode},
routing::get,
Json,
Router,
};
use base64::{engine::general_purpose::STANDARD, Engine};
use chrono::{DateTime, Utc};
use serde_json::json;
use sqlx::any::install_default_drivers;
use tower::ServiceExt;
use tokio::net::TcpListener;
use mikebase::{
build_router,
config::{AppConfig, AuthConfig, DatabaseConfig, S3Config, ServerConfig},
config::{AppConfig, AuthConfig, DatabaseConfig, ImportConfig, S3Config, ServerConfig},
db::Database,
models::new_uuid,
storage::S3Storage,
@@ -20,6 +26,14 @@ use mikebase::{
// ── Helpers ───────────────────────────────────────────────────────────────────
async fn test_state() -> AppState {
test_state_with_s3(
"http://localhost:19999".to_string(),
"http://localhost:19999/test-bucket".to_string(),
)
.await
}
async fn test_state_with_s3(s3_endpoint: String, s3_public_url: String) -> AppState {
install_default_drivers();
let pool = sqlx::pool::PoolOptions::<sqlx::Any>::new()
.max_connections(1)
@@ -31,12 +45,12 @@ async fn test_state() -> AppState {
let cfg = Arc::new(AppConfig {
s3: S3Config {
endpoint: "http://localhost:19999".to_string(),
endpoint: s3_endpoint,
region: "us-east-1".to_string(),
bucket: "test-bucket".to_string(),
access_key: "test".to_string(),
secret_key: "test".to_string(),
public_url: "http://localhost:19999/test-bucket".to_string(),
public_url: s3_public_url,
},
database: DatabaseConfig { url: "sqlite::memory:".to_string() },
server: ServerConfig::default(),
@@ -44,12 +58,84 @@ async fn test_state() -> AppState {
username: "admin".to_string(),
password: "secret".to_string(),
}),
import: ImportConfig {
allowed_hosts: vec!["smutba.se".to_string(), "localhost".to_string(), "127.0.0.1".to_string()],
},
});
let storage = S3Storage::new(&cfg);
AppState { db, storage, cfg }
}
async fn spawn_mock_s3_server() -> (String, tokio::task::JoinHandle<()>) {
async fn ok() -> StatusCode {
StatusCode::OK
}
let listener = TcpListener::bind("127.0.0.1:0").await.unwrap();
let addr = listener.local_addr().unwrap();
let base = format!("http://{}", addr);
let app = Router::new()
.route("/", get(ok))
.route("/{*path}", axum::routing::any(ok));
let handle = tokio::spawn(async move {
axum::serve(listener, app).await.unwrap();
});
(base, handle)
}
async fn spawn_legacy_source_server() -> (String, tokio::task::JoinHandle<()>) {
async fn image_new() -> ([(&'static str, &'static str); 1], &'static [u8]) {
([ ("content-type", "image/png") ], b"PNGDATA")
}
async fn image_dup() -> ([(&'static str, &'static str); 1], &'static [u8]) {
([ ("content-type", "image/png") ], b"PNGDATA2")
}
let listener = TcpListener::bind("127.0.0.1:0").await.unwrap();
let addr = listener.local_addr().unwrap();
let base = format!("http://{}", addr);
let payload_base = base.clone();
let app = Router::new()
.route(
"/emoji/json/",
get(move || {
let payload_base = payload_base.clone();
async move {
Json(json!({
"emotes": [
{
"name": "legacy_new",
"url": format!("{}/images/new.png", payload_base),
"created": "2020-01-01T00:00:00+00:00",
"modified": "2020-02-02T00:00:00+00:00"
},
{
"name": "legacy_duplicate",
"url": format!("{}/images/duplicate.png", payload_base),
"created": "2021-01-01T00:00:00+00:00",
"modified": "2021-01-02T00:00:00+00:00"
}
]
}))
}
}),
)
.route("/images/new.png", get(image_new))
.route("/images/duplicate.png", get(image_dup));
let handle = tokio::spawn(async move {
axum::serve(listener, app).await.unwrap();
});
(base, handle)
}
fn auth_header() -> String {
format!("Basic {}", STANDARD.encode("admin:secret"))
}
@@ -204,6 +290,151 @@ async fn manage_emotes_requires_auth() {
assert_eq!(resp.status(), StatusCode::UNAUTHORIZED);
}
#[tokio::test]
async fn manage_import_requires_auth() {
let app = build_router(test_state().await);
let resp = app
.oneshot(
Request::builder()
.method("POST")
.uri("/manage/import")
.header("content-type", "application/json")
.body(Body::from(r#"{"source_url":"https://smutba.se/emoji/json/"}"#))
.unwrap(),
)
.await
.unwrap();
assert_eq!(resp.status(), StatusCode::UNAUTHORIZED);
}
#[tokio::test]
async fn manage_import_rejects_non_allowlisted_host() {
let app = build_router(test_state().await);
let resp = app
.oneshot(
Request::builder()
.method("POST")
.uri("/manage/import")
.header("authorization", auth_header())
.header("content-type", "application/json")
.body(Body::from(r#"{"source_url":"https://example.com/emoji/json/"}"#))
.unwrap(),
)
.await
.unwrap();
assert_eq!(resp.status(), StatusCode::BAD_REQUEST);
}
#[tokio::test]
async fn manage_import_mirrors_and_skips_duplicates() {
let (legacy_base, server_handle) = spawn_legacy_source_server().await;
let (s3_base, s3_handle) = spawn_mock_s3_server().await;
let state = test_state_with_s3(s3_base.clone(), format!("{}/test-bucket", s3_base)).await;
// Pre-seed one duplicate emote name.
let existing_id = new_uuid();
state
.db
.create_emote(&existing_id, "legacy_duplicate", None, "emoji/existing.png")
.await
.unwrap();
let app = build_router(state.clone());
let resp = app
.oneshot(
Request::builder()
.method("POST")
.uri("/manage/import")
.header("authorization", auth_header())
.header("content-type", "application/json")
.body(Body::from(format!(
"{{\"source_url\":\"{}/emoji/json/\"}}",
legacy_base
)))
.unwrap(),
)
.await
.unwrap();
assert_eq!(resp.status(), StatusCode::OK);
let json = response_json(resp.into_body()).await;
assert_eq!(json["total"], 2);
assert_eq!(json["imported"], 1);
assert_eq!(json["skipped"], 1);
assert_eq!(json["failed"], 0);
let rows = state.db.list_emotes().await.unwrap();
let imported = rows.iter().find(|r| r.name == "legacy_new").unwrap();
let created = DateTime::parse_from_rfc3339(&imported.created)
.unwrap()
.with_timezone(&Utc)
.to_rfc3339();
let modified = DateTime::parse_from_rfc3339(&imported.modified)
.unwrap()
.with_timezone(&Utc)
.to_rfc3339();
assert_eq!(created, "2020-01-01T00:00:00+00:00");
assert_eq!(modified, "2020-02-02T00:00:00+00:00");
server_handle.abort();
s3_handle.abort();
}
#[tokio::test]
async fn manage_import_dry_run_does_not_persist() {
let (legacy_base, server_handle) = spawn_legacy_source_server().await;
let (s3_base, s3_handle) = spawn_mock_s3_server().await;
let state = test_state_with_s3(s3_base.clone(), format!("{}/test-bucket", s3_base)).await;
// Pre-seed one duplicate so we can verify would_skip detection.
let existing_id = new_uuid();
state
.db
.create_emote(&existing_id, "legacy_duplicate", None, "emoji/existing.png")
.await
.unwrap();
let app = build_router(state.clone());
let body = format!(
"{{\"source_url\":\"{}/emoji/json/\",\"dry_run\":true}}",
legacy_base
);
let resp = app
.oneshot(
Request::builder()
.method("POST")
.uri("/manage/import")
.header("authorization", auth_header())
.header("content-type", "application/json")
.body(Body::from(body))
.unwrap(),
)
.await
.unwrap();
assert_eq!(resp.status(), StatusCode::OK);
let json = response_json(resp.into_body()).await;
assert_eq!(json["dry_run"], true);
assert_eq!(json["total"], 2);
assert_eq!(json["imported"], 1);
assert_eq!(json["skipped"], 1);
assert_eq!(json["failed"], 0);
let results = json["results"].as_array().unwrap();
let new_result = results.iter().find(|r| r["name"] == "legacy_new").unwrap();
assert_eq!(new_result["status"], "would_import");
let dup_result = results.iter().find(|r| r["name"] == "legacy_duplicate").unwrap();
assert_eq!(dup_result["status"], "would_skip");
// Nothing new should have been written to the DB.
let rows = state.db.list_emotes().await.unwrap();
assert_eq!(rows.len(), 1, "dry-run must not insert any rows");
assert_eq!(rows[0].name, "legacy_duplicate");
server_handle.abort();
s3_handle.abort();
}
// ── POST /emotes input validation ─────────────────────────────────────────────
#[tokio::test]