feat: add emote import from legacy source with dry-run mode

Add POST /manage/import (auth-protected) that fetches emotes from a
legacy JSON endpoint, downloads each image, uploads it to S3, and
inserts it into the DB with the original timestamps preserved.

- Skip emotes whose name already exists (best-effort duplicate detection
  across SQLite and PostgreSQL via error code + message fallback)
- Validate source_url against a configurable host allowlist
  ([import] allowed_hosts in config, default ["smutba.se"])
- dry_run: true previews the import without writing to S3 or DB;
  result statuses are "would_import" / "would_skip" instead of
  "imported" / "skipped"
- Add db.name_exists() for efficient per-name existence checks used
  by dry-run
- Add reqwest (rustls-tls + json) and url dependencies
- Integration tests: auth guard, allowlist rejection, mirror +
  skip-duplicates, dry-run no-persist
This commit is contained in:
Ganonmaster
2026-05-02 02:48:24 +02:00
parent b7365ef1e9
commit 9c5212de05
10 changed files with 915 additions and 4 deletions
+20
View File
@@ -43,6 +43,16 @@ pub struct AuthConfig {
pub password: String,
}
#[derive(Debug, Deserialize, Clone)]
pub struct ImportConfig {
#[serde(default = "default_allowed_hosts")]
pub allowed_hosts: Vec<String>,
}
fn default_allowed_hosts() -> Vec<String> {
vec!["smutba.se".to_string()]
}
#[derive(Debug, Deserialize, Clone)]
pub struct AppConfig {
pub s3: S3Config,
@@ -50,6 +60,8 @@ pub struct AppConfig {
#[serde(default)]
pub server: ServerConfig,
pub auth: Option<AuthConfig>,
#[serde(default)]
pub import: ImportConfig,
}
impl Default for ServerConfig {
@@ -61,6 +73,14 @@ impl Default for ServerConfig {
}
}
impl Default for ImportConfig {
fn default() -> Self {
Self {
allowed_hosts: default_allowed_hosts(),
}
}
}
impl AppConfig {
pub fn load() -> Result<Self, ConfigError> {
let cfg = Config::builder()