feat: add emote import from legacy source with dry-run mode
Add POST /manage/import (auth-protected) that fetches emotes from a legacy JSON endpoint, downloads each image, uploads it to S3, and inserts it into the DB with the original timestamps preserved. - Skip emotes whose name already exists (best-effort duplicate detection across SQLite and PostgreSQL via error code + message fallback) - Validate source_url against a configurable host allowlist ([import] allowed_hosts in config, default ["smutba.se"]) - dry_run: true previews the import without writing to S3 or DB; result statuses are "would_import" / "would_skip" instead of "imported" / "skipped" - Add db.name_exists() for efficient per-name existence checks used by dry-run - Add reqwest (rustls-tls + json) and url dependencies - Integration tests: auth guard, allowlist rejection, mirror + skip-duplicates, dry-run no-persist
This commit is contained in:
Generated
+179
@@ -703,6 +703,12 @@ version = "1.0.4"
|
|||||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
checksum = "9330f8b2ff13f34540b44e946ef35111825727b38d33286ef986142615121801"
|
checksum = "9330f8b2ff13f34540b44e946ef35111825727b38d33286ef986142615121801"
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "cfg_aliases"
|
||||||
|
version = "0.2.1"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "613afe47fcd5fac7ccf1db93babcb082c5994d996f20b8b159f2ad1658eb5724"
|
||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "chrono"
|
name = "chrono"
|
||||||
version = "0.4.44"
|
version = "0.4.44"
|
||||||
@@ -1214,8 +1220,10 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
|
|||||||
checksum = "ff2abc00be7fca6ebc474524697ae276ad847ad0a6b3faa4bcb027e9a4614ad0"
|
checksum = "ff2abc00be7fca6ebc474524697ae276ad847ad0a6b3faa4bcb027e9a4614ad0"
|
||||||
dependencies = [
|
dependencies = [
|
||||||
"cfg-if",
|
"cfg-if",
|
||||||
|
"js-sys",
|
||||||
"libc",
|
"libc",
|
||||||
"wasi",
|
"wasi",
|
||||||
|
"wasm-bindgen",
|
||||||
]
|
]
|
||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
@@ -1225,9 +1233,11 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
|
|||||||
checksum = "899def5c37c4fd7b2664648c28120ecec138e4d395b459e5ca34f9cce2dd77fd"
|
checksum = "899def5c37c4fd7b2664648c28120ecec138e4d395b459e5ca34f9cce2dd77fd"
|
||||||
dependencies = [
|
dependencies = [
|
||||||
"cfg-if",
|
"cfg-if",
|
||||||
|
"js-sys",
|
||||||
"libc",
|
"libc",
|
||||||
"r-efi 5.3.0",
|
"r-efi 5.3.0",
|
||||||
"wasip2",
|
"wasip2",
|
||||||
|
"wasm-bindgen",
|
||||||
]
|
]
|
||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
@@ -1520,6 +1530,7 @@ dependencies = [
|
|||||||
"tokio",
|
"tokio",
|
||||||
"tokio-rustls 0.26.4",
|
"tokio-rustls 0.26.4",
|
||||||
"tower-service",
|
"tower-service",
|
||||||
|
"webpki-roots",
|
||||||
]
|
]
|
||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
@@ -1695,6 +1706,16 @@ version = "2.12.0"
|
|||||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
checksum = "d98f6fed1fde3f8c21bc40a1abb88dd75e67924f9cffc3ef95607bad8017f8e2"
|
checksum = "d98f6fed1fde3f8c21bc40a1abb88dd75e67924f9cffc3ef95607bad8017f8e2"
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "iri-string"
|
||||||
|
version = "0.7.12"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "25e659a4bb38e810ebc252e53b5814ff908a8c58c2a9ce2fae1bbec24cbf4e20"
|
||||||
|
dependencies = [
|
||||||
|
"memchr",
|
||||||
|
"serde",
|
||||||
|
]
|
||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "itoa"
|
name = "itoa"
|
||||||
version = "1.0.17"
|
version = "1.0.17"
|
||||||
@@ -1812,6 +1833,12 @@ dependencies = [
|
|||||||
"hashbrown 0.15.5",
|
"hashbrown 0.15.5",
|
||||||
]
|
]
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "lru-slab"
|
||||||
|
version = "0.1.2"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "112b39cec0b298b6c1999fee3e31427f74f676e4cb9879ed1a121b43661a4154"
|
||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "matchers"
|
name = "matchers"
|
||||||
version = "0.2.0"
|
version = "0.2.0"
|
||||||
@@ -1856,6 +1883,7 @@ dependencies = [
|
|||||||
"chrono",
|
"chrono",
|
||||||
"config",
|
"config",
|
||||||
"mime_guess",
|
"mime_guess",
|
||||||
|
"reqwest",
|
||||||
"serde",
|
"serde",
|
||||||
"serde_json",
|
"serde_json",
|
||||||
"sqlx",
|
"sqlx",
|
||||||
@@ -1866,6 +1894,7 @@ dependencies = [
|
|||||||
"tower-http",
|
"tower-http",
|
||||||
"tracing",
|
"tracing",
|
||||||
"tracing-subscriber",
|
"tracing-subscriber",
|
||||||
|
"url",
|
||||||
"uuid",
|
"uuid",
|
||||||
]
|
]
|
||||||
|
|
||||||
@@ -2220,6 +2249,61 @@ dependencies = [
|
|||||||
"unicode-ident",
|
"unicode-ident",
|
||||||
]
|
]
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "quinn"
|
||||||
|
version = "0.11.9"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "b9e20a958963c291dc322d98411f541009df2ced7b5a4f2bd52337638cfccf20"
|
||||||
|
dependencies = [
|
||||||
|
"bytes",
|
||||||
|
"cfg_aliases",
|
||||||
|
"pin-project-lite",
|
||||||
|
"quinn-proto",
|
||||||
|
"quinn-udp",
|
||||||
|
"rustc-hash",
|
||||||
|
"rustls 0.23.37",
|
||||||
|
"socket2 0.6.3",
|
||||||
|
"thiserror 2.0.18",
|
||||||
|
"tokio",
|
||||||
|
"tracing",
|
||||||
|
"web-time",
|
||||||
|
]
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "quinn-proto"
|
||||||
|
version = "0.11.14"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "434b42fec591c96ef50e21e886936e66d3cc3f737104fdb9b737c40ffb94c098"
|
||||||
|
dependencies = [
|
||||||
|
"bytes",
|
||||||
|
"getrandom 0.3.4",
|
||||||
|
"lru-slab",
|
||||||
|
"rand 0.9.2",
|
||||||
|
"ring",
|
||||||
|
"rustc-hash",
|
||||||
|
"rustls 0.23.37",
|
||||||
|
"rustls-pki-types",
|
||||||
|
"slab",
|
||||||
|
"thiserror 2.0.18",
|
||||||
|
"tinyvec",
|
||||||
|
"tracing",
|
||||||
|
"web-time",
|
||||||
|
]
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "quinn-udp"
|
||||||
|
version = "0.5.14"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "addec6a0dcad8a8d96a771f815f0eaf55f9d1805756410b39f5fa81332574cbd"
|
||||||
|
dependencies = [
|
||||||
|
"cfg_aliases",
|
||||||
|
"libc",
|
||||||
|
"once_cell",
|
||||||
|
"socket2 0.6.3",
|
||||||
|
"tracing",
|
||||||
|
"windows-sys 0.52.0",
|
||||||
|
]
|
||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "quote"
|
name = "quote"
|
||||||
version = "1.0.45"
|
version = "1.0.45"
|
||||||
@@ -2353,6 +2437,44 @@ version = "0.8.10"
|
|||||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
checksum = "dc897dd8d9e8bd1ed8cdad82b5966c3e0ecae09fb1907d58efaa013543185d0a"
|
checksum = "dc897dd8d9e8bd1ed8cdad82b5966c3e0ecae09fb1907d58efaa013543185d0a"
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "reqwest"
|
||||||
|
version = "0.12.28"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "eddd3ca559203180a307f12d114c268abf583f59b03cb906fd0b3ff8646c1147"
|
||||||
|
dependencies = [
|
||||||
|
"base64 0.22.1",
|
||||||
|
"bytes",
|
||||||
|
"futures-core",
|
||||||
|
"http 1.4.0",
|
||||||
|
"http-body 1.0.1",
|
||||||
|
"http-body-util",
|
||||||
|
"hyper 1.8.1",
|
||||||
|
"hyper-rustls 0.27.7",
|
||||||
|
"hyper-util",
|
||||||
|
"js-sys",
|
||||||
|
"log",
|
||||||
|
"percent-encoding",
|
||||||
|
"pin-project-lite",
|
||||||
|
"quinn",
|
||||||
|
"rustls 0.23.37",
|
||||||
|
"rustls-pki-types",
|
||||||
|
"serde",
|
||||||
|
"serde_json",
|
||||||
|
"serde_urlencoded",
|
||||||
|
"sync_wrapper",
|
||||||
|
"tokio",
|
||||||
|
"tokio-rustls 0.26.4",
|
||||||
|
"tower",
|
||||||
|
"tower-http",
|
||||||
|
"tower-service",
|
||||||
|
"url",
|
||||||
|
"wasm-bindgen",
|
||||||
|
"wasm-bindgen-futures",
|
||||||
|
"web-sys",
|
||||||
|
"webpki-roots",
|
||||||
|
]
|
||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "rfc6979"
|
name = "rfc6979"
|
||||||
version = "0.3.1"
|
version = "0.3.1"
|
||||||
@@ -2420,6 +2542,12 @@ dependencies = [
|
|||||||
"ordered-multimap",
|
"ordered-multimap",
|
||||||
]
|
]
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "rustc-hash"
|
||||||
|
version = "2.1.2"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "94300abf3f1ae2e2b8ffb7b58043de3d399c73fa6f4b73826402a5c457614dbe"
|
||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "rustc_version"
|
name = "rustc_version"
|
||||||
version = "0.4.1"
|
version = "0.4.1"
|
||||||
@@ -2449,6 +2577,7 @@ checksum = "758025cb5fccfd3bc2fd74708fd4682be41d99e5dff73c377c0646c6012c73a4"
|
|||||||
dependencies = [
|
dependencies = [
|
||||||
"aws-lc-rs",
|
"aws-lc-rs",
|
||||||
"once_cell",
|
"once_cell",
|
||||||
|
"ring",
|
||||||
"rustls-pki-types",
|
"rustls-pki-types",
|
||||||
"rustls-webpki 0.103.9",
|
"rustls-webpki 0.103.9",
|
||||||
"subtle",
|
"subtle",
|
||||||
@@ -2473,6 +2602,7 @@ version = "1.14.0"
|
|||||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
checksum = "be040f8b0a225e40375822a563fa9524378b9d63112f53e19ffff34df5d33fdd"
|
checksum = "be040f8b0a225e40375822a563fa9524378b9d63112f53e19ffff34df5d33fdd"
|
||||||
dependencies = [
|
dependencies = [
|
||||||
|
"web-time",
|
||||||
"zeroize",
|
"zeroize",
|
||||||
]
|
]
|
||||||
|
|
||||||
@@ -3019,6 +3149,9 @@ name = "sync_wrapper"
|
|||||||
version = "1.0.2"
|
version = "1.0.2"
|
||||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
checksum = "0bf256ce5efdfa370213c1dabab5935a12e49f2c58d15e9eac2870d3b4f27263"
|
checksum = "0bf256ce5efdfa370213c1dabab5935a12e49f2c58d15e9eac2870d3b4f27263"
|
||||||
|
dependencies = [
|
||||||
|
"futures-core",
|
||||||
|
]
|
||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "synstructure"
|
name = "synstructure"
|
||||||
@@ -3281,9 +3414,12 @@ checksum = "d4e6559d53cc268e5031cd8429d05415bc4cb4aefc4aa5d6cc35fbf5b924a1f8"
|
|||||||
dependencies = [
|
dependencies = [
|
||||||
"bitflags",
|
"bitflags",
|
||||||
"bytes",
|
"bytes",
|
||||||
|
"futures-util",
|
||||||
"http 1.4.0",
|
"http 1.4.0",
|
||||||
"http-body 1.0.1",
|
"http-body 1.0.1",
|
||||||
|
"iri-string",
|
||||||
"pin-project-lite",
|
"pin-project-lite",
|
||||||
|
"tower",
|
||||||
"tower-layer",
|
"tower-layer",
|
||||||
"tower-service",
|
"tower-service",
|
||||||
"tracing",
|
"tracing",
|
||||||
@@ -3544,6 +3680,20 @@ dependencies = [
|
|||||||
"wasm-bindgen-shared",
|
"wasm-bindgen-shared",
|
||||||
]
|
]
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "wasm-bindgen-futures"
|
||||||
|
version = "0.4.64"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "e9c5522b3a28661442748e09d40924dfb9ca614b21c00d3fd135720e48b67db8"
|
||||||
|
dependencies = [
|
||||||
|
"cfg-if",
|
||||||
|
"futures-util",
|
||||||
|
"js-sys",
|
||||||
|
"once_cell",
|
||||||
|
"wasm-bindgen",
|
||||||
|
"web-sys",
|
||||||
|
]
|
||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "wasm-bindgen-macro"
|
name = "wasm-bindgen-macro"
|
||||||
version = "0.2.114"
|
version = "0.2.114"
|
||||||
@@ -3610,6 +3760,35 @@ dependencies = [
|
|||||||
"semver",
|
"semver",
|
||||||
]
|
]
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "web-sys"
|
||||||
|
version = "0.3.91"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "854ba17bb104abfb26ba36da9729addc7ce7f06f5c0f90f3c391f8461cca21f9"
|
||||||
|
dependencies = [
|
||||||
|
"js-sys",
|
||||||
|
"wasm-bindgen",
|
||||||
|
]
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "web-time"
|
||||||
|
version = "1.1.0"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "5a6580f308b1fad9207618087a65c04e7a10bc77e02c8e84e9b00dd4b12fa0bb"
|
||||||
|
dependencies = [
|
||||||
|
"js-sys",
|
||||||
|
"wasm-bindgen",
|
||||||
|
]
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "webpki-roots"
|
||||||
|
version = "1.0.7"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "52f5ee44c96cf55f1b349600768e3ece3a8f26010c05265ab73f945bb1a2eb9d"
|
||||||
|
dependencies = [
|
||||||
|
"rustls-pki-types",
|
||||||
|
]
|
||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "whoami"
|
name = "whoami"
|
||||||
version = "1.6.1"
|
version = "1.6.1"
|
||||||
|
|||||||
@@ -23,6 +23,8 @@ mime_guess = "2"
|
|||||||
tokio-util = { version = "0.7", features = ["io"] }
|
tokio-util = { version = "0.7", features = ["io"] }
|
||||||
bytes = "1"
|
bytes = "1"
|
||||||
base64 = "0.22"
|
base64 = "0.22"
|
||||||
|
reqwest = { version = "0.12", default-features = false, features = ["rustls-tls", "json"] }
|
||||||
|
url = "2"
|
||||||
|
|
||||||
[dev-dependencies]
|
[dev-dependencies]
|
||||||
tower = { version = "0.5", features = ["util"] }
|
tower = { version = "0.5", features = ["util"] }
|
||||||
|
|||||||
@@ -1,2 +1,57 @@
|
|||||||
# mikebase
|
# mikebase
|
||||||
A Rust-based emote database and API.
|
A Rust-based emote database and API.
|
||||||
|
|
||||||
|
## Importing legacy emotes
|
||||||
|
|
||||||
|
Use the protected management endpoint to mirror emotes from a legacy JSON feed.
|
||||||
|
|
||||||
|
### Endpoint
|
||||||
|
|
||||||
|
POST /manage/import
|
||||||
|
|
||||||
|
- Auth: HTTP Basic (same credentials used for other protected routes)
|
||||||
|
- Content-Type: application/json
|
||||||
|
- Body:
|
||||||
|
|
||||||
|
```json
|
||||||
|
{
|
||||||
|
"source_url": "https://smutba.se/emoji/json/"
|
||||||
|
}
|
||||||
|
```
|
||||||
|
|
||||||
|
### Behavior
|
||||||
|
|
||||||
|
- Fetches source JSON in the format `{"emotes": [{name, url, created, modified}, ...]}`
|
||||||
|
- Downloads each image URL and uploads bytes to this app's configured S3 bucket
|
||||||
|
- Inserts emote rows preserving source `created` and `modified` timestamps
|
||||||
|
- Skips entries where `name` already exists locally
|
||||||
|
- Continues processing after per-item failures and returns a batch summary
|
||||||
|
|
||||||
|
### Example response
|
||||||
|
|
||||||
|
```json
|
||||||
|
{
|
||||||
|
"source_url": "https://smutba.se/emoji/json/",
|
||||||
|
"total": 2,
|
||||||
|
"imported": 1,
|
||||||
|
"skipped": 1,
|
||||||
|
"failed": 0,
|
||||||
|
"results": [
|
||||||
|
{"name": "legacy_new", "status": "imported", "reason": null},
|
||||||
|
{"name": "legacy_duplicate", "status": "skipped", "reason": "Name already exists"}
|
||||||
|
]
|
||||||
|
}
|
||||||
|
```
|
||||||
|
|
||||||
|
### Allowlisted hosts
|
||||||
|
|
||||||
|
Import is restricted to hosts in configuration:
|
||||||
|
|
||||||
|
```toml
|
||||||
|
[import]
|
||||||
|
allowed_hosts = ["smutba.se"]
|
||||||
|
```
|
||||||
|
|
||||||
|
Environment override example:
|
||||||
|
|
||||||
|
- `APP__IMPORT__ALLOWED_HOSTS=["smutba.se","legacy.example.org"]`
|
||||||
|
|||||||
@@ -19,6 +19,11 @@ port = 3000
|
|||||||
username = "admin"
|
username = "admin"
|
||||||
password = "changeme"
|
password = "changeme"
|
||||||
|
|
||||||
|
[import]
|
||||||
|
# Hosts allowed as migration sources for POST /manage/import.
|
||||||
|
# The request is rejected unless the source URL host matches one of these.
|
||||||
|
allowed_hosts = ["smutba.se"]
|
||||||
|
|
||||||
[s3]
|
[s3]
|
||||||
endpoint = "https://s3.eu-central-1.wasabisys.com"
|
endpoint = "https://s3.eu-central-1.wasabisys.com"
|
||||||
region = "eu-central-1"
|
region = "eu-central-1"
|
||||||
|
|||||||
@@ -43,6 +43,16 @@ pub struct AuthConfig {
|
|||||||
pub password: String,
|
pub password: String,
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[derive(Debug, Deserialize, Clone)]
|
||||||
|
pub struct ImportConfig {
|
||||||
|
#[serde(default = "default_allowed_hosts")]
|
||||||
|
pub allowed_hosts: Vec<String>,
|
||||||
|
}
|
||||||
|
|
||||||
|
fn default_allowed_hosts() -> Vec<String> {
|
||||||
|
vec!["smutba.se".to_string()]
|
||||||
|
}
|
||||||
|
|
||||||
#[derive(Debug, Deserialize, Clone)]
|
#[derive(Debug, Deserialize, Clone)]
|
||||||
pub struct AppConfig {
|
pub struct AppConfig {
|
||||||
pub s3: S3Config,
|
pub s3: S3Config,
|
||||||
@@ -50,6 +60,8 @@ pub struct AppConfig {
|
|||||||
#[serde(default)]
|
#[serde(default)]
|
||||||
pub server: ServerConfig,
|
pub server: ServerConfig,
|
||||||
pub auth: Option<AuthConfig>,
|
pub auth: Option<AuthConfig>,
|
||||||
|
#[serde(default)]
|
||||||
|
pub import: ImportConfig,
|
||||||
}
|
}
|
||||||
|
|
||||||
impl Default for ServerConfig {
|
impl Default for ServerConfig {
|
||||||
@@ -61,6 +73,14 @@ impl Default for ServerConfig {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
impl Default for ImportConfig {
|
||||||
|
fn default() -> Self {
|
||||||
|
Self {
|
||||||
|
allowed_hosts: default_allowed_hosts(),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
impl AppConfig {
|
impl AppConfig {
|
||||||
pub fn load() -> Result<Self, ConfigError> {
|
pub fn load() -> Result<Self, ConfigError> {
|
||||||
let cfg = Config::builder()
|
let cfg = Config::builder()
|
||||||
|
|||||||
@@ -40,6 +40,14 @@ impl Database {
|
|||||||
Ok(row)
|
Ok(row)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
pub async fn name_exists(&self, name: &str) -> Result<bool, sqlx::Error> {
|
||||||
|
let row: (i64,) = sqlx::query_as("SELECT COUNT(*) FROM emotes WHERE name = $1")
|
||||||
|
.bind(name)
|
||||||
|
.fetch_one(&self.pool)
|
||||||
|
.await?;
|
||||||
|
Ok(row.0 > 0)
|
||||||
|
}
|
||||||
|
|
||||||
pub async fn create_emote(
|
pub async fn create_emote(
|
||||||
&self,
|
&self,
|
||||||
uuid: &str,
|
uuid: &str,
|
||||||
@@ -70,6 +78,37 @@ impl Database {
|
|||||||
})
|
})
|
||||||
}
|
}
|
||||||
|
|
||||||
|
pub async fn create_emote_with_timestamps(
|
||||||
|
&self,
|
||||||
|
uuid: &str,
|
||||||
|
name: &str,
|
||||||
|
alias: Option<&str>,
|
||||||
|
image_key: &str,
|
||||||
|
created: &str,
|
||||||
|
modified: &str,
|
||||||
|
) -> Result<EmoteRow, sqlx::Error> {
|
||||||
|
sqlx::query(
|
||||||
|
"INSERT INTO emotes (uuid, name, alias, image_key, created, modified) VALUES ($1, $2, $3, $4, $5, $6)",
|
||||||
|
)
|
||||||
|
.bind(uuid)
|
||||||
|
.bind(name)
|
||||||
|
.bind(alias)
|
||||||
|
.bind(image_key)
|
||||||
|
.bind(created)
|
||||||
|
.bind(modified)
|
||||||
|
.execute(&self.pool)
|
||||||
|
.await?;
|
||||||
|
|
||||||
|
Ok(EmoteRow {
|
||||||
|
uuid: uuid.to_string(),
|
||||||
|
name: name.to_string(),
|
||||||
|
alias: alias.map(|s| s.to_string()),
|
||||||
|
image_key: image_key.to_string(),
|
||||||
|
created: created.to_string(),
|
||||||
|
modified: modified.to_string(),
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
pub async fn update_emote(
|
pub async fn update_emote(
|
||||||
&self,
|
&self,
|
||||||
uuid: &str,
|
uuid: &str,
|
||||||
@@ -180,6 +219,26 @@ mod tests {
|
|||||||
assert_eq!(updated.image_key, "emoji/x.png"); // unchanged
|
assert_eq!(updated.image_key, "emoji/x.png"); // unchanged
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[tokio::test]
|
||||||
|
async fn create_with_preserved_timestamps() {
|
||||||
|
let db = test_db().await;
|
||||||
|
let id = new_uuid();
|
||||||
|
let created = "2020-01-01T10:00:00+00:00";
|
||||||
|
let modified = "2021-02-03T11:30:00+00:00";
|
||||||
|
|
||||||
|
let row = db
|
||||||
|
.create_emote_with_timestamps(&id, "legacy", None, "emoji/legacy.png", created, modified)
|
||||||
|
.await
|
||||||
|
.unwrap();
|
||||||
|
|
||||||
|
assert_eq!(row.created, created);
|
||||||
|
assert_eq!(row.modified, modified);
|
||||||
|
|
||||||
|
let fetched = db.get_emote_by_id(&id).await.unwrap().unwrap();
|
||||||
|
assert_eq!(fetched.created, created);
|
||||||
|
assert_eq!(fetched.modified, modified);
|
||||||
|
}
|
||||||
|
|
||||||
#[tokio::test]
|
#[tokio::test]
|
||||||
async fn update_unspecified_fields_are_kept() {
|
async fn update_unspecified_fields_are_kept() {
|
||||||
let db = test_db().await;
|
let db = test_db().await;
|
||||||
|
|||||||
@@ -29,6 +29,7 @@ pub fn build_router(state: AppState) -> Router {
|
|||||||
let protected = Router::new()
|
let protected = Router::new()
|
||||||
.route("/manage", get(routes::manage::manage_root))
|
.route("/manage", get(routes::manage::manage_root))
|
||||||
.route("/manage/emotes", get(routes::manage::list_admin_emotes))
|
.route("/manage/emotes", get(routes::manage::list_admin_emotes))
|
||||||
|
.route("/manage/import", post(routes::manage::import_emotes))
|
||||||
.route("/emotes", post(routes::emotes::create_emote))
|
.route("/emotes", post(routes::emotes::create_emote))
|
||||||
.route("/emotes/{uuid}", put(routes::emotes::update_emote))
|
.route("/emotes/{uuid}", put(routes::emotes::update_emote))
|
||||||
.route("/emotes/{uuid}", delete(routes::emotes::delete_emote))
|
.route("/emotes/{uuid}", delete(routes::emotes::delete_emote))
|
||||||
|
|||||||
@@ -51,6 +51,44 @@ pub struct AdminEmoteResponse {
|
|||||||
pub modified: DateTime<Utc>,
|
pub modified: DateTime<Utc>,
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[derive(Debug, Deserialize)]
|
||||||
|
pub struct ImportEmotesRequest {
|
||||||
|
pub source_url: String,
|
||||||
|
#[serde(default)]
|
||||||
|
pub dry_run: bool,
|
||||||
|
}
|
||||||
|
|
||||||
|
#[derive(Debug, Deserialize)]
|
||||||
|
pub struct LegacyEmotesPayload {
|
||||||
|
pub emotes: Vec<LegacyEmote>,
|
||||||
|
}
|
||||||
|
|
||||||
|
#[derive(Debug, Deserialize)]
|
||||||
|
pub struct LegacyEmote {
|
||||||
|
pub name: String,
|
||||||
|
pub url: String,
|
||||||
|
pub created: DateTime<Utc>,
|
||||||
|
pub modified: DateTime<Utc>,
|
||||||
|
}
|
||||||
|
|
||||||
|
#[derive(Debug, Serialize)]
|
||||||
|
pub struct ImportEmotesResponse {
|
||||||
|
pub source_url: String,
|
||||||
|
pub dry_run: bool,
|
||||||
|
pub total: usize,
|
||||||
|
pub imported: usize,
|
||||||
|
pub skipped: usize,
|
||||||
|
pub failed: usize,
|
||||||
|
pub results: Vec<ImportEmoteResult>,
|
||||||
|
}
|
||||||
|
|
||||||
|
#[derive(Debug, Serialize)]
|
||||||
|
pub struct ImportEmoteResult {
|
||||||
|
pub name: String,
|
||||||
|
pub status: String,
|
||||||
|
pub reason: Option<String>,
|
||||||
|
}
|
||||||
|
|
||||||
/// Payload for updating an existing emote.
|
/// Payload for updating an existing emote.
|
||||||
///
|
///
|
||||||
/// `alias` uses a double-Option to distinguish three states:
|
/// `alias` uses a double-Option to distinguish three states:
|
||||||
|
|||||||
+322
-1
@@ -2,10 +2,20 @@ use axum::{
|
|||||||
extract::State,
|
extract::State,
|
||||||
http::StatusCode,
|
http::StatusCode,
|
||||||
response::{Html, IntoResponse, Json},
|
response::{Html, IntoResponse, Json},
|
||||||
|
Json as AxumJson,
|
||||||
};
|
};
|
||||||
|
use reqwest::header::CONTENT_TYPE;
|
||||||
use serde_json::json;
|
use serde_json::json;
|
||||||
|
use std::time::Duration;
|
||||||
|
use url::Url;
|
||||||
|
|
||||||
use crate::{models::AdminEmoteResponse, AppState};
|
use crate::{
|
||||||
|
models::{
|
||||||
|
new_uuid, AdminEmoteResponse, ImportEmoteResult, ImportEmotesRequest, ImportEmotesResponse,
|
||||||
|
LegacyEmotesPayload,
|
||||||
|
},
|
||||||
|
AppState,
|
||||||
|
};
|
||||||
|
|
||||||
/// GET /manage
|
/// GET /manage
|
||||||
/// Serves the emote management HTML page.
|
/// Serves the emote management HTML page.
|
||||||
@@ -41,3 +51,314 @@ pub async fn list_admin_emotes(State(state): State<AppState>) -> impl IntoRespon
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// POST /manage/import
|
||||||
|
/// Import emotes from a legacy JSON endpoint and mirror image bytes to local storage.
|
||||||
|
/// When `dry_run` is true, validates and previews the import without writing anything.
|
||||||
|
pub async fn import_emotes(
|
||||||
|
State(state): State<AppState>,
|
||||||
|
AxumJson(payload): AxumJson<ImportEmotesRequest>,
|
||||||
|
) -> impl IntoResponse {
|
||||||
|
let dry_run = payload.dry_run;
|
||||||
|
let source_url = payload.source_url.trim();
|
||||||
|
if source_url.is_empty() {
|
||||||
|
return (
|
||||||
|
StatusCode::BAD_REQUEST,
|
||||||
|
Json(json!({"error": "source_url is required"})),
|
||||||
|
)
|
||||||
|
.into_response();
|
||||||
|
}
|
||||||
|
|
||||||
|
let parsed_source = match Url::parse(source_url) {
|
||||||
|
Ok(url) => url,
|
||||||
|
Err(_) => {
|
||||||
|
return (
|
||||||
|
StatusCode::BAD_REQUEST,
|
||||||
|
Json(json!({"error": "Invalid source_url"})),
|
||||||
|
)
|
||||||
|
.into_response();
|
||||||
|
}
|
||||||
|
};
|
||||||
|
|
||||||
|
if !is_host_allowed(parsed_source.host_str(), &state.cfg.import.allowed_hosts) {
|
||||||
|
return (
|
||||||
|
StatusCode::BAD_REQUEST,
|
||||||
|
Json(json!({"error": "source_url host is not allowlisted"})),
|
||||||
|
)
|
||||||
|
.into_response();
|
||||||
|
}
|
||||||
|
|
||||||
|
let client = match reqwest::Client::builder()
|
||||||
|
.connect_timeout(Duration::from_secs(10))
|
||||||
|
.timeout(Duration::from_secs(30))
|
||||||
|
.build()
|
||||||
|
{
|
||||||
|
Ok(c) => c,
|
||||||
|
Err(e) => {
|
||||||
|
tracing::error!("Failed to build HTTP client: {e}");
|
||||||
|
return (
|
||||||
|
StatusCode::INTERNAL_SERVER_ERROR,
|
||||||
|
Json(json!({"error": "Failed to initialize import client"})),
|
||||||
|
)
|
||||||
|
.into_response();
|
||||||
|
}
|
||||||
|
};
|
||||||
|
|
||||||
|
let source_resp = match client.get(parsed_source.clone()).send().await {
|
||||||
|
Ok(resp) => resp,
|
||||||
|
Err(e) => {
|
||||||
|
tracing::error!("Failed to fetch source payload: {e}");
|
||||||
|
return (
|
||||||
|
StatusCode::BAD_GATEWAY,
|
||||||
|
Json(json!({"error": "Failed to fetch source payload"})),
|
||||||
|
)
|
||||||
|
.into_response();
|
||||||
|
}
|
||||||
|
};
|
||||||
|
|
||||||
|
if !source_resp.status().is_success() {
|
||||||
|
return (
|
||||||
|
StatusCode::BAD_GATEWAY,
|
||||||
|
Json(json!({
|
||||||
|
"error": "Source payload request failed",
|
||||||
|
"status": source_resp.status().as_u16()
|
||||||
|
})),
|
||||||
|
)
|
||||||
|
.into_response();
|
||||||
|
}
|
||||||
|
|
||||||
|
let payload: LegacyEmotesPayload = match source_resp.json().await {
|
||||||
|
Ok(p) => p,
|
||||||
|
Err(e) => {
|
||||||
|
tracing::error!("Failed to parse source payload JSON: {e}");
|
||||||
|
return (
|
||||||
|
StatusCode::BAD_REQUEST,
|
||||||
|
Json(json!({"error": "Invalid source payload schema"})),
|
||||||
|
)
|
||||||
|
.into_response();
|
||||||
|
}
|
||||||
|
};
|
||||||
|
|
||||||
|
let total = payload.emotes.len();
|
||||||
|
let mut imported = 0usize;
|
||||||
|
let mut skipped = 0usize;
|
||||||
|
let mut failed = 0usize;
|
||||||
|
let mut results = Vec::with_capacity(total);
|
||||||
|
|
||||||
|
for legacy in payload.emotes {
|
||||||
|
let name = legacy.name.trim().to_string();
|
||||||
|
if name.is_empty() {
|
||||||
|
failed += 1;
|
||||||
|
results.push(ImportEmoteResult {
|
||||||
|
name: legacy.name,
|
||||||
|
status: "failed".to_string(),
|
||||||
|
reason: Some("Missing emote name".to_string()),
|
||||||
|
});
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
|
||||||
|
let image_url = match Url::parse(&legacy.url) {
|
||||||
|
Ok(url) => url,
|
||||||
|
Err(_) => {
|
||||||
|
failed += 1;
|
||||||
|
results.push(ImportEmoteResult {
|
||||||
|
name,
|
||||||
|
status: "failed".to_string(),
|
||||||
|
reason: Some("Invalid image URL".to_string()),
|
||||||
|
});
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
};
|
||||||
|
|
||||||
|
let image_resp = match client.get(image_url.clone()).send().await {
|
||||||
|
Ok(resp) => resp,
|
||||||
|
Err(e) => {
|
||||||
|
failed += 1;
|
||||||
|
results.push(ImportEmoteResult {
|
||||||
|
name,
|
||||||
|
status: "failed".to_string(),
|
||||||
|
reason: Some(format!("Image download failed: {e}")),
|
||||||
|
});
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
};
|
||||||
|
|
||||||
|
if !image_resp.status().is_success() {
|
||||||
|
failed += 1;
|
||||||
|
results.push(ImportEmoteResult {
|
||||||
|
name,
|
||||||
|
status: "failed".to_string(),
|
||||||
|
reason: Some(format!("Image download failed with status {}", image_resp.status())),
|
||||||
|
});
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
|
||||||
|
let content_type = image_resp
|
||||||
|
.headers()
|
||||||
|
.get(CONTENT_TYPE)
|
||||||
|
.and_then(|h| h.to_str().ok())
|
||||||
|
.map(|ct| ct.split(';').next().unwrap_or(ct).trim().to_string())
|
||||||
|
.unwrap_or_else(|| {
|
||||||
|
mime_guess::from_path(image_url.path())
|
||||||
|
.first_or_octet_stream()
|
||||||
|
.to_string()
|
||||||
|
});
|
||||||
|
|
||||||
|
let data = match image_resp.bytes().await {
|
||||||
|
Ok(b) => b,
|
||||||
|
Err(e) => {
|
||||||
|
failed += 1;
|
||||||
|
results.push(ImportEmoteResult {
|
||||||
|
name,
|
||||||
|
status: "failed".to_string(),
|
||||||
|
reason: Some(format!("Failed reading image bytes: {e}")),
|
||||||
|
});
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
};
|
||||||
|
|
||||||
|
if dry_run {
|
||||||
|
// Check for a name collision without writing anything.
|
||||||
|
let exists = match state.db.name_exists(&name).await {
|
||||||
|
Ok(v) => v,
|
||||||
|
Err(e) => {
|
||||||
|
failed += 1;
|
||||||
|
results.push(ImportEmoteResult {
|
||||||
|
name,
|
||||||
|
status: "failed".to_string(),
|
||||||
|
reason: Some(format!("DB check failed: {e}")),
|
||||||
|
});
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
};
|
||||||
|
if exists {
|
||||||
|
skipped += 1;
|
||||||
|
results.push(ImportEmoteResult {
|
||||||
|
name,
|
||||||
|
status: "would_skip".to_string(),
|
||||||
|
reason: Some("Name already exists".to_string()),
|
||||||
|
});
|
||||||
|
} else {
|
||||||
|
imported += 1;
|
||||||
|
results.push(ImportEmoteResult {
|
||||||
|
name,
|
||||||
|
status: "would_import".to_string(),
|
||||||
|
reason: None,
|
||||||
|
});
|
||||||
|
}
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
|
||||||
|
let ext = infer_extension(&image_url, &content_type);
|
||||||
|
let id = new_uuid();
|
||||||
|
let key = format!("emoji/{id}.{ext}");
|
||||||
|
|
||||||
|
if let Err(e) = state.storage.upload(&key, data, &content_type).await {
|
||||||
|
failed += 1;
|
||||||
|
results.push(ImportEmoteResult {
|
||||||
|
name,
|
||||||
|
status: "failed".to_string(),
|
||||||
|
reason: Some(format!("Storage upload failed: {e}")),
|
||||||
|
});
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
|
||||||
|
let created = legacy.created.to_rfc3339();
|
||||||
|
let modified = legacy.modified.to_rfc3339();
|
||||||
|
match state
|
||||||
|
.db
|
||||||
|
.create_emote_with_timestamps(&id, &name, None, &key, &created, &modified)
|
||||||
|
.await
|
||||||
|
{
|
||||||
|
Ok(_) => {
|
||||||
|
imported += 1;
|
||||||
|
results.push(ImportEmoteResult {
|
||||||
|
name,
|
||||||
|
status: "imported".to_string(),
|
||||||
|
reason: None,
|
||||||
|
});
|
||||||
|
}
|
||||||
|
Err(e) if is_unique_name_violation(&e) => {
|
||||||
|
skipped += 1;
|
||||||
|
if let Err(del_err) = state.storage.delete(&key).await {
|
||||||
|
tracing::warn!("Failed to delete skipped upload key {key}: {del_err}");
|
||||||
|
}
|
||||||
|
results.push(ImportEmoteResult {
|
||||||
|
name,
|
||||||
|
status: "skipped".to_string(),
|
||||||
|
reason: Some("Name already exists".to_string()),
|
||||||
|
});
|
||||||
|
}
|
||||||
|
Err(e) => {
|
||||||
|
failed += 1;
|
||||||
|
if let Err(del_err) = state.storage.delete(&key).await {
|
||||||
|
tracing::warn!("Failed to cleanup failed upload key {key}: {del_err}");
|
||||||
|
}
|
||||||
|
results.push(ImportEmoteResult {
|
||||||
|
name,
|
||||||
|
status: "failed".to_string(),
|
||||||
|
reason: Some(format!("Database insert failed: {e}")),
|
||||||
|
});
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
let response = ImportEmotesResponse {
|
||||||
|
source_url: parsed_source.to_string(),
|
||||||
|
dry_run,
|
||||||
|
total,
|
||||||
|
imported,
|
||||||
|
skipped,
|
||||||
|
failed,
|
||||||
|
results,
|
||||||
|
};
|
||||||
|
|
||||||
|
(StatusCode::OK, Json(json!(response))).into_response()
|
||||||
|
}
|
||||||
|
|
||||||
|
fn is_host_allowed(host: Option<&str>, allowed_hosts: &[String]) -> bool {
|
||||||
|
let Some(host) = host else {
|
||||||
|
return false;
|
||||||
|
};
|
||||||
|
let host = host.to_ascii_lowercase();
|
||||||
|
allowed_hosts
|
||||||
|
.iter()
|
||||||
|
.any(|allowed| allowed.eq_ignore_ascii_case(&host))
|
||||||
|
}
|
||||||
|
|
||||||
|
fn infer_extension(url: &Url, content_type: &str) -> String {
|
||||||
|
if let Some(ext) = std::path::Path::new(url.path())
|
||||||
|
.extension()
|
||||||
|
.and_then(|e| e.to_str())
|
||||||
|
.filter(|e| !e.is_empty())
|
||||||
|
{
|
||||||
|
let lower = ext.to_ascii_lowercase();
|
||||||
|
if lower.chars().all(|c| c.is_ascii_alphanumeric()) {
|
||||||
|
return lower;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
if let Some(mime) = mime_guess::get_mime_extensions_str(content_type)
|
||||||
|
.and_then(|exts| exts.first())
|
||||||
|
{
|
||||||
|
return (*mime).to_string();
|
||||||
|
}
|
||||||
|
|
||||||
|
"bin".to_string()
|
||||||
|
}
|
||||||
|
|
||||||
|
fn is_unique_name_violation(err: &sqlx::Error) -> bool {
|
||||||
|
match err {
|
||||||
|
sqlx::Error::Database(db_err) => {
|
||||||
|
if let Some(code) = db_err.code() {
|
||||||
|
if code == "23505" || code == "2067" || code == "1555" {
|
||||||
|
return true;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
let msg = db_err.message().to_ascii_lowercase();
|
||||||
|
msg.contains("unique") || msg.contains("duplicate")
|
||||||
|
}
|
||||||
|
_ => false,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|||||||
+234
-3
@@ -3,14 +3,20 @@ use std::sync::Arc;
|
|||||||
use axum::{
|
use axum::{
|
||||||
body::Body,
|
body::Body,
|
||||||
http::{Request, StatusCode},
|
http::{Request, StatusCode},
|
||||||
|
routing::get,
|
||||||
|
Json,
|
||||||
|
Router,
|
||||||
};
|
};
|
||||||
use base64::{engine::general_purpose::STANDARD, Engine};
|
use base64::{engine::general_purpose::STANDARD, Engine};
|
||||||
|
use chrono::{DateTime, Utc};
|
||||||
|
use serde_json::json;
|
||||||
use sqlx::any::install_default_drivers;
|
use sqlx::any::install_default_drivers;
|
||||||
use tower::ServiceExt;
|
use tower::ServiceExt;
|
||||||
|
use tokio::net::TcpListener;
|
||||||
|
|
||||||
use mikebase::{
|
use mikebase::{
|
||||||
build_router,
|
build_router,
|
||||||
config::{AppConfig, AuthConfig, DatabaseConfig, S3Config, ServerConfig},
|
config::{AppConfig, AuthConfig, DatabaseConfig, ImportConfig, S3Config, ServerConfig},
|
||||||
db::Database,
|
db::Database,
|
||||||
models::new_uuid,
|
models::new_uuid,
|
||||||
storage::S3Storage,
|
storage::S3Storage,
|
||||||
@@ -20,6 +26,14 @@ use mikebase::{
|
|||||||
// ── Helpers ───────────────────────────────────────────────────────────────────
|
// ── Helpers ───────────────────────────────────────────────────────────────────
|
||||||
|
|
||||||
async fn test_state() -> AppState {
|
async fn test_state() -> AppState {
|
||||||
|
test_state_with_s3(
|
||||||
|
"http://localhost:19999".to_string(),
|
||||||
|
"http://localhost:19999/test-bucket".to_string(),
|
||||||
|
)
|
||||||
|
.await
|
||||||
|
}
|
||||||
|
|
||||||
|
async fn test_state_with_s3(s3_endpoint: String, s3_public_url: String) -> AppState {
|
||||||
install_default_drivers();
|
install_default_drivers();
|
||||||
let pool = sqlx::pool::PoolOptions::<sqlx::Any>::new()
|
let pool = sqlx::pool::PoolOptions::<sqlx::Any>::new()
|
||||||
.max_connections(1)
|
.max_connections(1)
|
||||||
@@ -31,12 +45,12 @@ async fn test_state() -> AppState {
|
|||||||
|
|
||||||
let cfg = Arc::new(AppConfig {
|
let cfg = Arc::new(AppConfig {
|
||||||
s3: S3Config {
|
s3: S3Config {
|
||||||
endpoint: "http://localhost:19999".to_string(),
|
endpoint: s3_endpoint,
|
||||||
region: "us-east-1".to_string(),
|
region: "us-east-1".to_string(),
|
||||||
bucket: "test-bucket".to_string(),
|
bucket: "test-bucket".to_string(),
|
||||||
access_key: "test".to_string(),
|
access_key: "test".to_string(),
|
||||||
secret_key: "test".to_string(),
|
secret_key: "test".to_string(),
|
||||||
public_url: "http://localhost:19999/test-bucket".to_string(),
|
public_url: s3_public_url,
|
||||||
},
|
},
|
||||||
database: DatabaseConfig { url: "sqlite::memory:".to_string() },
|
database: DatabaseConfig { url: "sqlite::memory:".to_string() },
|
||||||
server: ServerConfig::default(),
|
server: ServerConfig::default(),
|
||||||
@@ -44,12 +58,84 @@ async fn test_state() -> AppState {
|
|||||||
username: "admin".to_string(),
|
username: "admin".to_string(),
|
||||||
password: "secret".to_string(),
|
password: "secret".to_string(),
|
||||||
}),
|
}),
|
||||||
|
import: ImportConfig {
|
||||||
|
allowed_hosts: vec!["smutba.se".to_string(), "localhost".to_string(), "127.0.0.1".to_string()],
|
||||||
|
},
|
||||||
});
|
});
|
||||||
|
|
||||||
let storage = S3Storage::new(&cfg);
|
let storage = S3Storage::new(&cfg);
|
||||||
AppState { db, storage, cfg }
|
AppState { db, storage, cfg }
|
||||||
}
|
}
|
||||||
|
|
||||||
|
async fn spawn_mock_s3_server() -> (String, tokio::task::JoinHandle<()>) {
|
||||||
|
async fn ok() -> StatusCode {
|
||||||
|
StatusCode::OK
|
||||||
|
}
|
||||||
|
|
||||||
|
let listener = TcpListener::bind("127.0.0.1:0").await.unwrap();
|
||||||
|
let addr = listener.local_addr().unwrap();
|
||||||
|
let base = format!("http://{}", addr);
|
||||||
|
|
||||||
|
let app = Router::new()
|
||||||
|
.route("/", get(ok))
|
||||||
|
.route("/{*path}", axum::routing::any(ok));
|
||||||
|
|
||||||
|
let handle = tokio::spawn(async move {
|
||||||
|
axum::serve(listener, app).await.unwrap();
|
||||||
|
});
|
||||||
|
|
||||||
|
(base, handle)
|
||||||
|
}
|
||||||
|
|
||||||
|
async fn spawn_legacy_source_server() -> (String, tokio::task::JoinHandle<()>) {
|
||||||
|
async fn image_new() -> ([(&'static str, &'static str); 1], &'static [u8]) {
|
||||||
|
([ ("content-type", "image/png") ], b"PNGDATA")
|
||||||
|
}
|
||||||
|
|
||||||
|
async fn image_dup() -> ([(&'static str, &'static str); 1], &'static [u8]) {
|
||||||
|
([ ("content-type", "image/png") ], b"PNGDATA2")
|
||||||
|
}
|
||||||
|
|
||||||
|
let listener = TcpListener::bind("127.0.0.1:0").await.unwrap();
|
||||||
|
let addr = listener.local_addr().unwrap();
|
||||||
|
let base = format!("http://{}", addr);
|
||||||
|
|
||||||
|
let payload_base = base.clone();
|
||||||
|
let app = Router::new()
|
||||||
|
.route(
|
||||||
|
"/emoji/json/",
|
||||||
|
get(move || {
|
||||||
|
let payload_base = payload_base.clone();
|
||||||
|
async move {
|
||||||
|
Json(json!({
|
||||||
|
"emotes": [
|
||||||
|
{
|
||||||
|
"name": "legacy_new",
|
||||||
|
"url": format!("{}/images/new.png", payload_base),
|
||||||
|
"created": "2020-01-01T00:00:00+00:00",
|
||||||
|
"modified": "2020-02-02T00:00:00+00:00"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "legacy_duplicate",
|
||||||
|
"url": format!("{}/images/duplicate.png", payload_base),
|
||||||
|
"created": "2021-01-01T00:00:00+00:00",
|
||||||
|
"modified": "2021-01-02T00:00:00+00:00"
|
||||||
|
}
|
||||||
|
]
|
||||||
|
}))
|
||||||
|
}
|
||||||
|
}),
|
||||||
|
)
|
||||||
|
.route("/images/new.png", get(image_new))
|
||||||
|
.route("/images/duplicate.png", get(image_dup));
|
||||||
|
|
||||||
|
let handle = tokio::spawn(async move {
|
||||||
|
axum::serve(listener, app).await.unwrap();
|
||||||
|
});
|
||||||
|
|
||||||
|
(base, handle)
|
||||||
|
}
|
||||||
|
|
||||||
fn auth_header() -> String {
|
fn auth_header() -> String {
|
||||||
format!("Basic {}", STANDARD.encode("admin:secret"))
|
format!("Basic {}", STANDARD.encode("admin:secret"))
|
||||||
}
|
}
|
||||||
@@ -204,6 +290,151 @@ async fn manage_emotes_requires_auth() {
|
|||||||
assert_eq!(resp.status(), StatusCode::UNAUTHORIZED);
|
assert_eq!(resp.status(), StatusCode::UNAUTHORIZED);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[tokio::test]
|
||||||
|
async fn manage_import_requires_auth() {
|
||||||
|
let app = build_router(test_state().await);
|
||||||
|
let resp = app
|
||||||
|
.oneshot(
|
||||||
|
Request::builder()
|
||||||
|
.method("POST")
|
||||||
|
.uri("/manage/import")
|
||||||
|
.header("content-type", "application/json")
|
||||||
|
.body(Body::from(r#"{"source_url":"https://smutba.se/emoji/json/"}"#))
|
||||||
|
.unwrap(),
|
||||||
|
)
|
||||||
|
.await
|
||||||
|
.unwrap();
|
||||||
|
assert_eq!(resp.status(), StatusCode::UNAUTHORIZED);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[tokio::test]
|
||||||
|
async fn manage_import_rejects_non_allowlisted_host() {
|
||||||
|
let app = build_router(test_state().await);
|
||||||
|
let resp = app
|
||||||
|
.oneshot(
|
||||||
|
Request::builder()
|
||||||
|
.method("POST")
|
||||||
|
.uri("/manage/import")
|
||||||
|
.header("authorization", auth_header())
|
||||||
|
.header("content-type", "application/json")
|
||||||
|
.body(Body::from(r#"{"source_url":"https://example.com/emoji/json/"}"#))
|
||||||
|
.unwrap(),
|
||||||
|
)
|
||||||
|
.await
|
||||||
|
.unwrap();
|
||||||
|
assert_eq!(resp.status(), StatusCode::BAD_REQUEST);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[tokio::test]
|
||||||
|
async fn manage_import_mirrors_and_skips_duplicates() {
|
||||||
|
let (legacy_base, server_handle) = spawn_legacy_source_server().await;
|
||||||
|
let (s3_base, s3_handle) = spawn_mock_s3_server().await;
|
||||||
|
|
||||||
|
let state = test_state_with_s3(s3_base.clone(), format!("{}/test-bucket", s3_base)).await;
|
||||||
|
// Pre-seed one duplicate emote name.
|
||||||
|
let existing_id = new_uuid();
|
||||||
|
state
|
||||||
|
.db
|
||||||
|
.create_emote(&existing_id, "legacy_duplicate", None, "emoji/existing.png")
|
||||||
|
.await
|
||||||
|
.unwrap();
|
||||||
|
|
||||||
|
let app = build_router(state.clone());
|
||||||
|
let resp = app
|
||||||
|
.oneshot(
|
||||||
|
Request::builder()
|
||||||
|
.method("POST")
|
||||||
|
.uri("/manage/import")
|
||||||
|
.header("authorization", auth_header())
|
||||||
|
.header("content-type", "application/json")
|
||||||
|
.body(Body::from(format!(
|
||||||
|
"{{\"source_url\":\"{}/emoji/json/\"}}",
|
||||||
|
legacy_base
|
||||||
|
)))
|
||||||
|
.unwrap(),
|
||||||
|
)
|
||||||
|
.await
|
||||||
|
.unwrap();
|
||||||
|
|
||||||
|
assert_eq!(resp.status(), StatusCode::OK);
|
||||||
|
let json = response_json(resp.into_body()).await;
|
||||||
|
assert_eq!(json["total"], 2);
|
||||||
|
assert_eq!(json["imported"], 1);
|
||||||
|
assert_eq!(json["skipped"], 1);
|
||||||
|
assert_eq!(json["failed"], 0);
|
||||||
|
|
||||||
|
let rows = state.db.list_emotes().await.unwrap();
|
||||||
|
let imported = rows.iter().find(|r| r.name == "legacy_new").unwrap();
|
||||||
|
let created = DateTime::parse_from_rfc3339(&imported.created)
|
||||||
|
.unwrap()
|
||||||
|
.with_timezone(&Utc)
|
||||||
|
.to_rfc3339();
|
||||||
|
let modified = DateTime::parse_from_rfc3339(&imported.modified)
|
||||||
|
.unwrap()
|
||||||
|
.with_timezone(&Utc)
|
||||||
|
.to_rfc3339();
|
||||||
|
assert_eq!(created, "2020-01-01T00:00:00+00:00");
|
||||||
|
assert_eq!(modified, "2020-02-02T00:00:00+00:00");
|
||||||
|
|
||||||
|
server_handle.abort();
|
||||||
|
s3_handle.abort();
|
||||||
|
}
|
||||||
|
|
||||||
|
#[tokio::test]
|
||||||
|
async fn manage_import_dry_run_does_not_persist() {
|
||||||
|
let (legacy_base, server_handle) = spawn_legacy_source_server().await;
|
||||||
|
let (s3_base, s3_handle) = spawn_mock_s3_server().await;
|
||||||
|
|
||||||
|
let state = test_state_with_s3(s3_base.clone(), format!("{}/test-bucket", s3_base)).await;
|
||||||
|
// Pre-seed one duplicate so we can verify would_skip detection.
|
||||||
|
let existing_id = new_uuid();
|
||||||
|
state
|
||||||
|
.db
|
||||||
|
.create_emote(&existing_id, "legacy_duplicate", None, "emoji/existing.png")
|
||||||
|
.await
|
||||||
|
.unwrap();
|
||||||
|
|
||||||
|
let app = build_router(state.clone());
|
||||||
|
let body = format!(
|
||||||
|
"{{\"source_url\":\"{}/emoji/json/\",\"dry_run\":true}}",
|
||||||
|
legacy_base
|
||||||
|
);
|
||||||
|
let resp = app
|
||||||
|
.oneshot(
|
||||||
|
Request::builder()
|
||||||
|
.method("POST")
|
||||||
|
.uri("/manage/import")
|
||||||
|
.header("authorization", auth_header())
|
||||||
|
.header("content-type", "application/json")
|
||||||
|
.body(Body::from(body))
|
||||||
|
.unwrap(),
|
||||||
|
)
|
||||||
|
.await
|
||||||
|
.unwrap();
|
||||||
|
|
||||||
|
assert_eq!(resp.status(), StatusCode::OK);
|
||||||
|
let json = response_json(resp.into_body()).await;
|
||||||
|
assert_eq!(json["dry_run"], true);
|
||||||
|
assert_eq!(json["total"], 2);
|
||||||
|
assert_eq!(json["imported"], 1);
|
||||||
|
assert_eq!(json["skipped"], 1);
|
||||||
|
assert_eq!(json["failed"], 0);
|
||||||
|
|
||||||
|
let results = json["results"].as_array().unwrap();
|
||||||
|
let new_result = results.iter().find(|r| r["name"] == "legacy_new").unwrap();
|
||||||
|
assert_eq!(new_result["status"], "would_import");
|
||||||
|
let dup_result = results.iter().find(|r| r["name"] == "legacy_duplicate").unwrap();
|
||||||
|
assert_eq!(dup_result["status"], "would_skip");
|
||||||
|
|
||||||
|
// Nothing new should have been written to the DB.
|
||||||
|
let rows = state.db.list_emotes().await.unwrap();
|
||||||
|
assert_eq!(rows.len(), 1, "dry-run must not insert any rows");
|
||||||
|
assert_eq!(rows[0].name, "legacy_duplicate");
|
||||||
|
|
||||||
|
server_handle.abort();
|
||||||
|
s3_handle.abort();
|
||||||
|
}
|
||||||
|
|
||||||
// ── POST /emotes input validation ─────────────────────────────────────────────
|
// ── POST /emotes input validation ─────────────────────────────────────────────
|
||||||
|
|
||||||
#[tokio::test]
|
#[tokio::test]
|
||||||
|
|||||||
Reference in New Issue
Block a user