From 4c62ba284508075bb20f3417154eaee073504c2e Mon Sep 17 00:00:00 2001 From: J-onasJones Date: Sun, 11 Feb 2024 06:28:44 +0100 Subject: [PATCH 01/16] added /status route --- src/server.rs | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/src/server.rs b/src/server.rs index 7aa133e..cdf5e6d 100644 --- a/src/server.rs +++ b/src/server.rs @@ -20,9 +20,13 @@ pub async fn serve() { let favicon = warp::path("favicon.ico").and(warp::fs::file("./src/favicon.png")); + // /status => 200 OK + let status = warp::path("status") + .map(|| warp::reply()); + // GET (any) => reply with return from handle_path - let routes = favicon.or(get_v1_routes()) - .recover(handle_rejection); + let routes = favicon.or(status.or(get_v1_routes()) + .recover(handle_rejection)); async fn handle_rejection(err: warp::Rejection) -> Result { From bdfd5a74a69502d3771518eae1f8f1b291ac5997 Mon Sep 17 00:00:00 2001 From: J-onasJones Date: Sun, 11 Feb 2024 08:44:20 +0100 Subject: [PATCH 02/16] Added docker compose instructions --- README.md | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) diff --git a/README.md b/README.md index 5cbba9f..f4f1748 100644 --- a/README.md +++ b/README.md @@ -34,3 +34,24 @@ export API_IP={ip_address} export LASTFM_API_KEY={lastfm_api_key} export LASTFM_API_SECRET={lastfm_api_secret} ``` + +## Docker Compose + +`docker-compose.yaml`: +```yaml +version: '3.8' +services: + arch-linux: + image: archlinux:latest + container_name: jonas_jones-api + ports: + - "3030:3030" + volumes: + - /home/jonas_jones/jonas_jones-api:/home/jonas_jones/jonas_jones-api + command: ["sh", "-c", "pacman -Syu --noconfirm --needed pkg-config openssl cargo && cd /home/jonas_jones/jonas_jones-api && /usr/bin/cargo run"] +``` + +run container: +```sh +docker-compose up -d +``` From 7bee1f5bae27aeebc31332eea9ed613dfc6dc17f Mon Sep 17 00:00:00 2001 From: J-onasJones Date: Mon, 12 Feb 2024 03:46:08 +0100 Subject: [PATCH 03/16] added update runner thread --- src/main.rs | 35 ++++++++++++++- src/v1/mod.rs | 9 +++- src/v1/{update => run}/mod.rs | 84 +++++++++++++++++------------------ 3 files changed, 82 insertions(+), 46 deletions(-) rename src/v1/{update => run}/mod.rs (77%) diff --git a/src/main.rs b/src/main.rs index 86eb624..a55572c 100644 --- a/src/main.rs +++ b/src/main.rs @@ -1,4 +1,7 @@ use dotenv::dotenv; +use tokio::time::{sleep, Duration}; +use v1::{run_kcomebacks_command, run_likedsongs_command, run_projects_command}; +use std::fs; pub mod v1; pub mod logger; @@ -10,10 +13,38 @@ pub use logger::Logger; pub use tools::parse_ip; +async fn periodic_script_runner() { + loop { + Logger::info("Running periodic scripts..."); + // Run all Functions + let _ = run_kcomebacks_command(); + let _ = run_projects_command(); + let _ = run_likedsongs_command(); + + // Sleep for 6 hours + sleep(Duration::from_secs(6 * 60 * 60)).await; + } +} + #[tokio::main(flavor = "current_thread")] async fn main() { - // load .env file + // Load .env file dotenv().ok(); - server::serve().await; + // Start the api + let server_task = tokio::spawn(async { + server::serve().await; + }); + + // periodic script runner + let second_task = tokio::spawn(async { + // check if the local repository exists, if not, clone it + if !fs::metadata("./resources/turbo_octo_potato").is_ok() { + v1::run_setup().unwrap(); + }; + periodic_script_runner().await; + }); + + // Wait for both tasks to complete + let _ = tokio::try_join!(server_task, second_task); } diff --git a/src/v1/mod.rs b/src/v1/mod.rs index d0ff506..ff4bf8f 100644 --- a/src/v1/mod.rs +++ b/src/v1/mod.rs @@ -2,13 +2,18 @@ mod builtin; mod debug; mod kcomebacks; mod projects; -mod update; +mod run; + +pub use run::setup as run_setup; +pub use run::run_kcomebacks_command; +pub use run::run_projects_command; +pub use run::run_likedsongs_command; pub use builtin::get_builtin_routes as get_v1_builtin_routes; pub use debug::get_debug_routes as get_v1_debug_routes; pub use kcomebacks::get_kcomebacks_routes as get_v1_kcomebacks_routes; pub use projects::get_project_routes as get_v1_project_routes; -pub use update::get_run_routes as get_v1_updates_routes; +pub use run::get_run_routes as get_v1_updates_routes; use warp::Filter; diff --git a/src/v1/update/mod.rs b/src/v1/run/mod.rs similarity index 77% rename from src/v1/update/mod.rs rename to src/v1/run/mod.rs index 15a7f01..849c133 100644 --- a/src/v1/update/mod.rs +++ b/src/v1/run/mod.rs @@ -1,10 +1,10 @@ use std::fs; -use std::io::BufRead; +// use std::io::BufRead; use std::process::{Stdio, Command}; use serde::{Deserialize, Serialize}; use serde_json::json; -use tokio::sync::mpsc; +// use tokio::sync::mpsc; use tokio::task; use warp::Filter; use crate::error_responses::InternalServerError; @@ -80,7 +80,7 @@ async fn sync_liked_songs() -> Result { } -fn setup() -> Result<(), git2::Error> { +pub fn setup() -> Result<(), git2::Error> { let repository_url = "https://github.com/JonasunderscoreJones/turbo-octo-potato.git"; let local_directory = "resources/turbo_octo_potato"; @@ -131,8 +131,8 @@ fn setup() -> Result<(), git2::Error> { // run_command with python file and args as parameters -fn run_kcomebacks_command() -> Result<(), std::io::Error> { - let (tx, mut rx) = mpsc::channel(1); +pub fn run_kcomebacks_command() -> Result<(), std::io::Error> { + // let (tx, mut rx) = mpsc::channel(1); task::spawn_blocking(move || { let mut child = Command::new("python3") @@ -143,20 +143,20 @@ fn run_kcomebacks_command() -> Result<(), std::io::Error> { .spawn() .expect("failed to execute child"); - let stdout = child.stdout.as_mut().unwrap(); + // let stdout = child.stdout.as_mut().unwrap(); - let mut reader = std::io::BufReader::new(stdout); + // let mut reader = std::io::BufReader::new(stdout); - let mut line = String::new(); + // let mut line = String::new(); - loop { - let len = reader.read_line(&mut line).unwrap(); - if len == 0 { - break; - } - tx.blocking_send(line.clone()).unwrap(); - line.clear(); - } + // loop { + // let len = reader.read_line(&mut line).unwrap(); + // if len == 0 { + // break; + // } + // tx.blocking_send(line.clone()).unwrap(); + // line.clear(); + // } child.wait().unwrap(); }); @@ -171,8 +171,8 @@ fn run_kcomebacks_command() -> Result<(), std::io::Error> { Ok(()) } -fn run_projects_command() -> Result<(), std::io::Error> { - let (tx, mut rx) = mpsc::channel(1); +pub fn run_projects_command() -> Result<(), std::io::Error> { + // let (tx, mut rx) = mpsc::channel(1); task::spawn_blocking(move || { let mut child = Command::new("python3") @@ -183,20 +183,20 @@ fn run_projects_command() -> Result<(), std::io::Error> { .spawn() .expect("failed to execute child"); - let stdout = child.stdout.as_mut().unwrap(); + // let stdout = child.stdout.as_mut().unwrap(); - let mut reader = std::io::BufReader::new(stdout); + // let mut reader = std::io::BufReader::new(stdout); - let mut line = String::new(); + // let mut line = String::new(); - loop { - let len = reader.read_line(&mut line).unwrap(); - if len == 0 { - break; - } - tx.blocking_send(line.clone()).unwrap(); - line.clear(); - } + // loop { + // let len = reader.read_line(&mut line).unwrap(); + // if len == 0 { + // break; + // } + // tx.blocking_send(line.clone()).unwrap(); + // line.clear(); + // } child.wait().unwrap(); }); @@ -211,8 +211,8 @@ fn run_projects_command() -> Result<(), std::io::Error> { Ok(()) } -fn run_likedsongs_command() -> Result<(), std::io::Error> { - let (tx, mut rx) = mpsc::channel(1); +pub fn run_likedsongs_command() -> Result<(), std::io::Error> { + // let (tx, mut rx) = mpsc::channel(1); task::spawn_blocking(move || { let mut child = Command::new("python3") @@ -222,20 +222,20 @@ fn run_likedsongs_command() -> Result<(), std::io::Error> { .spawn() .expect("failed to execute child"); - let stdout = child.stdout.as_mut().unwrap(); + // let stdout = child.stdout.as_mut().unwrap(); - let mut reader = std::io::BufReader::new(stdout); + // let mut reader = std::io::BufReader::new(stdout); - let mut line = String::new(); + // let mut line = String::new(); - loop { - let len = reader.read_line(&mut line).unwrap(); - if len == 0 { - break; - } - tx.blocking_send(line.clone()).unwrap(); - line.clear(); - } + // loop { + // let len = reader.read_line(&mut line).unwrap(); + // if len == 0 { + // break; + // } + // tx.blocking_send(line.clone()).unwrap(); + // line.clear(); + // } child.wait().unwrap(); }); From 5e9ed3738c3418969d13ebb7916ec3b377eaad98 Mon Sep 17 00:00:00 2001 From: J-onasJones Date: Mon, 12 Feb 2024 03:49:41 +0100 Subject: [PATCH 04/16] fixed logs --- src/v1/run/mod.rs | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/v1/run/mod.rs b/src/v1/run/mod.rs index 849c133..9861d53 100644 --- a/src/v1/run/mod.rs +++ b/src/v1/run/mod.rs @@ -166,7 +166,7 @@ pub fn run_kcomebacks_command() -> Result<(), std::io::Error> { // Logger::info(&format!("[/v1/kcomebacks/update]: {}", line)); // } // }); - Logger::info("kcomebacks updated"); + Logger::info("Updating kcomebacks..."); Ok(()) } @@ -206,7 +206,7 @@ pub fn run_projects_command() -> Result<(), std::io::Error> { // Logger::info(&format!("[/v1/projects/update]: {}", line)); // } // }); - Logger::info("projects updated"); + Logger::info("Updating projects..."); Ok(()) } @@ -245,7 +245,7 @@ pub fn run_likedsongs_command() -> Result<(), std::io::Error> { // Logger::info(&format!("[/v1/synclikedsongs]: {}", line)); // } // }); - Logger::info("liked songs synced"); + Logger::info("Syncing liked songs..."); Ok(()) } \ No newline at end of file From 48da0d7f95a3c2e28e50be6aa839151420d299ae Mon Sep 17 00:00:00 2001 From: J-onasJones Date: Mon, 12 Feb 2024 04:05:44 +0100 Subject: [PATCH 05/16] Fixed docker compose and python lib requirements --- README.md | 2 +- requirements.txt | 13 +++++++++++++ 2 files changed, 14 insertions(+), 1 deletion(-) create mode 100644 requirements.txt diff --git a/README.md b/README.md index f4f1748..43d7ecd 100644 --- a/README.md +++ b/README.md @@ -48,7 +48,7 @@ services: - "3030:3030" volumes: - /home/jonas_jones/jonas_jones-api:/home/jonas_jones/jonas_jones-api - command: ["sh", "-c", "pacman -Syu --noconfirm --needed pkg-config openssl cargo && cd /home/jonas_jones/jonas_jones-api && /usr/bin/cargo run"] + command: ["sh", "-c", "pacman -Syu --noconfirm --needed pkg-config openssl python3 python-pip cargo && pip install -r requirements.txt && cd /home/jonas_jones/jonas_jones-api && /usr/bin/cargo run"] ``` run container: diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 0000000..560c864 --- /dev/null +++ b/requirements.txt @@ -0,0 +1,13 @@ +os +sys +json +time +requests +datetime +python-dotenv +spotipy +praw +re +spotipy +pylast +typing \ No newline at end of file From a63f933dd94f8c244dfbe75a4189ca9605e9a4a7 Mon Sep 17 00:00:00 2001 From: J-onasJones Date: Mon, 12 Feb 2024 04:21:05 +0100 Subject: [PATCH 06/16] Fixed dependencies --- requirements.txt | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/requirements.txt b/requirements.txt index 560c864..ce908f5 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,13 +1,9 @@ -os -sys -json -time requests datetime python-dotenv spotipy praw -re spotipy pylast -typing \ No newline at end of file +typing +markdown \ No newline at end of file From 4ecbd7aa98d774c457ac56e9171457cdb3de4bc5 Mon Sep 17 00:00:00 2001 From: J-onasJones Date: Mon, 12 Feb 2024 04:32:37 +0100 Subject: [PATCH 07/16] fixed docker compose --- README.md | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 43d7ecd..6b0acca 100644 --- a/README.md +++ b/README.md @@ -37,7 +37,7 @@ export LASTFM_API_SECRET={lastfm_api_secret} ## Docker Compose -`docker-compose.yaml`: +`docker-compose.yaml` (folder paths need adjusting): ```yaml version: '3.8' services: @@ -48,7 +48,8 @@ services: - "3030:3030" volumes: - /home/jonas_jones/jonas_jones-api:/home/jonas_jones/jonas_jones-api - command: ["sh", "-c", "pacman -Syu --noconfirm --needed pkg-config openssl python3 python-pip cargo && pip install -r requirements.txt && cd /home/jonas_jones/jonas_jones-api && /usr/bin/cargo run"] + - /home/jonas_jones/.config/rclone/:/root/.config/rclone/ + command: ["sh", "-c", "pacman -Syu --noconfirm --needed pkg-config openssl python3 python-pip rclone cargo && python3 -m venv api-venv && source api-venv/bin/activate && cd /home/jonas_jones/jonas_jones-api && pip install -r requirements.txt && /usr/bin/cargo run"] ``` run container: From 1cdb34503d82470215f004c1aec1722cfb485c1e Mon Sep 17 00:00:00 2001 From: J-onasJones Date: Mon, 12 Feb 2024 04:34:34 +0100 Subject: [PATCH 08/16] Added docker-compose file --- docker-compose.yaml | 11 +++++++++++ 1 file changed, 11 insertions(+) create mode 100644 docker-compose.yaml diff --git a/docker-compose.yaml b/docker-compose.yaml new file mode 100644 index 0000000..3a3650d --- /dev/null +++ b/docker-compose.yaml @@ -0,0 +1,11 @@ +version: '3.8' +services: + arch-linux: + image: archlinux:latest + container_name: jonas_jones-api + ports: + - "3030:3030" + volumes: + - /home/jonas_jones/jonas_jones-api:/home/jonas_jones/jonas_jones-api + - /home/jonas_jones/.config/rclone/:/root/.config/rclone/ + command: ["sh", "-c", "pacman -Syu --noconfirm --needed pkg-config openssl python3 python-pip rclone cargo && python3 -m venv api-venv && source api-venv/bin/activate && cd /home/jonas_jones/jonas_jones-api && pip install -r requirements.txt && /usr/bin/cargo run"] \ No newline at end of file From 3b5378c6f280c20677bb4b1561734888a8936709 Mon Sep 17 00:00:00 2001 From: J-onasJones Date: Mon, 12 Feb 2024 04:41:23 +0100 Subject: [PATCH 09/16] Fixed version (0verdue!!!) --- Cargo.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Cargo.toml b/Cargo.toml index 6d174f1..c2c6cb0 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "jonas_jones-api" -version = "0.3.0" +version = "0.4.1" edition = "2021" # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html From d2a89d506713a686f4254772031da1b9cc347368 Mon Sep 17 00:00:00 2001 From: Jonas_Jones Date: Wed, 14 Feb 2024 03:41:25 +0100 Subject: [PATCH 10/16] roadmap draft --- README.md | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/README.md b/README.md index 6b0acca..4c7b7b1 100644 --- a/README.md +++ b/README.md @@ -56,3 +56,12 @@ run container: ```sh docker-compose up -d ``` + +## Roadmap + +- analytics backend. track request origin through IP from header (store IP hash, region and time) +- rewrite all scripts in rust +- DB implementation for projects, kcomebacks, minecraft mod versions +- session backend, auth token system +- implementation for dashboard front-end with analytics/config +- complete minecraft mod implementation \ No newline at end of file From 41fa898a6729c86f850a71ca9e20bfcff04d5a77 Mon Sep 17 00:00:00 2001 From: J-onasJones Date: Wed, 15 May 2024 13:10:18 +0200 Subject: [PATCH 11/16] Updated dependencies --- Cargo.lock | 267 +++++++++++++++++++++++++++++++++++++++++++++-------- Cargo.toml | 4 +- 2 files changed, 233 insertions(+), 38 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 26fde4a..b15b2cb 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -90,6 +90,12 @@ version = "0.21.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "35636a1494ede3b646cc98f74f8e62c773a38a659ebc777a2cf26b9b74171df9" +[[package]] +name = "base64" +version = "0.22.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "72b3254f16251a8381aa12e40e3c4d2f0199f8c6508fbecb9d91f575e0fbb8c6" + [[package]] name = "bitflags" version = "1.3.2" @@ -373,7 +379,26 @@ dependencies = [ "futures-core", "futures-sink", "futures-util", - "http", + "http 0.2.11", + "indexmap", + "slab", + "tokio", + "tokio-util", + "tracing", +] + +[[package]] +name = "h2" +version = "0.4.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "816ec7294445779408f36fe57bc5b7fc1cf59664059096c65f905c1c61f58069" +dependencies = [ + "bytes", + "fnv", + "futures-core", + "futures-sink", + "futures-util", + "http 1.1.0", "indexmap", "slab", "tokio", @@ -393,10 +418,10 @@ version = "0.3.9" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "06683b93020a07e3dbcf5f8c0f6d40080d725bea7936fc01ad345c01b97dc270" dependencies = [ - "base64", + "base64 0.21.5", "bytes", "headers-core", - "http", + "http 0.2.11", "httpdate", "mime", "sha1", @@ -408,7 +433,7 @@ version = "0.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e7f66481bfee273957b1f20485a4ff3362987f85b2c236580d81b4eb7a326429" dependencies = [ - "http", + "http 0.2.11", ] [[package]] @@ -428,6 +453,17 @@ dependencies = [ "itoa", ] +[[package]] +name = "http" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "21b9ddb458710bc376481b842f5da65cdf31522de232c1ca8146abce2a358258" +dependencies = [ + "bytes", + "fnv", + "itoa", +] + [[package]] name = "http-body" version = "0.4.5" @@ -435,7 +471,30 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d5f38f16d184e36f2408a55281cd658ecbd3ca05cce6d6510a176eca393e26d1" dependencies = [ "bytes", - "http", + "http 0.2.11", + "pin-project-lite", +] + +[[package]] +name = "http-body" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1cac85db508abc24a2e48553ba12a996e87244a0395ce011e62b37158745d643" +dependencies = [ + "bytes", + "http 1.1.0", +] + +[[package]] +name = "http-body-util" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0475f8b2ac86659c21b64320d5d653f9efe42acd2a4e560073ec61a155a34f1d" +dependencies = [ + "bytes", + "futures-core", + "http 1.1.0", + "http-body 1.0.0", "pin-project-lite", ] @@ -461,9 +520,9 @@ dependencies = [ "futures-channel", "futures-core", "futures-util", - "h2", - "http", - "http-body", + "h2 0.3.22", + "http 0.2.11", + "http-body 0.4.5", "httparse", "httpdate", "itoa", @@ -475,6 +534,26 @@ dependencies = [ "want", ] +[[package]] +name = "hyper" +version = "1.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fe575dd17d0862a9a33781c8c4696a55c320909004a67a00fb286ba8b1bc496d" +dependencies = [ + "bytes", + "futures-channel", + "futures-util", + "h2 0.4.4", + "http 1.1.0", + "http-body 1.0.0", + "httparse", + "itoa", + "pin-project-lite", + "smallvec", + "tokio", + "want", +] + [[package]] name = "hyper-rustls" version = "0.24.2" @@ -482,8 +561,8 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ec3efd23720e2049821a693cbc7e65ea87c72f1c58ff2f9522ff332b1491e590" dependencies = [ "futures-util", - "http", - "hyper", + "http 0.2.11", + "hyper 0.14.27", "rustls", "tokio", "tokio-rustls", @@ -491,15 +570,38 @@ dependencies = [ [[package]] name = "hyper-tls" -version = "0.5.0" +version = "0.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d6183ddfa99b85da61a140bea0efc93fdf56ceaa041b37d553518030827f9905" +checksum = "70206fc6890eaca9fde8a0bf71caa2ddfc9fe045ac9e5c70df101a7dbde866e0" dependencies = [ "bytes", - "hyper", + "http-body-util", + "hyper 1.3.1", + "hyper-util", "native-tls", "tokio", "tokio-native-tls", + "tower-service", +] + +[[package]] +name = "hyper-util" +version = "0.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ca38ef113da30126bbff9cd1705f9273e15d45498615d138b0c20279ac7a76aa" +dependencies = [ + "bytes", + "futures-channel", + "futures-util", + "http 1.1.0", + "http-body 1.0.0", + "hyper 1.3.1", + "pin-project-lite", + "socket2 0.5.5", + "tokio", + "tower", + "tower-service", + "tracing", ] [[package]] @@ -568,7 +670,7 @@ dependencies = [ [[package]] name = "jonas_jones-api" -version = "0.3.0" +version = "0.4.1" dependencies = [ "chrono", "dotenv", @@ -577,7 +679,7 @@ dependencies = [ "log", "parking_lot", "regex", - "reqwest", + "reqwest 0.12.4", "serde", "serde_json", "tokio", @@ -596,16 +698,16 @@ dependencies = [ [[package]] name = "lastfm" -version = "0.8.1" +version = "0.10.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "aed1b6ed29669ba37e7a2a8a939b7e2ff456883f0eb71a173635fae0ddc1cb5b" +checksum = "6e1c7d19b3dabcb2ccef2ffe389604248cdcbc063e3455e9079992e4ec336158" dependencies = [ "async-stream", "chrono", "dotenv", "lazy_static", "rand", - "reqwest", + "reqwest 0.11.22", "serde", "serde_json", "thiserror", @@ -741,7 +843,7 @@ dependencies = [ "bytes", "encoding_rs", "futures-util", - "http", + "http 0.2.11", "httparse", "log", "memchr", @@ -1011,17 +1113,58 @@ version = "0.11.22" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "046cd98826c46c2ac8ddecae268eb5c2e58628688a5fc7a2643704a73faba95b" dependencies = [ - "base64", + "base64 0.21.5", "bytes", "encoding_rs", "futures-core", "futures-util", - "h2", - "http", - "http-body", - "hyper", + "h2 0.3.22", + "http 0.2.11", + "http-body 0.4.5", + "hyper 0.14.27", "hyper-rustls", + "ipnet", + "js-sys", + "log", + "mime", + "once_cell", + "percent-encoding", + "pin-project-lite", + "rustls", + "rustls-pemfile 1.0.4", + "serde", + "serde_json", + "serde_urlencoded", + "system-configuration", + "tokio", + "tokio-rustls", + "tower-service", + "url", + "wasm-bindgen", + "wasm-bindgen-futures", + "web-sys", + "webpki-roots", + "winreg 0.50.0", +] + +[[package]] +name = "reqwest" +version = "0.12.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "566cafdd92868e0939d3fb961bd0dc25fcfaaed179291093b3d43e6b3150ea10" +dependencies = [ + "base64 0.22.1", + "bytes", + "encoding_rs", + "futures-core", + "futures-util", + "h2 0.4.4", + "http 1.1.0", + "http-body 1.0.0", + "http-body-util", + "hyper 1.3.1", "hyper-tls", + "hyper-util", "ipnet", "js-sys", "log", @@ -1030,22 +1173,20 @@ dependencies = [ "once_cell", "percent-encoding", "pin-project-lite", - "rustls", - "rustls-pemfile", + "rustls-pemfile 2.1.2", "serde", "serde_json", "serde_urlencoded", + "sync_wrapper", "system-configuration", "tokio", "tokio-native-tls", - "tokio-rustls", "tower-service", "url", "wasm-bindgen", "wasm-bindgen-futures", "web-sys", - "webpki-roots", - "winreg", + "winreg 0.52.0", ] [[package]] @@ -1099,9 +1240,25 @@ version = "1.0.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1c74cae0a4cf6ccbbf5f359f08efdf8ee7e1dc532573bf0db71968cb56b1448c" dependencies = [ - "base64", + "base64 0.21.5", ] +[[package]] +name = "rustls-pemfile" +version = "2.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "29993a25686778eb88d4189742cd713c9bce943bc54251a33509dc63cbacf73d" +dependencies = [ + "base64 0.22.1", + "rustls-pki-types", +] + +[[package]] +name = "rustls-pki-types" +version = "1.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "976295e77ce332211c0d24d92c0e83e50f5c5f046d11082cea19f3df13a3562d" + [[package]] name = "rustls-webpki" version = "0.101.7" @@ -1255,9 +1412,9 @@ dependencies = [ [[package]] name = "smallvec" -version = "1.11.2" +version = "1.13.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4dccd0940a2dcdf68d092b8cbab7dc0ad8fa938bf95787e1b916b0e3d0e8e970" +checksum = "3c5e1a9a646d36c3599cd173a41282daf47c44583ad367b8e6837255952e5c67" [[package]] name = "socket2" @@ -1296,6 +1453,12 @@ dependencies = [ "unicode-ident", ] +[[package]] +name = "sync_wrapper" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2047c6ded9c721764247e62cd3b03c09ffc529b2ba5b10ec482ae507a4a70160" + [[package]] name = "system-configuration" version = "0.5.1" @@ -1486,6 +1649,28 @@ dependencies = [ "winnow", ] +[[package]] +name = "tower" +version = "0.4.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b8fa9be0de6cf49e536ce1851f987bd21a43b771b09473c3549a6c853db37c1c" +dependencies = [ + "futures-core", + "futures-util", + "pin-project", + "pin-project-lite", + "tokio", + "tower-layer", + "tower-service", + "tracing", +] + +[[package]] +name = "tower-layer" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c20c8dbed6283a09604c3e69b4b7eeb54e298b8a600d4d5ecb5ad39de609f1d0" + [[package]] name = "tower-service" version = "0.3.2" @@ -1539,7 +1724,7 @@ dependencies = [ "byteorder", "bytes", "data-encoding", - "http", + "http 0.2.11", "httparse", "log", "rand", @@ -1659,15 +1844,15 @@ dependencies = [ "futures-channel", "futures-util", "headers", - "http", - "hyper", + "http 0.2.11", + "hyper 0.14.27", "log", "mime", "mime_guess", "multer", "percent-encoding", "pin-project", - "rustls-pemfile", + "rustls-pemfile 1.0.4", "scoped-tls", "serde", "serde_json", @@ -1949,3 +2134,13 @@ dependencies = [ "cfg-if", "windows-sys 0.48.0", ] + +[[package]] +name = "winreg" +version = "0.52.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a277a57398d4bfa075df44f501a17cfdf8542d224f0d36095a2adc7aee4ef0a5" +dependencies = [ + "cfg-if", + "windows-sys 0.48.0", +] diff --git a/Cargo.toml b/Cargo.toml index c2c6cb0..4676fb9 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -11,11 +11,11 @@ parking_lot = "0.12.1" serde = { version = "1.0", features = ["derive"] } tokio = { version = "1.35", features = ["macros"] } dotenv = "0.15.0" -lastfm = "0.8.1" +lastfm = "0.10.0" log = "0.4.20" chrono = "0.4.31" toml = "0.8.8" -reqwest = { version = "0.11.22", features = ["json"] } +reqwest = { version = "0.12.4", features = ["json"] } serde_json = "1.0.108" regex = "1" git2 = "0.18.1" From cdcf27b0fa58d6f5e5950c1047e415fe0ae21b87 Mon Sep 17 00:00:00 2001 From: J-onasJones Date: Fri, 28 Jun 2024 05:00:38 +0200 Subject: [PATCH 12/16] added request logging --- Cargo.lock | 193 ++++++++++++++++++++++++++++++++++++++++++++++-- Cargo.toml | 3 +- src/iplookup.rs | 14 ++++ src/main.rs | 1 + src/server.rs | 25 ++++++- 5 files changed, 223 insertions(+), 13 deletions(-) create mode 100644 src/iplookup.rs diff --git a/Cargo.lock b/Cargo.lock index b15b2cb..3c750e2 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -201,12 +201,57 @@ dependencies = [ "typenum", ] +[[package]] +name = "darling" +version = "0.20.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "83b2eb4d90d12bdda5ed17de686c2acb4c57914f8f921b8da7e112b5a36f3fe1" +dependencies = [ + "darling_core", + "darling_macro", +] + +[[package]] +name = "darling_core" +version = "0.20.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "622687fe0bac72a04e5599029151f5796111b90f1baaa9b544d807a5e31cd120" +dependencies = [ + "fnv", + "ident_case", + "proc-macro2", + "quote", + "strsim", + "syn", +] + +[[package]] +name = "darling_macro" +version = "0.20.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "733cabb43482b1a1b53eee8583c2b9e8684d592215ea83efd305dd31bc2f0178" +dependencies = [ + "darling_core", + "quote", + "syn", +] + [[package]] name = "data-encoding" version = "2.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7e962a19be5cfc3f3bf6dd8f61eb50107f356ad6270fbb3ed41476571db78be5" +[[package]] +name = "deranged" +version = "0.3.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b42b6fa04a440b495c8b04d0e71b707c585f83cb9cb28cf8cd0d976c315e31b4" +dependencies = [ + "powerfmt", + "serde", +] + [[package]] name = "digest" version = "0.10.7" @@ -355,9 +400,9 @@ checksum = "4271d37baee1b8c7e4b708028c57d816cf9d2434acb33a549475f78c181f6253" [[package]] name = "git2" -version = "0.18.1" +version = "0.19.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fbf97ba92db08df386e10c8ede66a2a0369bd277090afd8710e19e38de9ec0cd" +checksum = "b903b73e45dc0c6c596f2d37eccece7c1c8bb6e4407b001096387c63d0d93724" dependencies = [ "bitflags 2.4.1", "libc", @@ -380,7 +425,7 @@ dependencies = [ "futures-sink", "futures-util", "http 0.2.11", - "indexmap", + "indexmap 2.1.0", "slab", "tokio", "tokio-util", @@ -399,13 +444,19 @@ dependencies = [ "futures-sink", "futures-util", "http 1.1.0", - "indexmap", + "indexmap 2.1.0", "slab", "tokio", "tokio-util", "tracing", ] +[[package]] +name = "hashbrown" +version = "0.12.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8a9ee70c43aaf417c914396645a0fa852624801b24ebb7ae78fe8272889ac888" + [[package]] name = "hashbrown" version = "0.14.3" @@ -442,6 +493,12 @@ version = "0.3.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d77f7ec81a6d05a3abb01ab6eb7590f6083d08449fe5a1c8b1e620283546ccb7" +[[package]] +name = "hex" +version = "0.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7f24254aa9a54b5c858eaee2f5bccdb46aaf0e486a595ed5fd8f86ba55232a70" + [[package]] name = "http" version = "0.2.11" @@ -627,6 +684,12 @@ dependencies = [ "cc", ] +[[package]] +name = "ident_case" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b9e0384b61958566e926dc50660321d12159025e767c18e043daf26b70104c39" + [[package]] name = "idna" version = "0.5.0" @@ -637,6 +700,17 @@ dependencies = [ "unicode-normalization", ] +[[package]] +name = "indexmap" +version = "1.9.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bd070e393353796e801d209ad339e89596eb4c8d430d18ede6a1cced8fafbd99" +dependencies = [ + "autocfg", + "hashbrown 0.12.3", + "serde", +] + [[package]] name = "indexmap" version = "2.1.0" @@ -644,7 +718,20 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d530e1a18b1cb4c484e6e34556a0d948706958449fca0cab753d649f2bce3d1f" dependencies = [ "equivalent", - "hashbrown", + "hashbrown 0.14.3", + "serde", +] + +[[package]] +name = "ip2location" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7c1511093b9295abf32d5b63cfefa2c59cd237fde31c339fabf9710a54c16c69" +dependencies = [ + "memmap", + "serde", + "serde_json", + "serde_with", ] [[package]] @@ -675,6 +762,7 @@ dependencies = [ "chrono", "dotenv", "git2", + "ip2location", "lastfm", "log", "parking_lot", @@ -732,9 +820,9 @@ checksum = "89d92a4743f9a61002fae18374ed11e7973f530cb3a3255fb354818118b2203c" [[package]] name = "libgit2-sys" -version = "0.16.1+1.7.1" +version = "0.17.0+1.8.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f2a2bb3680b094add03bb3732ec520ece34da31a8cd2d633d1389d0f0fb60d0c" +checksum = "10472326a8a6477c3c20a64547b0059e4b0d086869eee31e6d7da728a8eb7224" dependencies = [ "cc", "libc", @@ -798,6 +886,16 @@ version = "2.6.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f665ee40bc4a3c5590afb1e9677db74a508659dfd71e126420da8274909a0167" +[[package]] +name = "memmap" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6585fd95e7bb50d6cc31e20d4cf9afb4e2ba16c5846fc76793f11218da9c475b" +dependencies = [ + "libc", + "winapi", +] + [[package]] name = "mime" version = "0.3.17" @@ -870,6 +968,12 @@ dependencies = [ "tempfile", ] +[[package]] +name = "num-conv" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "51d515d32fb182ee37cda2ccdcb92950d6a3c2893aa280e540671c2cd0f3b1d9" + [[package]] name = "num-traits" version = "0.2.17" @@ -1015,6 +1119,12 @@ version = "0.3.27" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "26072860ba924cbfa98ea39c8c19b4dd6a4a25423dbdf219c1eca91aa0cf6964" +[[package]] +name = "powerfmt" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "439ee305def115ba05938db6eb1644ff94165c5ab5e9420d1c1bcedbba909391" + [[package]] name = "ppv-lite86" version = "0.2.17" @@ -1381,6 +1491,36 @@ dependencies = [ "serde", ] +[[package]] +name = "serde_with" +version = "3.8.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0ad483d2ab0149d5a5ebcd9972a3852711e0153d863bf5a5d0391d28883c4a20" +dependencies = [ + "base64 0.22.1", + "chrono", + "hex", + "indexmap 1.9.3", + "indexmap 2.1.0", + "serde", + "serde_derive", + "serde_json", + "serde_with_macros", + "time", +] + +[[package]] +name = "serde_with_macros" +version = "3.8.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "65569b702f41443e8bc8bbb1c5779bd0450bbe723b56198980e80ec45780bce2" +dependencies = [ + "darling", + "proc-macro2", + "quote", + "syn", +] + [[package]] name = "sha1" version = "0.10.6" @@ -1442,6 +1582,12 @@ version = "0.9.8" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6980e8d7511241f8acf4aebddbb1ff938df5eebe98691418c4468d0b72a96a67" +[[package]] +name = "strsim" +version = "0.11.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7da8b5736845d9f2fcb837ea5d9e2628564b3b043a70948a3f0b778838c5fb4f" + [[package]] name = "syn" version = "2.0.39" @@ -1513,6 +1659,37 @@ dependencies = [ "syn", ] +[[package]] +name = "time" +version = "0.3.36" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5dfd88e563464686c916c7e46e623e520ddc6d79fa6641390f2e3fa86e83e885" +dependencies = [ + "deranged", + "itoa", + "num-conv", + "powerfmt", + "serde", + "time-core", + "time-macros", +] + +[[package]] +name = "time-core" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ef927ca75afb808a4d64dd374f00a2adf8d0fcff8e7b184af886c3c87ec4a3f3" + +[[package]] +name = "time-macros" +version = "0.2.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3f252a68540fde3a3877aeea552b832b40ab9a69e318efd078774a01ddee1ccf" +dependencies = [ + "num-conv", + "time-core", +] + [[package]] name = "tinyvec" version = "1.6.0" @@ -1642,7 +1819,7 @@ version = "0.21.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d34d383cd00a163b4a5b85053df514d45bc330f6de7737edfe0a93311d1eaa03" dependencies = [ - "indexmap", + "indexmap 2.1.0", "serde", "serde_spanned", "toml_datetime", diff --git a/Cargo.toml b/Cargo.toml index 4676fb9..7e3e587 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -18,4 +18,5 @@ toml = "0.8.8" reqwest = { version = "0.12.4", features = ["json"] } serde_json = "1.0.108" regex = "1" -git2 = "0.18.1" +git2 = "0.19.0" +ip2location = "0.5.0" diff --git a/src/iplookup.rs b/src/iplookup.rs new file mode 100644 index 0000000..ddfafba --- /dev/null +++ b/src/iplookup.rs @@ -0,0 +1,14 @@ +use ip2location::{DB, Record}; + +pub fn ip_lookup(ip: &str) -> String { + let database_path = "resources/IP2LOCATION-LITE-DB5.IPV6.BIN/IP2LOCATION-LITE-DB5.IPV6.BIN"; + + let mut db = DB::from_file(database_path).unwrap(); + + let geo_info = db.ip_lookup(ip.parse().unwrap()).unwrap(); + + let record = if let Record::LocationDb(rec) = geo_info { + Some(rec) + } else { None }; + return record.unwrap().country.unwrap().short_name.to_string(); +} \ No newline at end of file diff --git a/src/main.rs b/src/main.rs index a55572c..e845d29 100644 --- a/src/main.rs +++ b/src/main.rs @@ -8,6 +8,7 @@ pub mod logger; pub mod tools; pub mod server; pub mod error_responses; +pub mod iplookup; pub use logger::Logger; pub use tools::parse_ip; diff --git a/src/server.rs b/src/server.rs index cdf5e6d..3c675a8 100644 --- a/src/server.rs +++ b/src/server.rs @@ -1,13 +1,16 @@ use std::convert::Infallible; +use std::net::SocketAddr; use std::env; -use reqwest::StatusCode; +use lastfm::reqwest::StatusCode; +use warp::filters::path::FullPath; use warp::Filter; use warp::reply::Reply; use crate::error_responses::{ErrorMessage, InternalServerError, BadRequestError, NotFoundError, NotImplementedError}; use crate::v1::get_v1_routes; use crate::{Logger, parse_ip}; +use crate::iplookup::ip_lookup; pub async fn serve() { @@ -24,9 +27,23 @@ pub async fn serve() { let status = warp::path("status") .map(|| warp::reply()); + // Middleware filter to log request details + let log_request = warp::any() + .and(warp::method()) + .and(warp::path::full()) + .and(warp::addr::remote()) + .and(warp::header::optional::("x-forwarded-for")) + .map(|method, path: FullPath, addr: Option, fwd_for: Option| { + let client_ip = fwd_for.unwrap_or_else(|| addr.map(|a| a.ip().to_string()).unwrap_or_else(|| String::from("unknown"))); + let path_str = path.as_str(); + + Logger::info(&format!(" {} {} from {}", method, path_str, ip_lookup(&client_ip))); + }); + // GET (any) => reply with return from handle_path - let routes = favicon.or(status.or(get_v1_routes()) - .recover(handle_rejection)); + let routes = log_request + .clone().untuple_one().and(favicon.or(status.or(get_v1_routes()) + .recover(handle_rejection))); async fn handle_rejection(err: warp::Rejection) -> Result { @@ -47,7 +64,7 @@ pub async fn serve() { message: message.into(), }); - Ok(warp::reply::with_status(json, code)) + Ok(warp::reply::with_status(json, StatusCode::from_u16(code.as_u16()).unwrap())) } From 350c9f722adf6e23f0d6f59dea91c4f3911bfae2 Mon Sep 17 00:00:00 2001 From: J-onasJones Date: Fri, 28 Jun 2024 05:12:47 +0200 Subject: [PATCH 13/16] added IP in brackets --- src/server.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/server.rs b/src/server.rs index 3c675a8..f658c9a 100644 --- a/src/server.rs +++ b/src/server.rs @@ -37,7 +37,7 @@ pub async fn serve() { let client_ip = fwd_for.unwrap_or_else(|| addr.map(|a| a.ip().to_string()).unwrap_or_else(|| String::from("unknown"))); let path_str = path.as_str(); - Logger::info(&format!(" {} {} from {}", method, path_str, ip_lookup(&client_ip))); + Logger::info(&format!(" {} {} from {} ({})", method, path_str, ip_lookup(&client_ip), client_ip)); }); // GET (any) => reply with return from handle_path From 2c8a9db949893f08d40532cc6e6b93eb256750f8 Mon Sep 17 00:00:00 2001 From: J-onasJones Date: Sat, 29 Jun 2024 23:27:05 +0200 Subject: [PATCH 14/16] added request logger --- Cargo.lock | 110 +++++++++++++++++++++++++++++++++++----- Cargo.toml | 3 +- src/main.rs | 1 + src/request_logger.rs | 68 +++++++++++++++++++++++++ src/server.rs | 3 +- src/v1/analytics/mod.rs | 8 +++ src/v1/auth/mod.rs | 7 +++ 7 files changed, 186 insertions(+), 14 deletions(-) create mode 100644 src/request_logger.rs create mode 100644 src/v1/analytics/mod.rs create mode 100644 src/v1/auth/mod.rs diff --git a/Cargo.lock b/Cargo.lock index 3c750e2..be3a161 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -345,6 +345,12 @@ version = "0.3.29" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "eb1d22c66e66d9d72e1758f0bd7d4fd0bee04cad842ee34587d68c07e45d088c" +[[package]] +name = "futures-io" +version = "0.3.30" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a44623e20b9681a318efdd71c299b6b222ed6f231972bfe2f224ebad6311f0c1" + [[package]] name = "futures-sink" version = "0.3.29" @@ -364,8 +370,10 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a19526d624e703a3179b3d322efec918b6246ea0fa51d41124525f00f1cc8104" dependencies = [ "futures-core", + "futures-io", "futures-sink", "futures-task", + "memchr", "pin-project-lite", "pin-utils", "slab", @@ -620,9 +628,26 @@ dependencies = [ "futures-util", "http 0.2.11", "hyper 0.14.27", - "rustls", + "rustls 0.21.10", "tokio", - "tokio-rustls", + "tokio-rustls 0.24.1", +] + +[[package]] +name = "hyper-rustls" +version = "0.27.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5ee4be2c948921a1a5320b629c4193916ed787a7f7f293fd3f7f5a6c9de74155" +dependencies = [ + "futures-util", + "http 1.1.0", + "hyper 1.3.1", + "hyper-util", + "rustls 0.23.10", + "rustls-pki-types", + "tokio", + "tokio-rustls 0.26.0", + "tower-service", ] [[package]] @@ -767,9 +792,10 @@ dependencies = [ "log", "parking_lot", "regex", - "reqwest 0.12.4", + "reqwest 0.12.5", "serde", "serde_json", + "sha2", "tokio", "toml", "warp", @@ -1232,7 +1258,7 @@ dependencies = [ "http 0.2.11", "http-body 0.4.5", "hyper 0.14.27", - "hyper-rustls", + "hyper-rustls 0.24.2", "ipnet", "js-sys", "log", @@ -1240,14 +1266,14 @@ dependencies = [ "once_cell", "percent-encoding", "pin-project-lite", - "rustls", + "rustls 0.21.10", "rustls-pemfile 1.0.4", "serde", "serde_json", "serde_urlencoded", "system-configuration", "tokio", - "tokio-rustls", + "tokio-rustls 0.24.1", "tower-service", "url", "wasm-bindgen", @@ -1259,13 +1285,14 @@ dependencies = [ [[package]] name = "reqwest" -version = "0.12.4" +version = "0.12.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "566cafdd92868e0939d3fb961bd0dc25fcfaaed179291093b3d43e6b3150ea10" +checksum = "c7d6d2a27d57148378eb5e111173f4276ad26340ecc5c49a4a2152167a2d6a37" dependencies = [ "base64 0.22.1", "bytes", "encoding_rs", + "futures-channel", "futures-core", "futures-util", "h2 0.4.4", @@ -1273,6 +1300,7 @@ dependencies = [ "http-body 1.0.0", "http-body-util", "hyper 1.3.1", + "hyper-rustls 0.27.2", "hyper-tls", "hyper-util", "ipnet", @@ -1340,10 +1368,23 @@ checksum = "f9d5a6813c0759e4609cd494e8e725babae6a2ca7b62a5536a13daaec6fcb7ba" dependencies = [ "log", "ring", - "rustls-webpki", + "rustls-webpki 0.101.7", "sct", ] +[[package]] +name = "rustls" +version = "0.23.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "05cff451f60db80f490f3c182b77c35260baace73209e9cdbbe526bfe3a4d402" +dependencies = [ + "once_cell", + "rustls-pki-types", + "rustls-webpki 0.102.4", + "subtle", + "zeroize", +] + [[package]] name = "rustls-pemfile" version = "1.0.4" @@ -1379,6 +1420,17 @@ dependencies = [ "untrusted", ] +[[package]] +name = "rustls-webpki" +version = "0.102.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ff448f7e92e913c4b7d4c6d8e4540a1724b319b4152b8aef6d4cf8339712b33e" +dependencies = [ + "ring", + "rustls-pki-types", + "untrusted", +] + [[package]] name = "ryu" version = "1.0.15" @@ -1532,6 +1584,17 @@ dependencies = [ "digest", ] +[[package]] +name = "sha2" +version = "0.10.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "793db75ad2bcafc3ffa7c68b215fee268f537982cd901d132f89c6343f3a3dc8" +dependencies = [ + "cfg-if", + "cpufeatures", + "digest", +] + [[package]] name = "signal-hook-registry" version = "1.4.1" @@ -1588,6 +1651,12 @@ version = "0.11.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7da8b5736845d9f2fcb837ea5d9e2628564b3b043a70948a3f0b778838c5fb4f" +[[package]] +name = "subtle" +version = "2.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "13c2bddecc57b384dee18652358fb23172facb8a2c51ccc10d74c157bdea3292" + [[package]] name = "syn" version = "2.0.39" @@ -1601,9 +1670,9 @@ dependencies = [ [[package]] name = "sync_wrapper" -version = "0.1.2" +version = "1.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2047c6ded9c721764247e62cd3b03c09ffc529b2ba5b10ec482ae507a4a70160" +checksum = "a7065abeca94b6a8a577f9bd45aa0867a2238b74e8eb67cf10d492bc39351394" [[package]] name = "system-configuration" @@ -1751,7 +1820,18 @@ version = "0.24.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c28327cf380ac148141087fbfb9de9d7bd4e84ab5d2c28fbc911d753de8a7081" dependencies = [ - "rustls", + "rustls 0.21.10", + "tokio", +] + +[[package]] +name = "tokio-rustls" +version = "0.26.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0c7bc40d0e5a97695bb96e27995cd3a08538541b0a846f65bba7a359f36700d4" +dependencies = [ + "rustls 0.23.10", + "rustls-pki-types", "tokio", ] @@ -2321,3 +2401,9 @@ dependencies = [ "cfg-if", "windows-sys 0.48.0", ] + +[[package]] +name = "zeroize" +version = "1.8.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ced3678a2879b30306d323f4542626697a464a97c0a07c9aebf7ebca65cd4dde" diff --git a/Cargo.toml b/Cargo.toml index 7e3e587..17ea181 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -15,8 +15,9 @@ lastfm = "0.10.0" log = "0.4.20" chrono = "0.4.31" toml = "0.8.8" -reqwest = { version = "0.12.4", features = ["json"] } +reqwest = { version = "0.12.5", features = ["json", "blocking"] } serde_json = "1.0.108" regex = "1" git2 = "0.19.0" ip2location = "0.5.0" +sha2 = "0.10.8" diff --git a/src/main.rs b/src/main.rs index e845d29..abc4dd1 100644 --- a/src/main.rs +++ b/src/main.rs @@ -9,6 +9,7 @@ pub mod tools; pub mod server; pub mod error_responses; pub mod iplookup; +pub mod request_logger; pub use logger::Logger; pub use tools::parse_ip; diff --git a/src/request_logger.rs b/src/request_logger.rs new file mode 100644 index 0000000..e559389 --- /dev/null +++ b/src/request_logger.rs @@ -0,0 +1,68 @@ +use chrono::Utc; +use reqwest::blocking::get; +use serde::{Deserialize, Serialize}; +use serde_json::Value; +use sha2::{Digest, Sha256}; +use std::fs::OpenOptions; +use std::io::{BufReader, BufWriter}; +use std::path::Path; + +#[derive(Serialize, Deserialize)] +struct RequestLog { + timestamp: String, + method: String, + pathname: String, + ip_country_code: String, + ip_hash: String, +} + +fn get_ip_country_code(ip: &str) -> Result> { + let url = format!("http://ip-api.com/json/{}", ip); + let response: Value = get(&url)?.json()?; + if let Some(country_code) = response["countryCode"].as_str() { + Ok(country_code.to_string()) + } else { + Err("Could not fetch country code".into()) + } +} + +fn hash_ip(ip: &str) -> String { + let mut hasher = Sha256::new(); + hasher.update(ip); + format!("{:x}", hasher.finalize()) +} + +pub fn log_request(ip: &str, pathname: &str, method: &str, file_path: &str) /*-> Result<(), Box>*/ { + let timestamp = Utc::now().to_rfc3339(); + let ip_country_code = get_ip_country_code(ip)?; + let ip_hash = hash_ip(ip); + + let log_entry = RequestLog { + timestamp, + method: method.to_string(), + pathname: pathname.to_string(), + ip_country_code, + ip_hash, + }; + + let path = Path::new(file_path); + let file = OpenOptions::new() + .read(true) + .write(true) + .create(true) + .open(path)?; + + let mut logs: Vec = if path.exists() { + let reader = BufReader::new(&file); + serde_json::from_reader(reader)? + } else { + Vec::new() + }; + + logs.push(log_entry); + + let writer = BufWriter::new(&file); + serde_json::to_writer_pretty(writer, &logs)?; + + //Ok(()) +} \ No newline at end of file diff --git a/src/server.rs b/src/server.rs index f658c9a..90bb641 100644 --- a/src/server.rs +++ b/src/server.rs @@ -9,7 +9,7 @@ use warp::reply::Reply; use crate::error_responses::{ErrorMessage, InternalServerError, BadRequestError, NotFoundError, NotImplementedError}; use crate::v1::get_v1_routes; -use crate::{Logger, parse_ip}; +use crate::{parse_ip, request_logger, Logger}; use crate::iplookup::ip_lookup; @@ -37,6 +37,7 @@ pub async fn serve() { let client_ip = fwd_for.unwrap_or_else(|| addr.map(|a| a.ip().to_string()).unwrap_or_else(|| String::from("unknown"))); let path_str = path.as_str(); + request_logger::log_request(&client_ip, path_str, method, "requests.json"); Logger::info(&format!(" {} {} from {} ({})", method, path_str, ip_lookup(&client_ip), client_ip)); }); diff --git a/src/v1/analytics/mod.rs b/src/v1/analytics/mod.rs new file mode 100644 index 0000000..98bbf68 --- /dev/null +++ b/src/v1/analytics/mod.rs @@ -0,0 +1,8 @@ +use warp::Filter; + +pub fn get_analytics_routes() -> impl warp::Filter + Clone { + warp::path("v1").and(warp::path("analytics")) + .and((warp::path("logcdnrequest").map(|| "Please refer to the wiki at https://wiki.jonasjones.dev/Api/")) + .or(warp::path("ping").map(|| "pong")) + .or(warp::path("version").map(|| warp::reply::json(&[option_env!("CARGO_PKG_VERSION").unwrap_or("unknown")])))) +} \ No newline at end of file diff --git a/src/v1/auth/mod.rs b/src/v1/auth/mod.rs new file mode 100644 index 0000000..0a37d4f --- /dev/null +++ b/src/v1/auth/mod.rs @@ -0,0 +1,7 @@ +use warp::Filter; + +pub fn get_auth_routes() -> impl warp::Filter + Clone { + warp::path("v1").and(warp::path("auth")) + .and((warp::path("requestsession").and(warp)) + .or(warp::path("login"))) +} \ No newline at end of file From 9ef64b5b82eaa7c62cd24a68f5f78da14d4289d6 Mon Sep 17 00:00:00 2001 From: Jonas_Jones Date: Sun, 30 Jun 2024 00:34:55 +0200 Subject: [PATCH 15/16] added more request logging code --- src/request_logger.rs | 17 ++++++++++------- src/server.rs | 21 +++++++++++++++++---- 2 files changed, 27 insertions(+), 11 deletions(-) diff --git a/src/request_logger.rs b/src/request_logger.rs index e559389..e742d4a 100644 --- a/src/request_logger.rs +++ b/src/request_logger.rs @@ -1,10 +1,12 @@ use chrono::Utc; +use reqwest::Client; use reqwest::blocking::get; use serde::{Deserialize, Serialize}; use serde_json::Value; use sha2::{Digest, Sha256}; use std::fs::OpenOptions; use std::io::{BufReader, BufWriter}; +use warp::http::Method; use std::path::Path; #[derive(Serialize, Deserialize)] @@ -16,7 +18,7 @@ struct RequestLog { ip_hash: String, } -fn get_ip_country_code(ip: &str) -> Result> { +async fn get_ip_country_code(ip: &str) -> Result> { let url = format!("http://ip-api.com/json/{}", ip); let response: Value = get(&url)?.json()?; if let Some(country_code) = response["countryCode"].as_str() { @@ -26,20 +28,21 @@ fn get_ip_country_code(ip: &str) -> Result> { } } -fn hash_ip(ip: &str) -> String { +async fn hash_ip(ip: &str) -> String { let mut hasher = Sha256::new(); hasher.update(ip); format!("{:x}", hasher.finalize()) } -pub fn log_request(ip: &str, pathname: &str, method: &str, file_path: &str) /*-> Result<(), Box>*/ { +pub async fn log_request(client: &Client, ip: &str, pathname: &str, method: &Method, file_path: &str) -> Result<(), Box> { + let method_str = method.as_str(); let timestamp = Utc::now().to_rfc3339(); - let ip_country_code = get_ip_country_code(ip)?; - let ip_hash = hash_ip(ip); + let ip_country_code = get_ip_country_code(ip).await?; + let ip_hash = hash_ip(ip).await; let log_entry = RequestLog { timestamp, - method: method.to_string(), + method: method_str.to_string(), pathname: pathname.to_string(), ip_country_code, ip_hash, @@ -64,5 +67,5 @@ pub fn log_request(ip: &str, pathname: &str, method: &str, file_path: &str) /*-> let writer = BufWriter::new(&file); serde_json::to_writer_pretty(writer, &logs)?; - //Ok(()) + Ok(()) } \ No newline at end of file diff --git a/src/server.rs b/src/server.rs index 90bb641..51ec63b 100644 --- a/src/server.rs +++ b/src/server.rs @@ -4,9 +4,12 @@ use std::env; use lastfm::reqwest::StatusCode; use warp::filters::path::FullPath; +use warp::http::Method; use warp::Filter; use warp::reply::Reply; +use reqwest::Client; + use crate::error_responses::{ErrorMessage, InternalServerError, BadRequestError, NotFoundError, NotImplementedError}; use crate::v1::get_v1_routes; use crate::{parse_ip, request_logger, Logger}; @@ -33,12 +36,22 @@ pub async fn serve() { .and(warp::path::full()) .and(warp::addr::remote()) .and(warp::header::optional::("x-forwarded-for")) - .map(|method, path: FullPath, addr: Option, fwd_for: Option| { + .map(|method: Method, path: FullPath, addr: Option, fwd_for: Option| { let client_ip = fwd_for.unwrap_or_else(|| addr.map(|a| a.ip().to_string()).unwrap_or_else(|| String::from("unknown"))); - let path_str = path.as_str(); + let path_str = path.as_str().to_string(); // Convert to owned String + let method_clone = method.clone(); + let method_str = method_clone.clone().as_str().to_string(); + let client_ip_clone = client_ip.clone(); // Clone for use outside the async block + let path_str_clone = path_str.clone(); - request_logger::log_request(&client_ip, path_str, method, "requests.json"); - Logger::info(&format!(" {} {} from {} ({})", method, path_str, ip_lookup(&client_ip), client_ip)); + /*tokio::spawn(async move { + let client = Client::new(); + if let Err(e) = request_logger::log_request(&client, &client_ip, &path_str, &method_clone, "requests.json").await { + eprintln!("Failed to log request: {:?}", e); + } + });*/ + + Logger::info(&format!("{} {} from {} ({})", method_str, path_str_clone, ip_lookup(&client_ip_clone), client_ip_clone)); }); // GET (any) => reply with return from handle_path From c25f2032d7b49b2ea6c0d16af4707295ef529d4e Mon Sep 17 00:00:00 2001 From: Jonas_Jones Date: Wed, 2 Apr 2025 23:18:53 +0200 Subject: [PATCH 16/16] migrated to new version of sync script --- src/main.rs | 6 ++---- src/v1/mod.rs | 4 +--- src/v1/run/mod.rs | 33 +++++++++++++++++++++++++++++++++ 3 files changed, 36 insertions(+), 7 deletions(-) diff --git a/src/main.rs b/src/main.rs index abc4dd1..e8a62ef 100644 --- a/src/main.rs +++ b/src/main.rs @@ -1,6 +1,6 @@ use dotenv::dotenv; use tokio::time::{sleep, Duration}; -use v1::{run_kcomebacks_command, run_likedsongs_command, run_projects_command}; +use v1::{run_sync_all_command}; use std::fs; pub mod v1; @@ -19,9 +19,7 @@ async fn periodic_script_runner() { loop { Logger::info("Running periodic scripts..."); // Run all Functions - let _ = run_kcomebacks_command(); - let _ = run_projects_command(); - let _ = run_likedsongs_command(); + let _ = run_sync_all_command(); // Sleep for 6 hours sleep(Duration::from_secs(6 * 60 * 60)).await; diff --git a/src/v1/mod.rs b/src/v1/mod.rs index ff4bf8f..199d74d 100644 --- a/src/v1/mod.rs +++ b/src/v1/mod.rs @@ -5,9 +5,7 @@ mod projects; mod run; pub use run::setup as run_setup; -pub use run::run_kcomebacks_command; -pub use run::run_projects_command; -pub use run::run_likedsongs_command; +pub use run::run_sync_all_command; pub use builtin::get_builtin_routes as get_v1_builtin_routes; pub use debug::get_debug_routes as get_v1_debug_routes; diff --git a/src/v1/run/mod.rs b/src/v1/run/mod.rs index 9861d53..4402b6b 100644 --- a/src/v1/run/mod.rs +++ b/src/v1/run/mod.rs @@ -26,6 +26,7 @@ pub fn get_run_routes() -> impl warp::Filter Result { } +async fn sync_all() -> Result { + // check if the local repository exists, if not, clone it + if !fs::metadata("./resources/turbo_octo_potato").is_ok() { + setup().unwrap(); + }; + + if let Err(err) = run_sync_all_command() { + // Handle the error here + eprintln!("Error: {}", err); + // Return an appropriate response or error + return Err(warp::reject::custom(InternalServerError)); + } + + Ok(warp::reply::json(&json!({"status": "syncing..."}))) + +} + pub fn setup() -> Result<(), git2::Error> { let repository_url = "https://github.com/JonasunderscoreJones/turbo-octo-potato.git"; let local_directory = "resources/turbo_octo_potato"; @@ -247,5 +265,20 @@ pub fn run_likedsongs_command() -> Result<(), std::io::Error> { // }); Logger::info("Syncing liked songs..."); + Ok(()) +} + +pub fn run_sync_all_command() -> Result<(), std::io::Error> { + task::spawn_blocking(move || { + let mut child = Command::new("python3") + .arg("script_interval_runner.py") + .current_dir("resources/turbo_octo_potato") + .stdout(Stdio::piped()) + .spawn() + .expect("failed to execute child"); + child.wait().unwrap(); + }); + Logger::info("Running all Sync Scripts..."); + Ok(()) } \ No newline at end of file