added more request logging code

Cargo.lock

@@ -345,6 +345,12 @@ version = "0.3.29"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "eb1d22c66e66d9d72e1758f0bd7d4fd0bee04cad842ee34587d68c07e45d088c"
 
+[[package]]
+name = "futures-io"
+version = "0.3.30"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "a44623e20b9681a318efdd71c299b6b222ed6f231972bfe2f224ebad6311f0c1"
+
 [[package]]
 name = "futures-sink"
 version = "0.3.29"
@@ -364,8 +370,10 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "a19526d624e703a3179b3d322efec918b6246ea0fa51d41124525f00f1cc8104"
 dependencies = [
  "futures-core",
+ "futures-io",
  "futures-sink",
  "futures-task",
+ "memchr",
  "pin-project-lite",
  "pin-utils",
  "slab",
@@ -620,9 +628,26 @@ dependencies = [
  "futures-util",
  "http 0.2.11",
  "hyper 0.14.27",
- "rustls",
+ "rustls 0.21.10",
  "tokio",
- "tokio-rustls",
+ "tokio-rustls 0.24.1",
+]
+
+[[package]]
+name = "hyper-rustls"
+version = "0.27.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "5ee4be2c948921a1a5320b629c4193916ed787a7f7f293fd3f7f5a6c9de74155"
+dependencies = [
+ "futures-util",
+ "http 1.1.0",
+ "hyper 1.3.1",
+ "hyper-util",
+ "rustls 0.23.10",
+ "rustls-pki-types",
+ "tokio",
+ "tokio-rustls 0.26.0",
+ "tower-service",
 ]
 
 [[package]]
@@ -767,9 +792,10 @@ dependencies = [
  "log",
  "parking_lot",
  "regex",
- "reqwest 0.12.4",
+ "reqwest 0.12.5",
  "serde",
  "serde_json",
+ "sha2",
  "tokio",
  "toml",
  "warp",
@@ -1232,7 +1258,7 @@ dependencies = [
  "http 0.2.11",
  "http-body 0.4.5",
  "hyper 0.14.27",
- "hyper-rustls",
+ "hyper-rustls 0.24.2",
  "ipnet",
  "js-sys",
  "log",
@@ -1240,14 +1266,14 @@ dependencies = [
  "once_cell",
  "percent-encoding",
  "pin-project-lite",
- "rustls",
+ "rustls 0.21.10",
  "rustls-pemfile 1.0.4",
  "serde",
  "serde_json",
  "serde_urlencoded",
  "system-configuration",
  "tokio",
- "tokio-rustls",
+ "tokio-rustls 0.24.1",
  "tower-service",
  "url",
  "wasm-bindgen",
@@ -1259,13 +1285,14 @@ dependencies = [
 
 [[package]]
 name = "reqwest"
-version = "0.12.4"
+version = "0.12.5"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "566cafdd92868e0939d3fb961bd0dc25fcfaaed179291093b3d43e6b3150ea10"
+checksum = "c7d6d2a27d57148378eb5e111173f4276ad26340ecc5c49a4a2152167a2d6a37"
 dependencies = [
  "base64 0.22.1",
  "bytes",
  "encoding_rs",
+ "futures-channel",
  "futures-core",
  "futures-util",
  "h2 0.4.4",
@@ -1273,6 +1300,7 @@ dependencies = [
  "http-body 1.0.0",
  "http-body-util",
  "hyper 1.3.1",
+ "hyper-rustls 0.27.2",
  "hyper-tls",
  "hyper-util",
  "ipnet",
@@ -1340,10 +1368,23 @@ checksum = "f9d5a6813c0759e4609cd494e8e725babae6a2ca7b62a5536a13daaec6fcb7ba"
 dependencies = [
  "log",
  "ring",
- "rustls-webpki",
+ "rustls-webpki 0.101.7",
  "sct",
 ]
 
+[[package]]
+name = "rustls"
+version = "0.23.10"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "05cff451f60db80f490f3c182b77c35260baace73209e9cdbbe526bfe3a4d402"
+dependencies = [
+ "once_cell",
+ "rustls-pki-types",
+ "rustls-webpki 0.102.4",
+ "subtle",
+ "zeroize",
+]
+
 [[package]]
 name = "rustls-pemfile"
 version = "1.0.4"
@@ -1379,6 +1420,17 @@ dependencies = [
  "untrusted",
 ]
 
+[[package]]
+name = "rustls-webpki"
+version = "0.102.4"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "ff448f7e92e913c4b7d4c6d8e4540a1724b319b4152b8aef6d4cf8339712b33e"
+dependencies = [
+ "ring",
+ "rustls-pki-types",
+ "untrusted",
+]
+
 [[package]]
 name = "ryu"
 version = "1.0.15"
@@ -1532,6 +1584,17 @@ dependencies = [
  "digest",
 ]
 
+[[package]]
+name = "sha2"
+version = "0.10.8"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "793db75ad2bcafc3ffa7c68b215fee268f537982cd901d132f89c6343f3a3dc8"
+dependencies = [
+ "cfg-if",
+ "cpufeatures",
+ "digest",
+]
+
 [[package]]
 name = "signal-hook-registry"
 version = "1.4.1"
@@ -1588,6 +1651,12 @@ version = "0.11.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "7da8b5736845d9f2fcb837ea5d9e2628564b3b043a70948a3f0b778838c5fb4f"
 
+[[package]]
+name = "subtle"
+version = "2.6.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "13c2bddecc57b384dee18652358fb23172facb8a2c51ccc10d74c157bdea3292"
+
 [[package]]
 name = "syn"
 version = "2.0.39"
@@ -1601,9 +1670,9 @@ dependencies = [
 
 [[package]]
 name = "sync_wrapper"
-version = "0.1.2"
+version = "1.0.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "2047c6ded9c721764247e62cd3b03c09ffc529b2ba5b10ec482ae507a4a70160"
+checksum = "a7065abeca94b6a8a577f9bd45aa0867a2238b74e8eb67cf10d492bc39351394"
 
 [[package]]
 name = "system-configuration"
@@ -1751,7 +1820,18 @@ version = "0.24.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "c28327cf380ac148141087fbfb9de9d7bd4e84ab5d2c28fbc911d753de8a7081"
 dependencies = [
- "rustls",
+ "rustls 0.21.10",
+ "tokio",
+]
+
+[[package]]
+name = "tokio-rustls"
+version = "0.26.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "0c7bc40d0e5a97695bb96e27995cd3a08538541b0a846f65bba7a359f36700d4"
+dependencies = [
+ "rustls 0.23.10",
+ "rustls-pki-types",
  "tokio",
 ]
 
@@ -2321,3 +2401,9 @@ dependencies = [
  "cfg-if",
  "windows-sys 0.48.0",
 ]
+
+[[package]]
+name = "zeroize"
+version = "1.8.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "ced3678a2879b30306d323f4542626697a464a97c0a07c9aebf7ebca65cd4dde"

Cargo.toml

@@ -15,8 +15,9 @@ lastfm = "0.10.0"
 log = "0.4.20"
 chrono = "0.4.31"
 toml = "0.8.8"
-reqwest = { version = "0.12.4", features = ["json"] }
+reqwest = { version = "0.12.5", features = ["json", "blocking"] }
 serde_json = "1.0.108"
 regex = "1"
 git2 = "0.19.0"
 ip2location = "0.5.0"
+sha2 = "0.10.8"

README.md

@@ -56,3 +56,12 @@ run container:
 ```sh
 docker-compose up -d
 ```
+
+## Roadmap
+
+- analytics backend. track request origin through IP from header (store IP hash, region and time)
+- rewrite all scripts in rust
+- DB implementation for projects, kcomebacks, minecraft mod versions
+- session backend, auth token system
+- implementation for dashboard front-end with analytics/config
+- complete minecraft mod implementation

src/main.rs

@@ -9,6 +9,7 @@ pub mod tools;
 pub mod server;
 pub mod error_responses;
 pub mod iplookup;
+pub mod request_logger;
 
 pub use logger::Logger;
 pub use tools::parse_ip;

src/request_logger.rs

@@ -0,0 +1,71 @@
+use chrono::Utc;
+use reqwest::Client;
+use reqwest::blocking::get;
+use serde::{Deserialize, Serialize};
+use serde_json::Value;
+use sha2::{Digest, Sha256};
+use std::fs::OpenOptions;
+use std::io::{BufReader, BufWriter};
+use warp::http::Method;
+use std::path::Path;
+
+#[derive(Serialize, Deserialize)]
+struct RequestLog {
+    timestamp: String,
+    method: String,
+    pathname: String,
+    ip_country_code: String,
+    ip_hash: String,
+}
+
+async fn get_ip_country_code(ip: &str) -> Result<String, Box<dyn std::error::Error>> {
+    let url = format!("http://ip-api.com/json/{}", ip);
+    let response: Value = get(&url)?.json()?;
+    if let Some(country_code) = response["countryCode"].as_str() {
+        Ok(country_code.to_string())
+    } else {
+        Err("Could not fetch country code".into())
+    }
+}
+
+async fn hash_ip(ip: &str) -> String {
+    let mut hasher = Sha256::new();
+    hasher.update(ip);
+    format!("{:x}", hasher.finalize())
+}
+
+pub async fn log_request(client: &Client, ip: &str, pathname: &str, method: &Method, file_path: &str) -> Result<(), Box<dyn std::error::Error>> {
+    let method_str = method.as_str();
+    let timestamp = Utc::now().to_rfc3339();
+    let ip_country_code = get_ip_country_code(ip).await?;
+    let ip_hash = hash_ip(ip).await;
+
+    let log_entry = RequestLog {
+        timestamp,
+        method: method_str.to_string(),
+        pathname: pathname.to_string(),
+        ip_country_code,
+        ip_hash,
+    };
+
+    let path = Path::new(file_path);
+    let file = OpenOptions::new()
+        .read(true)
+        .write(true)
+        .create(true)
+        .open(path)?;
+
+    let mut logs: Vec<RequestLog> = if path.exists() {
+        let reader = BufReader::new(&file);
+        serde_json::from_reader(reader)?
+    } else {
+        Vec::new()
+    };
+
+    logs.push(log_entry);
+
+    let writer = BufWriter::new(&file);
+    serde_json::to_writer_pretty(writer, &logs)?;
+
+    Ok(())
+}

src/server.rs

@@ -4,12 +4,15 @@ use std::env;
 
 use lastfm::reqwest::StatusCode;
 use warp::filters::path::FullPath;
+use warp::http::Method;
 use warp::Filter;
 use warp::reply::Reply;
 
+use reqwest::Client;
+
 use crate::error_responses::{ErrorMessage, InternalServerError, BadRequestError, NotFoundError, NotImplementedError};
 use crate::v1::get_v1_routes;
-use crate::{Logger, parse_ip};
+use crate::{parse_ip, request_logger, Logger};
 use crate::iplookup::ip_lookup;
 
 
@@ -33,11 +36,22 @@ pub async fn serve() {
         .and(warp::path::full())
         .and(warp::addr::remote())
         .and(warp::header::optional::<String>("x-forwarded-for"))
-        .map(|method, path: FullPath, addr: Option<SocketAddr>, fwd_for: Option<String>| {
+        .map(|method: Method, path: FullPath, addr: Option<SocketAddr>, fwd_for: Option<String>| {
             let client_ip = fwd_for.unwrap_or_else(|| addr.map(|a| a.ip().to_string()).unwrap_or_else(|| String::from("unknown")));
-            let path_str = path.as_str();
+            let path_str = path.as_str().to_string(); // Convert to owned String
+            let method_clone = method.clone();
+            let method_str = method_clone.clone().as_str().to_string();
+            let client_ip_clone = client_ip.clone(); // Clone for use outside the async block
+            let path_str_clone = path_str.clone();
 
-            Logger::info(&format!(" {} {} from {} ({})", method, path_str, ip_lookup(&client_ip), client_ip));
+            /*tokio::spawn(async move {
+                let client = Client::new();
+                if let Err(e) = request_logger::log_request(&client, &client_ip, &path_str, &method_clone, "requests.json").await {
+                    eprintln!("Failed to log request: {:?}", e);
+                }
+            });*/
+
+            Logger::info(&format!("{} {} from {} ({})", method_str, path_str_clone, ip_lookup(&client_ip_clone), client_ip_clone));
         });
 
     // GET (any) => reply with return from handle_path

src/v1/analytics/mod.rs

@@ -0,0 +1,8 @@
+use warp::Filter;
+
+pub fn get_analytics_routes() -> impl warp::Filter<Extract = impl warp::Reply, Error = warp::Rejection> + Clone {
+    warp::path("v1").and(warp::path("analytics"))
+        .and((warp::path("logcdnrequest").map(|| "Please refer to the wiki at https://wiki.jonasjones.dev/Api/"))
+        .or(warp::path("ping").map(|| "pong"))
+        .or(warp::path("version").map(|| warp::reply::json(&[option_env!("CARGO_PKG_VERSION").unwrap_or("unknown")]))))
+}

src/v1/auth/mod.rs

@@ -0,0 +1,7 @@
+use warp::Filter;
+
+pub fn get_auth_routes() -> impl warp::Filter<Extract = impl warp::Reply, Error = warp::Rejection> + Clone {
+    warp::path("v1").and(warp::path("auth"))
+        .and((warp::path("requestsession").and(warp))
+        .or(warp::path("login")))
+}