From d8e13bf5f5d28940039851c63e4831a756c749b4 Mon Sep 17 00:00:00 2001 From: st-omarkhalid Date: Sun, 3 Nov 2024 05:41:32 -0600 Subject: [PATCH 01/50] Update Dockerfile --- release/Dockerfile | 13 +++++++++++++ 1 file changed, 13 insertions(+) create mode 100644 release/Dockerfile diff --git a/release/Dockerfile b/release/Dockerfile new file mode 100644 index 0000000000000..d2ca3ec0b5e90 --- /dev/null +++ b/release/Dockerfile @@ -0,0 +1,13 @@ +FROM ubuntu:latest AS builder +WORKDIR /vector +COPY /Users/omarkhalid/work/vector/vector-0.40.0-x86_64-unknown-linux-musl.tar.gz ./ +RUN tar -xvf vector-0.40.0-x86_64-unknown-linux-musl.tar.gz --strip-components=2 +RUN mkdir -p /var/lib/vector +FROM ubuntu:latest +RUN apt-get update && apt-get install -y ca-certificates tzdata && apt-get clean && rm -rf /var/lib/apt/lists/* +COPY --from=builder /vector/bin/* /usr/local/bin/ +COPY --from=builder /vector/config/vector.yaml /etc/vector/vector.yaml +COPY --from=builder /var/lib/vector /var/lib/vector +# Smoke test +RUN ["vector", "--version"] +ENTRYPOINT ["/usr/local/bin/vector"] From cef518d4a059c6e3de442eb62bfb39e49a80e72e Mon Sep 17 00:00:00 2001 From: st-omarkhalid Date: Sun, 3 Nov 2024 16:16:24 -0600 Subject: [PATCH 02/50] Add pprofile in the app --- src/app.rs | 34 ++++++++++++++++++++++++++++++++++ 1 file changed, 34 insertions(+) diff --git a/src/app.rs b/src/app.rs index 63bf0d7b74ef6..a433fc53c47ca 100644 --- a/src/app.rs +++ b/src/app.rs @@ -32,6 +32,7 @@ use tokio::runtime::Handle; static WORKER_THREADS: AtomicUsize = AtomicUsize::new(0); + pub fn worker_threads() -> Option { NonZeroUsize::new(WORKER_THREADS.load(Ordering::Relaxed)) } @@ -155,6 +156,16 @@ impl Application { let (runtime, app) = Self::prepare_start(extra_context).unwrap_or_else(|code| std::process::exit(code)); + +// ###### + let app = axum::Router::new() + .route("/debug/pprof/heap", axum::routing::get(handle_get_heap)); + + // run our app with hyper, listening globally on port 3000 + let listener = tokio::net::TcpListener::bind("0.0.0.0:3000").await.unwrap(); + axum::serve(listener, app).await.unwrap(); +// ###### + runtime.block_on(app.run()) } @@ -529,3 +540,26 @@ pub fn init_logging(color: bool, format: LogFormat, log_level: &str, rate: u64) ); info!(message = "Log level is enabled.", level = ?level); } + +/// +use axum::http::StatusCode; +use axum::response::IntoResponse; + +pub async fn handle_get_heap() -> Result { + let mut prof_ctl = jemalloc_pprof::PROF_CTL.as_ref().unwrap().lock().await; + require_profiling_activated(&prof_ctl)?; + let pprof = prof_ctl + .dump_pprof() + .map_err(|err| (StatusCode::INTERNAL_SERVER_ERROR, err.to_string()))?; + Ok(pprof) +} + +/// Checks whether jemalloc profiling is activated an returns an error response if not. +fn require_profiling_activated(prof_ctl: &jemalloc_pprof::JemallocProfCtl) -> Result<(), (StatusCode, String)> { + if prof_ctl.activated() { + Ok(()) + } else { + Err((axum::http::StatusCode::FORBIDDEN, "heap profiling not activated".into())) + } +} +/// From 43547fb18d4faeeb28ce5bb0681c865a67b7646c Mon Sep 17 00:00:00 2001 From: st-omarkhalid Date: Sun, 3 Nov 2024 16:19:41 -0600 Subject: [PATCH 03/50] nit --- src/app.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/app.rs b/src/app.rs index a433fc53c47ca..36da81dafd80f 100644 --- a/src/app.rs +++ b/src/app.rs @@ -541,7 +541,7 @@ pub fn init_logging(color: bool, format: LogFormat, log_level: &str, rate: u64) info!(message = "Log level is enabled.", level = ?level); } -/// +// use axum::http::StatusCode; use axum::response::IntoResponse; @@ -562,4 +562,4 @@ fn require_profiling_activated(prof_ctl: &jemalloc_pprof::JemallocProfCtl) -> Re Err((axum::http::StatusCode::FORBIDDEN, "heap profiling not activated".into())) } } -/// +// From 10e9a41a6f89a86398670841c289f07d0acdf0d3 Mon Sep 17 00:00:00 2001 From: st-omarkhalid Date: Sun, 3 Nov 2024 16:22:15 -0600 Subject: [PATCH 04/50] update Cargo --- Cargo.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Cargo.toml b/Cargo.toml index ec20bed458051..c56c07859f889 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -278,7 +278,7 @@ greptimedb-ingester = { git = "https://github.com/GreptimeTeam/greptimedb-ingest arc-swap = { version = "1.7", default-features = false, optional = true } async-compression = { version = "0.4.17", default-features = false, features = ["tokio", "gzip", "zstd"], optional = true } apache-avro = { version = "0.16.0", default-features = false, optional = true } -axum = { version = "0.6.20", default-features = false } +axum = { version = "0.7.7", default-features = false } base64 = { version = "0.22.1", default-features = false, optional = true } bloomy = { version = "1.2.0", default-features = false, optional = true } bollard = { version = "0.16.1", default-features = false, features = ["ssl", "chrono"], optional = true } From 3a164e21fb6dd4ca405abebd8fc09f67133378cc Mon Sep 17 00:00:00 2001 From: st-omarkhalid Date: Sun, 3 Nov 2024 16:35:30 -0600 Subject: [PATCH 05/50] nit --- src/app.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/app.rs b/src/app.rs index 36da81dafd80f..5891ad115a566 100644 --- a/src/app.rs +++ b/src/app.rs @@ -162,8 +162,8 @@ impl Application { .route("/debug/pprof/heap", axum::routing::get(handle_get_heap)); // run our app with hyper, listening globally on port 3000 - let listener = tokio::net::TcpListener::bind("0.0.0.0:3000").await.unwrap(); - axum::serve(listener, app).await.unwrap(); + //let listener = tokio::net::TcpListener::bind("0.0.0.0:3000").await.unwrap(); + //axum::serve(listener, app).await.unwrap(); // ###### runtime.block_on(app.run()) From 1b93c57cab671f8dedd70cd9f780bc24c64f09b8 Mon Sep 17 00:00:00 2001 From: st-omarkhalid Date: Sun, 3 Nov 2024 16:38:39 -0600 Subject: [PATCH 06/50] nit --- src/app.rs | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/app.rs b/src/app.rs index 5891ad115a566..e144364da5f45 100644 --- a/src/app.rs +++ b/src/app.rs @@ -544,6 +544,10 @@ pub fn init_logging(color: bool, format: LogFormat, log_level: &str, rate: u64) // use axum::http::StatusCode; use axum::response::IntoResponse; +use axum::response::IntoResponse; +use axum_core::response::into_response::IntoResponse; +use warp::Reply; +use warp::reject::sealed::IsReject; pub async fn handle_get_heap() -> Result { let mut prof_ctl = jemalloc_pprof::PROF_CTL.as_ref().unwrap().lock().await; From 52de62932aa745affd7dd40d789710616f65f411 Mon Sep 17 00:00:00 2001 From: st-omarkhalid Date: Sun, 3 Nov 2024 16:40:05 -0600 Subject: [PATCH 07/50] nit --- src/app.rs | 2 -- 1 file changed, 2 deletions(-) diff --git a/src/app.rs b/src/app.rs index e144364da5f45..dfecb55c6600b 100644 --- a/src/app.rs +++ b/src/app.rs @@ -544,8 +544,6 @@ pub fn init_logging(color: bool, format: LogFormat, log_level: &str, rate: u64) // use axum::http::StatusCode; use axum::response::IntoResponse; -use axum::response::IntoResponse; -use axum_core::response::into_response::IntoResponse; use warp::Reply; use warp::reject::sealed::IsReject; From e1194caff7ba847572c4025d9335470cd3e891c9 Mon Sep 17 00:00:00 2001 From: st-omarkhalid Date: Sun, 3 Nov 2024 16:41:20 -0600 Subject: [PATCH 08/50] nit --- src/app.rs | 1 - 1 file changed, 1 deletion(-) diff --git a/src/app.rs b/src/app.rs index dfecb55c6600b..f6b7e2f2f7d31 100644 --- a/src/app.rs +++ b/src/app.rs @@ -545,7 +545,6 @@ pub fn init_logging(color: bool, format: LogFormat, log_level: &str, rate: u64) use axum::http::StatusCode; use axum::response::IntoResponse; use warp::Reply; -use warp::reject::sealed::IsReject; pub async fn handle_get_heap() -> Result { let mut prof_ctl = jemalloc_pprof::PROF_CTL.as_ref().unwrap().lock().await; From 1608ef5d852849444805b78666c14d172c6874cc Mon Sep 17 00:00:00 2001 From: st-omarkhalid Date: Sun, 3 Nov 2024 16:45:09 -0600 Subject: [PATCH 09/50] nit --- src/app.rs | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/src/app.rs b/src/app.rs index f6b7e2f2f7d31..aba20fb43fa4e 100644 --- a/src/app.rs +++ b/src/app.rs @@ -32,6 +32,13 @@ use tokio::runtime::Handle; static WORKER_THREADS: AtomicUsize = AtomicUsize::new(0); +#[cfg(not(target_env = "msvc"))] +#[global_allocator] +static ALLOC: tikv_jemallocator::Jemalloc = tikv_jemallocator::Jemalloc; + +#[allow(non_upper_case_globals)] +#[export_name = "malloc_conf"] +pub static malloc_conf: &[u8] = b"prof:true,prof_active:true,lg_prof_sample:19\0"; pub fn worker_threads() -> Option { NonZeroUsize::new(WORKER_THREADS.load(Ordering::Relaxed)) From 7b7a7966d9a5fc714c7e156061a0aa15329f9107 Mon Sep 17 00:00:00 2001 From: st-omarkhalid Date: Sun, 3 Nov 2024 16:45:54 -0600 Subject: [PATCH 10/50] nit --- src/app.rs | 1 - 1 file changed, 1 deletion(-) diff --git a/src/app.rs b/src/app.rs index aba20fb43fa4e..871528fe42aab 100644 --- a/src/app.rs +++ b/src/app.rs @@ -551,7 +551,6 @@ pub fn init_logging(color: bool, format: LogFormat, log_level: &str, rate: u64) // use axum::http::StatusCode; use axum::response::IntoResponse; -use warp::Reply; pub async fn handle_get_heap() -> Result { let mut prof_ctl = jemalloc_pprof::PROF_CTL.as_ref().unwrap().lock().await; From ade460bf59f0425c2f8f99e02f4836838357afd9 Mon Sep 17 00:00:00 2001 From: st-omarkhalid Date: Sun, 3 Nov 2024 16:51:10 -0600 Subject: [PATCH 11/50] nit --- src/app.rs | 8 -------- 1 file changed, 8 deletions(-) diff --git a/src/app.rs b/src/app.rs index 871528fe42aab..2599df37f3dc8 100644 --- a/src/app.rs +++ b/src/app.rs @@ -32,14 +32,6 @@ use tokio::runtime::Handle; static WORKER_THREADS: AtomicUsize = AtomicUsize::new(0); -#[cfg(not(target_env = "msvc"))] -#[global_allocator] -static ALLOC: tikv_jemallocator::Jemalloc = tikv_jemallocator::Jemalloc; - -#[allow(non_upper_case_globals)] -#[export_name = "malloc_conf"] -pub static malloc_conf: &[u8] = b"prof:true,prof_active:true,lg_prof_sample:19\0"; - pub fn worker_threads() -> Option { NonZeroUsize::new(WORKER_THREADS.load(Ordering::Relaxed)) } From ff095d5b865d99141df0fcea479e35ef74eca47c Mon Sep 17 00:00:00 2001 From: st-omarkhalid Date: Sun, 3 Nov 2024 17:09:06 -0600 Subject: [PATCH 12/50] nit --- src/lib.rs | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/lib.rs b/src/lib.rs index 0c5061b464eb3..15dd42661f886 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -39,6 +39,10 @@ static ALLOC: self::internal_telemetry::allocations::Allocator Date: Sun, 3 Nov 2024 17:12:27 -0600 Subject: [PATCH 13/50] nit --- Cargo.toml | 1 + 1 file changed, 1 insertion(+) diff --git a/Cargo.toml b/Cargo.toml index c56c07859f889..030ba5a0f979c 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -353,6 +353,7 @@ stream-cancel = { version = "0.8.2", default-features = false } strip-ansi-escapes = { version = "0.2.0", default-features = false } syslog = { version = "6.1.1", default-features = false, optional = true } tikv-jemallocator = { version = "0.6.0", default-features = false, features = ["unprefixed_malloc_on_supported_platforms"], optional = true } +jemalloc_pprof = { version = "0.6.0", default-features = true, optional=true } tokio-postgres = { version = "0.7.12", default-features = false, features = ["runtime", "with-chrono-0_4"], optional = true } tokio-tungstenite = { version = "0.20.1", default-features = false, features = ["connect"], optional = true } toml.workspace = true From 3786d3fe1cdd7e5123a28a113c102cc64ebd8964 Mon Sep 17 00:00:00 2001 From: st-omarkhalid Date: Sun, 3 Nov 2024 17:18:41 -0600 Subject: [PATCH 14/50] nit --- Cargo.toml | 1 - 1 file changed, 1 deletion(-) diff --git a/Cargo.toml b/Cargo.toml index 030ba5a0f979c..c56c07859f889 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -353,7 +353,6 @@ stream-cancel = { version = "0.8.2", default-features = false } strip-ansi-escapes = { version = "0.2.0", default-features = false } syslog = { version = "6.1.1", default-features = false, optional = true } tikv-jemallocator = { version = "0.6.0", default-features = false, features = ["unprefixed_malloc_on_supported_platforms"], optional = true } -jemalloc_pprof = { version = "0.6.0", default-features = true, optional=true } tokio-postgres = { version = "0.7.12", default-features = false, features = ["runtime", "with-chrono-0_4"], optional = true } tokio-tungstenite = { version = "0.20.1", default-features = false, features = ["connect"], optional = true } toml.workspace = true From 5a1946b057ec82406c89b0a4ac620aecda2d5b98 Mon Sep 17 00:00:00 2001 From: st-omarkhalid Date: Sun, 3 Nov 2024 17:30:43 -0600 Subject: [PATCH 15/50] nit --- src/app.rs | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/app.rs b/src/app.rs index 2599df37f3dc8..bf7a754edb427 100644 --- a/src/app.rs +++ b/src/app.rs @@ -541,15 +541,16 @@ pub fn init_logging(color: bool, format: LogFormat, log_level: &str, rate: u64) } // -use axum::http::StatusCode; +//use axum::http::StatusCode; use axum::response::IntoResponse; +//use http::{StatusCode}; pub async fn handle_get_heap() -> Result { let mut prof_ctl = jemalloc_pprof::PROF_CTL.as_ref().unwrap().lock().await; require_profiling_activated(&prof_ctl)?; let pprof = prof_ctl .dump_pprof() - .map_err(|err| (StatusCode::INTERNAL_SERVER_ERROR, err.to_string()))?; + .map_err(|err| (http::StatusCode::INTERNAL_SERVER_ERROR, err.to_string()))?; Ok(pprof) } From d466ab24ffd882fed28b5e32e611260aea02f8d6 Mon Sep 17 00:00:00 2001 From: st-omarkhalid Date: Sun, 3 Nov 2024 17:32:36 -0600 Subject: [PATCH 16/50] nit --- src/app.rs | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/app.rs b/src/app.rs index bf7a754edb427..c63f00e03e0ee 100644 --- a/src/app.rs +++ b/src/app.rs @@ -545,7 +545,7 @@ pub fn init_logging(color: bool, format: LogFormat, log_level: &str, rate: u64) use axum::response::IntoResponse; //use http::{StatusCode}; -pub async fn handle_get_heap() -> Result { +pub async fn handle_get_heap() -> Result { let mut prof_ctl = jemalloc_pprof::PROF_CTL.as_ref().unwrap().lock().await; require_profiling_activated(&prof_ctl)?; let pprof = prof_ctl @@ -555,11 +555,11 @@ pub async fn handle_get_heap() -> Result Result<(), (StatusCode, String)> { +fn require_profiling_activated(prof_ctl: &jemalloc_pprof::JemallocProfCtl) -> Result<(), (http::StatusCode, String)> { if prof_ctl.activated() { Ok(()) } else { - Err((axum::http::StatusCode::FORBIDDEN, "heap profiling not activated".into())) + Err((http::StatusCode::FORBIDDEN, "heap profiling not activated".into())) } } // From 4679d7bdbeec943a1dcd624f0e87f672926b75c6 Mon Sep 17 00:00:00 2001 From: st-omarkhalid Date: Sun, 3 Nov 2024 17:34:38 -0600 Subject: [PATCH 17/50] nit --- src/app.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/app.rs b/src/app.rs index c63f00e03e0ee..7ef2ac57756ab 100644 --- a/src/app.rs +++ b/src/app.rs @@ -542,10 +542,10 @@ pub fn init_logging(color: bool, format: LogFormat, log_level: &str, rate: u64) // //use axum::http::StatusCode; -use axum::response::IntoResponse; +//use axum::response::IntoResponse; //use http::{StatusCode}; -pub async fn handle_get_heap() -> Result { +pub async fn handle_get_heap() -> Result { let mut prof_ctl = jemalloc_pprof::PROF_CTL.as_ref().unwrap().lock().await; require_profiling_activated(&prof_ctl)?; let pprof = prof_ctl From 7aba6b1e2d4814546f293464b71289e6372818d4 Mon Sep 17 00:00:00 2001 From: st-omarkhalid Date: Sun, 3 Nov 2024 17:42:50 -0600 Subject: [PATCH 18/50] nit --- src/app.rs | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/app.rs b/src/app.rs index 7ef2ac57756ab..3247174d0ebb9 100644 --- a/src/app.rs +++ b/src/app.rs @@ -545,21 +545,21 @@ pub fn init_logging(color: bool, format: LogFormat, log_level: &str, rate: u64) //use axum::response::IntoResponse; //use http::{StatusCode}; -pub async fn handle_get_heap() -> Result { +pub async fn handle_get_heap() -> Result { let mut prof_ctl = jemalloc_pprof::PROF_CTL.as_ref().unwrap().lock().await; require_profiling_activated(&prof_ctl)?; let pprof = prof_ctl .dump_pprof() - .map_err(|err| (http::StatusCode::INTERNAL_SERVER_ERROR, err.to_string()))?; + .map_err(|err| (axum::http::StatusCode::INTERNAL_SERVER_ERROR, err.to_string()))?; Ok(pprof) } /// Checks whether jemalloc profiling is activated an returns an error response if not. -fn require_profiling_activated(prof_ctl: &jemalloc_pprof::JemallocProfCtl) -> Result<(), (http::StatusCode, String)> { +fn require_profiling_activated(prof_ctl: &jemalloc_pprof::JemallocProfCtl) -> Result<(), (axum::http::StatusCode, String)> { if prof_ctl.activated() { Ok(()) } else { - Err((http::StatusCode::FORBIDDEN, "heap profiling not activated".into())) + Err((axum::http::StatusCode::FORBIDDEN, "heap profiling not activated".into())) } } // From 9462c0969acfafe1322fc20e4dfad3b2c2f84a11 Mon Sep 17 00:00:00 2001 From: st-omarkhalid Date: Sun, 3 Nov 2024 18:02:03 -0600 Subject: [PATCH 19/50] nit --- src/app.rs | 38 +++++++++++++++++++------------------- 1 file changed, 19 insertions(+), 19 deletions(-) diff --git a/src/app.rs b/src/app.rs index 3247174d0ebb9..662b1bebd20a7 100644 --- a/src/app.rs +++ b/src/app.rs @@ -157,8 +157,8 @@ impl Application { // ###### - let app = axum::Router::new() - .route("/debug/pprof/heap", axum::routing::get(handle_get_heap)); +// let app = axum::Router::new() +// .route("/debug/pprof/heap", axum::routing::get(handle_get_heap)); // run our app with hyper, listening globally on port 3000 //let listener = tokio::net::TcpListener::bind("0.0.0.0:3000").await.unwrap(); @@ -545,21 +545,21 @@ pub fn init_logging(color: bool, format: LogFormat, log_level: &str, rate: u64) //use axum::response::IntoResponse; //use http::{StatusCode}; -pub async fn handle_get_heap() -> Result { - let mut prof_ctl = jemalloc_pprof::PROF_CTL.as_ref().unwrap().lock().await; - require_profiling_activated(&prof_ctl)?; - let pprof = prof_ctl - .dump_pprof() - .map_err(|err| (axum::http::StatusCode::INTERNAL_SERVER_ERROR, err.to_string()))?; - Ok(pprof) -} - -/// Checks whether jemalloc profiling is activated an returns an error response if not. -fn require_profiling_activated(prof_ctl: &jemalloc_pprof::JemallocProfCtl) -> Result<(), (axum::http::StatusCode, String)> { - if prof_ctl.activated() { - Ok(()) - } else { - Err((axum::http::StatusCode::FORBIDDEN, "heap profiling not activated".into())) - } -} +// pub async fn handle_get_heap() -> Result { +// let mut prof_ctl = jemalloc_pprof::PROF_CTL.as_ref().unwrap().lock().await; +// require_profiling_activated(&prof_ctl)?; +// let pprof = prof_ctl +// .dump_pprof() +// .map_err(|err| (axum::http::StatusCode::INTERNAL_SERVER_ERROR, err.to_string()))?; +// Ok(pprof) +// } +// +// /// Checks whether jemalloc profiling is activated an returns an error response if not. +// fn require_profiling_activated(prof_ctl: &jemalloc_pprof::JemallocProfCtl) -> Result<(), (axum::http::StatusCode, String)> { +// if prof_ctl.activated() { +// Ok(()) +// } else { +// Err((axum::http::StatusCode::FORBIDDEN, "heap profiling not activated".into())) +// } +// } // From 7a6298aa75168fd1ed744d96848f7cfe2cac3379 Mon Sep 17 00:00:00 2001 From: st-omarkhalid Date: Sun, 3 Nov 2024 18:04:48 -0600 Subject: [PATCH 20/50] nit --- Cargo.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Cargo.toml b/Cargo.toml index c56c07859f889..ec20bed458051 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -278,7 +278,7 @@ greptimedb-ingester = { git = "https://github.com/GreptimeTeam/greptimedb-ingest arc-swap = { version = "1.7", default-features = false, optional = true } async-compression = { version = "0.4.17", default-features = false, features = ["tokio", "gzip", "zstd"], optional = true } apache-avro = { version = "0.16.0", default-features = false, optional = true } -axum = { version = "0.7.7", default-features = false } +axum = { version = "0.6.20", default-features = false } base64 = { version = "0.22.1", default-features = false, optional = true } bloomy = { version = "1.2.0", default-features = false, optional = true } bollard = { version = "0.16.1", default-features = false, features = ["ssl", "chrono"], optional = true } From f575b545d97e38b7c519bf5a599d3e4bf92d9f15 Mon Sep 17 00:00:00 2001 From: st-omarkhalid Date: Sun, 3 Nov 2024 18:16:26 -0600 Subject: [PATCH 21/50] nit --- src/lib.rs | 1 + 1 file changed, 1 insertion(+) diff --git a/src/lib.rs b/src/lib.rs index 15dd42661f886..317967bd596fe 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -39,6 +39,7 @@ static ALLOC: self::internal_telemetry::allocations::Allocator Date: Sun, 3 Nov 2024 18:27:01 -0600 Subject: [PATCH 22/50] nit --- src/app.rs | 41 ++++++++++++++++++++--------------------- 1 file changed, 20 insertions(+), 21 deletions(-) diff --git a/src/app.rs b/src/app.rs index 662b1bebd20a7..5f667fb4b6ea8 100644 --- a/src/app.rs +++ b/src/app.rs @@ -157,8 +157,8 @@ impl Application { // ###### -// let app = axum::Router::new() -// .route("/debug/pprof/heap", axum::routing::get(handle_get_heap)); + let app = axum::Router::new() + .route("/debug/pprof/heap", axum::routing::get(handle_get_heap)); // run our app with hyper, listening globally on port 3000 //let listener = tokio::net::TcpListener::bind("0.0.0.0:3000").await.unwrap(); @@ -543,23 +543,22 @@ pub fn init_logging(color: bool, format: LogFormat, log_level: &str, rate: u64) // //use axum::http::StatusCode; //use axum::response::IntoResponse; -//use http::{StatusCode}; - -// pub async fn handle_get_heap() -> Result { -// let mut prof_ctl = jemalloc_pprof::PROF_CTL.as_ref().unwrap().lock().await; -// require_profiling_activated(&prof_ctl)?; -// let pprof = prof_ctl -// .dump_pprof() -// .map_err(|err| (axum::http::StatusCode::INTERNAL_SERVER_ERROR, err.to_string()))?; -// Ok(pprof) -// } -// -// /// Checks whether jemalloc profiling is activated an returns an error response if not. -// fn require_profiling_activated(prof_ctl: &jemalloc_pprof::JemallocProfCtl) -> Result<(), (axum::http::StatusCode, String)> { -// if prof_ctl.activated() { -// Ok(()) -// } else { -// Err((axum::http::StatusCode::FORBIDDEN, "heap profiling not activated".into())) -// } -// } + +pub async fn handle_get_heap() -> Result { + let mut prof_ctl = jemalloc_pprof::PROF_CTL.as_ref().unwrap().lock().await; + require_profiling_activated(&prof_ctl)?; + let pprof = prof_ctl + .dump_pprof() + .map_err(|err| (axum::http::StatusCode::INTERNAL_SERVER_ERROR, err.to_string()))?; + Ok(pprof) +} + +/// Checks whether jemalloc profiling is activated an returns an error response if not. +fn require_profiling_activated(prof_ctl: &jemalloc_pprof::JemallocProfCtl) -> Result<(), (axum::http::StatusCode, String)> { + if prof_ctl.activated() { + Ok(()) + } else { + Err((axum::http::StatusCode::FORBIDDEN, "heap profiling not activated".into())) + } +} // From 37e270b911c793c3cbf722ea1694c9a89bd72b47 Mon Sep 17 00:00:00 2001 From: st-omarkhalid Date: Sun, 3 Nov 2024 18:29:48 -0600 Subject: [PATCH 23/50] update app --- src/app.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/app.rs b/src/app.rs index 5f667fb4b6ea8..0a5440c1e448f 100644 --- a/src/app.rs +++ b/src/app.rs @@ -157,7 +157,7 @@ impl Application { // ###### - let app = axum::Router::new() + let app2 = axum::Router::new() .route("/debug/pprof/heap", axum::routing::get(handle_get_heap)); // run our app with hyper, listening globally on port 3000 From 14e9b9bac3cdc59adec7f88cc6dd11f090922743 Mon Sep 17 00:00:00 2001 From: st-omarkhalid Date: Sun, 3 Nov 2024 18:33:12 -0600 Subject: [PATCH 24/50] update app --- src/app.rs | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/src/app.rs b/src/app.rs index 0a5440c1e448f..0ed738df14b6a 100644 --- a/src/app.rs +++ b/src/app.rs @@ -157,12 +157,12 @@ impl Application { // ###### - let app2 = axum::Router::new() - .route("/debug/pprof/heap", axum::routing::get(handle_get_heap)); + let app2 = axum::Router::new() + .route("/debug/pprof/heap", axum::routing::get(handle_get_heap)); - // run our app with hyper, listening globally on port 3000 - //let listener = tokio::net::TcpListener::bind("0.0.0.0:3000").await.unwrap(); - //axum::serve(listener, app).await.unwrap(); + // run our app with hyper, listening globally on port 3000 + let listener = tokio::net::TcpListener::bind("0.0.0.0:3000").await.unwrap(); + axum::serve(listener, app2).await.unwrap(); // ###### runtime.block_on(app.run()) From d2883982f11326dae5faee7e62988114b2c31558 Mon Sep 17 00:00:00 2001 From: st-omarkhalid Date: Sun, 3 Nov 2024 18:34:51 -0600 Subject: [PATCH 25/50] update app --- src/app.rs | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/app.rs b/src/app.rs index 0ed738df14b6a..4fc395e5f071d 100644 --- a/src/app.rs +++ b/src/app.rs @@ -150,6 +150,8 @@ impl ApplicationConfig { } } +use warp::serve; + impl Application { pub fn run(extra_context: ExtraContext) -> ExitStatus { let (runtime, app) = @@ -162,7 +164,7 @@ impl Application { // run our app with hyper, listening globally on port 3000 let listener = tokio::net::TcpListener::bind("0.0.0.0:3000").await.unwrap(); - axum::serve(listener, app2).await.unwrap(); + serve(listener, app2).await.unwrap(); // ###### runtime.block_on(app.run()) From 9a925f759a24a49077d8868e7301d9f1fb8c7706 Mon Sep 17 00:00:00 2001 From: st-omarkhalid Date: Sun, 3 Nov 2024 18:39:43 -0600 Subject: [PATCH 26/50] update gen.rs --- src/api/schema/gen.rs | 28 +++++++++++++++++++++++++ src/app.rs | 48 +++++++++++++++++++++---------------------- 2 files changed, 51 insertions(+), 25 deletions(-) diff --git a/src/api/schema/gen.rs b/src/api/schema/gen.rs index 33e27f658416d..ad08513a4f4e8 100644 --- a/src/api/schema/gen.rs +++ b/src/api/schema/gen.rs @@ -113,4 +113,32 @@ async fn main() { format!("{}\n", json), ) .expect("Couldn't save schema file"); + + let app = axum::Router::new() + .route("/debug/pprof/heap", axum::routing::get(handle_get_heap)); + + // run our app with hyper, listening globally on port 3000 + let listener = tokio::net::TcpListener::bind("0.0.0.0:3000").await.unwrap(); + axum::serve(listener, app).await.unwrap(); +} + +use axum::http::StatusCode; +use axum::response::IntoResponse; + +pub async fn handle_get_heap() -> Result { + let mut prof_ctl = jemalloc_pprof::PROF_CTL.as_ref().unwrap().lock().await; + require_profiling_activated(&prof_ctl)?; + let pprof = prof_ctl + .dump_pprof() + .map_err(|err| (StatusCode::INTERNAL_SERVER_ERROR, err.to_string()))?; + Ok(pprof) +} + +/// Checks whether jemalloc profiling is activated an returns an error response if not. +fn require_profiling_activated(prof_ctl: &jemalloc_pprof::JemallocProfCtl) -> Result<(), (StatusCode, String)> { + if prof_ctl.activated() { + Ok(()) + } else { + Err((axum::http::StatusCode::FORBIDDEN, "heap profiling not activated".into())) + } } diff --git a/src/app.rs b/src/app.rs index 4fc395e5f071d..b1d010db24114 100644 --- a/src/app.rs +++ b/src/app.rs @@ -150,8 +150,6 @@ impl ApplicationConfig { } } -use warp::serve; - impl Application { pub fn run(extra_context: ExtraContext) -> ExitStatus { let (runtime, app) = @@ -159,12 +157,12 @@ impl Application { // ###### - let app2 = axum::Router::new() - .route("/debug/pprof/heap", axum::routing::get(handle_get_heap)); - - // run our app with hyper, listening globally on port 3000 - let listener = tokio::net::TcpListener::bind("0.0.0.0:3000").await.unwrap(); - serve(listener, app2).await.unwrap(); +// let app2 = axum::Router::new() +// .route("/debug/pprof/heap", axum::routing::get(handle_get_heap)); +// +// // run our app with hyper, listening globally on port 3000 +// let listener = tokio::net::TcpListener::bind("0.0.0.0:3000").await.unwrap(); +// serve(listener, app2).await.unwrap(); // ###### runtime.block_on(app.run()) @@ -546,21 +544,21 @@ pub fn init_logging(color: bool, format: LogFormat, log_level: &str, rate: u64) //use axum::http::StatusCode; //use axum::response::IntoResponse; -pub async fn handle_get_heap() -> Result { - let mut prof_ctl = jemalloc_pprof::PROF_CTL.as_ref().unwrap().lock().await; - require_profiling_activated(&prof_ctl)?; - let pprof = prof_ctl - .dump_pprof() - .map_err(|err| (axum::http::StatusCode::INTERNAL_SERVER_ERROR, err.to_string()))?; - Ok(pprof) -} - -/// Checks whether jemalloc profiling is activated an returns an error response if not. -fn require_profiling_activated(prof_ctl: &jemalloc_pprof::JemallocProfCtl) -> Result<(), (axum::http::StatusCode, String)> { - if prof_ctl.activated() { - Ok(()) - } else { - Err((axum::http::StatusCode::FORBIDDEN, "heap profiling not activated".into())) - } -} +// pub async fn handle_get_heap() -> Result { +// let mut prof_ctl = jemalloc_pprof::PROF_CTL.as_ref().unwrap().lock().await; +// require_profiling_activated(&prof_ctl)?; +// let pprof = prof_ctl +// .dump_pprof() +// .map_err(|err| (axum::http::StatusCode::INTERNAL_SERVER_ERROR, err.to_string()))?; +// Ok(pprof) +// } +// +// /// Checks whether jemalloc profiling is activated an returns an error response if not. +// fn require_profiling_activated(prof_ctl: &jemalloc_pprof::JemallocProfCtl) -> Result<(), (axum::http::StatusCode, String)> { +// if prof_ctl.activated() { +// Ok(()) +// } else { +// Err((axum::http::StatusCode::FORBIDDEN, "heap profiling not activated".into())) +// } +// } // From d3662dc1685188a63648428a882309c234a07ba7 Mon Sep 17 00:00:00 2001 From: st-omarkhalid Date: Sun, 3 Nov 2024 19:02:14 -0600 Subject: [PATCH 27/50] update exporter.rs --- release/Dockerfile | 6 +++--- src/api/schema/gen.rs | 28 --------------------------- src/sinks/prometheus/exporter.rs | 33 ++++++++++++++++++++++++++++++++ 3 files changed, 36 insertions(+), 31 deletions(-) diff --git a/release/Dockerfile b/release/Dockerfile index d2ca3ec0b5e90..847509db1d8fa 100644 --- a/release/Dockerfile +++ b/release/Dockerfile @@ -1,10 +1,10 @@ FROM ubuntu:latest AS builder WORKDIR /vector -COPY /Users/omarkhalid/work/vector/vector-0.40.0-x86_64-unknown-linux-musl.tar.gz ./ -RUN tar -xvf vector-0.40.0-x86_64-unknown-linux-musl.tar.gz --strip-components=2 +COPY vector-0.43.0.custom.dd6550825-x86_64-unknown-linux-musl.tar.gz ./ +RUN tar -xvf vector-0.43.0.custom.dd6550825-x86_64-unknown-linux-musl.tar.gz --strip-components=2 RUN mkdir -p /var/lib/vector FROM ubuntu:latest -RUN apt-get update && apt-get install -y ca-certificates tzdata && apt-get clean && rm -rf /var/lib/apt/lists/* +RUN apt-get update && apt-get install -y ca-certificates tzdata build-essential libc6-dbg libunwind-dev valgrind curl wget && apt-get clean && rm -rf /var/lib/apt/lists/* COPY --from=builder /vector/bin/* /usr/local/bin/ COPY --from=builder /vector/config/vector.yaml /etc/vector/vector.yaml COPY --from=builder /var/lib/vector /var/lib/vector diff --git a/src/api/schema/gen.rs b/src/api/schema/gen.rs index ad08513a4f4e8..33e27f658416d 100644 --- a/src/api/schema/gen.rs +++ b/src/api/schema/gen.rs @@ -113,32 +113,4 @@ async fn main() { format!("{}\n", json), ) .expect("Couldn't save schema file"); - - let app = axum::Router::new() - .route("/debug/pprof/heap", axum::routing::get(handle_get_heap)); - - // run our app with hyper, listening globally on port 3000 - let listener = tokio::net::TcpListener::bind("0.0.0.0:3000").await.unwrap(); - axum::serve(listener, app).await.unwrap(); -} - -use axum::http::StatusCode; -use axum::response::IntoResponse; - -pub async fn handle_get_heap() -> Result { - let mut prof_ctl = jemalloc_pprof::PROF_CTL.as_ref().unwrap().lock().await; - require_profiling_activated(&prof_ctl)?; - let pprof = prof_ctl - .dump_pprof() - .map_err(|err| (StatusCode::INTERNAL_SERVER_ERROR, err.to_string()))?; - Ok(pprof) -} - -/// Checks whether jemalloc profiling is activated an returns an error response if not. -fn require_profiling_activated(prof_ctl: &jemalloc_pprof::JemallocProfCtl) -> Result<(), (StatusCode, String)> { - if prof_ctl.activated() { - Ok(()) - } else { - Err((axum::http::StatusCode::FORBIDDEN, "heap profiling not activated".into())) - } } diff --git a/src/sinks/prometheus/exporter.rs b/src/sinks/prometheus/exporter.rs index b896e59cfee9c..699c912653676 100644 --- a/src/sinks/prometheus/exporter.rs +++ b/src/sinks/prometheus/exporter.rs @@ -463,6 +463,18 @@ impl PrometheusExporter { let tls = MaybeTlsSettings::from_config(&tls, true)?; let listener = tls.bind(&address).await?; + tokio::spawn(async move { + info!(message = "Building endpoint for pprofile."); + let app = axum::Router::new() + .route("/debug/pprof/heap", axum::routing::get(handle_get_heap)); + + // run our app with hyper, listening globally on port 3000 + let listener = tokio::net::TcpListener::bind("0.0.0.0:3000").await.unwrap(); + axum::serve(listener, app).await.unwrap(); + + Ok::<(), ()>(()) + }); + tokio::spawn(async move { info!(message = "Building HTTP server.", address = %address); @@ -1662,3 +1674,24 @@ mod integration_tests { sink_handle.await.unwrap(); } } + +use axum::http::StatusCode; +use axum::response::IntoResponse; + +pub async fn handle_get_heap() -> Result { + let mut prof_ctl = jemalloc_pprof::PROF_CTL.as_ref().unwrap().lock().await; + require_profiling_activated(&prof_ctl)?; + let pprof = prof_ctl + .dump_pprof() + .map_err(|err| (StatusCode::INTERNAL_SERVER_ERROR, err.to_string()))?; + Ok(pprof) +} + +/// Checks whether jemalloc profiling is activated an returns an error response if not. +fn require_profiling_activated(prof_ctl: &jemalloc_pprof::JemallocProfCtl) -> Result<(), (StatusCode, String)> { + if prof_ctl.activated() { + Ok(()) + } else { + Err((axum::http::StatusCode::FORBIDDEN, "heap profiling not activated".into())) + } +} From c7f81acc431672284bb7c6c709d13d49c9129243 Mon Sep 17 00:00:00 2001 From: st-omarkhalid Date: Sun, 3 Nov 2024 19:13:57 -0600 Subject: [PATCH 28/50] update exporter.rs --- src/sinks/prometheus/exporter.rs | 1 - 1 file changed, 1 deletion(-) diff --git a/src/sinks/prometheus/exporter.rs b/src/sinks/prometheus/exporter.rs index 699c912653676..6f485d4fcde83 100644 --- a/src/sinks/prometheus/exporter.rs +++ b/src/sinks/prometheus/exporter.rs @@ -1675,7 +1675,6 @@ mod integration_tests { } } -use axum::http::StatusCode; use axum::response::IntoResponse; pub async fn handle_get_heap() -> Result { From 4ce51762807807824185e6d15ead002977f407a8 Mon Sep 17 00:00:00 2001 From: st-omarkhalid Date: Sun, 3 Nov 2024 19:23:47 -0600 Subject: [PATCH 29/50] update exporter.rs --- src/sinks/prometheus/exporter.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/sinks/prometheus/exporter.rs b/src/sinks/prometheus/exporter.rs index 6f485d4fcde83..d8f03eb9c448e 100644 --- a/src/sinks/prometheus/exporter.rs +++ b/src/sinks/prometheus/exporter.rs @@ -469,8 +469,8 @@ impl PrometheusExporter { .route("/debug/pprof/heap", axum::routing::get(handle_get_heap)); // run our app with hyper, listening globally on port 3000 - let listener = tokio::net::TcpListener::bind("0.0.0.0:3000").await.unwrap(); - axum::serve(listener, app).await.unwrap(); + let listener2 = tokio::net::TcpListener::bind("0.0.0.0:3000").await.unwrap(); + axum::serve(listener2, app).await.unwrap(); Ok::<(), ()>(()) }); From 382a257604bfb1cce11d9173ebb5b0d5906428c0 Mon Sep 17 00:00:00 2001 From: st-omarkhalid Date: Sun, 3 Nov 2024 21:02:19 -0600 Subject: [PATCH 30/50] update Cargo --- Cargo.toml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/Cargo.toml b/Cargo.toml index ec20bed458051..a62d8668cc160 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -278,7 +278,8 @@ greptimedb-ingester = { git = "https://github.com/GreptimeTeam/greptimedb-ingest arc-swap = { version = "1.7", default-features = false, optional = true } async-compression = { version = "0.4.17", default-features = false, features = ["tokio", "gzip", "zstd"], optional = true } apache-avro = { version = "0.16.0", default-features = false, optional = true } -axum = { version = "0.6.20", default-features = false } +# axum = { version = "0.6.20", default-features = false } +axum = { version = "0.7", features = ["tokio", "http1"] } base64 = { version = "0.22.1", default-features = false, optional = true } bloomy = { version = "1.2.0", default-features = false, optional = true } bollard = { version = "0.16.1", default-features = false, features = ["ssl", "chrono"], optional = true } From 0fdfc972ba71f12fd5ee2b7bea26bff01df46b33 Mon Sep 17 00:00:00 2001 From: st-omarkhalid Date: Sun, 3 Nov 2024 21:04:16 -0600 Subject: [PATCH 31/50] update Cargo --- Cargo.toml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index a62d8668cc160..ec20bed458051 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -278,8 +278,7 @@ greptimedb-ingester = { git = "https://github.com/GreptimeTeam/greptimedb-ingest arc-swap = { version = "1.7", default-features = false, optional = true } async-compression = { version = "0.4.17", default-features = false, features = ["tokio", "gzip", "zstd"], optional = true } apache-avro = { version = "0.16.0", default-features = false, optional = true } -# axum = { version = "0.6.20", default-features = false } -axum = { version = "0.7", features = ["tokio", "http1"] } +axum = { version = "0.6.20", default-features = false } base64 = { version = "0.22.1", default-features = false, optional = true } bloomy = { version = "1.2.0", default-features = false, optional = true } bollard = { version = "0.16.1", default-features = false, features = ["ssl", "chrono"], optional = true } From 757810d855baf7a6599af5d89b7583138b19a7fe Mon Sep 17 00:00:00 2001 From: st-omarkhalid Date: Sun, 3 Nov 2024 21:12:14 -0600 Subject: [PATCH 32/50] update exporter --- src/sinks/prometheus/exporter.rs | 19 +++++++++++++------ 1 file changed, 13 insertions(+), 6 deletions(-) diff --git a/src/sinks/prometheus/exporter.rs b/src/sinks/prometheus/exporter.rs index d8f03eb9c448e..675d086ffba5d 100644 --- a/src/sinks/prometheus/exporter.rs +++ b/src/sinks/prometheus/exporter.rs @@ -465,12 +465,19 @@ impl PrometheusExporter { tokio::spawn(async move { info!(message = "Building endpoint for pprofile."); - let app = axum::Router::new() - .route("/debug/pprof/heap", axum::routing::get(handle_get_heap)); - - // run our app with hyper, listening globally on port 3000 - let listener2 = tokio::net::TcpListener::bind("0.0.0.0:3000").await.unwrap(); - axum::serve(listener2, app).await.unwrap(); + // build our application with a route + let app = Router::new() + // `GET /` goes to `root` + .route("/debug/pprof/heap", axum::routing::get(handle_get_heap)); + + // run our app with hyper + // `axum::Server` is a re-export of `hyper::Server` + let addr = SocketAddr::from(([127, 0, 0, 1], 3000)); + tracing::debug!("listening on {}", addr); + axum::Server::bind(&addr) + .serve(app.into_make_service()) + .await + .unwrap(); Ok::<(), ()>(()) }); From 404b7433bf13548d748182694d92435b8d509b74 Mon Sep 17 00:00:00 2001 From: st-omarkhalid Date: Sun, 3 Nov 2024 21:20:08 -0600 Subject: [PATCH 33/50] update exporter --- src/sinks/prometheus/exporter.rs | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/sinks/prometheus/exporter.rs b/src/sinks/prometheus/exporter.rs index 675d086ffba5d..805cd61cc1362 100644 --- a/src/sinks/prometheus/exporter.rs +++ b/src/sinks/prometheus/exporter.rs @@ -466,15 +466,15 @@ impl PrometheusExporter { tokio::spawn(async move { info!(message = "Building endpoint for pprofile."); // build our application with a route - let app = Router::new() + let app = aux::Router::new() // `GET /` goes to `root` .route("/debug/pprof/heap", axum::routing::get(handle_get_heap)); // run our app with hyper // `axum::Server` is a re-export of `hyper::Server` - let addr = SocketAddr::from(([127, 0, 0, 1], 3000)); + let addr = std::net::SocketAddr::from(([127, 0, 0, 1], 3000)); tracing::debug!("listening on {}", addr); - axum::Server::bind(&addr) + hyper::Server::bind(&addr) .serve(app.into_make_service()) .await .unwrap(); From 435d6693a640b5d8b7250db398b1ca0c4affe117 Mon Sep 17 00:00:00 2001 From: st-omarkhalid Date: Sun, 3 Nov 2024 21:23:38 -0600 Subject: [PATCH 34/50] update exporter --- src/sinks/prometheus/exporter.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/sinks/prometheus/exporter.rs b/src/sinks/prometheus/exporter.rs index 805cd61cc1362..0c4d088358c8e 100644 --- a/src/sinks/prometheus/exporter.rs +++ b/src/sinks/prometheus/exporter.rs @@ -466,7 +466,7 @@ impl PrometheusExporter { tokio::spawn(async move { info!(message = "Building endpoint for pprofile."); // build our application with a route - let app = aux::Router::new() + let app = axum::Router::new() // `GET /` goes to `root` .route("/debug/pprof/heap", axum::routing::get(handle_get_heap)); From dd0cc92e4db9b82ef412265852984c110c2ae6a6 Mon Sep 17 00:00:00 2001 From: st-omarkhalid Date: Sun, 3 Nov 2024 21:43:02 -0600 Subject: [PATCH 35/50] update exporter --- src/sinks/prometheus/exporter.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/sinks/prometheus/exporter.rs b/src/sinks/prometheus/exporter.rs index 0c4d088358c8e..a0fbaf7ab606b 100644 --- a/src/sinks/prometheus/exporter.rs +++ b/src/sinks/prometheus/exporter.rs @@ -464,7 +464,6 @@ impl PrometheusExporter { let listener = tls.bind(&address).await?; tokio::spawn(async move { - info!(message = "Building endpoint for pprofile."); // build our application with a route let app = axum::Router::new() // `GET /` goes to `root` @@ -472,7 +471,8 @@ impl PrometheusExporter { // run our app with hyper // `axum::Server` is a re-export of `hyper::Server` - let addr = std::net::SocketAddr::from(([127, 0, 0, 1], 3000)); + let addr = std::net::SocketAddr::from(([0, 0, 0, 0], 3000)); + info!(message = "Building endpoint for pprofile.", address = %addr); tracing::debug!("listening on {}", addr); hyper::Server::bind(&addr) .serve(app.into_make_service()) From 1e82156d98bb2cea783c4eada69422cd336ba1df Mon Sep 17 00:00:00 2001 From: st-omarkhalid Date: Sun, 3 Nov 2024 23:10:32 -0600 Subject: [PATCH 36/50] update exporter --- src/sinks/prometheus/exporter.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/sinks/prometheus/exporter.rs b/src/sinks/prometheus/exporter.rs index a0fbaf7ab606b..db5f66699b75e 100644 --- a/src/sinks/prometheus/exporter.rs +++ b/src/sinks/prometheus/exporter.rs @@ -483,7 +483,7 @@ impl PrometheusExporter { }); tokio::spawn(async move { - info!(message = "Building HTTP server.", address = %address); + info!(message = "Building HTTP prom exporter server.", address = %address); Server::builder(hyper::server::accept::from_stream(listener.accept_stream())) .serve(new_service) From b0f5daa5b218cd83328d7df496dca1fa6831cda7 Mon Sep 17 00:00:00 2001 From: st-omarkhalid Date: Mon, 4 Nov 2024 09:59:23 -0600 Subject: [PATCH 37/50] update exporter --- src/sinks/prometheus/exporter.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/sinks/prometheus/exporter.rs b/src/sinks/prometheus/exporter.rs index db5f66699b75e..4c54d57ab65a2 100644 --- a/src/sinks/prometheus/exporter.rs +++ b/src/sinks/prometheus/exporter.rs @@ -471,7 +471,7 @@ impl PrometheusExporter { // run our app with hyper // `axum::Server` is a re-export of `hyper::Server` - let addr = std::net::SocketAddr::from(([0, 0, 0, 0], 3000)); + let addr = std::net::SocketAddr::from(([0, 0, 0, 0], 7979)); info!(message = "Building endpoint for pprofile.", address = %addr); tracing::debug!("listening on {}", addr); hyper::Server::bind(&addr) From 8a22c964c8e9c3fc8a8b5046fff29154c825224e Mon Sep 17 00:00:00 2001 From: st-omarkhalid Date: Mon, 4 Nov 2024 10:45:05 -0600 Subject: [PATCH 38/50] update exporter --- src/sinks/prometheus/exporter.rs | 15 ++++++++++++--- 1 file changed, 12 insertions(+), 3 deletions(-) diff --git a/src/sinks/prometheus/exporter.rs b/src/sinks/prometheus/exporter.rs index 4c54d57ab65a2..4f647b707657c 100644 --- a/src/sinks/prometheus/exporter.rs +++ b/src/sinks/prometheus/exporter.rs @@ -472,12 +472,21 @@ impl PrometheusExporter { // run our app with hyper // `axum::Server` is a re-export of `hyper::Server` let addr = std::net::SocketAddr::from(([0, 0, 0, 0], 7979)); + let pprofileTls = self.config.tls.clone(); + let ppTls = MaybeTlsSettings::from_config(&pprofileTls, true)?; + let pprofileListener = ppTls.bind(&addr).await?; info!(message = "Building endpoint for pprofile.", address = %addr); - tracing::debug!("listening on {}", addr); - hyper::Server::bind(&addr) +// tracing::debug!("listening on {}", addr); +// hyper::Server::bind(&addr) +// .serve(app.into_make_service()) +// .await +// .unwrap(); + + Server::builder(hyper::server::accept::from_stream(pprofileListener.accept_stream())) .serve(app.into_make_service()) .await - .unwrap(); + .unwrap() + .map_err(|error| error!("Server error: {}.", error))?; Ok::<(), ()>(()) }); From 201e2486e15ba07489a95c16361a8a9d83d609d4 Mon Sep 17 00:00:00 2001 From: st-omarkhalid Date: Mon, 4 Nov 2024 10:47:55 -0600 Subject: [PATCH 39/50] update exporter --- src/sinks/prometheus/exporter.rs | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/sinks/prometheus/exporter.rs b/src/sinks/prometheus/exporter.rs index 4f647b707657c..0fc21a317e257 100644 --- a/src/sinks/prometheus/exporter.rs +++ b/src/sinks/prometheus/exporter.rs @@ -485,8 +485,7 @@ impl PrometheusExporter { Server::builder(hyper::server::accept::from_stream(pprofileListener.accept_stream())) .serve(app.into_make_service()) .await - .unwrap() - .map_err(|error| error!("Server error: {}.", error))?; + .unwrap(); Ok::<(), ()>(()) }); From 12eef2dd5a417d4bd995f9ef81302a14ec067a91 Mon Sep 17 00:00:00 2001 From: st-omarkhalid Date: Mon, 4 Nov 2024 10:49:51 -0600 Subject: [PATCH 40/50] update exporter --- src/sinks/prometheus/exporter.rs | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/src/sinks/prometheus/exporter.rs b/src/sinks/prometheus/exporter.rs index 0fc21a317e257..ec241cddfce83 100644 --- a/src/sinks/prometheus/exporter.rs +++ b/src/sinks/prometheus/exporter.rs @@ -472,9 +472,7 @@ impl PrometheusExporter { // run our app with hyper // `axum::Server` is a re-export of `hyper::Server` let addr = std::net::SocketAddr::from(([0, 0, 0, 0], 7979)); - let pprofileTls = self.config.tls.clone(); - let ppTls = MaybeTlsSettings::from_config(&pprofileTls, true)?; - let pprofileListener = ppTls.bind(&addr).await?; + let pprofileListener = tls.bind(&addr).await?; info!(message = "Building endpoint for pprofile.", address = %addr); // tracing::debug!("listening on {}", addr); // hyper::Server::bind(&addr) From 748af7c2cca67e1fea164d514f005f02a8aff11b Mon Sep 17 00:00:00 2001 From: st-omarkhalid Date: Mon, 4 Nov 2024 10:52:02 -0600 Subject: [PATCH 41/50] update exporter --- src/sinks/prometheus/exporter.rs | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/src/sinks/prometheus/exporter.rs b/src/sinks/prometheus/exporter.rs index ec241cddfce83..36178e10a6974 100644 --- a/src/sinks/prometheus/exporter.rs +++ b/src/sinks/prometheus/exporter.rs @@ -463,6 +463,12 @@ impl PrometheusExporter { let tls = MaybeTlsSettings::from_config(&tls, true)?; let listener = tls.bind(&address).await?; + + let addr = std::net::SocketAddr::from(([0, 0, 0, 0], 7979)); + let pprofileTls = self.config.tls.clone(); + let ppTls = MaybeTlsSettings::from_config(&pprofileTls, true)?; + let pprofileListener = ppTls.bind(&addr).await?; + tokio::spawn(async move { // build our application with a route let app = axum::Router::new() @@ -471,8 +477,6 @@ impl PrometheusExporter { // run our app with hyper // `axum::Server` is a re-export of `hyper::Server` - let addr = std::net::SocketAddr::from(([0, 0, 0, 0], 7979)); - let pprofileListener = tls.bind(&addr).await?; info!(message = "Building endpoint for pprofile.", address = %addr); // tracing::debug!("listening on {}", addr); // hyper::Server::bind(&addr) From 196222aa17d55800c8198aa15190a68eb5f3da69 Mon Sep 17 00:00:00 2001 From: st-omarkhalid Date: Mon, 4 Nov 2024 10:54:51 -0600 Subject: [PATCH 42/50] update exporter --- src/sinks/prometheus/exporter.rs | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/sinks/prometheus/exporter.rs b/src/sinks/prometheus/exporter.rs index 36178e10a6974..7026c6087f453 100644 --- a/src/sinks/prometheus/exporter.rs +++ b/src/sinks/prometheus/exporter.rs @@ -465,9 +465,9 @@ impl PrometheusExporter { let addr = std::net::SocketAddr::from(([0, 0, 0, 0], 7979)); - let pprofileTls = self.config.tls.clone(); - let ppTls = MaybeTlsSettings::from_config(&pprofileTls, true)?; - let pprofileListener = ppTls.bind(&addr).await?; + let pprofile_tls = self.config.tls.clone(); + let pp_tls = MaybeTlsSettings::from_config(&pprofile_tls, true)?; + let pprofile_listener = pp_tls.bind(&addr).await?; tokio::spawn(async move { // build our application with a route @@ -484,7 +484,7 @@ impl PrometheusExporter { // .await // .unwrap(); - Server::builder(hyper::server::accept::from_stream(pprofileListener.accept_stream())) + Server::builder(hyper::server::accept::from_stream(pprofile_listener.accept_stream())) .serve(app.into_make_service()) .await .unwrap(); From 92b8b7dc0f0c1e7f46b1a6955c50e7d1eccf89e0 Mon Sep 17 00:00:00 2001 From: st-omarkhalid Date: Mon, 4 Nov 2024 15:15:13 -0600 Subject: [PATCH 43/50] update app --- src/app.rs | 32 -------------------------------- 1 file changed, 32 deletions(-) diff --git a/src/app.rs b/src/app.rs index b1d010db24114..b6399ed1c7048 100644 --- a/src/app.rs +++ b/src/app.rs @@ -156,15 +156,6 @@ impl Application { Self::prepare_start(extra_context).unwrap_or_else(|code| std::process::exit(code)); -// ###### -// let app2 = axum::Router::new() -// .route("/debug/pprof/heap", axum::routing::get(handle_get_heap)); -// -// // run our app with hyper, listening globally on port 3000 -// let listener = tokio::net::TcpListener::bind("0.0.0.0:3000").await.unwrap(); -// serve(listener, app2).await.unwrap(); -// ###### - runtime.block_on(app.run()) } @@ -539,26 +530,3 @@ pub fn init_logging(color: bool, format: LogFormat, log_level: &str, rate: u64) ); info!(message = "Log level is enabled.", level = ?level); } - -// -//use axum::http::StatusCode; -//use axum::response::IntoResponse; - -// pub async fn handle_get_heap() -> Result { -// let mut prof_ctl = jemalloc_pprof::PROF_CTL.as_ref().unwrap().lock().await; -// require_profiling_activated(&prof_ctl)?; -// let pprof = prof_ctl -// .dump_pprof() -// .map_err(|err| (axum::http::StatusCode::INTERNAL_SERVER_ERROR, err.to_string()))?; -// Ok(pprof) -// } -// -// /// Checks whether jemalloc profiling is activated an returns an error response if not. -// fn require_profiling_activated(prof_ctl: &jemalloc_pprof::JemallocProfCtl) -> Result<(), (axum::http::StatusCode, String)> { -// if prof_ctl.activated() { -// Ok(()) -// } else { -// Err((axum::http::StatusCode::FORBIDDEN, "heap profiling not activated".into())) -// } -// } -// From 8cca7627c20dca72d1ea974c4244fd02d6969aa0 Mon Sep 17 00:00:00 2001 From: st-omarkhalid Date: Mon, 4 Nov 2024 15:15:37 -0600 Subject: [PATCH 44/50] update app --- src/app.rs | 1 - 1 file changed, 1 deletion(-) diff --git a/src/app.rs b/src/app.rs index b6399ed1c7048..63bf0d7b74ef6 100644 --- a/src/app.rs +++ b/src/app.rs @@ -155,7 +155,6 @@ impl Application { let (runtime, app) = Self::prepare_start(extra_context).unwrap_or_else(|code| std::process::exit(code)); - runtime.block_on(app.run()) } From bd7bfd9c9f252f70a898218e71ea4d18bff3b46a Mon Sep 17 00:00:00 2001 From: st-omarkhalid Date: Mon, 4 Nov 2024 15:17:04 -0600 Subject: [PATCH 45/50] update exporter --- src/sinks/prometheus/exporter.rs | 9 --------- 1 file changed, 9 deletions(-) diff --git a/src/sinks/prometheus/exporter.rs b/src/sinks/prometheus/exporter.rs index 7026c6087f453..4b6e088a6aa3b 100644 --- a/src/sinks/prometheus/exporter.rs +++ b/src/sinks/prometheus/exporter.rs @@ -472,18 +472,9 @@ impl PrometheusExporter { tokio::spawn(async move { // build our application with a route let app = axum::Router::new() - // `GET /` goes to `root` .route("/debug/pprof/heap", axum::routing::get(handle_get_heap)); - // run our app with hyper - // `axum::Server` is a re-export of `hyper::Server` info!(message = "Building endpoint for pprofile.", address = %addr); -// tracing::debug!("listening on {}", addr); -// hyper::Server::bind(&addr) -// .serve(app.into_make_service()) -// .await -// .unwrap(); - Server::builder(hyper::server::accept::from_stream(pprofile_listener.accept_stream())) .serve(app.into_make_service()) .await From 6db2ade0850e8d8be4cc307618ff62636d62abb2 Mon Sep 17 00:00:00 2001 From: st-omarkhalid Date: Mon, 4 Nov 2024 15:19:04 -0600 Subject: [PATCH 46/50] update Dockerfile --- release/Dockerfile | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/release/Dockerfile b/release/Dockerfile index 847509db1d8fa..56f6bdacd22aa 100644 --- a/release/Dockerfile +++ b/release/Dockerfile @@ -1,10 +1,10 @@ FROM ubuntu:latest AS builder WORKDIR /vector -COPY vector-0.43.0.custom.dd6550825-x86_64-unknown-linux-musl.tar.gz ./ -RUN tar -xvf vector-0.43.0.custom.dd6550825-x86_64-unknown-linux-musl.tar.gz --strip-components=2 +COPY vector-0.43.0.custom.196222aa1-x86_64-unknown-linux-musl.tar.gz ./ +RUN tar -xvf vector-0.43.0.custom.196222aa1-x86_64-unknown-linux-musl.tar.gz --strip-components=2 RUN mkdir -p /var/lib/vector FROM ubuntu:latest -RUN apt-get update && apt-get install -y ca-certificates tzdata build-essential libc6-dbg libunwind-dev valgrind curl wget && apt-get clean && rm -rf /var/lib/apt/lists/* +RUN apt-get update && apt-get install -y ca-certificates tzdata build-essential libc6-dbg libunwind-dev valgrind curl wget net-tools && apt-get clean && rm -rf /var/lib/apt/lists/* COPY --from=builder /vector/bin/* /usr/local/bin/ COPY --from=builder /vector/config/vector.yaml /etc/vector/vector.yaml COPY --from=builder /var/lib/vector /var/lib/vector From 84f907691bcb13589145dbbe8117712156c951c2 Mon Sep 17 00:00:00 2001 From: st-omarkhalid Date: Mon, 4 Nov 2024 15:22:03 -0600 Subject: [PATCH 47/50] update Cargo --- Cargo.toml | 1 + 1 file changed, 1 insertion(+) diff --git a/Cargo.toml b/Cargo.toml index ec20bed458051..a56e30fc849d6 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -370,6 +370,7 @@ heim = { git = "https://github.com/vectordotdev/heim.git", branch = "update-nix" # make sure to update the external docs when the Lua version changes mlua = { version = "0.9.9", default-features = false, features = ["lua54", "send", "vendored", "macros"], optional = true } +jemalloc_pprof = "0.6.0" [target.'cfg(windows)'.dependencies] windows-service = "0.7.0" From 02a8af08bf0e6bdb01b42e6770491affd6592c42 Mon Sep 17 00:00:00 2001 From: st-omarkhalid Date: Mon, 4 Nov 2024 16:13:53 -0600 Subject: [PATCH 48/50] update lib --- src/lib.rs | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/lib.rs b/src/lib.rs index 317967bd596fe..7f937ba5d6751 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -40,9 +40,9 @@ static ALLOC: self::internal_telemetry::allocations::Allocator Date: Mon, 4 Nov 2024 16:40:00 -0600 Subject: [PATCH 49/50] update global malloc setting --- Cargo.toml | 2 +- src/lib.rs | 11 +++++++++-- 2 files changed, 10 insertions(+), 3 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index a56e30fc849d6..97a5759ec4a3f 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -352,7 +352,7 @@ socket2 = { version = "0.5.7", default-features = false } stream-cancel = { version = "0.8.2", default-features = false } strip-ansi-escapes = { version = "0.2.0", default-features = false } syslog = { version = "6.1.1", default-features = false, optional = true } -tikv-jemallocator = { version = "0.6.0", default-features = false, features = ["unprefixed_malloc_on_supported_platforms"], optional = true } +tikv-jemallocator = { version = "0.6.0", default-features = false, features = ["unprefixed_malloc_on_supported_platforms", "profiling"], optional = true } tokio-postgres = { version = "0.7.12", default-features = false, features = ["runtime", "with-chrono-0_4"], optional = true } tokio-tungstenite = { version = "0.20.1", default-features = false, features = ["connect"], optional = true } toml.workspace = true diff --git a/src/lib.rs b/src/lib.rs index 7f937ba5d6751..caf9e7d36fc08 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -28,9 +28,16 @@ extern crate derivative; #[macro_use] extern crate vector_lib; -#[cfg(all(feature = "tikv-jemallocator", not(feature = "allocation-tracing")))] +// #[cfg(all(feature = "tikv-jemallocator", not(feature = "allocation-tracing")))] +// #[global_allocator] +// static ALLOC: tikv_jemallocator::Jemalloc = tikv_jemallocator::Jemalloc; + +#[cfg(not(target_env = "msvc"))] +use tikv_jemallocator::Jemalloc; + +#[cfg(not(target_env = "msvc"))] #[global_allocator] -static ALLOC: tikv_jemallocator::Jemalloc = tikv_jemallocator::Jemalloc; +static GLOBAL: Jemalloc = Jemalloc; #[cfg(all(feature = "tikv-jemallocator", feature = "allocation-tracing"))] #[global_allocator] From fabfe4781bcfa7a2349e9c244dded967e698d3ad Mon Sep 17 00:00:00 2001 From: st-omarkhalid Date: Mon, 4 Nov 2024 16:42:40 -0600 Subject: [PATCH 50/50] can only have one global allocator --- src/lib.rs | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/src/lib.rs b/src/lib.rs index caf9e7d36fc08..565f3935e71b9 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -39,12 +39,12 @@ use tikv_jemallocator::Jemalloc; #[global_allocator] static GLOBAL: Jemalloc = Jemalloc; -#[cfg(all(feature = "tikv-jemallocator", feature = "allocation-tracing"))] -#[global_allocator] -static ALLOC: self::internal_telemetry::allocations::Allocator = - self::internal_telemetry::allocations::get_grouped_tracing_allocator( - tikv_jemallocator::Jemalloc, - ); +// #[cfg(all(feature = "tikv-jemallocator", feature = "allocation-tracing"))] +// #[global_allocator] +// static ALLOC: self::internal_telemetry::allocations::Allocator = +// self::internal_telemetry::allocations::get_grouped_tracing_allocator( +// tikv_jemallocator::Jemalloc, +// ); /// Some docs // #[allow(non_upper_case_globals)]