From 1e5deb18b3293fdce4684936d6f360dd3216235a Mon Sep 17 00:00:00 2001 From: Meghdad Date: Thu, 7 May 2026 22:57:00 +0330 Subject: [PATCH] initial module --- .dockerignore | 9 + .gitignore | 7 + Makefile | 28 + README.md | 112 +++ assets/dashboard.html | 391 +++++++++++ config | 21 + dockerized/Dockerfile | 90 +++ dockerized/README.md | 67 ++ dockerized/build.sh | 15 + dockerized/docker-compose.yml | 12 + dockerized/nginx.conf | 57 ++ dockerized/run.sh | 13 + dockerized/test.sh | 22 + docs/API.md | 92 +++ docs/ARCHITECTURE.md | 60 ++ docs/CONFIGURATION.md | 51 ++ docs/PERFORMANCE.md | 48 ++ examples/nginx.conf | 59 ++ scripts/build.sh | 18 + scripts/smoke.sh | 26 + src/ngx_http_monitoring.h | 304 +++++++++ src/ngx_http_monitoring_collectors.c | 841 +++++++++++++++++++++++ src/ngx_http_monitoring_dashboard.c | 114 ++++ src/ngx_http_monitoring_json.c | 975 ++++++++++++++++++++++++++ src/ngx_http_monitoring_module.c | 985 +++++++++++++++++++++++++++ src/ngx_http_monitoring_shm.c | 558 +++++++++++++++ src/ngx_http_monitoring_sse.c | 296 ++++++++ 27 files changed, 5271 insertions(+) create mode 100644 .dockerignore create mode 100644 .gitignore create mode 100644 Makefile create mode 100644 README.md create mode 100644 assets/dashboard.html create mode 100644 config create mode 100644 dockerized/Dockerfile create mode 100644 dockerized/README.md create mode 100644 dockerized/build.sh create mode 100644 dockerized/docker-compose.yml create mode 100644 dockerized/nginx.conf create mode 100644 dockerized/run.sh create mode 100644 dockerized/test.sh create mode 100644 docs/API.md create mode 100644 docs/ARCHITECTURE.md create mode 100644 docs/CONFIGURATION.md create mode 100644 docs/PERFORMANCE.md create mode 100644 examples/nginx.conf create mode 100644 scripts/build.sh create mode 100644 scripts/smoke.sh create mode 100644 src/ngx_http_monitoring.h create mode 100644 src/ngx_http_monitoring_collectors.c create mode 100644 src/ngx_http_monitoring_dashboard.c create mode 100644 src/ngx_http_monitoring_json.c create mode 100644 src/ngx_http_monitoring_module.c create mode 100644 src/ngx_http_monitoring_shm.c create mode 100644 src/ngx_http_monitoring_sse.c diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 0000000..b010c36 --- /dev/null +++ b/.dockerignore @@ -0,0 +1,9 @@ +.git +.idea +build +objs +*.o +*.so +*.a +*.log +dockerized/.cache diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..7dc70af --- /dev/null +++ b/.gitignore @@ -0,0 +1,7 @@ +.idea +build/ +objs/ +*.o +*.so +*.a +*.log diff --git a/Makefile b/Makefile new file mode 100644 index 0000000..c28bdad --- /dev/null +++ b/Makefile @@ -0,0 +1,28 @@ +NGINX_SRC ?= +NGINX_CONFIGURE_ARGS ?= --with-compat --with-http_ssl_module --with-http_stub_status_module +BUILD_DIR ?= $(CURDIR)/build + +.PHONY: all module clean install check + +all: module + +module: + @test -n "$(NGINX_SRC)" || (echo "Set NGINX_SRC=/path/to/nginx-1.24+ source tree"; exit 1) + @test -d "$(NGINX_SRC)" || (echo "NGINX_SRC does not exist: $(NGINX_SRC)"; exit 1) + cd "$(NGINX_SRC)" && ./configure $(NGINX_CONFIGURE_ARGS) --add-dynamic-module="$(CURDIR)" + $(MAKE) -C "$(NGINX_SRC)" modules + @mkdir -p "$(BUILD_DIR)" + @cp "$(NGINX_SRC)/objs/ngx_http_monitoring_module.so" "$(BUILD_DIR)/" + @echo "Built $(BUILD_DIR)/ngx_http_monitoring_module.so" + +install: module + @test -n "$(DESTDIR)" || (echo "Set DESTDIR to nginx modules directory"; exit 1) + @mkdir -p "$(DESTDIR)" + @cp "$(BUILD_DIR)/ngx_http_monitoring_module.so" "$(DESTDIR)/" + +check: + @test -n "$(NGINX_BIN)" || (echo "Set NGINX_BIN=/path/to/nginx"; exit 1) + "$(NGINX_BIN)" -t -c "$(CURDIR)/examples/nginx.conf" + +clean: + @rm -rf "$(BUILD_DIR)" diff --git a/README.md b/README.md new file mode 100644 index 0000000..36911ed --- /dev/null +++ b/README.md @@ -0,0 +1,112 @@ +# ngx_http_monitoring_module + +`ngx_http_monitoring_module` is a Linux-only dynamic Nginx HTTP module for live server monitoring, JSON APIs, Server-Sent Events, and an embedded dashboard. + +## Features + +- Dynamic module build for Nginx 1.24+ +- Shared-memory metrics across workers +- Atomic request counters and bounded top-N tables +- Timer-based `/proc` collectors with cached snapshots +- JSON REST API under `/monitor/api` +- Embedded dark dashboard at `/monitor` +- Server-Sent Events stream at `/monitor/live` +- Prometheus text endpoint at `/monitor/metrics` +- ACL, optional basic auth, optional API token, CORS, and simple global API rate limiting +- Historical ring buffer with configurable retention and resolution + +## Build + +Install Nginx build prerequisites and use an Nginx source tree configured similarly to the Nginx binary you will load the module into. + +```sh +NGINX_SRC=/usr/local/src/nginx-1.24.0 make module +``` + +Recommended configure options: + +```sh +NGINX_CONFIGURE_ARGS="--with-compat --with-http_ssl_module --with-http_stub_status_module" \ +NGINX_SRC=/usr/local/src/nginx-1.24.0 make module +``` + +The compiled module is written to: + +```text +build/ngx_http_monitoring_module.so +``` + +## Docker Image + +A Dockerized Nginx image is available in [dockerized](): + +```sh +docker build -f dockerized/Dockerfile -t ngx-http-monitoring-module:local . +docker run --rm -p 8080:8080 ngx-http-monitoring-module:local +``` + +Then open: + +```text +http://127.0.0.1:8080/monitor +``` + +Load it from `nginx.conf`: + +```nginx +load_module modules/ngx_http_monitoring_module.so; +``` + +## Minimal Configuration + +```nginx +http { + monitor_refresh_interval 1s; + monitor_history 5m; + monitor_resolution 1s; + + server { + listen 8080; + + location /monitor { + monitor on; + monitor_allow 127.0.0.1/32; + monitor_deny all; + } + } +} +``` + +Open: + +```text +http://127.0.0.1:8080/monitor +``` + +After Nginx is running, run endpoint smoke checks: + +```sh +BASE_URL=http://127.0.0.1:8080 sh scripts/smoke.sh +``` + +## Endpoints + +- `GET /monitor` - embedded dashboard +- `GET /monitor/api` - full JSON document +- `GET /monitor/api/system` - CPU, load, memory, swap, uptime +- `GET /monitor/api/nginx` - Nginx connection/request/worker metrics +- `GET /monitor/api/network` - interfaces and traffic counters +- `GET /monitor/api/disk` - block device and filesystem counters +- `GET /monitor/api/processes` - process count, TCP/socket stats, workers +- `GET /monitor/api/upstreams` - observed upstream peer stats +- `GET /monitor/api/connections` - connection and SSE counters +- `GET /monitor/api/requests` - status, methods, histograms, top URLs, user agents +- `GET /monitor/live` - SSE metrics stream +- `GET /monitor/metrics` - Prometheus-compatible text metrics +- `GET /monitor/health` - lightweight health JSON + +## Notes + +The module is intentionally dependency-free at runtime. System data is collected by worker timers from Linux `/proc`, `statvfs()`, and `getifaddrs()`, then served from shared memory. API requests never parse `/proc` directly. + +For exact active/reading/writing/waiting connection counters, build Nginx with `--with-http_stub_status_module`. diff --git a/assets/dashboard.html b/assets/dashboard.html new file mode 100644 index 0000000..9cd0524 --- /dev/null +++ b/assets/dashboard.html @@ -0,0 +1,391 @@ + + + + + + Nginx Monitor + + + +
+
+
Nginx Monitor
+
waiting
+
+
offline
+
+
+ +
+
+
+
+

CPU

+

Requests

+

Network

+

Latency

+
+
+
+
+

CPU Load

+

Load Average

+
+
+
+
+

Memory

+

Memory Stats

+
+
+
+
+
+
+
+
+
+
+

Devices

+

Filesystems

+
+
+
+
+
+
+
User agents
+
+
+
+
+
+
+

Status

+

Error Rate

+
+
+
+
+ + + diff --git a/config b/config new file mode 100644 index 0000000..bee93d7 --- /dev/null +++ b/config @@ -0,0 +1,21 @@ +ngx_addon_name=ngx_http_monitoring_module + +NGX_HTTP_MONITORING_SRCS="$ngx_addon_dir/src/ngx_http_monitoring_module.c \ + $ngx_addon_dir/src/ngx_http_monitoring_shm.c \ + $ngx_addon_dir/src/ngx_http_monitoring_collectors.c \ + $ngx_addon_dir/src/ngx_http_monitoring_json.c \ + $ngx_addon_dir/src/ngx_http_monitoring_dashboard.c \ + $ngx_addon_dir/src/ngx_http_monitoring_sse.c" + +if test -n "$ngx_module_link"; then + ngx_module_type=HTTP + ngx_module_name=ngx_http_monitoring_module + ngx_module_incs="$ngx_addon_dir/src" + ngx_module_srcs="$NGX_HTTP_MONITORING_SRCS" + ngx_module_libs= + . auto/module +else + HTTP_MODULES="$HTTP_MODULES ngx_http_monitoring_module" + NGX_ADDON_SRCS="$NGX_ADDON_SRCS $NGX_HTTP_MONITORING_SRCS" + CORE_INCS="$CORE_INCS $ngx_addon_dir/src" +fi diff --git a/dockerized/Dockerfile b/dockerized/Dockerfile new file mode 100644 index 0000000..b59a117 --- /dev/null +++ b/dockerized/Dockerfile @@ -0,0 +1,90 @@ +ARG DEBIAN_VERSION=bookworm +ARG NGINX_VERSION=1.28.0 + +FROM debian:${DEBIAN_VERSION}-slim AS builder + +ARG NGINX_VERSION +ARG NGINX_CONFIGURE_ARGS="--with-compat --with-http_ssl_module --with-http_stub_status_module --with-http_realip_module --with-http_v2_module --with-http_gzip_static_module" + +RUN apt-get update \ + && apt-get install -y --no-install-recommends \ + build-essential \ + ca-certificates \ + curl \ + libpcre2-dev \ + libssl-dev \ + zlib1g-dev \ + && rm -rf /var/lib/apt/lists/* + +WORKDIR /usr/src + +RUN curl -fsSLO "https://nginx.org/download/nginx-${NGINX_VERSION}.tar.gz" \ + && tar -xzf "nginx-${NGINX_VERSION}.tar.gz" \ + && rm "nginx-${NGINX_VERSION}.tar.gz" + +COPY . /usr/src/ngx_http_monitoring_module + +WORKDIR /usr/src/nginx-${NGINX_VERSION} + +RUN ./configure \ + --prefix=/usr/local/nginx \ + --sbin-path=/usr/local/sbin/nginx \ + --modules-path=/usr/local/nginx/modules \ + --conf-path=/etc/nginx/nginx.conf \ + --error-log-path=/var/log/nginx/error.log \ + --http-log-path=/var/log/nginx/access.log \ + --pid-path=/var/run/nginx.pid \ + --lock-path=/var/run/nginx.lock \ + --http-client-body-temp-path=/var/cache/nginx/client_temp \ + --http-proxy-temp-path=/var/cache/nginx/proxy_temp \ + --http-fastcgi-temp-path=/var/cache/nginx/fastcgi_temp \ + --http-uwsgi-temp-path=/var/cache/nginx/uwsgi_temp \ + --http-scgi-temp-path=/var/cache/nginx/scgi_temp \ + ${NGINX_CONFIGURE_ARGS} \ + --add-dynamic-module=/usr/src/ngx_http_monitoring_module \ + && make -j"$(nproc)" \ + && make install \ + && mkdir -p /usr/local/nginx/modules \ + && cp objs/ngx_http_monitoring_module.so /usr/local/nginx/modules/ + + +FROM debian:${DEBIAN_VERSION}-slim AS runtime + +ARG NGINX_VERSION + +LABEL org.opencontainers.image.title="ngx_http_monitoring_module nginx image" \ + org.opencontainers.image.description="Nginx built with ngx_http_monitoring_module and embedded monitoring dashboard" \ + org.opencontainers.image.version="${NGINX_VERSION}" + +RUN apt-get update \ + && apt-get install -y --no-install-recommends \ + ca-certificates \ + libpcre2-8-0 \ + libssl3 \ + zlib1g \ + && rm -rf /var/lib/apt/lists/* \ + && groupadd --system --gid 101 nginx \ + && useradd --system --gid nginx --no-create-home --home /nonexistent \ + --shell /usr/sbin/nologin --uid 101 nginx \ + && mkdir -p /var/cache/nginx/client_temp \ + /var/cache/nginx/proxy_temp \ + /var/cache/nginx/fastcgi_temp \ + /var/cache/nginx/uwsgi_temp \ + /var/cache/nginx/scgi_temp \ + /var/log/nginx \ + /etc/nginx/conf.d \ + && chown -R nginx:nginx /var/cache/nginx /var/log/nginx + +COPY --from=builder /usr/local/nginx /usr/local/nginx +COPY --from=builder /usr/local/sbin/nginx /usr/local/sbin/nginx +COPY dockerized/nginx.conf /etc/nginx/nginx.conf + +EXPOSE 8080 + +STOPSIGNAL SIGQUIT + +HEALTHCHECK --interval=10s --timeout=3s --start-period=5s --retries=3 \ + CMD /usr/local/sbin/nginx -t -q && \ + test -r /usr/local/nginx/modules/ngx_http_monitoring_module.so + +CMD ["/usr/local/sbin/nginx", "-g", "daemon off;"] diff --git a/dockerized/README.md b/dockerized/README.md new file mode 100644 index 0000000..bca0cd9 --- /dev/null +++ b/dockerized/README.md @@ -0,0 +1,67 @@ +# Dockerized Nginx Image + +This folder builds an Nginx image with `ngx_http_monitoring_module` compiled and loaded as a dynamic module. + +## Build + +From the repository root: + +```sh +docker build -f dockerized/Dockerfile -t ngx-http-monitoring-module:local . +``` + +Or: + +```sh +sh dockerized/build.sh +``` + +To choose a different Nginx version: + +```sh +NGINX_VERSION=1.28.0 sh dockerized/build.sh +``` + +## Run + +```sh +docker run --rm -p 8080:8080 ngx-http-monitoring-module:local +``` + +Open: + +```text +http://127.0.0.1:8080/monitor +``` + +## Docker Compose + +```sh +cd dockerized +docker compose up --build +``` + +## Smoke Test + +With the container running: + +```sh +sh dockerized/test.sh +``` + +## Image Layout + +- Nginx binary: `/usr/local/sbin/nginx` +- Module: `/usr/local/nginx/modules/ngx_http_monitoring_module.so` +- Config: `/etc/nginx/nginx.conf` +- Listen port: `8080` + +The supplied `nginx.conf` enables: + +- `/monitor` +- `/monitor/api` +- `/monitor/live` +- `/monitor/metrics` +- `/monitor/health` + +This demo config allows access from any client. Restrict `location /monitor` before exposing the container outside a trusted network. diff --git a/dockerized/build.sh b/dockerized/build.sh new file mode 100644 index 0000000..5907f86 --- /dev/null +++ b/dockerized/build.sh @@ -0,0 +1,15 @@ +#!/usr/bin/env sh +set -eu + +IMAGE="${IMAGE:-ngx-http-monitoring-module:local}" +NGINX_VERSION="${NGINX_VERSION:-1.28.0}" + +cd "$(dirname "$0")/.." + +docker build \ + --build-arg "NGINX_VERSION=$NGINX_VERSION" \ + -f dockerized/Dockerfile \ + -t "$IMAGE" \ + . + +echo "Built $IMAGE" diff --git a/dockerized/docker-compose.yml b/dockerized/docker-compose.yml new file mode 100644 index 0000000..38a5c96 --- /dev/null +++ b/dockerized/docker-compose.yml @@ -0,0 +1,12 @@ +services: + nginx-monitor: + build: + context: .. + dockerfile: dockerized/Dockerfile + args: + NGINX_VERSION: "1.28.0" + image: ngx-http-monitoring-module:local + container_name: ngx-http-monitoring-module + ports: + - "8080:8080" + restart: unless-stopped diff --git a/dockerized/nginx.conf b/dockerized/nginx.conf new file mode 100644 index 0000000..889b35d --- /dev/null +++ b/dockerized/nginx.conf @@ -0,0 +1,57 @@ +load_module /usr/local/nginx/modules/ngx_http_monitoring_module.so; + +user nginx; +worker_processes auto; + +error_log /var/log/nginx/error.log info; +pid /var/run/nginx.pid; + +events { + worker_connections 65535; + multi_accept on; +} + +http { + default_type application/octet-stream; + sendfile on; + tcp_nopush on; + tcp_nodelay on; + keepalive_timeout 65; + server_tokens off; + + access_log /var/log/nginx/access.log; + + monitor_refresh_interval 1s; + monitor_history 5m; + monitor_resolution 1s; + monitor_shm_size 8m; + monitor_collect_system on; + monitor_collect_nginx on; + monitor_collect_network on; + monitor_access_log on; + monitor_max_top_urls 100; + + server { + listen 8080; + server_name _; + + location /monitor { + monitor on; + monitor_dashboard on; + monitor_api on; + monitor_sse on; + monitor_rate_limit 240; + monitor_cors "*"; + } + + location = / { + default_type text/plain; + return 200 "ngx_http_monitoring_module docker image\nopen /monitor for the dashboard\n"; + } + + location = /workload { + default_type application/json; + return 200 "{\"status\":\"ok\",\"source\":\"dockerized-nginx\"}\n"; + } + } +} diff --git a/dockerized/run.sh b/dockerized/run.sh new file mode 100644 index 0000000..3de6294 --- /dev/null +++ b/dockerized/run.sh @@ -0,0 +1,13 @@ +#!/usr/bin/env sh +set -eu + +IMAGE="${IMAGE:-ngx-http-monitoring-module:local}" +NAME="${NAME:-ngx-http-monitoring-module}" +PORT="${PORT:-8080}" + +docker rm -f "$NAME" >/dev/null 2>&1 || true + +docker run \ + --name "$NAME" \ + -p "$PORT:8080" \ + "$IMAGE" diff --git a/dockerized/test.sh b/dockerized/test.sh new file mode 100644 index 0000000..d1973a9 --- /dev/null +++ b/dockerized/test.sh @@ -0,0 +1,22 @@ +#!/usr/bin/env sh +set -eu + +BASE_URL="${BASE_URL:-http://127.0.0.1:8080}" + +curl -fsS "$BASE_URL/" >/dev/null +curl -fsS "$BASE_URL/monitor" >/dev/null +curl -fsS "$BASE_URL/monitor/api" >/dev/null +curl -fsS "$BASE_URL/monitor/api/system" >/dev/null +curl -fsS "$BASE_URL/monitor/api/nginx" >/dev/null +curl -fsS "$BASE_URL/monitor/api/network" >/dev/null +curl -fsS "$BASE_URL/monitor/api/disk" >/dev/null +curl -fsS "$BASE_URL/monitor/api/processes" >/dev/null +curl -fsS "$BASE_URL/monitor/api/upstreams" >/dev/null +curl -fsS "$BASE_URL/monitor/api/connections" >/dev/null +curl -fsS "$BASE_URL/monitor/api/requests" >/dev/null +curl -fsS "$BASE_URL/monitor/metrics" >/dev/null +curl -fsS "$BASE_URL/monitor/health" >/dev/null + +timeout 3 curl -fsS -N "$BASE_URL/monitor/live" >/dev/null || true + +echo "Dockerized module endpoints are reachable at $BASE_URL" diff --git a/docs/API.md b/docs/API.md new file mode 100644 index 0000000..fa40bd5 --- /dev/null +++ b/docs/API.md @@ -0,0 +1,92 @@ +# API + +All JSON responses are compact, versioned, and timestamped. + +```json +{ + "version": 1, + "module": "1.0.0", + "timestamp": 1710000000, + "scope": "full" +} +``` + +## Authentication + +If `monitor_api_token` is set, non-dashboard endpoints require one of: + +- `Authorization: Bearer ` +- `X-Monitor-Token: ` +- `?token=` + +If `monitor_basic_auth` is set to `user:password`, every monitoring endpoint requires HTTP Basic auth. For public production deployments, prefer terminating authentication at Nginx with `auth_basic` and `auth_basic_user_file`. + +## JSON Endpoints + +### `/monitor/api` + +Returns all sections: + +- `system` +- `nginx` +- `network` +- `disk` +- `processes` +- `upstreams` +- `connections` +- `requests` +- `history` + +### `/monitor/api/system` + +Includes: + +- `cpu.usage` +- `cpu.cores` +- `cpu.load` +- `memory.total` +- `memory.available` +- `memory.used` +- `memory.used_pct` +- `swap.total` +- `swap.free` +- `swap.used_pct` +- `uptime` + +### `/monitor/api/requests` + +Includes: + +- total request and response counters +- request and response moving averages +- status counters +- method distribution +- response size histogram +- latency histogram +- latency percentiles `p50`, `p90`, `p95`, `p99` +- `top_urls` +- `user_agents` + +### `/monitor/live` + +SSE stream: + +```text +retry: 3000 +: heartbeat 42 +id: 42 +event: metrics +data: {"version":1,"timestamp":1710000000,...} +``` + +The dashboard uses SSE first and falls back to polling `/monitor/api`. + +### `/monitor/metrics` + +Prometheus text format with core counters and gauges: + +- `nginx_monitor_requests_total` +- `nginx_monitor_active_connections` +- `nginx_monitor_cpu_usage_ratio` +- `nginx_monitor_memory_used_ratio` +- `nginx_monitor_latency_p95_ms` diff --git a/docs/ARCHITECTURE.md b/docs/ARCHITECTURE.md new file mode 100644 index 0000000..5eca9e2 --- /dev/null +++ b/docs/ARCHITECTURE.md @@ -0,0 +1,60 @@ +# Architecture + +## Request Path + +The module registers two HTTP phase handlers: + +- content phase: handles `/monitor`, `/monitor/api*`, `/monitor/live`, `/monitor/metrics`, and `/monitor/health` +- log phase: accounts completed application requests + +Monitoring endpoints are excluded from request accounting to avoid dashboard polling skewing the service metrics. + +## Shared Memory + +The module allocates one shared memory zone named `ngx_http_monitoring`. The zone contains: + +- global request, response, connection, and system counters +- fixed-size top URL and user-agent tables +- fixed-size upstream peer table +- per-worker metric slots keyed by `ngx_process_slot` +- fixed-size historical ring buffer +- collector lock and API rate-limit counters + +Hot request-path updates use Nginx atomic operations. The shared slab mutex is used only for rare top-N slot creation or replacement and rate-limit window resets. + +## Collection + +Each worker owns a timer, but collection is guarded by an atomic shared lock, so only one worker performs a collection pass at a time. Collection reads: + +- `/proc/stat` +- `/proc/loadavg` +- `/proc/meminfo` +- `/proc/net/dev` +- `/proc/diskstats` +- `/proc/uptime` +- `/proc/net/tcp` +- `/proc/net/tcp6` +- `/proc/net/sockstat` +- `/proc/mounts` +- `/proc/self/status` +- `/proc` +- `statvfs()` +- `getifaddrs()` + +The collector computes deltas and moving averages, then appends a sample to the ring buffer according to `monitor_resolution`. + +## Serialization + +JSON and Prometheus responses are generated from shared-memory snapshots only. API handlers do not read `/proc`, do not allocate persistent state, and use one request-pool buffer per response. + +## SSE + +`/monitor/live` is a long-lived request. Each client owns two reusable output buffers and a timer. The writer skips a tick if the connection still has pending buffered output, preventing slow clients from accumulating unbounded memory. + +## Upstreams + +Upstream metrics are passively observed from Nginx upstream state recorded on proxied requests. This avoids active backend probing in workers. Failures are counted from zero or `5xx` upstream statuses. + +## Compatibility + +The module is designed as an Nginx dynamic HTTP module and should be built with `--with-compat` against the target Nginx source tree. Stub-status counters are compiled in when the target Nginx build includes `--with-http_stub_status_module`; otherwise request counters still work but low-level active/reading/writing/waiting counters remain zero. diff --git a/docs/CONFIGURATION.md b/docs/CONFIGURATION.md new file mode 100644 index 0000000..d541e8b --- /dev/null +++ b/docs/CONFIGURATION.md @@ -0,0 +1,51 @@ +# Configuration + +## Core + +```nginx +monitor on | off; +monitor_dashboard on | off; +monitor_api on | off; +monitor_sse on | off; +``` + +`monitor` can be used at `http`, `server`, or `location` scope. A prefix location is recommended: + +```nginx +location /monitor { + monitor on; +} +``` + +## Collection + +```nginx +monitor_refresh_interval 1s; +monitor_history 5m; +monitor_resolution 1s; +monitor_shm_size 8m; +monitor_collect_system on; +monitor_collect_nginx on; +monitor_collect_network on; +monitor_access_log on; +monitor_max_top_urls 100; +``` + +`monitor_history / monitor_resolution` is capped by the compiled maximum of 3600 samples. + +## Security + +```nginx +monitor_allow 10.0.0.0/8; +monitor_allow 127.0.0.1/32; +monitor_deny all; + +monitor_basic_auth off; +monitor_api_token "change-me"; +monitor_rate_limit 120; +monitor_cors off; +``` + +ACL rules are evaluated in order. `monitor_basic_auth` accepts `off` or a literal `user:password`. For strong password storage, use Nginx `auth_basic` in the same location instead. + +`monitor_cors` accepts `off`, `*`, or a literal origin such as `https://ops.example.com`. diff --git a/docs/PERFORMANCE.md b/docs/PERFORMANCE.md new file mode 100644 index 0000000..3d64421 --- /dev/null +++ b/docs/PERFORMANCE.md @@ -0,0 +1,48 @@ +# Performance + +## Hot Path + +The log-phase request accounting path performs: + +- one timestamp delta calculation +- atomic increments for request, status, method, latency, and size buckets +- one bounded top-URL table update +- optional user-agent and upstream updates + +Top-N table insertion or replacement takes the shared slab mutex, but existing entries update with atomics only. + +## Collector Cost + +Collectors run from Nginx timers at `monitor_refresh_interval`. API requests do not trigger collection. Default collection is once per second. + +For very large hosts, `/proc/net/tcp` and `/proc/net/tcp6` can be expensive because they scale with connection count. If needed, increase `monitor_refresh_interval` or disable system collection: + +```nginx +monitor_collect_system off; +``` + +## Memory + +Default shared memory is 8 MiB: + +```nginx +monitor_shm_size 8m; +``` + +The default fixed-size structures are bounded: + +- history samples: 3600 maximum +- top URLs: 256 maximum +- user agents: 64 maximum +- upstream peers: 128 maximum +- workers: 128 maximum +- interfaces, disks, filesystems: 32 each + +## Operational Guidance + +- Build with `--with-compat`. +- Build with `--with-http_stub_status_module` for exact Nginx connection counters. +- Keep dashboard access private with `monitor_allow` / `monitor_deny` or standard Nginx auth modules. +- Use `monitor_api_token` when exposing API or SSE across trust boundaries. +- Put `/monitor` behind an internal listener or VPN for production. +- Keep `monitor_resolution` at `1s` or higher on very busy systems. diff --git a/examples/nginx.conf b/examples/nginx.conf new file mode 100644 index 0000000..fbbb5b6 --- /dev/null +++ b/examples/nginx.conf @@ -0,0 +1,59 @@ +load_module modules/ngx_http_monitoring_module.so; + +worker_processes auto; + +events { + worker_connections 65535; +} + +http { + sendfile on; + keepalive_timeout 65; + + monitor_refresh_interval 1s; + monitor_history 5m; + monitor_resolution 1s; + monitor_shm_size 8m; + + monitor_collect_system on; + monitor_collect_nginx on; + monitor_collect_network on; + monitor_access_log on; + monitor_max_top_urls 100; + + upstream demo_backend { + server 127.0.0.1:9000 max_fails=3 fail_timeout=10s; + keepalive 32; + } + + server { + listen 8080; + server_name localhost; + + location /monitor { + monitor on; + monitor_dashboard on; + monitor_api on; + monitor_sse on; + + monitor_allow 127.0.0.1/32; + monitor_allow ::1/128; + monitor_deny all; + + monitor_basic_auth off; + monitor_api_token ""; + monitor_cors off; + monitor_rate_limit 120; + } + + location /proxy { + proxy_pass http://demo_backend; + proxy_http_version 1.1; + proxy_set_header Connection ""; + } + + location / { + return 200 "ngx_http_monitoring_module example\n"; + } + } +} diff --git a/scripts/build.sh b/scripts/build.sh new file mode 100644 index 0000000..0516675 --- /dev/null +++ b/scripts/build.sh @@ -0,0 +1,18 @@ +#!/usr/bin/env sh +set -eu + +if [ "${NGINX_SRC:-}" = "" ]; then + echo "NGINX_SRC is required, for example: NGINX_SRC=/usr/local/src/nginx-1.24.0 ./scripts/build.sh" >&2 + exit 1 +fi + +NGINX_CONFIGURE_ARGS="${NGINX_CONFIGURE_ARGS:---with-compat --with-http_ssl_module --with-http_stub_status_module}" +BUILD_DIR="${BUILD_DIR:-$(pwd)/build}" + +cd "$NGINX_SRC" +./configure $NGINX_CONFIGURE_ARGS --add-dynamic-module="$(cd -P "$(dirname "$0")/.." && pwd)" +make modules + +mkdir -p "$BUILD_DIR" +cp "$NGINX_SRC/objs/ngx_http_monitoring_module.so" "$BUILD_DIR/" +echo "Built $BUILD_DIR/ngx_http_monitoring_module.so" diff --git a/scripts/smoke.sh b/scripts/smoke.sh new file mode 100644 index 0000000..a96ab13 --- /dev/null +++ b/scripts/smoke.sh @@ -0,0 +1,26 @@ +#!/usr/bin/env sh +set -eu + +BASE_URL="${BASE_URL:-http://127.0.0.1:8080}" +TOKEN="${TOKEN:-}" + +auth_arg="" +if [ "$TOKEN" != "" ]; then + auth_arg="?token=$TOKEN" +fi + +curl -fsS "$BASE_URL/monitor" >/dev/null +curl -fsS "$BASE_URL/monitor/api$auth_arg" >/dev/null +curl -fsS "$BASE_URL/monitor/api/system$auth_arg" >/dev/null +curl -fsS "$BASE_URL/monitor/api/nginx$auth_arg" >/dev/null +curl -fsS "$BASE_URL/monitor/api/network$auth_arg" >/dev/null +curl -fsS "$BASE_URL/monitor/api/disk$auth_arg" >/dev/null +curl -fsS "$BASE_URL/monitor/api/processes$auth_arg" >/dev/null +curl -fsS "$BASE_URL/monitor/api/upstreams$auth_arg" >/dev/null +curl -fsS "$BASE_URL/monitor/api/connections$auth_arg" >/dev/null +curl -fsS "$BASE_URL/monitor/api/requests$auth_arg" >/dev/null +curl -fsS "$BASE_URL/monitor/metrics$auth_arg" >/dev/null +curl -fsS "$BASE_URL/monitor/health$auth_arg" >/dev/null + +timeout 3 curl -fsS -N "$BASE_URL/monitor/live$auth_arg" >/dev/null || true +echo "smoke checks completed" diff --git a/src/ngx_http_monitoring.h b/src/ngx_http_monitoring.h new file mode 100644 index 0000000..0ad1ba4 --- /dev/null +++ b/src/ngx_http_monitoring.h @@ -0,0 +1,304 @@ +#ifndef _NGX_HTTP_MONITORING_H_INCLUDED_ +#define _NGX_HTTP_MONITORING_H_INCLUDED_ + +#include +#include +#include + +#include +#include + +#define NGX_HTTP_MONITORING_VERSION 1 +#define NGX_HTTP_MONITORING_MODULE_VERSION "1.0.0" + +#define NGX_HTTP_MONITORING_DEFAULT_SHM_SIZE (8 * 1024 * 1024) +#define NGX_HTTP_MONITORING_MIN_INTERVAL 100 +#define NGX_HTTP_MONITORING_HISTORY_MAX 3600 +#define NGX_HTTP_MONITORING_TOP_URLS_MAX 256 +#define NGX_HTTP_MONITORING_TOP_UA_MAX 64 +#define NGX_HTTP_MONITORING_UPSTREAMS_MAX 128 +#define NGX_HTTP_MONITORING_WORKERS_MAX 128 +#define NGX_HTTP_MONITORING_IFACES_MAX 32 +#define NGX_HTTP_MONITORING_DISKS_MAX 32 +#define NGX_HTTP_MONITORING_FILESYSTEMS_MAX 32 +#define NGX_HTTP_MONITORING_LATENCY_BUCKETS 14 +#define NGX_HTTP_MONITORING_SIZE_BUCKETS 12 +#define NGX_HTTP_MONITORING_METHODS 8 + +#define NGX_HTTP_MONITORING_KEY_LEN 128 +#define NGX_HTTP_MONITORING_NAME_LEN 64 + +typedef enum { + NGX_HTTP_MONITORING_EP_NONE = 0, + NGX_HTTP_MONITORING_EP_DASHBOARD, + NGX_HTTP_MONITORING_EP_API_FULL, + NGX_HTTP_MONITORING_EP_API_SYSTEM, + NGX_HTTP_MONITORING_EP_API_NGINX, + NGX_HTTP_MONITORING_EP_API_NETWORK, + NGX_HTTP_MONITORING_EP_API_DISK, + NGX_HTTP_MONITORING_EP_API_PROCESSES, + NGX_HTTP_MONITORING_EP_API_UPSTREAMS, + NGX_HTTP_MONITORING_EP_API_CONNECTIONS, + NGX_HTTP_MONITORING_EP_API_REQUESTS, + NGX_HTTP_MONITORING_EP_LIVE, + NGX_HTTP_MONITORING_EP_PROMETHEUS, + NGX_HTTP_MONITORING_EP_HEALTH +} ngx_http_monitoring_endpoint_e; + +typedef enum { + NGX_HTTP_MONITORING_METHOD_GET = 0, + NGX_HTTP_MONITORING_METHOD_POST, + NGX_HTTP_MONITORING_METHOD_PUT, + NGX_HTTP_MONITORING_METHOD_DELETE, + NGX_HTTP_MONITORING_METHOD_HEAD, + NGX_HTTP_MONITORING_METHOD_OPTIONS, + NGX_HTTP_MONITORING_METHOD_PATCH, + NGX_HTTP_MONITORING_METHOD_OTHER +} ngx_http_monitoring_method_e; + +typedef struct { + ngx_uint_t deny; + ngx_cidr_t cidr; +} ngx_http_monitoring_access_rule_t; + +typedef struct { + ngx_msec_t refresh_interval; + ngx_msec_t history; + ngx_msec_t resolution; + size_t shm_size; + ngx_uint_t max_top_urls; + ngx_flag_t collect_system; + ngx_flag_t collect_nginx; + ngx_flag_t collect_network; + ngx_flag_t access_log; + + ngx_shm_zone_t *shm_zone; + ngx_event_t collector_event; + ngx_uint_t collector_active; + void *sh; + ngx_slab_pool_t *shpool; +} ngx_http_monitoring_main_conf_t; + +typedef struct { + ngx_flag_t enabled; + ngx_flag_t dashboard; + ngx_flag_t api; + ngx_flag_t sse; + ngx_str_t api_token; + ngx_str_t basic_auth; + ngx_str_t cors; + ngx_uint_t rate_limit; + ngx_array_t *access_rules; +} ngx_http_monitoring_loc_conf_t; + +typedef struct { + time_t timestamp; + ngx_msec_t msec; + double cpu_usage; + double load1; + double load5; + double load15; + double memory_used_pct; + double swap_used_pct; + double requests_per_sec; + double responses_per_sec; + double network_rx_per_sec; + double network_tx_per_sec; + double disk_read_per_sec; + double disk_write_per_sec; + double latency_p95; + ngx_atomic_uint_t requests_total; + ngx_atomic_uint_t status_4xx; + ngx_atomic_uint_t status_5xx; +} ngx_http_monitoring_history_sample_t; + +typedef struct { + ngx_atomic_t hits; + ngx_atomic_t errors; + ngx_atomic_t bytes; + ngx_atomic_t latency_ms_total; + ngx_atomic_t last_seen; + uint32_t hash; + u_char key[NGX_HTTP_MONITORING_KEY_LEN]; +} ngx_http_monitoring_top_entry_t; + +typedef struct { + ngx_atomic_t requests; + ngx_atomic_t failures; + ngx_atomic_t latency_ms_total; + ngx_atomic_t status_4xx; + ngx_atomic_t status_5xx; + ngx_atomic_t last_seen; + uint32_t hash; + u_char peer[NGX_HTTP_MONITORING_KEY_LEN]; +} ngx_http_monitoring_upstream_entry_t; + +typedef struct { + ngx_atomic_t pid; + ngx_atomic_t requests; + ngx_atomic_t bytes; + ngx_atomic_t last_seen; + ngx_atomic_t active; + ngx_atomic_t errors; + ngx_atomic_t vm_size; + ngx_atomic_t vm_rss; + ngx_atomic_t voluntary_ctxt; + ngx_atomic_t nonvoluntary_ctxt; +} ngx_http_monitoring_worker_metric_t; + +typedef struct { + u_char name[NGX_HTTP_MONITORING_NAME_LEN]; + ngx_atomic_t rx_bytes; + ngx_atomic_t tx_bytes; + ngx_atomic_t rx_packets; + ngx_atomic_t tx_packets; + ngx_atomic_t rx_errors; + ngx_atomic_t tx_errors; + ngx_atomic_t flags; +} ngx_http_monitoring_iface_metric_t; + +typedef struct { + u_char name[NGX_HTTP_MONITORING_NAME_LEN]; + ngx_atomic_t reads; + ngx_atomic_t writes; + ngx_atomic_t read_bytes; + ngx_atomic_t write_bytes; + ngx_atomic_t io_ms; +} ngx_http_monitoring_disk_metric_t; + +typedef struct { + u_char path[NGX_HTTP_MONITORING_KEY_LEN]; + u_char type[NGX_HTTP_MONITORING_NAME_LEN]; + ngx_atomic_t total; + ngx_atomic_t used; + ngx_atomic_t free; + ngx_atomic_t avail; + ngx_atomic_t files; + ngx_atomic_t files_free; +} ngx_http_monitoring_fs_metric_t; + +typedef struct { + ngx_atomic_t usage_milli; + ngx_atomic_t cores; + ngx_atomic_t load1_milli; + ngx_atomic_t load5_milli; + ngx_atomic_t load15_milli; + ngx_atomic_t mem_total; + ngx_atomic_t mem_available; + ngx_atomic_t mem_free; + ngx_atomic_t swap_total; + ngx_atomic_t swap_free; + ngx_atomic_t uptime; + ngx_atomic_t process_count; + ngx_atomic_t tcp_established; + ngx_atomic_t tcp_listen; + ngx_atomic_t sockets_used; + ngx_atomic_t sockets_tcp; + ngx_atomic_t sockets_udp; +} ngx_http_monitoring_system_metric_t; + +typedef struct { + ngx_atomic_t total; + ngx_atomic_t accepted; + ngx_atomic_t handled; + ngx_atomic_t active; + ngx_atomic_t reading; + ngx_atomic_t writing; + ngx_atomic_t waiting; + ngx_atomic_t ssl_requests; + ngx_atomic_t ssl_handshakes; + ngx_atomic_t keepalive_requests; + ngx_atomic_t sse_clients; + ngx_atomic_t sse_events; + ngx_atomic_t rate_limited; +} ngx_http_monitoring_connection_metric_t; + +typedef struct { + ngx_atomic_t total; + ngx_atomic_t responses; + ngx_atomic_t bytes; + ngx_atomic_t request_time_ms_total; + ngx_atomic_t status_1xx; + ngx_atomic_t status_2xx; + ngx_atomic_t status_3xx; + ngx_atomic_t status_4xx; + ngx_atomic_t status_5xx; + ngx_atomic_t method[NGX_HTTP_MONITORING_METHODS]; + ngx_atomic_t latency_bucket[NGX_HTTP_MONITORING_LATENCY_BUCKETS]; + ngx_atomic_t size_bucket[NGX_HTTP_MONITORING_SIZE_BUCKETS]; +} ngx_http_monitoring_request_metric_t; + +typedef struct { + ngx_atomic_t collector_lock; + ngx_atomic_t rate_limit_window; + ngx_atomic_t rate_limit_count; + ngx_atomic_t history_write; + ngx_atomic_t history_count; + ngx_atomic_t history_capacity; + ngx_atomic_t generation; + ngx_atomic_t initialized; + + ngx_http_monitoring_system_metric_t system; + ngx_http_monitoring_connection_metric_t connections; + ngx_http_monitoring_request_metric_t requests; + + ngx_atomic_t iface_count; + ngx_http_monitoring_iface_metric_t ifaces[NGX_HTTP_MONITORING_IFACES_MAX]; + + ngx_atomic_t disk_count; + ngx_http_monitoring_disk_metric_t disks[NGX_HTTP_MONITORING_DISKS_MAX]; + + ngx_atomic_t fs_count; + ngx_http_monitoring_fs_metric_t filesystems[NGX_HTTP_MONITORING_FILESYSTEMS_MAX]; + + ngx_http_monitoring_worker_metric_t workers[NGX_HTTP_MONITORING_WORKERS_MAX]; + ngx_http_monitoring_top_entry_t urls[NGX_HTTP_MONITORING_TOP_URLS_MAX]; + ngx_http_monitoring_top_entry_t user_agents[NGX_HTTP_MONITORING_TOP_UA_MAX]; + ngx_http_monitoring_upstream_entry_t upstreams[NGX_HTTP_MONITORING_UPSTREAMS_MAX]; + + ngx_http_monitoring_history_sample_t history[NGX_HTTP_MONITORING_HISTORY_MAX]; + + ngx_atomic_t prev_req_total; + ngx_atomic_t prev_resp_total; + ngx_atomic_t prev_net_rx; + ngx_atomic_t prev_net_tx; + ngx_atomic_t prev_disk_read; + ngx_atomic_t prev_disk_write; + ngx_atomic_t prev_collect_msec; + ngx_atomic_t prev_history_msec; + ngx_atomic_t ewma_rps_milli; + ngx_atomic_t ewma_resp_milli; + + uint64_t prev_cpu_total; + uint64_t prev_cpu_idle; +} ngx_http_monitoring_shctx_t; + +extern ngx_module_t ngx_http_monitoring_module; + +ngx_int_t ngx_http_monitoring_init_shm(ngx_shm_zone_t *shm_zone, void *data); +ngx_int_t ngx_http_monitoring_init_collector(ngx_cycle_t *cycle); +void ngx_http_monitoring_collect(ngx_event_t *ev); +void ngx_http_monitoring_account_request(ngx_http_request_t *r); +void ngx_http_monitoring_account_upstream(ngx_http_request_t *r, ngx_msec_t latency); + +ngx_http_monitoring_endpoint_e ngx_http_monitoring_match_endpoint(ngx_http_request_t *r); +ngx_int_t ngx_http_monitoring_send_json(ngx_http_request_t *r, + ngx_http_monitoring_endpoint_e endpoint); +ngx_int_t ngx_http_monitoring_send_dashboard(ngx_http_request_t *r); +ngx_int_t ngx_http_monitoring_send_sse(ngx_http_request_t *r); +ngx_int_t ngx_http_monitoring_send_prometheus(ngx_http_request_t *r); +ngx_int_t ngx_http_monitoring_send_health(ngx_http_request_t *r); + +ngx_http_monitoring_shctx_t *ngx_http_monitoring_get_shctx(ngx_http_request_t *r); +ngx_http_monitoring_main_conf_t *ngx_http_monitoring_get_main_conf(ngx_http_request_t *r); +ngx_uint_t ngx_http_monitoring_method_index(ngx_uint_t method); +ngx_uint_t ngx_http_monitoring_latency_bucket(ngx_msec_t ms); +ngx_uint_t ngx_http_monitoring_size_bucket(off_t bytes); +double ngx_http_monitoring_percentile(ngx_http_monitoring_shctx_t *sh, + double percentile); +uint32_t ngx_http_monitoring_hash(ngx_str_t *value); +void ngx_http_monitoring_update_top(ngx_slab_pool_t *shpool, + ngx_http_monitoring_top_entry_t *entries, ngx_uint_t capacity, + ngx_str_t *key, ngx_msec_t latency, off_t bytes, ngx_uint_t is_error); + +#endif /* _NGX_HTTP_MONITORING_H_INCLUDED_ */ diff --git a/src/ngx_http_monitoring_collectors.c b/src/ngx_http_monitoring_collectors.c new file mode 100644 index 0000000..5f7e1d8 --- /dev/null +++ b/src/ngx_http_monitoring_collectors.c @@ -0,0 +1,841 @@ +#include "ngx_http_monitoring.h" + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#ifndef O_CLOEXEC +#define O_CLOEXEC 0 +#endif + +static ssize_t ngx_http_monitoring_read_file(const char *path, u_char *buf, + size_t size); +static void ngx_http_monitoring_collect_cpu(ngx_http_monitoring_shctx_t *sh); +static void ngx_http_monitoring_collect_load(ngx_http_monitoring_shctx_t *sh); +static void ngx_http_monitoring_collect_meminfo(ngx_http_monitoring_shctx_t *sh); +static void ngx_http_monitoring_collect_uptime(ngx_http_monitoring_shctx_t *sh); +static void ngx_http_monitoring_collect_processes(ngx_http_monitoring_shctx_t *sh); +static void ngx_http_monitoring_collect_self_status( + ngx_http_monitoring_shctx_t *sh); +static void ngx_http_monitoring_collect_tcp(ngx_http_monitoring_shctx_t *sh); +static void ngx_http_monitoring_collect_sockstat(ngx_http_monitoring_shctx_t *sh); +static void ngx_http_monitoring_collect_network(ngx_http_monitoring_shctx_t *sh); +static void ngx_http_monitoring_collect_diskstats(ngx_http_monitoring_shctx_t *sh); +static void ngx_http_monitoring_collect_filesystems(ngx_http_monitoring_shctx_t *sh); +static void ngx_http_monitoring_collect_nginx(ngx_http_monitoring_shctx_t *sh); +static void ngx_http_monitoring_collect_history( + ngx_http_monitoring_main_conf_t *mmcf, ngx_http_monitoring_shctx_t *sh); +static ngx_atomic_uint_t ngx_http_monitoring_sum_network( + ngx_http_monitoring_shctx_t *sh, ngx_uint_t tx); +static ngx_atomic_uint_t ngx_http_monitoring_sum_disk( + ngx_http_monitoring_shctx_t *sh, ngx_uint_t write); +static ngx_atomic_uint_t ngx_http_monitoring_delta(ngx_atomic_uint_t now, + ngx_atomic_uint_t prev); +static ngx_uint_t ngx_http_monitoring_skip_fs(const char *type); + + +ngx_int_t +ngx_http_monitoring_init_collector(ngx_cycle_t *cycle) +{ + ngx_http_monitoring_main_conf_t *mmcf; + ngx_http_monitoring_shctx_t *sh; + ngx_uint_t slot; + + if (ngx_process != NGX_PROCESS_WORKER) { + return NGX_OK; + } + + mmcf = ngx_http_cycle_get_module_main_conf(cycle, + ngx_http_monitoring_module); + if (mmcf == NULL || mmcf->sh == NULL) { + return NGX_OK; + } + + sh = (ngx_http_monitoring_shctx_t *) mmcf->sh; + + if (ngx_process_slot >= 0) { + slot = ((ngx_uint_t) ngx_process_slot) + % NGX_HTTP_MONITORING_WORKERS_MAX; + sh->workers[slot].pid = ngx_pid; + sh->workers[slot].active = 1; + sh->workers[slot].last_seen = ngx_time(); + } + + ngx_memzero(&mmcf->collector_event, sizeof(ngx_event_t)); + mmcf->collector_event.handler = ngx_http_monitoring_collect; + mmcf->collector_event.log = cycle->log; + mmcf->collector_event.data = cycle; + mmcf->collector_active = 1; + + ngx_add_timer(&mmcf->collector_event, + 50 + ((ngx_process_slot >= 0 ? ngx_process_slot : 0) % 16) + * 25); + + return NGX_OK; +} + + +void +ngx_http_monitoring_collect(ngx_event_t *ev) +{ + ngx_cycle_t *cycle; + ngx_http_monitoring_main_conf_t *mmcf; + ngx_http_monitoring_shctx_t *sh; + ngx_uint_t slot; + + cycle = ev->data; + mmcf = ngx_http_cycle_get_module_main_conf(cycle, + ngx_http_monitoring_module); + if (mmcf == NULL || mmcf->sh == NULL) { + return; + } + + sh = (ngx_http_monitoring_shctx_t *) mmcf->sh; + + if (ngx_process_slot >= 0) { + slot = ((ngx_uint_t) ngx_process_slot) + % NGX_HTTP_MONITORING_WORKERS_MAX; + sh->workers[slot].pid = ngx_pid; + sh->workers[slot].active = 1; + sh->workers[slot].last_seen = ngx_time(); + } + + ngx_http_monitoring_collect_self_status(sh); + + if (ngx_atomic_cmp_set(&sh->collector_lock, 0, (ngx_atomic_uint_t) ngx_pid)) + { + if (mmcf->collect_nginx) { + ngx_http_monitoring_collect_nginx(sh); + } + + if (mmcf->collect_system) { + ngx_http_monitoring_collect_cpu(sh); + ngx_http_monitoring_collect_load(sh); + ngx_http_monitoring_collect_meminfo(sh); + ngx_http_monitoring_collect_uptime(sh); + ngx_http_monitoring_collect_processes(sh); + ngx_http_monitoring_collect_tcp(sh); + ngx_http_monitoring_collect_sockstat(sh); + ngx_http_monitoring_collect_diskstats(sh); + ngx_http_monitoring_collect_filesystems(sh); + } + + if (mmcf->collect_network) { + ngx_http_monitoring_collect_network(sh); + } + + ngx_http_monitoring_collect_history(mmcf, sh); + + sh->generation++; + sh->collector_lock = 0; + } + + if (!ngx_exiting && mmcf->collector_active) { + ngx_add_timer(ev, mmcf->refresh_interval); + } +} + + +static ssize_t +ngx_http_monitoring_read_file(const char *path, u_char *buf, size_t size) +{ + int fd; + ssize_t n; + + if (size == 0) { + return -1; + } + + fd = open(path, O_RDONLY | O_CLOEXEC); + if (fd == -1) { + return -1; + } + + n = read(fd, buf, size - 1); + close(fd); + + if (n < 0) { + return -1; + } + + buf[n] = '\0'; + return n; +} + + +static void +ngx_http_monitoring_collect_cpu(ngx_http_monitoring_shctx_t *sh) +{ + u_char buf[32768]; + char *line, *next; + unsigned long long user, nice, system, idle, iowait, irq, softirq, steal; + uint64_t total, idle_all, delta_total, delta_idle; + ngx_uint_t cores; + + if (ngx_http_monitoring_read_file("/proc/stat", buf, sizeof(buf)) <= 0) { + return; + } + + user = nice = system = idle = iowait = irq = softirq = steal = 0; + + if (sscanf((char *) buf, "cpu %llu %llu %llu %llu %llu %llu %llu %llu", + &user, &nice, &system, &idle, &iowait, &irq, &softirq, &steal) + < 4) + { + return; + } + + total = user + nice + system + idle + iowait + irq + softirq + steal; + idle_all = idle + iowait; + + if (sh->prev_cpu_total > 0 && total > sh->prev_cpu_total) { + delta_total = total - sh->prev_cpu_total; + delta_idle = idle_all - sh->prev_cpu_idle; + if (delta_total > 0) { + sh->system.usage_milli = + (ngx_atomic_uint_t) (((delta_total - delta_idle) * 100000) + / delta_total); + } + } + + sh->prev_cpu_total = total; + sh->prev_cpu_idle = idle_all; + + cores = 0; + line = (char *) buf; + while (line != NULL && *line != '\0') { + next = strchr(line, '\n'); + if (next != NULL) { + *next++ = '\0'; + } + + if (line[0] == 'c' && line[1] == 'p' && line[2] == 'u' + && isdigit((unsigned char) line[3])) + { + cores++; + } + + line = next; + } + + if (cores > 0) { + sh->system.cores = cores; + } +} + + +static void +ngx_http_monitoring_collect_load(ngx_http_monitoring_shctx_t *sh) +{ + u_char buf[256]; + double l1, l5, l15; + + if (ngx_http_monitoring_read_file("/proc/loadavg", buf, sizeof(buf)) <= 0) { + return; + } + + if (sscanf((char *) buf, "%lf %lf %lf", &l1, &l5, &l15) != 3) { + return; + } + + sh->system.load1_milli = (ngx_atomic_uint_t) (l1 * 1000.0); + sh->system.load5_milli = (ngx_atomic_uint_t) (l5 * 1000.0); + sh->system.load15_milli = (ngx_atomic_uint_t) (l15 * 1000.0); +} + + +static void +ngx_http_monitoring_collect_meminfo(ngx_http_monitoring_shctx_t *sh) +{ + u_char buf[8192]; + char *line, *next; + unsigned long long value; + + if (ngx_http_monitoring_read_file("/proc/meminfo", buf, sizeof(buf)) <= 0) { + return; + } + + line = (char *) buf; + while (line != NULL && *line != '\0') { + next = strchr(line, '\n'); + if (next != NULL) { + *next++ = '\0'; + } + + if (sscanf(line, "MemTotal: %llu kB", &value) == 1) { + sh->system.mem_total = (ngx_atomic_uint_t) (value * 1024); + } else if (sscanf(line, "MemAvailable: %llu kB", &value) == 1) { + sh->system.mem_available = (ngx_atomic_uint_t) (value * 1024); + } else if (sscanf(line, "MemFree: %llu kB", &value) == 1) { + sh->system.mem_free = (ngx_atomic_uint_t) (value * 1024); + } else if (sscanf(line, "SwapTotal: %llu kB", &value) == 1) { + sh->system.swap_total = (ngx_atomic_uint_t) (value * 1024); + } else if (sscanf(line, "SwapFree: %llu kB", &value) == 1) { + sh->system.swap_free = (ngx_atomic_uint_t) (value * 1024); + } + + line = next; + } +} + + +static void +ngx_http_monitoring_collect_uptime(ngx_http_monitoring_shctx_t *sh) +{ + u_char buf[256]; + double uptime; + + if (ngx_http_monitoring_read_file("/proc/uptime", buf, sizeof(buf)) <= 0) { + return; + } + + if (sscanf((char *) buf, "%lf", &uptime) == 1) { + sh->system.uptime = (ngx_atomic_uint_t) uptime; + } +} + + +static void +ngx_http_monitoring_collect_processes(ngx_http_monitoring_shctx_t *sh) +{ + DIR *dir; + struct dirent *de; + ngx_uint_t count; + + dir = opendir("/proc"); + if (dir == NULL) { + return; + } + + count = 0; + while ((de = readdir(dir)) != NULL) { + if (isdigit((unsigned char) de->d_name[0])) { + count++; + } + } + + closedir(dir); + sh->system.process_count = count; +} + + +static void +ngx_http_monitoring_collect_self_status(ngx_http_monitoring_shctx_t *sh) +{ + u_char buf[8192]; + char *line, *next; + unsigned long long value; + ngx_uint_t slot; + ngx_http_monitoring_worker_metric_t *w; + + if (ngx_process_slot < 0) { + return; + } + + if (ngx_http_monitoring_read_file("/proc/self/status", buf, sizeof(buf)) + <= 0) + { + return; + } + + slot = ((ngx_uint_t) ngx_process_slot) % NGX_HTTP_MONITORING_WORKERS_MAX; + w = &sh->workers[slot]; + + line = (char *) buf; + while (line != NULL && *line != '\0') { + next = strchr(line, '\n'); + if (next != NULL) { + *next++ = '\0'; + } + + if (sscanf(line, "VmSize: %llu kB", &value) == 1) { + w->vm_size = (ngx_atomic_uint_t) (value * 1024); + } else if (sscanf(line, "VmRSS: %llu kB", &value) == 1) { + w->vm_rss = (ngx_atomic_uint_t) (value * 1024); + } else if (sscanf(line, "voluntary_ctxt_switches: %llu", &value) + == 1) + { + w->voluntary_ctxt = (ngx_atomic_uint_t) value; + } else if (sscanf(line, "nonvoluntary_ctxt_switches: %llu", &value) + == 1) + { + w->nonvoluntary_ctxt = (ngx_atomic_uint_t) value; + } + + line = next; + } +} + + +static void +ngx_http_monitoring_collect_tcp(ngx_http_monitoring_shctx_t *sh) +{ + FILE *fp; + char line[512]; + unsigned state; + ngx_uint_t established, listen; + const char *paths[2] = { "/proc/net/tcp", "/proc/net/tcp6" }; + ngx_uint_t i; + + established = 0; + listen = 0; + + for (i = 0; i < 2; i++) { + fp = fopen(paths[i], "r"); + if (fp == NULL) { + continue; + } + + (void) fgets(line, sizeof(line), fp); + + while (fgets(line, sizeof(line), fp) != NULL) { + state = 0; + if (sscanf(line, " %*u: %*64s %*64s %x", &state) == 1) { + if (state == 0x01) { + established++; + } else if (state == 0x0a) { + listen++; + } + } + } + + fclose(fp); + } + + sh->system.tcp_established = established; + sh->system.tcp_listen = listen; +} + + +static void +ngx_http_monitoring_collect_sockstat(ngx_http_monitoring_shctx_t *sh) +{ + u_char buf[4096]; + char *line, *next; + unsigned used, tcp, udp; + + if (ngx_http_monitoring_read_file("/proc/net/sockstat", buf, sizeof(buf)) + <= 0) + { + return; + } + + line = (char *) buf; + while (line != NULL && *line != '\0') { + next = strchr(line, '\n'); + if (next != NULL) { + *next++ = '\0'; + } + + if (sscanf(line, "sockets: used %u", &used) == 1) { + sh->system.sockets_used = used; + } else if (sscanf(line, "TCP: inuse %u", &tcp) == 1) { + sh->system.sockets_tcp = tcp; + } else if (sscanf(line, "UDP: inuse %u", &udp) == 1) { + sh->system.sockets_udp = udp; + } + + line = next; + } +} + + +static void +ngx_http_monitoring_collect_network(ngx_http_monitoring_shctx_t *sh) +{ + u_char buf[32768]; + char *line, *next, *colon, name[NGX_HTTP_MONITORING_NAME_LEN]; + unsigned long long rx_bytes, rx_packets, rx_errs; + unsigned long long tx_bytes, tx_packets, tx_errs; + ngx_uint_t count, i; + struct ifaddrs *ifaddr, *ifa; + + if (ngx_http_monitoring_read_file("/proc/net/dev", buf, sizeof(buf)) <= 0) { + return; + } + + count = 0; + line = (char *) buf; + + while (line != NULL && *line != '\0') { + next = strchr(line, '\n'); + if (next != NULL) { + *next++ = '\0'; + } + + colon = strchr(line, ':'); + if (colon != NULL && count < NGX_HTTP_MONITORING_IFACES_MAX) { + *colon = '\0'; + while (*line == ' ') { + line++; + } + + ngx_memzero(name, sizeof(name)); + if (sscanf(line, "%63s", name) != 1) { + line = next; + continue; + } + + rx_bytes = rx_packets = rx_errs = 0; + tx_bytes = tx_packets = tx_errs = 0; + if (sscanf(colon + 1, + "%llu %llu %llu %*u %*u %*u %*u %*u " + "%llu %llu %llu", + &rx_bytes, &rx_packets, &rx_errs, + &tx_bytes, &tx_packets, &tx_errs) == 6) + { + ngx_cpystrn(sh->ifaces[count].name, (u_char *) name, + NGX_HTTP_MONITORING_NAME_LEN); + sh->ifaces[count].rx_bytes = (ngx_atomic_uint_t) rx_bytes; + sh->ifaces[count].tx_bytes = (ngx_atomic_uint_t) tx_bytes; + sh->ifaces[count].rx_packets = (ngx_atomic_uint_t) rx_packets; + sh->ifaces[count].tx_packets = (ngx_atomic_uint_t) tx_packets; + sh->ifaces[count].rx_errors = (ngx_atomic_uint_t) rx_errs; + sh->ifaces[count].tx_errors = (ngx_atomic_uint_t) tx_errs; + sh->ifaces[count].flags = 0; + count++; + } + } + + line = next; + } + + if (getifaddrs(&ifaddr) == 0) { + for (ifa = ifaddr; ifa != NULL; ifa = ifa->ifa_next) { + if (ifa->ifa_name == NULL) { + continue; + } + + for (i = 0; i < count; i++) { + if (ngx_strncmp(sh->ifaces[i].name, ifa->ifa_name, + NGX_HTTP_MONITORING_NAME_LEN) == 0) + { + sh->ifaces[i].flags = ifa->ifa_flags; + break; + } + } + } + + freeifaddrs(ifaddr); + } + + sh->iface_count = count; +} + + +static void +ngx_http_monitoring_collect_diskstats(ngx_http_monitoring_shctx_t *sh) +{ + u_char buf[65536]; + char *line, *next, name[NGX_HTTP_MONITORING_NAME_LEN]; + unsigned major, minor; + unsigned long long reads, sectors_read, writes, sectors_written, io_ms; + ngx_uint_t count; + + if (ngx_http_monitoring_read_file("/proc/diskstats", buf, sizeof(buf)) + <= 0) + { + return; + } + + count = 0; + line = (char *) buf; + + while (line != NULL && *line != '\0' + && count < NGX_HTTP_MONITORING_DISKS_MAX) + { + next = strchr(line, '\n'); + if (next != NULL) { + *next++ = '\0'; + } + + ngx_memzero(name, sizeof(name)); + reads = sectors_read = writes = sectors_written = io_ms = 0; + + if (sscanf(line, + " %u %u %63s %llu %*u %llu %*u %llu %*u %llu %*u %*u %llu", + &major, &minor, name, &reads, §ors_read, + &writes, §ors_written, &io_ms) == 8) + { + (void) major; + (void) minor; + + if (ngx_strncmp(name, "loop", 4) != 0 + && ngx_strncmp(name, "ram", 3) != 0) + { + ngx_cpystrn(sh->disks[count].name, (u_char *) name, + NGX_HTTP_MONITORING_NAME_LEN); + sh->disks[count].reads = (ngx_atomic_uint_t) reads; + sh->disks[count].writes = (ngx_atomic_uint_t) writes; + sh->disks[count].read_bytes = + (ngx_atomic_uint_t) (sectors_read * 512); + sh->disks[count].write_bytes = + (ngx_atomic_uint_t) (sectors_written * 512); + sh->disks[count].io_ms = (ngx_atomic_uint_t) io_ms; + count++; + } + } + + line = next; + } + + sh->disk_count = count; +} + + +static void +ngx_http_monitoring_collect_filesystems(ngx_http_monitoring_shctx_t *sh) +{ + FILE *fp; + struct mntent *mnt; + struct statvfs st; + ngx_uint_t count; + uint64_t total, free_bytes, avail, used; + + fp = setmntent("/proc/mounts", "r"); + if (fp == NULL) { + return; + } + + count = 0; + while ((mnt = getmntent(fp)) != NULL + && count < NGX_HTTP_MONITORING_FILESYSTEMS_MAX) + { + if (ngx_http_monitoring_skip_fs(mnt->mnt_type)) { + continue; + } + + if (statvfs(mnt->mnt_dir, &st) != 0 || st.f_blocks == 0) { + continue; + } + + total = (uint64_t) st.f_blocks * st.f_frsize; + free_bytes = (uint64_t) st.f_bfree * st.f_frsize; + avail = (uint64_t) st.f_bavail * st.f_frsize; + used = total - free_bytes; + + ngx_cpystrn(sh->filesystems[count].path, (u_char *) mnt->mnt_dir, + NGX_HTTP_MONITORING_KEY_LEN); + ngx_cpystrn(sh->filesystems[count].type, (u_char *) mnt->mnt_type, + NGX_HTTP_MONITORING_NAME_LEN); + sh->filesystems[count].total = (ngx_atomic_uint_t) total; + sh->filesystems[count].used = (ngx_atomic_uint_t) used; + sh->filesystems[count].free = (ngx_atomic_uint_t) free_bytes; + sh->filesystems[count].avail = (ngx_atomic_uint_t) avail; + sh->filesystems[count].files = (ngx_atomic_uint_t) st.f_files; + sh->filesystems[count].files_free = (ngx_atomic_uint_t) st.f_ffree; + + count++; + } + + endmntent(fp); + sh->fs_count = count; +} + + +static void +ngx_http_monitoring_collect_nginx(ngx_http_monitoring_shctx_t *sh) +{ +#if (NGX_STAT_STUB) + sh->connections.accepted = *ngx_stat_accepted; + sh->connections.handled = *ngx_stat_handled; + sh->connections.total = *ngx_stat_active; + sh->connections.active = *ngx_stat_active; + sh->connections.reading = *ngx_stat_reading; + sh->connections.writing = *ngx_stat_writing; + sh->connections.waiting = *ngx_stat_waiting; +#else + (void) sh; +#endif +} + + +static void +ngx_http_monitoring_collect_history(ngx_http_monitoring_main_conf_t *mmcf, + ngx_http_monitoring_shctx_t *sh) +{ + ngx_time_t *tp; + ngx_msec_t now_msec; + ngx_atomic_uint_t req, resp, net_rx, net_tx; + ngx_atomic_uint_t disk_read, disk_write, prev_msec; + ngx_atomic_uint_t dt, rps_milli, resp_milli; + ngx_atomic_uint_t net_rx_rate, net_tx_rate; + ngx_atomic_uint_t disk_read_rate, disk_write_rate; + ngx_uint_t pos, capacity; + ngx_http_monitoring_history_sample_t *sample; + double mem_pct, swap_pct; + + tp = ngx_timeofday(); + now_msec = (ngx_msec_t) (tp->sec * 1000 + tp->msec); + + if (sh->prev_history_msec != 0 + && now_msec - sh->prev_history_msec < mmcf->resolution) + { + return; + } + + req = sh->requests.total; + resp = sh->requests.responses; + net_rx = ngx_http_monitoring_sum_network(sh, 0); + net_tx = ngx_http_monitoring_sum_network(sh, 1); + disk_read = ngx_http_monitoring_sum_disk(sh, 0); + disk_write = ngx_http_monitoring_sum_disk(sh, 1); + + prev_msec = sh->prev_collect_msec; + dt = (prev_msec == 0 || now_msec <= prev_msec) + ? mmcf->refresh_interval + : now_msec - prev_msec; + if (dt == 0) { + dt = 1; + } + + rps_milli = (ngx_http_monitoring_delta(req, sh->prev_req_total) + * 1000000) / dt; + resp_milli = (ngx_http_monitoring_delta(resp, sh->prev_resp_total) + * 1000000) / dt; + net_rx_rate = (ngx_http_monitoring_delta(net_rx, sh->prev_net_rx) + * 1000) / dt; + net_tx_rate = (ngx_http_monitoring_delta(net_tx, sh->prev_net_tx) + * 1000) / dt; + disk_read_rate = (ngx_http_monitoring_delta(disk_read, + sh->prev_disk_read) * 1000) / dt; + disk_write_rate = (ngx_http_monitoring_delta(disk_write, + sh->prev_disk_write) * 1000) / dt; + + if (sh->ewma_rps_milli == 0) { + sh->ewma_rps_milli = rps_milli; + sh->ewma_resp_milli = resp_milli; + } else { + sh->ewma_rps_milli = (sh->ewma_rps_milli * 7 + rps_milli * 3) / 10; + sh->ewma_resp_milli = (sh->ewma_resp_milli * 7 + + resp_milli * 3) / 10; + } + + capacity = sh->history_capacity; + if (capacity == 0 || capacity > NGX_HTTP_MONITORING_HISTORY_MAX) { + capacity = NGX_HTTP_MONITORING_HISTORY_MAX; + } + + pos = ngx_atomic_fetch_add(&sh->history_write, 1) % capacity; + sample = &sh->history[pos]; + ngx_memzero(sample, sizeof(ngx_http_monitoring_history_sample_t)); + + mem_pct = 0; + if (sh->system.mem_total > 0) { + mem_pct = ((double) (sh->system.mem_total + - sh->system.mem_available) * 100.0) + / (double) sh->system.mem_total; + } + + swap_pct = 0; + if (sh->system.swap_total > 0) { + swap_pct = ((double) (sh->system.swap_total - sh->system.swap_free) + * 100.0) / (double) sh->system.swap_total; + } + + sample->timestamp = tp->sec; + sample->msec = tp->msec; + sample->cpu_usage = (double) sh->system.usage_milli / 1000.0; + sample->load1 = (double) sh->system.load1_milli / 1000.0; + sample->load5 = (double) sh->system.load5_milli / 1000.0; + sample->load15 = (double) sh->system.load15_milli / 1000.0; + sample->memory_used_pct = mem_pct; + sample->swap_used_pct = swap_pct; + sample->requests_per_sec = (double) sh->ewma_rps_milli / 1000.0; + sample->responses_per_sec = (double) sh->ewma_resp_milli / 1000.0; + sample->network_rx_per_sec = (double) net_rx_rate; + sample->network_tx_per_sec = (double) net_tx_rate; + sample->disk_read_per_sec = (double) disk_read_rate; + sample->disk_write_per_sec = (double) disk_write_rate; + sample->latency_p95 = ngx_http_monitoring_percentile(sh, 95.0); + sample->requests_total = req; + sample->status_4xx = sh->requests.status_4xx; + sample->status_5xx = sh->requests.status_5xx; + + if (sh->history_count < capacity) { + ngx_atomic_fetch_add(&sh->history_count, 1); + } + + sh->prev_req_total = req; + sh->prev_resp_total = resp; + sh->prev_net_rx = net_rx; + sh->prev_net_tx = net_tx; + sh->prev_disk_read = disk_read; + sh->prev_disk_write = disk_write; + sh->prev_collect_msec = now_msec; + sh->prev_history_msec = now_msec; +} + + +static ngx_atomic_uint_t +ngx_http_monitoring_sum_network(ngx_http_monitoring_shctx_t *sh, ngx_uint_t tx) +{ + ngx_uint_t i, count; + ngx_atomic_uint_t total; + + total = 0; + count = ngx_min((ngx_uint_t) sh->iface_count, + NGX_HTTP_MONITORING_IFACES_MAX); + + for (i = 0; i < count; i++) { + total += tx ? sh->ifaces[i].tx_bytes : sh->ifaces[i].rx_bytes; + } + + return total; +} + + +static ngx_atomic_uint_t +ngx_http_monitoring_sum_disk(ngx_http_monitoring_shctx_t *sh, ngx_uint_t write) +{ + ngx_uint_t i, count; + ngx_atomic_uint_t total; + + total = 0; + count = ngx_min((ngx_uint_t) sh->disk_count, + NGX_HTTP_MONITORING_DISKS_MAX); + + for (i = 0; i < count; i++) { + total += write ? sh->disks[i].write_bytes : sh->disks[i].read_bytes; + } + + return total; +} + + +static ngx_atomic_uint_t +ngx_http_monitoring_delta(ngx_atomic_uint_t now, ngx_atomic_uint_t prev) +{ + return now >= prev ? now - prev : 0; +} + + +static ngx_uint_t +ngx_http_monitoring_skip_fs(const char *type) +{ + static const char *skip[] = { + "proc", "sysfs", "devtmpfs", "devpts", "securityfs", "cgroup", + "cgroup2", "pstore", "autofs", "mqueue", "hugetlbfs", "debugfs", + "tracefs", "configfs", "fusectl", "binfmt_misc", "rpc_pipefs", + "overlay", "nsfs", NULL + }; + + ngx_uint_t i; + + for (i = 0; skip[i] != NULL; i++) { + if (strcmp(type, skip[i]) == 0) { + return 1; + } + } + + return 0; +} diff --git a/src/ngx_http_monitoring_dashboard.c b/src/ngx_http_monitoring_dashboard.c new file mode 100644 index 0000000..0af80de --- /dev/null +++ b/src/ngx_http_monitoring_dashboard.c @@ -0,0 +1,114 @@ +#include "ngx_http_monitoring.h" + +static const u_char ngx_http_monitoring_dashboard_html[] = +"" +"" +"Nginx Monitor
Nginx Monitor
waiting
offline
" +"
" +"

CPU

Requests

Network

Latency

" +"

CPU Load

Load Average

" +"

Memory

Memory Stats

" +"
" +"

Devices

Filesystems

" +"
User agents
" +"
" +"
" +"

Status

Error Rate

" +"
"; + + +ngx_int_t +ngx_http_monitoring_send_dashboard(ngx_http_request_t *r) +{ + ngx_buf_t *b; + ngx_chain_t out; + ngx_table_elt_t *h; + + ngx_http_discard_request_body(r); + + h = ngx_list_push(&r->headers_out.headers); + if (h != NULL) { + h->hash = 1; + ngx_str_set(&h->key, "Cache-Control"); + ngx_str_set(&h->value, "no-store"); + } + + h = ngx_list_push(&r->headers_out.headers); + if (h != NULL) { + h->hash = 1; + ngx_str_set(&h->key, "X-Frame-Options"); + ngx_str_set(&h->value, "DENY"); + } + + h = ngx_list_push(&r->headers_out.headers); + if (h != NULL) { + h->hash = 1; + ngx_str_set(&h->key, "Content-Security-Policy"); + ngx_str_set(&h->value, + "default-src 'self'; script-src 'unsafe-inline' 'self'; " + "style-src 'unsafe-inline' 'self'; connect-src 'self'"); + } + + r->headers_out.status = NGX_HTTP_OK; + r->headers_out.content_length_n = + sizeof(ngx_http_monitoring_dashboard_html) - 1; + ngx_str_set(&r->headers_out.content_type, "text/html"); + + if (r->method == NGX_HTTP_HEAD) { + r->header_only = 1; + return ngx_http_send_header(r); + } + + b = ngx_calloc_buf(r->pool); + if (b == NULL) { + return NGX_HTTP_INTERNAL_SERVER_ERROR; + } + + b->pos = (u_char *) ngx_http_monitoring_dashboard_html; + b->last = (u_char *) ngx_http_monitoring_dashboard_html + + sizeof(ngx_http_monitoring_dashboard_html) - 1; + b->memory = 1; + b->last_buf = 1; + + out.buf = b; + out.next = NULL; + + ngx_http_send_header(r); + return ngx_http_output_filter(r, &out); +} diff --git a/src/ngx_http_monitoring_json.c b/src/ngx_http_monitoring_json.c new file mode 100644 index 0000000..b3e3893 --- /dev/null +++ b/src/ngx_http_monitoring_json.c @@ -0,0 +1,975 @@ +#include "ngx_http_monitoring.h" + +#include +#include + +typedef struct { + u_char *pos; + u_char *last; + ngx_int_t failed; +} ngx_http_monitoring_json_writer_t; + +typedef struct { + u_char key[NGX_HTTP_MONITORING_KEY_LEN]; + ngx_atomic_uint_t hits; + ngx_atomic_uint_t errors; + ngx_atomic_uint_t bytes; + ngx_atomic_uint_t latency_ms_total; + ngx_atomic_uint_t last_seen; +} ngx_http_monitoring_top_view_t; + +static void ngx_http_monitoring_json_append( + ngx_http_monitoring_json_writer_t *jw, const char *s); +static void ngx_http_monitoring_json_printf( + ngx_http_monitoring_json_writer_t *jw, const char *fmt, ...); +static void ngx_http_monitoring_json_string( + ngx_http_monitoring_json_writer_t *jw, const u_char *data, size_t len); +static void ngx_http_monitoring_json_header( + ngx_http_monitoring_json_writer_t *jw, ngx_http_request_t *r, + const char *scope); +static void ngx_http_monitoring_json_system( + ngx_http_monitoring_json_writer_t *jw, ngx_http_monitoring_shctx_t *sh); +static void ngx_http_monitoring_json_network( + ngx_http_monitoring_json_writer_t *jw, ngx_http_monitoring_shctx_t *sh); +static void ngx_http_monitoring_json_disk( + ngx_http_monitoring_json_writer_t *jw, ngx_http_monitoring_shctx_t *sh); +static void ngx_http_monitoring_json_processes( + ngx_http_monitoring_json_writer_t *jw, ngx_http_monitoring_shctx_t *sh); +static void ngx_http_monitoring_json_nginx( + ngx_http_monitoring_json_writer_t *jw, ngx_http_monitoring_shctx_t *sh); +static void ngx_http_monitoring_json_connections( + ngx_http_monitoring_json_writer_t *jw, ngx_http_monitoring_shctx_t *sh); +static void ngx_http_monitoring_json_requests( + ngx_http_monitoring_json_writer_t *jw, ngx_http_request_t *r, + ngx_http_monitoring_shctx_t *sh); +static void ngx_http_monitoring_json_upstreams( + ngx_http_monitoring_json_writer_t *jw, ngx_http_monitoring_shctx_t *sh); +static void ngx_http_monitoring_json_history( + ngx_http_monitoring_json_writer_t *jw, ngx_http_monitoring_shctx_t *sh); +static void ngx_http_monitoring_json_workers( + ngx_http_monitoring_json_writer_t *jw, ngx_http_monitoring_shctx_t *sh); +static void ngx_http_monitoring_json_top_entries( + ngx_http_monitoring_json_writer_t *jw, ngx_http_request_t *r, + ngx_http_monitoring_top_entry_t *entries, ngx_uint_t capacity, + const char *key_name); +static ngx_int_t ngx_http_monitoring_send_buffer(ngx_http_request_t *r, + ngx_buf_t *b, ngx_str_t *type); +static void ngx_http_monitoring_add_no_store(ngx_http_request_t *r); +static void ngx_http_monitoring_sort_top(ngx_http_monitoring_top_view_t *v, + ngx_uint_t n); + +static ngx_uint_t ngx_http_monitoring_latency_bounds[ + NGX_HTTP_MONITORING_LATENCY_BUCKETS - 1 +] = { 1, 5, 10, 25, 50, 100, 250, 500, 1000, 2500, 5000, 10000, 30000 }; + +static off_t ngx_http_monitoring_size_bounds[ + NGX_HTTP_MONITORING_SIZE_BUCKETS - 1 +] = { 512, 1024, 2048, 4096, 8192, 16384, 32768, 65536, + 262144, 1048576, 10485760 }; + +static const char *ngx_http_monitoring_methods[NGX_HTTP_MONITORING_METHODS] = { + "GET", "POST", "PUT", "DELETE", "HEAD", "OPTIONS", "PATCH", "OTHER" +}; + + +ngx_int_t +ngx_http_monitoring_send_json(ngx_http_request_t *r, + ngx_http_monitoring_endpoint_e endpoint) +{ + ngx_buf_t *b; + ngx_http_monitoring_json_writer_t jw; + ngx_http_monitoring_shctx_t *sh; + ngx_str_t type = ngx_string("application/json"); + const char *scope; + + sh = ngx_http_monitoring_get_shctx(r); + if (sh == NULL) { + return NGX_HTTP_SERVICE_UNAVAILABLE; + } + + b = ngx_create_temp_buf(r->pool, 512 * 1024); + if (b == NULL) { + return NGX_HTTP_INTERNAL_SERVER_ERROR; + } + + jw.pos = b->pos; + jw.last = b->end; + jw.failed = 0; + + switch (endpoint) { + case NGX_HTTP_MONITORING_EP_API_SYSTEM: + scope = "system"; + break; + case NGX_HTTP_MONITORING_EP_API_NGINX: + scope = "nginx"; + break; + case NGX_HTTP_MONITORING_EP_API_NETWORK: + scope = "network"; + break; + case NGX_HTTP_MONITORING_EP_API_DISK: + scope = "disk"; + break; + case NGX_HTTP_MONITORING_EP_API_PROCESSES: + scope = "processes"; + break; + case NGX_HTTP_MONITORING_EP_API_UPSTREAMS: + scope = "upstreams"; + break; + case NGX_HTTP_MONITORING_EP_API_CONNECTIONS: + scope = "connections"; + break; + case NGX_HTTP_MONITORING_EP_API_REQUESTS: + scope = "requests"; + break; + default: + scope = "full"; + break; + } + + ngx_http_monitoring_json_header(&jw, r, scope); + + if (endpoint == NGX_HTTP_MONITORING_EP_API_FULL + || endpoint == NGX_HTTP_MONITORING_EP_API_SYSTEM) + { + ngx_http_monitoring_json_append(&jw, ",\"system\":"); + ngx_http_monitoring_json_system(&jw, sh); + } + + if (endpoint == NGX_HTTP_MONITORING_EP_API_FULL + || endpoint == NGX_HTTP_MONITORING_EP_API_NGINX) + { + ngx_http_monitoring_json_append(&jw, ",\"nginx\":"); + ngx_http_monitoring_json_nginx(&jw, sh); + } + + if (endpoint == NGX_HTTP_MONITORING_EP_API_FULL + || endpoint == NGX_HTTP_MONITORING_EP_API_NETWORK) + { + ngx_http_monitoring_json_append(&jw, ",\"network\":"); + ngx_http_monitoring_json_network(&jw, sh); + } + + if (endpoint == NGX_HTTP_MONITORING_EP_API_FULL + || endpoint == NGX_HTTP_MONITORING_EP_API_DISK) + { + ngx_http_monitoring_json_append(&jw, ",\"disk\":"); + ngx_http_monitoring_json_disk(&jw, sh); + } + + if (endpoint == NGX_HTTP_MONITORING_EP_API_FULL + || endpoint == NGX_HTTP_MONITORING_EP_API_PROCESSES) + { + ngx_http_monitoring_json_append(&jw, ",\"processes\":"); + ngx_http_monitoring_json_processes(&jw, sh); + } + + if (endpoint == NGX_HTTP_MONITORING_EP_API_FULL + || endpoint == NGX_HTTP_MONITORING_EP_API_UPSTREAMS) + { + ngx_http_monitoring_json_append(&jw, ",\"upstreams\":"); + ngx_http_monitoring_json_upstreams(&jw, sh); + } + + if (endpoint == NGX_HTTP_MONITORING_EP_API_FULL + || endpoint == NGX_HTTP_MONITORING_EP_API_CONNECTIONS) + { + ngx_http_monitoring_json_append(&jw, ",\"connections\":"); + ngx_http_monitoring_json_connections(&jw, sh); + } + + if (endpoint == NGX_HTTP_MONITORING_EP_API_FULL + || endpoint == NGX_HTTP_MONITORING_EP_API_REQUESTS) + { + ngx_http_monitoring_json_append(&jw, ",\"requests\":"); + ngx_http_monitoring_json_requests(&jw, r, sh); + } + + if (endpoint == NGX_HTTP_MONITORING_EP_API_FULL) { + ngx_http_monitoring_json_append(&jw, ",\"history\":"); + ngx_http_monitoring_json_history(&jw, sh); + } + + ngx_http_monitoring_json_append(&jw, "}\n"); + + if (jw.failed) { + return NGX_HTTP_INTERNAL_SERVER_ERROR; + } + + b->last = jw.pos; + b->last_buf = 1; + + return ngx_http_monitoring_send_buffer(r, b, &type); +} + + +ngx_int_t +ngx_http_monitoring_send_prometheus(ngx_http_request_t *r) +{ + ngx_buf_t *b; + ngx_http_monitoring_json_writer_t jw; + ngx_http_monitoring_shctx_t *sh; + ngx_str_t type = ngx_string("text/plain"); + + sh = ngx_http_monitoring_get_shctx(r); + if (sh == NULL) { + return NGX_HTTP_SERVICE_UNAVAILABLE; + } + + b = ngx_create_temp_buf(r->pool, 128 * 1024); + if (b == NULL) { + return NGX_HTTP_INTERNAL_SERVER_ERROR; + } + + jw.pos = b->pos; + jw.last = b->end; + jw.failed = 0; + + ngx_http_monitoring_json_append(&jw, + "# HELP nginx_monitor_requests_total Total HTTP requests observed by " + "ngx_http_monitoring_module\n" + "# TYPE nginx_monitor_requests_total counter\n"); + ngx_http_monitoring_json_printf(&jw, "nginx_monitor_requests_total %uA\n", + sh->requests.total); + + ngx_http_monitoring_json_append(&jw, + "# HELP nginx_monitor_active_connections Active nginx connections\n" + "# TYPE nginx_monitor_active_connections gauge\n"); + ngx_http_monitoring_json_printf(&jw, + "nginx_monitor_active_connections %uA\n", sh->connections.active); + + ngx_http_monitoring_json_append(&jw, + "# HELP nginx_monitor_cpu_usage_ratio CPU usage ratio\n" + "# TYPE nginx_monitor_cpu_usage_ratio gauge\n"); + ngx_http_monitoring_json_printf(&jw, + "nginx_monitor_cpu_usage_ratio %.3f\n", + (double) sh->system.usage_milli / 100000.0); + + ngx_http_monitoring_json_append(&jw, + "# HELP nginx_monitor_memory_used_ratio Memory usage ratio\n" + "# TYPE nginx_monitor_memory_used_ratio gauge\n"); + if (sh->system.mem_total > 0) { + ngx_http_monitoring_json_printf(&jw, + "nginx_monitor_memory_used_ratio %.3f\n", + (double) (sh->system.mem_total - sh->system.mem_available) + / (double) sh->system.mem_total); + } else { + ngx_http_monitoring_json_append(&jw, + "nginx_monitor_memory_used_ratio 0\n"); + } + + ngx_http_monitoring_json_append(&jw, + "# HELP nginx_monitor_latency_p95_ms Estimated p95 request latency\n" + "# TYPE nginx_monitor_latency_p95_ms gauge\n"); + ngx_http_monitoring_json_printf(&jw, + "nginx_monitor_latency_p95_ms %.3f\n", + ngx_http_monitoring_percentile(sh, 95.0)); + + if (jw.failed) { + return NGX_HTTP_INTERNAL_SERVER_ERROR; + } + + b->last = jw.pos; + b->last_buf = 1; + + return ngx_http_monitoring_send_buffer(r, b, &type); +} + + +ngx_int_t +ngx_http_monitoring_send_health(ngx_http_request_t *r) +{ + ngx_buf_t *b; + ngx_http_monitoring_json_writer_t jw; + ngx_http_monitoring_shctx_t *sh; + ngx_str_t type = ngx_string("application/json"); + + sh = ngx_http_monitoring_get_shctx(r); + if (sh == NULL) { + return NGX_HTTP_SERVICE_UNAVAILABLE; + } + + b = ngx_create_temp_buf(r->pool, 4096); + if (b == NULL) { + return NGX_HTTP_INTERNAL_SERVER_ERROR; + } + + jw.pos = b->pos; + jw.last = b->end; + jw.failed = 0; + + ngx_http_monitoring_json_header(&jw, r, "health"); + ngx_http_monitoring_json_printf(&jw, + ",\"status\":\"ok\",\"generation\":%uA,\"sse_clients\":%uA}\n", + sh->generation, sh->connections.sse_clients); + + if (jw.failed) { + return NGX_HTTP_INTERNAL_SERVER_ERROR; + } + + b->last = jw.pos; + b->last_buf = 1; + + return ngx_http_monitoring_send_buffer(r, b, &type); +} + + +static void +ngx_http_monitoring_json_append(ngx_http_monitoring_json_writer_t *jw, + const char *s) +{ + size_t len; + + if (jw->failed) { + return; + } + + len = ngx_strlen(s); + if ((size_t) (jw->last - jw->pos) < len) { + jw->failed = 1; + return; + } + + jw->pos = ngx_cpymem(jw->pos, s, len); +} + + +static void +ngx_http_monitoring_json_printf(ngx_http_monitoring_json_writer_t *jw, + const char *fmt, ...) +{ + va_list args; + u_char *p; + + if (jw->failed) { + return; + } + + if (jw->last - jw->pos <= 1) { + jw->failed = 1; + return; + } + + va_start(args, fmt); + p = ngx_vslprintf(jw->pos, jw->last, fmt, args); + va_end(args); + + if (p >= jw->last) { + jw->failed = 1; + return; + } + + jw->pos = p; +} + + +static void +ngx_http_monitoring_json_string(ngx_http_monitoring_json_writer_t *jw, + const u_char *data, size_t len) +{ + size_t i; + u_char c; + + ngx_http_monitoring_json_append(jw, "\""); + + for (i = 0; i < len && !jw->failed; i++) { + c = data[i]; + + if (c == '"' || c == '\\') { + if (jw->last - jw->pos < 2) { + jw->failed = 1; + return; + } + *jw->pos++ = '\\'; + *jw->pos++ = c; + } else if (c == '\n') { + ngx_http_monitoring_json_append(jw, "\\n"); + } else if (c == '\r') { + ngx_http_monitoring_json_append(jw, "\\r"); + } else if (c == '\t') { + ngx_http_monitoring_json_append(jw, "\\t"); + } else if (c < 0x20) { + ngx_http_monitoring_json_printf(jw, "\\u%04x", c); + } else { + if (jw->last - jw->pos < 1) { + jw->failed = 1; + return; + } + *jw->pos++ = c; + } + } + + ngx_http_monitoring_json_append(jw, "\""); +} + + +static void +ngx_http_monitoring_json_header(ngx_http_monitoring_json_writer_t *jw, + ngx_http_request_t *r, const char *scope) +{ + ngx_time_t *tp; + + tp = ngx_timeofday(); + + ngx_http_monitoring_json_printf(jw, + "{\"version\":%d,\"module\":\"%s\",\"timestamp\":%T," + "\"msec\":%M,\"scope\":\"%s\",\"pid\":%P", + NGX_HTTP_MONITORING_VERSION, + NGX_HTTP_MONITORING_MODULE_VERSION, + tp->sec, tp->msec, scope, ngx_pid); + + (void) r; +} + + +static void +ngx_http_monitoring_json_system(ngx_http_monitoring_json_writer_t *jw, + ngx_http_monitoring_shctx_t *sh) +{ + double mem_used, mem_pct, swap_pct; + + mem_used = 0; + mem_pct = 0; + if (sh->system.mem_total > 0) { + mem_used = (double) (sh->system.mem_total - sh->system.mem_available); + mem_pct = mem_used * 100.0 / (double) sh->system.mem_total; + } + + swap_pct = 0; + if (sh->system.swap_total > 0) { + swap_pct = (double) (sh->system.swap_total - sh->system.swap_free) + * 100.0 / (double) sh->system.swap_total; + } + + ngx_http_monitoring_json_printf(jw, + "{\"cpu\":{\"usage\":%.3f,\"cores\":%uA,\"load\":[%.3f,%.3f,%.3f]}," + "\"memory\":{\"total\":%uA,\"available\":%uA,\"free\":%uA," + "\"used\":%.0f,\"used_pct\":%.3f}," + "\"swap\":{\"total\":%uA,\"free\":%uA,\"used_pct\":%.3f}," + "\"uptime\":%uA}", + (double) sh->system.usage_milli / 1000.0, + sh->system.cores, + (double) sh->system.load1_milli / 1000.0, + (double) sh->system.load5_milli / 1000.0, + (double) sh->system.load15_milli / 1000.0, + sh->system.mem_total, sh->system.mem_available, sh->system.mem_free, + mem_used, mem_pct, + sh->system.swap_total, sh->system.swap_free, swap_pct, + sh->system.uptime); +} + + +static void +ngx_http_monitoring_json_network(ngx_http_monitoring_json_writer_t *jw, + ngx_http_monitoring_shctx_t *sh) +{ + ngx_uint_t i, count; + ngx_atomic_uint_t rx, tx, rx_packets, tx_packets, rx_errors, tx_errors; + + rx = tx = rx_packets = tx_packets = rx_errors = tx_errors = 0; + count = ngx_min((ngx_uint_t) sh->iface_count, + NGX_HTTP_MONITORING_IFACES_MAX); + + for (i = 0; i < count; i++) { + rx += sh->ifaces[i].rx_bytes; + tx += sh->ifaces[i].tx_bytes; + rx_packets += sh->ifaces[i].rx_packets; + tx_packets += sh->ifaces[i].tx_packets; + rx_errors += sh->ifaces[i].rx_errors; + tx_errors += sh->ifaces[i].tx_errors; + } + + ngx_http_monitoring_json_printf(jw, + "{\"rx_bytes\":%uA,\"tx_bytes\":%uA,\"rx_packets\":%uA," + "\"tx_packets\":%uA,\"rx_errors\":%uA,\"tx_errors\":%uA," + "\"interfaces\":[", + rx, tx, rx_packets, tx_packets, rx_errors, tx_errors); + + for (i = 0; i < count; i++) { + if (i) { + ngx_http_monitoring_json_append(jw, ","); + } + + ngx_http_monitoring_json_append(jw, "{\"name\":"); + ngx_http_monitoring_json_string(jw, sh->ifaces[i].name, + ngx_strlen(sh->ifaces[i].name)); + ngx_http_monitoring_json_printf(jw, + ",\"rx_bytes\":%uA,\"tx_bytes\":%uA,\"rx_packets\":%uA," + "\"tx_packets\":%uA,\"rx_errors\":%uA,\"tx_errors\":%uA," + "\"up\":%s}", + sh->ifaces[i].rx_bytes, sh->ifaces[i].tx_bytes, + sh->ifaces[i].rx_packets, sh->ifaces[i].tx_packets, + sh->ifaces[i].rx_errors, sh->ifaces[i].tx_errors, + (sh->ifaces[i].flags & IFF_UP) ? "true" : "false"); + } + + ngx_http_monitoring_json_append(jw, "]}"); +} + + +static void +ngx_http_monitoring_json_disk(ngx_http_monitoring_json_writer_t *jw, + ngx_http_monitoring_shctx_t *sh) +{ + ngx_uint_t i, count; + ngx_atomic_uint_t reads, writes, read_bytes, write_bytes; + + reads = writes = read_bytes = write_bytes = 0; + count = ngx_min((ngx_uint_t) sh->disk_count, + NGX_HTTP_MONITORING_DISKS_MAX); + + for (i = 0; i < count; i++) { + reads += sh->disks[i].reads; + writes += sh->disks[i].writes; + read_bytes += sh->disks[i].read_bytes; + write_bytes += sh->disks[i].write_bytes; + } + + ngx_http_monitoring_json_printf(jw, + "{\"reads\":%uA,\"writes\":%uA,\"read_bytes\":%uA," + "\"write_bytes\":%uA,\"devices\":[", + reads, writes, read_bytes, write_bytes); + + for (i = 0; i < count; i++) { + if (i) { + ngx_http_monitoring_json_append(jw, ","); + } + + ngx_http_monitoring_json_append(jw, "{\"name\":"); + ngx_http_monitoring_json_string(jw, sh->disks[i].name, + ngx_strlen(sh->disks[i].name)); + ngx_http_monitoring_json_printf(jw, + ",\"reads\":%uA,\"writes\":%uA,\"read_bytes\":%uA," + "\"write_bytes\":%uA,\"io_ms\":%uA}", + sh->disks[i].reads, sh->disks[i].writes, + sh->disks[i].read_bytes, sh->disks[i].write_bytes, + sh->disks[i].io_ms); + } + + ngx_http_monitoring_json_append(jw, "],\"filesystems\":["); + + count = ngx_min((ngx_uint_t) sh->fs_count, + NGX_HTTP_MONITORING_FILESYSTEMS_MAX); + + for (i = 0; i < count; i++) { + if (i) { + ngx_http_monitoring_json_append(jw, ","); + } + + ngx_http_monitoring_json_append(jw, "{\"path\":"); + ngx_http_monitoring_json_string(jw, sh->filesystems[i].path, + ngx_strlen(sh->filesystems[i].path)); + ngx_http_monitoring_json_append(jw, ",\"type\":"); + ngx_http_monitoring_json_string(jw, sh->filesystems[i].type, + ngx_strlen(sh->filesystems[i].type)); + ngx_http_monitoring_json_printf(jw, + ",\"total\":%uA,\"used\":%uA,\"free\":%uA,\"avail\":%uA," + "\"files\":%uA,\"files_free\":%uA}", + sh->filesystems[i].total, sh->filesystems[i].used, + sh->filesystems[i].free, sh->filesystems[i].avail, + sh->filesystems[i].files, sh->filesystems[i].files_free); + } + + ngx_http_monitoring_json_append(jw, "]}"); +} + + +static void +ngx_http_monitoring_json_processes(ngx_http_monitoring_json_writer_t *jw, + ngx_http_monitoring_shctx_t *sh) +{ + ngx_http_monitoring_json_printf(jw, + "{\"process_count\":%uA,\"tcp\":{\"established\":%uA," + "\"listen\":%uA},\"sockets\":{\"used\":%uA,\"tcp\":%uA," + "\"udp\":%uA},\"workers\":", + sh->system.process_count, sh->system.tcp_established, + sh->system.tcp_listen, sh->system.sockets_used, + sh->system.sockets_tcp, sh->system.sockets_udp); + + ngx_http_monitoring_json_workers(jw, sh); + ngx_http_monitoring_json_append(jw, "}"); +} + + +static void +ngx_http_monitoring_json_nginx(ngx_http_monitoring_json_writer_t *jw, + ngx_http_monitoring_shctx_t *sh) +{ + ngx_http_monitoring_json_printf(jw, + "{\"connections\":{\"accepted\":%uA,\"handled\":%uA,\"active\":%uA," + "\"reading\":%uA,\"writing\":%uA,\"waiting\":%uA}," + "\"requests\":{\"total\":%uA,\"responses\":%uA," + "\"requests_per_sec\":%.3f,\"responses_per_sec\":%.3f}," + "\"ssl\":{\"requests\":%uA,\"handshakes\":%uA}," + "\"keepalive\":{\"requests\":%uA},\"workers\":", + sh->connections.accepted, sh->connections.handled, + sh->connections.active, sh->connections.reading, + sh->connections.writing, sh->connections.waiting, + sh->requests.total, sh->requests.responses, + (double) sh->ewma_rps_milli / 1000.0, + (double) sh->ewma_resp_milli / 1000.0, + sh->connections.ssl_requests, sh->connections.ssl_handshakes, + sh->connections.keepalive_requests); + + ngx_http_monitoring_json_workers(jw, sh); + ngx_http_monitoring_json_append(jw, "}"); +} + + +static void +ngx_http_monitoring_json_connections(ngx_http_monitoring_json_writer_t *jw, + ngx_http_monitoring_shctx_t *sh) +{ + ngx_http_monitoring_json_printf(jw, + "{\"accepted\":%uA,\"handled\":%uA,\"active\":%uA," + "\"reading\":%uA,\"writing\":%uA,\"waiting\":%uA," + "\"ssl_requests\":%uA,\"ssl_handshakes\":%uA," + "\"keepalive_requests\":%uA,\"sse_clients\":%uA," + "\"sse_events\":%uA,\"rate_limited\":%uA}", + sh->connections.accepted, sh->connections.handled, + sh->connections.active, sh->connections.reading, + sh->connections.writing, sh->connections.waiting, + sh->connections.ssl_requests, sh->connections.ssl_handshakes, + sh->connections.keepalive_requests, sh->connections.sse_clients, + sh->connections.sse_events, sh->connections.rate_limited); +} + + +static void +ngx_http_monitoring_json_requests(ngx_http_monitoring_json_writer_t *jw, + ngx_http_request_t *r, ngx_http_monitoring_shctx_t *sh) +{ + ngx_uint_t i; + double avg_latency; + + avg_latency = 0; + if (sh->requests.total > 0) { + avg_latency = (double) sh->requests.request_time_ms_total + / (double) sh->requests.total; + } + + ngx_http_monitoring_json_printf(jw, + "{\"total\":%uA,\"responses\":%uA,\"bytes\":%uA," + "\"requests_per_sec\":%.3f,\"responses_per_sec\":%.3f," + "\"error_rate\":%.6f," + "\"latency\":{\"avg\":%.3f,\"p50\":%.3f,\"p90\":%.3f," + "\"p95\":%.3f,\"p99\":%.3f}," + "\"status\":{\"1xx\":%uA,\"2xx\":%uA,\"3xx\":%uA," + "\"4xx\":%uA,\"5xx\":%uA},\"methods\":{", + sh->requests.total, sh->requests.responses, sh->requests.bytes, + (double) sh->ewma_rps_milli / 1000.0, + (double) sh->ewma_resp_milli / 1000.0, + sh->requests.total > 0 + ? (double) (sh->requests.status_4xx + sh->requests.status_5xx) + / (double) sh->requests.total + : 0.0, + avg_latency, + ngx_http_monitoring_percentile(sh, 50.0), + ngx_http_monitoring_percentile(sh, 90.0), + ngx_http_monitoring_percentile(sh, 95.0), + ngx_http_monitoring_percentile(sh, 99.0), + sh->requests.status_1xx, sh->requests.status_2xx, + sh->requests.status_3xx, sh->requests.status_4xx, + sh->requests.status_5xx); + + for (i = 0; i < NGX_HTTP_MONITORING_METHODS; i++) { + if (i) { + ngx_http_monitoring_json_append(jw, ","); + } + ngx_http_monitoring_json_printf(jw, "\"%s\":%uA", + ngx_http_monitoring_methods[i], + sh->requests.method[i]); + } + + ngx_http_monitoring_json_append(jw, "},\"latency_histogram\":["); + + for (i = 0; i < NGX_HTTP_MONITORING_LATENCY_BUCKETS; i++) { + if (i) { + ngx_http_monitoring_json_append(jw, ","); + } + if (i < NGX_HTTP_MONITORING_LATENCY_BUCKETS - 1) { + ngx_http_monitoring_json_printf(jw, + "{\"le\":%ui,\"count\":%uA}", + ngx_http_monitoring_latency_bounds[i], + sh->requests.latency_bucket[i]); + } else { + ngx_http_monitoring_json_printf(jw, + "{\"le\":\"+Inf\",\"count\":%uA}", + sh->requests.latency_bucket[i]); + } + } + + ngx_http_monitoring_json_append(jw, "],\"size_histogram\":["); + + for (i = 0; i < NGX_HTTP_MONITORING_SIZE_BUCKETS; i++) { + if (i) { + ngx_http_monitoring_json_append(jw, ","); + } + if (i < NGX_HTTP_MONITORING_SIZE_BUCKETS - 1) { + ngx_http_monitoring_json_printf(jw, + "{\"le\":%O,\"count\":%uA}", + ngx_http_monitoring_size_bounds[i], + sh->requests.size_bucket[i]); + } else { + ngx_http_monitoring_json_printf(jw, + "{\"le\":\"+Inf\",\"count\":%uA}", + sh->requests.size_bucket[i]); + } + } + + ngx_http_monitoring_json_append(jw, "],\"top_urls\":"); + ngx_http_monitoring_json_top_entries(jw, r, sh->urls, + NGX_HTTP_MONITORING_TOP_URLS_MAX, "url"); + + ngx_http_monitoring_json_append(jw, ",\"user_agents\":"); + ngx_http_monitoring_json_top_entries(jw, r, sh->user_agents, + NGX_HTTP_MONITORING_TOP_UA_MAX, "user_agent"); + + ngx_http_monitoring_json_append(jw, "}"); +} + + +static void +ngx_http_monitoring_json_upstreams(ngx_http_monitoring_json_writer_t *jw, + ngx_http_monitoring_shctx_t *sh) +{ + ngx_uint_t i, count, emitted; + ngx_http_monitoring_upstream_entry_t *u; + double avg; + + count = NGX_HTTP_MONITORING_UPSTREAMS_MAX; + emitted = 0; + + ngx_http_monitoring_json_append(jw, "["); + + for (i = 0; i < count; i++) { + u = &sh->upstreams[i]; + if (u->requests == 0) { + continue; + } + + if (emitted++) { + ngx_http_monitoring_json_append(jw, ","); + } + + avg = u->requests > 0 + ? (double) u->latency_ms_total / (double) u->requests + : 0.0; + + ngx_http_monitoring_json_append(jw, "{\"peer\":"); + ngx_http_monitoring_json_string(jw, u->peer, ngx_strlen(u->peer)); + ngx_http_monitoring_json_printf(jw, + ",\"requests\":%uA,\"failures\":%uA,\"status_4xx\":%uA," + "\"status_5xx\":%uA,\"avg_latency\":%.3f,\"last_seen\":%uA}", + u->requests, u->failures, u->status_4xx, u->status_5xx, + avg, u->last_seen); + } + + ngx_http_monitoring_json_append(jw, "]"); +} + + +static void +ngx_http_monitoring_json_history(ngx_http_monitoring_json_writer_t *jw, + ngx_http_monitoring_shctx_t *sh) +{ + ngx_uint_t i, idx, count, capacity, write; + ngx_http_monitoring_history_sample_t *s; + + count = ngx_min((ngx_uint_t) sh->history_count, + NGX_HTTP_MONITORING_HISTORY_MAX); + capacity = ngx_min((ngx_uint_t) sh->history_capacity, + NGX_HTTP_MONITORING_HISTORY_MAX); + + if (capacity == 0) { + ngx_http_monitoring_json_append(jw, "[]"); + return; + } + + write = (ngx_uint_t) sh->history_write; + + ngx_http_monitoring_json_append(jw, "["); + + for (i = 0; i < count; i++) { + idx = (write + capacity - count + i) % capacity; + s = &sh->history[idx]; + + if (i) { + ngx_http_monitoring_json_append(jw, ","); + } + + ngx_http_monitoring_json_printf(jw, + "{\"timestamp\":%T,\"cpu\":%.3f,\"memory\":%.3f," + "\"swap\":%.3f,\"rps\":%.3f,\"responses_per_sec\":%.3f," + "\"network_rx_per_sec\":%.3f,\"network_tx_per_sec\":%.3f," + "\"disk_read_per_sec\":%.3f,\"disk_write_per_sec\":%.3f," + "\"latency_p95\":%.3f,\"requests_total\":%uA," + "\"status_4xx\":%uA,\"status_5xx\":%uA}", + s->timestamp, s->cpu_usage, s->memory_used_pct, + s->swap_used_pct, s->requests_per_sec, s->responses_per_sec, + s->network_rx_per_sec, s->network_tx_per_sec, + s->disk_read_per_sec, s->disk_write_per_sec, + s->latency_p95, s->requests_total, s->status_4xx, + s->status_5xx); + } + + ngx_http_monitoring_json_append(jw, "]"); +} + + +static void +ngx_http_monitoring_json_workers(ngx_http_monitoring_json_writer_t *jw, + ngx_http_monitoring_shctx_t *sh) +{ + ngx_uint_t i, emitted; + + emitted = 0; + ngx_http_monitoring_json_append(jw, "["); + + for (i = 0; i < NGX_HTTP_MONITORING_WORKERS_MAX; i++) { + if (sh->workers[i].pid == 0) { + continue; + } + + if (emitted++) { + ngx_http_monitoring_json_append(jw, ","); + } + + ngx_http_monitoring_json_printf(jw, + "{\"slot\":%ui,\"pid\":%uA,\"active\":%s,\"requests\":%uA," + "\"bytes\":%uA,\"errors\":%uA,\"vm_size\":%uA," + "\"vm_rss\":%uA,\"voluntary_ctxt\":%uA," + "\"nonvoluntary_ctxt\":%uA,\"last_seen\":%uA}", + i, sh->workers[i].pid, + sh->workers[i].active ? "true" : "false", + sh->workers[i].requests, sh->workers[i].bytes, + sh->workers[i].errors, sh->workers[i].vm_size, + sh->workers[i].vm_rss, sh->workers[i].voluntary_ctxt, + sh->workers[i].nonvoluntary_ctxt, sh->workers[i].last_seen); + } + + ngx_http_monitoring_json_append(jw, "]"); +} + + +static void +ngx_http_monitoring_json_top_entries(ngx_http_monitoring_json_writer_t *jw, + ngx_http_request_t *r, ngx_http_monitoring_top_entry_t *entries, + ngx_uint_t capacity, const char *key_name) +{ + ngx_uint_t i, n; + ngx_http_monitoring_top_view_t *view; + double avg; + + view = ngx_pcalloc(r->pool, capacity * sizeof(ngx_http_monitoring_top_view_t)); + if (view == NULL) { + jw->failed = 1; + return; + } + + n = 0; + for (i = 0; i < capacity; i++) { + if (entries[i].hits == 0) { + continue; + } + + ngx_cpystrn(view[n].key, entries[i].key, NGX_HTTP_MONITORING_KEY_LEN); + view[n].hits = entries[i].hits; + view[n].errors = entries[i].errors; + view[n].bytes = entries[i].bytes; + view[n].latency_ms_total = entries[i].latency_ms_total; + view[n].last_seen = entries[i].last_seen; + n++; + } + + ngx_http_monitoring_sort_top(view, n); + + ngx_http_monitoring_json_append(jw, "["); + for (i = 0; i < n; i++) { + if (i) { + ngx_http_monitoring_json_append(jw, ","); + } + + avg = view[i].hits > 0 + ? (double) view[i].latency_ms_total / (double) view[i].hits + : 0.0; + + ngx_http_monitoring_json_append(jw, "{\""); + ngx_http_monitoring_json_append(jw, key_name); + ngx_http_monitoring_json_append(jw, "\":"); + ngx_http_monitoring_json_string(jw, view[i].key, + ngx_strlen(view[i].key)); + ngx_http_monitoring_json_printf(jw, + ",\"hits\":%uA,\"errors\":%uA,\"bytes\":%uA," + "\"avg_latency\":%.3f,\"last_seen\":%uA}", + view[i].hits, view[i].errors, view[i].bytes, avg, + view[i].last_seen); + } + ngx_http_monitoring_json_append(jw, "]"); +} + + +static ngx_int_t +ngx_http_monitoring_send_buffer(ngx_http_request_t *r, ngx_buf_t *b, + ngx_str_t *type) +{ + ngx_chain_t out; + + ngx_http_discard_request_body(r); + ngx_http_monitoring_add_no_store(r); + + r->headers_out.status = NGX_HTTP_OK; + r->headers_out.content_length_n = b->last - b->pos; + r->headers_out.content_type = *type; + + if (r->method == NGX_HTTP_HEAD) { + r->header_only = 1; + return ngx_http_send_header(r); + } + + out.buf = b; + out.next = NULL; + + ngx_http_send_header(r); + return ngx_http_output_filter(r, &out); +} + + +static void +ngx_http_monitoring_add_no_store(ngx_http_request_t *r) +{ + ngx_table_elt_t *h; + + h = ngx_list_push(&r->headers_out.headers); + if (h != NULL) { + h->hash = 1; + ngx_str_set(&h->key, "Cache-Control"); + ngx_str_set(&h->value, "no-store"); + } + + h = ngx_list_push(&r->headers_out.headers); + if (h != NULL) { + h->hash = 1; + ngx_str_set(&h->key, "X-Content-Type-Options"); + ngx_str_set(&h->value, "nosniff"); + } +} + + +static void +ngx_http_monitoring_sort_top(ngx_http_monitoring_top_view_t *v, ngx_uint_t n) +{ + ngx_uint_t i, j; + ngx_http_monitoring_top_view_t tmp; + + for (i = 1; i < n; i++) { + tmp = v[i]; + j = i; + + while (j > 0 && v[j - 1].hits < tmp.hits) { + v[j] = v[j - 1]; + j--; + } + + v[j] = tmp; + } +} diff --git a/src/ngx_http_monitoring_module.c b/src/ngx_http_monitoring_module.c new file mode 100644 index 0000000..cdd8c62 --- /dev/null +++ b/src/ngx_http_monitoring_module.c @@ -0,0 +1,985 @@ +#include "ngx_http_monitoring.h" + +static void *ngx_http_monitoring_create_main_conf(ngx_conf_t *cf); +static char *ngx_http_monitoring_init_main_conf(ngx_conf_t *cf, void *conf); +static void *ngx_http_monitoring_create_loc_conf(ngx_conf_t *cf); +static char *ngx_http_monitoring_merge_loc_conf(ngx_conf_t *cf, void *parent, + void *child); +static ngx_int_t ngx_http_monitoring_init(ngx_conf_t *cf); +static ngx_int_t ngx_http_monitoring_init_process(ngx_cycle_t *cycle); +static void ngx_http_monitoring_exit_process(ngx_cycle_t *cycle); + +static char *ngx_http_monitoring_access_rule(ngx_conf_t *cf, ngx_command_t *cmd, + void *conf); + +static ngx_int_t ngx_http_monitoring_content_handler(ngx_http_request_t *r); +static ngx_int_t ngx_http_monitoring_log_handler(ngx_http_request_t *r); +static ngx_int_t ngx_http_monitoring_authorize(ngx_http_request_t *r, + ngx_http_monitoring_loc_conf_t *mlcf, + ngx_http_monitoring_endpoint_e endpoint); +static ngx_int_t ngx_http_monitoring_preflight(ngx_http_request_t *r, + ngx_http_monitoring_loc_conf_t *mlcf); +static void ngx_http_monitoring_add_cors(ngx_http_request_t *r, + ngx_http_monitoring_loc_conf_t *mlcf); +static ngx_int_t ngx_http_monitoring_plain_response(ngx_http_request_t *r, + ngx_uint_t status, ngx_str_t *body); +static ngx_int_t ngx_http_monitoring_match_access(ngx_http_request_t *r, + ngx_http_monitoring_loc_conf_t *mlcf); +static ngx_int_t ngx_http_monitoring_check_basic_auth(ngx_http_request_t *r, + ngx_http_monitoring_loc_conf_t *mlcf); +static ngx_int_t ngx_http_monitoring_check_api_token(ngx_http_request_t *r, + ngx_http_monitoring_loc_conf_t *mlcf); +static ngx_int_t ngx_http_monitoring_check_rate_limit(ngx_http_request_t *r, + ngx_http_monitoring_loc_conf_t *mlcf); +static ngx_table_elt_t *ngx_http_monitoring_find_header(ngx_http_request_t *r, + ngx_str_t *name); +static ngx_int_t ngx_http_monitoring_consttime_eq(ngx_str_t *a, ngx_str_t *b); + + +static ngx_command_t ngx_http_monitoring_commands[] = { + + { ngx_string("monitor"), + NGX_HTTP_MAIN_CONF|NGX_HTTP_SRV_CONF|NGX_HTTP_LOC_CONF|NGX_CONF_FLAG, + ngx_conf_set_flag_slot, + NGX_HTTP_LOC_CONF_OFFSET, + offsetof(ngx_http_monitoring_loc_conf_t, enabled), + NULL }, + + { ngx_string("monitor_dashboard"), + NGX_HTTP_MAIN_CONF|NGX_HTTP_SRV_CONF|NGX_HTTP_LOC_CONF|NGX_CONF_FLAG, + ngx_conf_set_flag_slot, + NGX_HTTP_LOC_CONF_OFFSET, + offsetof(ngx_http_monitoring_loc_conf_t, dashboard), + NULL }, + + { ngx_string("monitor_api"), + NGX_HTTP_MAIN_CONF|NGX_HTTP_SRV_CONF|NGX_HTTP_LOC_CONF|NGX_CONF_FLAG, + ngx_conf_set_flag_slot, + NGX_HTTP_LOC_CONF_OFFSET, + offsetof(ngx_http_monitoring_loc_conf_t, api), + NULL }, + + { ngx_string("monitor_sse"), + NGX_HTTP_MAIN_CONF|NGX_HTTP_SRV_CONF|NGX_HTTP_LOC_CONF|NGX_CONF_FLAG, + ngx_conf_set_flag_slot, + NGX_HTTP_LOC_CONF_OFFSET, + offsetof(ngx_http_monitoring_loc_conf_t, sse), + NULL }, + + { ngx_string("monitor_refresh_interval"), + NGX_HTTP_MAIN_CONF|NGX_CONF_TAKE1, + ngx_conf_set_msec_slot, + NGX_HTTP_MAIN_CONF_OFFSET, + offsetof(ngx_http_monitoring_main_conf_t, refresh_interval), + NULL }, + + { ngx_string("monitor_history"), + NGX_HTTP_MAIN_CONF|NGX_CONF_TAKE1, + ngx_conf_set_msec_slot, + NGX_HTTP_MAIN_CONF_OFFSET, + offsetof(ngx_http_monitoring_main_conf_t, history), + NULL }, + + { ngx_string("monitor_resolution"), + NGX_HTTP_MAIN_CONF|NGX_CONF_TAKE1, + ngx_conf_set_msec_slot, + NGX_HTTP_MAIN_CONF_OFFSET, + offsetof(ngx_http_monitoring_main_conf_t, resolution), + NULL }, + + { ngx_string("monitor_shm_size"), + NGX_HTTP_MAIN_CONF|NGX_CONF_TAKE1, + ngx_conf_set_size_slot, + NGX_HTTP_MAIN_CONF_OFFSET, + offsetof(ngx_http_monitoring_main_conf_t, shm_size), + NULL }, + + { ngx_string("monitor_collect_system"), + NGX_HTTP_MAIN_CONF|NGX_CONF_FLAG, + ngx_conf_set_flag_slot, + NGX_HTTP_MAIN_CONF_OFFSET, + offsetof(ngx_http_monitoring_main_conf_t, collect_system), + NULL }, + + { ngx_string("monitor_collect_nginx"), + NGX_HTTP_MAIN_CONF|NGX_CONF_FLAG, + ngx_conf_set_flag_slot, + NGX_HTTP_MAIN_CONF_OFFSET, + offsetof(ngx_http_monitoring_main_conf_t, collect_nginx), + NULL }, + + { ngx_string("monitor_collect_network"), + NGX_HTTP_MAIN_CONF|NGX_CONF_FLAG, + ngx_conf_set_flag_slot, + NGX_HTTP_MAIN_CONF_OFFSET, + offsetof(ngx_http_monitoring_main_conf_t, collect_network), + NULL }, + + { ngx_string("monitor_access_log"), + NGX_HTTP_MAIN_CONF|NGX_CONF_FLAG, + ngx_conf_set_flag_slot, + NGX_HTTP_MAIN_CONF_OFFSET, + offsetof(ngx_http_monitoring_main_conf_t, access_log), + NULL }, + + { ngx_string("monitor_max_top_urls"), + NGX_HTTP_MAIN_CONF|NGX_CONF_TAKE1, + ngx_conf_set_num_slot, + NGX_HTTP_MAIN_CONF_OFFSET, + offsetof(ngx_http_monitoring_main_conf_t, max_top_urls), + NULL }, + + { ngx_string("monitor_api_token"), + NGX_HTTP_MAIN_CONF|NGX_HTTP_SRV_CONF|NGX_HTTP_LOC_CONF|NGX_CONF_TAKE1, + ngx_conf_set_str_slot, + NGX_HTTP_LOC_CONF_OFFSET, + offsetof(ngx_http_monitoring_loc_conf_t, api_token), + NULL }, + + { ngx_string("monitor_basic_auth"), + NGX_HTTP_MAIN_CONF|NGX_HTTP_SRV_CONF|NGX_HTTP_LOC_CONF|NGX_CONF_TAKE1, + ngx_conf_set_str_slot, + NGX_HTTP_LOC_CONF_OFFSET, + offsetof(ngx_http_monitoring_loc_conf_t, basic_auth), + NULL }, + + { ngx_string("monitor_cors"), + NGX_HTTP_MAIN_CONF|NGX_HTTP_SRV_CONF|NGX_HTTP_LOC_CONF|NGX_CONF_TAKE1, + ngx_conf_set_str_slot, + NGX_HTTP_LOC_CONF_OFFSET, + offsetof(ngx_http_monitoring_loc_conf_t, cors), + NULL }, + + { ngx_string("monitor_rate_limit"), + NGX_HTTP_MAIN_CONF|NGX_HTTP_SRV_CONF|NGX_HTTP_LOC_CONF|NGX_CONF_TAKE1, + ngx_conf_set_num_slot, + NGX_HTTP_LOC_CONF_OFFSET, + offsetof(ngx_http_monitoring_loc_conf_t, rate_limit), + NULL }, + + { ngx_string("monitor_allow"), + NGX_HTTP_MAIN_CONF|NGX_HTTP_SRV_CONF|NGX_HTTP_LOC_CONF|NGX_CONF_TAKE1, + ngx_http_monitoring_access_rule, + NGX_HTTP_LOC_CONF_OFFSET, + 0, + NULL }, + + { ngx_string("monitor_deny"), + NGX_HTTP_MAIN_CONF|NGX_HTTP_SRV_CONF|NGX_HTTP_LOC_CONF|NGX_CONF_TAKE1, + ngx_http_monitoring_access_rule, + NGX_HTTP_LOC_CONF_OFFSET, + 0, + NULL }, + + ngx_null_command +}; + + +static ngx_http_module_t ngx_http_monitoring_module_ctx = { + NULL, /* preconfiguration */ + ngx_http_monitoring_init, /* postconfiguration */ + + ngx_http_monitoring_create_main_conf, /* create main configuration */ + ngx_http_monitoring_init_main_conf, /* init main configuration */ + + NULL, /* create server configuration */ + NULL, /* merge server configuration */ + + ngx_http_monitoring_create_loc_conf, /* create location configuration */ + ngx_http_monitoring_merge_loc_conf /* merge location configuration */ +}; + + +ngx_module_t ngx_http_monitoring_module = { + NGX_MODULE_V1, + &ngx_http_monitoring_module_ctx, + ngx_http_monitoring_commands, + NGX_HTTP_MODULE, + NULL, /* init master */ + NULL, /* init module */ + ngx_http_monitoring_init_process, /* init process */ + NULL, /* init thread */ + NULL, /* exit thread */ + ngx_http_monitoring_exit_process, /* exit process */ + NULL, /* exit master */ + NGX_MODULE_V1_PADDING +}; + + +static void * +ngx_http_monitoring_create_main_conf(ngx_conf_t *cf) +{ + ngx_http_monitoring_main_conf_t *mmcf; + + mmcf = ngx_pcalloc(cf->pool, sizeof(ngx_http_monitoring_main_conf_t)); + if (mmcf == NULL) { + return NULL; + } + + mmcf->refresh_interval = NGX_CONF_UNSET_MSEC; + mmcf->history = NGX_CONF_UNSET_MSEC; + mmcf->resolution = NGX_CONF_UNSET_MSEC; + mmcf->shm_size = NGX_CONF_UNSET_SIZE; + mmcf->max_top_urls = NGX_CONF_UNSET_UINT; + mmcf->collect_system = NGX_CONF_UNSET; + mmcf->collect_nginx = NGX_CONF_UNSET; + mmcf->collect_network = NGX_CONF_UNSET; + mmcf->access_log = NGX_CONF_UNSET; + + return mmcf; +} + + +static char * +ngx_http_monitoring_init_main_conf(ngx_conf_t *cf, void *conf) +{ + ngx_http_monitoring_main_conf_t *mmcf = conf; + + ngx_conf_init_msec_value(mmcf->refresh_interval, 1000); + ngx_conf_init_msec_value(mmcf->history, 300000); + ngx_conf_init_msec_value(mmcf->resolution, 1000); + ngx_conf_init_size_value(mmcf->shm_size, + NGX_HTTP_MONITORING_DEFAULT_SHM_SIZE); + ngx_conf_init_uint_value(mmcf->max_top_urls, 100); + ngx_conf_init_value(mmcf->collect_system, 1); + ngx_conf_init_value(mmcf->collect_nginx, 1); + ngx_conf_init_value(mmcf->collect_network, 1); + ngx_conf_init_value(mmcf->access_log, 1); + + if (mmcf->refresh_interval < NGX_HTTP_MONITORING_MIN_INTERVAL) { + ngx_conf_log_error(NGX_LOG_EMERG, cf, 0, + "monitor_refresh_interval must be at least 100ms"); + return NGX_CONF_ERROR; + } + + if (mmcf->resolution < NGX_HTTP_MONITORING_MIN_INTERVAL) { + ngx_conf_log_error(NGX_LOG_EMERG, cf, 0, + "monitor_resolution must be at least 100ms"); + return NGX_CONF_ERROR; + } + + if (mmcf->max_top_urls == 0 + || mmcf->max_top_urls > NGX_HTTP_MONITORING_TOP_URLS_MAX) + { + ngx_conf_log_error(NGX_LOG_EMERG, cf, 0, + "monitor_max_top_urls must be between 1 and %d", + NGX_HTTP_MONITORING_TOP_URLS_MAX); + return NGX_CONF_ERROR; + } + + return NGX_CONF_OK; +} + + +static void * +ngx_http_monitoring_create_loc_conf(ngx_conf_t *cf) +{ + ngx_http_monitoring_loc_conf_t *mlcf; + + mlcf = ngx_pcalloc(cf->pool, sizeof(ngx_http_monitoring_loc_conf_t)); + if (mlcf == NULL) { + return NULL; + } + + mlcf->enabled = NGX_CONF_UNSET; + mlcf->dashboard = NGX_CONF_UNSET; + mlcf->api = NGX_CONF_UNSET; + mlcf->sse = NGX_CONF_UNSET; + mlcf->rate_limit = NGX_CONF_UNSET_UINT; + + return mlcf; +} + + +static char * +ngx_http_monitoring_merge_loc_conf(ngx_conf_t *cf, void *parent, void *child) +{ + ngx_http_monitoring_loc_conf_t *prev = parent; + ngx_http_monitoring_loc_conf_t *conf = child; + + ngx_conf_merge_value(conf->enabled, prev->enabled, 0); + ngx_conf_merge_value(conf->dashboard, prev->dashboard, 1); + ngx_conf_merge_value(conf->api, prev->api, 1); + ngx_conf_merge_value(conf->sse, prev->sse, 1); + ngx_conf_merge_uint_value(conf->rate_limit, prev->rate_limit, 60); + ngx_conf_merge_str_value(conf->api_token, prev->api_token, ""); + ngx_conf_merge_str_value(conf->basic_auth, prev->basic_auth, ""); + ngx_conf_merge_str_value(conf->cors, prev->cors, ""); + + if (conf->access_rules == NULL) { + conf->access_rules = prev->access_rules; + } + + return NGX_CONF_OK; +} + + +static char * +ngx_http_monitoring_access_rule(ngx_conf_t *cf, ngx_command_t *cmd, void *conf) +{ + ngx_http_monitoring_loc_conf_t *mlcf = conf; + ngx_http_monitoring_access_rule_t *rule; + ngx_str_t *value; + ngx_int_t rc; + + if (mlcf->access_rules == NULL) { + mlcf->access_rules = ngx_array_create(cf->pool, 4, + sizeof(ngx_http_monitoring_access_rule_t)); + if (mlcf->access_rules == NULL) { + return NGX_CONF_ERROR; + } + } + + rule = ngx_array_push(mlcf->access_rules); + if (rule == NULL) { + return NGX_CONF_ERROR; + } + + value = cf->args->elts; + ngx_memzero(rule, sizeof(ngx_http_monitoring_access_rule_t)); + rule->deny = (cmd->name.len == sizeof("monitor_deny") - 1); + + if (value[1].len == 3 && ngx_strncmp(value[1].data, "all", 3) == 0) { + rule->cidr.family = AF_UNSPEC; + return NGX_CONF_OK; + } + + rc = ngx_ptocidr(&value[1], &rule->cidr); + if (rc == NGX_ERROR) { + ngx_conf_log_error(NGX_LOG_EMERG, cf, 0, + "invalid CIDR \"%V\"", &value[1]); + return NGX_CONF_ERROR; + } + + if (rc == NGX_DONE) { + ngx_conf_log_error(NGX_LOG_WARN, cf, 0, + "low address bits of \"%V\" are meaningless", + &value[1]); + } + + return NGX_CONF_OK; +} + + +static ngx_int_t +ngx_http_monitoring_init(ngx_conf_t *cf) +{ + ngx_http_handler_pt *h; + ngx_http_core_main_conf_t *cmcf; + ngx_http_monitoring_main_conf_t *mmcf; + ngx_str_t name = ngx_string("ngx_http_monitoring"); + + mmcf = ngx_http_conf_get_module_main_conf(cf, ngx_http_monitoring_module); + + mmcf->shm_zone = ngx_shared_memory_add(cf, &name, mmcf->shm_size, + &ngx_http_monitoring_module); + if (mmcf->shm_zone == NULL) { + return NGX_ERROR; + } + + mmcf->shm_zone->init = ngx_http_monitoring_init_shm; + mmcf->shm_zone->data = mmcf; + + cmcf = ngx_http_conf_get_module_main_conf(cf, ngx_http_core_module); + + h = ngx_array_push(&cmcf->phases[NGX_HTTP_CONTENT_PHASE].handlers); + if (h == NULL) { + return NGX_ERROR; + } + *h = ngx_http_monitoring_content_handler; + + h = ngx_array_push(&cmcf->phases[NGX_HTTP_LOG_PHASE].handlers); + if (h == NULL) { + return NGX_ERROR; + } + *h = ngx_http_monitoring_log_handler; + + return NGX_OK; +} + + +static ngx_int_t +ngx_http_monitoring_init_process(ngx_cycle_t *cycle) +{ + return ngx_http_monitoring_init_collector(cycle); +} + + +static void +ngx_http_monitoring_exit_process(ngx_cycle_t *cycle) +{ + ngx_http_monitoring_main_conf_t *mmcf; + ngx_http_monitoring_shctx_t *sh; + ngx_uint_t slot; + + mmcf = ngx_http_cycle_get_module_main_conf(cycle, + ngx_http_monitoring_module); + if (mmcf == NULL) { + return; + } + + mmcf->collector_active = 0; + if (mmcf->collector_event.timer_set) { + ngx_del_timer(&mmcf->collector_event); + } + + sh = (ngx_http_monitoring_shctx_t *) mmcf->sh; + if (sh == NULL || ngx_process_slot < 0) { + return; + } + + slot = ((ngx_uint_t) ngx_process_slot) % NGX_HTTP_MONITORING_WORKERS_MAX; + sh->workers[slot].active = 0; + sh->workers[slot].last_seen = ngx_time(); +} + + +static ngx_int_t +ngx_http_monitoring_content_handler(ngx_http_request_t *r) +{ + ngx_int_t rc; + ngx_http_monitoring_endpoint_e endpoint; + ngx_http_monitoring_loc_conf_t *mlcf; + + endpoint = ngx_http_monitoring_match_endpoint(r); + if (endpoint == NGX_HTTP_MONITORING_EP_NONE) { + return NGX_DECLINED; + } + + mlcf = ngx_http_get_module_loc_conf(r, ngx_http_monitoring_module); + if (mlcf == NULL || !mlcf->enabled) { + return NGX_DECLINED; + } + + if (r->method == NGX_HTTP_OPTIONS) { + return ngx_http_monitoring_preflight(r, mlcf); + } + + if (!(r->method & (NGX_HTTP_GET|NGX_HTTP_HEAD))) { + return NGX_HTTP_NOT_ALLOWED; + } + + if (endpoint == NGX_HTTP_MONITORING_EP_DASHBOARD && !mlcf->dashboard) { + return NGX_HTTP_NOT_FOUND; + } + + if (endpoint >= NGX_HTTP_MONITORING_EP_API_FULL + && endpoint <= NGX_HTTP_MONITORING_EP_API_REQUESTS + && !mlcf->api) + { + return NGX_HTTP_NOT_FOUND; + } + + if (endpoint == NGX_HTTP_MONITORING_EP_LIVE && !mlcf->sse) { + return NGX_HTTP_NOT_FOUND; + } + + rc = ngx_http_monitoring_authorize(r, mlcf, endpoint); + if (rc != NGX_OK) { + return rc; + } + + ngx_http_monitoring_add_cors(r, mlcf); + + switch (endpoint) { + case NGX_HTTP_MONITORING_EP_DASHBOARD: + return ngx_http_monitoring_send_dashboard(r); + case NGX_HTTP_MONITORING_EP_LIVE: + return ngx_http_monitoring_send_sse(r); + case NGX_HTTP_MONITORING_EP_PROMETHEUS: + return ngx_http_monitoring_send_prometheus(r); + case NGX_HTTP_MONITORING_EP_HEALTH: + return ngx_http_monitoring_send_health(r); + default: + return ngx_http_monitoring_send_json(r, endpoint); + } +} + + +static ngx_int_t +ngx_http_monitoring_log_handler(ngx_http_request_t *r) +{ + ngx_http_monitoring_main_conf_t *mmcf; + + mmcf = ngx_http_get_module_main_conf(r, ngx_http_monitoring_module); + if (mmcf == NULL || !mmcf->access_log || mmcf->sh == NULL) { + return NGX_OK; + } + + if (ngx_http_monitoring_match_endpoint(r) != NGX_HTTP_MONITORING_EP_NONE) { + return NGX_OK; + } + + ngx_http_monitoring_account_request(r); + + return NGX_OK; +} + + +ngx_http_monitoring_endpoint_e +ngx_http_monitoring_match_endpoint(ngx_http_request_t *r) +{ + ngx_str_t *u = &r->uri; + + if (u->len == sizeof("/monitor") - 1 + && ngx_strncmp(u->data, "/monitor", u->len) == 0) + { + return NGX_HTTP_MONITORING_EP_DASHBOARD; + } + + if (u->len == sizeof("/monitor/") - 1 + && ngx_strncmp(u->data, "/monitor/", u->len) == 0) + { + return NGX_HTTP_MONITORING_EP_DASHBOARD; + } + + if (u->len == sizeof("/monitor/live") - 1 + && ngx_strncmp(u->data, "/monitor/live", u->len) == 0) + { + return NGX_HTTP_MONITORING_EP_LIVE; + } + + if (u->len == sizeof("/monitor/metrics") - 1 + && ngx_strncmp(u->data, "/monitor/metrics", u->len) == 0) + { + return NGX_HTTP_MONITORING_EP_PROMETHEUS; + } + + if (u->len == sizeof("/monitor/health") - 1 + && ngx_strncmp(u->data, "/monitor/health", u->len) == 0) + { + return NGX_HTTP_MONITORING_EP_HEALTH; + } + + if (u->len == sizeof("/monitor/api") - 1 + && ngx_strncmp(u->data, "/monitor/api", u->len) == 0) + { + return NGX_HTTP_MONITORING_EP_API_FULL; + } + + if (u->len == sizeof("/monitor/api/system") - 1 + && ngx_strncmp(u->data, "/monitor/api/system", u->len) == 0) + { + return NGX_HTTP_MONITORING_EP_API_SYSTEM; + } + + if (u->len == sizeof("/monitor/api/nginx") - 1 + && ngx_strncmp(u->data, "/monitor/api/nginx", u->len) == 0) + { + return NGX_HTTP_MONITORING_EP_API_NGINX; + } + + if (u->len == sizeof("/monitor/api/network") - 1 + && ngx_strncmp(u->data, "/monitor/api/network", u->len) == 0) + { + return NGX_HTTP_MONITORING_EP_API_NETWORK; + } + + if (u->len == sizeof("/monitor/api/disk") - 1 + && ngx_strncmp(u->data, "/monitor/api/disk", u->len) == 0) + { + return NGX_HTTP_MONITORING_EP_API_DISK; + } + + if (u->len == sizeof("/monitor/api/processes") - 1 + && ngx_strncmp(u->data, "/monitor/api/processes", u->len) == 0) + { + return NGX_HTTP_MONITORING_EP_API_PROCESSES; + } + + if (u->len == sizeof("/monitor/api/upstreams") - 1 + && ngx_strncmp(u->data, "/monitor/api/upstreams", u->len) == 0) + { + return NGX_HTTP_MONITORING_EP_API_UPSTREAMS; + } + + if (u->len == sizeof("/monitor/api/connections") - 1 + && ngx_strncmp(u->data, "/monitor/api/connections", u->len) == 0) + { + return NGX_HTTP_MONITORING_EP_API_CONNECTIONS; + } + + if (u->len == sizeof("/monitor/api/requests") - 1 + && ngx_strncmp(u->data, "/monitor/api/requests", u->len) == 0) + { + return NGX_HTTP_MONITORING_EP_API_REQUESTS; + } + + return NGX_HTTP_MONITORING_EP_NONE; +} + + +static ngx_int_t +ngx_http_monitoring_authorize(ngx_http_request_t *r, + ngx_http_monitoring_loc_conf_t *mlcf, + ngx_http_monitoring_endpoint_e endpoint) +{ + ngx_str_t forbidden = ngx_string("forbidden\n"); + ngx_str_t unauthorized = ngx_string("unauthorized\n"); + ngx_str_t limited = ngx_string("rate limited\n"); + + if (ngx_http_monitoring_match_access(r, mlcf) != NGX_OK) { + return ngx_http_monitoring_plain_response(r, NGX_HTTP_FORBIDDEN, + &forbidden); + } + + if (ngx_http_monitoring_check_basic_auth(r, mlcf) != NGX_OK) { + r->headers_out.www_authenticate = + ngx_list_push(&r->headers_out.headers); + if (r->headers_out.www_authenticate == NULL) { + return NGX_HTTP_INTERNAL_SERVER_ERROR; + } + + r->headers_out.www_authenticate->hash = 1; + ngx_str_set(&r->headers_out.www_authenticate->key, + "WWW-Authenticate"); + ngx_str_set(&r->headers_out.www_authenticate->value, + "Basic realm=\"nginx monitoring\""); + + return ngx_http_monitoring_plain_response(r, NGX_HTTP_UNAUTHORIZED, + &unauthorized); + } + + if (endpoint != NGX_HTTP_MONITORING_EP_DASHBOARD + && ngx_http_monitoring_check_api_token(r, mlcf) != NGX_OK) + { + return ngx_http_monitoring_plain_response(r, NGX_HTTP_UNAUTHORIZED, + &unauthorized); + } + + if (ngx_http_monitoring_check_rate_limit(r, mlcf) != NGX_OK) { + return ngx_http_monitoring_plain_response(r, 429, &limited); + } + + return NGX_OK; +} + + +static ngx_int_t +ngx_http_monitoring_preflight(ngx_http_request_t *r, + ngx_http_monitoring_loc_conf_t *mlcf) +{ + ngx_http_monitoring_add_cors(r, mlcf); + + r->headers_out.status = NGX_HTTP_NO_CONTENT; + r->headers_out.content_length_n = 0; + r->header_only = 1; + + return ngx_http_send_header(r); +} + + +static void +ngx_http_monitoring_add_cors(ngx_http_request_t *r, + ngx_http_monitoring_loc_conf_t *mlcf) +{ + ngx_table_elt_t *h; + + if (mlcf->cors.len == 0 + || (mlcf->cors.len == 3 + && ngx_strncmp(mlcf->cors.data, "off", 3) == 0)) + { + return; + } + + h = ngx_list_push(&r->headers_out.headers); + if (h == NULL) { + return; + } + h->hash = 1; + ngx_str_set(&h->key, "Access-Control-Allow-Origin"); + h->value = mlcf->cors; + + h = ngx_list_push(&r->headers_out.headers); + if (h == NULL) { + return; + } + h->hash = 1; + ngx_str_set(&h->key, "Access-Control-Allow-Methods"); + ngx_str_set(&h->value, "GET, HEAD, OPTIONS"); + + h = ngx_list_push(&r->headers_out.headers); + if (h == NULL) { + return; + } + h->hash = 1; + ngx_str_set(&h->key, "Access-Control-Allow-Headers"); + ngx_str_set(&h->value, "Authorization, X-Monitor-Token, Content-Type"); +} + + +static ngx_int_t +ngx_http_monitoring_plain_response(ngx_http_request_t *r, ngx_uint_t status, + ngx_str_t *body) +{ + ngx_buf_t *b; + ngx_chain_t out; + + ngx_http_discard_request_body(r); + + r->headers_out.status = status; + r->headers_out.content_length_n = body->len; + ngx_str_set(&r->headers_out.content_type, "text/plain"); + + if (r->method == NGX_HTTP_HEAD) { + r->header_only = 1; + return ngx_http_send_header(r); + } + + b = ngx_calloc_buf(r->pool); + if (b == NULL) { + return NGX_HTTP_INTERNAL_SERVER_ERROR; + } + + b->pos = body->data; + b->last = body->data + body->len; + b->memory = 1; + b->last_buf = 1; + + out.buf = b; + out.next = NULL; + + ngx_http_send_header(r); + return ngx_http_output_filter(r, &out); +} + + +static ngx_int_t +ngx_http_monitoring_match_access(ngx_http_request_t *r, + ngx_http_monitoring_loc_conf_t *mlcf) +{ + ngx_uint_t i, j; + ngx_http_monitoring_access_rule_t *rule; + struct sockaddr *sa; + struct sockaddr_in *sin; +#if (NGX_HAVE_INET6) + struct sockaddr_in6 *sin6; +#endif + u_char *addr; + u_char *mask; + u_char *cidr; + + if (mlcf->access_rules == NULL || mlcf->access_rules->nelts == 0) { + return NGX_OK; + } + + rule = mlcf->access_rules->elts; + sa = r->connection->sockaddr; + + for (i = 0; i < mlcf->access_rules->nelts; i++) { + if (rule[i].cidr.family == AF_UNSPEC) { + return rule[i].deny ? NGX_DECLINED : NGX_OK; + } + + if (sa->sa_family != (ngx_int_t) rule[i].cidr.family) { + continue; + } + + if (sa->sa_family == AF_INET) { + sin = (struct sockaddr_in *) sa; + if ((sin->sin_addr.s_addr & rule[i].cidr.u.in.mask) + == rule[i].cidr.u.in.addr) + { + return rule[i].deny ? NGX_DECLINED : NGX_OK; + } + continue; + } + +#if (NGX_HAVE_INET6) + if (sa->sa_family == AF_INET6) { + sin6 = (struct sockaddr_in6 *) sa; + addr = sin6->sin6_addr.s6_addr; + mask = rule[i].cidr.u.in6.mask.s6_addr; + cidr = rule[i].cidr.u.in6.addr.s6_addr; + + for (j = 0; j < 16; j++) { + if ((addr[j] & mask[j]) != cidr[j]) { + break; + } + } + + if (j == 16) { + return rule[i].deny ? NGX_DECLINED : NGX_OK; + } + } +#endif + } + + return NGX_OK; +} + + +static ngx_int_t +ngx_http_monitoring_check_basic_auth(ngx_http_request_t *r, + ngx_http_monitoring_loc_conf_t *mlcf) +{ + ngx_table_elt_t *authorization; + ngx_str_t encoded, decoded, expected; + + if (mlcf->basic_auth.len == 0 + || (mlcf->basic_auth.len == 3 + && ngx_strncmp(mlcf->basic_auth.data, "off", 3) == 0)) + { + return NGX_OK; + } + + authorization = r->headers_in.authorization; + if (authorization == NULL || authorization->value.len <= 6 + || ngx_strncasecmp(authorization->value.data, + (u_char *) "Basic ", 6) != 0) + { + return NGX_DECLINED; + } + + encoded.data = authorization->value.data + 6; + encoded.len = authorization->value.len - 6; + decoded.len = ngx_base64_decoded_length(encoded.len); + decoded.data = ngx_pnalloc(r->pool, decoded.len); + if (decoded.data == NULL) { + return NGX_DECLINED; + } + + if (ngx_decode_base64(&decoded, &encoded) != NGX_OK) { + return NGX_DECLINED; + } + + expected = mlcf->basic_auth; + return ngx_http_monitoring_consttime_eq(&decoded, &expected); +} + + +static ngx_int_t +ngx_http_monitoring_check_api_token(ngx_http_request_t *r, + ngx_http_monitoring_loc_conf_t *mlcf) +{ + ngx_table_elt_t *h; + ngx_str_t header = ngx_string("X-Monitor-Token"); + ngx_str_t token, bearer, query; + + if (mlcf->api_token.len == 0) { + return NGX_OK; + } + + h = ngx_http_monitoring_find_header(r, &header); + if (h != NULL) { + token = h->value; + if (ngx_http_monitoring_consttime_eq(&token, &mlcf->api_token) + == NGX_OK) + { + return NGX_OK; + } + } + + h = r->headers_in.authorization; + if (h != NULL && h->value.len > 7 + && ngx_strncasecmp(h->value.data, (u_char *) "Bearer ", 7) == 0) + { + bearer.data = h->value.data + 7; + bearer.len = h->value.len - 7; + + if (ngx_http_monitoring_consttime_eq(&bearer, &mlcf->api_token) + == NGX_OK) + { + return NGX_OK; + } + } + + if (ngx_http_arg(r, (u_char *) "token", 5, &query) == NGX_OK + && ngx_http_monitoring_consttime_eq(&query, &mlcf->api_token) + == NGX_OK) + { + return NGX_OK; + } + + return NGX_DECLINED; +} + + +static ngx_int_t +ngx_http_monitoring_check_rate_limit(ngx_http_request_t *r, + ngx_http_monitoring_loc_conf_t *mlcf) +{ + time_t now; + ngx_atomic_uint_t count; + ngx_http_monitoring_shctx_t *sh; + ngx_http_monitoring_main_conf_t *mmcf; + + if (mlcf->rate_limit == 0) { + return NGX_OK; + } + + sh = ngx_http_monitoring_get_shctx(r); + mmcf = ngx_http_monitoring_get_main_conf(r); + if (sh == NULL || mmcf == NULL || mmcf->shpool == NULL) { + return NGX_OK; + } + + now = ngx_time(); + + if (sh->rate_limit_window != (ngx_atomic_uint_t) now) { + ngx_shmtx_lock(&mmcf->shpool->mutex); + if (sh->rate_limit_window != (ngx_atomic_uint_t) now) { + sh->rate_limit_window = (ngx_atomic_uint_t) now; + sh->rate_limit_count = 0; + } + ngx_shmtx_unlock(&mmcf->shpool->mutex); + } + + count = ngx_atomic_fetch_add(&sh->rate_limit_count, 1) + 1; + if (count > mlcf->rate_limit) { + ngx_atomic_fetch_add(&sh->connections.rate_limited, 1); + return NGX_DECLINED; + } + + return NGX_OK; +} + + +static ngx_table_elt_t * +ngx_http_monitoring_find_header(ngx_http_request_t *r, ngx_str_t *name) +{ + ngx_list_part_t *part; + ngx_table_elt_t *h; + ngx_uint_t i; + + part = &r->headers_in.headers.part; + h = part->elts; + + for (i = 0; /* void */; i++) { + if (i >= part->nelts) { + if (part->next == NULL) { + break; + } + part = part->next; + h = part->elts; + i = 0; + } + + if (h[i].key.len == name->len + && ngx_strncasecmp(h[i].key.data, name->data, name->len) == 0) + { + return &h[i]; + } + } + + return NULL; +} + + +static ngx_int_t +ngx_http_monitoring_consttime_eq(ngx_str_t *a, ngx_str_t *b) +{ + size_t i, max; + ngx_uint_t diff; + + max = (a->len > b->len) ? a->len : b->len; + diff = (ngx_uint_t) (a->len ^ b->len); + + for (i = 0; i < max; i++) { + diff |= (ngx_uint_t) + ((i < a->len ? a->data[i] : 0) + ^ (i < b->len ? b->data[i] : 0)); + } + + return diff == 0 ? NGX_OK : NGX_DECLINED; +} diff --git a/src/ngx_http_monitoring_shm.c b/src/ngx_http_monitoring_shm.c new file mode 100644 index 0000000..343de1c --- /dev/null +++ b/src/ngx_http_monitoring_shm.c @@ -0,0 +1,558 @@ +#include "ngx_http_monitoring.h" + +static void ngx_http_monitoring_update_worker(ngx_http_monitoring_shctx_t *sh, + ngx_msec_t latency, off_t bytes, ngx_uint_t is_error); +static void ngx_http_monitoring_update_status(ngx_http_monitoring_shctx_t *sh, + ngx_uint_t status); +static void ngx_http_monitoring_update_upstream_slot(ngx_slab_pool_t *shpool, + ngx_http_monitoring_upstream_entry_t *entries, ngx_uint_t capacity, + ngx_str_t *peer, ngx_msec_t latency, ngx_uint_t status); +static ngx_uint_t ngx_http_monitoring_worker_slot(void); + +static ngx_uint_t ngx_http_monitoring_latency_bounds[ + NGX_HTTP_MONITORING_LATENCY_BUCKETS - 1 +] = { 1, 5, 10, 25, 50, 100, 250, 500, 1000, 2500, 5000, 10000, 30000 }; + +static off_t ngx_http_monitoring_size_bounds[ + NGX_HTTP_MONITORING_SIZE_BUCKETS - 1 +] = { 512, 1024, 2048, 4096, 8192, 16384, 32768, 65536, + 262144, 1048576, 10485760 }; + + +ngx_int_t +ngx_http_monitoring_init_shm(ngx_shm_zone_t *shm_zone, void *data) +{ + ngx_http_monitoring_main_conf_t *ommcf = data; + ngx_http_monitoring_main_conf_t *mmcf; + ngx_http_monitoring_shctx_t *sh; + ngx_slab_pool_t *shpool; + ngx_uint_t capacity; + + mmcf = shm_zone->data; + + if (ommcf) { + mmcf->sh = ommcf->sh; + mmcf->shpool = ommcf->shpool; + sh = (ngx_http_monitoring_shctx_t *) mmcf->sh; + capacity = mmcf->history / ngx_max(mmcf->resolution, 1); + if (capacity == 0) { + capacity = 1; + } + if (capacity > NGX_HTTP_MONITORING_HISTORY_MAX) { + capacity = NGX_HTTP_MONITORING_HISTORY_MAX; + } + sh->history_capacity = capacity; + if ((ngx_uint_t) sh->history_count > capacity) { + sh->history_count = capacity; + } + return NGX_OK; + } + + shpool = (ngx_slab_pool_t *) shm_zone->shm.addr; + + if (shm_zone->shm.exists) { + sh = shpool->data; + if (sh == NULL) { + return NGX_ERROR; + } + + mmcf->sh = sh; + mmcf->shpool = shpool; + capacity = mmcf->history / ngx_max(mmcf->resolution, 1); + if (capacity == 0) { + capacity = 1; + } + if (capacity > NGX_HTTP_MONITORING_HISTORY_MAX) { + capacity = NGX_HTTP_MONITORING_HISTORY_MAX; + } + sh->history_capacity = capacity; + if ((ngx_uint_t) sh->history_count > capacity) { + sh->history_count = capacity; + } + return NGX_OK; + } + + sh = ngx_slab_alloc(shpool, sizeof(ngx_http_monitoring_shctx_t)); + if (sh == NULL) { + return NGX_ERROR; + } + + ngx_memzero(sh, sizeof(ngx_http_monitoring_shctx_t)); + + capacity = mmcf->history / ngx_max(mmcf->resolution, 1); + if (capacity == 0) { + capacity = 1; + } + if (capacity > NGX_HTTP_MONITORING_HISTORY_MAX) { + capacity = NGX_HTTP_MONITORING_HISTORY_MAX; + } + + sh->history_capacity = capacity; + sh->initialized = 1; + sh->generation = 1; + shpool->data = sh; + + mmcf->sh = sh; + mmcf->shpool = shpool; + + return NGX_OK; +} + + +ngx_http_monitoring_shctx_t * +ngx_http_monitoring_get_shctx(ngx_http_request_t *r) +{ + ngx_http_monitoring_main_conf_t *mmcf; + + mmcf = ngx_http_get_module_main_conf(r, ngx_http_monitoring_module); + if (mmcf == NULL) { + return NULL; + } + + return (ngx_http_monitoring_shctx_t *) mmcf->sh; +} + + +ngx_http_monitoring_main_conf_t * +ngx_http_monitoring_get_main_conf(ngx_http_request_t *r) +{ + return ngx_http_get_module_main_conf(r, ngx_http_monitoring_module); +} + + +void +ngx_http_monitoring_account_request(ngx_http_request_t *r) +{ + ngx_time_t *tp; + ngx_msec_int_t ms; + ngx_msec_t latency; + off_t bytes; + ngx_uint_t status, method, lb, sb, is_error; + ngx_http_monitoring_shctx_t *sh; + ngx_http_monitoring_main_conf_t *mmcf; + ngx_str_t uri, ua; + + sh = ngx_http_monitoring_get_shctx(r); + mmcf = ngx_http_monitoring_get_main_conf(r); + if (sh == NULL || mmcf == NULL) { + return; + } + + tp = ngx_timeofday(); + ms = (ngx_msec_int_t) ((tp->sec - r->start_sec) * 1000 + + (tp->msec - r->start_msec)); + latency = (ms > 0) ? (ngx_msec_t) ms : 0; + + status = r->headers_out.status; + if (status == 0) { + status = 499; + } + + bytes = r->headers_out.content_length_n; + if (bytes < 0) { + bytes = 0; + } + + ngx_atomic_fetch_add(&sh->requests.total, 1); + ngx_atomic_fetch_add(&sh->requests.responses, 1); + ngx_atomic_fetch_add(&sh->requests.bytes, (ngx_atomic_int_t) bytes); + ngx_atomic_fetch_add(&sh->requests.request_time_ms_total, latency); + + method = ngx_http_monitoring_method_index(r->method); + ngx_atomic_fetch_add(&sh->requests.method[method], 1); + + lb = ngx_http_monitoring_latency_bucket(latency); + ngx_atomic_fetch_add(&sh->requests.latency_bucket[lb], 1); + + sb = ngx_http_monitoring_size_bucket(bytes); + ngx_atomic_fetch_add(&sh->requests.size_bucket[sb], 1); + + ngx_http_monitoring_update_status(sh, status); + +#if (NGX_HTTP_SSL) + if (r->connection->ssl) { + ngx_atomic_fetch_add(&sh->connections.ssl_requests, 1); + if (r->connection->requests <= 1) { + ngx_atomic_fetch_add(&sh->connections.ssl_handshakes, 1); + } + } +#endif + + if (r->connection->requests > 1) { + ngx_atomic_fetch_add(&sh->connections.keepalive_requests, 1); + } + + is_error = status >= 500; + ngx_http_monitoring_update_worker(sh, latency, bytes, is_error); + + uri = r->uri; + ngx_http_monitoring_update_top(mmcf->shpool, sh->urls, + ngx_min(mmcf->max_top_urls, + NGX_HTTP_MONITORING_TOP_URLS_MAX), + &uri, latency, bytes, + status >= 400); + + if (r->headers_in.user_agent) { + ua = r->headers_in.user_agent->value; + ngx_http_monitoring_update_top(mmcf->shpool, sh->user_agents, + NGX_HTTP_MONITORING_TOP_UA_MAX, + &ua, latency, bytes, + status >= 400); + } + + ngx_http_monitoring_account_upstream(r, latency); +} + + +void +ngx_http_monitoring_account_upstream(ngx_http_request_t *r, ngx_msec_t latency) +{ + ngx_uint_t i; + ngx_http_upstream_state_t *state; + ngx_http_monitoring_shctx_t *sh; + ngx_http_monitoring_main_conf_t *mmcf; + ngx_str_t peer; + ngx_msec_t response_time; + + if (r->upstream_states == NULL || r->upstream_states->nelts == 0) { + return; + } + + sh = ngx_http_monitoring_get_shctx(r); + mmcf = ngx_http_monitoring_get_main_conf(r); + if (sh == NULL || mmcf == NULL || mmcf->shpool == NULL) { + return; + } + + state = r->upstream_states->elts; + + for (i = 0; i < r->upstream_states->nelts; i++) { + if (state[i].peer && state[i].peer->len) { + peer = *state[i].peer; + } else { + ngx_str_set(&peer, "unknown"); + } + + response_time = state[i].response_time; + if (response_time == (ngx_msec_t) -1) { + response_time = latency; + } + + ngx_http_monitoring_update_upstream_slot(mmcf->shpool, sh->upstreams, + NGX_HTTP_MONITORING_UPSTREAMS_MAX, &peer, response_time, + state[i].status); + } +} + + +static void +ngx_http_monitoring_update_worker(ngx_http_monitoring_shctx_t *sh, + ngx_msec_t latency, off_t bytes, ngx_uint_t is_error) +{ + ngx_uint_t slot; + ngx_http_monitoring_worker_metric_t *w; + + slot = ngx_http_monitoring_worker_slot(); + w = &sh->workers[slot]; + + w->pid = ngx_pid; + w->last_seen = ngx_time(); + ngx_atomic_fetch_add(&w->requests, 1); + ngx_atomic_fetch_add(&w->bytes, (ngx_atomic_int_t) bytes); + + if (latency > 30000) { + ngx_atomic_fetch_add(&w->errors, 1); + } + + if (is_error) { + ngx_atomic_fetch_add(&w->errors, 1); + } +} + + +static void +ngx_http_monitoring_update_status(ngx_http_monitoring_shctx_t *sh, + ngx_uint_t status) +{ + if (status < 200) { + ngx_atomic_fetch_add(&sh->requests.status_1xx, 1); + } else if (status < 300) { + ngx_atomic_fetch_add(&sh->requests.status_2xx, 1); + } else if (status < 400) { + ngx_atomic_fetch_add(&sh->requests.status_3xx, 1); + } else if (status < 500) { + ngx_atomic_fetch_add(&sh->requests.status_4xx, 1); + } else { + ngx_atomic_fetch_add(&sh->requests.status_5xx, 1); + } +} + + +static void +ngx_http_monitoring_update_upstream_slot(ngx_slab_pool_t *shpool, + ngx_http_monitoring_upstream_entry_t *entries, ngx_uint_t capacity, + ngx_str_t *peer, ngx_msec_t latency, ngx_uint_t status) +{ + ngx_uint_t i, victim, empty, failure; + ngx_atomic_uint_t min_hits, hits; + uint32_t hash; + size_t len; + ngx_http_monitoring_upstream_entry_t *e; + + if (peer->len == 0 || capacity == 0) { + return; + } + + hash = ngx_http_monitoring_hash(peer); + empty = capacity; + victim = 0; + min_hits = (ngx_atomic_uint_t) -1; + + for (i = 0; i < capacity; i++) { + e = &entries[i]; + + if (e->requests == 0 && empty == capacity) { + empty = i; + continue; + } + + if (e->hash == hash + && ngx_strncmp(e->peer, peer->data, + ngx_min(peer->len, NGX_HTTP_MONITORING_KEY_LEN - 1)) + == 0 + && e->peer[ngx_min(peer->len, NGX_HTTP_MONITORING_KEY_LEN - 1)] + == '\0') + { + failure = (status == 0 || status >= 500); + ngx_atomic_fetch_add(&e->requests, 1); + ngx_atomic_fetch_add(&e->latency_ms_total, latency); + e->last_seen = ngx_time(); + if (failure) { + ngx_atomic_fetch_add(&e->failures, 1); + } + if (status >= 400 && status < 500) { + ngx_atomic_fetch_add(&e->status_4xx, 1); + } + if (status >= 500) { + ngx_atomic_fetch_add(&e->status_5xx, 1); + } + return; + } + + hits = e->requests; + if (hits < min_hits) { + min_hits = hits; + victim = i; + } + } + + ngx_shmtx_lock(&shpool->mutex); + + i = (empty != capacity) ? empty : victim; + e = &entries[i]; + ngx_memzero(e, sizeof(ngx_http_monitoring_upstream_entry_t)); + + len = ngx_min(peer->len, NGX_HTTP_MONITORING_KEY_LEN - 1); + ngx_memcpy(e->peer, peer->data, len); + e->peer[len] = '\0'; + e->hash = hash; + e->last_seen = ngx_time(); + + ngx_shmtx_unlock(&shpool->mutex); + + failure = (status == 0 || status >= 500); + ngx_atomic_fetch_add(&e->requests, 1); + ngx_atomic_fetch_add(&e->latency_ms_total, latency); + if (failure) { + ngx_atomic_fetch_add(&e->failures, 1); + } + if (status >= 400 && status < 500) { + ngx_atomic_fetch_add(&e->status_4xx, 1); + } + if (status >= 500) { + ngx_atomic_fetch_add(&e->status_5xx, 1); + } +} + + +void +ngx_http_monitoring_update_top(ngx_slab_pool_t *shpool, + ngx_http_monitoring_top_entry_t *entries, ngx_uint_t capacity, + ngx_str_t *key, ngx_msec_t latency, off_t bytes, ngx_uint_t is_error) +{ + ngx_uint_t i, empty, victim; + ngx_atomic_uint_t min_hits, hits; + uint32_t hash; + size_t len; + ngx_http_monitoring_top_entry_t *e; + + if (shpool == NULL || key->len == 0 || capacity == 0) { + return; + } + + hash = ngx_http_monitoring_hash(key); + empty = capacity; + victim = 0; + min_hits = (ngx_atomic_uint_t) -1; + len = ngx_min(key->len, NGX_HTTP_MONITORING_KEY_LEN - 1); + + for (i = 0; i < capacity; i++) { + e = &entries[i]; + + if (e->hits == 0 && empty == capacity) { + empty = i; + continue; + } + + if (e->hash == hash + && ngx_strncmp(e->key, key->data, len) == 0 + && e->key[len] == '\0') + { + ngx_atomic_fetch_add(&e->hits, 1); + ngx_atomic_fetch_add(&e->bytes, (ngx_atomic_int_t) bytes); + ngx_atomic_fetch_add(&e->latency_ms_total, latency); + e->last_seen = ngx_time(); + if (is_error) { + ngx_atomic_fetch_add(&e->errors, 1); + } + return; + } + + hits = e->hits; + if (hits < min_hits) { + min_hits = hits; + victim = i; + } + } + + ngx_shmtx_lock(&shpool->mutex); + + i = (empty != capacity) ? empty : victim; + e = &entries[i]; + ngx_memzero(e, sizeof(ngx_http_monitoring_top_entry_t)); + + ngx_memcpy(e->key, key->data, len); + e->key[len] = '\0'; + e->hash = hash; + e->last_seen = ngx_time(); + + ngx_shmtx_unlock(&shpool->mutex); + + ngx_atomic_fetch_add(&e->hits, 1); + ngx_atomic_fetch_add(&e->bytes, (ngx_atomic_int_t) bytes); + ngx_atomic_fetch_add(&e->latency_ms_total, latency); + if (is_error) { + ngx_atomic_fetch_add(&e->errors, 1); + } +} + + +ngx_uint_t +ngx_http_monitoring_method_index(ngx_uint_t method) +{ + if (method & NGX_HTTP_GET) { + return NGX_HTTP_MONITORING_METHOD_GET; + } + if (method & NGX_HTTP_POST) { + return NGX_HTTP_MONITORING_METHOD_POST; + } + if (method & NGX_HTTP_PUT) { + return NGX_HTTP_MONITORING_METHOD_PUT; + } + if (method & NGX_HTTP_DELETE) { + return NGX_HTTP_MONITORING_METHOD_DELETE; + } + if (method & NGX_HTTP_HEAD) { + return NGX_HTTP_MONITORING_METHOD_HEAD; + } + if (method & NGX_HTTP_OPTIONS) { + return NGX_HTTP_MONITORING_METHOD_OPTIONS; + } +#ifdef NGX_HTTP_PATCH + if (method & NGX_HTTP_PATCH) { + return NGX_HTTP_MONITORING_METHOD_PATCH; + } +#endif + return NGX_HTTP_MONITORING_METHOD_OTHER; +} + + +ngx_uint_t +ngx_http_monitoring_latency_bucket(ngx_msec_t ms) +{ + ngx_uint_t i; + + for (i = 0; i < NGX_HTTP_MONITORING_LATENCY_BUCKETS - 1; i++) { + if (ms <= ngx_http_monitoring_latency_bounds[i]) { + return i; + } + } + + return NGX_HTTP_MONITORING_LATENCY_BUCKETS - 1; +} + + +ngx_uint_t +ngx_http_monitoring_size_bucket(off_t bytes) +{ + ngx_uint_t i; + + for (i = 0; i < NGX_HTTP_MONITORING_SIZE_BUCKETS - 1; i++) { + if (bytes <= ngx_http_monitoring_size_bounds[i]) { + return i; + } + } + + return NGX_HTTP_MONITORING_SIZE_BUCKETS - 1; +} + + +double +ngx_http_monitoring_percentile(ngx_http_monitoring_shctx_t *sh, + double percentile) +{ + ngx_uint_t i; + ngx_atomic_uint_t total, count, target; + + total = 0; + for (i = 0; i < NGX_HTTP_MONITORING_LATENCY_BUCKETS; i++) { + total += sh->requests.latency_bucket[i]; + } + + if (total == 0) { + return 0; + } + + target = (ngx_atomic_uint_t) ((double) total * percentile / 100.0); + if (target == 0) { + target = 1; + } + + count = 0; + for (i = 0; i < NGX_HTTP_MONITORING_LATENCY_BUCKETS - 1; i++) { + count += sh->requests.latency_bucket[i]; + if (count >= target) { + return (double) ngx_http_monitoring_latency_bounds[i]; + } + } + + return 30000.0; +} + + +uint32_t +ngx_http_monitoring_hash(ngx_str_t *value) +{ + return ngx_crc32_short(value->data, value->len); +} + + +static ngx_uint_t +ngx_http_monitoring_worker_slot(void) +{ + if (ngx_process_slot < 0) { + return 0; + } + + return ((ngx_uint_t) ngx_process_slot) % NGX_HTTP_MONITORING_WORKERS_MAX; +} diff --git a/src/ngx_http_monitoring_sse.c b/src/ngx_http_monitoring_sse.c new file mode 100644 index 0000000..270b56e --- /dev/null +++ b/src/ngx_http_monitoring_sse.c @@ -0,0 +1,296 @@ +#include "ngx_http_monitoring.h" + +typedef struct { + ngx_http_request_t *r; + ngx_event_t event; + ngx_msec_t interval; + ngx_uint_t closed; + ngx_uint_t sequence; + ngx_uint_t slot; + u_char data[2][16384]; + ngx_buf_t bufs[2]; + ngx_chain_t chains[2]; +} ngx_http_monitoring_sse_client_t; + +static void ngx_http_monitoring_sse_timer(ngx_event_t *ev); +static void ngx_http_monitoring_sse_cleanup(void *data); +static ngx_int_t ngx_http_monitoring_sse_write( + ngx_http_monitoring_sse_client_t *client); +static u_char *ngx_http_monitoring_sse_payload(u_char *p, u_char *last, + ngx_http_monitoring_sse_client_t *client, + ngx_http_monitoring_shctx_t *sh); +static ngx_atomic_uint_t ngx_http_monitoring_sse_sum_net( + ngx_http_monitoring_shctx_t *sh, ngx_uint_t tx); +static ngx_atomic_uint_t ngx_http_monitoring_sse_sum_disk( + ngx_http_monitoring_shctx_t *sh, ngx_uint_t write); +static void ngx_http_monitoring_sse_add_header(ngx_http_request_t *r, + const char *key, const char *value); + + +ngx_int_t +ngx_http_monitoring_send_sse(ngx_http_request_t *r) +{ + ngx_int_t rc; + ngx_pool_cleanup_t *cln; + ngx_http_monitoring_sse_client_t *client; + ngx_http_monitoring_main_conf_t *mmcf; + ngx_http_monitoring_shctx_t *sh; + + mmcf = ngx_http_monitoring_get_main_conf(r); + sh = ngx_http_monitoring_get_shctx(r); + if (mmcf == NULL || sh == NULL) { + return NGX_HTTP_SERVICE_UNAVAILABLE; + } + + ngx_http_discard_request_body(r); + + r->headers_out.status = NGX_HTTP_OK; + r->headers_out.content_length_n = -1; + ngx_str_set(&r->headers_out.content_type, "text/event-stream"); + ngx_http_monitoring_sse_add_header(r, "Cache-Control", "no-store"); + ngx_http_monitoring_sse_add_header(r, "X-Accel-Buffering", "no"); + ngx_http_monitoring_sse_add_header(r, "Connection", "keep-alive"); + + rc = ngx_http_send_header(r); + if (rc == NGX_ERROR || rc > NGX_OK || r->header_only) { + return rc; + } + + client = ngx_pcalloc(r->pool, sizeof(ngx_http_monitoring_sse_client_t)); + if (client == NULL) { + return NGX_HTTP_INTERNAL_SERVER_ERROR; + } + + client->r = r; + client->interval = ngx_max(mmcf->refresh_interval, 1000); + client->event.handler = ngx_http_monitoring_sse_timer; + client->event.data = client; + client->event.log = r->connection->log; + + cln = ngx_pool_cleanup_add(r->pool, 0); + if (cln == NULL) { + return NGX_HTTP_INTERNAL_SERVER_ERROR; + } + cln->handler = ngx_http_monitoring_sse_cleanup; + cln->data = client; + + ngx_atomic_fetch_add(&sh->connections.sse_clients, 1); + + r->main->count++; + + if (ngx_http_monitoring_sse_write(client) == NGX_ERROR) { + ngx_http_finalize_request(r, NGX_ERROR); + return NGX_DONE; + } + + ngx_add_timer(&client->event, client->interval); + + return NGX_DONE; +} + + +static void +ngx_http_monitoring_sse_timer(ngx_event_t *ev) +{ + ngx_http_monitoring_sse_client_t *client; + ngx_http_request_t *r; + + client = ev->data; + r = client->r; + + if (client->closed || r->connection->error) { + ngx_http_finalize_request(r, NGX_ERROR); + return; + } + + if (r->connection->buffered) { + ngx_add_timer(ev, client->interval); + return; + } + + if (ngx_http_monitoring_sse_write(client) == NGX_ERROR) { + ngx_http_finalize_request(r, NGX_ERROR); + return; + } + + ngx_add_timer(ev, client->interval); +} + + +static void +ngx_http_monitoring_sse_cleanup(void *data) +{ + ngx_http_monitoring_sse_client_t *client = data; + ngx_http_monitoring_shctx_t *sh; + + if (client->closed) { + return; + } + + client->closed = 1; + + if (client->event.timer_set) { + ngx_del_timer(&client->event); + } + + sh = ngx_http_monitoring_get_shctx(client->r); + if (sh != NULL && sh->connections.sse_clients > 0) { + ngx_atomic_fetch_add(&sh->connections.sse_clients, -1); + } +} + + +static ngx_int_t +ngx_http_monitoring_sse_write(ngx_http_monitoring_sse_client_t *client) +{ + ngx_http_request_t *r; + ngx_http_monitoring_shctx_t *sh; + ngx_buf_t *b; + ngx_chain_t *out; + u_char *p, *last; + + r = client->r; + sh = ngx_http_monitoring_get_shctx(r); + if (sh == NULL) { + return NGX_ERROR; + } + + client->slot ^= 1; + p = client->data[client->slot]; + last = p + sizeof(client->data[client->slot]); + + p = ngx_slprintf(p, last, "retry: 3000\n: heartbeat %ui\n", + client->sequence); + p = ngx_http_monitoring_sse_payload(p, last, client, sh); + if (p == last) { + return NGX_ERROR; + } + + b = &client->bufs[client->slot]; + ngx_memzero(b, sizeof(ngx_buf_t)); + b->pos = client->data[client->slot]; + b->last = p; + b->memory = 1; + b->flush = 1; + + out = &client->chains[client->slot]; + out->buf = b; + out->next = NULL; + + if (ngx_http_output_filter(r, out) == NGX_ERROR) { + return NGX_ERROR; + } + + ngx_atomic_fetch_add(&sh->connections.sse_events, 1); + client->sequence++; + + return NGX_OK; +} + + +static u_char * +ngx_http_monitoring_sse_payload(u_char *p, u_char *last, + ngx_http_monitoring_sse_client_t *client, + ngx_http_monitoring_shctx_t *sh) +{ + ngx_time_t *tp; + ngx_atomic_uint_t rx, tx, dr, dw; + double mem_pct; + + tp = ngx_timeofday(); + rx = ngx_http_monitoring_sse_sum_net(sh, 0); + tx = ngx_http_monitoring_sse_sum_net(sh, 1); + dr = ngx_http_monitoring_sse_sum_disk(sh, 0); + dw = ngx_http_monitoring_sse_sum_disk(sh, 1); + + mem_pct = 0; + if (sh->system.mem_total > 0) { + mem_pct = ((double) (sh->system.mem_total - sh->system.mem_available) + * 100.0) / (double) sh->system.mem_total; + } + + return ngx_slprintf(p, last, + "id: %ui\n" + "event: metrics\n" + "data: {\"version\":%d,\"timestamp\":%T,\"sequence\":%ui," + "\"system\":{\"cpu\":{\"usage\":%.3f,\"cores\":%uA}," + "\"memory\":{\"used_pct\":%.3f,\"total\":%uA," + "\"available\":%uA}}," + "\"requests\":{\"total\":%uA,\"requests_per_sec\":%.3f," + "\"responses_per_sec\":%.3f,\"latency\":{\"p95\":%.3f," + "\"p99\":%.3f},\"status\":{\"4xx\":%uA,\"5xx\":%uA}}," + "\"connections\":{\"active\":%uA,\"reading\":%uA," + "\"writing\":%uA,\"waiting\":%uA,\"sse_clients\":%uA," + "\"keepalive_requests\":%uA}," + "\"network\":{\"rx_bytes\":%uA,\"tx_bytes\":%uA}," + "\"disk\":{\"read_bytes\":%uA,\"write_bytes\":%uA}}\n\n", + client->sequence, NGX_HTTP_MONITORING_VERSION, tp->sec, + client->sequence, + (double) sh->system.usage_milli / 1000.0, sh->system.cores, + mem_pct, sh->system.mem_total, sh->system.mem_available, + sh->requests.total, (double) sh->ewma_rps_milli / 1000.0, + (double) sh->ewma_resp_milli / 1000.0, + ngx_http_monitoring_percentile(sh, 95.0), + ngx_http_monitoring_percentile(sh, 99.0), + sh->requests.status_4xx, sh->requests.status_5xx, + sh->connections.active, sh->connections.reading, + sh->connections.writing, sh->connections.waiting, + sh->connections.sse_clients, sh->connections.keepalive_requests, + rx, tx, dr, dw); +} + + +static ngx_atomic_uint_t +ngx_http_monitoring_sse_sum_net(ngx_http_monitoring_shctx_t *sh, ngx_uint_t tx) +{ + ngx_uint_t i, count; + ngx_atomic_uint_t total; + + total = 0; + count = ngx_min((ngx_uint_t) sh->iface_count, + NGX_HTTP_MONITORING_IFACES_MAX); + + for (i = 0; i < count; i++) { + total += tx ? sh->ifaces[i].tx_bytes : sh->ifaces[i].rx_bytes; + } + + return total; +} + + +static ngx_atomic_uint_t +ngx_http_monitoring_sse_sum_disk(ngx_http_monitoring_shctx_t *sh, + ngx_uint_t write) +{ + ngx_uint_t i, count; + ngx_atomic_uint_t total; + + total = 0; + count = ngx_min((ngx_uint_t) sh->disk_count, + NGX_HTTP_MONITORING_DISKS_MAX); + + for (i = 0; i < count; i++) { + total += write ? sh->disks[i].write_bytes : sh->disks[i].read_bytes; + } + + return total; +} + + +static void +ngx_http_monitoring_sse_add_header(ngx_http_request_t *r, const char *key, + const char *value) +{ + ngx_table_elt_t *h; + + h = ngx_list_push(&r->headers_out.headers); + if (h == NULL) { + return; + } + + h->hash = 1; + h->key.len = ngx_strlen(key); + h->key.data = (u_char *) key; + h->value.len = ngx_strlen(value); + h->value.data = (u_char *) value; +}