chore: refactor release workflow for modular distro and upstream builds

2026-06-09 21:20:20 +03:30
parent f97ac2f7c5
commit 058d835f07
10 changed files with 1423 additions and 200 deletions
--- a/scripts/build-distro-release.sh
+++ b/scripts/build-distro-release.sh
@@ -0,0 +1,537 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+MODULE_NAME="${MODULE_NAME:-ngx_http_monitoring_module}"
+DEB_PACKAGE_NAME="${DEB_PACKAGE_NAME:-libnginx-mod-http-monitoring}"
+NGINX_INSTALL_PACKAGE="${NGINX_INSTALL_PACKAGE:-nginx}"
+NEXT_VERSION="${NEXT_VERSION:-0.0.0}"
+NEXT_TAG="${NEXT_TAG:-v${NEXT_VERSION}}"
+LATEST_TAG="${LATEST_TAG:-none}"
+TARGET_ARCH="${TARGET_ARCH:-$(uname -m)}"
+DOCKER_PLATFORM="${DOCKER_PLATFORM:-unknown}"
+DISTRO_IMAGE="${DISTRO_IMAGE:-unknown}"
+DIST_DIR="${DIST_DIR:-/work/dist}"
+WORK_ROOT="${WORK_ROOT:-/work/.release-work}"
+VALIDATION_PORT="${VALIDATION_PORT:-18080}"
+GITHUB_REF="${GITHUB_REF:-unknown}"
+GITHUB_SHA="${GITHUB_SHA:-unknown}"
+
+export DEBIAN_FRONTEND=noninteractive
+
+slugify() {
+    tr '[:upper:]' '[:lower:]' \
+        | sed -E 's/[^a-z0-9.+~]+/./g; s/^[.]+//; s/[.]+$//'
+}
+
+fail() {
+    echo "error: $*" >&2
+    exit 1
+}
+
+apt_install() {
+    apt-get install -y --no-install-recommends "$@"
+}
+
+find_nginx_abi() {
+    local pkg
+    local abi
+
+    for pkg in "$@"; do
+        abi="$(
+            apt-cache show "$pkg" 2>/dev/null \
+            | awk -F': ' '
+                /^Provides:/ {
+                    n = split($2, provides, ",")
+                    for (i = 1; i <= n; i++) {
+                        gsub(/^[ \t]+|[ \t]+$/, "", provides[i])
+                        if (provides[i] ~ /^nginx-abi-/) {
+                            print provides[i]
+                            exit
+                        }
+                    }
+                }
+            ' \
+            | head -n 1 \
+            || true
+        )"
+
+        if [[ -n "$abi" ]]; then
+            printf '%s\n' "$abi"
+            return 0
+        fi
+    done
+}
+
+validate_module() {
+    local module_path="$1"
+    local runtime_root="$2"
+    local config_path="$runtime_root/nginx.conf"
+    local base_url="http://127.0.0.1:${VALIDATION_PORT}"
+    local status
+    local dashboard
+    local sse_output
+
+    rm -rf "$runtime_root"
+    mkdir -p \
+        "$runtime_root/logs" \
+        "$runtime_root/temp/client_body" \
+        "$runtime_root/temp/proxy" \
+        "$runtime_root/temp/fastcgi" \
+        "$runtime_root/temp/uwsgi" \
+        "$runtime_root/temp/scgi"
+
+    {
+        printf 'load_module %s;\n\n' "$module_path"
+        printf 'worker_processes 1;\n'
+        printf 'error_log %s/logs/error.log info;\n' "$runtime_root"
+        printf 'pid %s/logs/nginx.pid;\n\n' "$runtime_root"
+        printf 'events {\n'
+        printf '    worker_connections 1024;\n'
+        printf '}\n\n'
+        printf 'http {\n'
+        printf '    access_log off;\n'
+        printf '    default_type application/octet-stream;\n'
+        printf '    client_body_temp_path %s/temp/client_body;\n' "$runtime_root"
+        printf '    proxy_temp_path %s/temp/proxy;\n' "$runtime_root"
+        printf '    fastcgi_temp_path %s/temp/fastcgi;\n' "$runtime_root"
+        printf '    uwsgi_temp_path %s/temp/uwsgi;\n' "$runtime_root"
+        printf '    scgi_temp_path %s/temp/scgi;\n\n' "$runtime_root"
+        printf '    monitor_refresh_interval 1s;\n'
+        printf '    monitor_history 1m;\n'
+        printf '    monitor_resolution 1s;\n'
+        printf '    monitor_shm_size 8m;\n'
+        printf '    monitor_collect_system on;\n'
+        printf '    monitor_collect_nginx on;\n'
+        printf '    monitor_collect_network on;\n'
+        printf '    monitor_access_log on;\n'
+        printf '    monitor_max_top_urls 100;\n\n'
+        printf '    server {\n'
+        printf '        listen 127.0.0.1:%s;\n' "$VALIDATION_PORT"
+        printf '        server_name localhost;\n\n'
+        printf '        location /monitor {\n'
+        printf '            monitor on;\n'
+        printf '            monitor_dashboard on;\n'
+        printf '            monitor_api on;\n'
+        printf '            monitor_sse on;\n'
+        printf '            monitor_allow all;\n'
+        printf '            monitor_basic_auth off;\n'
+        printf '            monitor_cors off;\n'
+        printf '            monitor_rate_limit 120;\n'
+        printf '        }\n\n'
+        printf '        location / {\n'
+        printf '            return 200 "ok\\n";\n'
+        printf '        }\n'
+        printf '    }\n'
+        printf '}\n'
+    } > "$config_path"
+
+    nginx -t -p "$runtime_root" -c "$config_path"
+
+    cleanup() {
+        nginx -p "$runtime_root" -c "$config_path" -s quit >/dev/null 2>&1 || true
+    }
+    trap cleanup RETURN
+
+    nginx -p "$runtime_root" -c "$config_path"
+
+    for attempt in $(seq 1 50); do
+        if curl -fsS "$base_url/" >/dev/null; then
+            break
+        fi
+
+        if [[ "$attempt" -eq 50 ]]; then
+            echo "Nginx did not become ready" >&2
+            return 1
+        fi
+
+        sleep 0.2
+    done
+
+    curl -fsS "$base_url/" | grep -q '^ok'
+
+    status="$(curl -fsS -o "$runtime_root/dashboard.html" -w '%{http_code}' "$base_url/monitor")"
+    [[ "$status" == "200" ]] || fail "/monitor returned HTTP $status"
+    dashboard="$(cat "$runtime_root/dashboard.html")"
+    printf '%s' "$dashboard" | grep -q '<title>Nginx Monitor</title>'
+    printf '%s' "$dashboard" | grep -q '/monitor/api'
+    printf '%s' "$dashboard" | grep -q '/monitor/live'
+
+    curl -fsS "$base_url/monitor/api" \
+        | jq -e '.version == 1 and has("system") and has("nginx") and has("requests") and has("history")' >/dev/null
+    curl -fsS "$base_url/monitor/api/system" \
+        | jq -e '.version == 1 and .scope == "system" and has("system")' >/dev/null
+    curl -fsS "$base_url/monitor/api/nginx" \
+        | jq -e '.version == 1 and .scope == "nginx" and has("nginx")' >/dev/null
+    curl -fsS "$base_url/monitor/health" \
+        | jq -e '.version == 1 and .scope == "health" and .status == "ok"' >/dev/null
+    curl -fsS "$base_url/monitor/metrics" \
+        | grep -q '^nginx_monitor_requests_total '
+
+    sse_output="$(timeout 5s curl -fsS -N "$base_url/monitor/live" || true)"
+    printf '%s' "$sse_output" | grep -q '^event: metrics'
+    printf '%s' "$sse_output" | grep -q '^data: {'
+
+    cleanup
+    trap - RETURN
+}
+
+mkdir -p "$DIST_DIR" "$WORK_ROOT"
+
+printf '#!/bin/sh\nexit 101\n' > /usr/sbin/policy-rc.d
+chmod +x /usr/sbin/policy-rc.d
+
+apt-get update
+apt_install ca-certificates curl gnupg
+bash /work/scripts/enable-apt-source-repos.sh
+apt-get update
+apt_install \
+    bash \
+    binutils \
+    build-essential \
+    ca-certificates \
+    curl \
+    dpkg-dev \
+    file \
+    findutils \
+    jq \
+    libpcre2-dev \
+    libssl-dev \
+    python3 \
+    xz-utils \
+    zlib1g-dev \
+    "$NGINX_INSTALL_PACKAGE"
+
+. /etc/os-release
+distro_id="${ID:-unknown}"
+distro_version="${VERSION_ID:-unknown}"
+distro_codename="${VERSION_CODENAME:-${UBUNTU_CODENAME:-unknown}}"
+distro_slug="$(printf '%s-%s-%s' "$distro_id" "$distro_version" "$distro_codename" | slugify)"
+
+deb_arch="$(dpkg --print-architecture)"
+nginx_bin="$(command -v nginx)"
+nginx_binary_package="$(dpkg-query -S "$nginx_bin" | head -n 1 | cut -d: -f1)"
+nginx_binary_version="$(dpkg-query -W -f='${Version}' "$nginx_binary_package")"
+source_package="$(dpkg-query -W -f='${source:Package}' "$nginx_binary_package" 2>/dev/null || true)"
+source_version="$(dpkg-query -W -f='${source:Version}' "$nginx_binary_package" 2>/dev/null || true)"
+
+if [[ -z "$source_package" || "$source_package" == *'{'* ]]; then
+    source_package="$nginx_binary_package"
+fi
+
+if [[ -z "$source_version" || "$source_version" == *'{'* ]]; then
+    source_version="$nginx_binary_version"
+fi
+
+nginx_abi="$(find_nginx_abi "$nginx_binary_package" "$NGINX_INSTALL_PACKAGE" nginx-core nginx-full nginx-light nginx-extras nginx)"
+
+echo "Distro: ${distro_id} ${distro_version} (${distro_codename}), arch ${deb_arch}"
+echo "Nginx binary package: ${nginx_binary_package} ${nginx_binary_version}"
+echo "Nginx source package: ${source_package} ${source_version}"
+echo "Nginx ABI package: ${nginx_abi:-none found}"
+
+src_root="$WORK_ROOT/${distro_slug}/${deb_arch}/source"
+build_root="$WORK_ROOT/${distro_slug}/${deb_arch}/build"
+rm -rf "$src_root" "$build_root"
+mkdir -p "$src_root" "$build_root"
+
+(
+    cd "$src_root"
+    apt-get source "${source_package}=${source_version}" \
+        || apt-get source "$source_package"
+)
+
+dsc_path="$(find "$src_root" -maxdepth 1 -name '*.dsc' | head -n 1)"
+[[ -n "$dsc_path" ]] || fail "apt-get source did not download a .dsc file"
+
+dsc_version="$(awk '/^Version:/ { print $2; exit }' "$dsc_path")"
+if [[ "$dsc_version" != "$source_version" ]]; then
+    fail "downloaded source version ${dsc_version}, expected ${source_version}"
+fi
+
+nginx_src="$(find "$src_root" -mindepth 1 -maxdepth 1 -type d | head -n 1)"
+[[ -n "$nginx_src" ]] || fail "apt-get source did not unpack an nginx source directory"
+
+apt-get build-dep -y --no-install-recommends "${source_package}=${source_version}" \
+    || apt-get build-dep -y --no-install-recommends "$source_package"
+
+nginx_build_info="$build_root/nginx-build-info.json"
+python3 /work/scripts/nginx-build-info.py --nginx-src "$nginx_src" --json > "$nginx_build_info"
+modules_path="$(python3 /work/scripts/nginx-build-info.py --modules-path)"
+configure_line="$(python3 /work/scripts/nginx-build-info.py --nginx-src "$nginx_src" --configure-line)"
+
+case "$modules_path" in
+    /*) ;;
+    *) modules_path="/${modules_path}" ;;
+esac
+
+source_version_slug="$(printf '%s' "$source_version" | slugify)"
+deb_revision_slug="$(printf '%s%s.nginx%s' "$distro_id" "$distro_version" "$source_version" | slugify)"
+deb_version="${NEXT_VERSION}-1+${deb_revision_slug}"
+build_metadata="$(printf '%s.%s.%s' "$distro_slug" "$deb_arch" "$source_version_slug" | slugify)"
+module_cflags="-DNGX_HTTP_MONITORING_MODULE_VERSION=\\\"${NEXT_VERSION}\\\" -DNGX_HTTP_MONITORING_BUILD_METADATA=\\\"${build_metadata}\\\""
+
+cd "$nginx_src"
+eval "./configure ${configure_line} --with-cc-opt=\"${module_cflags}\" --add-dynamic-module=/work"
+
+build_jobs="$(nproc)"
+if [[ "$TARGET_ARCH" != "linux-x86_64" && "$build_jobs" -gt 2 ]]; then
+    build_jobs=2
+fi
+
+make -j"$build_jobs" modules
+
+built_module="$nginx_src/objs/${MODULE_NAME}.so"
+[[ -f "$built_module" ]] || fail "module was not built at ${built_module}"
+
+artifact_base="${MODULE_NAME}-${NEXT_TAG}-${distro_slug}-nginx-${source_version_slug}-${deb_arch}"
+deb_file="${DEB_PACKAGE_NAME}_${deb_version}_${deb_arch}.deb"
+tarball_file="${artifact_base}.tar.gz"
+package_root="$build_root/package-root"
+doc_dir="$package_root/usr/share/doc/$DEB_PACKAGE_NAME"
+module_install_dir="$package_root/${modules_path#/}"
+modules_available_dir="$package_root/usr/share/nginx/modules-available"
+metadata_dir="$doc_dir/metadata"
+
+rm -rf "$package_root"
+mkdir -p "$module_install_dir" "$modules_available_dir" "$metadata_dir" "$package_root/DEBIAN"
+
+cp "$built_module" "$module_install_dir/${MODULE_NAME}.so"
+strip --strip-unneeded "$module_install_dir/${MODULE_NAME}.so" || true
+
+cat > "$modules_available_dir/mod-http-monitoring.conf" <<EOF
+load_module ${modules_path}/${MODULE_NAME}.so;
+EOF
+
+cp /work/README.md "$doc_dir/README.md"
+cp /work/LICENSE "$doc_dir/copyright"
+cp /work/docs/API.md "$doc_dir/API.md"
+cp /work/docs/CONFIGURATION.md "$doc_dir/CONFIGURATION.md"
+cp /work/docs/PERFORMANCE.md "$doc_dir/PERFORMANCE.md"
+cp /work/examples/monitoring-site.conf "$doc_dir/monitoring-site.conf.example"
+
+cat > "$doc_dir/changelog.Debian" <<EOF
+$DEB_PACKAGE_NAME (${deb_version}) unstable; urgency=medium
+
+  * Build ${NEXT_TAG} against ${distro_id} ${distro_version} nginx source ${source_version}.
+
+ -- GitHub Actions <actions@github.com>  $(date -Ru)
+EOF
+gzip -9n "$doc_dir/changelog.Debian"
+
+module_sha256="$(sha256sum "$module_install_dir/${MODULE_NAME}.so" | awk '{ print $1 }')"
+dsc_sha256="$(sha256sum "$dsc_path" | awk '{ print $1 }')"
+nginx_v_text="$metadata_dir/nginx-V.txt"
+nginx -V > "$nginx_v_text" 2>&1
+
+compatibility_json="$metadata_dir/compatibility.json"
+jq -n \
+    --slurpfile nginxBuild "$nginx_build_info" \
+    --arg schema "1" \
+    --arg module "$MODULE_NAME" \
+    --arg debPackage "$DEB_PACKAGE_NAME" \
+    --arg debVersion "$deb_version" \
+    --arg releaseTag "$NEXT_TAG" \
+    --arg releaseVersion "$NEXT_VERSION" \
+    --arg previousTag "$LATEST_TAG" \
+    --arg gitRef "$GITHUB_REF" \
+    --arg gitSha "$GITHUB_SHA" \
+    --arg distroImage "$DISTRO_IMAGE" \
+    --arg dockerPlatform "$DOCKER_PLATFORM" \
+    --arg targetArch "$TARGET_ARCH" \
+    --arg osId "$distro_id" \
+    --arg osVersion "$distro_version" \
+    --arg osCodename "$distro_codename" \
+    --arg debArch "$deb_arch" \
+    --arg nginxBinaryPackage "$nginx_binary_package" \
+    --arg nginxBinaryVersion "$nginx_binary_version" \
+    --arg nginxSourcePackage "$source_package" \
+    --arg nginxSourceVersion "$source_version" \
+    --arg nginxAbi "$nginx_abi" \
+    --arg modulesPath "$modules_path" \
+    --arg dscPath "$(basename "$dsc_path")" \
+    --arg dscSha256 "$dsc_sha256" \
+    --arg moduleSha256 "$module_sha256" \
+    --arg builtAt "$(date -u +%Y-%m-%dT%H:%M:%SZ)" \
+    --arg artifactBase "$artifact_base" \
+    --arg debFile "$deb_file" \
+    --arg tarballFile "$tarball_file" \
+    '{
+        schema: ($schema | tonumber),
+        module: $module,
+        release: {
+            tag: $releaseTag,
+            version: $releaseVersion,
+            previous_tag: $previousTag,
+            git_ref: $gitRef,
+            git_sha: $gitSha
+        },
+        target: {
+            os_id: $osId,
+            os_version: $osVersion,
+            os_codename: $osCodename,
+            architecture: $debArch,
+            requested_target: $targetArch,
+            docker_image: $distroImage,
+            docker_platform: $dockerPlatform
+        },
+        nginx: {
+            binary_package: $nginxBinaryPackage,
+            binary_version: $nginxBinaryVersion,
+            source_package: $nginxSourcePackage,
+            source_version: $nginxSourceVersion,
+            abi: $nginxAbi,
+            modules_path: $modulesPath,
+            build: $nginxBuild[0]
+        },
+        build: {
+            method: "distro-source-package",
+            dsc_file: $dscPath,
+            dsc_sha256: $dscSha256,
+            module_sha256: $moduleSha256,
+            built_at: $builtAt
+        },
+        artifacts: {
+            base_name: $artifactBase,
+            deb_package: $debPackage,
+            deb_version: $debVersion,
+            deb_file: $debFile,
+            tarball_file: $tarballFile,
+            compatibility_file: ($artifactBase + ".compatibility.json")
+        },
+        packaging: {
+            deb_depends: (
+                if $nginxAbi == "" then
+                    [($nginxBinaryPackage + " (= " + $nginxBinaryVersion + ")"), "libc6"]
+                else
+                    [$nginxAbi, ($nginxBinaryPackage + " (= " + $nginxBinaryVersion + ")"), "libc6"]
+                end
+            )
+        },
+        validation: {
+            packaged_nginx_config_test: false,
+            packaged_nginx_endpoint_smoke: false
+        }
+    }' > "$compatibility_json"
+
+cat > "$doc_dir/INSTALL.md" <<EOF
+# ${MODULE_NAME} for ${distro_id} ${distro_version} ${deb_arch}
+
+This package was built from the ${source_package} source package version
+${source_version} and validated against the packaged ${nginx_binary_package}
+binary version ${nginx_binary_version}.
+
+Recommended install:
+
+\`\`\`sh
+sudo apt install ./$(printf '%s_%s_%s.deb' "$DEB_PACKAGE_NAME" "$deb_version" "$deb_arch")
+\`\`\`
+
+The package installs the module at:
+
+\`\`\`text
+${modules_path}/${MODULE_NAME}.so
+\`\`\`
+
+Load it near the top of nginx.conf:
+
+\`\`\`nginx
+load_module ${modules_path}/${MODULE_NAME}.so;
+\`\`\`
+
+Compatibility guard:
+
+\`\`\`text
+$(if [[ -n "$nginx_abi" ]]; then printf 'Depends: %s\nDepends: %s (= %s)\n' "$nginx_abi" "$nginx_binary_package" "$nginx_binary_version"; else printf 'Depends: %s (= %s)\n' "$nginx_binary_package" "$nginx_binary_version"; fi)
+\`\`\`
+
+If a distro nginx update changes this ABI, apt will refuse to keep this module
+installed with the new nginx package. Rebuild and publish a new package for the
+new distro nginx package instead of reusing this binary.
+EOF
+
+depends=""
+if [[ -n "$nginx_abi" ]]; then
+    depends="$nginx_abi, $nginx_binary_package (= $nginx_binary_version), libc6"
+else
+    depends="$nginx_binary_package (= $nginx_binary_version), libc6"
+fi
+
+installed_size="$(du -ks "$package_root" | awk '{ print $1 }')"
+cat > "$package_root/DEBIAN/control" <<EOF
+Package: $DEB_PACKAGE_NAME
+Version: $deb_version
+Section: httpd
+Priority: optional
+Architecture: $deb_arch
+Maintainer: ngx_http_monitoring_module maintainers <noreply@github.com>
+Depends: $depends
+Installed-Size: $installed_size
+Description: Nginx HTTP monitoring dynamic module
+ Distro-built dynamic module for live Nginx monitoring, JSON APIs,
+ Server-Sent Events, Prometheus metrics, and an embedded dashboard.
+ This package is built against the target distribution nginx source package
+ and depends on the matching nginx module ABI.
+EOF
+
+deb_path="$DIST_DIR/$deb_file"
+dpkg-deb --build --root-owner-group "$package_root" "$deb_path"
+
+apt-get install -y --no-install-recommends "$deb_path"
+validate_module "${modules_path}/${MODULE_NAME}.so" "$build_root/runtime"
+
+jq \
+    --arg debFile "$deb_file" \
+    --arg validationTime "$(date -u +%Y-%m-%dT%H:%M:%SZ)" \
+    '.artifacts.deb_file = $debFile
+     | .validation.packaged_nginx_config_test = true
+     | .validation.packaged_nginx_endpoint_smoke = true
+     | .validation.validated_at = $validationTime' \
+    "$compatibility_json" > "$compatibility_json.tmp"
+mv "$compatibility_json.tmp" "$compatibility_json"
+dpkg-deb --build --root-owner-group "$package_root" "$deb_path"
+
+tar_root="$build_root/tar"
+tar_dir="$tar_root/$artifact_base"
+rm -rf "$tar_root"
+mkdir -p "$tar_dir/modules" "$tar_dir/docs" "$tar_dir/examples"
+cp "${modules_path}/${MODULE_NAME}.so" "$tar_dir/modules/${MODULE_NAME}.so"
+cp "$compatibility_json" "$tar_dir/COMPATIBILITY.json"
+cp "$doc_dir/INSTALL.md" "$tar_dir/INSTALL.md"
+cp "$doc_dir/README.md" "$tar_dir/README.md"
+cp "$doc_dir/API.md" "$tar_dir/docs/API.md"
+cp "$doc_dir/CONFIGURATION.md" "$tar_dir/docs/CONFIGURATION.md"
+cp "$doc_dir/PERFORMANCE.md" "$tar_dir/docs/PERFORMANCE.md"
+cp "$doc_dir/monitoring-site.conf.example" "$tar_dir/examples/monitoring-site.conf"
+cp "$nginx_v_text" "$tar_dir/NGINX-V.txt"
+
+{
+    printf 'module=%s\n' "$MODULE_NAME"
+    printf 'module_release=%s\n' "$NEXT_TAG"
+    printf 'module_version=%s\n' "$NEXT_VERSION"
+    printf 'module_build_metadata=%s\n' "$build_metadata"
+    printf 'distro=%s\n' "$distro_slug"
+    printf 'architecture=%s\n' "$deb_arch"
+    printf 'nginx_binary_package=%s\n' "$nginx_binary_package"
+    printf 'nginx_binary_version=%s\n' "$nginx_binary_version"
+    printf 'nginx_source_package=%s\n' "$source_package"
+    printf 'nginx_source_version=%s\n' "$source_version"
+    printf 'nginx_abi=%s\n' "$nginx_abi"
+    printf 'modules_path=%s\n' "$modules_path"
+    printf 'git_ref=%s\n' "$GITHUB_REF"
+    printf 'git_sha=%s\n' "$GITHUB_SHA"
+    printf 'built_at=%s\n' "$(date -u +%Y-%m-%dT%H:%M:%SZ)"
+} > "$tar_dir/METADATA.txt"
+
+(
+    cd "$tar_dir"
+    sha256sum "modules/${MODULE_NAME}.so" COMPATIBILITY.json > SHA256SUMS
+    file "modules/${MODULE_NAME}.so" > FILE.txt
+)
+
+tar -czf "$DIST_DIR/$tarball_file" -C "$tar_root" "$artifact_base"
+
+cp "$compatibility_json" "$DIST_DIR/${artifact_base}.compatibility.json"
+
+echo "Built $deb_path"
+echo "Built $DIST_DIR/$tarball_file"
+echo "Built $DIST_DIR/${artifact_base}.compatibility.json"
--- a/scripts/build-upstream-release.sh
+++ b/scripts/build-upstream-release.sh
@@ -0,0 +1,359 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+MODULE_NAME="${MODULE_NAME:-ngx_http_monitoring_module}"
+BUILD_NGINX_VERSIONS="${BUILD_NGINX_VERSIONS:-}"
+NEXT_VERSION="${NEXT_VERSION:-0.0.0}"
+NEXT_TAG="${NEXT_TAG:-v${NEXT_VERSION}}"
+LATEST_TAG="${LATEST_TAG:-none}"
+TARGET_ARCH="${TARGET_ARCH:-$(uname -m)}"
+DOCKER_PLATFORM="${DOCKER_PLATFORM:-unknown}"
+RELEASE_BUILD_IMAGE="${RELEASE_BUILD_IMAGE:-unknown}"
+RELEASE_LIBC_BASELINE="${RELEASE_LIBC_BASELINE:-unknown}"
+NGINX_CONFIGURE_ARGS="${NGINX_CONFIGURE_ARGS:---with-compat --with-http_ssl_module --with-http_stub_status_module}"
+DIST_DIR="${DIST_DIR:-/work/dist}"
+WORK_ROOT="${WORK_ROOT:-/work/.release-work/upstream}"
+VALIDATION_PORT="${VALIDATION_PORT:-18080}"
+GITHUB_REF="${GITHUB_REF:-unknown}"
+GITHUB_SHA="${GITHUB_SHA:-unknown}"
+
+export DEBIAN_FRONTEND=noninteractive
+
+fail() {
+    echo "error: $*" >&2
+    exit 1
+}
+
+slugify() {
+    tr '[:upper:]' '[:lower:]' \
+        | sed -E 's/[^a-z0-9.+~]+/./g; s/^[.]+//; s/[.]+$//'
+}
+
+validate_module() {
+    local nginx_bin="$1"
+    local module_path="$2"
+    local runtime_root="$3"
+    local config_path="$runtime_root/nginx.conf"
+    local base_url="http://127.0.0.1:${VALIDATION_PORT}"
+    local status
+    local dashboard
+    local sse_output
+
+    rm -rf "$runtime_root"
+    mkdir -p \
+        "$runtime_root/logs" \
+        "$runtime_root/temp/client_body" \
+        "$runtime_root/temp/proxy" \
+        "$runtime_root/temp/fastcgi" \
+        "$runtime_root/temp/uwsgi" \
+        "$runtime_root/temp/scgi"
+
+    {
+        printf 'load_module %s;\n\n' "$module_path"
+        printf 'worker_processes 1;\n'
+        printf 'error_log %s/logs/error.log info;\n' "$runtime_root"
+        printf 'pid %s/logs/nginx.pid;\n\n' "$runtime_root"
+        printf 'events {\n'
+        printf '    worker_connections 1024;\n'
+        printf '}\n\n'
+        printf 'http {\n'
+        printf '    access_log off;\n'
+        printf '    default_type application/octet-stream;\n'
+        printf '    client_body_temp_path %s/temp/client_body;\n' "$runtime_root"
+        printf '    proxy_temp_path %s/temp/proxy;\n' "$runtime_root"
+        printf '    fastcgi_temp_path %s/temp/fastcgi;\n' "$runtime_root"
+        printf '    uwsgi_temp_path %s/temp/uwsgi;\n' "$runtime_root"
+        printf '    scgi_temp_path %s/temp/scgi;\n\n' "$runtime_root"
+        printf '    monitor_refresh_interval 1s;\n'
+        printf '    monitor_history 1m;\n'
+        printf '    monitor_resolution 1s;\n'
+        printf '    monitor_shm_size 8m;\n'
+        printf '    monitor_collect_system on;\n'
+        printf '    monitor_collect_nginx on;\n'
+        printf '    monitor_collect_network on;\n'
+        printf '    monitor_access_log on;\n'
+        printf '    monitor_max_top_urls 100;\n\n'
+        printf '    server {\n'
+        printf '        listen 127.0.0.1:%s;\n' "$VALIDATION_PORT"
+        printf '        server_name localhost;\n\n'
+        printf '        location /monitor {\n'
+        printf '            monitor on;\n'
+        printf '            monitor_dashboard on;\n'
+        printf '            monitor_api on;\n'
+        printf '            monitor_sse on;\n'
+        printf '            monitor_allow all;\n'
+        printf '            monitor_basic_auth off;\n'
+        printf '            monitor_cors off;\n'
+        printf '            monitor_rate_limit 120;\n'
+        printf '        }\n\n'
+        printf '        location / {\n'
+        printf '            return 200 "ok\\n";\n'
+        printf '        }\n'
+        printf '    }\n'
+        printf '}\n'
+    } > "$config_path"
+
+    "$nginx_bin" -t -p "$runtime_root" -c "$config_path"
+
+    cleanup() {
+        "$nginx_bin" -p "$runtime_root" -c "$config_path" -s quit >/dev/null 2>&1 || true
+    }
+    trap cleanup RETURN
+
+    "$nginx_bin" -p "$runtime_root" -c "$config_path"
+
+    for attempt in $(seq 1 50); do
+        if curl -fsS "$base_url/" >/dev/null; then
+            break
+        fi
+
+        if [[ "$attempt" -eq 50 ]]; then
+            echo "Nginx did not become ready" >&2
+            return 1
+        fi
+
+        sleep 0.2
+    done
+
+    curl -fsS "$base_url/" | grep -q '^ok'
+
+    status="$(curl -fsS -o "$runtime_root/dashboard.html" -w '%{http_code}' "$base_url/monitor")"
+    [[ "$status" == "200" ]] || fail "/monitor returned HTTP $status"
+    dashboard="$(cat "$runtime_root/dashboard.html")"
+    printf '%s' "$dashboard" | grep -q '<title>Nginx Monitor</title>'
+    printf '%s' "$dashboard" | grep -q '/monitor/api'
+    printf '%s' "$dashboard" | grep -q '/monitor/live'
+
+    curl -fsS "$base_url/monitor/api" \
+        | jq -e '.version == 1 and has("system") and has("nginx") and has("requests") and has("history")' >/dev/null
+    curl -fsS "$base_url/monitor/api/system" \
+        | jq -e '.version == 1 and .scope == "system" and has("system")' >/dev/null
+    curl -fsS "$base_url/monitor/api/nginx" \
+        | jq -e '.version == 1 and .scope == "nginx" and has("nginx")' >/dev/null
+    curl -fsS "$base_url/monitor/health" \
+        | jq -e '.version == 1 and .scope == "health" and .status == "ok"' >/dev/null
+    curl -fsS "$base_url/monitor/metrics" \
+        | grep -q '^nginx_monitor_requests_total '
+
+    sse_output="$(timeout 5s curl -fsS -N "$base_url/monitor/live" || true)"
+    printf '%s' "$sse_output" | grep -q '^event: metrics'
+    printf '%s' "$sse_output" | grep -q '^data: {'
+
+    cleanup
+    trap - RETURN
+}
+
+[[ -n "$BUILD_NGINX_VERSIONS" ]] || fail "BUILD_NGINX_VERSIONS is required"
+
+apt-get update
+apt-get install -y --no-install-recommends \
+    binutils \
+    build-essential \
+    ca-certificates \
+    curl \
+    file \
+    jq \
+    libpcre2-dev \
+    libssl-dev \
+    zlib1g-dev
+rm -rf /var/lib/apt/lists/*
+
+mkdir -p "$DIST_DIR" "$WORK_ROOT"
+
+for nginx_version in $BUILD_NGINX_VERSIONS; do
+    archive="nginx-${nginx_version}.tar.gz"
+    archive_path="${WORK_ROOT}/${archive}"
+    workdir="${WORK_ROOT}/nginx-${nginx_version}"
+    runtime_root="${WORK_ROOT}/runtime/nginx-${nginx_version}"
+
+    if [[ ! -f "$archive_path" ]]; then
+        curl -fsSLo "$archive_path" "https://nginx.org/download/${archive}"
+    fi
+
+    rm -rf "$workdir"
+    tar -xzf "$archive_path" -C "$WORK_ROOT"
+
+    nginx_version_slug="$(printf '%s' "$nginx_version" | slugify)"
+    build_metadata="$(printf 'nginxorg.%s.%s.%s' "$nginx_version_slug" "$TARGET_ARCH" "$RELEASE_LIBC_BASELINE" | slugify)"
+    module_cflags="-DNGX_HTTP_MONITORING_MODULE_VERSION=\\\"${NEXT_VERSION}\\\" -DNGX_HTTP_MONITORING_BUILD_METADATA=\\\"${build_metadata}\\\""
+
+    cd "$workdir"
+    ./configure \
+        --prefix=/usr/local/nginx \
+        --sbin-path=/usr/local/sbin/nginx \
+        --modules-path=/usr/local/nginx/modules \
+        --conf-path=/etc/nginx/nginx.conf \
+        --error-log-path=/var/log/nginx/error.log \
+        --http-log-path=/var/log/nginx/access.log \
+        --pid-path=/var/run/nginx.pid \
+        --lock-path=/var/run/nginx.lock \
+        --http-client-body-temp-path=/var/cache/nginx/client_temp \
+        --http-proxy-temp-path=/var/cache/nginx/proxy_temp \
+        --http-fastcgi-temp-path=/var/cache/nginx/fastcgi_temp \
+        --http-uwsgi-temp-path=/var/cache/nginx/uwsgi_temp \
+        --http-scgi-temp-path=/var/cache/nginx/scgi_temp \
+        --with-cc-opt="$module_cflags" \
+        $NGINX_CONFIGURE_ARGS \
+        --add-dynamic-module=/work
+
+    build_jobs="$(nproc)"
+    if [[ "$TARGET_ARCH" != "linux-x86_64" && "$build_jobs" -gt 2 ]]; then
+        build_jobs=2
+    fi
+
+    make -j"$build_jobs"
+    make -j"$build_jobs" modules
+
+    module_path="$workdir/objs/${MODULE_NAME}.so"
+    nginx_bin="$workdir/objs/nginx"
+    [[ -x "$nginx_bin" ]] || fail "nginx binary was not built at ${nginx_bin}"
+    [[ -f "$module_path" ]] || fail "module was not built at ${module_path}"
+
+    strip --strip-unneeded "$module_path" || true
+    validate_module "$nginx_bin" "$module_path" "$runtime_root"
+
+    module_sha256="$(sha256sum "$module_path" | awk '{ print $1 }')"
+    archive_sha256="$(sha256sum "$archive_path" | awk '{ print $1 }')"
+    artifact_base="${MODULE_NAME}-${NEXT_TAG}-nginxorg-${nginx_version_slug}-${TARGET_ARCH}"
+    package_root="${WORK_ROOT}/packages/${artifact_base}"
+    compatibility_json="${package_root}/COMPATIBILITY.json"
+
+    rm -rf "$package_root"
+    mkdir -p "$package_root/modules" "$package_root/docs" "$package_root/examples"
+
+    cp "$module_path" "$package_root/modules/${MODULE_NAME}.so"
+    cp /work/examples/nginx.conf "$package_root/examples/nginx.conf"
+    cp /work/examples/monitoring-site.conf "$package_root/examples/monitoring-site.conf"
+    cp /work/README.md "$package_root/README.md"
+    cp /work/docs/API.md "$package_root/docs/API.md"
+    cp /work/docs/CONFIGURATION.md "$package_root/docs/CONFIGURATION.md"
+    cp /work/docs/PERFORMANCE.md "$package_root/docs/PERFORMANCE.md"
+    "$nginx_bin" -V > "$package_root/NGINX-V.txt" 2>&1
+
+    tarball_file="${artifact_base}.tar.gz"
+    sidecar_file="${artifact_base}.compatibility.json"
+
+    jq -n \
+        --arg schema "1" \
+        --arg module "$MODULE_NAME" \
+        --arg releaseTag "$NEXT_TAG" \
+        --arg releaseVersion "$NEXT_VERSION" \
+        --arg previousTag "$LATEST_TAG" \
+        --arg gitRef "$GITHUB_REF" \
+        --arg gitSha "$GITHUB_SHA" \
+        --arg buildImage "$RELEASE_BUILD_IMAGE" \
+        --arg libcBaseline "$RELEASE_LIBC_BASELINE" \
+        --arg dockerPlatform "$DOCKER_PLATFORM" \
+        --arg targetArch "$TARGET_ARCH" \
+        --arg nginxVersion "$nginx_version" \
+        --arg configureArgs "$NGINX_CONFIGURE_ARGS" \
+        --arg archive "$archive" \
+        --arg archiveSha256 "$archive_sha256" \
+        --arg moduleSha256 "$module_sha256" \
+        --arg builtAt "$(date -u +%Y-%m-%dT%H:%M:%SZ)" \
+        --arg artifactBase "$artifact_base" \
+        --arg tarballFile "$tarball_file" \
+        --arg sidecarFile "$sidecar_file" \
+        --arg validationTime "$(date -u +%Y-%m-%dT%H:%M:%SZ)" \
+        '{
+            schema: ($schema | tonumber),
+            module: $module,
+            release: {
+                tag: $releaseTag,
+                version: $releaseVersion,
+                previous_tag: $previousTag,
+                git_ref: $gitRef,
+                git_sha: $gitSha
+            },
+            target: {
+                os_id: "nginx.org-source",
+                os_version: $nginxVersion,
+                os_codename: "upstream",
+                architecture: $targetArch,
+                requested_target: $targetArch,
+                docker_image: $buildImage,
+                docker_platform: $dockerPlatform,
+                libc_baseline: $libcBaseline
+            },
+            nginx: {
+                binary_package: "nginx.org-source-build",
+                binary_version: $nginxVersion,
+                source_package: "nginx.org",
+                source_version: $nginxVersion,
+                abi: "",
+                modules_path: "/usr/local/nginx/modules",
+                build: {
+                    version: ("nginx/" + $nginxVersion),
+                    configure_args: ($configureArgs | split(" ")),
+                    configure_line: $configureArgs
+                }
+            },
+            build: {
+                method: "upstream-nginxorg-source",
+                source_archive: $archive,
+                source_archive_sha256: $archiveSha256,
+                module_sha256: $moduleSha256,
+                built_at: $builtAt
+            },
+            artifacts: {
+                base_name: $artifactBase,
+                deb_package: "",
+                deb_version: "",
+                deb_file: "",
+                tarball_file: $tarballFile,
+                compatibility_file: $sidecarFile
+            },
+            packaging: {
+                deb_depends: []
+            },
+            validation: {
+                packaged_nginx_config_test: false,
+                packaged_nginx_endpoint_smoke: false,
+                built_nginx_config_test: true,
+                built_nginx_endpoint_smoke: true,
+                validated_at: $validationTime
+            }
+        }' > "$compatibility_json"
+
+    {
+        printf '# %s for nginx.org %s\n\n' "$MODULE_NAME" "$nginx_version"
+        printf 'Release: `%s`\n\n' "$NEXT_TAG"
+        printf 'Target: `%s` (`%s`)\n\n' "$TARGET_ARCH" "$DOCKER_PLATFORM"
+        printf 'Build baseline: `%s`\n\n' "$RELEASE_LIBC_BASELINE"
+        printf 'This tarball is built against upstream nginx.org source, not a Debian or Ubuntu source package.\n\n'
+        printf 'Use this artifact for Nginx binaries built from the same upstream source version and compatible configure options. For distribution-packaged Nginx, prefer the distro `.deb` release artifacts.\n\n'
+        printf 'Load it near the top of `nginx.conf`:\n\n'
+        printf '```nginx\n'
+        printf 'load_module modules/%s.so;\n' "$MODULE_NAME"
+        printf '```\n'
+    } > "$package_root/INSTALL.md"
+
+    {
+        printf 'module=%s\n' "$MODULE_NAME"
+        printf 'module_release=%s\n' "$NEXT_TAG"
+        printf 'module_version=%s\n' "$NEXT_VERSION"
+        printf 'module_build_metadata=%s\n' "$build_metadata"
+        printf 'nginx_source=nginx.org\n'
+        printf 'nginx_version=%s\n' "$nginx_version"
+        printf 'target=%s\n' "$TARGET_ARCH"
+        printf 'docker_platform=%s\n' "$DOCKER_PLATFORM"
+        printf 'build_image=%s\n' "$RELEASE_BUILD_IMAGE"
+        printf 'libc_baseline=%s\n' "$RELEASE_LIBC_BASELINE"
+        printf 'configure_args=%s\n' "$NGINX_CONFIGURE_ARGS"
+        printf 'git_ref=%s\n' "$GITHUB_REF"
+        printf 'git_sha=%s\n' "$GITHUB_SHA"
+        printf 'built_at=%s\n' "$(date -u +%Y-%m-%dT%H:%M:%SZ)"
+    } > "$package_root/METADATA.txt"
+
+    (
+        cd "$package_root"
+        sha256sum "modules/${MODULE_NAME}.so" COMPATIBILITY.json > SHA256SUMS
+        file "modules/${MODULE_NAME}.so" > FILE.txt
+    )
+
+    tar -czf "$DIST_DIR/$tarball_file" -C "$(dirname "$package_root")" "$(basename "$package_root")"
+    cp "$compatibility_json" "$DIST_DIR/$sidecar_file"
+
+    echo "Built $DIST_DIR/$tarball_file"
+    echo "Built $DIST_DIR/$sidecar_file"
+done
--- a/scripts/enable-apt-source-repos.sh
+++ b/scripts/enable-apt-source-repos.sh
@@ -0,0 +1,53 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+enable_deb822_sources() {
+    local file="$1"
+    local tmp
+
+    tmp="$(mktemp)"
+    awk '
+        /^Types:/ {
+            if ($0 !~ /(^|[[:space:]])deb-src($|[[:space:]])/) {
+                print $0 " deb-src"
+                next
+            }
+        }
+        { print }
+    ' "$file" > "$tmp"
+    cat "$tmp" > "$file"
+    rm -f "$tmp"
+}
+
+enable_legacy_sources() {
+    local file="$1"
+    local output="$2"
+
+    awk '
+        /^[[:space:]]*deb[[:space:]]/ {
+            line = $0
+            sub(/^[[:space:]]*deb[[:space:]]+/, "deb-src ", line)
+            print line
+        }
+    ' "$file" > "$output"
+}
+
+found_sources=0
+
+for sources_file in /etc/apt/sources.list.d/*.sources; do
+    if [[ -f "$sources_file" ]]; then
+        found_sources=1
+        enable_deb822_sources "$sources_file"
+    fi
+done
+
+if [[ -f /etc/apt/sources.list ]]; then
+    found_sources=1
+    enable_legacy_sources /etc/apt/sources.list \
+        /etc/apt/sources.list.d/codex-deb-src.list
+fi
+
+if [[ "$found_sources" -eq 0 ]]; then
+    echo "No apt source list files were found under /etc/apt" >&2
+    exit 1
+fi
--- a/scripts/nginx-build-info.py
+++ b/scripts/nginx-build-info.py
@@ -0,0 +1,124 @@
+#!/usr/bin/env python3
+import argparse
+import json
+import os
+import shlex
+import subprocess
+import sys
+
+
+def nginx_v_output(nginx_bin):
+    proc = subprocess.run(
+        [nginx_bin, "-V"],
+        stdout=subprocess.PIPE,
+        stderr=subprocess.STDOUT,
+        text=True,
+        check=True,
+    )
+    return proc.stdout
+
+
+def parse_nginx_v(output):
+    version = ""
+    configure_line = ""
+
+    for line in output.splitlines():
+        if line.startswith("nginx version:"):
+            version = line.split(":", 1)[1].strip()
+        elif line.startswith("configure arguments:"):
+            configure_line = line.split(":", 1)[1].strip()
+
+    if not configure_line:
+        raise RuntimeError("nginx -V did not include configure arguments")
+
+    return version, configure_line, shlex.split(configure_line)
+
+
+def option_value(args, option):
+    prefix = option + "="
+    for arg in args:
+        if arg.startswith(prefix):
+            return arg[len(prefix) :]
+    return ""
+
+
+def rewrite_module_path(arg, nginx_src):
+    for option in ("--add-module=", "--add-dynamic-module="):
+        if not arg.startswith(option):
+            continue
+
+        module_path = arg[len(option) :]
+        if os.path.exists(module_path):
+            return arg
+
+        marker = "/debian/modules/"
+        if marker in module_path:
+            suffix = module_path.split(marker, 1)[1]
+            candidate = os.path.join(nginx_src, "debian", "modules", suffix)
+            if os.path.exists(candidate):
+                return option + candidate
+
+        marker = "/debian/"
+        if marker in module_path:
+            suffix = module_path.split(marker, 1)[1]
+            candidate = os.path.join(nginx_src, "debian", suffix)
+            if os.path.exists(candidate):
+                return option + candidate
+
+    return arg
+
+
+def rewrite_configure_args(args, nginx_src):
+    if not nginx_src:
+        return args
+
+    return [rewrite_module_path(arg, nginx_src) for arg in args]
+
+
+def main():
+    parser = argparse.ArgumentParser()
+    parser.add_argument("--nginx-bin", default="nginx")
+    parser.add_argument("--nginx-src", default="")
+    parser.add_argument("--json", action="store_true")
+    parser.add_argument("--configure-line", action="store_true")
+    parser.add_argument("--modules-path", action="store_true")
+    args = parser.parse_args()
+
+    output = nginx_v_output(args.nginx_bin)
+    version, configure_line, configure_args = parse_nginx_v(output)
+    rewritten_args = rewrite_configure_args(configure_args, args.nginx_src)
+
+    modules_path = option_value(configure_args, "--modules-path")
+    if not modules_path:
+        prefix = option_value(configure_args, "--prefix") or "/usr/share/nginx"
+        modules_path = os.path.join(prefix, "modules")
+
+    if args.configure_line:
+        print(shlex.join(rewritten_args))
+        return
+
+    if args.modules_path:
+        print(modules_path)
+        return
+
+    if args.json:
+        json.dump(
+            {
+                "version": version,
+                "configure_line": configure_line,
+                "configure_args": configure_args,
+                "rewritten_configure_args": rewritten_args,
+                "modules_path": modules_path,
+            },
+            sys.stdout,
+            indent=2,
+            sort_keys=True,
+        )
+        sys.stdout.write("\n")
+        return
+
+    parser.error("choose one of --json, --configure-line, or --modules-path")
+
+
+if __name__ == "__main__":
+    main()