Files
ngx_http_monitoring_module/scripts/build-distro-release.sh

560 lines
20 KiB
Bash

#!/usr/bin/env bash
set -Eeuo pipefail
MODULE_NAME="${MODULE_NAME:-ngx_http_monitoring_module}"
DEB_PACKAGE_NAME="${DEB_PACKAGE_NAME:-libnginx-mod-http-monitoring}"
NGINX_INSTALL_PACKAGE="${NGINX_INSTALL_PACKAGE:-nginx}"
NEXT_VERSION="${NEXT_VERSION:-0.0.0}"
NEXT_TAG="${NEXT_TAG:-v${NEXT_VERSION}}"
LATEST_TAG="${LATEST_TAG:-none}"
TARGET_ARCH="${TARGET_ARCH:-$(uname -m)}"
DOCKER_PLATFORM="${DOCKER_PLATFORM:-unknown}"
DISTRO_IMAGE="${DISTRO_IMAGE:-unknown}"
DIST_DIR="${DIST_DIR:-/work/dist}"
WORK_ROOT="${WORK_ROOT:-/work/.release-work}"
VALIDATION_PORT="${VALIDATION_PORT:-18080}"
GITHUB_REF="${GITHUB_REF:-unknown}"
GITHUB_SHA="${GITHUB_SHA:-unknown}"
export DEBIAN_FRONTEND=noninteractive
on_error() {
local status="$?"
local line="${BASH_LINENO[0]:-unknown}"
echo "::error::distro release build failed at line ${line} with exit code ${status}" >&2
if [[ -n "${WORK_ROOT:-}" && -d "$WORK_ROOT" ]]; then
find "$WORK_ROOT" -path '*/objs/autoconf.err' -type f -print -exec sed -n '1,160p' {} \; >&2 || true
find "$WORK_ROOT" -path '*/objs/*.log' -type f -print -exec sed -n '1,160p' {} \; >&2 || true
fi
exit "$status"
}
trap on_error ERR
slugify() {
tr '[:upper:]' '[:lower:]' \
| sed -E 's/[^a-z0-9.+~]+/./g; s/^[.]+//; s/[.]+$//'
}
fail() {
echo "error: $*" >&2
exit 1
}
apt_install() {
apt-get install -y --no-install-recommends "$@"
}
find_nginx_abi() {
local pkg
local abi
for pkg in "$@"; do
abi="$(
apt-cache show "$pkg" 2>/dev/null \
| awk -F': ' '
/^Provides:/ {
n = split($2, provides, ",")
for (i = 1; i <= n; i++) {
gsub(/^[ \t]+|[ \t]+$/, "", provides[i])
if (provides[i] ~ /^nginx-abi-/) {
print provides[i]
exit
}
}
}
' \
| head -n 1 \
|| true
)"
if [[ -n "$abi" ]]; then
printf '%s\n' "$abi"
return 0
fi
done
}
validate_module() {
local module_path="$1"
local runtime_root="$2"
local config_path="$runtime_root/nginx.conf"
local base_url="http://127.0.0.1:${VALIDATION_PORT}"
local status
local dashboard
local sse_output
rm -rf "$runtime_root"
mkdir -p \
"$runtime_root/logs" \
"$runtime_root/temp/client_body" \
"$runtime_root/temp/proxy" \
"$runtime_root/temp/fastcgi" \
"$runtime_root/temp/uwsgi" \
"$runtime_root/temp/scgi"
{
printf 'load_module %s;\n\n' "$module_path"
printf 'worker_processes 1;\n'
printf 'error_log %s/logs/error.log info;\n' "$runtime_root"
printf 'pid %s/logs/nginx.pid;\n\n' "$runtime_root"
printf 'events {\n'
printf ' worker_connections 1024;\n'
printf '}\n\n'
printf 'http {\n'
printf ' access_log off;\n'
printf ' default_type application/octet-stream;\n'
printf ' client_body_temp_path %s/temp/client_body;\n' "$runtime_root"
printf ' proxy_temp_path %s/temp/proxy;\n' "$runtime_root"
printf ' fastcgi_temp_path %s/temp/fastcgi;\n' "$runtime_root"
printf ' uwsgi_temp_path %s/temp/uwsgi;\n' "$runtime_root"
printf ' scgi_temp_path %s/temp/scgi;\n\n' "$runtime_root"
printf ' monitor_refresh_interval 1s;\n'
printf ' monitor_history 1m;\n'
printf ' monitor_resolution 1s;\n'
printf ' monitor_shm_size 8m;\n'
printf ' monitor_collect_system on;\n'
printf ' monitor_collect_nginx on;\n'
printf ' monitor_collect_network on;\n'
printf ' monitor_access_log on;\n'
printf ' monitor_max_top_urls 100;\n\n'
printf ' server {\n'
printf ' listen 127.0.0.1:%s;\n' "$VALIDATION_PORT"
printf ' server_name localhost;\n\n'
printf ' location /monitor {\n'
printf ' monitor on;\n'
printf ' monitor_dashboard on;\n'
printf ' monitor_api on;\n'
printf ' monitor_sse on;\n'
printf ' monitor_allow all;\n'
printf ' monitor_basic_auth off;\n'
printf ' monitor_cors off;\n'
printf ' monitor_rate_limit 120;\n'
printf ' }\n\n'
printf ' location / {\n'
printf ' return 200 "ok\\n";\n'
printf ' }\n'
printf ' }\n'
printf '}\n'
} > "$config_path"
nginx -t -p "$runtime_root" -c "$config_path"
cleanup() {
nginx -p "$runtime_root" -c "$config_path" -s quit >/dev/null 2>&1 || true
}
trap cleanup RETURN
nginx -p "$runtime_root" -c "$config_path"
for attempt in $(seq 1 50); do
if curl -fsS "$base_url/" >/dev/null; then
break
fi
if [[ "$attempt" -eq 50 ]]; then
echo "Nginx did not become ready" >&2
return 1
fi
sleep 0.2
done
curl -fsS "$base_url/" | grep -q '^ok'
status="$(curl -fsS -o "$runtime_root/dashboard.html" -w '%{http_code}' "$base_url/monitor")"
[[ "$status" == "200" ]] || fail "/monitor returned HTTP $status"
dashboard="$(cat "$runtime_root/dashboard.html")"
printf '%s' "$dashboard" | grep -q '<title>Nginx Monitor</title>'
printf '%s' "$dashboard" | grep -q '/monitor/api'
printf '%s' "$dashboard" | grep -q '/monitor/live'
curl -fsS "$base_url/monitor/api" \
| jq -e '.version == 1 and has("system") and has("nginx") and has("requests") and has("history")' >/dev/null
curl -fsS "$base_url/monitor/api/system" \
| jq -e '.version == 1 and .scope == "system" and has("system")' >/dev/null
curl -fsS "$base_url/monitor/api/nginx" \
| jq -e '.version == 1 and .scope == "nginx" and has("nginx")' >/dev/null
curl -fsS "$base_url/monitor/health" \
| jq -e '.version == 1 and .scope == "health" and .status == "ok"' >/dev/null
curl -fsS "$base_url/monitor/metrics" \
| grep -q '^nginx_monitor_requests_total '
sse_output="$(timeout 5s curl -fsS -N "$base_url/monitor/live" || true)"
printf '%s' "$sse_output" | grep -q '^event: metrics'
printf '%s' "$sse_output" | grep -q '^data: {'
cleanup
trap - RETURN
}
mkdir -p "$DIST_DIR" "$WORK_ROOT"
printf '#!/bin/sh\nexit 101\n' > /usr/sbin/policy-rc.d
chmod +x /usr/sbin/policy-rc.d
apt-get update
apt_install ca-certificates curl gnupg
bash /work/scripts/enable-apt-source-repos.sh
apt-get update
apt_install \
bash \
binutils \
build-essential \
ca-certificates \
curl \
dpkg-dev \
file \
findutils \
jq \
libpcre2-dev \
libssl-dev \
python3 \
xz-utils \
zlib1g-dev \
"$NGINX_INSTALL_PACKAGE"
. /etc/os-release
distro_id="${ID:-unknown}"
distro_version="${VERSION_ID:-unknown}"
distro_codename="${VERSION_CODENAME:-${UBUNTU_CODENAME:-unknown}}"
distro_slug="$(printf '%s-%s-%s' "$distro_id" "$distro_version" "$distro_codename" | slugify)"
deb_arch="$(dpkg --print-architecture)"
nginx_bin="$(command -v nginx)"
nginx_binary_package="$(dpkg-query -S "$nginx_bin" | head -n 1 | cut -d: -f1)"
nginx_binary_version="$(dpkg-query -W -f='${Version}' "$nginx_binary_package")"
source_package="$(dpkg-query -W -f='${source:Package}' "$nginx_binary_package" 2>/dev/null || true)"
source_version="$(dpkg-query -W -f='${source:Version}' "$nginx_binary_package" 2>/dev/null || true)"
if [[ -z "$source_package" || "$source_package" == *'{'* ]]; then
source_package="$nginx_binary_package"
fi
if [[ -z "$source_version" || "$source_version" == *'{'* ]]; then
source_version="$nginx_binary_version"
fi
nginx_abi="$(find_nginx_abi "$nginx_binary_package" "$NGINX_INSTALL_PACKAGE" nginx-core nginx-full nginx-light nginx-extras nginx)"
echo "Distro: ${distro_id} ${distro_version} (${distro_codename}), arch ${deb_arch}"
echo "Nginx binary package: ${nginx_binary_package} ${nginx_binary_version}"
echo "Nginx source package: ${source_package} ${source_version}"
echo "Nginx ABI package: ${nginx_abi:-none found}"
src_root="$WORK_ROOT/${distro_slug}/${deb_arch}/source"
build_root="$WORK_ROOT/${distro_slug}/${deb_arch}/build"
rm -rf "$src_root" "$build_root"
mkdir -p "$src_root" "$build_root"
(
cd "$src_root"
apt-get source "${source_package}=${source_version}" \
|| apt-get source "$source_package"
)
dsc_path="$(find "$src_root" -maxdepth 1 -name '*.dsc' | head -n 1)"
[[ -n "$dsc_path" ]] || fail "apt-get source did not download a .dsc file"
dsc_version="$(awk '/^Version:/ { print $2; exit }' "$dsc_path")"
if [[ "$dsc_version" != "$source_version" ]]; then
fail "downloaded source version ${dsc_version}, expected ${source_version}"
fi
nginx_src="$(find "$src_root" -mindepth 1 -maxdepth 1 -type d | head -n 1)"
[[ -n "$nginx_src" ]] || fail "apt-get source did not unpack an nginx source directory"
apt-get build-dep -y --no-install-recommends "${source_package}=${source_version}" \
|| apt-get build-dep -y --no-install-recommends "$source_package"
nginx_build_info="$build_root/nginx-build-info.json"
modules_path="$(python3 /work/scripts/nginx-build-info.py --modules-path)"
case "$modules_path" in
/*) ;;
*) modules_path="/${modules_path}" ;;
esac
source_version_slug="$(printf '%s' "$source_version" | slugify)"
deb_revision_slug="$(printf '%s%s.nginx%s' "$distro_id" "$distro_version" "$source_version" | slugify)"
deb_version="${NEXT_VERSION}-1+${deb_revision_slug}"
build_metadata="$(printf '%s.%s.%s' "$distro_slug" "$deb_arch" "$source_version_slug" | slugify)"
module_cflags="-DNGX_HTTP_MONITORING_MODULE_VERSION=\\\"${NEXT_VERSION}\\\" -DNGX_HTTP_MONITORING_BUILD_METADATA=\\\"${build_metadata}\\\""
python3 /work/scripts/nginx-build-info.py \
--nginx-src "$nginx_src" \
--extra-cc-opt "$module_cflags" \
--json > "$nginx_build_info"
configure_line="$(python3 /work/scripts/nginx-build-info.py \
--nginx-src "$nginx_src" \
--extra-cc-opt "$module_cflags" \
--configure-line)"
cd "$nginx_src"
eval "./configure ${configure_line} --add-dynamic-module=/work"
build_jobs="$(nproc)"
if [[ "$TARGET_ARCH" != "linux-x86_64" && "$build_jobs" -gt 2 ]]; then
build_jobs=2
fi
make -j"$build_jobs" modules
built_module="$nginx_src/objs/${MODULE_NAME}.so"
[[ -f "$built_module" ]] || fail "module was not built at ${built_module}"
artifact_base="${MODULE_NAME}-${NEXT_TAG}-${distro_slug}-nginx-${source_version_slug}-${deb_arch}"
deb_file="${DEB_PACKAGE_NAME}_${deb_version}_${deb_arch}.deb"
tarball_file="${artifact_base}.tar.gz"
package_root="$build_root/package-root"
doc_dir="$package_root/usr/share/doc/$DEB_PACKAGE_NAME"
module_install_dir="$package_root/${modules_path#/}"
modules_available_dir="$package_root/usr/share/nginx/modules-available"
metadata_dir="$doc_dir/metadata"
rm -rf "$package_root"
mkdir -p "$module_install_dir" "$modules_available_dir" "$metadata_dir" "$package_root/DEBIAN"
cp "$built_module" "$module_install_dir/${MODULE_NAME}.so"
strip --strip-unneeded "$module_install_dir/${MODULE_NAME}.so" || true
cat > "$modules_available_dir/mod-http-monitoring.conf" <<EOF
load_module ${modules_path}/${MODULE_NAME}.so;
EOF
cp /work/README.md "$doc_dir/README.md"
cp /work/LICENSE "$doc_dir/copyright"
cp /work/docs/API.md "$doc_dir/API.md"
cp /work/docs/CONFIGURATION.md "$doc_dir/CONFIGURATION.md"
cp /work/docs/PERFORMANCE.md "$doc_dir/PERFORMANCE.md"
cp /work/examples/monitoring-site.conf "$doc_dir/monitoring-site.conf.example"
cat > "$doc_dir/changelog.Debian" <<EOF
$DEB_PACKAGE_NAME (${deb_version}) unstable; urgency=medium
* Build ${NEXT_TAG} against ${distro_id} ${distro_version} nginx source ${source_version}.
-- GitHub Actions <actions@github.com> $(date -Ru)
EOF
gzip -9n "$doc_dir/changelog.Debian"
module_sha256="$(sha256sum "$module_install_dir/${MODULE_NAME}.so" | awk '{ print $1 }')"
dsc_sha256="$(sha256sum "$dsc_path" | awk '{ print $1 }')"
nginx_v_text="$metadata_dir/nginx-V.txt"
nginx -V > "$nginx_v_text" 2>&1
compatibility_json="$metadata_dir/compatibility.json"
jq -n \
--slurpfile nginxBuild "$nginx_build_info" \
--arg schema "1" \
--arg module "$MODULE_NAME" \
--arg debPackage "$DEB_PACKAGE_NAME" \
--arg debVersion "$deb_version" \
--arg releaseTag "$NEXT_TAG" \
--arg releaseVersion "$NEXT_VERSION" \
--arg previousTag "$LATEST_TAG" \
--arg gitRef "$GITHUB_REF" \
--arg gitSha "$GITHUB_SHA" \
--arg distroImage "$DISTRO_IMAGE" \
--arg dockerPlatform "$DOCKER_PLATFORM" \
--arg targetArch "$TARGET_ARCH" \
--arg osId "$distro_id" \
--arg osVersion "$distro_version" \
--arg osCodename "$distro_codename" \
--arg debArch "$deb_arch" \
--arg nginxBinaryPackage "$nginx_binary_package" \
--arg nginxBinaryVersion "$nginx_binary_version" \
--arg nginxSourcePackage "$source_package" \
--arg nginxSourceVersion "$source_version" \
--arg nginxAbi "$nginx_abi" \
--arg modulesPath "$modules_path" \
--arg dscPath "$(basename "$dsc_path")" \
--arg dscSha256 "$dsc_sha256" \
--arg moduleSha256 "$module_sha256" \
--arg builtAt "$(date -u +%Y-%m-%dT%H:%M:%SZ)" \
--arg artifactBase "$artifact_base" \
--arg debFile "$deb_file" \
--arg tarballFile "$tarball_file" \
'{
schema: ($schema | tonumber),
module: $module,
release: {
tag: $releaseTag,
version: $releaseVersion,
previous_tag: $previousTag,
git_ref: $gitRef,
git_sha: $gitSha
},
target: {
os_id: $osId,
os_version: $osVersion,
os_codename: $osCodename,
architecture: $debArch,
requested_target: $targetArch,
docker_image: $distroImage,
docker_platform: $dockerPlatform
},
nginx: {
binary_package: $nginxBinaryPackage,
binary_version: $nginxBinaryVersion,
source_package: $nginxSourcePackage,
source_version: $nginxSourceVersion,
abi: $nginxAbi,
modules_path: $modulesPath,
build: $nginxBuild[0]
},
build: {
method: "distro-source-package",
dsc_file: $dscPath,
dsc_sha256: $dscSha256,
module_sha256: $moduleSha256,
built_at: $builtAt
},
artifacts: {
base_name: $artifactBase,
deb_package: $debPackage,
deb_version: $debVersion,
deb_file: $debFile,
tarball_file: $tarballFile,
compatibility_file: ($artifactBase + ".compatibility.json")
},
packaging: {
deb_depends: (
if $nginxAbi == "" then
[($nginxBinaryPackage + " (= " + $nginxBinaryVersion + ")"), "libc6"]
else
[$nginxAbi, ($nginxBinaryPackage + " (= " + $nginxBinaryVersion + ")"), "libc6"]
end
)
},
validation: {
packaged_nginx_config_test: false,
packaged_nginx_endpoint_smoke: false
}
}' > "$compatibility_json"
cat > "$doc_dir/INSTALL.md" <<EOF
# ${MODULE_NAME} for ${distro_id} ${distro_version} ${deb_arch}
This package was built from the ${source_package} source package version
${source_version} and validated against the packaged ${nginx_binary_package}
binary version ${nginx_binary_version}.
Recommended install:
\`\`\`sh
sudo apt install ./$(printf '%s_%s_%s.deb' "$DEB_PACKAGE_NAME" "$deb_version" "$deb_arch")
\`\`\`
The package installs the module at:
\`\`\`text
${modules_path}/${MODULE_NAME}.so
\`\`\`
Load it near the top of nginx.conf:
\`\`\`nginx
load_module ${modules_path}/${MODULE_NAME}.so;
\`\`\`
Compatibility guard:
\`\`\`text
$(if [[ -n "$nginx_abi" ]]; then printf 'Depends: %s\nDepends: %s (= %s)\n' "$nginx_abi" "$nginx_binary_package" "$nginx_binary_version"; else printf 'Depends: %s (= %s)\n' "$nginx_binary_package" "$nginx_binary_version"; fi)
\`\`\`
If a distro nginx update changes this ABI, apt will refuse to keep this module
installed with the new nginx package. Rebuild and publish a new package for the
new distro nginx package instead of reusing this binary.
EOF
depends=""
if [[ -n "$nginx_abi" ]]; then
depends="$nginx_abi, $nginx_binary_package (= $nginx_binary_version), libc6"
else
depends="$nginx_binary_package (= $nginx_binary_version), libc6"
fi
installed_size="$(du -ks "$package_root" | awk '{ print $1 }')"
cat > "$package_root/DEBIAN/control" <<EOF
Package: $DEB_PACKAGE_NAME
Version: $deb_version
Section: httpd
Priority: optional
Architecture: $deb_arch
Maintainer: ngx_http_monitoring_module maintainers <noreply@github.com>
Depends: $depends
Installed-Size: $installed_size
Description: Nginx HTTP monitoring dynamic module
Distro-built dynamic module for live Nginx monitoring, JSON APIs,
Server-Sent Events, Prometheus metrics, and an embedded dashboard.
This package is built against the target distribution nginx source package
and depends on the matching nginx module ABI.
EOF
deb_path="$DIST_DIR/$deb_file"
dpkg-deb --build --root-owner-group "$package_root" "$deb_path"
apt-get install -y --no-install-recommends "$deb_path"
validate_module "${modules_path}/${MODULE_NAME}.so" "$build_root/runtime"
jq \
--arg debFile "$deb_file" \
--arg validationTime "$(date -u +%Y-%m-%dT%H:%M:%SZ)" \
'.artifacts.deb_file = $debFile
| .validation.packaged_nginx_config_test = true
| .validation.packaged_nginx_endpoint_smoke = true
| .validation.validated_at = $validationTime' \
"$compatibility_json" > "$compatibility_json.tmp"
mv "$compatibility_json.tmp" "$compatibility_json"
dpkg-deb --build --root-owner-group "$package_root" "$deb_path"
tar_root="$build_root/tar"
tar_dir="$tar_root/$artifact_base"
rm -rf "$tar_root"
mkdir -p "$tar_dir/modules" "$tar_dir/docs" "$tar_dir/examples"
cp "${modules_path}/${MODULE_NAME}.so" "$tar_dir/modules/${MODULE_NAME}.so"
cp "$compatibility_json" "$tar_dir/COMPATIBILITY.json"
cp "$doc_dir/INSTALL.md" "$tar_dir/INSTALL.md"
cp "$doc_dir/README.md" "$tar_dir/README.md"
cp "$doc_dir/API.md" "$tar_dir/docs/API.md"
cp "$doc_dir/CONFIGURATION.md" "$tar_dir/docs/CONFIGURATION.md"
cp "$doc_dir/PERFORMANCE.md" "$tar_dir/docs/PERFORMANCE.md"
cp "$doc_dir/monitoring-site.conf.example" "$tar_dir/examples/monitoring-site.conf"
cp "$nginx_v_text" "$tar_dir/NGINX-V.txt"
{
printf 'module=%s\n' "$MODULE_NAME"
printf 'module_release=%s\n' "$NEXT_TAG"
printf 'module_version=%s\n' "$NEXT_VERSION"
printf 'module_build_metadata=%s\n' "$build_metadata"
printf 'distro=%s\n' "$distro_slug"
printf 'architecture=%s\n' "$deb_arch"
printf 'nginx_binary_package=%s\n' "$nginx_binary_package"
printf 'nginx_binary_version=%s\n' "$nginx_binary_version"
printf 'nginx_source_package=%s\n' "$source_package"
printf 'nginx_source_version=%s\n' "$source_version"
printf 'nginx_abi=%s\n' "$nginx_abi"
printf 'modules_path=%s\n' "$modules_path"
printf 'git_ref=%s\n' "$GITHUB_REF"
printf 'git_sha=%s\n' "$GITHUB_SHA"
printf 'built_at=%s\n' "$(date -u +%Y-%m-%dT%H:%M:%SZ)"
} > "$tar_dir/METADATA.txt"
(
cd "$tar_dir"
sha256sum "modules/${MODULE_NAME}.so" COMPATIBILITY.json > SHA256SUMS
file "modules/${MODULE_NAME}.so" > FILE.txt
)
tar -czf "$DIST_DIR/$tarball_file" -C "$tar_root" "$artifact_base"
cp "$compatibility_json" "$DIST_DIR/${artifact_base}.compatibility.json"
echo "Built $deb_path"
echo "Built $DIST_DIR/$tarball_file"
echo "Built $DIST_DIR/${artifact_base}.compatibility.json"