diff --git a/poetry.lock b/poetry.lock index da61c53..84a31bb 100644 --- a/poetry.lock +++ b/poetry.lock @@ -212,13 +212,13 @@ test = ["flaky", "ipyparallel", "pre-commit", "pytest (>=7.0)", "pytest-asyncio [[package]] name = "ipython" -version = "8.21.0" +version = "8.22.0" description = "IPython: Productive Interactive Computing" optional = false python-versions = ">=3.10" files = [ - {file = "ipython-8.21.0-py3-none-any.whl", hash = "sha256:1050a3ab8473488d7eee163796b02e511d0735cf43a04ba2a8348bd0f2eaf8a5"}, - {file = "ipython-8.21.0.tar.gz", hash = "sha256:48fbc236fbe0e138b88773fa0437751f14c3645fb483f1d4c5dee58b37e5ce73"}, + {file = "ipython-8.22.0-py3-none-any.whl", hash = "sha256:a3e962e1d42927f5825dab048f4de617f5856cf3272fff1cb9245bea0c785a46"}, + {file = "ipython-8.22.0.tar.gz", hash = "sha256:bc649987e35a75ecccab7a245d7403710e3a289384c268d6d846ab8933ca0811"}, ] [package.dependencies] @@ -226,16 +226,16 @@ colorama = {version = "*", markers = "sys_platform == \"win32\""} decorator = "*" jedi = ">=0.16" matplotlib-inline = "*" -pexpect = {version = ">4.3", markers = "sys_platform != \"win32\""} +pexpect = {version = ">4.3", markers = "sys_platform != \"win32\" and sys_platform != \"emscripten\""} prompt-toolkit = ">=3.0.41,<3.1.0" pygments = ">=2.4.0" stack-data = "*" traitlets = ">=5" [package.extras] -all = ["black", "curio", "docrepr", "exceptiongroup", "ipykernel", "ipyparallel", "ipywidgets", "matplotlib", "matplotlib (!=3.2.0)", "nbconvert", "nbformat", "notebook", "numpy (>=1.23)", "pandas", "pickleshare", "pytest (<8)", "pytest-asyncio (<0.22)", "qtconsole", "setuptools (>=18.5)", "sphinx (>=1.3)", "sphinx-rtd-theme", "stack-data", "testpath", "trio", "typing-extensions"] +all = ["ipython[black,doc,kernel,nbconvert,nbformat,notebook,parallel,qtconsole,terminal]", "ipython[test,test-extra]"] black = ["black"] -doc = ["docrepr", "exceptiongroup", "ipykernel", "matplotlib", "pickleshare", "pytest (<8)", "pytest-asyncio (<0.22)", "setuptools (>=18.5)", "sphinx (>=1.3)", "sphinx-rtd-theme", "stack-data", "testpath", "typing-extensions"] +doc = ["docrepr", "exceptiongroup", "ipykernel", "ipython[test]", "matplotlib", "setuptools (>=18.5)", "sphinx (>=1.3)", "sphinx-rtd-theme", "sphinxcontrib-jquery", "stack-data", "typing-extensions"] kernel = ["ipykernel"] nbconvert = ["nbconvert"] nbformat = ["nbformat"] @@ -243,7 +243,7 @@ notebook = ["ipywidgets", "notebook"] parallel = ["ipyparallel"] qtconsole = ["qtconsole"] test = ["pickleshare", "pytest (<8)", "pytest-asyncio (<0.22)", "testpath"] -test-extra = ["curio", "matplotlib (!=3.2.0)", "nbformat", "numpy (>=1.23)", "pandas", "pickleshare", "pytest (<8)", "pytest-asyncio (<0.22)", "testpath", "trio"] +test-extra = ["curio", "ipython[test]", "matplotlib (!=3.2.0)", "nbformat", "numpy (>=1.23)", "pandas", "trio"] [[package]] name = "jedi" diff --git a/prometheus-exercises.ipynb b/prometheus-exercises.ipynb index 7ad0a3d..244f171 100644 --- a/prometheus-exercises.ipynb +++ b/prometheus-exercises.ipynb @@ -39,7 +39,9 @@ "DEMO_VERSION=\"0.11.1\"\n", "NODEEXP_VERSION=\"1.7.0\"\n", "CADVISOR_VERSION=\"0.36.0\"\n", - "CONSUL_VERSION=\"1.17.3\"" + "CONSUL_VERSION=\"1.17.3\"\n", + "BLACKBOX_EXPORTER_VERSION=\"0.24.0\"\n", + "PUSHGATEWAY_VERSION=\"1.7.0\"" ] }, { @@ -84,6 +86,7 @@ "metadata": {}, "outputs": [], "source": [ + "\n", "%%bash -s {WORKDIR}\n", "{\n", " echo \"Stopping all processes and containers that are started\"\n", @@ -92,7 +95,9 @@ " killall consul\n", " killall prometheus_demo_service\n", " killall cpu-metric-exporter.py\n", - " \n", + " killall blackbox_exporter\n", + " killall pushgateway\n", + " \n", " docker rm -f cadvisor\n", "}" ] @@ -169,7 +174,71 @@ "cell_type": "code", "execution_count": null, "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "ts=2024-02-22T19:05:29.278Z caller=consul.go:295 level=error component=\"discovery manager scrape\" discovery=consul config=consul-sd-demo msg=\"Error retrieving datacenter name\" err=\"Get \\\"http://localhost:8500/v1/agent/self\\\": dial tcp 127.0.0.1:8500: connect: connection refused\"\n", + "ts=2024-02-22T19:05:44.279Z caller=consul.go:295 level=error component=\"discovery manager scrape\" discovery=consul config=consul-sd-demo msg=\"Error retrieving datacenter name\" err=\"Get \\\"http://localhost:8500/v1/agent/self\\\": dial tcp 127.0.0.1:8500: connect: connection refused\"\n", + "ts=2024-02-22T19:05:59.280Z caller=consul.go:295 level=error component=\"discovery manager scrape\" discovery=consul config=consul-sd-demo msg=\"Error retrieving datacenter name\" err=\"Get \\\"http://localhost:8500/v1/agent/self\\\": dial tcp 127.0.0.1:8500: connect: connection refused\"\n", + "ts=2024-02-22T19:06:14.281Z caller=consul.go:295 level=error component=\"discovery manager scrape\" discovery=consul config=consul-sd-demo msg=\"Error retrieving datacenter name\" err=\"Get \\\"http://localhost:8500/v1/agent/self\\\": dial tcp 127.0.0.1:8500: connect: connection refused\"\n", + "ts=2024-02-22T19:06:29.285Z caller=consul.go:295 level=error component=\"discovery manager scrape\" discovery=consul config=consul-sd-demo msg=\"Error retrieving datacenter name\" err=\"Get \\\"http://localhost:8500/v1/agent/self\\\": dial tcp 127.0.0.1:8500: connect: connection refused\"\n", + "ts=2024-02-22T19:06:44.285Z caller=consul.go:295 level=error component=\"discovery manager scrape\" discovery=consul config=consul-sd-demo msg=\"Error retrieving datacenter name\" err=\"Get \\\"http://localhost:8500/v1/agent/self\\\": dial tcp 127.0.0.1:8500: connect: connection refused\"\n", + "ts=2024-02-22T19:06:59.286Z caller=consul.go:295 level=error component=\"discovery manager scrape\" discovery=consul config=consul-sd-demo msg=\"Error retrieving datacenter name\" err=\"Get \\\"http://localhost:8500/v1/agent/self\\\": dial tcp 127.0.0.1:8500: connect: connection refused\"\n", + "ts=2024-02-22T19:07:14.287Z caller=consul.go:295 level=error component=\"discovery manager scrape\" discovery=consul config=consul-sd-demo msg=\"Error retrieving datacenter name\" err=\"Get \\\"http://localhost:8500/v1/agent/self\\\": dial tcp 127.0.0.1:8500: connect: connection refused\"\n", + "ts=2024-02-22T19:07:29.288Z caller=consul.go:295 level=error component=\"discovery manager scrape\" discovery=consul config=consul-sd-demo msg=\"Error retrieving datacenter name\" err=\"Get \\\"http://localhost:8500/v1/agent/self\\\": dial tcp 127.0.0.1:8500: connect: connection refused\"\n", + "ts=2024-02-22T19:07:44.289Z caller=consul.go:295 level=error component=\"discovery manager scrape\" discovery=consul config=consul-sd-demo msg=\"Error retrieving datacenter name\" err=\"Get \\\"http://localhost:8500/v1/agent/self\\\": dial tcp 127.0.0.1:8500: connect: connection refused\"\n", + "ts=2024-02-22T19:07:59.290Z caller=consul.go:295 level=error component=\"discovery manager scrape\" discovery=consul config=consul-sd-demo msg=\"Error retrieving datacenter name\" err=\"Get \\\"http://localhost:8500/v1/agent/self\\\": dial tcp 127.0.0.1:8500: connect: connection refused\"\n", + "ts=2024-02-22T19:08:14.316Z caller=consul.go:295 level=error component=\"discovery manager scrape\" discovery=consul config=consul-sd-demo msg=\"Error retrieving datacenter name\" err=\"Get \\\"http://localhost:8500/v1/agent/self\\\": dial tcp 127.0.0.1:8500: connect: connection refused\"\n", + "ts=2024-02-22T19:08:29.316Z caller=consul.go:295 level=error component=\"discovery manager scrape\" discovery=consul config=consul-sd-demo msg=\"Error retrieving datacenter name\" err=\"Get \\\"http://localhost:8500/v1/agent/self\\\": dial tcp 127.0.0.1:8500: connect: connection refused\"\n", + "ts=2024-02-22T19:08:44.318Z caller=consul.go:295 level=error component=\"discovery manager scrape\" discovery=consul config=consul-sd-demo msg=\"Error retrieving datacenter name\" err=\"Get \\\"http://localhost:8500/v1/agent/self\\\": dial tcp 127.0.0.1:8500: connect: connection refused\"\n", + "ts=2024-02-22T19:08:59.318Z caller=consul.go:295 level=error component=\"discovery manager scrape\" discovery=consul config=consul-sd-demo msg=\"Error retrieving datacenter name\" err=\"Get \\\"http://localhost:8500/v1/agent/self\\\": dial tcp 127.0.0.1:8500: connect: connection refused\"\n", + "ts=2024-02-22T19:09:14.319Z caller=consul.go:295 level=error component=\"discovery manager scrape\" discovery=consul config=consul-sd-demo msg=\"Error retrieving datacenter name\" err=\"Get \\\"http://localhost:8500/v1/agent/self\\\": dial tcp 127.0.0.1:8500: connect: connection refused\"\n", + "ts=2024-02-22T19:09:29.319Z caller=consul.go:295 level=error component=\"discovery manager scrape\" discovery=consul config=consul-sd-demo msg=\"Error retrieving datacenter name\" err=\"Get \\\"http://localhost:8500/v1/agent/self\\\": dial tcp 127.0.0.1:8500: connect: connection refused\"\n", + "ts=2024-02-22T19:09:44.320Z caller=consul.go:295 level=error component=\"discovery manager scrape\" discovery=consul config=consul-sd-demo msg=\"Error retrieving datacenter name\" err=\"Get \\\"http://localhost:8500/v1/agent/self\\\": dial tcp 127.0.0.1:8500: connect: connection refused\"\n", + "ts=2024-02-22T19:09:59.322Z caller=consul.go:295 level=error component=\"discovery manager scrape\" discovery=consul config=consul-sd-demo msg=\"Error retrieving datacenter name\" err=\"Get \\\"http://localhost:8500/v1/agent/self\\\": dial tcp 127.0.0.1:8500: connect: connection refused\"\n", + "ts=2024-02-22T19:10:14.323Z caller=consul.go:295 level=error component=\"discovery manager scrape\" discovery=consul config=consul-sd-demo msg=\"Error retrieving datacenter name\" err=\"Get \\\"http://localhost:8500/v1/agent/self\\\": dial tcp 127.0.0.1:8500: connect: connection refused\"\n", + "ts=2024-02-22T19:10:29.324Z caller=consul.go:295 level=error component=\"discovery manager scrape\" discovery=consul config=consul-sd-demo msg=\"Error retrieving datacenter name\" err=\"Get \\\"http://localhost:8500/v1/agent/self\\\": dial tcp 127.0.0.1:8500: connect: connection refused\"\n", + "ts=2024-02-22T19:10:44.328Z caller=consul.go:295 level=error component=\"discovery manager scrape\" discovery=consul config=consul-sd-demo msg=\"Error retrieving datacenter name\" err=\"Get \\\"http://localhost:8500/v1/agent/self\\\": dial tcp 127.0.0.1:8500: connect: connection refused\"\n", + "ts=2024-02-22T19:10:59.329Z caller=consul.go:295 level=error component=\"discovery manager scrape\" discovery=consul config=consul-sd-demo msg=\"Error retrieving datacenter name\" err=\"Get \\\"http://localhost:8500/v1/agent/self\\\": dial tcp 127.0.0.1:8500: connect: connection refused\"\n", + "ts=2024-02-22T19:11:14.331Z caller=consul.go:295 level=error component=\"discovery manager scrape\" discovery=consul config=consul-sd-demo msg=\"Error retrieving datacenter name\" err=\"Get \\\"http://localhost:8500/v1/agent/self\\\": dial tcp 127.0.0.1:8500: connect: connection refused\"\n", + "ts=2024-02-22T19:11:29.355Z caller=consul.go:295 level=error component=\"discovery manager scrape\" discovery=consul config=consul-sd-demo msg=\"Error retrieving datacenter name\" err=\"Get \\\"http://localhost:8500/v1/agent/self\\\": dial tcp 127.0.0.1:8500: connect: connection refused\"\n", + "ts=2024-02-22T19:11:44.356Z caller=consul.go:295 level=error component=\"discovery manager scrape\" discovery=consul config=consul-sd-demo msg=\"Error retrieving datacenter name\" err=\"Get \\\"http://localhost:8500/v1/agent/self\\\": dial tcp 127.0.0.1:8500: connect: connection refused\"\n", + "ts=2024-02-22T19:11:59.357Z caller=consul.go:295 level=error component=\"discovery manager scrape\" discovery=consul config=consul-sd-demo msg=\"Error retrieving datacenter name\" err=\"Get \\\"http://localhost:8500/v1/agent/self\\\": dial tcp 127.0.0.1:8500: connect: connection refused\"\n", + "ts=2024-02-22T19:12:14.358Z caller=consul.go:295 level=error component=\"discovery manager scrape\" discovery=consul config=consul-sd-demo msg=\"Error retrieving datacenter name\" err=\"Get \\\"http://localhost:8500/v1/agent/self\\\": dial tcp 127.0.0.1:8500: connect: connection refused\"\n", + "ts=2024-02-22T19:12:29.359Z caller=consul.go:295 level=error component=\"discovery manager scrape\" discovery=consul config=consul-sd-demo msg=\"Error retrieving datacenter name\" err=\"Get \\\"http://localhost:8500/v1/agent/self\\\": dial tcp 127.0.0.1:8500: connect: connection refused\"\n", + "ts=2024-02-22T19:12:44.360Z caller=consul.go:295 level=error component=\"discovery manager scrape\" discovery=consul config=consul-sd-demo msg=\"Error retrieving datacenter name\" err=\"Get \\\"http://localhost:8500/v1/agent/self\\\": dial tcp 127.0.0.1:8500: connect: connection refused\"\n", + "ts=2024-02-22T19:12:59.361Z caller=consul.go:295 level=error component=\"discovery manager scrape\" discovery=consul config=consul-sd-demo msg=\"Error retrieving datacenter name\" err=\"Get \\\"http://localhost:8500/v1/agent/self\\\": dial tcp 127.0.0.1:8500: connect: connection refused\"\n", + "ts=2024-02-22T19:13:14.362Z caller=consul.go:295 level=error component=\"discovery manager scrape\" discovery=consul config=consul-sd-demo msg=\"Error retrieving datacenter name\" err=\"Get \\\"http://localhost:8500/v1/agent/self\\\": dial tcp 127.0.0.1:8500: connect: connection refused\"\n", + "ts=2024-02-22T19:13:29.364Z caller=consul.go:295 level=error component=\"discovery manager scrape\" discovery=consul config=consul-sd-demo msg=\"Error retrieving datacenter name\" err=\"Get \\\"http://localhost:8500/v1/agent/self\\\": dial tcp 127.0.0.1:8500: connect: connection refused\"\n", + "ts=2024-02-22T19:13:44.365Z caller=consul.go:295 level=error component=\"discovery manager scrape\" discovery=consul config=consul-sd-demo msg=\"Error retrieving datacenter name\" err=\"Get \\\"http://localhost:8500/v1/agent/self\\\": dial tcp 127.0.0.1:8500: connect: connection refused\"\n", + "ts=2024-02-22T19:13:59.365Z caller=consul.go:295 level=error component=\"discovery manager scrape\" discovery=consul config=consul-sd-demo msg=\"Error retrieving datacenter name\" err=\"Get \\\"http://localhost:8500/v1/agent/self\\\": dial tcp 127.0.0.1:8500: connect: connection refused\"\n", + "ts=2024-02-22T19:14:14.366Z caller=consul.go:295 level=error component=\"discovery manager scrape\" discovery=consul config=consul-sd-demo msg=\"Error retrieving datacenter name\" err=\"Get \\\"http://localhost:8500/v1/agent/self\\\": dial tcp 127.0.0.1:8500: connect: connection refused\"\n", + "ts=2024-02-22T19:14:29.367Z caller=consul.go:295 level=error component=\"discovery manager scrape\" discovery=consul config=consul-sd-demo msg=\"Error retrieving datacenter name\" err=\"Get \\\"http://localhost:8500/v1/agent/self\\\": dial tcp 127.0.0.1:8500: connect: connection refused\"\n", + "ts=2024-02-22T19:14:44.368Z caller=consul.go:295 level=error component=\"discovery manager scrape\" discovery=consul config=consul-sd-demo msg=\"Error retrieving datacenter name\" err=\"Get \\\"http://localhost:8500/v1/agent/self\\\": dial tcp 127.0.0.1:8500: connect: connection refused\"\n", + "ts=2024-02-22T19:14:59.368Z caller=consul.go:295 level=error component=\"discovery manager scrape\" discovery=consul config=consul-sd-demo msg=\"Error retrieving datacenter name\" err=\"Get \\\"http://localhost:8500/v1/agent/self\\\": dial tcp 127.0.0.1:8500: connect: connection refused\"\n", + "ts=2024-02-22T19:15:14.370Z caller=consul.go:295 level=error component=\"discovery manager scrape\" discovery=consul config=consul-sd-demo msg=\"Error retrieving datacenter name\" err=\"Get \\\"http://localhost:8500/v1/agent/self\\\": dial tcp 127.0.0.1:8500: connect: connection refused\"\n", + "ts=2024-02-22T19:15:29.370Z caller=consul.go:295 level=error component=\"discovery manager scrape\" discovery=consul config=consul-sd-demo msg=\"Error retrieving datacenter name\" err=\"Get \\\"http://localhost:8500/v1/agent/self\\\": dial tcp 127.0.0.1:8500: connect: connection refused\"\n", + "ts=2024-02-22T19:15:44.372Z caller=consul.go:295 level=error component=\"discovery manager scrape\" discovery=consul config=consul-sd-demo msg=\"Error retrieving datacenter name\" err=\"Get \\\"http://localhost:8500/v1/agent/self\\\": dial tcp 127.0.0.1:8500: connect: connection refused\"\n", + "ts=2024-02-22T19:15:59.373Z caller=consul.go:295 level=error component=\"discovery manager scrape\" discovery=consul config=consul-sd-demo msg=\"Error retrieving datacenter name\" err=\"Get \\\"http://localhost:8500/v1/agent/self\\\": dial tcp 127.0.0.1:8500: connect: connection refused\"\n", + "ts=2024-02-22T19:16:14.373Z caller=consul.go:295 level=error component=\"discovery manager scrape\" discovery=consul config=consul-sd-demo msg=\"Error retrieving datacenter name\" err=\"Get \\\"http://localhost:8500/v1/agent/self\\\": dial tcp 127.0.0.1:8500: connect: connection refused\"\n", + "ts=2024-02-22T19:16:29.374Z caller=consul.go:295 level=error component=\"discovery manager scrape\" discovery=consul config=consul-sd-demo msg=\"Error retrieving datacenter name\" err=\"Get \\\"http://localhost:8500/v1/agent/self\\\": dial tcp 127.0.0.1:8500: connect: connection refused\"\n", + "ts=2024-02-22T19:16:44.399Z caller=consul.go:295 level=error component=\"discovery manager scrape\" discovery=consul config=consul-sd-demo msg=\"Error retrieving datacenter name\" err=\"Get \\\"http://localhost:8500/v1/agent/self\\\": dial tcp 127.0.0.1:8500: connect: connection refused\"\n", + "ts=2024-02-22T19:16:59.400Z caller=consul.go:295 level=error component=\"discovery manager scrape\" discovery=consul config=consul-sd-demo msg=\"Error retrieving datacenter name\" err=\"Get \\\"http://localhost:8500/v1/agent/self\\\": dial tcp 127.0.0.1:8500: connect: connection refused\"\n", + "ts=2024-02-22T19:17:14.401Z caller=consul.go:295 level=error component=\"discovery manager scrape\" discovery=consul config=consul-sd-demo msg=\"Error retrieving datacenter name\" err=\"Get \\\"http://localhost:8500/v1/agent/self\\\": dial tcp 127.0.0.1:8500: connect: connection refused\"\n", + "ts=2024-02-22T19:17:29.402Z caller=consul.go:295 level=error component=\"discovery manager scrape\" discovery=consul config=consul-sd-demo msg=\"Error retrieving datacenter name\" err=\"Get \\\"http://localhost:8500/v1/agent/self\\\": dial tcp 127.0.0.1:8500: connect: connection refused\"\n", + "ts=2024-02-22T19:17:44.403Z caller=consul.go:295 level=error component=\"discovery manager scrape\" discovery=consul config=consul-sd-demo msg=\"Error retrieving datacenter name\" err=\"Get \\\"http://localhost:8500/v1/agent/self\\\": dial tcp 127.0.0.1:8500: connect: connection refused\"\n", + "ts=2024-02-22T19:17:59.404Z caller=consul.go:295 level=error component=\"discovery manager scrape\" discovery=consul config=consul-sd-demo msg=\"Error retrieving datacenter name\" err=\"Get \\\"http://localhost:8500/v1/agent/self\\\": dial tcp 127.0.0.1:8500: connect: connection refused\"\n", + "ts=2024-02-22T19:18:14.410Z caller=consul.go:295 level=error component=\"discovery manager scrape\" discovery=consul config=consul-sd-demo msg=\"Error retrieving datacenter name\" err=\"Get \\\"http://localhost:8500/v1/agent/self\\\": dial tcp 127.0.0.1:8500: connect: connection refused\"\n", + "ts=2024-02-22T19:18:29.413Z caller=consul.go:295 level=error component=\"discovery manager scrape\" discovery=consul config=consul-sd-demo msg=\"Error retrieving datacenter name\" err=\"Get \\\"http://localhost:8500/v1/agent/self\\\": dial tcp 127.0.0.1:8500: connect: connection refused\"\n", + "ts=2024-02-22T19:18:44.415Z caller=consul.go:295 level=error component=\"discovery manager scrape\" discovery=consul config=consul-sd-demo msg=\"Error retrieving datacenter name\" err=\"Get \\\"http://localhost:8500/v1/agent/self\\\": dial tcp 127.0.0.1:8500: connect: connection refused\"\n", + "ts=2024-02-22T19:18:59.436Z caller=consul.go:295 level=error component=\"discovery manager scrape\" discovery=consul config=consul-sd-demo msg=\"Error retrieving datacenter name\" err=\"Get \\\"http://localhost:8500/v1/agent/self\\\": dial tcp 127.0.0.1:8500: connect: connection refused\"\n", + "ts=2024-02-22T19:19:14.438Z caller=consul.go:295 level=error component=\"discovery manager scrape\" discovery=consul config=consul-sd-demo msg=\"Error retrieving datacenter name\" err=\"Get \\\"http://localhost:8500/v1/agent/self\\\": dial tcp 127.0.0.1:8500: connect: connection refused\"\n", + "ts=2024-02-22T19:19:29.443Z caller=consul.go:295 level=error component=\"discovery manager scrape\" discovery=consul config=consul-sd-demo msg=\"Error retrieving datacenter name\" err=\"Get \\\"http://localhost:8500/v1/agent/self\\\": dial tcp 127.0.0.1:8500: connect: connection refused\"\n" + ] + } + ], "source": [ "%%bash -s {WORKDIR}\n", "(\n", @@ -753,6 +822,336 @@ " killall -HUP prometheus\n", ")" ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### BlackBox Exporter\n", + "Instead of the target providing data, we can probe the target externally using protocols like HTTP, TCP, DNS, etc. The prometheus service discovery provides the targets to the BlackBox Exporter which then scrapes the target, here some relabelling is used." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "%%bash -s {WORKDIR} {BLACKBOX_EXPORTER_VERSION} \n", + "{\n", + " set -x\n", + " WORKDIR=$1\n", + " VERSION=${2:-0.24.0}\n", + " cd ${WORKDIR}\n", + " echo \"Downloading blackbox_exporter ${VERSION} to ${WORKDIR}\"\n", + " wget -q https://github.com/prometheus/blackbox_exporter/releases/download/v${VERSION}/blackbox_exporter-${VERSION}.linux-amd64.tar.gz\n", + " tar zxf blackbox_exporter-${VERSION}.linux-amd64.tar.gz\n", + " rm -f blackbox_exporter-${VERSION}.linux-amd64\n", + " \n", + " cd ./blackbox_exporter-${VERSION}.linux-amd64\n", + " \n", + " cat > ./blackbox.yml <<-EOD\n", + "modules:\n", + " http_2xx:\n", + " prober: http\n", + " timeout: 2s\n", + " http:\n", + " valid_http_versions: [ \"HTTP/1.1\", \"HTTP/2\" ]\n", + " valid_status_codes: [] # Defaults to 2xx\n", + " method: GET\n", + " preferred_ip_protocol: \"ip4\" # defaults to \"ip6\"\n", + "EOD\n", + " killall blackbox_exporter\n", + " \n", + " ./blackbox_exporter > /dev/null 2>&1 &\n", + " echo \"Started blackbox_exporter, http://localhost:9115/\"\n", + "}" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "#### Update the scrape config to probe some websites using blackbox exporter\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "%%bash -s {WORKDIR}\n", + "(\n", + " echo \"Updating the prometheus.yml to include blackbox_exporter\"\n", + " WORKDIR=$1\n", + " cat > ${WORKDIR}/prometheus-*/prometheus.yml <<-EOD \n", + "global:\n", + " scrape_interval: 5s\n", + " evaluation_interval: 5s\n", + "scrape_configs:\n", + " - job_name: 'prometheus'\n", + " static_configs:\n", + " - targets: ['localhost:9090']\n", + " - job_name: 'demo-service'\n", + " static_configs:\n", + " - targets:\n", + " - localhost:10001\n", + " - localhost:10002\n", + " - localhost:10003\n", + " metric_relabel_configs:\n", + " - action: keep\n", + " source_labels: [__name__]\n", + " regex: '(demo_|http_).*'\n", + " - job_name: 'node_exporter'\n", + " static_configs:\n", + " - targets:\n", + " - localhost:9100\n", + " - job_name: 'cadvisor'\n", + " static_configs:\n", + " - targets:\n", + " - localhost:8080\n", + " - job_name: 'cpu-metrics'\n", + " static_configs:\n", + " - targets:\n", + " - localhost:8100\n", + " - job_name: 'consul-sd-demo'\n", + " consul_sd_configs:\n", + " - server: 'localhost:8500'\n", + " relabel_configs:\n", + " - action: keep\n", + " source_labels: [__meta_consul_service]\n", + " regex: demo\n", + " - job_name: 'file-sd-demo'\n", + " file_sd_configs:\n", + " - files:\n", + " - 'targets.yml'\n", + " \n", + " - job_name: 'blackbox'\n", + " metrics_path: /probe\n", + " params:\n", + " module: [http_2xx]\n", + " static_configs:\n", + " - targets:\n", + " - http://prometheus.io\n", + " - https://prometheus.io\n", + " - http://example.com:8080\n", + " relabel_configs:\n", + " - source_labels: [__address__]\n", + " target_label: __param_target\n", + " - source_labels: [__param_target]\n", + " target_label: instance\n", + " - target_label: __address__\n", + " replacement: localhost:9115\n", + " \n", + "EOD\n", + "\n", + " # Create the targets.yml\n", + " echo \"Creating the targets.yml file\"\n", + " cd $WORKDIR/prometheus-*/\n", + " cat > targets.yml <<-EOD\n", + "- targets:\n", + " - localhost:10001\n", + " - localhost:10002\n", + " labels:\n", + " env: production\n", + "- targets:\n", + " - localhost:10003\n", + " labels:\n", + " env: staging\n", + "EOD\n", + " \n", + " # Send signal to reaload\n", + " killall -HUP prometheus\n", + ")" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Using PushGateway\n", + "\n", + "Its the way for ephemeral jobs to send metric data that get persisted until the next write, the prometheus server scrapes from the PushGateway" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "#### Installing PushGateway\n", + "https://github.com/prometheus/pushgateway/releases/download/v1.7.0/pushgateway-1.7.0.linux-amd64.tar.gz" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "%%bash -s {WORKDIR} {PUSHGATEWAY_VERSION} \n", + "{\n", + " set -x\n", + " WORKDIR=$1\n", + " VERSION=${2:-1.7.0}\n", + " cd ${WORKDIR}\n", + " echo \"Downloading PushGateweay ${VERSION} to ${WORKDIR}\"\n", + " wget -q https://github.com/prometheus/pushgateway/releases/download/v${VERSION}/pushgateway-${VERSION}.linux-amd64.tar.gz\n", + " tar zxf pushgateway-${VERSION}.linux-amd64.tar.gz\n", + " \n", + " cd ./pushgateway-${VERSION}.linux-amd64\n", + " \n", + " killall pushgateway\n", + " \n", + " ./pushgateway > /dev/null 2>&1 &\n", + " echo \"Started pushgateway, http://localhost:9091/\"\n", + "}" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Update scrape config and restart prometheus" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "%%bash -s {WORKDIR}\n", + "(\n", + " echo \"Updating the prometheus.yml to include pushgateway\"\n", + " WORKDIR=$1\n", + " cat > ${WORKDIR}/prometheus-*/prometheus.yml <<-EOD \n", + "global:\n", + " scrape_interval: 5s\n", + " evaluation_interval: 5s\n", + "scrape_configs:\n", + " - job_name: 'prometheus'\n", + " static_configs:\n", + " - targets: ['localhost:9090']\n", + " - job_name: 'demo-service'\n", + " static_configs:\n", + " - targets:\n", + " - localhost:10001\n", + " - localhost:10002\n", + " - localhost:10003\n", + " metric_relabel_configs:\n", + " - action: keep\n", + " source_labels: [__name__]\n", + " regex: '(demo_|http_).*'\n", + " - job_name: 'node_exporter'\n", + " static_configs:\n", + " - targets:\n", + " - localhost:9100\n", + " - job_name: 'cadvisor'\n", + " static_configs:\n", + " - targets:\n", + " - localhost:8080\n", + " - job_name: 'cpu-metrics'\n", + " static_configs:\n", + " - targets:\n", + " - localhost:8100\n", + " - job_name: 'consul-sd-demo'\n", + " consul_sd_configs:\n", + " - server: 'localhost:8500'\n", + " relabel_configs:\n", + " - action: keep\n", + " source_labels: [__meta_consul_service]\n", + " regex: demo\n", + " - job_name: 'file-sd-demo'\n", + " file_sd_configs:\n", + " - files:\n", + " - 'targets.yml'\n", + " \n", + " - job_name: 'blackbox'\n", + " metrics_path: /probe\n", + " params:\n", + " module: [http_2xx]\n", + " static_configs:\n", + " - targets:\n", + " - http://prometheus.io\n", + " - https://prometheus.io\n", + " - http://example.com:8080\n", + " relabel_configs:\n", + " - source_labels: [__address__]\n", + " target_label: __param_target\n", + " - source_labels: [__param_target]\n", + " target_label: instance\n", + " - target_label: __address__\n", + " replacement: localhost:9115\n", + " \n", + " - job_name: 'pushgateway'\n", + " honor_labels: true\n", + " static_configs:\n", + " - targets: ['localhost:9091']\n", + " \n", + "EOD\n", + "\n", + " # Create the targets.yml\n", + " echo \"Creating the targets.yml file\"\n", + " cd $WORKDIR/prometheus-*/\n", + " cat > targets.yml <<-EOD\n", + "- targets:\n", + " - localhost:10001\n", + " - localhost:10002\n", + " labels:\n", + " env: production\n", + "- targets:\n", + " - localhost:10003\n", + " labels:\n", + " env: staging\n", + "EOD\n", + " \n", + " # Send signal to reaload\n", + " killall -HUP prometheus\n", + ")" + ] + }, + { + "cell_type": "code", + "execution_count": 36, + "metadata": {}, + "outputs": [ + { + "ename": "SyntaxError", + "evalue": "invalid syntax (3217222304.py, line 4)", + "output_type": "error", + "traceback": [ + "\u001b[0;36m Cell \u001b[0;32mIn[36], line 4\u001b[0;36m\u001b[0m\n\u001b[0;31m curl --data-binary @- -- http://localhost:9091/metrics/job/demo_batch_job <