read namespace only once (#105)

* read namespace only once

* mention number of resources

* test name exclusion, clean up unused options

README.rst

@@ -153,9 +153,8 @@ Available command line options:
 ``--exclude-namespaces``
     Exclude namespaces from downscaling (default: kube-system), can also be configured via environment variable ``EXCLUDE_NAMESPACES``
 ``--exclude-deployments``
-    Exclude specific deployments from downscaling (default: kube-downscaler, downscaler), can also be configured via environment variable ``EXCLUDE_DEPLOYMENTS``
-``--exclude-statefulsets``
-    Exclude specific statefulsets from statefulsets, can also be configured via environment variable ``EXCLUDE_STATEFULSETS``
+    Exclude specific deployments/statefulsets/cronjobs from downscaling (default: kube-downscaler, downscaler), can also be configured via environment variable ``EXCLUDE_DEPLOYMENTS``.
+    Despite its name, this option will match the name of any included resource type (Deployment, StatefulSet, CronJob, ..).
 ``--downtime-replicas``
     Default value of replicas to downscale to, the annotation ``downscaler/downtime-replicas`` takes precedence over this value.
 ``--deployment-time-annotation``

kube_downscaler/cmd.py

@@ -76,16 +76,6 @@ def get_parser():
         help="Exclude specific deployments from downscaling (default: kube-downscaler,downscaler)",
         default=os.getenv("EXCLUDE_DEPLOYMENTS", "kube-downscaler,downscaler"),
     )
-    parser.add_argument(
-        "--exclude-statefulsets",
-        help="Exclude specific statefulsets from downscaling",
-        default=os.getenv("EXCLUDE_STATEFULSETS", ""),
-    )
-    parser.add_argument(
-        "--exclude-cronjobs",
-        help="Exclude specific cronjobs from downscaling",
-        default=os.getenv("EXCLUDE_CRONJOBS", ""),
-    )
     parser.add_argument(
         "--downtime-replicas",
         type=int,

kube_downscaler/main.py

@@ -35,8 +35,6 @@ def main(args=None):
         args.default_downtime,
         args.exclude_namespaces,
         args.exclude_deployments,
-        args.exclude_statefulsets,
-        args.exclude_cronjobs,
         args.grace_period,
         args.interval,
         args.dry_run,
@@ -55,8 +53,6 @@ def run_loop(
     default_downtime,
     exclude_namespaces,
     exclude_deployments,
-    exclude_statefulsets,
-    exclude_cronjobs,
     grace_period,
     interval,
     dry_run,
@@ -75,8 +71,6 @@ def run_loop(
                 include_resources=frozenset(include_resources.split(",")),
                 exclude_namespaces=frozenset(exclude_namespaces.split(",")),
                 exclude_deployments=frozenset(exclude_deployments.split(",")),
-                exclude_statefulsets=frozenset(exclude_statefulsets.split(",")),
-                exclude_cronjobs=frozenset(exclude_cronjobs.split(",")),
                 dry_run=dry_run,
                 grace_period=grace_period,
                 downtime_replicas=downtime_replicas,

kube_downscaler/scaler.py

@@ -1,3 +1,4 @@
+import collections
 import datetime
 import logging
 from typing import FrozenSet
@@ -7,6 +8,7 @@
 from pykube import CronJob
 from pykube import Deployment
 from pykube import HorizontalPodAutoscaler
+from pykube import Namespace
 from pykube import StatefulSet
 from pykube.objects import NamespacedAPIObject
 
@@ -329,15 +331,29 @@ def autoscale_resources(
     downtime_replicas: int,
     deployment_time_annotation: Optional[str] = None,
 ):
+    resources_by_namespace = collections.defaultdict(list)
     for resource in kind.objects(api, namespace=(namespace or pykube.all)):
-        if resource.namespace in exclude_namespaces or resource.name in exclude_names:
+        if resource.name in exclude_names:
             logger.debug(
-                f"Resource {resource.name} was excluded (either resource itself or namespace {resource.namespace} are excluded)"
+                f"{resource.kind} {resource.namespace}/{resource.name} was excluded (name matches exclusion list)"
             )
             continue
+        resources_by_namespace[resource.namespace].append(resource)
+
+    for current_namespace, resources in sorted(resources_by_namespace.items()):
+
+        if current_namespace in exclude_namespaces:
+            logger.debug(
+                f"Namespace {current_namespace} was excluded (exclusion list matches)"
+            )
+            continue
+
+        logger.debug(
+            f"Processing {len(resources)} {kind.endpoint} in namespace {current_namespace}.."
+        )
 
         # Override defaults with (optional) annotations from Namespace
-        namespace_obj = pykube.Namespace.objects(api).get_by_name(resource.namespace)
+        namespace_obj = Namespace.objects(api).get_by_name(current_namespace)
 
         excluded = ignore_resource(namespace_obj, now)
 
@@ -362,21 +378,21 @@ def autoscale_resources(
         forced_uptime_for_namespace = namespace_obj.annotations.get(
             FORCE_UPTIME_ANNOTATION, forced_uptime
         )
-
-        autoscale_resource(
-            resource,
-            upscale_period_for_namespace,
-            downscale_period_for_namespace,
-            default_uptime_for_namespace,
-            default_downtime_for_namespace,
-            forced_uptime_for_namespace,
-            dry_run,
-            now,
-            grace_period,
-            default_downtime_replicas_for_namespace,
-            namespace_excluded=excluded,
-            deployment_time_annotation=deployment_time_annotation,
-        )
+        for resource in resources:
+            autoscale_resource(
+                resource,
+                upscale_period_for_namespace,
+                downscale_period_for_namespace,
+                default_uptime_for_namespace,
+                default_downtime_for_namespace,
+                forced_uptime_for_namespace,
+                dry_run,
+                now,
+                grace_period,
+                default_downtime_replicas_for_namespace,
+                namespace_excluded=excluded,
+                deployment_time_annotation=deployment_time_annotation,
+            )
 
 
 def scale(
@@ -388,8 +404,6 @@ def scale(
     include_resources: FrozenSet[str],
     exclude_namespaces: FrozenSet[str],
     exclude_deployments: FrozenSet[str],
-    exclude_statefulsets: FrozenSet[str],
-    exclude_cronjobs: FrozenSet[str],
     dry_run: bool,
     grace_period: int,
     downtime_replicas: int = 0,

tests/test_scaler.py

@@ -53,8 +53,6 @@ def get(url, version, **kwargs):
         include_resources=include_resources,
         exclude_namespaces=[],
         exclude_deployments=[],
-        exclude_statefulsets=[],
-        exclude_cronjobs=[],
         dry_run=False,
         grace_period=300,
         downtime_replicas=0,
@@ -114,8 +112,6 @@ def get(url, version, **kwargs):
         include_resources=include_resources,
         exclude_namespaces=["system-ns"],
         exclude_deployments=[],
-        exclude_statefulsets=[],
-        exclude_cronjobs=[],
         dry_run=False,
         grace_period=300,
         downtime_replicas=0,
@@ -190,8 +186,6 @@ def get(url, version, **kwargs):
         include_resources=include_resources,
         exclude_namespaces=[],
         exclude_deployments=[],
-        exclude_statefulsets=[],
-        exclude_cronjobs=[],
         dry_run=False,
         grace_period=300,
         downtime_replicas=0,
@@ -258,8 +252,6 @@ def get(url, version, **kwargs):
         include_resources=include_resources,
         exclude_namespaces=[],
         exclude_deployments=[],
-        exclude_statefulsets=[],
-        exclude_cronjobs=[],
         dry_run=False,
         grace_period=300,
         downtime_replicas=0,
@@ -319,8 +311,6 @@ def get(url, version, **kwargs):
         include_resources=include_resources,
         exclude_namespaces=[],
         exclude_deployments=[],
-        exclude_statefulsets=[],
-        exclude_cronjobs=[],
         dry_run=False,
         grace_period=300,
         downtime_replicas=0,
@@ -381,8 +371,6 @@ def get(url, version, **kwargs):
         include_resources=include_resources,
         exclude_namespaces=[],
         exclude_deployments=[],
-        exclude_statefulsets=[],
-        exclude_cronjobs=[],
         dry_run=False,
         grace_period=300,
         downtime_replicas=0,
@@ -445,8 +433,6 @@ def get(url, version, **kwargs):
         include_resources=include_resources,
         exclude_namespaces=[],
         exclude_deployments=[],
-        exclude_statefulsets=[],
-        exclude_cronjobs=[],
         dry_run=False,
         grace_period=300,
         downtime_replicas=0,
@@ -506,8 +492,6 @@ def get(url, version, **kwargs):
         include_resources=include_resources,
         exclude_namespaces=[],
         exclude_deployments=[],
-        exclude_statefulsets=[],
-        exclude_cronjobs=[],
         dry_run=False,
         grace_period=300,
         downtime_replicas=0,
@@ -563,8 +547,6 @@ def get(url, version, **kwargs):
         include_resources=include_resources,
         exclude_namespaces=[],
         exclude_deployments=[],
-        exclude_statefulsets=[],
-        exclude_cronjobs=[],
         dry_run=False,
         grace_period=300,
         downtime_replicas=0,
@@ -619,8 +601,6 @@ def get(url, version, **kwargs):
         include_resources=include_resources,
         exclude_namespaces=[],
         exclude_deployments=[],
-        exclude_statefulsets=[],
-        exclude_cronjobs=[],
         dry_run=False,
         grace_period=300,
         downtime_replicas=0,
@@ -693,8 +673,6 @@ def get(url, version, **kwargs):
         include_resources=include_resources,
         exclude_namespaces=[],
         exclude_deployments=[],
-        exclude_statefulsets=[],
-        exclude_cronjobs=[],
         dry_run=False,
         grace_period=300,
         downtime_replicas=0,
@@ -759,8 +737,6 @@ def get(url, version, **kwargs):
         include_resources=include_resources,
         exclude_namespaces=[],
         exclude_deployments=[],
-        exclude_statefulsets=[],
-        exclude_cronjobs=[],
         dry_run=False,
         grace_period=300,
         downtime_replicas=0,
@@ -839,8 +815,6 @@ def get(url, version, **kwargs):
         include_resources=include_resources,
         exclude_namespaces=[],
         exclude_deployments=[],
-        exclude_statefulsets=[],
-        exclude_cronjobs=[],
         dry_run=False,
         grace_period=300,
     )
@@ -921,8 +895,6 @@ def get(url, version, **kwargs):
         include_resources=include_resources,
         exclude_namespaces=[],
         exclude_deployments=[],
-        exclude_statefulsets=[],
-        exclude_cronjobs=[],
         dry_run=False,
         grace_period=300,
         downtime_replicas=0,
@@ -942,3 +914,74 @@ def get(url, version, **kwargs):
     }
     assert api.patch.call_args[1]["url"] == "/deployments/deploy-2"
     assert json.loads(api.patch.call_args[1]["data"]) == patch_data
+
+
+def test_scaler_name_excluded(monkeypatch):
+    api = MagicMock()
+    monkeypatch.setattr(
+        "kube_downscaler.scaler.helper.get_kube_api", MagicMock(return_value=api)
+    )
+
+    def get(url, version, **kwargs):
+        if url == "pods":
+            data = {"items": []}
+        elif url == "deployments":
+            data = {
+                "items": [
+                    {
+                        "metadata": {
+                            "name": "sysdep-1",
+                            "namespace": "system-ns",
+                            "creationTimestamp": "2019-03-01T16:38:00Z",
+                        },
+                        "spec": {"replicas": 1},
+                    },
+                    {
+                        "metadata": {
+                            "name": "deploy-2",
+                            "namespace": "default",
+                            "creationTimestamp": "2019-03-01T16:38:00Z",
+                        },
+                        "spec": {"replicas": 2},
+                    },
+                ]
+            }
+        elif url == "namespaces/default":
+            data = {"metadata": {}}
+        else:
+            raise Exception(f"unexpected call: {url}, {version}, {kwargs}")
+
+        response = MagicMock()
+        response.json.return_value = data
+        return response
+
+    api.get = get
+
+    include_resources = frozenset(["deployments"])
+    scale(
+        namespace=None,
+        upscale_period="never",
+        downscale_period="never",
+        default_uptime="never",
+        default_downtime="always",
+        include_resources=include_resources,
+        exclude_namespaces=[],
+        exclude_deployments=["sysdep-1"],
+        dry_run=False,
+        grace_period=300,
+    )
+
+    assert api.patch.call_count == 1
+
+    # make sure that deploy-2 was updated (sysdep-1 was excluded)
+    patch_data = {
+        "metadata": {
+            "name": "deploy-2",
+            "namespace": "default",
+            "creationTimestamp": "2019-03-01T16:38:00Z",
+            "annotations": {ORIGINAL_REPLICAS_ANNOTATION: "2"},
+        },
+        "spec": {"replicas": 0},
+    }
+    assert api.patch.call_args[1]["url"] == "/deployments/deploy-2"
+    assert json.loads(api.patch.call_args[1]["data"]) == patch_data