[Bug] KubeRay operator failed to watch endpoint (ray-project#2080)

helm-chart/kuberay-operator/templates/multiple_namespaces_role.yaml

@@ -39,6 +39,7 @@ rules:
   verbs:
   - get
   - list
+  - watch
 - apiGroups:
   - ""
   resources:

helm-chart/kuberay-operator/templates/role.yaml

@@ -35,6 +35,7 @@ rules:
   verbs:
   - get
   - list
+  - watch
 - apiGroups:
   - ""
   resources:

ray-operator/config/rbac/role.yaml

@@ -32,6 +32,7 @@ rules:
   verbs:
   - get
   - list
+  - watch
 - apiGroups:
   - ""
   resources:

ray-operator/controllers/ray/rayservice_controller.go

@@ -82,7 +82,7 @@ func NewRayServiceReconciler(ctx context.Context, mgr manager.Manager, dashboard
 // +kubebuilder:rbac:groups=core,resources=events,verbs=get;list;watch;create;update;patch;delete
 // +kubebuilder:rbac:groups=core,resources=pods,verbs=get;list;watch;create;update;patch;delete
 // +kubebuilder:rbac:groups=core,resources=pods/status,verbs=get;list;watch;create;update;patch;delete
-// +kubebuilder:rbac:groups=core,resources=endpoints,verbs=get;list
+// +kubebuilder:rbac:groups=core,resources=endpoints,verbs=get;list;watch
 // +kubebuilder:rbac:groups=core,resources=services,verbs=get;list;watch;create;update;patch;delete
 // +kubebuilder:rbac:groups=core,resources=services/status,verbs=get;update;patch
 // +kubebuilder:rbac:groups=coordination.k8s.io,resources=leases,verbs=get;list;create;update

tests/framework/prototype.py

@@ -1,9 +1,9 @@
 """Configuration test framework for KubeRay"""
 import json
-import jsonpatch
-from typing import Dict, List, Optional
-import unittest
 import time
+import unittest
+from typing import Dict, List, Optional
+import jsonpatch
 
 from framework.utils import (
     create_custom_object,
@@ -306,17 +306,20 @@ def assert_rule(self, custom_resource, cr_namespace):
                 name=custom_resource["metadata"]["name"],
                 namespace=cr_namespace,
                 path=query.get("path").rstrip("/"),
-                json=json.dumps(query["json_args"]),
+                json=json.dumps(query["json_args"])
             )
+
             if self.start_in_background:
-                shell_subprocess_run(f"{cmd} &", hide_output=True)
+                shell_subprocess_run(f"{cmd} &", hide_output=False)
 
             else:
                 output = shell_subprocess_check_output(cmd)
+                logger.info("curl output: %s", output.decode('utf-8'))
                 if hasattr(query.get("expected_output"), "__iter__"):
                     assert output.decode('utf-8') in query["expected_output"]
                 else:
                     assert output.decode('utf-8') == query["expected_output"]
+            time.sleep(1)
 
 class AutoscaleRule(Rule):
     def __init__(

tests/test_sample_rayservice_yamls.py

@@ -1,12 +1,11 @@
 ''' Test sample RayService YAML files to catch invalid and outdated ones. '''
 from copy import deepcopy
-from kubernetes import client
 import logging
 import pytest
 import sys
 from tempfile import NamedTemporaryFile
 import time
-from typing import Any, Dict, List, Optional
+from typing import Dict, List, Optional
 import yaml
 
 from framework.prototype import (
@@ -15,10 +14,7 @@
     EasyJobRule,
     CurlServiceRule,
     AutoscaleRule,
-    get_expected_head_pods,
-    get_expected_worker_pods,
     show_cluster_info,
-    check_pod_running,
 )
 
 from framework.utils import (