Merge remote-tracking branch 'origin/develop' into develop

CHANGELOG.md

@@ -10,7 +10,6 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 - remove `accN` index from usage
 - scan each region in a separate process in order to reduce RAM usage
 - change `create_indexes` command. Now it ensures that indexes are up-to-date instead of recreating them
-- added `CAAS_HTTP_PROXY` and `CAAS_HTTPS_PROXY` envs. They only impact all boto3 clients, License manager client and job executor
 
 ## [5.4.0] - 2024-07-09
 - added `rule_source_id` and `excluded_rules` parameters to `POST /rulestets`.

src/helpers/constants.py

@@ -437,10 +437,6 @@ class CAASEnv(EnvEnum):
     # init envs
     SYSTEM_USER_PASSWORD = 'CAAS_SYSTEM_USER_PASSWORD'
 
-    # proxy, currently used only for scans and license manager api client
-    HTTP_PROXY = 'CAAS_HTTP_PROXY'
-    HTTPS_PROXY = 'CAAS_HTTPS_PROXY'
-
 
 class BatchJobEnv(EnvEnum):
     """

src/run.py

@@ -795,17 +795,27 @@ def get_credentials(tenant: Tenant,
     # 5
     if not credentials:
         _LOG.info(_log_start + 'instance profile')
-        # TODO refactor + do the same for other clouds. Try to resolve
-        #  from envs
-        try:
-            aid = StsClient.factory().build().get_caller_identity()['Account']
-            _LOG.debug('Instance profile found')
-            if aid == tenant.project:
-                _LOG.info('Instance profile credentials match to tenant id')
-                return {}
-        except (Exception, ClientError) as e:
-            _LOG.warning(f'No instance credentials found: {e}')
-
+        # TODO refactor
+        match tenant.cloud:
+            case Cloud.AWS:
+                try:
+                    aid = StsClient.factory().build().get_caller_identity()['Account']
+                    _LOG.debug('Instance profile found')
+                    if aid == tenant.project:
+                        _LOG.info('Instance profile credentials match to tenant id')
+                        return {}
+                except (Exception, ClientError) as e:
+                    _LOG.warning(f'No instance credentials found: {e}')
+            case Cloud.AZURE:
+                try:
+                    from c7n_azure.session import Session
+                    aid = Session().subscription_id
+                    _LOG.info('subscription id found')
+                    if aid == tenant.project:
+                        _LOG.info('Subscription id matches to tenant id')
+                        return {}
+                except BaseException:  # catch sys.exit(1)
+                    _LOG.warning('Could not find azure subscription id')
     if credentials:
         credentials = mcs.complete_credentials_dict(
             credentials=credentials,
@@ -1211,16 +1221,16 @@ def standard_job(job: Job, tenant: Tenant, work_dir: Path):
     _XRAY.put_annotation('tenant_name', tenant.name)
     _XRAY.put_metadata('cloud', cloud.value)
 
-    licensed_urls = map(operator.itemgetter('s3_path'),
-                        get_licensed_ruleset_dto_list(tenant, job))
-    standard_urls = map(SP.ruleset_service.download_url,
-                        BSP.policies_service.get_standard_rulesets(job))
-
     if platform:
         credentials = get_platform_credentials(platform)
     else:
         credentials = get_credentials(tenant)
 
+    licensed_urls = map(operator.itemgetter('s3_path'),
+                        get_licensed_ruleset_dto_list(tenant, job))
+    standard_urls = map(SP.ruleset_service.download_url,
+                        BSP.policies_service.get_standard_rulesets(job))
+
     policies = BSP.policies_service.get_policies(
         urls=chain(licensed_urls, standard_urls),
         keep=set(job.rules_to_scan),
@@ -1229,12 +1239,7 @@ def standard_job(job: Job, tenant: Tenant, work_dir: Path):
     with tempfile.NamedTemporaryFile(delete=False) as file:
         file.write(msgspec.json.encode(policies))
     failed = {}
-    proxies = {}
-    if url := CAASEnv.HTTP_PROXY.get():
-        proxies['HTTP_PROXY'] = url
-    if url := CAASEnv.HTTPS_PROXY.get():
-        proxies['HTTPS_PROXY'] = url
-    with EnvironmentContext(credentials | proxies, reset_all=False):
+    with EnvironmentContext(credentials, reset_all=False):
         q = multiprocessing.Queue()
         for region in [GLOBAL_REGION, ] + sorted(BSP.env.target_regions()):
             p = multiprocessing.Process(

src/services/clients/__init__.py

@@ -15,55 +15,36 @@
 class Boto3ClientFactory:
     _session = Session()  # class variable
 
-    def __init__(self, service: str, no_proxies: bool = False):
+    def __init__(self, service: str):
         self._service = service
-        self._no_proxies = no_proxies
-
-    def _build_default_config(self) -> Config:
-        proxy = {}
-        if url := CAASEnv.HTTP_PROXY.get():
-            proxy['http'] = url
-        if url := CAASEnv.HTTPS_PROXY.get():
-            proxy['https'] = url
-        if proxy and not self._no_proxies:
-            return Config(proxies=proxy)
-        return Config()
 
     def build(self, region_name: str = None, endpoint_url: str = None,
               aws_access_key_id: str = None, aws_secret_access_key: str = None,
               aws_session_token: str = None, config: Config = None,
               ) -> BaseClient:
-        _LOG.info(f'Building boto3 client for {self._service}')
-        conf = self._build_default_config()
-        if config:
-            conf = conf.merge(config)
         return self._session.client(
             service_name=self._service,
             region_name=region_name,
             endpoint_url=endpoint_url,
             aws_access_key_id=aws_access_key_id,
             aws_secret_access_key=aws_secret_access_key,
             aws_session_token=aws_session_token,
-            config=conf
+            config=config
         )
 
     def build_resource(self, region_name: str = None, endpoint_url: str = None,
                        aws_access_key_id: str = None,
                        aws_secret_access_key: str = None,
                        aws_session_token: str = None, config: Config = None,
                        ) -> ServiceResource:
-        _LOG.info(f'Building boto3 resource for {self._service}')
-        conf = self._build_default_config()
-        if config:
-            conf = conf.merge(config)
         return self._session.resource(
             service_name=self._service,
             region_name=region_name,
             endpoint_url=endpoint_url,
             aws_access_key_id=aws_access_key_id,
             aws_secret_access_key=aws_secret_access_key,
             aws_session_token=aws_session_token,
-            config=conf
+            config=config
         )
 
     def from_keys(self, aws_access_key_id: str, aws_secret_access_key: str,

src/services/clients/lm_client.py

@@ -153,13 +153,6 @@ def __init__(self, baseurl: str, token_producer: LmTokenProducer):
         self._token_producer = token_producer
 
         self._session = requests.Session()
-        proxies = {}
-        if url := CAASEnv.HTTPS_PROXY.get():
-            proxies['https'] = url
-        if url := CAASEnv.HTTP_PROXY.get():
-            proxies['http'] = url
-        if proxies:
-            self._session.proxies.update(proxies)
 
     def __del__(self):
         self._session.close()

src/services/clients/s3.py

@@ -56,8 +56,7 @@ def _base_config(cls) -> Config:
     def _minio_config(cls) -> Config:
         return cls._base_config().merge(Config(s3={
             'signature_version': 's3v4',
-            'addressing_style': 'path',
-            'proxies': {}
+            'addressing_style': 'path'
         }))
 
     def build_s3(self, region_name: str) -> 'S3Client':
@@ -81,7 +80,7 @@ def build_minio(self) -> 'S3Client':
 
         instance = self._wrapper.build()
         instance.resource = Boto3ClientFactory(
-            instance.service_name, no_proxies=True).build_resource(
+            instance.service_name).build_resource(
             aws_access_key_id=access_key,
             aws_secret_access_key=secret_key,
             endpoint_url=endpoint,