From d66976fd7b1731860ea3fd9a651857f37ffbc137 Mon Sep 17 00:00:00 2001 From: Victor Skvortsov Date: Thu, 17 Aug 2023 13:46:49 +0500 Subject: [PATCH 1/8] Drop storage region for AWS --- cli/dstack/_internal/backend/aws/__init__.py | 4 +- cli/dstack/_internal/backend/aws/compute.py | 4 +- cli/dstack/_internal/backend/aws/config.py | 16 ++-- cli/dstack/_internal/hub/schemas/__init__.py | 11 +-- .../hub/services/backends/aws/configurator.py | 94 +++++++------------ 5 files changed, 48 insertions(+), 81 deletions(-) diff --git a/cli/dstack/_internal/backend/aws/__init__.py b/cli/dstack/_internal/backend/aws/__init__.py index 58656a234..b872ea44b 100644 --- a/cli/dstack/_internal/backend/aws/__init__.py +++ b/cli/dstack/_internal/backend/aws/__init__.py @@ -27,12 +27,12 @@ def __init__( self.backend_config = backend_config if self.backend_config.credentials is not None: self._session = Session( - region_name=self.backend_config.region_name, + region_name=self.backend_config.region, aws_access_key_id=self.backend_config.credentials.get("access_key"), aws_secret_access_key=self.backend_config.credentials.get("secret_key"), ) else: - self._session = Session(region_name=self.backend_config.region_name) + self._session = Session(region_name=self.backend_config.region) self._storage = AWSStorage( s3_client=aws_utils.get_s3_client(self._session), bucket_name=self.backend_config.bucket_name, diff --git a/cli/dstack/_internal/backend/aws/compute.py b/cli/dstack/_internal/backend/aws/compute.py index ab49fbde1..76e2d9a3f 100644 --- a/cli/dstack/_internal/backend/aws/compute.py +++ b/cli/dstack/_internal/backend/aws/compute.py @@ -39,7 +39,7 @@ def get_instance_type(self, job: Job) -> Optional[InstanceType]: def get_supported_instances(self) -> List[InstanceType]: instances = {} - for region in [self.backend_config.region_name, *self.backend_config.extra_regions]: + for region in self.backend_config.regions: for i in runners._get_instance_types(self._get_ec2_client(region=region)): if i.instance_name not in instances: instances[i.instance_name] = i @@ -54,7 +54,7 @@ def run_instance( session=self.session, iam_client=self.iam_client, bucket_name=self.backend_config.bucket_name, - region_name=region or self.backend_config.region_name, + region_name=region or self.backend_config.region, subnet_id=self.backend_config.subnet_id, runner_id=job.runner_id, instance_type=instance_type, diff --git a/cli/dstack/_internal/backend/aws/config.py b/cli/dstack/_internal/backend/aws/config.py index 215979f96..46d0dd47b 100644 --- a/cli/dstack/_internal/backend/aws/config.py +++ b/cli/dstack/_internal/backend/aws/config.py @@ -4,25 +4,23 @@ from dstack._internal.backend.base.config import BackendConfig -DEFAULT_REGION_NAME = "us-east-1" +DEFAULT_REGION = "us-east-1" class AWSConfig(BackendConfig, BaseModel): bucket_name: str - region_name: Optional[str] = DEFAULT_REGION_NAME - extra_regions: List[str] = [] + regions: List[str] subnet_id: Optional[str] = None credentials: Optional[Dict] = None + # dynamically set + region: Optional[str] = DEFAULT_REGION def serialize(self) -> Dict: config_data = { "backend": "aws", "bucket": self.bucket_name, + "regions": self.regions, } - if self.region_name: - config_data["region"] = self.region_name - if self.extra_regions: - config_data["extra_regions"] = self.extra_regions if self.subnet_id: config_data["subnet"] = self.subnet_id return config_data @@ -37,7 +35,7 @@ def deserialize(cls, config_data: Dict) -> Optional["AWSConfig"]: return None return cls( bucket_name=bucket_name, - region_name=config_data.get("region"), - extra_regions=config_data.get("extra_regions", []), + regions=config_data.get("regions", []), subnet_id=config_data.get("subnet"), + region_name=config_data.get("region"), ) diff --git a/cli/dstack/_internal/hub/schemas/__init__.py b/cli/dstack/_internal/hub/schemas/__init__.py index b1c6ed23f..4ca93ce33 100644 --- a/cli/dstack/_internal/hub/schemas/__init__.py +++ b/cli/dstack/_internal/hub/schemas/__init__.py @@ -43,18 +43,14 @@ class LocalBackendConfig(BaseModel): class AWSBackendConfigPartial(BaseModel): type: Literal["aws"] = "aws" - region_name: Optional[str] - region_name_title: Optional[str] - extra_regions: Optional[List[str]] + regions: Optional[List[str]] s3_bucket_name: Optional[str] ec2_subnet_id: Optional[str] class AWSBackendConfig(BaseModel): type: Literal["aws"] = "aws" - region_name: str - region_name_title: Optional[str] - extra_regions: List[str] = [] + regions: List[str] s3_bucket_name: str ec2_subnet_id: Optional[str] @@ -274,8 +270,7 @@ class AWSBucketBackendElement(BaseModel): class AWSBackendValues(BaseModel): type: Literal["aws"] = "aws" default_credentials: bool = False - region_name: Optional[BackendElement] - extra_regions: Optional[BackendMultiElement] + regions: Optional[BackendMultiElement] s3_bucket_name: Optional[AWSBucketBackendElement] ec2_subnet_id: Optional[BackendElement] diff --git a/cli/dstack/_internal/hub/services/backends/aws/configurator.py b/cli/dstack/_internal/hub/services/backends/aws/configurator.py index fa6215f96..21bce4b64 100644 --- a/cli/dstack/_internal/hub/services/backends/aws/configurator.py +++ b/cli/dstack/_internal/hub/services/backends/aws/configurator.py @@ -5,9 +5,8 @@ from boto3.session import Session from dstack._internal.backend.aws import AwsBackend -from dstack._internal.backend.aws.config import DEFAULT_REGION_NAME, AWSConfig +from dstack._internal.backend.aws.config import DEFAULT_REGION, AWSConfig from dstack._internal.hub.db.models import Backend as DBBackend -from dstack._internal.hub.db.models import Project from dstack._internal.hub.schemas import ( AWSBackendConfig, AWSBackendConfigWithCreds, @@ -44,16 +43,10 @@ class AWSConfigurator(Configurator): def configure_backend( self, backend_config: AWSBackendConfigWithCredsPartial ) -> AWSBackendValues: - if ( - backend_config.region_name is not None - and backend_config.region_name not in REGION_VALUES - ): - raise BackendConfigError(f"Invalid AWS region {backend_config.region_name}") - backend_values = AWSBackendValues() session = Session() if session.region_name is None: - session = Session(region_name=backend_config.region_name or DEFAULT_REGION_NAME) + session = Session(region_name=DEFAULT_REGION) backend_values.default_credentials = self._valid_credentials(session=session) @@ -63,7 +56,7 @@ def configure_backend( project_credentials = backend_config.credentials.__root__ if project_credentials.type == "access_key": session = Session( - region_name=backend_config.region_name or DEFAULT_REGION_NAME, + region_name=DEFAULT_REGION, aws_access_key_id=project_credentials.access_key, aws_secret_access_key=project_credentials.secret_key, ) @@ -74,15 +67,11 @@ def configure_backend( elif not backend_values.default_credentials: self._raise_invalid_credentials_error(fields=[["credentials"]]) - # TODO validate config values - backend_values.region_name = self._get_hub_region_element(selected=session.region_name) - backend_values.extra_regions = self._get_hub_extra_regions_element( - region=session.region_name, - selected=backend_config.extra_regions or [], + backend_values.regions = self._get_hub_regions_element( + selected=backend_config.regions or [DEFAULT_REGION] ) backend_values.s3_bucket_name = self._get_hub_buckets_element( session=session, - region=session.region_name, selected=backend_config.s3_bucket_name, ) backend_values.ec2_subnet_id = self._get_hub_subnet_element( @@ -94,8 +83,7 @@ def create_backend( self, project_name: str, backend_config: AWSBackendConfigWithCreds ) -> Tuple[Dict, Dict]: config_data = { - "region_name": backend_config.region_name, - "extra_regions": backend_config.extra_regions, + "regions": backend_config.regions, "s3_bucket_name": backend_config.s3_bucket_name.replace("s3://", ""), "ec2_subnet_id": backend_config.ec2_subnet_id, } @@ -106,41 +94,36 @@ def get_backend_config( self, db_backend: DBBackend, include_creds: bool ) -> Union[AWSBackendConfig, AWSBackendConfigWithCreds]: json_config = json.loads(db_backend.config) - region_name = json_config["region_name"] + regions = json_config.get("regions") or ( + json_config.get("extra_regions", []) + [json_config.get("region_name")] + ) s3_bucket_name = json_config["s3_bucket_name"] ec2_subnet_id = json_config["ec2_subnet_id"] - extra_regions = json_config.get("extra_regions", []) if include_creds: json_auth = json.loads(db_backend.auth) return AWSBackendConfigWithCreds( - region_name=region_name, - region_name_title=region_name, - extra_regions=extra_regions, + regions=regions, s3_bucket_name=s3_bucket_name, ec2_subnet_id=ec2_subnet_id, credentials=AWSBackendCreds.parse_obj(json_auth), ) return AWSBackendConfig( - region_name=region_name, - region_name_title=region_name, - extra_regions=extra_regions, + regions=regions, s3_bucket_name=s3_bucket_name, ec2_subnet_id=ec2_subnet_id, ) def get_backend(self, db_backend: DBBackend) -> AwsBackend: - config_data = json.loads(db_backend.config) - auth_data = json.loads(db_backend.auth) + json_config = json.loads(db_backend.config) + json_auth = json.loads(db_backend.auth) + regions = json_config.get("regions") or ( + json_config.get("extra_regions", []) + [json_config.get("region_name")] + ) config = AWSConfig( - bucket_name=config_data.get("bucket") - or config_data.get("bucket_name") - or config_data.get("s3_bucket_name"), - region_name=config_data.get("region_name"), - extra_regions=config_data.get("extra_regions", []), - subnet_id=config_data.get("subnet_id") - or config_data.get("ec2_subnet_id") - or config_data.get("subnet"), - credentials=auth_data, + bucket_name=json_config.get("s3_bucket_name"), + regions=regions, + subnet_id=json_config.get("ec2_subnet_id"), + credentials=json_auth, ) return AwsBackend(config) @@ -159,26 +142,24 @@ def _raise_invalid_credentials_error(self, fields: Optional[List[List[str]]] = N fields=fields, ) - def _get_hub_region_element(self, selected: Optional[str]) -> BackendElement: - element = BackendElement(selected=selected) - for r in REGIONS: - element.values.append(BackendElementValue(value=r[1], label=r[1])) - return element - - def _get_hub_extra_regions_element( - self, region: str, selected: List[str] - ) -> BackendMultiElement: + def _get_hub_regions_element(self, selected: List[str]) -> BackendMultiElement: + for r in selected: + if r not in REGION_VALUES: + raise BackendConfigError( + f"The region {r} is invalid", + code="invalid_region", + fields=[["regions"]], + ) element = BackendMultiElement(selected=selected) for r in REGION_VALUES: - if r != region: - element.values.append(BackendElementValue(value=r, label=r)) + element.values.append(BackendElementValue(value=r, label=r)) return element def _get_hub_buckets_element( - self, session: Session, region: str, selected: Optional[str] + self, session: Session, selected: Optional[str] ) -> AWSBucketBackendElement: if selected: - self._validate_hub_bucket(session=session, region=region, bucket_name=selected) + self._validate_hub_bucket(session=session, bucket_name=selected) element = AWSBucketBackendElement(selected=selected) s3_client = session.client("s3") try: @@ -191,22 +172,15 @@ def _get_hub_buckets_element( AWSBucketBackendElementValue( name=bucket["Name"], created=bucket["CreationDate"].strftime("%d.%m.%Y %H:%M:%S"), - region=region, + region="", ) ) return element - def _validate_hub_bucket(self, session: Session, region: str, bucket_name: str): + def _validate_hub_bucket(self, session: Session, bucket_name: str): s3_client = session.client("s3") try: - response = s3_client.head_bucket(Bucket=bucket_name) - bucket_region = response["ResponseMetadata"]["HTTPHeaders"]["x-amz-bucket-region"] - if bucket_region.lower() != region: - raise BackendConfigError( - "The bucket belongs to another AWS region", - code="invalid_bucket", - fields=[["s3_bucket_name"]], - ) + s3_client.head_bucket(Bucket=bucket_name) except botocore.exceptions.ClientError as e: if ( hasattr(e, "response") From 4845f3a70ac919acaa0bfa03745009031e63c657 Mon Sep 17 00:00:00 2001 From: Victor Skvortsov Date: Thu, 17 Aug 2023 14:08:03 +0500 Subject: [PATCH 2/8] Drop storage region for GCP --- cli/dstack/_internal/backend/gcp/config.py | 13 +- cli/dstack/_internal/hub/schemas/__init__.py | 19 +-- .../hub/services/backends/aws/configurator.py | 14 +- .../hub/services/backends/gcp/configurator.py | 149 ++++-------------- 4 files changed, 50 insertions(+), 145 deletions(-) diff --git a/cli/dstack/_internal/backend/gcp/config.py b/cli/dstack/_internal/backend/gcp/config.py index fba45502b..5a7dd7a49 100644 --- a/cli/dstack/_internal/backend/gcp/config.py +++ b/cli/dstack/_internal/backend/gcp/config.py @@ -9,25 +9,26 @@ class GCPConfig(BackendConfig, BaseModel): backend: Literal["gcp"] = "gcp" project_id: str - region: str - zone: str + regions: List[str] bucket_name: str vpc: str subnet: str - extra_regions: List[str] = [] credentials_file: Optional[str] = None credentials: Optional[Dict] = None + # dynamically set + region: Optional[str] + zone: Optional[str] def serialize(self) -> Dict: res = { "backend": "gcp", "project": self.project_id, - "region": self.region, - "zone": self.zone, + "regions": self.regions, "bucket": self.bucket_name, "vpc": self.vpc, "subnet": self.subnet, - "extra_regions": self.extra_regions, + "region": self.region, + "zone": self.zone, } if self.credentials_file is not None: res["credentials_file"] = self.credentials_file diff --git a/cli/dstack/_internal/hub/schemas/__init__.py b/cli/dstack/_internal/hub/schemas/__init__.py index 4ca93ce33..30ee868c7 100644 --- a/cli/dstack/_internal/hub/schemas/__init__.py +++ b/cli/dstack/_internal/hub/schemas/__init__.py @@ -43,15 +43,15 @@ class LocalBackendConfig(BaseModel): class AWSBackendConfigPartial(BaseModel): type: Literal["aws"] = "aws" - regions: Optional[List[str]] s3_bucket_name: Optional[str] + regions: Optional[List[str]] ec2_subnet_id: Optional[str] class AWSBackendConfig(BaseModel): type: Literal["aws"] = "aws" - regions: List[str] s3_bucket_name: str + regions: List[str] ec2_subnet_id: Optional[str] @@ -81,24 +81,18 @@ class AWSBackendConfigWithCreds(AWSBackendConfig): class GCPBackendConfigPartial(BaseModel): type: Literal["gcp"] = "gcp" - area: Optional[str] - region: Optional[str] - zone: Optional[str] bucket_name: Optional[str] + regions: Optional[List[str]] vpc: Optional[str] subnet: Optional[str] - extra_regions: Optional[List[str]] class GCPBackendConfig(BaseModel): type: Literal["gcp"] = "gcp" - area: str - region: str - zone: str bucket_name: str + regions: List[str] vpc: str subnet: str - extra_regions: List[str] = [] class GCPBackendDefaultCreds(BaseModel): @@ -289,12 +283,9 @@ class GCPVPCSubnetBackendElement(BaseModel): class GCPBackendValues(BaseModel): type: Literal["gcp"] = "gcp" default_credentials: bool = False - area: Optional[BackendElement] - region: Optional[BackendElement] - zone: Optional[BackendElement] bucket_name: Optional[BackendElement] + regions: Optional[BackendMultiElement] vpc_subnet: Optional[GCPVPCSubnetBackendElement] - extra_regions: Optional[BackendMultiElement] class AzureBackendValues(BaseModel): diff --git a/cli/dstack/_internal/hub/services/backends/aws/configurator.py b/cli/dstack/_internal/hub/services/backends/aws/configurator.py index 21bce4b64..5bf69ca85 100644 --- a/cli/dstack/_internal/hub/services/backends/aws/configurator.py +++ b/cli/dstack/_internal/hub/services/backends/aws/configurator.py @@ -94,10 +94,11 @@ def get_backend_config( self, db_backend: DBBackend, include_creds: bool ) -> Union[AWSBackendConfig, AWSBackendConfigWithCreds]: json_config = json.loads(db_backend.config) - regions = json_config.get("regions") or ( - json_config.get("extra_regions", []) + [json_config.get("region_name")] - ) s3_bucket_name = json_config["s3_bucket_name"] + regions = json_config.get("regions") + if regions is None: + # old regions format + regions = json_config.get("extra_regions", []) + [json_config.get("region_name")] ec2_subnet_id = json_config["ec2_subnet_id"] if include_creds: json_auth = json.loads(db_backend.auth) @@ -116,9 +117,10 @@ def get_backend_config( def get_backend(self, db_backend: DBBackend) -> AwsBackend: json_config = json.loads(db_backend.config) json_auth = json.loads(db_backend.auth) - regions = json_config.get("regions") or ( - json_config.get("extra_regions", []) + [json_config.get("region_name")] - ) + regions = json_config.get("regions") + if regions is None: + # old regions format + regions = json_config.get("extra_regions", []) + [json_config.get("region_name")] config = AWSConfig( bucket_name=json_config.get("s3_bucket_name"), regions=regions, diff --git a/cli/dstack/_internal/hub/services/backends/gcp/configurator.py b/cli/dstack/_internal/hub/services/backends/gcp/configurator.py index 5576adf06..2604b37bb 100644 --- a/cli/dstack/_internal/hub/services/backends/gcp/configurator.py +++ b/cli/dstack/_internal/hub/services/backends/gcp/configurator.py @@ -27,9 +27,7 @@ ) from dstack._internal.hub.services.backends.base import BackendConfigError, Configurator -DEFAULT_GEOGRAPHIC_AREA = "North America" - -GCP_LOCATIONS = [ +LOCATIONS = [ { "name": "North America", "regions": [ @@ -108,6 +106,8 @@ "default_zone": "australia-southeast1-c", }, ] +REGIONS = [r for l in LOCATIONS for r in l["regions"]] +DEFAULT_REGION = "us-east1" class GCPConfigurator(Configurator): @@ -138,31 +138,17 @@ def configure_backend( elif not backend_values.default_credentials: self._raise_invalid_credentials_error(fields=[["credentials"]]) - backend_values.area = self._get_hub_geographic_area_element(backend_config.area) - location = self._get_location(backend_values.area.selected) - backend_values.region, regions = self._get_hub_region_element( - location=location, - selected=backend_config.region, - ) - backend_values.zone = self._get_hub_zone_element( - location=location, - region=regions.get(backend_values.region.selected), - selected=backend_config.zone, - ) backend_values.bucket_name = self._get_hub_buckets_element( - region=backend_values.region.selected, selected=backend_config.bucket_name, ) + backend_values.regions = self._get_hub_regions_element( + selected=backend_config.regions or [DEFAULT_REGION], + ) backend_values.vpc_subnet = self._get_hub_vpc_subnet_element( - region=backend_values.region.selected, + region=DEFAULT_REGION, selected_vpc=backend_config.vpc, selected_subnet=backend_config.subnet, ) - backend_values.extra_regions = self._get_hub_extra_regions_element( - region=backend_values.region.selected, - region_names=list(regions.keys()), - selected_regions=backend_config.extra_regions or [], - ) return backend_values def create_backend( @@ -179,13 +165,10 @@ def create_backend( auth_data["service_account_email"] = service_account_email config_data = { "project": self.project_id, - "area": backend_config.area, - "region": backend_config.region, - "zone": backend_config.zone, "bucket_name": backend_config.bucket_name, + "regions": backend_config.regions, "vpc": backend_config.vpc, "subnet": backend_config.subnet, - "extra_regions": backend_config.extra_regions, } return config_data, auth_data @@ -193,123 +176,65 @@ def get_backend_config( self, db_backend: DBBackend, include_creds: bool ) -> Union[GCPBackendConfig, GCPBackendConfigWithCreds]: config_data = json.loads(db_backend.config) - area = config_data["area"] - region = config_data["region"] - zone = config_data["zone"] + regions = config_data.get("regions") + if regions is None: + # old regions format + regions = config_data.get("extra_regions", []) + [config_data.get("region")] bucket_name = config_data["bucket_name"] vpc = config_data["vpc"] subnet = config_data["subnet"] - extra_regions = config_data.get("extra_regions", []) if include_creds: auth_data = json.loads(db_backend.auth) return GCPBackendConfigWithCreds( credentials=GCPBackendCreds.parse_obj(auth_data), - area=area, - region=region, - zone=zone, bucket_name=bucket_name, + regions=regions, vpc=vpc, subnet=subnet, - extra_regions=extra_regions, ) return GCPBackendConfig( - area=area, - region=region, - zone=zone, bucket_name=bucket_name, + regions=regions, vpc=vpc, subnet=subnet, - extra_regions=extra_regions, ) def get_backend(self, db_backend: DBBackend) -> GCPBackend: config_data = json.loads(db_backend.config) auth_data = json.loads(db_backend.auth) project_id = config_data.get("project") - # Legacy config_data does not store project - if project_id is None: - self._auth(auth_data) - project_id = self.project_id + regions = config_data.get("regions") + if regions is None: + # old regions format + regions = config_data.get("extra_regions", []) + [config_data.get("region")] config = GCPConfig( project_id=project_id, - region=config_data["region"], - zone=config_data["zone"], bucket_name=config_data["bucket_name"], + regions=regions, vpc=config_data["vpc"], subnet=config_data["subnet"], - extra_regions=config_data.get("extra_regions", []), credentials=auth_data, ) return GCPBackend(config) - def _get_hub_geographic_area_element(self, selected: Optional[str]) -> BackendElement: - area_names = sorted([l["name"] for l in GCP_LOCATIONS]) - if selected is None: - selected = DEFAULT_GEOGRAPHIC_AREA - if selected not in area_names: - raise BackendConfigError(f"Invalid GCP area {selected}") - element = BackendElement(selected=selected) - for area_name in area_names: - element.values.append(BackendElementValue(value=area_name, label=area_name)) - return element - - def _get_hub_region_element( - self, location: Dict, selected: Optional[str] - ) -> Tuple[BackendElement, Dict]: - regions_client = compute_v1.RegionsClient(credentials=self.credentials) - regions = regions_client.list(project=self.project_id) - region_names = sorted( - [r.name for r in regions if r.name in location["regions"]], - key=lambda name: (name != location["default_region"], name), - ) - if selected is None: - selected = region_names[0] - if selected not in region_names: - raise BackendConfigError(f"Invalid GCP region {selected} in area {location['name']}") - element = BackendElement(selected=selected) - for region_name in region_names: - element.values.append(BackendElementValue(value=region_name, label=region_name)) - return element, {r.name: r for r in regions} - - def _get_location(self, area: str) -> Optional[Dict]: - for location in GCP_LOCATIONS: - if location["name"] == area: - return location - return None - - def _get_hub_zone_element( - self, location: Dict, region: compute_v1.Region, selected: Optional[str] - ) -> BackendElement: - zone_names = sorted( - [gcp_utils.get_resource_name(z) for z in region.zones], - key=lambda name: (name != location["default_zone"], name), - ) - if selected is None: - selected = zone_names[0] - if selected not in zone_names: - raise BackendConfigError(f"Invalid GCP zone {selected} in region {region.name}") - element = BackendElement(selected=selected) - for zone_name in zone_names: - element.values.append(BackendElementValue(value=zone_name, label=zone_name)) - return element - - def _get_hub_buckets_element( - self, region: str, selected: Optional[str] = None - ) -> BackendElement: + def _get_hub_buckets_element(self, selected: Optional[str] = None) -> BackendElement: storage_client = storage.Client(credentials=self.credentials) buckets = storage_client.list_buckets() - bucket_names = [bucket.name for bucket in buckets if bucket.location.lower() == region] - if selected is not None and selected not in bucket_names: - raise BackendConfigError( - f"Invalid bucket {selected} for region {region}", - code="invalid_bucket", - fields=[["bucket_name"]], - ) + bucket_names = [bucket.name for bucket in buckets] element = BackendElement(selected=selected) for bucket_name in bucket_names: element.values.append(BackendElementValue(value=bucket_name, label=bucket_name)) return element + def _get_hub_regions_element( + self, + selected: List[str], + ) -> BackendMultiElement: + element = BackendMultiElement(selected=selected) + for region_name in REGIONS: + element.values.append(BackendElementValue(value=region_name, label=region_name)) + return element + def _get_hub_vpc_subnet_element( self, region: str, @@ -350,20 +275,6 @@ def _get_hub_vpc_subnet_element( ) return element - def _get_hub_extra_regions_element( - self, - region: str, - region_names: List[str], - selected_regions: List[str], - ) -> BackendMultiElement: - element = BackendMultiElement() - for region_name in region_names: - if region_name == region: - continue - element.values.append(BackendElementValue(value=region_name, label=region_name)) - element.selected = selected_regions - return element - def _auth(self, credentials_data: Dict): if credentials_data["type"] == "service_account": service_account_info = json.loads(credentials_data["data"]) From 1fd4edfa21a77fbbc51075219bb9f9de7d7ef360 Mon Sep 17 00:00:00 2001 From: Victor Skvortsov Date: Thu, 17 Aug 2023 15:00:08 +0500 Subject: [PATCH 3/8] Drop storage region for Azure --- cli/dstack/_internal/backend/azure/compute.py | 2 +- cli/dstack/_internal/backend/azure/config.py | 7 ++- cli/dstack/_internal/backend/gcp/compute.py | 16 ++----- cli/dstack/_internal/hub/schemas/__init__.py | 9 ++-- .../services/backends/azure/configurator.py | 48 ++++++------------- 5 files changed, 25 insertions(+), 57 deletions(-) diff --git a/cli/dstack/_internal/backend/azure/compute.py b/cli/dstack/_internal/backend/azure/compute.py index dda6f34c2..b166d8917 100644 --- a/cli/dstack/_internal/backend/azure/compute.py +++ b/cli/dstack/_internal/backend/azure/compute.py @@ -110,7 +110,7 @@ def get_instance_type(self, job: Job) -> Optional[InstanceType]: def get_supported_instances(self) -> List[InstanceType]: instances = {} - for location in [self.azure_config.location, *self.azure_config.extra_locations]: + for location in self.azure_config.locations: for i in _get_instance_types(client=self._compute_client, location=location): if i.instance_name not in instances: instances[i.instance_name] = i diff --git a/cli/dstack/_internal/backend/azure/config.py b/cli/dstack/_internal/backend/azure/config.py index 24e88aee9..0b9b6871a 100644 --- a/cli/dstack/_internal/backend/azure/config.py +++ b/cli/dstack/_internal/backend/azure/config.py @@ -10,13 +10,12 @@ class AzureConfig(BackendConfig, BaseModel): backend: Literal["azure"] = "azure" tenant_id: str subscription_id: str - location: str resource_group: str storage_account: str vault_url: str - extra_locations: List[str] - # network and subnet are location-dependent. - # Hub selects them dynamically when provisioning. + locations: List[str] + # set dynamically + location: Optional[str] network: Optional[str] subnet: Optional[str] credentials: Optional[Dict] = None diff --git a/cli/dstack/_internal/backend/gcp/compute.py b/cli/dstack/_internal/backend/gcp/compute.py index d8143c0f4..203517e07 100644 --- a/cli/dstack/_internal/backend/gcp/compute.py +++ b/cli/dstack/_internal/backend/gcp/compute.py @@ -118,9 +118,7 @@ def get_supported_instances(self) -> List[InstanceType]: zones = _get_zones( regions_client=self.regions_client, project_id=self.gcp_config.project_id, - primary_region=self.gcp_config.region, - primary_zone=self.gcp_config.zone, - extra_regions=self.gcp_config.extra_regions, + regions=self.gcp_config.regions, ) for zone in zones: instance_types = _list_instance_types( @@ -646,18 +644,10 @@ def _run_instance( def _get_zones( regions_client: compute_v1.RegionsClient, project_id: str, - primary_region: str, - primary_zone: str, - extra_regions: List[str], + regions: List[str], ) -> List[str]: regions = regions_client.list(project=project_id) - region_name_to_zones_map = { - r.name: [gcp_utils.get_resource_name(z) for z in r.zones] for r in regions - } - zones = region_name_to_zones_map[primary_region] - zones = sorted(zones, key=lambda x: x != primary_zone) - for extra_region in extra_regions: - zones += region_name_to_zones_map[extra_region] + zones = [gcp_utils.get_resource_name(z) for r in regions for z in r.zones] return zones diff --git a/cli/dstack/_internal/hub/schemas/__init__.py b/cli/dstack/_internal/hub/schemas/__init__.py index 30ee868c7..81465b52a 100644 --- a/cli/dstack/_internal/hub/schemas/__init__.py +++ b/cli/dstack/_internal/hub/schemas/__init__.py @@ -123,9 +123,8 @@ class AzureBackendConfigPartial(BaseModel): type: Literal["azure"] = "azure" tenant_id: Optional[str] subscription_id: Optional[str] - location: Optional[str] storage_account: Optional[str] - extra_locations: Optional[List[str]] + locations: Optional[List[str]] class AzureBackendClientCreds(BaseModel): @@ -152,9 +151,8 @@ class AzureBackendConfig(BaseModel): type: Literal["azure"] = "azure" tenant_id: str subscription_id: str - location: str storage_account: str - extra_locations: List[str] = [] + locations: List[str] class AzureBackendConfigWithCreds(AzureBackendConfig): @@ -293,9 +291,8 @@ class AzureBackendValues(BaseModel): default_credentials: bool = False tenant_id: Optional[BackendElement] subscription_id: Optional[BackendElement] - location: Optional[BackendElement] storage_account: Optional[BackendElement] - extra_locations: Optional[BackendMultiElement] + locations: Optional[BackendMultiElement] class AWSStorageBackendValues(BaseModel): diff --git a/cli/dstack/_internal/hub/services/backends/azure/configurator.py b/cli/dstack/_internal/hub/services/backends/azure/configurator.py index 20993ab02..98dfcb8fa 100644 --- a/cli/dstack/_internal/hub/services/backends/azure/configurator.py +++ b/cli/dstack/_internal/hub/services/backends/azure/configurator.py @@ -84,6 +84,7 @@ ("(South America) Brazil South", "brazilsouth"), ] LOCATION_VALUES = [l[1] for l in LOCATIONS] +DEFAULT_LOCATION = "eastus" class AzureConfigurator(Configurator): @@ -137,16 +138,14 @@ def configure_backend( self.subscription_id = backend_values.subscription_id.selected if self.subscription_id is None: return backend_values - backend_values.location = self._get_location_element(selected=backend_config.location) self.location = backend_values.location.selected if self.location is None: return backend_values backend_values.storage_account = self._get_storage_account_element( selected=backend_config.storage_account ) - backend_values.extra_locations = self._get_extra_locations_element( - location=backend_config.location, - selected_locations=backend_config.extra_locations or [], + backend_values.locations = self._get_locations_element( + selected=backend_config.extra_locations or [DEFAULT_LOCATION], ) return backend_values @@ -191,38 +190,40 @@ def get_backend_config( json_config = json.loads(db_backend.config) tenant_id = json_config["tenant_id"] subscription_id = json_config["subscription_id"] - location = json_config["location"] storage_account = json_config["storage_account"] - extra_locations = json_config.get("extra_locations", []) + if locations is None: + # old location format + locations = json_config.get("extra_locations", []) + [json_config.get("location")] if include_creds: auth_config = json.loads(db_backend.auth) return AzureBackendConfigWithCreds( tenant_id=tenant_id, subscription_id=subscription_id, - location=location, storage_account=storage_account, - extra_locations=extra_locations, + locations=locations, credentials=AzureBackendCreds.parse_obj(auth_config), ) return AzureBackendConfig( tenant_id=tenant_id, subscription_id=subscription_id, - location=location, storage_account=storage_account, - extra_locations=extra_locations, + locations=locations, ) def get_backend(self, db_backend: DBBackend) -> AzureBackend: config_data = json.loads(db_backend.config) auth_data = json.loads(db_backend.auth) + locations = config_data.get("locations") + if locations is None: + # old location format + locations = config_data.get("extra_locations", []) + [config_data.get("location")] config = AzureConfig( tenant_id=config_data["tenant_id"], subscription_id=config_data["subscription_id"], - location=config_data["location"], resource_group=config_data["resource_group"], storage_account=config_data["storage_account"], vault_url=config_data["vault_url"], - extra_locations=config_data.get("extra_locations", []), + locations=locations, credentials=auth_data, ) return AzureBackend(config) @@ -279,21 +280,6 @@ def _get_subscription_id_element(self, selected: Optional[str]) -> BackendElemen ) return element - def _get_location_element(self, selected: Optional[str]) -> BackendElement: - if selected is not None and selected not in LOCATION_VALUES: - raise BackendConfigError( - "Invalid location", code="invalid_location", fields=[["location"]] - ) - element = BackendElement(selected=selected) - for l in LOCATIONS: - element.values.append( - BackendElementValue( - value=l[1], - label=f"{l[0]} [{l[1]}]", - ) - ) - return element - def _get_storage_account_element(self, selected: Optional[str]) -> BackendElement: client = StorageManagementClient( credential=self.credential, subscription_id=self.subscription_id @@ -314,15 +300,11 @@ def _get_storage_account_element(self, selected: Optional[str]) -> BackendElemen element.selected = storage_accounts[0] return element - def _get_extra_locations_element( - self, location: str, selected_locations: List[str] - ) -> BackendMultiElement: + def _get_locations_element(self, selected: List[str]) -> BackendMultiElement: element = BackendMultiElement() for l in LOCATION_VALUES: - if l == location: - continue element.values.append(BackendElementValue(value=l, label=l)) - element.selected = selected_locations + element.selected = selected return element def _get_resource_group(self) -> str: From 67a52b485ba426f3f3bfd10084050fc021bee64c Mon Sep 17 00:00:00 2001 From: Victor Skvortsov Date: Thu, 17 Aug 2023 15:22:41 +0500 Subject: [PATCH 4/8] Fixes --- cli/dstack/_internal/backend/gcp/compute.py | 11 ++++++++--- .../hub/services/backends/azure/configurator.py | 1 + 2 files changed, 9 insertions(+), 3 deletions(-) diff --git a/cli/dstack/_internal/backend/gcp/compute.py b/cli/dstack/_internal/backend/gcp/compute.py index d46532661..85df215b5 100644 --- a/cli/dstack/_internal/backend/gcp/compute.py +++ b/cli/dstack/_internal/backend/gcp/compute.py @@ -121,7 +121,7 @@ def get_supported_instances(self) -> List[InstanceType]: zones = _get_zones( regions_client=self.regions_client, project_id=self.gcp_config.project_id, - regions=self.gcp_config.regions, + configured_regions=self.gcp_config.regions, ) for zone in zones: instance_types = _list_instance_types( @@ -648,10 +648,15 @@ def _run_instance( def _get_zones( regions_client: compute_v1.RegionsClient, project_id: str, - regions: List[str], + configured_regions: List[str], ) -> List[str]: regions = regions_client.list(project=project_id) - zones = [gcp_utils.get_resource_name(z) for r in regions for z in r.zones] + zones = [ + gcp_utils.get_resource_name(z) + for r in regions + for z in r.zones + if r.name in configured_regions + ] return zones diff --git a/cli/dstack/_internal/hub/services/backends/azure/configurator.py b/cli/dstack/_internal/hub/services/backends/azure/configurator.py index 98dfcb8fa..288eda7df 100644 --- a/cli/dstack/_internal/hub/services/backends/azure/configurator.py +++ b/cli/dstack/_internal/hub/services/backends/azure/configurator.py @@ -191,6 +191,7 @@ def get_backend_config( tenant_id = json_config["tenant_id"] subscription_id = json_config["subscription_id"] storage_account = json_config["storage_account"] + locations = json_config.get("locations") if locations is None: # old location format locations = json_config.get("extra_locations", []) + [json_config.get("location")] From 0be47705b37d433adbd6dea0c725a37822cb67c2 Mon Sep 17 00:00:00 2001 From: Oleg Vavilov Date: Thu, 17 Aug 2023 22:16:49 +0300 Subject: [PATCH 5/8] #656 Refactoring regions fields, replaced columns in backend table --- hub/src/locale/en.json | 46 +++---- .../Project/Backends/Form/AWS/constants.tsx | 11 +- .../pages/Project/Backends/Form/AWS/index.tsx | 53 +++----- .../Project/Backends/Form/Azure/constants.tsx | 23 +--- .../Project/Backends/Form/Azure/index.tsx | 64 +++------ .../Project/Backends/Form/GCP/constants.tsx | 50 +------ .../pages/Project/Backends/Form/GCP/index.tsx | 123 +++--------------- .../Table/hooks/useColumnsDefinitions.tsx | 99 +++++--------- .../Project/Backends/Table/styles.module.scss | 2 +- hub/src/types/backend.d.ts | 41 ++---- 10 files changed, 126 insertions(+), 386 deletions(-) diff --git a/hub/src/locale/en.json b/hub/src/locale/en.json index c0a13234b..e37ec81e3 100644 --- a/hub/src/locale/en.json +++ b/hub/src/locale/en.json @@ -65,10 +65,8 @@ }, "table": { - "region": "Region/Location", - "bucket": "Bucket", - "extra_regions": "Extra Region/Extra Locations", - "subnet": "Subnet" + "region": "Region", + "bucket": "Storage" }, "edit": { @@ -124,17 +122,14 @@ "secret_key": "Secret key", "secret_key_id": "Secret access key", "secret_key_id_description": "Specify the AWS secret access key", - "region_name": "Region", - "region_name_description": "Select a region to run workflows and store artifacts", - "region_name_placeholder": "Not selected", + "regions": "Regions", + "regions_description": "Select regions to run workflows and store artifacts", + "regions_placeholder": "Not selected", "s3_bucket_name": "Bucket", "s3_bucket_name_description": "Select an S3 bucket to store artifacts", "ec2_subnet_id": "Subnet", "ec2_subnet_id_description": "Select a subnet to run workflows in", - "ec2_subnet_id_placeholder": "Not selected", - "extra_regions": "Additional regions", - "extra_regions_description": "Select additional regions to run workflows", - "extra_regions_placeholder": "Select regions" + "ec2_subnet_id_placeholder": "Not selected" }, "azure" : { "authorization": "Authorization", @@ -150,15 +145,13 @@ "subscription_id": "Subscription ID", "subscription_id_description": "Select an Azure subscription ID", "subscription_id_placeholder": "Not selected", - "location": "Location", - "location_description": "Select an Azure location to run workflows and store artifacts", - "location_placeholder": "Not selected", + "locations": "Locations", + "locations_description": "Select locations to run workflows", + "locations_placeholder": "Select locations", "storage_account": "Storage account", "storage_account_description": "Select an Azure storage account to store artifacts", - "storage_account_placeholder": "Not selected", - "extra_locations": "Additional locations", - "extra_locations_description": "Select additional locations to run workflows", - "extra_locations_placeholder": "Select locations" + "storage_account_placeholder": "Not selected" + }, "gcp": { "authorization": "Authorization", @@ -166,15 +159,9 @@ "service_account": "Service account key", "credentials_description": "Credentials description", "credentials_placeholder": "Credentials placeholder", - "area": "Location", - "area_description": "Select a location to run workflows and store artifacts", - "area_placeholder": "Not selected", - "region": "Region", - "region_description": "Select a region to run workflows and store artifacts", - "region_placeholder": "Not selected", - "zone": "Zone", - "zone_description": "Select a zone to run workflows and store artifacts", - "zone_placeholder": "Not selected", + "regions": "Regions", + "regions_description": "Select regions to run workflows and store artifacts", + "regions_placeholder": "Not selected", "bucket_name": "Bucket", "bucket_name_description": "Select a storage bucket to store artifacts", "bucket_name_placeholder": "Not selected", @@ -183,10 +170,7 @@ "vpc_placeholder": "VPC placeholder", "subnet": "Subnet", "subnet_description": "Select a subnet to run workflows in", - "subnet_placeholder": "Not selected", - "extra_regions": "Additional regions", - "extra_regions_description": "Select additional regions to run workflows", - "extra_regions_placeholder": "Select regions" + "subnet_placeholder": "Not selected" }, "lambda": { "api_key": "API key", diff --git a/hub/src/pages/Project/Backends/Form/AWS/constants.tsx b/hub/src/pages/Project/Backends/Form/AWS/constants.tsx index e6f35a925..9fa11ddf0 100644 --- a/hub/src/pages/Project/Backends/Form/AWS/constants.tsx +++ b/hub/src/pages/Project/Backends/Form/AWS/constants.tsx @@ -6,10 +6,9 @@ export const FIELD_NAMES = { ACCESS_KEY: 'credentials.access_key', SECRET_KEY: 'credentials.secret_key', }, - REGION_NAME: 'region_name', + REGIONS: 'regions', S3_BUCKET_NAME: 's3_bucket_name', EC2_SUBNET_ID: 'ec2_subnet_id', - EXTRA_REGIONS: 'extra_regions', }; export const CREDENTIALS_HELP = { @@ -40,15 +39,15 @@ export const CREDENTIALS_HELP = { ), }; -export const REGION_HELP = { - header:

Region

, +export const REGIONS_HELP = { + header:

Regions

, body: ( <>

- Select a Region that dstack Hub will use to create resources in your AWS account. + Select Regions that dstack Hub will use to create resources in your AWS account.

- The selected Region will be used to run workflows and store artifacts. + The selected Regions will be used to run workflows and store artifacts.

), diff --git a/hub/src/pages/Project/Backends/Form/AWS/index.tsx b/hub/src/pages/Project/Backends/Form/AWS/index.tsx index d237f3f02..5eabf6824 100644 --- a/hub/src/pages/Project/Backends/Form/AWS/index.tsx +++ b/hub/src/pages/Project/Backends/Form/AWS/index.tsx @@ -21,7 +21,7 @@ import { isRequestFormErrors2, isRequestFormFieldError } from 'libs'; import { useBackendValuesMutation } from 'services/backend'; import { AWSCredentialTypeEnum } from 'types'; -import { ADDITIONAL_REGIONS_HELP, BUCKET_HELP, CREDENTIALS_HELP, FIELD_NAMES, REGION_HELP, SUBNET_HELP } from './constants'; +import { BUCKET_HELP, CREDENTIALS_HELP, FIELD_NAMES, REGIONS_HELP, SUBNET_HELP } from './constants'; import { IProps } from './types'; @@ -32,10 +32,9 @@ export const AWSBackend: React.FC = ({ loading }) => { const [pushNotification] = useNotifications(); const { control, getValues, setValue, setError, clearErrors, watch } = useFormContext(); const [valuesData, setValuesData] = useState(); - const [regions, setRegions] = useState([]); + const [regions, setRegions] = useState([]); const [buckets, setBuckets] = useState([]); const [subnets, setSubnets] = useState([]); - const [extraRegions, setExtraRegions] = useState([]); const [availableDefaultCredentials, setAvailableDefaultCredentials] = useState(null); const lastUpdatedField = useRef(null); const isFirstRender = useRef(true); @@ -93,12 +92,12 @@ export const AWSBackend: React.FC = ({ loading }) => { if (response.default_credentials) changeFormHandler().catch(console.log); } - if (response.region_name?.values) { - setRegions(response.region_name.values); + if (response.regions?.values) { + setRegions(response.regions.values); } - if (response.region_name?.selected !== undefined) { - setValue(FIELD_NAMES.REGION_NAME, response.region_name.selected); + if (response.regions?.selected !== undefined) { + setValue(FIELD_NAMES.REGIONS, response.regions.selected); } if (response.s3_bucket_name?.values) { @@ -116,14 +115,6 @@ export const AWSBackend: React.FC = ({ loading }) => { if (response.ec2_subnet_id?.selected !== undefined) { setValue(FIELD_NAMES.EC2_SUBNET_ID, response.ec2_subnet_id.selected ?? ''); } - - if (response.extra_regions?.values) { - setExtraRegions(response.extra_regions.values); - } - - if (response.extra_regions?.selected !== undefined) { - setValue(FIELD_NAMES.EXTRA_REGIONS, response.extra_regions.selected); - } } catch (errorResponse) { console.log('fetch backends values error:', errorResponse); // eslint-disable-next-line @typescript-eslint/ban-ts-comment @@ -235,18 +226,17 @@ export const AWSBackend: React.FC = ({ loading }) => { )} - openHelpPanel(REGION_HELP)} />} - label={t('projects.edit.aws.region_name')} - description={t('projects.edit.aws.region_name_description')} - placeholder={t('projects.edit.aws.region_name_placeholder')} + openHelpPanel(REGIONS_HELP)} />} + label={t('projects.edit.aws.regions')} + description={t('projects.edit.aws.regions_description')} + placeholder={t('projects.edit.aws.regions_placeholder')} control={control} - name={FIELD_NAMES.REGION_NAME} - disabled={getDisabledByFieldName(FIELD_NAMES.REGION_NAME)} - onChange={getOnChangeSelectField(FIELD_NAMES.REGION_NAME)} - options={regions} - rules={{ required: t('validation.required') }} + name={FIELD_NAMES.REGIONS} + onChange={getOnChangeSelectField(FIELD_NAMES.REGIONS)} + disabled={getDisabledByFieldName(FIELD_NAMES.REGIONS)} secondaryControl={renderSpinner()} + options={regions} /> = ({ loading }) => { options={subnets} secondaryControl={renderSpinner()} /> - - openHelpPanel(ADDITIONAL_REGIONS_HELP)} />} - label={t('projects.edit.aws.extra_regions')} - description={t('projects.edit.aws.extra_regions_description')} - placeholder={t('projects.edit.aws.extra_regions_placeholder')} - control={control} - name={FIELD_NAMES.EXTRA_REGIONS} - onChange={getOnChangeSelectField(FIELD_NAMES.EXTRA_REGIONS)} - disabled={getDisabledByFieldName(FIELD_NAMES.EXTRA_REGIONS)} - secondaryControl={renderSpinner()} - options={extraRegions} - /> ); }; diff --git a/hub/src/pages/Project/Backends/Form/Azure/constants.tsx b/hub/src/pages/Project/Backends/Form/Azure/constants.tsx index ab38ecb46..5fcdcd610 100644 --- a/hub/src/pages/Project/Backends/Form/Azure/constants.tsx +++ b/hub/src/pages/Project/Backends/Form/Azure/constants.tsx @@ -8,9 +8,8 @@ export const FIELD_NAMES = { CLIENT_SECRET: 'credentials.client_secret', }, SUBSCRIPTION_ID: 'subscription_id', - LOCATION: 'location', + LOCATIONS: 'locations', STORAGE_ACCOUNT: 'storage_account', - EXTRA_LOCATIONS: 'extra_locations', }; export const CREDENTIALS_HELP = { @@ -36,15 +35,15 @@ export const SUBSCRIPTION_HELP = { ), }; -export const LOCATION_HELP = { - header:

Location

, +export const LOCATIONS_HELP = { + header:

Locations

, body: ( <>

- Select a Location that dstack Hub will use to create resources in your Azure account. + Select Locations that dstack Hub will use to create resources in your Azure account.

- The selected Location will be used to run workflows and store artifacts. + The selected Locations will be used to run workflows and store artifacts.

), @@ -65,15 +64,3 @@ export const STORAGE_ACCOUNT_HELP = { ), }; - -export const ADDITIONAL_LOCATIONS_HELP = { - header:

Additional locations

, - body: ( - <> -

- dstack will try to provision the instance in additional locations if the primary location has no capacity.{' '} - Specifying additional location increases the chances of provisioning spot instances. -

- - ), -}; diff --git a/hub/src/pages/Project/Backends/Form/Azure/index.tsx b/hub/src/pages/Project/Backends/Form/Azure/index.tsx index 72ce32015..99c78f0e4 100644 --- a/hub/src/pages/Project/Backends/Form/Azure/index.tsx +++ b/hub/src/pages/Project/Backends/Form/Azure/index.tsx @@ -20,20 +20,13 @@ import { isRequestFormErrors2, isRequestFormFieldError } from 'libs'; import { useBackendValuesMutation } from 'services/backend'; import { AzureCredentialTypeEnum } from 'types'; -import { - ADDITIONAL_LOCATIONS_HELP, - CREDENTIALS_HELP, - FIELD_NAMES, - LOCATION_HELP, - STORAGE_ACCOUNT_HELP, - SUBSCRIPTION_HELP, -} from './constants'; +import { CREDENTIALS_HELP, FIELD_NAMES, LOCATIONS_HELP, STORAGE_ACCOUNT_HELP, SUBSCRIPTION_HELP } from './constants'; import { IProps } from './types'; import styles from './styles.module.scss'; -const FIELDS_QUEUE = [FIELD_NAMES.SUBSCRIPTION_ID, FIELD_NAMES.LOCATION, FIELD_NAMES.STORAGE_ACCOUNT]; +const FIELDS_QUEUE = [FIELD_NAMES.SUBSCRIPTION_ID, FIELD_NAMES.LOCATIONS, FIELD_NAMES.STORAGE_ACCOUNT]; export const AzureBackend: React.FC = ({ loading }) => { const { t } = useTranslation(); @@ -42,9 +35,8 @@ export const AzureBackend: React.FC = ({ loading }) => { const [valuesData, setValuesData] = useState(); const [subscriptionIds, setSubscriptionIds] = useState([]); const [tenantIds, setTenantIds] = useState([]); - const [locations, setLocations] = useState([]); + const [locations, setLocations] = useState([]); const [storageAccounts, setStorageAccounts] = useState([]); - const [extraLocations, setExtraLocations] = useState([]); const [availableDefaultCredentials, setAvailableDefaultCredentials] = useState(null); const lastUpdatedField = useRef(null); const isFirstRender = useRef(true); @@ -114,11 +106,11 @@ export const AzureBackend: React.FC = ({ loading }) => { if (response.subscription_id?.selected !== undefined) { setValue(FIELD_NAMES.SUBSCRIPTION_ID, response.subscription_id.selected); } - if (response.location?.values) { - setLocations(response.location.values); + if (response.locations?.values) { + setLocations(response.locations.values); } - if (response.location?.selected !== undefined) { - setValue(FIELD_NAMES.LOCATION, response.location.selected); + if (response.locations?.selected !== undefined) { + setValue(FIELD_NAMES.LOCATIONS, response.locations.selected); } if (response.storage_account?.values) { setStorageAccounts(response.storage_account.values); @@ -126,14 +118,6 @@ export const AzureBackend: React.FC = ({ loading }) => { if (response.storage_account?.selected !== undefined) { setValue(FIELD_NAMES.STORAGE_ACCOUNT, response.storage_account.selected); } - - if (response.extra_locations?.values) { - setExtraLocations(response.extra_locations.values); - } - - if (response.extra_locations?.selected !== undefined) { - setValue(FIELD_NAMES.EXTRA_LOCATIONS, response.extra_locations.selected); - } } catch (errorResponse) { console.log('fetch backends values error:', errorResponse); // eslint-disable-next-line @typescript-eslint/ban-ts-comment @@ -218,7 +202,7 @@ export const AzureBackend: React.FC = ({ loading }) => { case FIELD_NAMES.SUBSCRIPTION_ID: disabledField = disabledField || !subscriptionIds.length; break; - case FIELD_NAMES.LOCATION: + case FIELD_NAMES.LOCATIONS: disabledField = disabledField || !locations.length; break; case FIELD_NAMES.STORAGE_ACCOUNT: @@ -329,18 +313,17 @@ export const AzureBackend: React.FC = ({ loading }) => { secondaryControl={renderSpinner()} /> - openHelpPanel(LOCATION_HELP)} />} - label={t('projects.edit.azure.location')} - description={t('projects.edit.azure.location_description')} - placeholder={t('projects.edit.azure.location_placeholder')} + openHelpPanel(LOCATIONS_HELP)} />} + label={t('projects.edit.azure.locations')} + description={t('projects.edit.azure.locations_description')} + placeholder={t('projects.edit.azure.locations_placeholder')} control={control} - name={FIELD_NAMES.LOCATION} - disabled={getDisabledByFieldName(FIELD_NAMES.LOCATION)} - onChange={getOnChangeSelectField(FIELD_NAMES.LOCATION)} - options={locations} - rules={{ required: t('validation.required') }} + name={FIELD_NAMES.LOCATIONS} + onChange={getOnChangeSelectField(FIELD_NAMES.LOCATIONS)} + disabled={getDisabledByFieldName(FIELD_NAMES.LOCATIONS)} secondaryControl={renderSpinner()} + options={locations} /> = ({ loading }) => { options={storageAccounts} secondaryControl={renderSpinner()} /> - - openHelpPanel(ADDITIONAL_LOCATIONS_HELP)} />} - label={t('projects.edit.azure.extra_locations')} - description={t('projects.edit.azure.extra_locations_description')} - placeholder={t('projects.edit.azure.extra_locations_placeholder')} - control={control} - name={FIELD_NAMES.EXTRA_LOCATIONS} - onChange={getOnChangeSelectField(FIELD_NAMES.EXTRA_LOCATIONS)} - disabled={getDisabledByFieldName(FIELD_NAMES.EXTRA_LOCATIONS)} - secondaryControl={renderSpinner()} - options={extraLocations} - /> ); }; diff --git a/hub/src/pages/Project/Backends/Form/GCP/constants.tsx b/hub/src/pages/Project/Backends/Form/GCP/constants.tsx index 7528867f9..fcecde0fd 100644 --- a/hub/src/pages/Project/Backends/Form/GCP/constants.tsx +++ b/hub/src/pages/Project/Backends/Form/GCP/constants.tsx @@ -6,14 +6,11 @@ export const FIELD_NAMES = { FILENAME: 'credentials.filename', DATA: 'credentials.data', }, - AREA: 'area', - REGION: 'region', - ZONE: 'zone', + REGIONS: 'regions', BUCKET_NAME: 'bucket_name', VPC_SUBNET: 'vpc_subnet', VPC: 'vpc', SUBNET: 'subnet', - EXTRA_REGIONS: 'extra_regions', }; export const SERVICE_ACCOUNT_HELP = { @@ -59,40 +56,15 @@ export const SERVICE_ACCOUNT_HELP = { ), }; -export const AREA_HELP = { - header:

Location

, +export const REGIONS_HELP = { + header:

Regions

, body: ( <>

- Select a Location to see the available Regions and Zones. + Select Regions that dstack Hub will use to create resources in your GCP account.

- - ), -}; - -export const REGION_HELP = { - header:

Region

, - body: ( - <> -

- Select a Region that dstack Hub will use to create resources in your GCP account. -

-

- The selected Region will be used to run workflows and store artifacts. -

- - ), -}; - -export const ZONE_HELP = { - header:

Zone

, - body: ( - <>

- Select a Zone that dstack Hub will use to create resources in your GCP account. -

-

- The selected Zone will be used to run workflows and store artifacts. + The selected Regions will be used to run workflows and store artifacts.

), @@ -125,15 +97,3 @@ export const SUBNET_HELP = { ), }; - -export const ADDITIONAL_REGIONS_HELP = { - header:

Additional regions

, - body: ( - <> -

- dstack will try to provision the instance in additional regions if the primary region has no capacity.{' '} - Specifying additional regions increases the chances of provisioning spot instances. -

- - ), -}; diff --git a/hub/src/pages/Project/Backends/Form/GCP/index.tsx b/hub/src/pages/Project/Backends/Form/GCP/index.tsx index fef6eb0cd..fbd14e396 100644 --- a/hub/src/pages/Project/Backends/Form/GCP/index.tsx +++ b/hub/src/pages/Project/Backends/Form/GCP/index.tsx @@ -8,7 +8,6 @@ import { FormMultiselectOptions, FormS3BucketSelector, FormSelect, - FormSelectOptions, InfoLink, SpaceBetween, Spinner, @@ -20,16 +19,7 @@ import { isRequestFormErrors2, isRequestFormFieldError } from 'libs'; import { useBackendValuesMutation } from 'services/backend'; import { GCPCredentialTypeEnum } from 'types'; -import { - ADDITIONAL_REGIONS_HELP, - AREA_HELP, - BUCKET_HELP, - FIELD_NAMES, - REGION_HELP, - SERVICE_ACCOUNT_HELP, - SUBNET_HELP, - ZONE_HELP, -} from './constants'; +import { BUCKET_HELP, FIELD_NAMES, REGIONS_HELP, SERVICE_ACCOUNT_HELP, SUBNET_HELP } from './constants'; import { IProps, VPCSubnetOption } from './types'; @@ -38,11 +28,8 @@ import styles from './styles.module.scss'; const FIELDS_QUEUE = [ FIELD_NAMES.CREDENTIALS.TYPE, FIELD_NAMES.CREDENTIALS.DATA, - FIELD_NAMES.AREA, - FIELD_NAMES.REGION, - FIELD_NAMES.ZONE, + FIELD_NAMES.REGIONS, FIELD_NAMES.BUCKET_NAME, - FIELD_NAMES.VPC_SUBNET, FIELD_NAMES.VPC, FIELD_NAMES.SUBNET, ]; @@ -59,12 +46,9 @@ export const GCPBackend: React.FC = ({ loading }) => { } = useFormContext(); const [files, setFiles] = useState([]); const [valuesData, setValuesData] = useState(); - const [areaOptions, setAreaOptions] = useState([]); - const [regionOptions, setRegionOptions] = useState([]); - const [zoneOptions, setZoneOptions] = useState([]); + const [regionsOptions, setRegionsOptions] = useState([]); const [bucketNameOptions, setBucketNameOptions] = useState([]); const [subnetOptions, setSubnetOptions] = useState([]); - const [extraRegions, setExtraRegions] = useState([]); const [availableDefaultCredentials, setAvailableDefaultCredentials] = useState(null); const requestRef = useRef>(null); const [pushNotification] = useNotifications(); @@ -116,28 +100,12 @@ export const GCPBackend: React.FC = ({ loading }) => { if (response.default_credentials) changeFormHandler().catch(console.log); } - if (response.area?.values) { - setAreaOptions(response.area.values); - } - - if (response.area?.selected !== undefined) { - setValue(FIELD_NAMES.AREA, response.area.selected); - } - - if (response.region?.values) { - setRegionOptions(response.region.values); - } - - if (response.region?.selected !== undefined) { - setValue(FIELD_NAMES.REGION, response.region.selected); - } - - if (response.zone?.values) { - setZoneOptions(response.zone.values); + if (response.regions?.values) { + setRegionsOptions(response.regions.values); } - if (response.zone?.selected !== undefined) { - setValue(FIELD_NAMES.ZONE, response.zone.selected); + if (response.regions?.selected !== undefined) { + setValue(FIELD_NAMES.REGIONS, response.regions.selected); } if (response.bucket_name?.values) { @@ -174,14 +142,6 @@ export const GCPBackend: React.FC = ({ loading }) => { subnet: valueItem.subnet, }); } - - if (response.extra_regions?.values) { - setExtraRegions(response.extra_regions.values); - } - - if (response.extra_regions?.selected !== undefined) { - setValue(FIELD_NAMES.EXTRA_REGIONS, response.extra_regions.selected); - } } catch (errorResponse) { console.log('fetch backends values error:', errorResponse); // eslint-disable-next-line @typescript-eslint/ban-ts-comment @@ -271,14 +231,8 @@ export const GCPBackend: React.FC = ({ loading }) => { disabledField = disabledField || (lastUpdatedField.current !== fieldName && isLoadingValues); switch (fieldName) { - case FIELD_NAMES.AREA: - disabledField = disabledField || !areaOptions.length; - break; - case FIELD_NAMES.REGION: - disabledField = disabledField || !regionOptions.length; - break; - case FIELD_NAMES.ZONE: - disabledField = disabledField || !zoneOptions.length; + case FIELD_NAMES.REGIONS: + disabledField = disabledField || !regionsOptions.length; break; case FIELD_NAMES.VPC_SUBNET: disabledField = disabledField || !subnetOptions.length; @@ -357,44 +311,16 @@ export const GCPBackend: React.FC = ({ loading }) => { /> )} - openHelpPanel(AREA_HELP)} />} - label={t('projects.edit.gcp.area')} - description={t('projects.edit.gcp.area_description')} - placeholder={t('projects.edit.gcp.area_placeholder')} - control={control} - name={FIELD_NAMES.AREA} - options={areaOptions} - onChange={getOnChangeSelectFormField(FIELD_NAMES.AREA)} - disabled={getDisabledByFieldName(FIELD_NAMES.AREA)} - rules={{ required: t('validation.required') }} - secondaryControl={renderSpinner()} - /> - - openHelpPanel(REGION_HELP)} />} - label={t('projects.edit.gcp.region')} - description={t('projects.edit.gcp.region_description')} - placeholder={t('projects.edit.gcp.region_placeholder')} - control={control} - name={FIELD_NAMES.REGION} - options={regionOptions} - onChange={getOnChangeSelectFormField(FIELD_NAMES.REGION)} - disabled={getDisabledByFieldName(FIELD_NAMES.REGION)} - rules={{ required: t('validation.required') }} - secondaryControl={renderSpinner()} - /> - - openHelpPanel(ZONE_HELP)} />} - label={t('projects.edit.gcp.zone')} - description={t('projects.edit.gcp.zone_description')} - placeholder={t('projects.edit.gcp.zone_placeholder')} + openHelpPanel(REGIONS_HELP)} />} + label={t('projects.edit.gcp.regions')} + description={t('projects.edit.gcp.regions_description')} + placeholder={t('projects.edit.gcp.regions_placeholder')} control={control} - name={FIELD_NAMES.ZONE} - options={zoneOptions} - onChange={getOnChangeSelectFormField(FIELD_NAMES.ZONE)} - disabled={getDisabledByFieldName(FIELD_NAMES.ZONE)} + name={FIELD_NAMES.REGIONS} + options={regionsOptions} + onChange={getOnChangeSelectFormField(FIELD_NAMES.REGIONS)} + disabled={getDisabledByFieldName(FIELD_NAMES.REGIONS)} rules={{ required: t('validation.required') }} secondaryControl={renderSpinner()} /> @@ -431,19 +357,6 @@ export const GCPBackend: React.FC = ({ loading }) => { rules={{ required: t('validation.required') }} secondaryControl={renderSpinner()} /> - - openHelpPanel(ADDITIONAL_REGIONS_HELP)} />} - label={t('projects.edit.gcp.extra_regions')} - description={t('projects.edit.gcp.extra_regions_description')} - placeholder={t('projects.edit.gcp.extra_regions_placeholder')} - control={control} - name={FIELD_NAMES.EXTRA_REGIONS} - onChange={getOnChangeSelectFormField(FIELD_NAMES.EXTRA_REGIONS)} - disabled={getDisabledByFieldName(FIELD_NAMES.EXTRA_REGIONS)} - secondaryControl={renderSpinner()} - options={extraRegions} - /> ); diff --git a/hub/src/pages/Project/Backends/Table/hooks/useColumnsDefinitions.tsx b/hub/src/pages/Project/Backends/Table/hooks/useColumnsDefinitions.tsx index bd0ee9aae..6785682c7 100644 --- a/hub/src/pages/Project/Backends/Table/hooks/useColumnsDefinitions.tsx +++ b/hub/src/pages/Project/Backends/Table/hooks/useColumnsDefinitions.tsx @@ -19,81 +19,59 @@ export const useColumnsDefinitions = ({ loading, onDeleteClick, onEditClick }: h const getRegionByBackendType = (backend: IProjectBackend) => { switch (backend.config.type) { - case BackendTypesEnum.AWS: - return backend.config.region_name_title; - case BackendTypesEnum.AZURE: - return backend.config.location; - case BackendTypesEnum.GCP: - return backend.config.region; - case BackendTypesEnum.LAMBDA: { - const regions = backend.config?.regions.join(', ') ?? '-'; + case BackendTypesEnum.AWS: { + const regions = backend.config.regions?.join(', '); + return (
{regions}
); } - default: - return '-'; - } - }; - - const getBucketByBackendType = (backend: IProjectBackend) => { - switch (backend.config.type) { - case BackendTypesEnum.AWS: - return backend.config.s3_bucket_name; - case BackendTypesEnum.GCP: - return backend.config.bucket_name; - case BackendTypesEnum.LAMBDA: - return backend.config.storage_backend.bucket_name; - default: - return '-'; - } - }; - const getSubnetByBackendType = (backend: IProjectBackend) => { - switch (backend.config.type) { - case BackendTypesEnum.AWS: - return backend.config.ec2_subnet_id; - case BackendTypesEnum.GCP: - return backend.config.subnet; - default: - return '-'; - } - }; - - const getExtraRegionsByBackendType = (backend: IProjectBackend) => { - switch (backend.config.type) { - case BackendTypesEnum.AWS: { - const extraRegions = backend.config.extra_regions?.join(', '); + case BackendTypesEnum.AZURE: { + const locations = backend.config.locations?.join(', '); return ( -
- {extraRegions} +
+ {locations}
); } - case BackendTypesEnum.AZURE: { - const extraLocations = backend.config.extra_locations?.join(', '); + case BackendTypesEnum.GCP: { + const regions = backend.config.regions?.join(', '); return ( -
- {extraLocations} +
+ {regions}
); } - case BackendTypesEnum.GCP: { - const extraRegions = backend.config.extra_regions?.join(', '); - + case BackendTypesEnum.LAMBDA: { + const regions = backend.config?.regions.join(', ') ?? '-'; return ( -
- {extraRegions} +
+ {regions}
); } + default: + return '-'; + } + }; + const getBucketByBackendType = (backend: IProjectBackend) => { + switch (backend.config.type) { + case BackendTypesEnum.AWS: + return backend.config.s3_bucket_name; + case BackendTypesEnum.AZURE: + return backend.config.storage_account; + case BackendTypesEnum.GCP: + return backend.config.bucket_name; + case BackendTypesEnum.LAMBDA: + return backend.config.storage_backend.bucket_name; default: return '-'; } @@ -116,26 +94,11 @@ export const useColumnsDefinitions = ({ loading, onDeleteClick, onEditClick }: h { id: 'bucket', header: t('backend.table.bucket'), - cell: getBucketByBackendType, - }, - { - id: 'subnet', - header: t('backend.table.subnet'), - cell: getSubnetByBackendType, - }, - - { - id: 'extra_regions', - header: t('backend.table.extra_regions'), - cell: getExtraRegionsByBackendType, - }, - - { - id: 'actions', - header: '', cell: (backend: IProjectBackend) => (
+
{getBucketByBackendType(backend)}
+
{onEditClick && (