[Feature Request] Update supported cross-region inference #741
Labels
enhancement
New feature or request
Comments
|
just saw current inference profiles by AWS CLI. USus-east-2 us-east-1 ( Found 2 additional models ) EUAPAC |
|
They look complicated. Supported cross-region inferences are different by each regions. index.tsimport {
BedrockClient,
type InferenceProfileSummary,
ListInferenceProfilesCommand,
} from "@aws-sdk/client-bedrock";
const AWS_PROFILE = process.env["AWS_PROFILE"];
// Ref: https://github.com/aws-samples/bedrock-claude-chat/blob/a890a5f269f3c00e341c728b690e3f4d6d5395e8/backend/app/bedrock.py#L270-L286
const supportedBaseModelIds = [
"anthropic.claude-v2:1",
"anthropic.claude-instant-v1",
"anthropic.claude-3-sonnet-20240229-v1:0",
"anthropic.claude-3-haiku-20240307-v1:0",
"anthropic.claude-3-opus-20240229-v1:0",
"anthropic.claude-3-5-sonnet-20240620-v1:0",
"anthropic.claude-3-5-sonnet-20241022-v2:0",
"anthropic.claude-3-5-haiku-20241022-v1:0",
"mistral.mistral-7b-instruct-v0:2",
"mistral.mixtral-8x7b-instruct-v0:1",
"mistral.mistral-large-2402-v1:0",
"amazon.nova-pro-v1:0",
"amazon.nova-lite-v1:0",
"amazon.nova-micro-v1:0",
];
const regions: string[] = [
// US
"us-east-1",
"us-east-2",
"us-west-2",
"ap-south-1",
// // EU
"eu-central-1",
"eu-west-1",
"eu-west-2",
"eu-west-3",
"eu-north-1",
// // APAC
"ap-northeast-1",
"ap-northeast-2",
"ap-northeast-3",
"ap-southeast-1",
"ap-southeast-2",
];
interface RegionModelDict {
/**
* "us" | "eu" | "apac"
*/
area: string;
region: string;
model: string;
}
const sortByRegionAndModel = (dictList: RegionModelDict[]): RegionModelDict[] =>
dictList
.toSorted((a, b) => a.model.localeCompare(b.model))
.toSorted((a, b) => regions.indexOf(a.region) - regions.indexOf(b.region));
const uniq = (dict: RegionModelDict[]): RegionModelDict[] => {
const list: RegionModelDict[] = [];
for (const pair of dict) {
if (
!list.find(
(item) => item.model === pair.model && item.region === pair.region
)
)
list.push(pair);
}
return list;
};
const splitInferenceProfilesId = (
inferenceProfileId: string
): {
area: string;
model: string;
} => {
const [area, ...rest] = inferenceProfileId.split(".");
const model = rest.join(".");
return {
area,
model,
};
};
const getRegionFromArn = (arn: string): string | undefined => arn.split(":")[3];
const toRegionModelDictList = (
inferenceProfiles: InferenceProfileSummary[]
): RegionModelDict[] => {
const list: RegionModelDict[] = [];
for (const inferenceProfile of inferenceProfiles) {
const { inferenceProfileId, models } = inferenceProfile;
if (inferenceProfileId === undefined || models === undefined) {
continue;
}
const { area, model } = splitInferenceProfilesId(inferenceProfileId);
const regions = models
.map(({ modelArn }) => modelArn && getRegionFromArn(modelArn))
.filter((region) => region !== "")
.filter((region) => region !== undefined);
for (const region of regions) {
list.push({
area,
model,
region,
});
}
}
return list;
};
const listInferenceProfilesForRegion = async (
region: string
): Promise<InferenceProfileSummary[] | undefined> => {
const client = new BedrockClient({
region,
profile: AWS_PROFILE,
});
const command = new ListInferenceProfilesCommand({
typeEquals: "SYSTEM_DEFINED",
maxResults: 1000,
});
const response = await client.send(command);
return response.inferenceProfileSummaries;
};
const listInferenceProfilesForAllRegion = async (): Promise<
InferenceProfileSummary[]
> => {
let list: InferenceProfileSummary[] = [];
for (const region of regions) {
const inferenceProfiles = await listInferenceProfilesForRegion(region);
if (inferenceProfiles === undefined) {
continue;
}
list = [...list, ...inferenceProfiles];
}
return list;
};
const main = async () => {
const inferenceProfiles = await listInferenceProfilesForAllRegion();
const activeInferenceProfiles = inferenceProfiles.filter(
({ status }) => status === "ACTIVE"
);
// transform to RegionModelDict[]
const regionModelDictList = sortByRegionAndModel(
uniq(
toRegionModelDictList(activeInferenceProfiles).filter(({ model }) =>
supportedBaseModelIds.includes(model)
)
)
);
console.log(
JSON.stringify(
Object.groupBy(regionModelDictList, ({ region }) => region),
null,
2
)
);
};
main().catch((error) => {
console.error(error);
});Result{
"us-east-1": [
{
"area": "us",
"model": "amazon.nova-lite-v1:0",
"region": "us-east-1"
},
{
"area": "us",
"model": "amazon.nova-micro-v1:0",
"region": "us-east-1"
},
{
"area": "us",
"model": "amazon.nova-pro-v1:0",
"region": "us-east-1"
},
{
"area": "us",
"model": "anthropic.claude-3-5-haiku-20241022-v1:0",
"region": "us-east-1"
},
{
"area": "us",
"model": "anthropic.claude-3-5-sonnet-20240620-v1:0",
"region": "us-east-1"
},
{
"area": "us",
"model": "anthropic.claude-3-5-sonnet-20241022-v2:0",
"region": "us-east-1"
},
{
"area": "us",
"model": "anthropic.claude-3-haiku-20240307-v1:0",
"region": "us-east-1"
},
{
"area": "us",
"model": "anthropic.claude-3-opus-20240229-v1:0",
"region": "us-east-1"
},
{
"area": "us",
"model": "anthropic.claude-3-sonnet-20240229-v1:0",
"region": "us-east-1"
}
],
"us-east-2": [
{
"area": "us",
"model": "amazon.nova-lite-v1:0",
"region": "us-east-2"
},
{
"area": "us",
"model": "amazon.nova-micro-v1:0",
"region": "us-east-2"
},
{
"area": "us",
"model": "amazon.nova-pro-v1:0",
"region": "us-east-2"
},
{
"area": "us",
"model": "anthropic.claude-3-5-haiku-20241022-v1:0",
"region": "us-east-2"
},
{
"area": "us",
"model": "anthropic.claude-3-5-sonnet-20241022-v2:0",
"region": "us-east-2"
},
{
"area": "us",
"model": "anthropic.claude-3-haiku-20240307-v1:0",
"region": "us-east-2"
}
],
"us-west-2": [
{
"area": "us",
"model": "amazon.nova-lite-v1:0",
"region": "us-west-2"
},
{
"area": "us",
"model": "amazon.nova-micro-v1:0",
"region": "us-west-2"
},
{
"area": "us",
"model": "amazon.nova-pro-v1:0",
"region": "us-west-2"
},
{
"area": "us",
"model": "anthropic.claude-3-5-haiku-20241022-v1:0",
"region": "us-west-2"
},
{
"area": "us",
"model": "anthropic.claude-3-5-sonnet-20240620-v1:0",
"region": "us-west-2"
},
{
"area": "us",
"model": "anthropic.claude-3-5-sonnet-20241022-v2:0",
"region": "us-west-2"
},
{
"area": "us",
"model": "anthropic.claude-3-haiku-20240307-v1:0",
"region": "us-west-2"
},
{
"area": "us",
"model": "anthropic.claude-3-opus-20240229-v1:0",
"region": "us-west-2"
},
{
"area": "us",
"model": "anthropic.claude-3-sonnet-20240229-v1:0",
"region": "us-west-2"
}
],
"ap-south-1": [
{
"area": "apac",
"model": "anthropic.claude-3-5-sonnet-20240620-v1:0",
"region": "ap-south-1"
},
{
"area": "apac",
"model": "anthropic.claude-3-5-sonnet-20241022-v2:0",
"region": "ap-south-1"
},
{
"area": "apac",
"model": "anthropic.claude-3-haiku-20240307-v1:0",
"region": "ap-south-1"
},
{
"area": "apac",
"model": "anthropic.claude-3-sonnet-20240229-v1:0",
"region": "ap-south-1"
}
],
"eu-central-1": [
{
"area": "eu",
"model": "anthropic.claude-3-5-sonnet-20240620-v1:0",
"region": "eu-central-1"
},
{
"area": "eu",
"model": "anthropic.claude-3-haiku-20240307-v1:0",
"region": "eu-central-1"
},
{
"area": "eu",
"model": "anthropic.claude-3-sonnet-20240229-v1:0",
"region": "eu-central-1"
}
],
"eu-west-1": [
{
"area": "eu",
"model": "anthropic.claude-3-5-sonnet-20240620-v1:0",
"region": "eu-west-1"
},
{
"area": "eu",
"model": "anthropic.claude-3-haiku-20240307-v1:0",
"region": "eu-west-1"
},
{
"area": "eu",
"model": "anthropic.claude-3-sonnet-20240229-v1:0",
"region": "eu-west-1"
}
],
"eu-west-3": [
{
"area": "eu",
"model": "anthropic.claude-3-5-sonnet-20240620-v1:0",
"region": "eu-west-3"
},
{
"area": "eu",
"model": "anthropic.claude-3-haiku-20240307-v1:0",
"region": "eu-west-3"
},
{
"area": "eu",
"model": "anthropic.claude-3-sonnet-20240229-v1:0",
"region": "eu-west-3"
}
],
"ap-northeast-1": [
{
"area": "apac",
"model": "anthropic.claude-3-5-sonnet-20240620-v1:0",
"region": "ap-northeast-1"
},
{
"area": "apac",
"model": "anthropic.claude-3-5-sonnet-20241022-v2:0",
"region": "ap-northeast-1"
},
{
"area": "apac",
"model": "anthropic.claude-3-haiku-20240307-v1:0",
"region": "ap-northeast-1"
},
{
"area": "apac",
"model": "anthropic.claude-3-sonnet-20240229-v1:0",
"region": "ap-northeast-1"
}
],
"ap-northeast-2": [
{
"area": "apac",
"model": "anthropic.claude-3-5-sonnet-20240620-v1:0",
"region": "ap-northeast-2"
},
{
"area": "apac",
"model": "anthropic.claude-3-5-sonnet-20241022-v2:0",
"region": "ap-northeast-2"
},
{
"area": "apac",
"model": "anthropic.claude-3-haiku-20240307-v1:0",
"region": "ap-northeast-2"
},
{
"area": "apac",
"model": "anthropic.claude-3-sonnet-20240229-v1:0",
"region": "ap-northeast-2"
}
],
"ap-northeast-3": [
{
"area": "apac",
"model": "anthropic.claude-3-5-sonnet-20241022-v2:0",
"region": "ap-northeast-3"
}
],
"ap-southeast-1": [
{
"area": "apac",
"model": "anthropic.claude-3-5-sonnet-20240620-v1:0",
"region": "ap-southeast-1"
},
{
"area": "apac",
"model": "anthropic.claude-3-5-sonnet-20241022-v2:0",
"region": "ap-southeast-1"
},
{
"area": "apac",
"model": "anthropic.claude-3-haiku-20240307-v1:0",
"region": "ap-southeast-1"
},
{
"area": "apac",
"model": "anthropic.claude-3-sonnet-20240229-v1:0",
"region": "ap-southeast-1"
}
],
"ap-southeast-2": [
{
"area": "apac",
"model": "anthropic.claude-3-5-sonnet-20240620-v1:0",
"region": "ap-southeast-2"
},
{
"area": "apac",
"model": "anthropic.claude-3-5-sonnet-20241022-v2:0",
"region": "ap-southeast-2"
},
{
"area": "apac",
"model": "anthropic.claude-3-haiku-20240307-v1:0",
"region": "ap-southeast-2"
},
{
"area": "apac",
"model": "anthropic.claude-3-sonnet-20240229-v1:0",
"region": "ap-southeast-2"
}
]
} |
|
@sztm Thank you for the detailed reporting. We'd be appreciated if you could create a PR, Thank you. |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment
Describe the solution you'd like
Update supported cross region inference. Following list looks miss some regions and models.
bedrock-claude-chat/backend/app/bedrock.py
Lines 289 to 307 in a890a5f
Especially I want to add APAC for Claude 3.5 Sonnet to solve following error.
Why the solution needed
I can see a following error when use Claude 3.5 Sonnet v2 in
ap-northeast-1region even ifenableBedrockCrossRegionInferenceis true.Additional context
Claude 3.5 Sonnet v2 is available only on cross region inference in
ap-northeast-1. But bedrock-claude-chat app does not support it in APAC regions currently.Amazon Bedrock supports APAC as mentioned in following document.
https://docs.aws.amazon.com/bedrock/latest/userguide/inference-profiles-support.html
Probably this document is also not updated latest inference profiles.
Implementation feasibility
Are you willing to collaborate with us to discuss the solution, decide on the approach, and assist with the implementation?
I believe updating following solves this issue.
bedrock-claude-chat/backend/app/bedrock.py
Lines 301 to 307 in a890a5f
The text was updated successfully, but these errors were encountered: