Ecosystem
LLMs
- Overview
- OpenAI
- Anthropic
- Google Gemini
- Google Vertex AI
- Azure OpenAI
- Bedrock
- AWS SageMaker
- Ollama
- More
- Bring Your Own LLM
Agents
Perform batch inference with Bedrock
To perform batch inference with Bedrock, you need to upload files to S3. This process can be cumbersome and duplicative in nature because you need to transform your data into model specific formats.
With Portkey, you can upload the file in OpenAI format and portkey will handle transforming the file into the format required by Bedrock on the fly!
This is the most efficient way to
- Test your data with different foundation models
- Perform A/B testing with different foundation models
- Perform batch inference with different foundation models
Create Batch Job
from portkey_ai import Portkey
# Initialize the Portkey client
portkey = Portkey(
api_key="PORTKEY_API_KEY", # Replace with your Portkey API key
provider="bedrock",
aws_access_key_id="YOUR_AWS_ACCESS_KEY_ID",
aws_secret_access_key="YOUR_AWS_SECRET_ACCESS_KEY",
aws_region="YOUR_AWS_REGION",
aws_s3_bucket="YOUR_AWS_S3_BUCKET",
aws_s3_object_key="YOUR_AWS_S3_OBJECT_KEY",
aws_bedrock_model="YOUR_AWS_BEDROCK_MODEL"
)
start_batch_response = portkey.batches.create(
input_file_id="file_id", # file id of the input file
endpoint="endpoint", # ex: /v1/chat/completions
completion_window="completion_window", # ex: 24h
metadata={}, # metadata for the batch,
role_arn="arn:aws:iam::12312:role/BedrockBatchRole", # the role to use for creating the batch job
model="anthropic.claude-3-5-sonnet-20240620-v1:0", # the model to use for the batch
output_data_config={
"s3OutputDataConfig": {
"s3Uri": "s3://generations-raw/",
"s3EncryptionKeyId": "arn:aws:kms:us-west-2:517194595696:key/89b483cb-130d-497b-aa37-7db177e7cd32" # this is optional, if you want to use a KMS key to encrypt the output data
}
}, # output_data_config is optional, if you want to specify a different output location for the batch job, default is the same as the input file
job_name="anthropi-requests-test" # optional
)
print(start_batch_response)
List Batch Jobs
from portkey_ai import Portkey
# Initialize the Portkey client
portkey = Portkey(
api_key="PORTKEY_API_KEY", # Replace with your Portkey API key
provider="bedrock",
aws_access_key_id="YOUR_AWS_ACCESS_KEY_ID",
aws_secret_access_key="YOUR_AWS_SECRET_ACCESS_KEY",
aws_region="YOUR_AWS_REGION",
)
batches = portkey.batches.list()
print(batches)
Get Batch Job Details
from portkey_ai import Portkey
# Initialize the Portkey client
portkey = Portkey(
api_key="PORTKEY_API_KEY", # Replace with your Portkey API key
virtual_key="VIRTUAL_KEY", # Add your provider's virtual key
)
batch = portkey.batches.retrieve(batch_id="batch_id")
print(batch)
Get Batch Output
curl --location 'https://api.portkey.ai/v1/batches/<batch_id>/output' \
--header 'x-portkey-api-key: <portkey_api_key>' \
--header 'x-portkey-virtual-key: <virtual_key>' \
List Batch Jobs
from portkey_ai import Portkey
# Initialize the Portkey client
portkey = Portkey(
api_key="PORTKEY_API_KEY", # Replace with your Portkey API key
virtual_key="VIRTUAL_KEY", # Add your provider's virtual key
)
batches = portkey.batches.list()
print(batches)
Cancel Batch Job
from portkey_ai import Portkey
# Initialize the Portkey client
portkey = Portkey(
api_key="PORTKEY_API_KEY", # Replace with your Portkey API key
virtual_key="VIRTUAL_KEY", # Add your provider's virtual key
)
cancel_batch_response = portkey.batches.cancel(batch_id="batch_id")
print(cancel_batch_response)
Information about Permissions and IAM Roles
These are the minimum permissions required to use the Bedrock Batch APIs.
{
"Version": "2012-10-17",
"Statement": [
{
"Effect": "Allow",
"Action": [
"bedrock:ListFoundationModels",
"bedrock:GetFoundationModel",
"bedrock:ListInferenceProfiles",
"bedrock:GetInferenceProfile",
"bedrock:ListCustomModels",
"bedrock:GetCustomModel",
"bedrock:TagResource",
"bedrock:UntagResource",
"bedrock:ListTagsForResource",
"bedrock:CreateModelInvocationJob",
"bedrock:GetModelInvocationJob",
"bedrock:ListModelInvocationJobs",
"bedrock:StopModelInvocationJob"
],
"Resource": [
"arn:aws:bedrock:<region>:<account_id>:model-customization-job/*",
"arn:aws:bedrock:<region>:<account_id>:custom-model/*",
"arn:aws:bedrock:<region>::foundation-model/*"
]
},
{
"Effect": "Allow",
"Action": [
"s3:ListBucket",
"s3:PutObject",
"s3:GetObject",
"s3:GetObjectAttributes"
],
"Resource": [
"arn:aws:s3:::<bucket>",
"arn:aws:s3:::<bucket>/*"
]
},
{
"Action": [
"iam:PassRole"
],
"Effect": "Allow",
"Resource": "arn:aws:iam::<account_id>:role/<service_role_name>",
"Condition": {
"StringEquals": {
"iam:PassedToService": [
"bedrock.amazonaws.com"
]
}
}
}
]
}
These are the minimum permissions required to use the Bedrock Batch APIs.
Trust relationship:
{
"Version": "2012-10-17",
"Statement": [
{
"Effect": "Allow",
"Principal": {
"Service": "bedrock.amazonaws.com"
},
"Action": "sts:AssumeRole",
"Condition": {
"StringEquals": {
"aws:SourceAccount": "<account_id>"
},
"ArnEquals": {
"aws:SourceArn": "arn:aws:bedrock:<region>:<account_id>:model-invocation-job/*"
}
}
]
}
Permission Policy:
{
"Version": "2012-10-17",
"Statement": [
{
"Effect": "Allow",
"Action": [
"s3:GetObject",
"s3:PutObject",
"s3:ListBucket"
],
"Resource": [
"arn:aws:s3:::<bucket>",
"arn:aws:s3:::<bucket>/*"
]
}
]
}
Was this page helpful?