To perform batch inference with Bedrock, you need to upload files to S3. This process can be cumbersome and duplicative in nature because you need to transform your data into model specific formats.

With Portkey, you can upload the file in OpenAI format and portkey will handle transforming the file into the format required by Bedrock on the fly!

This is the most efficient way to

  • Test your data with different foundation models
  • Perform A/B testing with different foundation models
  • Perform batch inference with different foundation models

​
Create Batch Job

from portkey_ai import Portkey

# Initialize the Portkey client
portkey = Portkey(
    api_key="PORTKEY_API_KEY",  # Replace with your Portkey API key
    provider="bedrock",
    aws_access_key_id="YOUR_AWS_ACCESS_KEY_ID",
    aws_secret_access_key="YOUR_AWS_SECRET_ACCESS_KEY",
    aws_region="YOUR_AWS_REGION",
    aws_s3_bucket="YOUR_AWS_S3_BUCKET",
    aws_s3_object_key="YOUR_AWS_S3_OBJECT_KEY",
    aws_bedrock_model="YOUR_AWS_BEDROCK_MODEL"
)

start_batch_response = portkey.batches.create(
  input_file_id="file_id", # file id of the input file
  endpoint="endpoint", # ex: /v1/chat/completions
  completion_window="completion_window", # ex: 24h
  metadata={}, # metadata for the batch,
  role_arn="arn:aws:iam::12312:role/BedrockBatchRole", # the role to use for creating the batch job
  model="anthropic.claude-3-5-sonnet-20240620-v1:0", # the model to use for the batch
  output_data_config={
    "s3OutputDataConfig": {
      "s3Uri": "s3://generations-raw/",
      "s3EncryptionKeyId": "arn:aws:kms:us-west-2:517194595696:key/89b483cb-130d-497b-aa37-7db177e7cd32" # this is optional, if you want to use a KMS key to encrypt the output data
    }
  }, # output_data_config is optional, if you want to specify a different output location for the batch job, default is the same as the input file
  job_name="anthropi-requests-test" # optional
)

print(start_batch_response)

​
List Batch Jobs

from portkey_ai import Portkey

# Initialize the Portkey client
portkey = Portkey(
    api_key="PORTKEY_API_KEY",  # Replace with your Portkey API key
    provider="bedrock",
    aws_access_key_id="YOUR_AWS_ACCESS_KEY_ID",
    aws_secret_access_key="YOUR_AWS_SECRET_ACCESS_KEY",
    aws_region="YOUR_AWS_REGION",
)

batches = portkey.batches.list()

print(batches)

​
Get Batch Job Details

from portkey_ai import Portkey

# Initialize the Portkey client
portkey = Portkey(
    api_key="PORTKEY_API_KEY",  # Replace with your Portkey API key
    virtual_key="VIRTUAL_KEY",   # Add your provider's virtual key
)

batch = portkey.batches.retrieve(batch_id="batch_id")

print(batch)

​
Get Batch Output

curl --location 'https://api.portkey.ai/v1/batches/<batch_id>/output' \
--header 'x-portkey-api-key: <portkey_api_key>' \
--header 'x-portkey-virtual-key: <virtual_key>' \

​
List Batch Jobs

from portkey_ai import Portkey

# Initialize the Portkey client
portkey = Portkey(
    api_key="PORTKEY_API_KEY",  # Replace with your Portkey API key
    virtual_key="VIRTUAL_KEY",   # Add your provider's virtual key
)

batches = portkey.batches.list()

print(batches)

​
Cancel Batch Job

from portkey_ai import Portkey

# Initialize the Portkey client
portkey = Portkey(
    api_key="PORTKEY_API_KEY",  # Replace with your Portkey API key
    virtual_key="VIRTUAL_KEY",   # Add your provider's virtual key
)

cancel_batch_response = portkey.batches.cancel(batch_id="batch_id")

print(cancel_batch_response)

​
Information about Permissions and IAM Roles

Was this page helpful?

Suggest editsRaise issue