Ecosystem
LLMs
- Overview
- OpenAI
- Anthropic
- Google Gemini
- Google Vertex AI
- Azure OpenAI
- Bedrock
- AWS SageMaker
- Ollama
- More
- Bring Your Own LLM
Agents
Perform batch inference with Bedrock
To perform batch inference with Bedrock, you need to upload files to S3. This process can be cumbersome and duplicative in nature because you need to transform your data into model specific formats.
With Portkey, you can upload the file in OpenAI format and portkey will handle transforming the file into the format required by Bedrock on the fly!
This is the most efficient way to
- Test your data with different foundation models
- Perform A/B testing with different foundation models
- Perform batch inference with different foundation models
Create Batch Job
from portkey_ai import Portkey
# Initialize the Portkey client
portkey = Portkey(
api_key="PORTKEY_API_KEY", # Replace with your Portkey API key
provider="bedrock",
aws_access_key_id="YOUR_AWS_ACCESS_KEY_ID",
aws_secret_access_key="YOUR_AWS_SECRET_ACCESS_KEY",
aws_region="YOUR_AWS_REGION",
aws_s3_bucket="YOUR_AWS_S3_BUCKET",
aws_s3_object_key="YOUR_AWS_S3_OBJECT_KEY",
aws_bedrock_model="YOUR_AWS_BEDROCK_MODEL"
)
start_batch_response = portkey.batches.create(
input_file_id="file_id", # file id of the input file
endpoint="endpoint", # ex: /v1/chat/completions
completion_window="completion_window", # ex: 24h
metadata={}, # metadata for the batch,
role_arn="arn:aws:iam::12312:role/BedrockBatchRole", # the role to use for creating the batch job
model="anthropic.claude-3-5-sonnet-20240620-v1:0", # the model to use for the batch
output_data_config={
"s3OutputDataConfig": {
"s3Uri": "s3://generations-raw/",
"s3EncryptionKeyId": "arn:aws:kms:us-west-2:517194595696:key/89b483cb-130d-497b-aa37-7db177e7cd32" # this is optional, if you want to use a KMS key to encrypt the output data
}
}, # output_data_config is optional, if you want to specify a different output location for the batch job, default is the same as the input file
job_name="anthropi-requests-test" # optional
)
print(start_batch_response)
from portkey_ai import Portkey
# Initialize the Portkey client
portkey = Portkey(
api_key="PORTKEY_API_KEY", # Replace with your Portkey API key
provider="bedrock",
aws_access_key_id="YOUR_AWS_ACCESS_KEY_ID",
aws_secret_access_key="YOUR_AWS_SECRET_ACCESS_KEY",
aws_region="YOUR_AWS_REGION",
aws_s3_bucket="YOUR_AWS_S3_BUCKET",
aws_s3_object_key="YOUR_AWS_S3_OBJECT_KEY",
aws_bedrock_model="YOUR_AWS_BEDROCK_MODEL"
)
start_batch_response = portkey.batches.create(
input_file_id="file_id", # file id of the input file
endpoint="endpoint", # ex: /v1/chat/completions
completion_window="completion_window", # ex: 24h
metadata={}, # metadata for the batch,
role_arn="arn:aws:iam::12312:role/BedrockBatchRole", # the role to use for creating the batch job
model="anthropic.claude-3-5-sonnet-20240620-v1:0", # the model to use for the batch
output_data_config={
"s3OutputDataConfig": {
"s3Uri": "s3://generations-raw/",
"s3EncryptionKeyId": "arn:aws:kms:us-west-2:517194595696:key/89b483cb-130d-497b-aa37-7db177e7cd32" # this is optional, if you want to use a KMS key to encrypt the output data
}
}, # output_data_config is optional, if you want to specify a different output location for the batch job, default is the same as the input file
job_name="anthropi-requests-test" # optional
)
print(start_batch_response)
import { Portkey } from 'portkey-ai';
// Initialize the Portkey client
const portkey = new Portkey({
apiKey: "PORTKEY_API_KEY", // Replace with your Portkey API key
provider="bedrock",
awsAccessKeyId="YOUR_AWS_ACCESS_KEY_ID",
awsSecretAccessKey="YOUR_AWS_SECRET_ACCESS_KEY",
awsRegion="YOUR_AWS_REGION",
awsS3Bucket="YOUR_AWS_S3_BUCKET",
awsS3ObjectKey="YOUR_AWS_S3_OBJECT_KEY",
awsBedrockModel="YOUR_AWS_BEDROCK_MODEL"
});
const startBatch = async () => {
const startBatchResponse = await portkey.batches.create({
input_file_id: "file_id", // file id of the input file
endpoint: "endpoint", // ex: /v1/chat/completions
completion_window: "completion_window", // ex: 24h
metadata: {}, // metadata for the batch
role_arn: "arn:aws:iam::12312:role/BedrockBatchRole", // the role to use for creating the batch job
model: "anthropic.claude-3-5-sonnet-20240620-v1:0", // the model to use for the batch
output_data_config: {
s3OutputDataConfig: {
s3Uri: "s3://generations-raw/",
s3EncryptionKeyId: "arn:aws:kms:us-west-2:517194595696:key/89b483cb-130d-497b-aa37-7db177e7cd32" // this is optional, if you want to use a KMS key to encrypt the output data
}
}, // output_data_config is optional, if you want to specify a different output location for the batch job, default is the same as the input file
job_name: "anthropi-requests-test" // optional
});
console.log(startBatchResponse);
}
await startBatch();
curl --location 'https://api.portkey.ai/v1/batches' \
--header 'x-portkey-api-key: <portkey_api_key>' \
--header 'x-portkey-aws-access-key-id: {YOUR_AWS_ACCESS_KEY_ID}' \
--header 'x-portkey-aws-secret-access-key: {YOUR_AWS_SECRET_ACCESS_KEY}' \
--header 'x-portkey-aws-region: {YOUR_AWS_REGION}' \
--header 'Content-Type: application/json' \
--data '{
"model": "meta.llama3-1-8b-instruct-v1:0",
"input_file_id": "s3%3A%2F%2Fgenerations-raw-west-2%2Fbatch_files%2Fllama2%2Fbatch_chat_completions_101_requests.jsonl",
"role_arn": "arn:aws:iam::12312:role/BedrockBatchRole", // the role to use for creating the batch job
"output_data_config": { // output_data_config is optional, if you want to specify a different output location for the batch job, default is the same as the input file
"s3OutputDataConfig": {
"s3Uri": "s3://generations-raw/",
"s3EncryptionKeyId": "arn:aws:kms:us-west-2:517194595696:key/89b483cb-130d-497b-aa37-7db177e7cd32" // this is optional, if you want to use a KMS key to encrypt the output data
}
},
"job_name": "anthropi-requests" // optional
}'
import OpenAI from 'openai'; // We're using the v4 SDK
import { PORTKEY_GATEWAY_URL, createHeaders } from 'portkey-ai'
const openai = new OpenAI({
apiKey: 'PLACEHOLDER_NOT_USED', // defaults to process.env["OPENAI_API_KEY"],
baseURL: PORTKEY_GATEWAY_URL,
defaultHeaders: createHeaders({
provider: "openai",
apiKey: "PORTKEY_API_KEY", // defaults to process.env["PORTKEY_API_KEY"]
awsAccessKeyId: "YOUR_AWS_ACCESS_KEY_ID",
awsSecretAccessKey: "YOUR_AWS_SECRET_ACCESS_KEY",
awsRegion: "YOUR_AWS_REGION",
awsS3Bucket: "YOUR_AWS_S3_BUCKET",
awsS3ObjectKey: "YOUR_AWS_S3_OBJECT_KEY",
awsBedrockModel: "YOUR_AWS_BEDROCK_MODEL"
})
});
const startBatch = async () => {
const startBatchResponse = await openai.batches.create({
input_file_id: "file_id", // file id of the input file
endpoint: "endpoint", // ex: /v1/chat/completions
completion_window: "completion_window", // ex: 24h
metadata: {}, // metadata for the batch
role_arn: "arn:aws:iam::12312:role/BedrockBatchRole", // the role to use for creating the batch job
model: "anthropic.claude-3-5-sonnet-20240620-v1:0", // the model to use for the batch
output_data_config: {
s3OutputDataConfig: {
s3Uri: "s3://generations-raw/",
s3EncryptionKeyId: "arn:aws:kms:us-west-2:517194595696:key/89b483cb-130d-497b-aa37-7db177e7cd32" // this is optional, if you want to use a KMS key to encrypt the output data
}
}, // output_data_config is optional, if you want to specify a different output location for the batch job, default is the same as the input file
job_name: "anthropi-requests-test" // optional
});
console.log(startBatchResponse);
}
await startBatch();
from openai import OpenAI
from portkey_ai import PORTKEY_GATEWAY_URL, createHeaders
openai = OpenAI(
api_key='PLACEHOLDER_NOT_USED',
base_url=PORTKEY_GATEWAY_URL,
default_headers=createHeaders(
provider="openai",
api_key="PORTKEY_API_KEY",
aws_access_key_id="YOUR_AWS_ACCESS_KEY_ID",
aws_secret_access_key="YOUR_AWS_SECRET_ACCESS_KEY",
aws_region="YOUR_AWS_REGION",
aws_s3_bucket="YOUR_AWS_S3_BUCKET",
aws_s3_object_key="YOUR_AWS_S3_OBJECT_KEY",
aws_bedrock_model="YOUR_AWS_BEDROCK_MODEL"
)
)
start_batch_response = openai.batches.create(
input_file_id="file_id", # file id of the input file
endpoint="endpoint", # ex: /v1/chat/completions
completion_window="completion_window", # ex: 24h
metadata={}, # metadata for the batch
role_arn="arn:aws:iam::12312:role/BedrockBatchRole", # the role to use for creating the batch job
model="anthropic.claude-3-5-sonnet-20240620-v1:0", # the model to use for the batch
output_data_config={
"s3OutputDataConfig": {
"s3Uri": "s3://generations-raw/",
"s3EncryptionKeyId": "arn:aws:kms:us-west-2:517194595696:key/89b483cb-130d-497b-aa37-7db177e7cd32" // this is optional, if you want to use a KMS key to encrypt the output data
}
}, # output_data_config is optional, if you want to specify a different output location for the batch job, default is the same as the input file
job_name="anthropi-requests-test" # optional
)
print(start_batch_response)
List Batch Jobs
from portkey_ai import Portkey
# Initialize the Portkey client
portkey = Portkey(
api_key="PORTKEY_API_KEY", # Replace with your Portkey API key
provider="bedrock",
aws_access_key_id="YOUR_AWS_ACCESS_KEY_ID",
aws_secret_access_key="YOUR_AWS_SECRET_ACCESS_KEY",
aws_region="YOUR_AWS_REGION",
)
batches = portkey.batches.list()
print(batches)
from portkey_ai import Portkey
# Initialize the Portkey client
portkey = Portkey(
api_key="PORTKEY_API_KEY", # Replace with your Portkey API key
provider="bedrock",
aws_access_key_id="YOUR_AWS_ACCESS_KEY_ID",
aws_secret_access_key="YOUR_AWS_SECRET_ACCESS_KEY",
aws_region="YOUR_AWS_REGION",
)
batches = portkey.batches.list()
print(batches)
import { Portkey } from 'portkey-ai';
// Initialize the Portkey client
const portkey = new Portkey({
apiKey: "PORTKEY_API_KEY", // Replace with your Portkey API key
provider="bedrock",
awsAccessKeyId="YOUR_AWS_ACCESS_KEY_ID",
awsSecretAccessKey="YOUR_AWS_SECRET_ACCESS_KEY",
awsRegion="YOUR_AWS_REGION",
});
const listBatches = async () => {
const batches = await portkey.batches.list();
console.log(batches);
}
await listBatches();
curl --location 'https://api.portkey.ai/v1/batches' \
--header 'x-portkey-api-key: <portkey_api_key>' \
--header 'x-portkey-virtual-key: <virtual_key>' \
import OpenAI from 'openai'; // We're using the v4 SDK
import { PORTKEY_GATEWAY_URL, createHeaders } from 'portkey-ai'
const openai = new OpenAI({
apiKey: 'PLACEHOLDER_NOT_USED', // defaults to process.env["OPENAI_API_KEY"],
baseURL: PORTKEY_GATEWAY_URL,
defaultHeaders: createHeaders({
provider: "openai",
apiKey: "PORTKEY_API_KEY", // defaults to process.env["PORTKEY_API_KEY"]
awsAccessKeyId: "YOUR_AWS_ACCESS_KEY_ID",
awsSecretAccessKey: "YOUR_AWS_SECRET_ACCESS_KEY",
awsRegion: "YOUR_AWS_REGION"
})
});
const listBatches = async () => {
const batches = await openai.batches.list();
console.log(batches);
}
await listBatches();
from openai import OpenAI
from portkey_ai import PORTKEY_GATEWAY_URL, createHeaders
openai = OpenAI(
api_key='PLACEHOLDER_NOT_USED',
base_url=PORTKEY_GATEWAY_URL,
default_headers=createHeaders(
provider="openai",
api_key="PORTKEY_API_KEY",
aws_access_key_id="YOUR_AWS_ACCESS_KEY_ID",
aws_secret_access_key="YOUR_AWS_SECRET_ACCESS_KEY",
aws_region="YOUR_AWS_REGION"
)
)
batches = openai.batches.list()
print(batches)
Get Batch Job Details
from portkey_ai import Portkey
# Initialize the Portkey client
portkey = Portkey(
api_key="PORTKEY_API_KEY", # Replace with your Portkey API key
virtual_key="VIRTUAL_KEY", # Add your provider's virtual key
)
batch = portkey.batches.retrieve(batch_id="batch_id")
print(batch)
from portkey_ai import Portkey
# Initialize the Portkey client
portkey = Portkey(
api_key="PORTKEY_API_KEY", # Replace with your Portkey API key
virtual_key="VIRTUAL_KEY", # Add your provider's virtual key
)
batch = portkey.batches.retrieve(batch_id="batch_id")
print(batch)
import { Portkey } from 'portkey-ai';
// Initialize the Portkey client
const portkey = new Portkey({
apiKey: "PORTKEY_API_KEY", // Replace with your Portkey API key
virtualKey: "VIRTUAL_KEY", // Add your provider's virtual key
});
const getBatch = async () => {
const batch = await portkey.batches.retrieve(batch_id="batch_id");
console.log(batch);
}
await getBatch();
curl --location 'https://api.portkey.ai/v1/batches/<batch_id>' \
--header 'x-portkey-api-key: <portkey_api_key>' \
--header 'x-portkey-virtual-key: <virtual_key>' \
import OpenAI from 'openai'; // We're using the v4 SDK
import { PORTKEY_GATEWAY_URL, createHeaders } from 'portkey-ai'
const openai = new OpenAI({
apiKey: 'PLACEHOLDER_NOT_USED', // defaults to process.env["OPENAI_API_KEY"],
baseURL: PORTKEY_GATEWAY_URL,
defaultHeaders: createHeaders({
provider: "openai",
apiKey: "PORTKEY_API_KEY", // defaults to process.env["PORTKEY_API_KEY"]
virtualKey: "BEDROCK_VIRTUAL_KEY",
})
});
const getBatch = async () => {
const batch = await openai.batches.retrieve(batch_id="batch_id");
console.log(batch);
}
await getBatch();
from openai import OpenAI
from portkey_ai import PORTKEY_GATEWAY_URL, createHeaders
openai = OpenAI(
api_key='PLACEHOLDER_NOT_USED',
base_url=PORTKEY_GATEWAY_URL,
default_headers=createHeaders(
provider="openai",
api_key="PORTKEY_API_KEY",
virtualKey: "BEDROCK_VIRTUAL_KEY",
)
)
batch = openai.batches.retrieve(batch_id="batch_id")
print(batch)
Get Batch Output
curl --location 'https://api.portkey.ai/v1/batches/<batch_id>/output' \
--header 'x-portkey-api-key: <portkey_api_key>' \
--header 'x-portkey-virtual-key: <virtual_key>' \
curl --location 'https://api.portkey.ai/v1/batches/<batch_id>/output' \
--header 'x-portkey-api-key: <portkey_api_key>' \
--header 'x-portkey-virtual-key: <virtual_key>' \
List Batch Jobs
from portkey_ai import Portkey
# Initialize the Portkey client
portkey = Portkey(
api_key="PORTKEY_API_KEY", # Replace with your Portkey API key
virtual_key="VIRTUAL_KEY", # Add your provider's virtual key
)
batches = portkey.batches.list()
print(batches)
from portkey_ai import Portkey
# Initialize the Portkey client
portkey = Portkey(
api_key="PORTKEY_API_KEY", # Replace with your Portkey API key
virtual_key="VIRTUAL_KEY", # Add your provider's virtual key
)
batches = portkey.batches.list()
print(batches)
import { Portkey } from 'portkey-ai';
// Initialize the Portkey client
const portkey = new Portkey({
apiKey: "PORTKEY_API_KEY", // Replace with your Portkey API key
virtualKey: "VIRTUAL_KEY", // Add your provider's virtual key
});
const listBatchingJobs = async () => {
const batching_jobs = await portkey.batches.list();
console.log(batching_jobs);
}
await listBatchingJobs();
curl --location 'https://api.portkey.ai/v1/batches' \
--header 'x-portkey-api-key: <portkey_api_key>' \
--header 'x-portkey-virtual-key: <virtual_key>' \
import OpenAI from 'openai'; // We're using the v4 SDK
import { PORTKEY_GATEWAY_URL, createHeaders } from 'portkey-ai'
const openai = new OpenAI({
apiKey: 'PLACEHOLDER_NOT_USED', // defaults to process.env["OPENAI_API_KEY"],
baseURL: PORTKEY_GATEWAY_URL,
defaultHeaders: createHeaders({
provider: "openai",
apiKey: "PORTKEY_API_KEY", // defaults to process.env["PORTKEY_API_KEY"]
virtualKey: "BEDROCK_VIRTUAL_KEY",
})
});
const listBatchingJobs = async () => {
const batching_jobs = await openai.batches.list();
console.log(batching_jobs);
}
await listBatchingJobs();
from openai import OpenAI
from portkey_ai import PORTKEY_GATEWAY_URL, createHeaders
openai = OpenAI(
api_key='PLACEHOLDER_NOT_USED',
base_url=PORTKEY_GATEWAY_URL,
default_headers=createHeaders(
provider="openai",
api_key="PORTKEY_API_KEY",
virtualKey: "BEDROCK_VIRTUAL_KEY",
)
)
batching_jobs = openai.batches.list()
print(batching_jobs)
Cancel Batch Job
from portkey_ai import Portkey
# Initialize the Portkey client
portkey = Portkey(
api_key="PORTKEY_API_KEY", # Replace with your Portkey API key
virtual_key="VIRTUAL_KEY", # Add your provider's virtual key
)
cancel_batch_response = portkey.batches.cancel(batch_id="batch_id")
print(cancel_batch_response)
from portkey_ai import Portkey
# Initialize the Portkey client
portkey = Portkey(
api_key="PORTKEY_API_KEY", # Replace with your Portkey API key
virtual_key="VIRTUAL_KEY", # Add your provider's virtual key
)
cancel_batch_response = portkey.batches.cancel(batch_id="batch_id")
print(cancel_batch_response)
import { Portkey } from 'portkey-ai';
// Initialize the Portkey client
const portkey = new Portkey({
apiKey: "PORTKEY_API_KEY", // Replace with your Portkey API key
virtualKey: "VIRTUAL_KEY", // Add your provider's virtual key
});
const cancelBatch = async () => {
const cancel_batch_response = await portkey.batches.cancel(batch_id="batch_id");
console.log(cancel_batch_response);
}
await cancelBatch();
curl --request POST --location 'https://api.portkey.ai/v1/batches/<batch_id>/cancel' \
--header 'x-portkey-api-key: <portkey_api_key>' \
--header 'x-portkey-virtual-key: <virtual_key>' \
import OpenAI from 'openai'; // We're using the v4 SDK
import { PORTKEY_GATEWAY_URL, createHeaders } from 'portkey-ai'
const openai = new OpenAI({
apiKey: 'PLACEHOLDER_NOT_USED', // defaults to process.env["OPENAI_API_KEY"],
baseURL: PORTKEY_GATEWAY_URL,
defaultHeaders: createHeaders({
provider: "openai",
apiKey: "PORTKEY_API_KEY", // defaults to process.env["PORTKEY_API_KEY"]
virtualKey: "BEDROCK_VIRTUAL_KEY",
})
});
const cancelBatch = async () => {
const cancel_batch_response = await openai.batches.cancel(batch_id="batch_id");
console.log(cancel_batch_response);
}
await cancelBatch();
from openai import OpenAI
from portkey_ai import PORTKEY_GATEWAY_URL, createHeaders
openai = OpenAI(
api_key='PLACEHOLDER_NOT_USED',
base_url=PORTKEY_GATEWAY_URL,
default_headers=createHeaders(
provider="openai",
api_key="PORTKEY_API_KEY",
virtualKey: "BEDROCK_VIRTUAL_KEY",
)
)
cancel_batch_response = openai.batches.cancel(batch_id="batch_id")
print(cancel_batch_response)
Information about Permissions and IAM Roles
These are the minimum permissions required to use the Bedrock Batch APIs.
{
"Version": "2012-10-17",
"Statement": [
{
"Effect": "Allow",
"Action": [
"bedrock:ListFoundationModels",
"bedrock:GetFoundationModel",
"bedrock:ListInferenceProfiles",
"bedrock:GetInferenceProfile",
"bedrock:ListCustomModels",
"bedrock:GetCustomModel",
"bedrock:TagResource",
"bedrock:UntagResource",
"bedrock:ListTagsForResource",
"bedrock:CreateModelInvocationJob",
"bedrock:GetModelInvocationJob",
"bedrock:ListModelInvocationJobs",
"bedrock:StopModelInvocationJob"
],
"Resource": [
"arn:aws:bedrock:<region>:<account_id>:model-customization-job/*",
"arn:aws:bedrock:<region>:<account_id>:custom-model/*",
"arn:aws:bedrock:<region>::foundation-model/*"
]
},
{
"Effect": "Allow",
"Action": [
"s3:ListBucket",
"s3:PutObject",
"s3:GetObject",
"s3:GetObjectAttributes"
],
"Resource": [
"arn:aws:s3:::<bucket>",
"arn:aws:s3:::<bucket>/*"
]
},
{
"Action": [
"iam:PassRole"
],
"Effect": "Allow",
"Resource": "arn:aws:iam::<account_id>:role/<service_role_name>",
"Condition": {
"StringEquals": {
"iam:PassedToService": [
"bedrock.amazonaws.com"
]
}
}
}
]
}
These are the minimum permissions required to use the Bedrock Batch APIs.
Trust relationship:
{
"Version": "2012-10-17",
"Statement": [
{
"Effect": "Allow",
"Principal": {
"Service": "bedrock.amazonaws.com"
},
"Action": "sts:AssumeRole",
"Condition": {
"StringEquals": {
"aws:SourceAccount": "<account_id>"
},
"ArnEquals": {
"aws:SourceArn": "arn:aws:bedrock:<region>:<account_id>:model-invocation-job/*"
}
}
]
}
Permission Policy:
{
"Version": "2012-10-17",
"Statement": [
{
"Effect": "Allow",
"Action": [
"s3:GetObject",
"s3:PutObject",
"s3:ListBucket"
],
"Resource": [
"arn:aws:s3:::<bucket>",
"arn:aws:s3:::<bucket>/*"
]
}
]
}
Was this page helpful?