- Test your data with different foundation models
- Perform A/B testing with different foundation models
- Perform batch inference with different foundation models
The AI Gateway supports two modes on Azure OpenAI:
- Provider Batch API (cheapest, completion_window:
24h,48h, etc.) - AI Gateway Batch API (fast, provider-agnostic, completion_window:
immediate)
Using Azure OpenAI Batch API through the AI Gateway
Create Batch Job
curl --location 'https://aigw.portkey.ai/v1/batches' \
--header 'Authorization: Bearer <portkey_api_key>' \
--header 'x-portkey-provider: @provider' \
--header 'Content-Type: application/json' \
--data '{
"input_file_id": "<file_id>",
"endpoint": "<endpoint>",
"completion_window": "<completion_window>",
"metadata": {}
}'
import OpenAI from 'openai'; // We're using the v4 SDK
const openai = new OpenAI({
apiKey: "PORTKEY_API_KEY", // defaults to process.env["PORTKEY_API_KEY"]
baseURL: "https://aigw.portkey.ai/v1",
defaultHeaders: {
"x-portkey-provider": "openai",
}
});
const startBatch = async () => {
const startBatchResponse = await openai.batches.create({
input_file_id: "file_id", // file id of the input file
endpoint: "endpoint", // ex: /v1/chat/completions
completion_window: "completion_window", // ex: 24h
metadata: {} // metadata for the batch
});
console.log(startBatchResponse);
}
await startBatch();
from openai import OpenAI
openai = OpenAI(
api_key="PORTKEY_API_KEY",
base_url="https://aigw.portkey.ai/v1",
default_headers={"x-portkey-provider": "openai"}
)
start_batch_response = openai.batches.create(
input_file_id="file_id", # file id of the input file
endpoint="endpoint", # ex: /v1/chat/completions
completion_window="completion_window", # ex: 24h
metadata={} # metadata for the batch
)
print(start_batch_response)
Create Batch Job with Blob Storage
curl --location 'https://aigw.portkey.ai/v1/batches' \
--header 'Authorization: Bearer <portkey_api_key>' \
--header 'x-portkey-provider: @provider' \
--header 'Content-Type: application/json' \
--data '{
"endpoint": "<endpoint>",
"completion_window": "<completion_window>",
"metadata": {},
"input_blob": "<blob_url>",
"output_folder": {
"url": "<output_blob_folder>" # both error file and output file will be saved in this folder
}
}'
import OpenAI from 'openai'; // We're using the v4 SDK
const openai = new OpenAI({
apiKey: "PORTKEY_API_KEY", // defaults to process.env["PORTKEY_API_KEY"]
baseURL: "https://aigw.portkey.ai/v1",
defaultHeaders: {
"x-portkey-provider": "openai",
}
});
const startBatch = async () => {
const startBatchResponse = await openai.batches.create({
endpoint: "endpoint", // ex: /v1/chat/completions
completion_window: "completion_window", // ex: 24h
metadata: {}, // metadata for the batch
extra_body: {
input_blob: "<blob_url>",
output_folder: {
url: "<output_blob_folder>" // both error file and output file will be saved in this folder
}
}
});
console.log(startBatchResponse);
}
await startBatch();
from openai import OpenAI
openai = OpenAI(
api_key="PORTKEY_API_KEY",
base_url="https://aigw.portkey.ai/v1",
default_headers={"x-portkey-provider": "openai"}
)
start_batch_response = openai.batches.create(
endpoint="endpoint", # ex: /v1/chat/completions
completion_window="completion_window", # ex: 24h
metadata={}, # metadata for the batch
extra_body={
"input_blob": "<blob_url>",
"output_folder": {
"url": "<output_blob_folder>" # both error file and output file will be saved in this folder
}
}
)
print(start_batch_response)
List Batch Jobs
curl --location 'https://aigw.portkey.ai/v1/batches' \
--header 'Authorization: Bearer <portkey_api_key>' \
--header 'x-portkey-provider: @provider'
import OpenAI from 'openai'; // We're using the v4 SDK
const openai = new OpenAI({
apiKey: "PORTKEY_API_KEY", // defaults to process.env["PORTKEY_API_KEY"]
baseURL: "https://aigw.portkey.ai/v1",
defaultHeaders: {
"x-portkey-provider": "openai",
}
});
const listBatches = async () => {
const batches = await openai.batches.list();
console.log(batches);
}
await listBatches();
from openai import OpenAI
openai = OpenAI(
api_key="PORTKEY_API_KEY",
base_url="https://aigw.portkey.ai/v1",
default_headers={"x-portkey-provider": "openai"}
)
batches = openai.batches.list()
print(batches)
Get Batch Job Details
curl --location 'https://aigw.portkey.ai/v1/batches/<batch_id>' \
--header 'Authorization: Bearer <portkey_api_key>' \
--header 'x-portkey-provider: @provider'
import OpenAI from 'openai'; // We're using the v4 SDK
const openai = new OpenAI({
apiKey: "PORTKEY_API_KEY", // defaults to process.env["PORTKEY_API_KEY"]
baseURL: "https://aigw.portkey.ai/v1",
defaultHeaders: {
"x-portkey-provider": "openai",
}
});
const getBatch = async () => {
const batch = await openai.batches.retrieve(batch_id="batch_id");
console.log(batch);
}
await getBatch();
from openai import OpenAI
openai = OpenAI(
api_key="PORTKEY_API_KEY",
base_url="https://aigw.portkey.ai/v1",
default_headers={"x-portkey-provider": "openai"}
)
batch = openai.batches.retrieve(batch_id="batch_id")
print(batch)
Get Batch Output
curl --location 'https://aigw.portkey.ai/v1/batches/<batch_id>/output' \
--header 'Authorization: Bearer <portkey_api_key>' \
--header 'x-portkey-provider: @provider'
List Batch Jobs
curl --location 'https://aigw.portkey.ai/v1/batches' \
--header 'Authorization: Bearer <portkey_api_key>' \
--header 'x-portkey-provider: @provider'
import OpenAI from 'openai'; // We're using the v4 SDK
const openai = new OpenAI({
apiKey: "PORTKEY_API_KEY", // defaults to process.env["PORTKEY_API_KEY"]
baseURL: "https://aigw.portkey.ai/v1",
defaultHeaders: {
"x-portkey-provider": "openai",
}
});
const listBatchingJobs = async () => {
const batching_jobs = await openai.batches.list();
console.log(batching_jobs);
}
await listBatchingJobs();
from openai import OpenAI
openai = OpenAI(
api_key="PORTKEY_API_KEY",
base_url="https://aigw.portkey.ai/v1",
default_headers={"x-portkey-provider": "openai"}
)
batching_jobs = openai.batches.list()
print(batching_jobs)
Cancel Batch Job
curl --request POST --location 'https://aigw.portkey.ai/v1/batches/<batch_id>/cancel' \
--header 'Authorization: Bearer <portkey_api_key>' \
--header 'x-portkey-provider: @provider'
import OpenAI from 'openai'; // We're using the v4 SDK
const openai = new OpenAI({
apiKey: "PORTKEY_API_KEY", // defaults to process.env["PORTKEY_API_KEY"]
baseURL: "https://aigw.portkey.ai/v1",
defaultHeaders: {
"x-portkey-provider": "openai",
}
});
const cancelBatch = async () => {
const cancel_batch_response = await openai.batches.cancel(batch_id="batch_id");
console.log(cancel_batch_response);
}
await cancelBatch();
from openai import OpenAI
openai = OpenAI(
api_key="PORTKEY_API_KEY",
base_url="https://aigw.portkey.ai/v1",
default_headers={"x-portkey-provider": "openai"}
)
cancel_batch_response = openai.batches.cancel(batch_id="batch_id")
print(cancel_batch_response)