Google Vertex AI
Batches
Ecosystem
LLMs
- Overview
- OpenAI
- Anthropic
- Google Gemini
- Google Vertex AI
- Azure OpenAI
- Bedrock
- AWS SageMaker
- Ollama
- More
- Bring Your Own LLM
Agents
Batches
Perform batch inference with Vertex AI
With Portkey, you can perform batch inference operations with Vertex AI models. This is the most efficient way to:
- Process large volumes of data with Vertex AI models
- Test your data with different foundation models
- Perform A/B testing with different foundation models
Upload a file for batch inference
from portkey_ai import Portkey
# Initialize the Portkey client
portkey = Portkey(
api_key="PORTKEY_API_KEY", # Replace with your Portkey API key
virtual_key="VERTEX_VIRTUAL_KEY", # Add your Vertex virtual key
vertex_storage_bucket_name="your_bucket_name", # Specify the GCS bucket name
provider_file_name="your_file_name.jsonl", # Specify the file name in GCS
provider_model="gemini-1.5-flash-001" # Specify the model to use
)
# Upload a file for batch inference
file = portkey.files.create(
file=open("dataset.jsonl", "rb"),
purpose="batch"
)
print(file)
from portkey_ai import Portkey
# Initialize the Portkey client
portkey = Portkey(
api_key="PORTKEY_API_KEY", # Replace with your Portkey API key
virtual_key="VERTEX_VIRTUAL_KEY", # Add your Vertex virtual key
vertex_storage_bucket_name="your_bucket_name", # Specify the GCS bucket name
provider_file_name="your_file_name.jsonl", # Specify the file name in GCS
provider_model="gemini-1.5-flash-001" # Specify the model to use
)
# Upload a file for batch inference
file = portkey.files.create(
file=open("dataset.jsonl", "rb"),
purpose="batch"
)
print(file)
import { Portkey } from "portkey-ai";
import * as fs from 'fs';
// Initialize the Portkey client
const portkey = Portkey({
apiKey: "PORTKEY_API_KEY", // Replace with your Portkey API key
virtualKey: "VERTEX_VIRTUAL_KEY", // Add your Vertex virtual key
vertexStorageBucketName: "your_bucket_name", // Specify the GCS bucket name
providerFileName: "your_file_name.jsonl", // Specify the file name in GCS
providerModel: "gemini-1.5-flash-001" // Specify the model to use
});
(async () => {
// Upload a file for batch inference
const file = await portkey.files.create({
file: fs.createReadStream("dataset.jsonl"),
purpose: "batch"
});
console.log(file);
})();
from openai import OpenAI
from portkey_ai import PORTKEY_GATEWAY_URL, createHeaders
openai = OpenAI(
api_key='OPENAI_API_KEY',
base_url=PORTKEY_GATEWAY_URL,
default_headers=createHeaders(
virtual_key="VERTEX_VIRTUAL_KEY",
api_key="PORTKEY_API_KEY",
vertex_storage_bucket_name="your_bucket_name",
provider_file_name="your_file_name.jsonl",
provider_model="gemini-1.5-flash-001"
)
)
# Upload a file for batch inference
file = openai.files.create(
file=open("dataset.jsonl", "rb"),
purpose="batch"
)
print(file)
import OpenAI from 'openai';
import { PORTKEY_GATEWAY_URL, createHeaders } from 'portkey-ai';
import * as fs from 'fs';
const openai = new OpenAI({
apiKey: 'OPENAI_API_KEY',
baseURL: PORTKEY_GATEWAY_URL,
defaultHeaders: createHeaders({
virtualKey: "VERTEX_VIRTUAL_KEY",
apiKey: "PORTKEY_API_KEY",
vertexStorageBucketName: "your_bucket_name",
providerFileName: "your_file_name.jsonl",
providerModel: "gemini-1.5-flash-001"
})
});
(async () => {
// Upload a file for batch inference
const file = await openai.files.create({
file: fs.createReadStream("dataset.jsonl"),
purpose: "batch"
});
console.log(file);
})();
curl -X POST --header 'x-portkey-api-key: <portkey_api_key>' \
--header 'x-portkey-virtual-key: <vertex_virtual_key>' \
--header 'x-portkey-vertex-storage-bucket-name: <bucket_name>' \
--header 'x-portkey-provider-file-name: <file_name>.jsonl' \
--header 'x-portkey-provider-model: <model_name>' \
--form 'purpose="batch"' \
--form '[email protected]' \
'https://api.portkey.ai/v1/files'
Create a batch job
from portkey_ai import Portkey
# Initialize the Portkey client
portkey = Portkey(
api_key="PORTKEY_API_KEY", # Replace with your Portkey API key
virtual_key="VERTEX_VIRTUAL_KEY" # Add your Vertex virtual key
)
# Create a batch inference job
batch_job = portkey.batches.create(
input_file_id="<file_id>", # File ID from the upload step
endpoint="/v1/chat/completions", # API endpoint to use
completion_window="24h", # Time window for completion
model="gemini-1.5-flash-001"
)
print(batch_job)
from portkey_ai import Portkey
# Initialize the Portkey client
portkey = Portkey(
api_key="PORTKEY_API_KEY", # Replace with your Portkey API key
virtual_key="VERTEX_VIRTUAL_KEY" # Add your Vertex virtual key
)
# Create a batch inference job
batch_job = portkey.batches.create(
input_file_id="<file_id>", # File ID from the upload step
endpoint="/v1/chat/completions", # API endpoint to use
completion_window="24h", # Time window for completion
model="gemini-1.5-flash-001"
)
print(batch_job)
import { Portkey } from "portkey-ai";
// Initialize the Portkey client
const portkey = Portkey({
apiKey: "PORTKEY_API_KEY", // Replace with your Portkey API key
virtualKey: "VERTEX_VIRTUAL_KEY" // Add your Vertex virtual key
});
(async () => {
// Create a batch inference job
const batchJob = await portkey.batches.create({
input_file_id: "<file_id>", // File ID from the upload step
endpoint: "/v1/chat/completions", // API endpoint to use
completion_window: "24h", // Time window for completion
model:"gemini-1.5-flash-001"
});
console.log(batchJob);
})();
from openai import OpenAI
from portkey_ai import PORTKEY_GATEWAY_URL, createHeaders
openai = OpenAI(
api_key='OPENAI_API_KEY',
base_url=PORTKEY_GATEWAY_URL,
default_headers=createHeaders(
virtual_key="VERTEX_VIRTUAL_KEY",
api_key="PORTKEY_API_KEY"
)
)
# Create a batch inference job
batch_job = openai.batches.create(
input_file_id="<file_id>", # File ID from the upload step
endpoint="/v1/chat/completions", # API endpoint to use
completion_window="24h", # Time window for completion
model="gemini-1.5-flash-001"
)
print(batch_job)
import OpenAI from 'openai';
import { PORTKEY_GATEWAY_URL, createHeaders } from 'portkey-ai';
const openai = new OpenAI({
apiKey: 'OPENAI_API_KEY',
baseURL: PORTKEY_GATEWAY_URL,
defaultHeaders: createHeaders({
virtualKey: "VERTEX_VIRTUAL_KEY",
apiKey: "PORTKEY_API_KEY"
})
});
(async () => {
// Create a batch inference job
const batchJob = await openai.batches.create({
input_file_id: "<file_id>", // File ID from the upload step
endpoint: "/v1/chat/completions", // API endpoint to use
completion_window: "24h", // Time window for completion
model:"gemini-1.5-flash-001"
});
console.log(batchJob);
})();
curl -X POST --header 'Content-Type: application/json' \
--header 'x-portkey-api-key: <portkey_api_key>' \
--header 'x-portkey-virtual-key: <vertex_virtual_key>' \
--data \
$'{"input_file_id": "<file_id>", "endpoint": "/v1/chat/completions", "completion_window": "24h", "model":"gemini-1.5-flash-001"}' \
'https://api.portkey.ai/v1/batches'
List batch jobs
from portkey_ai import Portkey
# Initialize the Portkey client
portkey = Portkey(
api_key="PORTKEY_API_KEY", # Replace with your Portkey API key
virtual_key="VERTEX_VIRTUAL_KEY" # Add your Vertex virtual key
)
# List all batch jobs
jobs = portkey.batches.list(
limit=10 # Optional: Number of jobs to retrieve (default: 20)
)
print(jobs)
from portkey_ai import Portkey
# Initialize the Portkey client
portkey = Portkey(
api_key="PORTKEY_API_KEY", # Replace with your Portkey API key
virtual_key="VERTEX_VIRTUAL_KEY" # Add your Vertex virtual key
)
# List all batch jobs
jobs = portkey.batches.list(
limit=10 # Optional: Number of jobs to retrieve (default: 20)
)
print(jobs)
import { Portkey } from "portkey-ai";
// Initialize the Portkey client
const portkey = Portkey({
apiKey: "PORTKEY_API_KEY", // Replace with your Portkey API key
virtualKey: "VERTEX_VIRTUAL_KEY" // Add your Vertex virtual key
});
(async () => {
// List all batch jobs
const jobs = await portkey.batches.list({
limit: 10 // Optional: Number of jobs to retrieve (default: 20)
});
console.log(jobs);
})();
from openai import OpenAI
from portkey_ai import PORTKEY_GATEWAY_URL, createHeaders
openai = OpenAI(
api_key='OPENAI_API_KEY',
base_url=PORTKEY_GATEWAY_URL,
default_headers=createHeaders(
virtual_key="VERTEX_VIRTUAL_KEY",
api_key="PORTKEY_API_KEY"
)
)
# List all batch jobs
jobs = openai.batches.list(
limit=10 # Optional: Number of jobs to retrieve (default: 20)
)
print(jobs)
import OpenAI from 'openai';
import { PORTKEY_GATEWAY_URL, createHeaders } from 'portkey-ai';
const openai = new OpenAI({
apiKey: 'OPENAI_API_KEY',
baseURL: PORTKEY_GATEWAY_URL,
defaultHeaders: createHeaders({
virtualKey: "VERTEX_VIRTUAL_KEY",
apiKey: "PORTKEY_API_KEY"
})
});
(async () => {
// List all batch jobs
const jobs = await openai.batches.list({
limit: 10 // Optional:
});
console.log(jobs);
})();
curl -X GET --header 'x-portkey-api-key: <portkey_api_key>' \
--header 'x-portkey-virtual-key: <vertex_virtual_key>' \
'https://api.portkey.ai/v1/batches'
Get a batch job
from portkey_ai import Portkey
# Initialize the Portkey client
portkey = Portkey(
api_key="PORTKEY_API_KEY", # Replace with your Portkey API key
virtual_key="VERTEX_VIRTUAL_KEY" # Add your Vertex virtual key
)
# Retrieve a specific batch job
job = portkey.batches.retrieve(
"job_id" # The ID of the batch job to retrieve
)
print(job)
from portkey_ai import Portkey
# Initialize the Portkey client
portkey = Portkey(
api_key="PORTKEY_API_KEY", # Replace with your Portkey API key
virtual_key="VERTEX_VIRTUAL_KEY" # Add your Vertex virtual key
)
# Retrieve a specific batch job
job = portkey.batches.retrieve(
"job_id" # The ID of the batch job to retrieve
)
print(job)
import { Portkey } from "portkey-ai";
// Initialize the Portkey client
const portkey = Portkey({
apiKey: "PORTKEY_API_KEY", // Replace with your Portkey API key
virtualKey: "VERTEX_VIRTUAL_KEY" // Add your Vertex virtual key
});
(async () => {
// Retrieve a specific batch job
const job = await portkey.batches.retrieve(
"job_id" // The ID of the batch job to retrieve
);
console.log(job);
})();
from openai import OpenAI
from portkey_ai import PORTKEY_GATEWAY_URL, createHeaders
openai = OpenAI(
api_key='OPENAI_API_KEY',
base_url=PORTKEY_GATEWAY_URL,
default_headers=createHeaders(
virtual_key="VERTEX_VIRTUAL_KEY",
api_key="PORTKEY_API_KEY"
)
)
# Retrieve a specific batch job
job = openai.batches.retrieve(
"job_id" # The ID of the batch job to retrieve
)
print(job)
import OpenAI from 'openai';
import { PORTKEY_GATEWAY_URL, createHeaders } from 'portkey-ai';
const openai = new OpenAI({
apiKey: 'OPENAI_API_KEY',
baseURL: PORTKEY_GATEWAY_URL,
defaultHeaders: createHeaders({
virtualKey: "VERTEX_VIRTUAL_KEY",
apiKey: "PORTKEY_API_KEY"
})
});
(async () => {
// Retrieve a specific batch job
const job = await openai.batches.retrieve(
"job_id" // The ID of the batch job to retrieve
);
console.log(job);
})();
curl -X GET --header 'x-portkey-api-key: <portkey_api_key>' \
--header 'x-portkey-virtual-key: <vertex_virtual_key>' \
'https://api.portkey.ai/v1/batches/<job_id>'
Get batch job output
curl -X GET --header 'x-portkey-api-key: <portkey_api_key>' \
--header 'x-portkey-virtual-key: <vertex_virtual_key>' \
'https://api.portkey.ai/v1/batches/<job_id>/output'
curl -X GET --header 'x-portkey-api-key: <portkey_api_key>' \
--header 'x-portkey-virtual-key: <vertex_virtual_key>' \
'https://api.portkey.ai/v1/batches/<job_id>/output'
Was this page helpful?