> ## Documentation Index
> Fetch the complete documentation index at: https://docs.portkey.ai/docs/llms.txt
> Use this file to discover all available pages before exploring further.

# Batches

> Perform batch inference with Azure OpenAI

With Portkey, you can perform [Azure OpenAI Batch Inference](https://learn.microsoft.com/en-us/azure/ai-services/openai/how-to/batch?tabs=global-batch%2Cstandard-input%2Cpython-secure\&pivots=rest-api) operations.
This is the most efficient way to

* Test your data with different foundation models
* Perform A/B testing with different foundation models
* Perform batch inference with different foundation models

<Info>
  Portkey supports **two modes** on Azure OpenAI:

  * **[Provider Batch API](#using-azure-openai-batch-api-through-portkey)** (cheapest, completion\_window: `24h`, `48h`, etc.)
  * **[Portkey Batch API](/product/ai-gateway/batches#portkey-batch-api-mode)** (fast, provider-agnostic, completion\_window: `immediate`)
</Info>

## Using Azure OpenAI Batch API through Portkey

### Create Batch Job

<CodeGroup>
  ```python Python theme={"system"}
  from portkey_ai import Portkey

  # Initialize the Portkey client
  portkey = Portkey(
      api_key="PORTKEY_API_KEY",  # Replace with your Portkey API key
      provider="@PROVIDER"   
  )

  start_batch_response = portkey.batches.create(
    input_file_id="file_id", # file id of the input file
    endpoint="endpoint", # ex: /v1/chat/completions
    completion_window="completion_window", # ex: 24h
    metadata={} # metadata for the batch
  )

  print(start_batch_response)
  ```

  ```javascript Typescript theme={"system"}
  import { Portkey } from 'portkey-ai';

  // Initialize the Portkey client
  const portkey = new Portkey({
      apiKey: "PORTKEY_API_KEY",  // Replace with your Portkey API key
      provider:"@PROVIDER"   
  });

  const startBatch = async () => {
    const startBatchResponse = await portkey.batches.create({
      input_file_id: "file_id", // file id of the input file
      endpoint: "endpoint", // ex: /v1/chat/completions
      completion_window: "completion_window", // ex: 24h
      metadata: {} // metadata for the batch
    });

    console.log(startBatchResponse);
  }

  await startBatch();

  ```

  ```bash curl theme={"system"}
  curl --location 'https://api.portkey.ai/v1/batches' \
  --header 'x-portkey-api-key: <portkey_api_key>' \
  --header 'x-portkey-provider: @provider' \
  --header 'Content-Type: application/json' \
  --data '{
      "input_file_id": "<file_id>",
      "endpoint": "<endpoint>",
      "completion_window": "<completion_window>",
      "metadata": {}
  }'
  ```

  ```javascript OpenAI NodeJS theme={"system"}
  import OpenAI from 'openai'; // We're using the v4 SDK
  import { PORTKEY_GATEWAY_URL, createHeaders } from 'portkey-ai'

  const openai = new OpenAI({
    apiKey: 'OPENAI_API_KEY', // defaults to process.env["OPENAI_API_KEY"],
    baseURL: PORTKEY_GATEWAY_URL,
    defaultHeaders: createHeaders({
      provider: "openai",
      apiKey: "PORTKEY_API_KEY" // defaults to process.env["PORTKEY_API_KEY"]
    })
  });

  const startBatch = async () => {
    const startBatchResponse = await openai.batches.create({
      input_file_id: "file_id", // file id of the input file
      endpoint: "endpoint", // ex: /v1/chat/completions
      completion_window: "completion_window", // ex: 24h
      metadata: {} // metadata for the batch
    });

    console.log(startBatchResponse);
  }

  await startBatch();
  ```

  ```python OpenAI Python theme={"system"}
  from openai import OpenAI
  from portkey_ai import PORTKEY_GATEWAY_URL, createHeaders

  openai = OpenAI(
      api_key='OPENAI_API_KEY',
      base_url=PORTKEY_GATEWAY_URL,
      default_headers=createHeaders(
          provider="openai",
          api_key="PORTKEY_API_KEY"
      )
  )

  start_batch_response = openai.batches.create(
    input_file_id="file_id", # file id of the input file
    endpoint="endpoint", # ex: /v1/chat/completions
    completion_window="completion_window", # ex: 24h
    metadata={} # metadata for the batch
  )

  print(start_batch_response)
  ```
</CodeGroup>

### Create Batch Job with Blob Storage

<CodeGroup>
  ```python Python theme={"system"}
  from portkey_ai import Portkey

  # Initialize the Portkey client
  portkey = Portkey(
      api_key="PORTKEY_API_KEY",  # Replace with your Portkey API key
      provider="@PROVIDER"   
  )

  start_batch_response = portkey.batches.create(
    endpoint="endpoint",  # ex: /v1/chat/completions
    completion_window="completion_window",  # ex: 24h
    metadata={},  # metadata for the batch
    input_blob="<blob_url>",
    output_folder={
      "url": "<output_blob_folder>" # both error file and output file will be saved in this folder
    }
  )

  print(start_batch_response)
  ```

  ```javascript Typescript theme={"system"}
  import { Portkey } from 'portkey-ai';

  // Initialize the Portkey client
  const portkey = new Portkey({
      apiKey: "PORTKEY_API_KEY",  // Replace with your Portkey API key
      provider:"@PROVIDER"   
  });

  const startBatch = async () => {
    const startBatchResponse = await portkey.batches.create({
      endpoint: "endpoint", // ex: /v1/chat/completions
      completion_window: "completion_window", // ex: 24h
      metadata: {}, // metadata for the batch
      input_blob: "<blob_url>",
      output_folder: {
        url: "<output_blob_folder>" // both error file and output file will be saved in this folder
      }
    });

    console.log(startBatchResponse);
  }

  await startBatch();
  ```

  ```bash curl theme={"system"}
  curl --location 'https://api.portkey.ai/v1/batches' \
  --header 'x-portkey-api-key: <portkey_api_key>' \
  --header 'x-portkey-provider: @provider' \
  --header 'Content-Type: application/json' \
  --data '{
      "endpoint": "<endpoint>",
      "completion_window": "<completion_window>",
      "metadata": {},
      "input_blob": "<blob_url>",
      "output_folder": {
        "url": "<output_blob_folder>" # both error file and output file will be saved in this folder
      }
  }'
  ```

  ```javascript OpenAI NodeJS theme={"system"}
  import OpenAI from 'openai'; // We're using the v4 SDK
  import { PORTKEY_GATEWAY_URL, createHeaders } from 'portkey-ai'

  const openai = new OpenAI({
    apiKey: 'OPENAI_API_KEY', // defaults to process.env["OPENAI_API_KEY"],
    baseURL: PORTKEY_GATEWAY_URL,
    defaultHeaders: createHeaders({
      provider: "openai",
      apiKey: "PORTKEY_API_KEY" // defaults to process.env["PORTKEY_API_KEY"]
    })
  });

  const startBatch = async () => {
    const startBatchResponse = await openai.batches.create({
      endpoint: "endpoint", // ex: /v1/chat/completions
      completion_window: "completion_window", // ex: 24h
      metadata: {}, // metadata for the batch
      extra_body: {
        input_blob: "<blob_url>",
        output_folder: {
          url: "<output_blob_folder>" // both error file and output file will be saved in this folder
        }
      }
    });

    console.log(startBatchResponse);
  }

  await startBatch();
  ```

  ```python OpenAI Python theme={"system"}
  from openai import OpenAI
  from portkey_ai import PORTKEY_GATEWAY_URL, createHeaders

  openai = OpenAI(
      api_key='OPENAI_API_KEY',
      base_url=PORTKEY_GATEWAY_URL,
      default_headers=createHeaders(
          provider="openai",
          api_key="PORTKEY_API_KEY"
      )
  )

  start_batch_response = openai.batches.create(
    endpoint="endpoint",  # ex: /v1/chat/completions
    completion_window="completion_window",  # ex: 24h
    metadata={},  # metadata for the batch
    extra_body={
      "input_blob": "<blob_url>",
      "output_folder": {
        "url": "<output_blob_folder>"  # both error file and output file will be saved in this folder
      }
    }
  )

  print(start_batch_response)
  ```
</CodeGroup>

### List Batch Jobs

<CodeGroup>
  ```python Python theme={"system"}
  from portkey_ai import Portkey

  # Initialize the Portkey client
  portkey = Portkey(
      api_key="PORTKEY_API_KEY",  # Replace with your Portkey API key
      provider="@PROVIDER"   
  )

  batches = portkey.batches.list()

  print(batches)
  ```

  ```javascript Typescript theme={"system"}
  import { Portkey } from 'portkey-ai';

  // Initialize the Portkey client
  const portkey = new Portkey({
      apiKey: "PORTKEY_API_KEY",  // Replace with your Portkey API key
      provider:"@PROVIDER"   
  });

  const listBatches = async () => {
    const batches = await portkey.batches.list();

    console.log(batches);
  }

  await listBatches();

  ```

  ```bash curl theme={"system"}
  curl --location 'https://api.portkey.ai/v1/batches' \
  --header 'x-portkey-api-key: <portkey_api_key>' \
  --header 'x-portkey-provider: @provider'
  ```

  ```javascript OpenAI NodeJS theme={"system"}
  import OpenAI from 'openai'; // We're using the v4 SDK
  import { PORTKEY_GATEWAY_URL, createHeaders } from 'portkey-ai'

  const openai = new OpenAI({
    apiKey: 'OPENAI_API_KEY', // defaults to process.env["OPENAI_API_KEY"],
    baseURL: PORTKEY_GATEWAY_URL,
    defaultHeaders: createHeaders({
      provider: "openai",
      apiKey: "PORTKEY_API_KEY" // defaults to process.env["PORTKEY_API_KEY"]
    })
  });

  const listBatches = async () => {
    const batches = await openai.batches.list();

    console.log(batches);
  }

  await listBatches();
  ```

  ```python OpenAI Python theme={"system"}
  from openai import OpenAI
  from portkey_ai import PORTKEY_GATEWAY_URL, createHeaders

  openai = OpenAI(
      api_key='OPENAI_API_KEY',
      base_url=PORTKEY_GATEWAY_URL,
      default_headers=createHeaders(
          provider="openai",
          api_key="PORTKEY_API_KEY"
      )
  )

  batches = openai.batches.list()

  print(batches)
  ```
</CodeGroup>

### Get Batch Job Details

<CodeGroup>
  ```python Python theme={"system"}
  from portkey_ai import Portkey

  # Initialize the Portkey client
  portkey = Portkey(
      api_key="PORTKEY_API_KEY",  # Replace with your Portkey API key
      provider="@PROVIDER"   
  )

  batch = portkey.batches.retrieve(batch_id="batch_id")

  print(batch)
  ```

  ```javascript Typescript theme={"system"}
  import { Portkey } from 'portkey-ai';

  // Initialize the Portkey client
  const portkey = new Portkey({
      apiKey: "PORTKEY_API_KEY",  // Replace with your Portkey API key
      provider:"@PROVIDER"   
  });

  const getBatch = async () => {
    const batch = await portkey.batches.retrieve(batch_id="batch_id");

    console.log(batch);
  }

  await getBatch();

  ```

  ```bash curl theme={"system"}
  curl --location 'https://api.portkey.ai/v1/batches/<batch_id>' \
  --header 'x-portkey-api-key: <portkey_api_key>' \
  --header 'x-portkey-provider: @provider'
  ```

  ```javascript OpenAI NodeJS theme={"system"}
  import OpenAI from 'openai'; // We're using the v4 SDK
  import { PORTKEY_GATEWAY_URL, createHeaders } from 'portkey-ai'

  const openai = new OpenAI({
    apiKey: 'OPENAI_API_KEY', // defaults to process.env["OPENAI_API_KEY"],
    baseURL: PORTKEY_GATEWAY_URL,
    defaultHeaders: createHeaders({
      provider: "openai",
      apiKey: "PORTKEY_API_KEY" // defaults to process.env["PORTKEY_API_KEY"]
    })
  });

  const getBatch = async () => {
    const batch = await openai.batches.retrieve(batch_id="batch_id");

    console.log(batch);
  }

  await getBatch();
  ```

  ```python OpenAI Python theme={"system"}
  from openai import OpenAI
  from portkey_ai import PORTKEY_GATEWAY_URL, createHeaders

  openai = OpenAI(
      api_key='OPENAI_API_KEY',
      base_url=PORTKEY_GATEWAY_URL,
      default_headers=createHeaders(
          provider="openai",
          api_key="PORTKEY_API_KEY"
      )
  )

  batch = openai.batches.retrieve(batch_id="batch_id")

  print(batch)
  ```
</CodeGroup>

### Get Batch Output

<CodeGroup>
  ```bash curl theme={"system"}
  curl --location 'https://api.portkey.ai/v1/batches/<batch_id>/output' \
  --header 'x-portkey-api-key: <portkey_api_key>' \
  --header 'x-portkey-provider: @provider'
  ```
</CodeGroup>

### List Batch Jobs

<CodeGroup>
  ```python Python theme={"system"}
  from portkey_ai import Portkey

  # Initialize the Portkey client
  portkey = Portkey(
      api_key="PORTKEY_API_KEY",  # Replace with your Portkey API key
      provider="@PROVIDER"   
  )

  batches = portkey.batches.list()

  print(batches)
  ```

  ```javascript Typescript theme={"system"}
  import { Portkey } from 'portkey-ai';

  // Initialize the Portkey client
  const portkey = new Portkey({
      apiKey: "PORTKEY_API_KEY",  // Replace with your Portkey API key
      provider:"@PROVIDER"   
  });

  const listBatchingJobs = async () => {
    const batching_jobs = await portkey.batches.list();

    console.log(batching_jobs);
  }

  await listBatchingJobs();

  ```

  ```bash curl theme={"system"}
  curl --location 'https://api.portkey.ai/v1/batches' \
  --header 'x-portkey-api-key: <portkey_api_key>' \
  --header 'x-portkey-provider: @provider'
  ```

  ```javascript OpenAI NodeJS theme={"system"}
  import OpenAI from 'openai'; // We're using the v4 SDK
  import { PORTKEY_GATEWAY_URL, createHeaders } from 'portkey-ai'

  const openai = new OpenAI({
    apiKey: 'OPENAI_API_KEY', // defaults to process.env["OPENAI_API_KEY"],
    baseURL: PORTKEY_GATEWAY_URL,
    defaultHeaders: createHeaders({
      provider: "openai",
      apiKey: "PORTKEY_API_KEY" // defaults to process.env["PORTKEY_API_KEY"]
    })
  });

  const listBatchingJobs = async () => {
    const batching_jobs = await openai.batches.list();

    console.log(batching_jobs);
  }

  await listBatchingJobs();
  ```

  ```python OpenAI Python theme={"system"}
  from openai import OpenAI
  from portkey_ai import PORTKEY_GATEWAY_URL, createHeaders

  openai = OpenAI(
      api_key='OPENAI_API_KEY',
      base_url=PORTKEY_GATEWAY_URL,
      default_headers=createHeaders(
          provider="openai",
          api_key="PORTKEY_API_KEY"
      )
  )

  batching_jobs = openai.batches.list()

  print(batching_jobs)
  ```
</CodeGroup>

### Cancel Batch Job

<CodeGroup>
  ```python Python theme={"system"}
  from portkey_ai import Portkey

  # Initialize the Portkey client
  portkey = Portkey(
      api_key="PORTKEY_API_KEY",  # Replace with your Portkey API key
      provider="@PROVIDER"   
  )

  cancel_batch_response = portkey.batches.cancel(batch_id="batch_id")

  print(cancel_batch_response)
  ```

  ```javascript Typescript theme={"system"}
  import { Portkey } from 'portkey-ai';

  // Initialize the Portkey client
  const portkey = new Portkey({
      apiKey: "PORTKEY_API_KEY",  // Replace with your Portkey API key
      provider:"@PROVIDER"   
  });

  const cancelBatch = async () => {
    const cancel_batch_response = await portkey.batches.cancel(batch_id="batch_id");

    console.log(cancel_batch_response);
  }

  await cancelBatch();

  ```

  ```bash curl theme={"system"}
  curl --request POST --location 'https://api.portkey.ai/v1/batches/<batch_id>/cancel' \
  --header 'x-portkey-api-key: <portkey_api_key>' \
  --header 'x-portkey-provider: @provider'
  ```

  ```javascript OpenAI NodeJS theme={"system"}
  import OpenAI from 'openai'; // We're using the v4 SDK
  import { PORTKEY_GATEWAY_URL, createHeaders } from 'portkey-ai'

  const openai = new OpenAI({
    apiKey: 'OPENAI_API_KEY', // defaults to process.env["OPENAI_API_KEY"],
    baseURL: PORTKEY_GATEWAY_URL,
    defaultHeaders: createHeaders({
      provider: "openai",
      apiKey: "PORTKEY_API_KEY" // defaults to process.env["PORTKEY_API_KEY"]
    })
  });

  const cancelBatch = async () => {
    const cancel_batch_response = await openai.batches.cancel(batch_id="batch_id");

    console.log(cancel_batch_response);
  }

  await cancelBatch();
  ```

  ```python OpenAI Python theme={"system"}
  from openai import OpenAI
  from portkey_ai import PORTKEY_GATEWAY_URL, createHeaders

  openai = OpenAI(
      api_key='OPENAI_API_KEY',
      base_url=PORTKEY_GATEWAY_URL,
      default_headers=createHeaders(
          provider="openai",
          api_key="PORTKEY_API_KEY"
      )
  )

  cancel_batch_response = openai.batches.cancel(batch_id="batch_id")

  print(cancel_batch_response)
  ```
</CodeGroup>