> ## Documentation Index
> Fetch the complete documentation index at: https://docs.portkey.ai/docs/llms.txt
> Use this file to discover all available pages before exploring further.

# Batches

> Perform batch inference with Bedrock

To perform batch inference with Bedrock, you need to upload files to S3.
This process can be cumbersome and duplicative in nature because you need to transform your data into model specific formats.

With Portkey, you can upload the file in [OpenAI format](https://platform.openai.com/docs/guides/batch#1-preparing-your-batch-file) and portkey will handle transforming the file into the format required by Bedrock on the fly!

This is the most efficient way to

* Test your data with different foundation models
* Perform A/B testing with different foundation models
* Perform batch inference with different foundation models

## Create Batch Job

<Tabs>
  <Tab title="Python">
    ```python theme={"system"}
    from portkey_ai import Portkey

    # Initialize the Portkey client
    portkey = Portkey(
        api_key="PORTKEY_API_KEY",  # Replace with your Portkey API key
        provider="bedrock",
        aws_access_key_id="YOUR_AWS_ACCESS_KEY_ID",
        aws_secret_access_key="YOUR_AWS_SECRET_ACCESS_KEY",
        aws_region="YOUR_AWS_REGION",
        aws_s3_bucket="YOUR_AWS_S3_BUCKET",
        aws_s3_object_key="YOUR_AWS_S3_OBJECT_KEY",
        aws_bedrock_model="YOUR_AWS_BEDROCK_MODEL"
    )

    start_batch_response = portkey.batches.create(
      input_file_id="file_id", # file id of the input file
      endpoint="endpoint", # ex: /v1/chat/completions
      completion_window="completion_window", # ex: 24h
      metadata={}, # metadata for the batch,
      role_arn="arn:aws:iam::12312:role/BedrockBatchRole", # the role to use for creating the batch job
      model="anthropic.claude-3-5-sonnet-20240620-v1:0", # the model to use for the batch
      output_data_config={
        "s3OutputDataConfig": {
          "s3Uri": "s3://generations-raw/",
          "s3EncryptionKeyId": "arn:aws:kms:us-west-2:517194595696:key/89b483cb-130d-497b-aa37-7db177e7cd32" # this is optional, if you want to use a KMS key to encrypt the output data
        }
      }, # output_data_config is optional, if you want to specify a different output location for the batch job, default is the same as the input file
      job_name="anthropi-requests-test" # optional
    )

    print(start_batch_response)
    ```
  </Tab>

  <Tab title="NodeJS">
    ```js theme={"system"}
    import { Portkey } from 'portkey-ai';

    // Initialize the Portkey client
    const portkey = new Portkey({
        apiKey: "PORTKEY_API_KEY",  // Replace with your Portkey API key
        provider="bedrock",
        awsAccessKeyId="YOUR_AWS_ACCESS_KEY_ID",
        awsSecretAccessKey="YOUR_AWS_SECRET_ACCESS_KEY",
        awsRegion="YOUR_AWS_REGION",
        awsS3Bucket="YOUR_AWS_S3_BUCKET",
        awsS3ObjectKey="YOUR_AWS_S3_OBJECT_KEY",
        awsBedrockModel="YOUR_AWS_BEDROCK_MODEL"
    });

    const startBatch = async () => {
      const startBatchResponse = await portkey.batches.create({
        input_file_id: "file_id", // file id of the input file
        endpoint: "endpoint", // ex: /v1/chat/completions
        completion_window: "completion_window", // ex: 24h
        metadata: {}, // metadata for the batch
        role_arn: "arn:aws:iam::12312:role/BedrockBatchRole", // the role to use for creating the batch job
        model: "anthropic.claude-3-5-sonnet-20240620-v1:0", // the model to use for the batch
        output_data_config: {
          s3OutputDataConfig: {
            s3Uri: "s3://generations-raw/",
            s3EncryptionKeyId: "arn:aws:kms:us-west-2:517194595696:key/89b483cb-130d-497b-aa37-7db177e7cd32" // this is optional, if you want to use a KMS key to encrypt the output data
          }
        }, // output_data_config is optional, if you want to specify a different output location for the batch job, default is the same as the input file
        job_name: "anthropi-requests-test" // optional
      });

      console.log(startBatchResponse);
    }

    await startBatch();

    ```
  </Tab>

  <Tab title="REST">
    ```sh theme={"system"}
    curl --location 'https://api.portkey.ai/v1/batches' \
    --header 'x-portkey-api-key: <portkey_api_key>' \
    --header 'x-portkey-aws-access-key-id: {YOUR_AWS_ACCESS_KEY_ID}' \
    --header 'x-portkey-aws-secret-access-key: {YOUR_AWS_SECRET_ACCESS_KEY}' \
    --header 'x-portkey-aws-region: {YOUR_AWS_REGION}' \
    --header 'Content-Type: application/json' \
    --data '{
        "model": "meta.llama3-1-8b-instruct-v1:0",
        "input_file_id": "s3%3A%2F%2Fgenerations-raw-west-2%2Fbatch_files%2Fllama2%2Fbatch_chat_completions_101_requests.jsonl",
        "role_arn": "arn:aws:iam::12312:role/BedrockBatchRole", // the role to use for creating the batch job
        "output_data_config": {            // output_data_config is optional, if you want to specify a different output location for the batch job, default is the same as the input file
            "s3OutputDataConfig": {
                "s3Uri": "s3://generations-raw/",
                "s3EncryptionKeyId": "arn:aws:kms:us-west-2:517194595696:key/89b483cb-130d-497b-aa37-7db177e7cd32" // this is optional, if you want to use a KMS key to encrypt the output data
            }
        },
        "job_name": "anthropi-requests" // optional
    }'
    ```
  </Tab>

  <Tab title="OpenAI NodeJS">
    ```js theme={"system"}
    import OpenAI from 'openai'; // We're using the v4 SDK
    import { PORTKEY_GATEWAY_URL, createHeaders } from 'portkey-ai'

    const openai = new OpenAI({
      apiKey: 'PLACEHOLDER_NOT_USED', // defaults to process.env["OPENAI_API_KEY"],
      baseURL: PORTKEY_GATEWAY_URL,
      defaultHeaders: createHeaders({
        provider: "openai",
        apiKey: "PORTKEY_API_KEY", // defaults to process.env["PORTKEY_API_KEY"]
        awsAccessKeyId: "YOUR_AWS_ACCESS_KEY_ID",
        awsSecretAccessKey: "YOUR_AWS_SECRET_ACCESS_KEY",
        awsRegion: "YOUR_AWS_REGION",
        awsS3Bucket: "YOUR_AWS_S3_BUCKET",
        awsS3ObjectKey: "YOUR_AWS_S3_OBJECT_KEY",
        awsBedrockModel: "YOUR_AWS_BEDROCK_MODEL"
      })
    });

    const startBatch = async () => {
      const startBatchResponse = await openai.batches.create({
        input_file_id: "file_id", // file id of the input file
        endpoint: "endpoint", // ex: /v1/chat/completions
        completion_window: "completion_window", // ex: 24h
        metadata: {}, // metadata for the batch
        role_arn: "arn:aws:iam::12312:role/BedrockBatchRole", // the role to use for creating the batch job
        model: "anthropic.claude-3-5-sonnet-20240620-v1:0", // the model to use for the batch
        output_data_config: {
          s3OutputDataConfig: {
            s3Uri: "s3://generations-raw/",
            s3EncryptionKeyId: "arn:aws:kms:us-west-2:517194595696:key/89b483cb-130d-497b-aa37-7db177e7cd32" // this is optional, if you want to use a KMS key to encrypt the output data
          }
        }, // output_data_config is optional, if you want to specify a different output location for the batch job, default is the same as the input file
        job_name: "anthropi-requests-test" // optional
      });

      console.log(startBatchResponse);
    }

    await startBatch();
    ```
  </Tab>

  <Tab title="OpenAI Python">
    ```python theme={"system"}
    from openai import OpenAI
    from portkey_ai import PORTKEY_GATEWAY_URL, createHeaders

    openai = OpenAI(
        api_key='PLACEHOLDER_NOT_USED',
        base_url=PORTKEY_GATEWAY_URL,
        default_headers=createHeaders(
            provider="openai",
            api_key="PORTKEY_API_KEY",
            aws_access_key_id="YOUR_AWS_ACCESS_KEY_ID",
            aws_secret_access_key="YOUR_AWS_SECRET_ACCESS_KEY",
            aws_region="YOUR_AWS_REGION",
            aws_s3_bucket="YOUR_AWS_S3_BUCKET",
            aws_s3_object_key="YOUR_AWS_S3_OBJECT_KEY",
            aws_bedrock_model="YOUR_AWS_BEDROCK_MODEL"
        )
    )

    start_batch_response = openai.batches.create(
      input_file_id="file_id", # file id of the input file
      endpoint="endpoint", # ex: /v1/chat/completions
      completion_window="completion_window", # ex: 24h
      metadata={}, # metadata for the batch
      role_arn="arn:aws:iam::12312:role/BedrockBatchRole", # the role to use for creating the batch job
      model="anthropic.claude-3-5-sonnet-20240620-v1:0", # the model to use for the batch
      output_data_config={
        "s3OutputDataConfig": {
          "s3Uri": "s3://generations-raw/",
          "s3EncryptionKeyId": "arn:aws:kms:us-west-2:517194595696:key/89b483cb-130d-497b-aa37-7db177e7cd32" // this is optional, if you want to use a KMS key to encrypt the output data
        }
      }, # output_data_config is optional, if you want to specify a different output location for the batch job, default is the same as the input file
      job_name="anthropi-requests-test" # optional
    )
    print(start_batch_response)
    ```
  </Tab>
</Tabs>

## List Batch Jobs

<Tabs>
  <Tab title="Python">
    ```python theme={"system"}
    from portkey_ai import Portkey

    # Initialize the Portkey client
    portkey = Portkey(
        api_key="PORTKEY_API_KEY",  # Replace with your Portkey API key
        provider="bedrock",
        aws_access_key_id="YOUR_AWS_ACCESS_KEY_ID",
        aws_secret_access_key="YOUR_AWS_SECRET_ACCESS_KEY",
        aws_region="YOUR_AWS_REGION",
    )

    batches = portkey.batches.list()

    print(batches)
    ```
  </Tab>

  <Tab title="NodeJS">
    ```js theme={"system"}
    import { Portkey } from 'portkey-ai';

    // Initialize the Portkey client
    const portkey = new Portkey({
        apiKey: "PORTKEY_API_KEY",  // Replace with your Portkey API key
        provider="bedrock",
        awsAccessKeyId="YOUR_AWS_ACCESS_KEY_ID",
        awsSecretAccessKey="YOUR_AWS_SECRET_ACCESS_KEY",
        awsRegion="YOUR_AWS_REGION",
    });

    const listBatches = async () => {
      const batches = await portkey.batches.list();

      console.log(batches);
    }

    await listBatches();

    ```
  </Tab>

  <Tab title="REST">
    ```sh theme={"system"}
    curl --location 'https://api.portkey.ai/v1/batches' \
    --header 'x-portkey-api-key: <portkey_api_key>' \
    --header 'x-portkey-virtual-key: <virtual_key>' \
    ```
  </Tab>

  <Tab title="OpenAI NodeJS">
    ```js theme={"system"}
    import OpenAI from 'openai'; // We're using the v4 SDK
    import { PORTKEY_GATEWAY_URL, createHeaders } from 'portkey-ai'

    const openai = new OpenAI({
      apiKey: 'PLACEHOLDER_NOT_USED', // defaults to process.env["OPENAI_API_KEY"],
      baseURL: PORTKEY_GATEWAY_URL,
      defaultHeaders: createHeaders({
        provider: "openai",
        apiKey: "PORTKEY_API_KEY", // defaults to process.env["PORTKEY_API_KEY"]
        awsAccessKeyId: "YOUR_AWS_ACCESS_KEY_ID",
        awsSecretAccessKey: "YOUR_AWS_SECRET_ACCESS_KEY",
        awsRegion: "YOUR_AWS_REGION"
      })
    });

    const listBatches = async () => {
      const batches = await openai.batches.list();

      console.log(batches);
    }

    await listBatches();
    ```
  </Tab>

  <Tab title="OpenAI Python">
    ```python theme={"system"}
    from openai import OpenAI
    from portkey_ai import PORTKEY_GATEWAY_URL, createHeaders

    openai = OpenAI(
        api_key='PLACEHOLDER_NOT_USED',
        base_url=PORTKEY_GATEWAY_URL,
        default_headers=createHeaders(
            provider="openai",
            api_key="PORTKEY_API_KEY",
            aws_access_key_id="YOUR_AWS_ACCESS_KEY_ID",
            aws_secret_access_key="YOUR_AWS_SECRET_ACCESS_KEY",
            aws_region="YOUR_AWS_REGION"
        )
    )

    batches = openai.batches.list()

    print(batches)
    ```
  </Tab>
</Tabs>

## Get Batch Job Details

<Tabs>
  <Tab title="Python">
    ```python theme={"system"}
    from portkey_ai import Portkey

    # Initialize the Portkey client
    portkey = Portkey(
        api_key="PORTKEY_API_KEY",  # Replace with your Portkey API key
        virtual_key="VIRTUAL_KEY",   # Add your provider's virtual key
    )

    batch = portkey.batches.retrieve(batch_id="batch_id")

    print(batch)
    ```
  </Tab>

  <Tab title="NodeJS">
    ```js theme={"system"}
    import { Portkey } from 'portkey-ai';

    // Initialize the Portkey client
    const portkey = new Portkey({
        apiKey: "PORTKEY_API_KEY",  // Replace with your Portkey API key
        virtualKey: "VIRTUAL_KEY",   // Add your provider's virtual key
    });

    const getBatch = async () => {
      const batch = await portkey.batches.retrieve(batch_id="batch_id");

      console.log(batch);
    }

    await getBatch();

    ```
  </Tab>

  <Tab title="REST">
    ```sh theme={"system"}
    curl --location 'https://api.portkey.ai/v1/batches/<batch_id>' \
    --header 'x-portkey-api-key: <portkey_api_key>' \
    --header 'x-portkey-virtual-key: <virtual_key>' \
    ```
  </Tab>

  <Tab title="OpenAI NodeJS">
    ```js theme={"system"}
    import OpenAI from 'openai'; // We're using the v4 SDK
    import { PORTKEY_GATEWAY_URL, createHeaders } from 'portkey-ai'

    const openai = new OpenAI({
      apiKey: 'PLACEHOLDER_NOT_USED', // defaults to process.env["OPENAI_API_KEY"],
      baseURL: PORTKEY_GATEWAY_URL,
      defaultHeaders: createHeaders({
        provider: "openai",
        apiKey: "PORTKEY_API_KEY", // defaults to process.env["PORTKEY_API_KEY"]
        virtualKey: "BEDROCK_VIRTUAL_KEY",
      })
    });

    const getBatch = async () => {
      const batch = await openai.batches.retrieve(batch_id="batch_id");

      console.log(batch);
    }

    await getBatch();
    ```
  </Tab>

  <Tab title="OpenAI Python">
    ```python theme={"system"}
    from openai import OpenAI
    from portkey_ai import PORTKEY_GATEWAY_URL, createHeaders

    openai = OpenAI(
        api_key='PLACEHOLDER_NOT_USED',
        base_url=PORTKEY_GATEWAY_URL,
        default_headers=createHeaders(
            provider="openai",
            api_key="PORTKEY_API_KEY",
            virtualKey: "BEDROCK_VIRTUAL_KEY",
        )
    )

    batch = openai.batches.retrieve(batch_id="batch_id")

    print(batch)
    ```
  </Tab>
</Tabs>

## Get Batch Output

<Tabs>
  <Tab title="REST">
    ```sh theme={"system"}
    curl --location 'https://api.portkey.ai/v1/batches/<batch_id>/output' \
    --header 'x-portkey-api-key: <portkey_api_key>' \
    --header 'x-portkey-virtual-key: <virtual_key>' \
    ```
  </Tab>
</Tabs>

## List Batch Jobs

<Tabs>
  <Tab title="Python">
    ```python theme={"system"}
    from portkey_ai import Portkey

    # Initialize the Portkey client
    portkey = Portkey(
        api_key="PORTKEY_API_KEY",  # Replace with your Portkey API key
        virtual_key="VIRTUAL_KEY",   # Add your provider's virtual key
    )

    batches = portkey.batches.list()

    print(batches)
    ```
  </Tab>

  <Tab title="NodeJS">
    ```js theme={"system"}
    import { Portkey } from 'portkey-ai';

    // Initialize the Portkey client
    const portkey = new Portkey({
        apiKey: "PORTKEY_API_KEY",  // Replace with your Portkey API key
        virtualKey: "VIRTUAL_KEY",   // Add your provider's virtual key
    });

    const listBatchingJobs = async () => {
      const batching_jobs = await portkey.batches.list();

      console.log(batching_jobs);
    }

    await listBatchingJobs();

    ```
  </Tab>

  <Tab title="REST">
    ```sh theme={"system"}
    curl --location 'https://api.portkey.ai/v1/batches' \
    --header 'x-portkey-api-key: <portkey_api_key>' \
    --header 'x-portkey-virtual-key: <virtual_key>' \
    ```
  </Tab>

  <Tab title="OpenAI NodeJS">
    ```js theme={"system"}
    import OpenAI from 'openai'; // We're using the v4 SDK
    import { PORTKEY_GATEWAY_URL, createHeaders } from 'portkey-ai'

    const openai = new OpenAI({
      apiKey: 'PLACEHOLDER_NOT_USED', // defaults to process.env["OPENAI_API_KEY"],
      baseURL: PORTKEY_GATEWAY_URL,
      defaultHeaders: createHeaders({
        provider: "openai",
        apiKey: "PORTKEY_API_KEY", // defaults to process.env["PORTKEY_API_KEY"]
        virtualKey: "BEDROCK_VIRTUAL_KEY",
      })
    });

    const listBatchingJobs = async () => {
      const batching_jobs = await openai.batches.list();

      console.log(batching_jobs);
    }

    await listBatchingJobs();
    ```
  </Tab>

  <Tab title="OpenAI Python">
    ```python theme={"system"}
    from openai import OpenAI
    from portkey_ai import PORTKEY_GATEWAY_URL, createHeaders

    openai = OpenAI(
        api_key='PLACEHOLDER_NOT_USED',
        base_url=PORTKEY_GATEWAY_URL,
        default_headers=createHeaders(
            provider="openai",
            api_key="PORTKEY_API_KEY",
            virtualKey: "BEDROCK_VIRTUAL_KEY",
        )
    )

    batching_jobs = openai.batches.list()

    print(batching_jobs)
    ```
  </Tab>
</Tabs>

## Cancel Batch Job

<Tabs>
  <Tab title="Python">
    ```python theme={"system"}
    from portkey_ai import Portkey

    # Initialize the Portkey client
    portkey = Portkey(
        api_key="PORTKEY_API_KEY",  # Replace with your Portkey API key
        virtual_key="VIRTUAL_KEY",   # Add your provider's virtual key
    )

    cancel_batch_response = portkey.batches.cancel(batch_id="batch_id")

    print(cancel_batch_response)
    ```
  </Tab>

  <Tab title="NodeJS">
    ```js theme={"system"}
    import { Portkey } from 'portkey-ai';

    // Initialize the Portkey client
    const portkey = new Portkey({
        apiKey: "PORTKEY_API_KEY",  // Replace with your Portkey API key
        virtualKey: "VIRTUAL_KEY",   // Add your provider's virtual key
    });

    const cancelBatch = async () => {
      const cancel_batch_response = await portkey.batches.cancel(batch_id="batch_id");

      console.log(cancel_batch_response);
    }

    await cancelBatch();

    ```
  </Tab>

  <Tab title="REST">
    ```sh theme={"system"}
    curl --request POST --location 'https://api.portkey.ai/v1/batches/<batch_id>/cancel' \
    --header 'x-portkey-api-key: <portkey_api_key>' \
    --header 'x-portkey-virtual-key: <virtual_key>' \
    ```
  </Tab>

  <Tab title="OpenAI NodeJS">
    ```js theme={"system"}
    import OpenAI from 'openai'; // We're using the v4 SDK
    import { PORTKEY_GATEWAY_URL, createHeaders } from 'portkey-ai'

    const openai = new OpenAI({
      apiKey: 'PLACEHOLDER_NOT_USED', // defaults to process.env["OPENAI_API_KEY"],
      baseURL: PORTKEY_GATEWAY_URL,
      defaultHeaders: createHeaders({
        provider: "openai",
        apiKey: "PORTKEY_API_KEY", // defaults to process.env["PORTKEY_API_KEY"]
        virtualKey: "BEDROCK_VIRTUAL_KEY",
      })
    });

    const cancelBatch = async () => {
      const cancel_batch_response = await openai.batches.cancel(batch_id="batch_id");

      console.log(cancel_batch_response);
    }

    await cancelBatch();
    ```
  </Tab>

  <Tab title="OpenAI Python">
    ```python theme={"system"}
    from openai import OpenAI
    from portkey_ai import PORTKEY_GATEWAY_URL, createHeaders

    openai = OpenAI(
        api_key='PLACEHOLDER_NOT_USED',
        base_url=PORTKEY_GATEWAY_URL,
        default_headers=createHeaders(
            provider="openai",
            api_key="PORTKEY_API_KEY",
            virtualKey: "BEDROCK_VIRTUAL_KEY",
        )
    )

    cancel_batch_response = openai.batches.cancel(batch_id="batch_id")

    print(cancel_batch_response)
    ```
  </Tab>
</Tabs>

## Information about Permissions and IAM Roles

<Accordion title="For Principal Role (Used in Virtual Key)">
  These are the minimum permissions required to use the Bedrock Batch APIs.

  ```json theme={"system"}
  {
    "Version": "2012-10-17",
    "Statement": [
        {
            "Effect": "Allow",
            "Action": [
                "bedrock:ListFoundationModels",
                "bedrock:GetFoundationModel",
                "bedrock:ListInferenceProfiles",
                "bedrock:GetInferenceProfile",
                "bedrock:ListCustomModels",
                "bedrock:GetCustomModel",
                "bedrock:TagResource", 
                "bedrock:UntagResource", 
                "bedrock:ListTagsForResource",
                "bedrock:CreateModelInvocationJob",
                "bedrock:GetModelInvocationJob",
                "bedrock:ListModelInvocationJobs",
                "bedrock:StopModelInvocationJob"
            ],
            "Resource": [
                "arn:aws:bedrock:<region>:<account_id>:model-customization-job/*",
                "arn:aws:bedrock:<region>:<account_id>:custom-model/*",
                "arn:aws:bedrock:<region>::foundation-model/*"
            ]
        },
        {
            "Effect": "Allow",
            "Action": [
                "s3:ListBucket",
                "s3:PutObject",
                "s3:GetObject",
                "s3:GetObjectAttributes"
            ],
            "Resource": [
                "arn:aws:s3:::<bucket>",
                "arn:aws:s3:::<bucket>/*"
            ]
        },
        {
            "Action": [
                "iam:PassRole"
            ],
            "Effect": "Allow",
            "Resource": "arn:aws:iam::<account_id>:role/<service_role_name>",
            "Condition": {
                "StringEquals": {
                    "iam:PassedToService": [
                        "bedrock.amazonaws.com"
                    ]
                }
            }
        }
    ]
  }
  ```
</Accordion>

<Accordion title="For Service Role (role_arn) used for creating and executing the batch job">
  These are the minimum permissions required to use the Bedrock Batch APIs.

  Trust relationship:

  ```json theme={"system"}
  {
    "Version": "2012-10-17",
    "Statement": [
        {
            "Effect": "Allow",
            "Principal": {
                "Service": "bedrock.amazonaws.com"
            },
            "Action": "sts:AssumeRole",
            "Condition": {
                "StringEquals": {
                    "aws:SourceAccount": "<account_id>"
                },
                "ArnEquals": {
                    "aws:SourceArn": "arn:aws:bedrock:<region>:<account_id>:model-invocation-job/*"
                }
            }
    ]
  }
  ```

  Permission Policy:

  ```json theme={"system"}
  {
    "Version": "2012-10-17",
    "Statement": [
        {
            "Effect": "Allow",
            "Action": [
                "s3:GetObject",
                "s3:PutObject",
                "s3:ListBucket"
            ],
            "Resource": [
                "arn:aws:s3:::<bucket>",
                "arn:aws:s3:::<bucket>/*"
            ]
        }
    ]
  }
  ```
</Accordion>