Query the OpenAI Vision API with an Image URL

Easily query the OpenAI Vision API with an image URL and map the 'response' output variable to a variable you have set on your Voiceflow project.
Created By
FlowBridge
Voiceflow Built
INPUT VARIABLES
{
max_tokens
}
The maximum amount of tokens you want to set for the OpenAI API (300 - 500 is a good start)
{
openai_token
}
Your OpenAI API token (Found on platform.openai.com)
{
query
}
What you want to ask the Vision API (i.e: What is in this image?)
{
image_url
}
The URL to the image in question
{
}
{
}
OUTPUT VARIABLES
{
response
}
This is the response from the OpenAI Vision API
{
}
{
}
{
}
{
}
{
}
PATHS
{
error
}
If something goes wrong with querying the API or running the function, it will use this path
{
success
}
If the query was made successfully, this path is chosen. You can use the 'response' output variable to do something with the result
{
}
{
}
{
}
{
}

Function Code Snippet


export default async function main({ inputVars }) {
  const { openai_token, query, image_url, max_tokens } = inputVars

    if (!image_url || !openai_token || !query || !max_tokens) {
        return {
          next: {
            path: 'error'
          },
          trace: [
            {
              type: "debug",
              payload: {
                message: "Not all values were filled in for the Vision API function"
              }
            }
          ]
        }
    }

    let payload = {
      "model": "gpt-4-vision-preview",
      "messages": [
        {
          "role": "user",
          "content": [
            {
              "type": "text",
              "text": query
            },
            {
              "type": "image_url",
              "image_url": {
                "url": image_url
              }
            }
          ]
        }
      ],
      "max_tokens": max_tokens
    }
  
    const response = (
      await fetch('https://api.openai.com/v1/chat/completions', {
          method: 'POST',
          headers: {
              'Authorization': 'Bearer '+ openai_token,
              'Content-Type': 'application/json'
          },
          body: JSON.stringify(payload)
        },
        { parseType: 'json' }
      )
    ).json

  if(!response || !response?.choices[0]?.message?.content) {
    return {
      next: {
        path: 'error'
      },
      trace: [
        {
          type: "debug",
          payload: {
            message: "Something went wrong while querying the OpenAI Vision API"
          }
        },
        {
          type: "debug",
          payload: {
            message: "Open AI Response: "+ JSON.stringify(response)
          }
        }
      ]
    }
  }
  
  return {
    outputVars: {
      response: response.choices[0].message.content
    },
    next: {
      path: 'success'
    },
    trace: [
      {
        type: "debug",
        payload: {
          message: "Open AI Response: "+ JSON.stringify(response)
        }
      },
      {
        type: "debug",
        payload: {
          message: "Vision Message: "+ response.choices[0].message.content
        }
      }
    ]
  }
       
}

Function Walkthrough