Content Types

Multi-modal content types for messages and variables in the Adaline SDK.

Overview

Content types support text, images, PDFs, tool calls/responses, and reasoning. All content uses the modality field as a discriminator.

Content (Union Type)

Polymorphic content type supporting multiple modalities.

type Content = 
  | TextContent
  | ImageContent
  | PdfContent
  | ToolCallContent
  | ToolResponseContent
  | ReasoningContent;

Type Narrowing:

function processContent(content: Content) {
  switch (content.modality) {
    case 'text':
      console.log('Text:', content.value);
      break;
    case 'image':
      console.log('Image:', content.value.type);
      break;
    case 'pdf':
      console.log('PDF:', content.file.name);
      break;
    case 'tool-call':
      console.log('Tool call:', content.name);
      break;
    case 'tool-response':
      console.log('Tool response:', content.name);
      break;
    case 'reasoning':
      console.log('Reasoning...');
      break;
  }
}

Text Content

TextContent

Plain text content for messages.

interface TextContent {
  modality: 'text';
  value: string;
}

Example:

const text: TextContent = {
  modality: 'text',
  value: 'Hello, how are you?'
};

const message: Message = {
  role: 'user',
  content: [text]
};

JSON:

{
  "modality": "text",
  "value": "Hello, how are you?"
}

Image Content

Base64ImageContentValue

Base64-encoded image data with media type.

interface Base64ImageContentValue {
  type: 'base64';
  base64: string;
  mediaType: 'png' | 'jpeg' | 'webp' | 'gif';
}

Example:

const base64Image: Base64ImageContentValue = {
  type: 'base64',
  base64: 'iVBORw0KGgoAAAANSUhEUgA...',
  mediaType: 'png'
};

UrlImageContentValue

URL reference to an externally hosted image.

interface UrlImageContentValue {
  type: 'url';
  url: string;
}

Example:

const urlImage: UrlImageContentValue = {
  type: 'url',
  url: 'https://example.com/image.jpg'
};

ImageContentValue

Union of base64 or URL image content.

type ImageContentValue = Base64ImageContentValue | UrlImageContentValue;

Example:

// URL variant
const urlValue: ImageContentValue = {
  type: 'url',
  url: 'https://example.com/photo.jpg'
};

// Base64 variant
const base64Value: ImageContentValue = {
  type: 'base64',
  base64: 'iVBORw0KG...',
  mediaType: 'png'
};

ImageContent

Image content with detail level specification.

interface ImageContent {
  modality: 'image';
  detail: 'low' | 'medium' | 'high' | 'auto';
  value: ImageContentValue;
}

Example:

// With URL
const image: ImageContent = {
  modality: 'image',
  detail: 'high',
  value: {
    type: 'url',
    url: 'https://example.com/chart.png'
  }
};

// With base64
const base64Image: ImageContent = {
  modality: 'image',
  detail: 'auto',
  value: {
    type: 'base64',
    base64: 'iVBORw0KGgoAAAANSUhEUgA...',
    mediaType: 'jpeg'
  }
};

// In message
const message: Message = {
  role: 'user',
  content: [
    { modality: 'text', value: 'Describe this' },
    image
  ]
};

JSON:

{
  "modality": "image",
  "detail": "high",
  "value": {
    "type": "url",
    "url": "https://example.com/image.jpg"
  }
}

PDF Content

Base64PdfContentValue

Base64-encoded PDF document.

interface Base64PdfContentValue {
  type: 'base64';
  base64: string;
}

Example:

const pdfValue: Base64PdfContentValue = {
  type: 'base64',
  base64: 'JVBERi0xLjcKCjEgMCBvYmo...'
};

UrlPdfContentValue

URL reference to a PDF document.

interface UrlPdfContentValue {
  type: 'url';
  url: string;
}

Example:

const pdfValue: UrlPdfContentValue = {
  type: 'url',
  url: 'https://example.com/document.pdf'
};

PdfContentValue

Union of base64 or URL PDF content.

type PdfContentValue = Base64PdfContentValue | UrlPdfContentValue;

PdfContent

PDF document content with file metadata.

interface PdfContent {
  modality: 'pdf';
  value: PdfContentValue;
  file: {
    name: string;
    id: string;
    size?: number | null;
  };
}

Example:

const pdf: PdfContent = {
  modality: 'pdf',
  value: {
    type: 'url',
    url: 'https://example.com/report.pdf'
  },
  file: {
    name: 'Q4_Report.pdf',
    id: 'file_abc123',
    size: 1024000
  }
};

const message: Message = {
  role: 'user',
  content: [
    { modality: 'text', value: 'Summarize this document' },
    pdf
  ]
};

JSON:

{
  "modality": "pdf",
  "value": {
    "type": "url",
    "url": "https://example.com/report.pdf"
  },
  "file": {
    "name": "report.pdf",
    "id": "file_123",
    "size": 1024000
  }
}

Tool Content

ToolCallContent

Tool/function call request from LLM.

interface ToolCallContent {
  modality: 'tool-call';
  index: number;                   // minimum: 0
  id: string;
  name: string;
  arguments: string;
  serverName?: string | null;
}

Example:

const toolCall: ToolCallContent = {
  modality: 'tool-call',
  index: 0,
  id: 'call_abc123',
  name: 'get_weather',
  arguments: JSON.stringify({
    city: 'San Francisco',
    units: 'fahrenheit'
  }),
  serverName: 'weather-api'
};

// In assistant message
const assistantMessage: Message = {
  role: 'assistant',
  content: [toolCall]
};

JSON:

{
  "modality": "tool-call",
  "index": 0,
  "id": "call_abc123",
  "name": "get_weather",
  "arguments": "{\"city\":\"San Francisco\"}"
}

ToolResponseContent

Tool/function execution response.

interface ToolResponseContent {
  modality: 'tool-response';
  index: number;                   // minimum: 0
  id: string;
  name: string;
  data: string;
  apiResponse?: {
    statusCode?: number;
  } | null;
}

Example:

const toolResponse: ToolResponseContent = {
  modality: 'tool-response',
  index: 0,
  id: 'call_abc123',
  name: 'get_weather',
  data: JSON.stringify({
    temperature: 72,
    conditions: 'sunny',
    humidity: 65
  }),
  apiResponse: {
    statusCode: 200
  }
};

// In tool message
const toolMessage: Message = {
  role: 'tool',
  content: [toolResponse]
};

JSON:

{
  "modality": "tool-response",
  "index": 0,
  "id": "call_abc123",
  "name": "get_weather",
  "data": "{\"temperature\":72,\"conditions\":\"sunny\"}",
  "apiResponse": {
    "statusCode": 200
  }
}

Reasoning Content

ReasoningContentValue

LLM reasoning/thinking content with signature.

interface ReasoningContentValue {
  type: 'thinking';
  thinking: string;
  signature: string;
}

RedactedReasoningContentValue

Redacted reasoning content for privacy.

interface RedactedReasoningContentValue {
  type: 'redacted';
  data: string;
}

ReasoningContentValueUnion

Union of reasoning content types.

type ReasoningContentValueUnion = 
  | ReasoningContentValue 
  | RedactedReasoningContentValue;

ReasoningContent

Reasoning content for chain-of-thought responses.

interface ReasoningContent {
  modality: 'reasoning';
  value: ReasoningContentValueUnion;
}

Example:

const reasoning: ReasoningContent = {
  modality: 'reasoning',
  value: {
    type: 'thinking',
    thinking: 'Let me analyze this step by step...',
    signature: 'sig_abc123'
  }
};

// Redacted version
const redacted: ReasoningContent = {
  modality: 'reasoning',
  value: {
    type: 'redacted',
    data: '[REDACTED]'
  }
};

Complete Examples

import type { Message, Content } from '@adaline/api';

const message: Message = {
  role: 'user',
  content: [
    {
      modality: 'text',
      value: 'Analyze this image and document'
    },
    {
      modality: 'image',
      detail: 'high',
      value: {
        type: 'url',
        url: 'https://example.com/image.jpg'
      }
    },
    {
      modality: 'pdf',
      value: {
        type: 'url',
        url: 'https://example.com/report.pdf'
      },
      file: {
        name: 'report.pdf',
        id: 'file_123'
      }
    }
  ]
};

Tool Call Flow

// 1. Assistant makes tool call
const assistantWithTool: Message = {
  role: 'assistant',
  content: [
    {
      modality: 'tool-call',
      index: 0,
      id: 'call_123',
      name: 'get_weather',
      arguments: JSON.stringify({ city: 'Paris' })
    }
  ]
};

// 2. Tool responds
const toolResult: Message = {
  role: 'tool',
  content: [
    {
      modality: 'tool-response',
      index: 0,
      id: 'call_123',
      name: 'get_weather',
      data: JSON.stringify({ temp: 24, conditions: 'sunny' })
    }
  ]
};

// 3. Assistant uses tool result
const finalResponse: Message = {
  role: 'assistant',
  content: [
    {
      modality: 'text',
      value: 'The weather in Paris is sunny with a temperature of 24°C.'
    }
  ]
};

Helper Functions

// Create text content
function createText(value: string): TextContent {
  return { modality: 'text', value };
}

// Create image content
function createImage(url: string, detail: 'low' | 'medium' | 'high' | 'auto' = 'auto'): ImageContent {
  return {
    modality: 'image',
    detail,
    value: { type: 'url', url }
  };
}

// Create PDF content
function createPdf(url: string, name: string): PdfContent {
  return {
    modality: 'pdf',
    value: { type: 'url', url },
    file: { name, id: `file_${Date.now()}` }
  };
}

// Usage
const message: Message = {
  role: 'user',
  content: [
    createText('Analyze these'),
    createImage('https://example.com/chart.png', 'high'),
    createPdf('https://example.com/report.pdf', 'Q4 Report')
  ]
};

Content Filtering

const message: Message = {
  role: 'user',
  content: [/* mixed content */]
};

// Get all text content
const textItems = message.content.filter(
  (c): c is TextContent => c.modality === 'text'
);

// Get all images
const images = message.content.filter(
  (c): c is ImageContent => c.modality === 'image'
);

// Get tool calls
const toolCalls = message.content.filter(
  (c): c is ToolCallContent => c.modality === 'tool-call'
);

// Extract text values
const textValues = textItems.map(t => t.value).join(' ');

Overview

Types

TypeScript

Python

Content

Content Types

Overview

Content (Union Type)

Text Content

TextContent

Image Content

Base64ImageContentValue

UrlImageContentValue

ImageContentValue

ImageContent

PDF Content

Base64PdfContentValue

UrlPdfContentValue

PdfContentValue

PdfContent

Tool Content

ToolCallContent

ToolResponseContent

Reasoning Content

ReasoningContentValue

RedactedReasoningContentValue

ReasoningContentValueUnion

ReasoningContent

Complete Examples

Tool Call Flow

Helper Functions

Content Filtering

Overview

Types

TypeScript

Python

​Content Types

​Overview

​Content (Union Type)

​Text Content

​TextContent

​Image Content

​Base64ImageContentValue

​UrlImageContentValue

​ImageContentValue

​ImageContent

​PDF Content

​Base64PdfContentValue

​UrlPdfContentValue

​PdfContentValue

​PdfContent

​Tool Content

​ToolCallContent

​ToolResponseContent

​Reasoning Content

​ReasoningContentValue

​RedactedReasoningContentValue

​ReasoningContentValueUnion

​ReasoningContent

​Complete Examples

​Multi-Modal Message

​Tool Call Flow

​Helper Functions

​Content Filtering

Content Types

Overview

Content (Union Type)

Text Content

TextContent

Image Content

Base64ImageContentValue

UrlImageContentValue

ImageContentValue

ImageContent

PDF Content

Base64PdfContentValue

UrlPdfContentValue

PdfContentValue

PdfContent

Tool Content

ToolCallContent

ToolResponseContent

Reasoning Content

ReasoningContentValue

RedactedReasoningContentValue

ReasoningContentValueUnion

ReasoningContent

Complete Examples

Multi-Modal Message

Tool Call Flow

Helper Functions

Content Filtering