BlockNote AI Reference

// Usage:
const aiExtension = createAIExtension(opts: AIExtensionOptions);
 
// Definitions:
function createAIExtension(options: AIExtensionOptions): (editor: BlockNoteEditor) => AIExtension;
 
type AIExtensionOptions = {
    /**
   * The default language model to use for LLM calls
   */
  model: LanguageModel;
  /**
   * Whether to stream the LLM response
   * @default true
   */
  stream?: boolean;
  /**
   * The default data format to use for LLM calls
   * html format is recommended, the other formats are experimental
   * @default llmFormats.html
   */
  dataFormat?: LLMFormat;
  /**
   * A function that can be used to customize the prompt sent to the LLM
   * @default the default prompt builder for the selected {@link dataFormat}
   */
  promptBuilder?: PromptBuilder;
  /**
   * The name and color of the agent cursor when the AI is writing
   *
   * @default { name: "AI", color: "#8bc6ff" }
   */
  agentCursor?: { name: string; color: string };
};

function getAIExtension(editor: BlockNoteEditor): AIExtension;

class AIExtension {
  /**
   * Execute a call to an LLM and apply the result to the editor
   */
  callLLM(
    opts: MakeOptional<LLMRequestOptions, "model">,
  ): Promise<LLMResponse | undefined>;
  /**
   * Returns a read-only zustand store with the state of the AI Menu
   */
  get store(): ReadonlyStoreApi<{
    aiMenuState:
      | ({
          /**
           * The ID of the block that the AI menu is opened at.
           * This changes as the AI is making changes to the document
           */
          blockId: string;
        } & (
          | {
              status: "error";
              error: any;
            }
          | {
              status:
                | "user-input"
                | "thinking"
                | "ai-writing"
                | "user-reviewing";
            }
        ))
      | "closed";
    /**
     * The previous response from the LLM, used for multi-step LLM calls
     */
    llmResponse?: LLMResponse;
  }>;
  /**
   * Returns a zustand store with the global configuration of the AI Extension,
   * these options are used as default across all LLM calls when calling {@link callLLM}
   */
  readonly options: StoreApi<{
    model: LanguageModel;
    dataFormat: LLMFormat;
    stream: boolean;
    promptBuilder?: PromptBuilder;
  }>;
 
  /**
   * Open the AI menu at a specific block
   */
  openAIMenuAtBlock(blockID: string): void;
  /**
   * Close the AI menu
   */
  closeAIMenu(): void;
  /**
   * Accept the changes made by the LLM
   */
  acceptChanges(): void;
  /**
   * Reject the changes made by the LLM
   */
  rejectChanges(): void;
  /**
   * Retry the previous LLM call.
   *
   * Only valid if the current status is "error"
   */
  retry(): Promise<void>;
  /**
   * Update the status of a call to an LLM
   *
   * @warning This method should usually only be used for advanced use-cases
   * if you want to implement how an LLM call is executed. Usually, you should
   * use {@link callLLM} instead which will handle the status updates for you.
   */
  setAIResponseStatus(
    status:
      | "user-input"
      | "thinking"
      | "ai-writing"
      | "user-reviewing"
      | {
          status: "error";
          error: any;
        },
  ): void;
}

type LLMRequestOptions = {
  /**
   * The language model to use for the LLM call (AI SDK)
   *
   * (when invoking `callLLM` via the `AIExtension` this will default to the
   * model set in the `AIExtension` options)
   */
  model: LanguageModelV1;
  /**
   * The user prompt to use for the LLM call
   */
  userPrompt: string;
  /**
   * Previous response from the LLM, used for multi-step LLM calls
   *
   * (populated automatically when invoking `callLLM` via the `AIExtension` class)
   */
  previousResponse?: LLMResponse;
  /**
   * The default data format to use for LLM calls
   * "html" is recommended, the other formats are experimental
   * @default html format (`llm.html`)
   */
  dataFormat?: LLMFormat;
  /**
   * The `PromptBuilder` to use for the LLM call
   *
   * (A PromptBuilder is a function that takes a BlockNoteEditor and details about the user's prompt
   * and turns it into an AI SDK `CoreMessage` array to be passed to the LLM)
   *
   * @default provided by the format (e.g. `llm.html.defaultPromptBuilder`)
   */
  promptBuilder?: PromptBuilder;
  /**
   * The maximum number of retries for the LLM call
   *
   * @default 2
   */
  maxRetries?: number;
  /**
   * Whether to use the editor selection for the LLM call
   *
   * @default true
   */
  useSelection?: boolean;
  /**
   * Defines whether the LLM can add, update, or delete blocks
   *
   * @default { add: true, update: true, delete: true }
   */
  defaultStreamTools?: {
    /** Enable the add tool (default: true) */
    add?: boolean;
    /** Enable the update tool (default: true) */
    update?: boolean;
    /** Enable the delete tool (default: true) */
    delete?: boolean;
  };
  /**
   * Whether to stream the LLM response or not
   *
   * When streaming, we use the AI SDK `streamObject` function,
   * otherwise, we use the AI SDK `generateObject` function.
   *
   * @default true
   */
  stream?: boolean;
  /**
   * If the user's cursor is in an empty paragraph, automatically delete it when the AI
   * is starting to write.
   *
   * (This is used when a user starts typing `/ai` in an empty block)
   *
   * @default true
   */
  deleteEmptyCursorBlock?: boolean;
  /**
   * Callback when a specific block is updated by the LLM
   *
   * (used by `AIExtension` to update the `AIMenu` position)
   */
  onBlockUpdate?: (blockId: string) => void;
  /**
   * Callback when the AI Agent starts writing
   */
  onStart?: () => void;
  /**
   * Whether to add delays between text update operations, to make the AI simulate a human typing
   *
   * @default true
   */
  withDelays?: boolean;
  /**
   * Additional options to pass to the AI SDK `generateObject` function
   * (only used when `stream` is `false`)
   */
  _generateObjectOptions?: Partial<Parameters<typeof generateObject<any>>[0]>;
  /**
   * Additional options to pass to the AI SDK `streamObject` function
   * (only used when `stream` is `true`)
   */
  _streamObjectOptions?: Partial<Parameters<typeof streamObject<any>>[0]>;
};

/**
 * Execute an LLM call
 *
 * @param editor - The BlockNoteEditor the LLM should operate on
 * @param opts - The options for the LLM call (@link {CallLLMOptions})
 * @returns A `LLMResponse` object containing the LLM response which can be applied to the editor
 */
function doLLMRequest(
  editor: BlockNoteEditor<any, any, any>,
  opts: LLMRequestOptions,
): Promise<LLMResponse>;

/**
 * A PromptBuilder is a function that takes a BlockNoteEditor and details about the user's promot
 * and turns it into an array of CoreMessage (AI SDK) to be passed to the LLM.
 */
type PromptBuilder = (
  editor: BlockNoteEditor<any, any, any>,
  opts: PromptBuilderInput,
) => Promise<Array<CoreMessage>>;
 
/**
 * The input passed to a PromptBuilder
 */
type PromptBuilderInput = {
  /**
   * The user's prompt
   */
  userPrompt: string;
  /**
   * The selection of the editor which the LLM should operate on
   */
  selectedBlocks?: Block<any, any, any>[];
  /**
   * The ids of blocks that should be excluded from the prompt
   * (e.g.: if `deleteEmptyCursorBlock` is true in the LLMRequest,
   * this will be the id of the block that should be ignored)
   */
  excludeBlockIds?: string[];
  /**
   * When following a multi-step conversation, or repairing a previous error,
   * the previous messages that have been sent to the LLM
   */
  previousMessages?: Array<CoreMessage>;
};

type LLMFormat = {
  /**
   * Function to get th format specific stream tools that the LLM can choose to invoke
   */
  getStreamTools: (
    editor: BlockNoteEditor<any, any, any>,
    withDelays: boolean,
    defaultStreamTools?: {
      add?: boolean;
      update?: boolean;
      delete?: boolean;
    },
    selectionInfo?: {
      from: number;
      to: number;
    },
    onBlockUpdate?: (blockId: string) => void,
  ) => StreamTool<any>[];
  /**
   * The default PromptBuilder that determines how a userPrompt is converted to an array of
   * LLM Messages (CoreMessage[])
   */
  defaultPromptBuilder: PromptBuilder;
  /**
   * Helper functions which can be used when implementing a custom PromptBuilder.
   * The signature depends on the specific format
   */
  promptHelpers: any;
};
 
// The default LLMFormats are exported under `llmFormats`:
 
export const llmFormats = {
  _experimental_json,
  _experimental_markdown,
  html,
};