Audio

/**
 * Adapter: UI and utility re-exports for copy-standalone portability.
 */
"use client";

export { cn } from "@/lib/utils";
export { Button } from "@/components/ui/button";
export { Slider } from "@/components/ui/slider";

"use client";

import * as React from "react";
import { Pause, Play } from "lucide-react";
import { cn, Button, Slider } from "./_adapter";

import { AudioProvider, useAudio } from "./context";
import type { SerializableAudio, AudioVariant } from "./schema";

const FALLBACK_LOCALE = "en-US";

function formatTime(seconds: number): string {
  if (!Number.isFinite(seconds)) return "0:00";
  const mins = Math.floor(seconds / 60);
  const secs = Math.floor(seconds % 60);
  return `${mins}:${secs.toString().padStart(2, "0")}`;
}

export interface AudioProps extends SerializableAudio {
  variant?: AudioVariant;
  className?: string;
  onMediaEvent?: (type: "play" | "pause" | "mute" | "unmute") => void;
}

export function Audio(props: AudioProps) {
  return (
    <AudioProvider>
      <AudioInner {...props} />
    </AudioProvider>
  );
}

interface PlayerControls {
  isPlaying: boolean;
  currentTime: number;
  duration: number;
  onPlayPause: () => void;
  onSeek: (value: number[]) => void;
  onSeekStart: () => void;
  onSeekEnd: () => void;
}

interface FullPlayerProps {
  artwork?: string;
  title?: string;
  description?: string;
  controls: PlayerControls;
}

function FullPlayer({
  artwork,
  title,
  description,
  controls,
}: FullPlayerProps) {
  return (
    <div className="flex w-full flex-col">
      {artwork && (
        <div className="bg-muted relative aspect-[4/3] w-full overflow-hidden">
          <img
            src={artwork}
            alt=""
            aria-hidden="true"
            loading="lazy"
            decoding="async"
            className="absolute inset-0 h-full w-full object-cover"
          />
        </div>
      )}
      <div className="flex flex-col gap-5 p-4">
        {(title || description) && (
          <div className="space-y-0.5">
            {title && (
              <div className="text-foreground line-clamp-2 font-semibold leading-snug">
                {title}
              </div>
            )}
            {description && (
              <div className="text-muted-foreground line-clamp-2 text-sm leading-snug">
                {description}
              </div>
            )}
          </div>
        )}
        <div className="flex items-start gap-3">
          <div className="flex flex-1 flex-col gap-2">
            <Slider
              value={[controls.currentTime]}
              max={controls.duration || 100}
              step={0.1}
              onValueChange={controls.onSeek}
              onPointerDown={controls.onSeekStart}
              onPointerUp={controls.onSeekEnd}
              className="cursor-pointer [&_[data-slot=range]]:bg-foreground [&_[data-slot=thumb]]:size-3 [&_[data-slot=thumb]]:border-2 [&_[data-slot=thumb]]:border-background [&_[data-slot=thumb]]:bg-foreground"
              aria-label="Audio progress"
            />
            <div className="text-muted-foreground flex items-center justify-between text-xs tabular-nums">
              <span>{formatTime(controls.currentTime)}</span>
              <span>{formatTime(controls.duration)}</span>
            </div>
          </div>
          <Button
            variant="default"
            size="icon"
            onClick={controls.onPlayPause}
            className="-mt-4 size-10 shrink-0 rounded-full"
            aria-label={controls.isPlaying ? "Pause" : "Play"}
          >
            {controls.isPlaying ? (
              <Pause className="size-4" fill="currentColor" />
            ) : (
              <Play className="size-4 ml-0.5" fill="currentColor" />
            )}
          </Button>
        </div>
      </div>
    </div>
  );
}

interface CompactPlayerProps {
  artwork?: string;
  title?: string;
  description?: string;
  controls: PlayerControls;
}

function CompactPlayer({
  artwork,
  title,
  description,
  controls,
}: CompactPlayerProps) {
  const progress =
    controls.duration > 0
      ? (controls.currentTime / controls.duration) * 100
      : 0;

  return (
    <div className="relative flex w-full items-center gap-3 overflow-hidden p-3">
      {artwork && (
        <>
          <img
            src={artwork}
            alt=""
            aria-hidden="true"
            className="pointer-events-none absolute -left-1/4 top-1/2 h-[200%] w-auto -translate-y-1/2 object-cover opacity-40 blur-2xl saturate-150"
          />
          <div className="from-card/60 to-card/90 pointer-events-none absolute inset-0 bg-gradient-to-r" />
        </>
      )}
      {artwork && (
        <div className="ring-background/20 relative size-12 shrink-0 overflow-hidden rounded-lg shadow-lg ring-1">
          <img
            src={artwork}
            alt=""
            aria-hidden="true"
            loading="lazy"
            decoding="async"
            className="absolute inset-0 h-full w-full object-cover"
          />
        </div>
      )}
      <div className="relative flex min-w-0 flex-1 flex-col justify-center">
        {title && (
          <div className="text-foreground truncate text-sm font-semibold leading-tight">
            {title}
          </div>
        )}
        {description && (
          <div className="text-muted-foreground mt-0.5 truncate text-xs leading-tight">
            {description}
          </div>
        )}
        {controls.duration > 0 && (
          <div className="mt-1 flex items-center gap-2">
            <div className="bg-foreground/20 relative h-1 flex-1 overflow-hidden rounded-full">
              <div
                className="bg-foreground absolute inset-y-0 left-0 rounded-full transition-all duration-150"
                style={{ width: `${progress}%` }}
              />
            </div>
            <span className="text-muted-foreground text-xs tabular-nums">
              {formatTime(controls.currentTime)}
            </span>
          </div>
        )}
      </div>
      <Button
        variant="default"
        size="icon"
        onClick={controls.onPlayPause}
        className="relative size-10 shrink-0 rounded-full shadow-md"
        aria-label={controls.isPlaying ? "Pause" : "Play"}
      >
        {controls.isPlaying ? (
          <Pause className="size-4" fill="currentColor" />
        ) : (
          <Play className="size-4 ml-0.5" fill="currentColor" />
        )}
      </Button>
    </div>
  );
}

function AudioInner(props: AudioProps) {
  const { variant = "full", className, onMediaEvent, ...serializable } = props;

  const {
    id,
    src,
    title,
    description,
    artwork,
    locale: providedLocale,
  } = serializable;

  const locale = providedLocale ?? FALLBACK_LOCALE;

  const { state, setState, setAudioElement } = useAudio();
  const audioRef = React.useRef<HTMLAudioElement | null>(null);
  const [currentTime, setCurrentTime] = React.useState(0);
  const [duration, setDuration] = React.useState(0);
  const [isSeeking, setIsSeeking] = React.useState(false);

  React.useEffect(() => {
    setAudioElement(audioRef.current);
    return () => setAudioElement(null);
  }, [setAudioElement]);

  React.useEffect(() => {
    const audio = audioRef.current;
    if (!audio) return;
    if (state.playing && audio.paused) {
      void audio.play().catch(() => undefined);
    } else if (!state.playing && !audio.paused) {
      audio.pause();
    }
  }, [state.playing]);

  const handlePlayPause = () => {
    const audio = audioRef.current;
    if (!audio) return;
    if (audio.paused) {
      void audio.play().catch(() => undefined);
    } else {
      audio.pause();
    }
  };

  const handleSeek = (value: number[]) => {
    const audio = audioRef.current;
    if (!audio) return;
    const newTime = value[0];
    audio.currentTime = newTime;
    setCurrentTime(newTime);
  };

  const handleSeekStart = () => {
    setIsSeeking(true);
  };

  const handleSeekEnd = () => {
    setIsSeeking(false);
  };

  const controls: PlayerControls = {
    isPlaying: state.playing,
    currentTime,
    duration,
    onPlayPause: handlePlayPause,
    onSeek: handleSeek,
    onSeekStart: handleSeekStart,
    onSeekEnd: handleSeekEnd,
  };

  const isCompact = variant === "compact";

  return (
    <article
      className={cn(
        "@container/actions relative w-full",
        isCompact ? "min-w-72 max-w-md" : "min-w-52 max-w-sm",
        className,
      )}
      lang={locale}
      data-tool-ui-id={id}
      data-slot="audio"
    >
      <div
        className={cn(
          "group @container relative isolate flex w-full min-w-0 flex-col overflow-hidden",
          "border-border bg-card border text-sm shadow-xs",
          "rounded-xl",
        )}
      >
        {isCompact ? (
          <CompactPlayer
            artwork={artwork}
            title={title}
            description={description}
            controls={controls}
          />
        ) : (
          <FullPlayer
            artwork={artwork}
            title={title}
            description={description}
            controls={controls}
          />
        )}

        <audio
          ref={audioRef}
          src={src}
          preload="metadata"
          className="hidden"
          onPlay={() => {
            setState({ playing: true });
            onMediaEvent?.("play");
          }}
          onPause={() => {
            setState({ playing: false });
            onMediaEvent?.("pause");
          }}
          onTimeUpdate={(event) => {
            if (!isSeeking) {
              setCurrentTime(event.currentTarget.currentTime);
            }
          }}
          onLoadedMetadata={(event) => {
            setDuration(event.currentTarget.duration);
          }}
          onDurationChange={(event) => {
            setDuration(event.currentTarget.duration);
          }}
        />
      </div>
    </article>
  );
}

"use client";

import * as React from "react";

export interface AudioPlaybackState {
  playing: boolean;
  muted: boolean;
}

export interface AudioContextValue {
  state: AudioPlaybackState;
  setState: (patch: Partial<AudioPlaybackState>) => void;
  audioElement: HTMLAudioElement | null;
  setAudioElement: (node: HTMLAudioElement | null) => void;
}

const AudioContext = React.createContext<AudioContextValue | null>(null);

export function useAudio() {
  const ctx = React.use(AudioContext);
  if (!ctx) {
    throw new Error("useAudio must be used within an <AudioProvider />");
  }
  return ctx;
}

export interface AudioProviderProps {
  children: React.ReactNode;
  defaultState?: Partial<AudioPlaybackState>;
}

export function AudioProvider({ children, defaultState }: AudioProviderProps) {
  const [state, setStateInternal] = React.useState<AudioPlaybackState>({
    playing: defaultState?.playing ?? false,
    muted: defaultState?.muted ?? false,
  });

  const [audioElement, setAudioElement] =
    React.useState<HTMLAudioElement | null>(null);

  const setState = React.useCallback((patch: Partial<AudioPlaybackState>) => {
    setStateInternal((prev) => ({ ...prev, ...patch }));
  }, []);

  const value = React.useMemo(
    () => ({ state, setState, audioElement, setAudioElement }),
    [state, setState, audioElement],
  );

  return (
    <AudioContext.Provider value={value}>{children}</AudioContext.Provider>
  );
}

export { Audio } from "./audio";
export type { AudioProps } from "./audio";
export { AudioProvider, useAudio } from "./context";
export type { AudioPlaybackState, AudioContextValue } from "./context";
export type { SerializableAudio, Source, AudioVariant } from "./schema";

# Audio

Implementation for the "audio" Tool UI surface.

## Files

- public exports: components/tool-ui/audio/index.ts
- serializable schema + parse helpers: components/tool-ui/audio/schema.ts

## Companion assets

- Docs page: app/docs/audio/content.mdx
- Preset payload: lib/presets/audio.ts

## Quick check

Run this after edits:

pnpm test

import { z } from "zod";
import { defineToolUiContract } from "../shared/contract";
import {
  ToolUIIdSchema,
  ToolUIReceiptSchema,
  ToolUIRoleSchema,
} from "../shared/schema";

export const SourceSchema = z.object({
  label: z.string(),
  iconUrl: z.url().optional(),
  url: z.url().optional(),
});

export type Source = z.infer<typeof SourceSchema>;

export const SerializableAudioSchema = z.object({
  id: ToolUIIdSchema,
  role: ToolUIRoleSchema.optional(),
  receipt: ToolUIReceiptSchema.optional(),
  assetId: z.string(),
  src: z.url(),
  title: z.string().optional(),
  description: z.string().optional(),
  artwork: z.url().optional(),
  durationMs: z.number().int().positive().optional(),
  fileSizeBytes: z.number().int().positive().optional(),
  createdAt: z.string().datetime().optional(),
  locale: z.string().optional(),
  source: SourceSchema.optional(),
});

export type SerializableAudio = z.infer<typeof SerializableAudioSchema>;

const SerializableAudioSchemaContract = defineToolUiContract(
  "Audio",
  SerializableAudioSchema,
);

export const parseSerializableAudio: (input: unknown) => SerializableAudio =
  SerializableAudioSchemaContract.parse;

export const safeParseSerializableAudio: (
  input: unknown,
) => SerializableAudio | null = SerializableAudioSchemaContract.safeParse;
export type AudioVariant = "full" | "compact";

import { z } from "zod";
import { parseWithSchema, safeParseWithSchema } from "./parse";

export interface ToolUiContract<T> {
  schema: z.ZodType<T>;
  parse: (input: unknown) => T;
  safeParse: (input: unknown) => T | null;
}

export function defineToolUiContract<T>(
  componentName: string,
  schema: z.ZodType<T>,
): ToolUiContract<T> {
  return {
    schema,
    parse: (input: unknown) => parseWithSchema(schema, input, componentName),
    safeParse: (input: unknown) => safeParseWithSchema(schema, input),
  };
}

import { z } from "zod";

function formatZodPath(path: Array<string | number | symbol>): string {
  if (path.length === 0) return "root";
  return path
    .map((segment) =>
      typeof segment === "number" ? `[${segment}]` : String(segment),
    )
    .join(".");
}

/**
 * Format Zod errors into a compact `path: message` string.
 */
export function formatZodError(error: z.ZodError): string {
  const parts = error.issues.map((issue) => {
    const path = formatZodPath(issue.path);
    return `${path}: ${issue.message}`;
  });

  return Array.from(new Set(parts)).join("; ");
}

/**
 * Parse unknown input and throw a readable error.
 */
export function parseWithSchema<T>(
  schema: z.ZodType<T>,
  input: unknown,
  name: string,
): T {
  const res = schema.safeParse(input);
  if (!res.success) {
    throw new Error(`Invalid ${name} payload: ${formatZodError(res.error)}`);
  }
  return res.data;
}

/**
 * Parse unknown input, returning `null` instead of throwing on failure.
 *
 * Use this in assistant-ui `render` functions where `args` stream in
 * incrementally and may be incomplete until the tool call finishes.
 */
export function safeParseWithSchema<T>(
  schema: z.ZodType<T>,
  input: unknown,
): T | null {
  const res = schema.safeParse(input);
  return res.success ? res.data : null;
}

import { z } from "zod";
import type { ReactNode } from "react";

/**
 * Tool UI conventions:
 * - Serializable schemas are JSON-safe (no callbacks/ReactNode/`className`).
 * - Schema: `SerializableXSchema`
 * - Parser: `parseSerializableX(input: unknown)` (throws on invalid)
 * - Safe parser: `safeParseSerializableX(input: unknown)` (returns `null` on invalid)
 * - Actions: `LocalActions` for non-receipt actions and `DecisionActions` for consequential actions
 * - Root attrs: `data-tool-ui-id` + `data-slot`
 */

/**
 * Schema for tool UI identity.
 *
 * Every tool UI should have a unique identifier that:
 * - Is stable across re-renders
 * - Is meaningful (not auto-generated)
 * - Is unique within the conversation
 *
 * Format recommendation: `{component-type}-{semantic-identifier}`
 * Examples: "data-table-expenses-q3", "option-list-deploy-target"
 */
export const ToolUIIdSchema = z.string().min(1);

export type ToolUIId = z.infer<typeof ToolUIIdSchema>;

/**
 * Primary role of a Tool UI surface in a chat context.
 */
export const ToolUIRoleSchema = z.enum([
  "information",
  "decision",
  "control",
  "state",
  "composite",
]);

export type ToolUIRole = z.infer<typeof ToolUIRoleSchema>;

export const ToolUIReceiptOutcomeSchema = z.enum([
  "success",
  "partial",
  "failed",
  "cancelled",
]);

export type ToolUIReceiptOutcome = z.infer<typeof ToolUIReceiptOutcomeSchema>;

/**
 * Optional receipt metadata: a durable summary of an outcome.
 */
export const ToolUIReceiptSchema = z.object({
  outcome: ToolUIReceiptOutcomeSchema,
  summary: z.string().min(1),
  identifiers: z.record(z.string(), z.string()).optional(),
  at: z.string().datetime(),
});

export type ToolUIReceipt = z.infer<typeof ToolUIReceiptSchema>;

/**
 * Base schema for Tool UI payloads (id + optional role/receipt).
 */
export const ToolUISurfaceSchema = z.object({
  id: ToolUIIdSchema,
  role: ToolUIRoleSchema.optional(),
  receipt: ToolUIReceiptSchema.optional(),
});

export type ToolUISurface = z.infer<typeof ToolUISurfaceSchema>;

export const ActionSchema = z.object({
  id: z.string().min(1),
  label: z.string().min(1),
  /**
   * Canonical narration the assistant can use after this action is taken.
   *
   * Example: "I exported the table as CSV." / "I opened the link in a new tab."
   */
  sentence: z.string().optional(),
  confirmLabel: z.string().optional(),
  variant: z
    .enum(["default", "destructive", "secondary", "ghost", "outline"])
    .optional(),
  icon: z.custom<ReactNode>().optional(),
  loading: z.boolean().optional(),
  disabled: z.boolean().optional(),
  shortcut: z.string().optional(),
});

export type Action = z.infer<typeof ActionSchema>;
export type LocalAction = Action;
export type DecisionAction = Action;

export const DecisionResultSchema = z.object({
  kind: z.literal("decision"),
  version: z.literal(1),
  decisionId: z.string().min(1),
  actionId: z.string().min(1),
  actionLabel: z.string().min(1),
  at: z.string().datetime(),
  payload: z.record(z.string(), z.unknown()).optional(),
});

export type DecisionResult<
  TPayload extends Record<string, unknown> = Record<string, unknown>,
> = Omit<z.infer<typeof DecisionResultSchema>, "payload"> & {
  payload?: TPayload;
};

export function createDecisionResult<
  TPayload extends Record<string, unknown> = Record<string, unknown>,
>(args: {
  decisionId: string;
  action: { id: string; label: string };
  payload?: TPayload;
}): DecisionResult<TPayload> {
  return {
    kind: "decision",
    version: 1,
    decisionId: args.decisionId,
    actionId: args.action.id,
    actionLabel: args.action.label,
    at: new Date().toISOString(),
    payload: args.payload,
  };
}

export const ActionButtonsPropsSchema = z.object({
  actions: z.array(ActionSchema).min(1),
  align: z.enum(["left", "center", "right"]).optional(),
  confirmTimeout: z.number().positive().optional(),
  className: z.string().optional(),
});

export const SerializableActionSchema = ActionSchema.omit({ icon: true });
export const SerializableActionsSchema = ActionButtonsPropsSchema.extend({
  actions: z.array(SerializableActionSchema),
}).omit({ className: true });

export interface ActionsConfig {
  items: Action[];
  align?: "left" | "center" | "right";
  confirmTimeout?: number;
}

export const SerializableActionsConfigSchema = z.object({
  items: z.array(SerializableActionSchema).min(1),
  align: z.enum(["left", "center", "right"]).optional(),
  confirmTimeout: z.number().positive().optional(),
});

export type SerializableActionsConfig = z.infer<
  typeof SerializableActionsConfigSchema
>;

export type SerializableAction = z.infer<typeof SerializableActionSchema>;