Multimodal Vision Playground
blockaiGPT-4o/Claude 3.5 Vision multimodal visual analysis studio with interactive bounding box overlays, OCR entity extraction, tabular line item parsing, raw RFC-8259 JSON viewer, and visual question answering (VQA).
Installation
$pnpm dlx uipkge-ng@latest add multimodal-vision-playground$npx uipkge-ng@latest add multimodal-vision-playground$yarn dlx uipkge-ng@latest add multimodal-vision-playground$bunx uipkge-ng@latest add multimodal-vision-playgroundManifest:
https://uipkge.dev/r/angular/multimodal-vision-playground.jsonInstalls to:src/app/components/blocks/multimodal-vision-playground/Variants
Loading interactive previews…
Loading interactive previews…
Schema
Type aliases exported from this item's source. Use these to shape the data you pass in.
ModelOptioninterface ModelOption {
id: string
name: string
provider: string
latency: string
tokenUsage: number
meanConfidence: number
}VisionTaskOptioninterface VisionTaskOption {
id: string
name: string
description: string
}BoundingBoxinterface BoundingBox {
id: string
label: string
text: string
category: string
confidence: number
coordinates: string
top: string
left: string
width: string
height: string
colorBorder: string
colorBg: string
colorBadge: string
}ExtractedEntityinterface ExtractedEntity {
id: string
fieldName: string
extractedValue: string
confidence: number
boxId: string
dataType: string
}LineIteminterface LineItem {
id: number
description: string
qty: number
unit: string
unitPrice: number
totalPrice: number
confidence: number
}VqaPresetinterface VqaPreset {
id: string
question: string
answer: string
targetBoxId: string
}Files installed (2)
src/app/components/blocks/multimodal-vision-playground/multimodal-vision-playground.component.ts51 kB
// GPT-4o/Claude 3.5 Vision multimodal visual analysis studio with interactive bounding // box overlays, OCR entity extraction, tabular line item parsing, raw RFC-8259 JSON viewer, // and visual question answering (VQA). Port of the React block 1:1. import { ChangeDetectionStrategy, Component, DestroyRef, Input, OnDestroy, OnInit, computed, inject, signal, } from '@angular/core' import { Bot, Braces, Check, CheckCircle2, Copy, Cpu, Crosshair, Eye, FileCheck, Info, Layers, LucideAngularModule, RefreshCw, Scan, Send, ShieldCheck, Sparkles, Tag, } from 'lucide-angular' import { cn } from '@/lib/utils' import { UiBadgeComponent } from '@/ui/badge/badge.component' import { UiButtonComponent } from '@/ui/button/button.component' import { UiCardComponent, UiCardContentComponent, UiCardDescriptionComponent, UiCardFooterComponent, UiCardHeaderComponent, UiCardTitleComponent, } from '@/ui/card/card.component' import { UiInputComponent } from '@/ui/input/input.component' import { UiSelectComponent, UiSelectContentComponent, UiSelectItemComponent, UiSelectTriggerComponent, UiSelectValueComponent, } from '@/ui/select/select.component' import { UiSeparatorComponent } from '@/ui/separator/separator.component' import { UiSwitchComponent } from '@/ui/switch/switch.component' import { UiTableBodyComponent, UiTableCellComponent, UiTableComponent, UiTableHeadComponent, UiTableHeaderComponent, UiTableRowComponent, } from '@/ui/table/table.component' import { UiTabsComponent, UiTabsContentComponent, UiTabsListComponent, UiTabsTriggerComponent, } from '@/ui/tabs/tabs.component' interface ModelOption { id: string name: string provider: string latency: string tokenUsage: number meanConfidence: number } interface VisionTaskOption { id: string name: string description: string } interface BoundingBox { id: string label: string text: string category: string confidence: number coordinates: string top: string left: string width: string height: string colorBorder: string colorBg: string colorBadge: string } interface ExtractedEntity { id: string fieldName: string extractedValue: string confidence: number boxId: string dataType: string } interface LineItem { id: number description: string qty: number unit: string unitPrice: number totalPrice: number confidence: number } interface VqaPreset { id: string question: string answer: string targetBoxId: string } const models: ModelOption[] = [ { id: 'claude-3-5-sonnet-vision', name: 'Claude 3.5 Sonnet Vision', provider: 'Anthropic', latency: '642ms', tokenUsage: 1420, meanConfidence: 99.4, }, { id: 'gpt-4o-vision', name: 'GPT-4o Vision', provider: 'OpenAI', latency: '584ms', tokenUsage: 1380, meanConfidence: 99.2, }, ] const visionTasks: VisionTaskOption[] = [ { id: 'ocr-extraction', name: 'Document OCR & Extraction', description: 'High-precision layout bounding boxes, key-value extraction & structured RFC-8259 JSON parsing.', }, { id: 'object-detection', name: 'Object & Bounding Box Detection', description: 'Semantic region segmentation, visual entity localization, and coordinate spatial mapping.', }, { id: 'vqa', name: 'Visual Question Answering', description: 'Multi-step visual reasoning and spatial evidence grounding directly on the document image.', }, ] const boundingBoxes: BoundingBox[] = [ { id: 'box-logo', label: 'Vendor Logo', text: 'APEX CLOUD SYSTEMS INC.', category: 'vendor', confidence: 99.9, coordinates: '[42, 48, 110, 320]', top: '4%', left: '4%', width: '42%', height: '11%', colorBorder: 'border-warning/70', colorBg: 'bg-warning/10', colorBadge: 'bg-warning/15 text-warning border-warning/30', }, { id: 'box-inv', label: 'Invoice #', text: 'INV-84920', category: 'identifier', confidence: 99.8, coordinates: '[42, 680, 110, 1140]', top: '4%', left: '54%', width: '42%', height: '11%', colorBorder: 'border-info/70', colorBg: 'bg-info/10', colorBadge: 'bg-info/15 text-info border-info/30', }, { id: 'box-billto', label: 'Customer Entity', text: 'Cyberdyne Systems Corp', category: 'customer', confidence: 99.3, coordinates: '[145, 48, 250, 480]', top: '17%', left: '4%', width: '46%', height: '13%', colorBorder: 'border-info/70', colorBg: 'bg-info/10', colorBadge: 'bg-info/15 text-info border-info/30', }, { id: 'box-due', label: 'Due Date & Terms', text: 'Due: 2026-09-01 (Net 10)', category: 'date', confidence: 99.4, coordinates: '[145, 680, 250, 1140]', top: '17%', left: '54%', width: '42%', height: '13%', colorBorder: 'border-destructive/70', colorBg: 'bg-destructive/10', colorBadge: 'bg-destructive/15 text-destructive border-destructive/30', }, { id: 'box-table', label: 'Line Items Table', text: '3 Items (GPU, NVMe, Bandwidth)', category: 'table', confidence: 99.5, coordinates: '[320, 48, 710, 1140]', top: '32%', left: '4%', width: '92%', height: '39%', colorBorder: 'border-chart-1/70', colorBg: 'bg-chart-1/10', colorBadge: 'bg-chart-1/15 text-chart-1 border-chart-1/30', }, { id: 'box-subtotal', label: 'Subtotal & VAT', text: 'Subtotal: $1,150.00 · VAT: $95.00', category: 'financial', confidence: 99.1, coordinates: '[730, 620, 830, 1140]', top: '73%', left: '50%', width: '46%', height: '10%', colorBorder: 'border-chart-2/70', colorBg: 'bg-chart-2/10', colorBadge: 'bg-chart-2/15 text-chart-2 border-chart-2/30', }, { id: 'box-total', label: 'Total Amount $1,245.00', text: 'Total Due: USD $1,245.00', category: 'total', confidence: 99.9, coordinates: '[850, 620, 960, 1140]', top: '85%', left: '50%', width: '46%', height: '11%', colorBorder: 'border-success/80', colorBg: 'bg-success/15', colorBadge: 'bg-success/20 text-success border-success/40 font-semibold', }, ] const extractedEntities: ExtractedEntity[] = [ { id: 'ent-1', fieldName: 'Invoice Number', extractedValue: 'INV-84920', confidence: 99.8, boxId: 'box-inv', dataType: 'string', }, { id: 'ent-2', fieldName: 'Due Date', extractedValue: '2026-09-01', confidence: 99.4, boxId: 'box-due', dataType: 'date', }, { id: 'ent-3', fieldName: 'Issue Date', extractedValue: '2026-08-21', confidence: 99.6, boxId: 'box-due', dataType: 'date', }, { id: 'ent-4', fieldName: 'Vendor Name', extractedValue: 'Apex Cloud Systems Inc.', confidence: 99.9, boxId: 'box-logo', dataType: 'string', }, { id: 'ent-5', fieldName: 'Vendor Tax ID', extractedValue: 'US-EIN-94-8271923', confidence: 98.9, boxId: 'box-logo', dataType: 'string', }, { id: 'ent-6', fieldName: 'Customer Entity', extractedValue: 'Cyberdyne Systems Corp', confidence: 99.3, boxId: 'box-billto', dataType: 'string', }, { id: 'ent-7', fieldName: 'Subtotal Amount', extractedValue: '$1,150.00', confidence: 99.7, boxId: 'box-subtotal', dataType: 'currency', }, { id: 'ent-8', fieldName: 'VAT / Tax Rate (8.25%)', extractedValue: '$95.00', confidence: 99.1, boxId: 'box-subtotal', dataType: 'currency', }, { id: 'ent-9', fieldName: 'Total Amount Due', extractedValue: '$1,245.00', confidence: 99.9, boxId: 'box-total', dataType: 'currency', }, ] const lineItems: LineItem[] = [ { id: 1, description: 'GPU Compute Instance (H100 NVLink 80GB)', qty: 2, unit: 'hours', unitPrice: 450.0, totalPrice: 900.0, confidence: 99.5, }, { id: 2, description: 'High-Throughput NVMe Block Storage 4TB', qty: 1, unit: 'month', unitPrice: 150.0, totalPrice: 150.0, confidence: 99.2, }, { id: 3, description: 'Dedicated Egress Bandwidth Allocation (10Gbps)', qty: 1, unit: 'allocation', unitPrice: 100.0, totalPrice: 100.0, confidence: 98.8, }, ] const vqaPresets: VqaPreset[] = [ { id: 'q-total', question: 'What is the total amount due on this invoice?', answer: 'The total amount due is $1,245.00 USD, parsed from the summary block with 99.9% confidence. It includes $1,150.00 subtotal plus $95.00 VAT (8.25%).', targetBoxId: 'box-total', }, { id: 'q-due', question: 'What is the payment due date and terms?', answer: 'The invoice payment due date is September 01, 2026 under Net 10 Day terms (issued on August 21, 2026), extracted with 99.4% confidence.', targetBoxId: 'box-due', }, { id: 'q-math', question: 'Are all line item totals and taxes mathematically verified?', answer: 'Yes. Calculated ($450 × 2) + $150 + $100 = $1,150.00 subtotal. VAT of 8.25% ($94.88 rounded to $95.00) brings grand total to $1,245.00 with 0 arithmetic errors.', targetBoxId: 'box-table', }, { id: 'q-vendor', question: 'What is the verified vendor entity and tax identifier?', answer: 'Vendor is Apex Cloud Systems Inc. with verified Federal Tax EIN: US-EIN-94-8271923, extracted with 99.9% confidence.', targetBoxId: 'box-logo', }, ] @Component({ changeDetection: ChangeDetectionStrategy.Eager, selector: 'ui-multimodal-vision-playground, [ui-multimodal-vision-playground]', standalone: true, // React renders the root <div> itself: the host stays out of layout and `class` goes to the root. host: { '[attr.class]': '"contents"' }, imports: [ LucideAngularModule, UiBadgeComponent, UiButtonComponent, UiCardComponent, UiCardContentComponent, UiCardDescriptionComponent, UiCardFooterComponent, UiCardHeaderComponent, UiCardTitleComponent, UiInputComponent, UiSelectComponent, UiSelectContentComponent, UiSelectItemComponent, UiSelectTriggerComponent, UiSelectValueComponent, UiSeparatorComponent, UiSwitchComponent, UiTableBodyComponent, UiTableCellComponent, UiTableComponent, UiTableHeadComponent, UiTableHeaderComponent, UiTableRowComponent, UiTabsComponent, UiTabsContentComponent, UiTabsListComponent, UiTabsTriggerComponent, ], template: ` <div data-slot="multimodal-vision-playground" [class]="rootClass"> <!-- Header Controls Strip --> <div class="border-border bg-card flex flex-wrap items-center justify-between gap-4 rounded-lg border p-4 shadow-xs" > <div class="flex min-w-0 flex-1 flex-wrap items-center gap-3"> <div class="flex items-center gap-2.5"> <div class="bg-primary/10 text-primary flex size-8 items-center justify-center rounded-md"> <lucide-icon [img]="Scan" class="size-4.5" /> </div> <div class="min-w-[12rem]"> <h2 class="text-foreground text-sm font-semibold tracking-tight"> Multimodal Vision & Document Intelligence </h2> <p class="text-muted-foreground text-xs">Visual reasoning, layout parsing & entity extraction studio</p> </div> </div> <ui-separator orientation="vertical" class="hidden h-7 sm:block" /> <!-- Model Selector --> <div class="w-52"> <ui-select [value]="selectedModel()" (valueChange)="selectedModel.set($event)"> <button ui-select-trigger class="h-9 w-full text-xs font-medium" aria-label="Select Vision Model"> <ui-select-value placeholder="Select Vision Model" /> </button> <ui-select-content> @for (m of models; track m.id) { <ui-select-item [value]="m.id"> <div class="flex items-center gap-2"> <lucide-icon [img]="Sparkles" class="text-primary size-3.5" /> <span>{{ m.name }}</span> </div> </ui-select-item> } </ui-select-content> </ui-select> </div> <!-- Vision Task Selector --> <div class="w-56"> <ui-select [value]="selectedTask()" (valueChange)="selectedTask.set($event)"> <button ui-select-trigger class="h-9 w-full text-xs" aria-label="Select Vision Task"> <ui-select-value placeholder="Select Vision Task" /> </button> <ui-select-content> @for (task of visionTasks; track task.id) { <ui-select-item [value]="task.id"> <span>{{ task.name }}</span> </ui-select-item> } </ui-select-content> </ui-select> </div> <!-- Telemetry Pill --> <div class="border-border bg-muted/60 text-muted-foreground hidden items-center gap-2 rounded-full border px-3 py-1 font-mono text-xs lg:inline-flex" > <lucide-icon [img]="Cpu" class="text-primary size-3.5" /> <span>{{ activeModel().latency }}</span> <span>{{ activeModel().tokenUsage }} tokens</span> <span class="text-success font-medium">{{ activeModel().meanConfidence }}% conf</span> </div> </div> <div class="flex items-center gap-2"> <!-- Run Vision Model Primary Button --> <button ui-button size="sm" class="bg-primary text-primary-foreground hover:bg-primary/90 h-9 gap-1.5 text-xs font-medium shadow-xs" [disabled]="isAnalyzing()" (click)="runVisionAnalysis()" > @if (isAnalyzing()) { <lucide-icon [img]="RefreshCw" class="size-3.5 animate-spin" /> } @else { <lucide-icon [img]="Eye" class="size-3.5" /> } <span>{{ isAnalyzing() ? 'Analyzing Image...' : 'Run Vision Model' }}</span> <kbd class="bg-primary-foreground/20 hidden items-center rounded px-1 py-0.5 font-mono text-xs sm:inline-flex" > ⌘↵ </kbd> </button> </div> </div> <!-- 2-Column Vision Studio (50% / 50%) --> <div class="grid grid-cols-1 gap-6 lg:grid-cols-12"> <!-- Left Column: Visual Document Preview & Interactive Bounding Box Canvas (50%) --> <div class="space-y-4 lg:col-span-6"> <div ui-card class="border-border bg-card overflow-hidden shadow-xs"> <div ui-card-header class="pb-3"> <div class="flex flex-wrap items-center justify-between gap-2"> <div class="flex items-center gap-2"> <div class="bg-primary/10 text-primary flex size-7 items-center justify-center rounded-md"> <lucide-icon [img]="Layers" class="size-4" /> </div> <div> <h3 ui-card-title class="text-sm font-semibold">Visual Document Canvas</h3> <p ui-card-description class="text-xs">Spatial bounding polygon detection & OCR anchor layer</p> </div> </div> <span ui-badge variant="outline" class="text-muted-foreground font-mono text-xs"> 1240 × 1754 · 300 DPI </span> </div> <!-- Bounding Box Layer Toggles --> <div class="border-border/80 bg-muted/40 mt-3 flex flex-wrap items-center justify-between gap-3 rounded-md border px-3 py-2 text-xs" > <span class="text-muted-foreground flex items-center gap-1.5 font-medium"> <lucide-icon [img]="Crosshair" class="text-primary size-3.5" /> <span>Layer Overlays:</span> </span> <div class="flex flex-wrap items-center gap-4"> <!-- Toggle: OCR Text --> <div class="flex items-center gap-1.5"> <ui-switch id="angular-show-ocr" [checked]="showOcrText()" (checkedChange)="showOcrText.set($event)" /> <label for="angular-show-ocr" class="text-foreground cursor-pointer text-xs font-medium"> OCR Text </label> </div> <!-- Toggle: Confidence Scores --> <div class="flex items-center gap-1.5"> <ui-switch id="angular-show-conf" [checked]="showConfidence()" (checkedChange)="showConfidence.set($event)" /> <label for="angular-show-conf" class="text-foreground cursor-pointer text-xs font-medium"> Confidence </label> </div> <!-- Toggle: Coordinates --> <div class="flex items-center gap-1.5"> <ui-switch id="angular-show-coords" [checked]="showCoordinates()" (checkedChange)="showCoordinates.set($event)" /> <label for="angular-show-coords" class="text-foreground cursor-pointer text-xs font-medium"> Coordinates </label> </div> </div> </div> </div> <div ui-card-content class="pt-0"> <!-- Document Viewport Container --> <div class="border-border bg-muted/30 relative w-full overflow-hidden rounded-lg border p-3 select-none sm:p-4" > <!-- The Rendered High-Fidelity Document Sheet --> <div class="border-border/90 bg-card text-card-foreground relative mx-auto flex aspect-[1/1.36] min-h-[420px] w-full max-w-[560px] max-w-full flex-col justify-between overflow-auto rounded-md border p-4 shadow-sm sm:p-6" > <!-- Watermark Background Texture --> <div class="pointer-events-none absolute inset-0 bg-[radial-gradient(#000_1px,transparent_1px)] [background-size:16px_16px] opacity-[0.03] dark:opacity-[0.05]" ></div> <!-- Document Header Section --> <div class="border-border/70 relative z-10 flex items-start justify-between border-b pb-4"> <!-- Vendor Brand Info --> <div class="space-y-1"> <div class="flex items-center gap-2"> <div class="bg-primary/20 text-primary flex size-6 items-center justify-center rounded text-xs font-bold" > ▲ </div> <span class="text-foreground text-xs font-bold tracking-wider uppercase"> Apex Cloud Systems </span> </div> <p class="text-muted-foreground text-xs">500 Howard St, Suite 400</p> <p class="text-muted-foreground text-xs">San Francisco, CA 94105</p> <p class="text-muted-foreground font-mono text-xs">EIN: US-94-8271923</p> </div> <!-- Invoice Meta --> <div class="space-y-1 text-right"> <div class="bg-primary/10 text-primary inline-block rounded px-2 py-0.5 text-xs font-bold"> TAX INVOICE </div> <p class="text-foreground font-mono text-xs font-semibold">INV-84920</p> <p class="text-muted-foreground text-xs">Issued: Aug 21, 2026</p> <p class="text-destructive text-xs font-medium">Due: Sep 01, 2026</p> </div> </div> <!-- Bill To & Account Details --> <div class="border-border/60 relative z-10 grid grid-cols-1 gap-3 border-b py-3 text-xs sm:grid-cols-2" > <div> <span class="text-muted-foreground text-xs font-semibold uppercase">Billed To:</span> <p class="text-foreground mt-0.5 font-semibold">Cyberdyne Systems Corp</p> <p class="text-muted-foreground">100 Cyberdyne Blvd, Suite 800</p> <p class="text-muted-foreground">San Francisco, CA 94107</p> </div> <div class="text-right"> <span class="text-muted-foreground text-xs font-semibold uppercase">Payment Terms:</span> <p class="text-foreground mt-0.5 font-medium">Net 10 Days (Wire Transfer)</p> <p class="text-muted-foreground font-mono">PO Ref: PO-2026-90412</p> <p class="text-muted-foreground">Currency: USD ($)</p> </div> </div> <!-- Document Line Items Table Mockup --> <div class="relative z-10 flex-1 py-3"> <div class="w-full text-xs"> <div class="border-border/80 text-muted-foreground grid grid-cols-12 gap-1 border-b pb-1.5 font-semibold" > <div class="col-span-6">Item & Description</div> <div class="col-span-2 text-center">Qty</div> <div class="col-span-2 text-right">Rate</div> <div class="col-span-2 text-right">Amount</div> </div> <div class="divide-border/40 divide-y text-xs"> <div class="grid grid-cols-12 items-center gap-1 py-2"> <div class="col-span-6"> <p class="text-foreground font-medium">GPU Instance (H100 NVLink 80GB)</p> <p class="text-muted-foreground text-xs">Dedicated high-speed cluster</p> </div> <div class="col-span-2 text-center font-mono">2 hrs</div> <div class="col-span-2 text-right font-mono">$450.00</div> <div class="col-span-2 text-right font-mono font-medium">$900.00</div> </div> <div class="grid grid-cols-12 items-center gap-1 py-2"> <div class="col-span-6"> <p class="text-foreground font-medium">High-Throughput NVMe Storage</p> <p class="text-muted-foreground text-xs">4TB Tier-1 SSD Volume</p> </div> <div class="col-span-2 text-center font-mono">1 mo</div> <div class="col-span-2 text-right font-mono">$150.00</div> <div class="col-span-2 text-right font-mono font-medium">$150.00</div> </div> <div class="grid grid-cols-12 items-center gap-1 py-2"> <div class="col-span-6"> <p class="text-foreground font-medium">Dedicated Egress Bandwidth</p> <p class="text-muted-foreground text-xs">10Gbps Burstable Pipe</p> </div> <div class="col-span-2 text-center font-mono">1 alloc</div> <div class="col-span-2 text-right font-mono">$100.00</div> <div class="col-span-2 text-right font-mono font-medium">$100.00</div> </div> </div> </div> </div> <!-- Financial Calculation Bottom Strip --> <div class="border-border/80 relative z-10 flex items-end justify-between border-t pt-3"> <div class="space-y-1"> <div class="text-muted-foreground flex items-center gap-1.5 text-xs"> <lucide-icon [img]="ShieldCheck" class="text-success size-3.5" /> <span>Cryptographic Checksum: 0x8F9A2B</span> </div> <p class="text-muted-foreground text-xs">Wire: Wells Fargo Routing 121000358</p> </div> <div class="w-48 space-y-1 text-right text-xs"> <div class="text-muted-foreground flex justify-between"> <span>Subtotal:</span> <span class="text-foreground font-mono font-medium">$1,150.00</span> </div> <div class="text-muted-foreground flex justify-between"> <span>VAT (8.25%):</span> <span class="text-foreground font-mono font-medium">$95.00</span> </div> <div class="border-border text-foreground flex justify-between border-t pt-1 text-xs font-bold sm:text-sm" > <span>Total Due:</span> <span class="text-success font-mono">$1,245.00</span> </div> </div> </div> <!-- INTERACTIVE BOUNDING BOX OVERLAYS LAYER --> <div class="pointer-events-auto absolute inset-0 z-20"> @for (box of boundingBoxes; track box.id) { <div tabindex="0" role="button" [attr.aria-label]="box.label + ': ' + box.text" [style.top]="box.top" [style.left]="box.left" [style.width]="box.width" [style.height]="box.height" [class]="boxClass(box)" (mouseenter)="activeBoxId.set(box.id)" (mouseleave)="activeBoxId.set(null)" (click)="activeBoxId.set(box.id)" (focus)="activeBoxId.set(box.id)" (blur)="activeBoxId.set(null)" > <!-- Corner Anchor Dots --> <span class="bg-primary border-background absolute -top-1 -left-1 size-2 rounded-full border" ></span> <span class="bg-primary border-background absolute -top-1 -right-1 size-2 rounded-full border" ></span> <span class="bg-primary border-background absolute -bottom-1 -left-1 size-2 rounded-full border" ></span> <span class="bg-primary border-background absolute -right-1 -bottom-1 size-2 rounded-full border" ></span> <!-- Floating Metadata Tag on Box --> <div [class]="boxBadgeClass(box)"> <lucide-icon [img]="Tag" class="size-3" /> <span class="font-semibold">{{ box.label }}</span> @if (showConfidence()) { <span class="font-mono text-xs opacity-90"> {{ box.confidence }}% </span> } </div> <!-- OCR Text Preview Ribbon (Bottom) --> @if (showOcrText()) { <div class="bg-foreground/90 text-background py-0.2 absolute -bottom-5 left-0 max-w-full truncate rounded px-1.5 font-mono text-xs shadow-xs" > {{ box.text }} </div> } <!-- Coordinates Pill --> @if (showCoordinates() && activeBoxId() === box.id) { <div class="bg-background/95 border-border text-muted-foreground absolute top-1 right-1 rounded border px-1 py-0.5 font-mono text-xs shadow-xs" > {{ box.coordinates }} </div> } </div> } </div> </div> </div> </div> <div ui-card-footer class="border-border text-muted-foreground flex items-center justify-between border-t pt-3 text-xs" > <span class="flex items-center gap-1.5"> <lucide-icon [img]="Info" class="text-primary size-3.5" /> <span>Hover or click any bounding box to isolate extracted entities</span> </span> @if (activeBox(); as box) { <span class="text-foreground font-mono font-medium"> Active: {{ box.label }} ({{ box.confidence }}%) </span> } @else { <span class="font-mono text-xs">7 Bounding Anchors</span> } </div> </div> </div> <!-- Right Column: Structured Output & Analysis Studio (50%) --> <div class="space-y-4 lg:col-span-6"> <div ui-card class="border-border bg-card shadow-xs"> <div ui-card-header class="pb-3"> <div class="flex flex-wrap items-center justify-between gap-3"> <div class="flex items-center gap-2"> <div class="bg-primary/10 text-primary flex size-7 items-center justify-center rounded-md"> <lucide-icon [img]="Bot" class="size-4" /> </div> <div> <h3 ui-card-title class="text-sm font-semibold">Structured Output & Visual Analysis</h3> <p ui-card-description class="text-xs"> Multi-modal entity parsing, JSON schemas & visual reasoning </p> </div> </div> <!-- Status Badge --> <span ui-badge variant="outline" class="border-success/30 bg-success/10 text-success font-mono text-xs"> <lucide-icon [img]="CheckCircle2" class="text-success mr-1 size-3" /> Validated · 0 Errors </span> </div> </div> <div ui-card-content class="space-y-4 pt-0"> <ui-tabs [value]="activeTab()" (valueChange)="activeTab.set($event)" class="w-full"> <ui-tabs-list class="grid h-9 w-full grid-cols-4"> <button ui-tabs-trigger value="entities" class="text-xs">Key-Values</button> <button ui-tabs-trigger value="table" class="text-xs">Line Items</button> <button ui-tabs-trigger value="json" class="text-xs">Raw JSON</button> <button ui-tabs-trigger value="reasoning" class="text-xs">Reasoning</button> </ui-tabs-list> <!-- TAB 1: Extracted Key-Value Data Table --> <div ui-tabs-content value="entities" class="mt-4 space-y-3"> <div class="border-border overflow-hidden rounded-md border"> <div class="overflow-x-auto"> <table ui-table> <thead ui-table-header> <tr ui-table-row class="bg-muted/50 text-xs"> <th ui-table-head class="font-semibold">Field Name</th> <th ui-table-head class="font-semibold">Extracted Value</th> <th ui-table-head class="text-center font-semibold">Confidence</th> <th ui-table-head class="text-right font-semibold">Action</th> </tr> </thead> <tbody ui-table-body class="text-xs"> @for (field of extractedEntities; track field.id) { <tr ui-table-row [class]="entityRowClass(field)" (mouseenter)="activeBoxId.set(field.boxId)" (mouseleave)="activeBoxId.set(null)" > <td ui-table-cell class="text-foreground py-2.5 font-medium"> <div class="flex items-center gap-1.5"> <span [class]="entityDotClass(field)"></span> <span>{{ field.fieldName }}</span> </div> </td> <td ui-table-cell class="text-foreground py-2.5 font-mono"> {{ field.extractedValue }} </td> <td ui-table-cell class="py-2.5 text-center"> <span ui-badge variant="secondary" [class]="entityBadgeConfidenceClass(field)"> {{ field.confidence }}% </span> </td> <td ui-table-cell class="py-2.5 text-right"> <button ui-button variant="ghost" size="sm" class="text-muted-foreground hover:text-foreground h-7 w-7 p-0" aria-label="Copy field value" (click)="handleCopyEntity(field)" > @if (copiedFieldId() === field.id) { <lucide-icon [img]="Check" class="text-success size-3.5" /> } @else { <lucide-icon [img]="Copy" class="size-3.5" /> } </button> </td> </tr> } </tbody> </table> </div> </div> </div> <!-- TAB 2: Line Items Tabular Structure --> <div ui-tabs-content value="table" class="mt-4 space-y-3"> <div class="border-border overflow-hidden rounded-md border"> <div class="overflow-x-auto"> <table ui-table> <thead ui-table-header> <tr ui-table-row class="bg-muted/50 text-xs"> <th ui-table-head class="w-8 font-semibold">#</th> <th ui-table-head class="font-semibold">Description</th> <th ui-table-head class="text-center font-semibold">Qty</th> <th ui-table-head class="text-right font-semibold">Unit Price</th> <th ui-table-head class="text-right font-semibold">Total</th> </tr> </thead> <tbody ui-table-body class="text-xs"> @for (item of lineItems; track item.id) { <tr ui-table-row class="hover:bg-muted/40 cursor-pointer" (mouseenter)="activeBoxId.set('box-table')" (mouseleave)="activeBoxId.set(null)" > <td ui-table-cell class="text-muted-foreground py-2.5 font-mono">{{ item.id }}</td> <td ui-table-cell class="text-foreground py-2.5 font-medium"> <p>{{ item.description }}</p> <span class="text-muted-foreground font-mono text-xs"> Confidence: {{ item.confidence }}% </span> </td> <td ui-table-cell class="py-2.5 text-center font-mono">{{ item.qty }} {{ item.unit }}</td> <td ui-table-cell class="py-2.5 text-right font-mono"> {{ '$' + item.unitPrice.toFixed(2) }} </td> <td ui-table-cell class="text-foreground py-2.5 text-right font-mono font-semibold"> {{ '$' + item.totalPrice.toFixed(2) }} </td> </tr> } </tbody> </table> </div> </div> </div> <!-- TAB 3: Raw Structured JSON Schema --> <div ui-tabs-content value="json" class="mt-4 space-y-3"> <div class="bg-muted/60 border-border flex items-center justify-between rounded-t-md border border-b-0 px-3 py-2 text-xs" > <span class="text-muted-foreground flex items-center gap-1.5 font-mono"> <lucide-icon [img]="Braces" class="text-primary size-3.5" /> <span>rfc8259-invoice-schema.json</span> </span> <button ui-button variant="ghost" size="sm" class="text-muted-foreground hover:text-foreground h-7 px-2 text-xs" (click)="handleCopyJson()" > @if (isCopied()) { <lucide-icon [img]="Check" class="text-success mr-1 size-3.5" /> } @else { <lucide-icon [img]="Copy" class="mr-1 size-3.5" /> } <span>{{ isCopied() ? 'Copied JSON!' : 'Copy Schema' }}</span> </button> </div> <pre class="border-border bg-muted/30 text-foreground max-h-[320px] overflow-x-auto overflow-y-auto rounded-b-md border p-3.5 font-mono text-xs leading-relaxed" ><code>{{ rawJsonSchema() }}</code></pre> </div> <!-- TAB 4: Visual Reasoning & Question Answering --> <div ui-tabs-content value="reasoning" class="mt-4 space-y-4"> <!-- Prose Summary Box --> <div class="border-border/90 bg-muted/30 space-y-2.5 rounded-lg border p-3.5 text-xs"> <div class="flex items-center gap-2"> <lucide-icon [img]="Sparkles" class="text-primary size-4" /> <h4 class="text-foreground text-xs font-semibold">Multimodal Reasoning Summary</h4> </div> <p class="text-muted-foreground leading-relaxed"> Identified standard commercial invoice with 3 line items. VAT calculation validated with 0 arithmetic errors. Vendor entity verified against active directory with clean 0.0° skew angle and 300 DPI clarity. </p> <div class="border-border/60 grid grid-cols-1 gap-2 border-t pt-1 sm:grid-cols-3"> <div class="border-border/70 bg-card rounded border p-2"> <span class="text-muted-foreground text-xs">Arithmetic Check</span> <p class="text-success mt-0.5 font-semibold">100% Match</p> </div> <div class="border-border/70 bg-card rounded border p-2"> <span class="text-muted-foreground text-xs">Layout Skew</span> <p class="text-foreground mt-0.5 font-mono font-medium">0.0° (Aligned)</p> </div> <div class="border-border/70 bg-card rounded border p-2"> <span class="text-muted-foreground text-xs">Optical Clarity</span> <p class="text-foreground mt-0.5 font-semibold">300 DPI High</p> </div> </div> </div> <!-- Interactive VQA Console --> <div class="space-y-2.5"> <label class="text-foreground flex items-center justify-between text-xs font-semibold"> <span>Visual Question Answering (VQA):</span> <span class="text-muted-foreground text-xs font-normal">Click a sample prompt</span> </label> <!-- Preset Question Chips --> <div class="flex flex-wrap gap-1.5"> @for (preset of vqaPresets; track preset.id) { <button type="button" class="border-border/80 bg-muted/40 hover:border-primary/60 hover:bg-primary/5 rounded-md border px-2 py-1 text-left text-xs transition-colors" (click)="selectVqaPreset(preset)" > {{ preset.question }} </button> } </div> <!-- Query Input Box --> <div class="flex items-center gap-2"> <ui-input [value]="vqaInput()" (valueChange)="vqaInput.set($event)" placeholder="Ask a visual question about this document..." class="h-9 text-xs" (keydown.enter)="submitCustomVqa()" /> <button ui-button size="sm" class="h-9 gap-1.5 px-3 text-xs font-medium" [disabled]="isAnsweringVqa()" (click)="submitCustomVqa()" > @if (isAnsweringVqa()) { <lucide-icon [img]="RefreshCw" class="size-3.5 animate-spin" /> } @else { <lucide-icon [img]="Send" class="size-3.5" /> } <span>Ask</span> </button> </div> <!-- Reasoning Answer Box --> <div class="border-primary/20 bg-primary/5 text-foreground flex items-start gap-2.5 rounded-md border p-3 text-xs leading-relaxed" > <lucide-icon [img]="Bot" class="text-primary mt-0.5 size-4 shrink-0" /> <div> <p class="text-foreground font-medium">{{ currentVqaAnswer() }}</p> </div> </div> </div> </div> </ui-tabs> </div> <div ui-card-footer class="border-border text-muted-foreground flex items-center justify-between border-t pt-3 text-xs" > <span class="flex items-center gap-1.5"> <lucide-icon [img]="FileCheck" class="text-success size-3.5" /> <span>Extracted 10 Entities · 3 Line Items</span> </span> <span class="font-mono">Throughput: ~412 tok/s</span> </div> </div> </div> </div> </div> `, }) export class UiMultimodalVisionPlaygroundComponent implements OnInit, OnDestroy { protected readonly Bot = Bot protected readonly Braces = Braces protected readonly Check = Check protected readonly CheckCircle2 = CheckCircle2 protected readonly Copy = Copy protected readonly Cpu = Cpu protected readonly Crosshair = Crosshair protected readonly Eye = Eye protected readonly FileCheck = FileCheck protected readonly Info = Info protected readonly Layers = Layers protected readonly RefreshCw = RefreshCw protected readonly Scan = Scan protected readonly Send = Send protected readonly ShieldCheck = ShieldCheck protected readonly Sparkles = Sparkles protected readonly Tag = Tag protected readonly models = models protected readonly visionTasks = visionTasks protected readonly boundingBoxes = boundingBoxes protected readonly extractedEntities = extractedEntities protected readonly lineItems = lineItems protected readonly vqaPresets = vqaPresets @Input('class') className?: string readonly selectedModel = signal('claude-3-5-sonnet-vision') readonly selectedTask = signal('ocr-extraction') readonly isAnalyzing = signal(false) readonly activeBoxId = signal<string | null>(null) readonly activeTab = signal('entities') readonly isCopied = signal(false) readonly copiedFieldId = signal<string | null>(null) readonly showOcrText = signal(true) readonly showConfidence = signal(true) readonly showCoordinates = signal(true) readonly vqaInput = signal(vqaPresets[0]!.question) readonly currentVqaAnswer = signal(vqaPresets[0]!.answer) readonly isAnsweringVqa = signal(false) private timers = new Set<ReturnType<typeof setTimeout>>() private readonly keyHandler = (e: KeyboardEvent) => { if ((e.ctrlKey || e.metaKey) && e.key === 'Enter') { e.preventDefault() this.runVisionAnalysis() } } constructor() { inject(DestroyRef).onDestroy(() => { for (const t of this.timers) clearTimeout(t) this.timers.clear() }) } ngOnInit(): void { window.addEventListener('keydown', this.keyHandler) } ngOnDestroy(): void { window.removeEventListener('keydown', this.keyHandler) } readonly activeModel = computed(() => this.models.find((m) => m.id === this.selectedModel()) ?? this.models[0]!) readonly activeBox = computed(() => { const id = this.activeBoxId() if (!id) return null return this.boundingBoxes.find((b) => b.id === id) ?? null }) readonly rawJsonSchema = computed(() => { const model = this.selectedModel() const meanConfidence = this.activeModel().meanConfidence / 100 return JSON.stringify( { $schema: 'https://json-schema.org/draft/2020-12/schema', document_type: 'commercial_invoice', document_id: 'doc_vision_9824f8a1', model, mean_confidence: meanConfidence, metadata: { dimensions: { width: 1240, height: 1754, dpi: 300 }, orientation_degrees: 0, language: 'en-US', pages_analyzed: 1, processed_at: new Date().toISOString(), }, extracted_fields: { invoice_number: { value: 'INV-84920', confidence: 0.998, bbox: [42, 680, 110, 1140] }, issue_date: { value: '2026-08-21', confidence: 0.996, bbox: [145, 680, 250, 1140] }, due_date: { value: '2026-09-01', confidence: 0.994, bbox: [145, 680, 250, 1140] }, vendor: { name: 'Apex Cloud Systems Inc.', tax_id: 'US-EIN-94-8271923', address: '500 Howard St, Suite 400, San Francisco, CA 94105', confidence: 0.999, }, customer: { name: 'Cyberdyne Systems Corp', account_id: 'ACC-90412', confidence: 0.993 }, currency: 'USD', subtotal: 1150.0, tax_amount: 95.0, tax_rate_pct: 8.25, total_amount: 1245.0, }, line_items: this.lineItems.map((item) => ({ item_id: item.id, description: item.description, quantity: item.qty, unit: item.unit, unit_price: item.unitPrice, total_price: item.totalPrice, confidence: item.confidence / 100, })), validation_rules: { math_checksum_passed: true, tax_arithmetic_valid: true, vendor_sanctions_screened: true, flagged_anomalies_count: 0, }, }, null, 2, ) }) get rootClass(): string { return cn('w-full space-y-6', this.className) } boxClass(box: BoundingBox): string { return cn( 'focus-visible:ring-primary absolute cursor-pointer rounded border-2 transition-colors duration-150 focus-visible:ring-2 focus-visible:outline-hidden', box.colorBorder, box.colorBg, this.activeBoxId() === box.id ? 'ring-primary z-30 scale-[1.01] opacity-100 shadow-md ring-2' : 'hover:ring-primary/60 opacity-85 hover:opacity-100 hover:ring-1', ) } boxBadgeClass(box: BoundingBox): string { return cn( 'absolute -top-5.5 left-0 flex items-center gap-1 rounded px-1.5 py-0.5 text-xs whitespace-nowrap shadow-xs transition-opacity', box.colorBadge, ) } entityRowClass(field: ExtractedEntity): string { return cn( 'cursor-pointer transition-colors', this.activeBoxId() === field.boxId ? 'bg-primary/10 font-medium' : 'hover:bg-muted/40', ) } entityDotClass(field: ExtractedEntity): string { return cn('size-1.5 rounded-full', this.activeBoxId() === field.boxId ? 'bg-primary' : 'bg-muted-foreground/40') } entityBadgeConfidenceClass(field: ExtractedEntity): string { return cn( 'font-mono text-xs', field.confidence >= 99.5 ? 'border-success/20 bg-success/10 text-success border' : 'border-info/20 bg-info/10 text-info border', ) } private later(fn: () => void, ms: number): void { const t = setTimeout(() => { this.timers.delete(t) fn() }, ms) this.timers.add(t) } runVisionAnalysis(): void { if (this.isAnalyzing()) return this.isAnalyzing.set(true) this.activeBoxId.set(null) this.later(() => this.isAnalyzing.set(false), 650) } handleCopyJson(): void { void navigator.clipboard?.writeText(this.rawJsonSchema()) this.isCopied.set(true) this.later(() => this.isCopied.set(false), 2000) } handleCopyEntity(field: ExtractedEntity): void { void navigator.clipboard?.writeText(field.extractedValue) this.copiedFieldId.set(field.id) this.later(() => this.copiedFieldId.set(null), 2000) } selectVqaPreset(preset: VqaPreset): void { this.vqaInput.set(preset.question) this.isAnsweringVqa.set(true) this.activeBoxId.set(preset.targetBoxId) this.later(() => { this.currentVqaAnswer.set(preset.answer) this.isAnsweringVqa.set(false) }, 350) } submitCustomVqa(): void { if (!this.vqaInput().trim() || this.isAnsweringVqa()) return this.isAnsweringVqa.set(true) this.later(() => { const input = this.vqaInput() const match = this.vqaPresets.find((p) => p.question.toLowerCase().includes(input.toLowerCase().slice(0, 8))) if (match) { this.currentVqaAnswer.set(match.answer) this.activeBoxId.set(match.targetBoxId) } else { this.currentVqaAnswer.set( 'Multimodal reasoning on visual region: Extracted relevant visual context from document canvas. Invoice INV-84920 Total is $1,245.00 due Sep 01, 2026. Arithmetic verified.', ) this.activeBoxId.set('box-total') } this.isAnsweringVqa.set(false) }, 400) } }src/app/components/blocks/multimodal-vision-playground/index.ts0.1 kB
export { UiMultimodalVisionPlaygroundComponent } from './multimodal-vision-playground.component'
Raw manifest:https://uipkge.dev/r/angular/multimodal-vision-playground.json