Lakehouse Table Optimizer
blockdevopsApache Iceberg and Delta Lake table maintenance manager, compaction optimizer, snapshot history visualizer, and time-travel SQL query helper.
Installation
$pnpm dlx uipkge-ng@latest add lakehouse-table-optimizer$npx uipkge-ng@latest add lakehouse-table-optimizer$yarn dlx uipkge-ng@latest add lakehouse-table-optimizer$bunx uipkge-ng@latest add lakehouse-table-optimizerManifest:
https://uipkge.dev/r/angular/lakehouse-table-optimizer.jsonInstalls to:src/app/components/blocks/lakehouse-table-optimizer/Variants
Loading interactive previews…
Loading interactive previews…
Schema
Type aliases exported from this item's source. Use these to shape the data you pass in.
TableSnapshotinterface TableSnapshot {
id: string
numericId: string
timestamp: string
relativeTime: string
operation: SnapshotOperation
rowsAdded: string
rowsDeleted: string
author: string
summary: string
manifestFile: string
filesAdded: number
filesRemoved: number
byteDelta: string
}PartitionRecordinterface PartitionRecord {
partitionKey: string
fileCount: number
avgFileSize: string
totalSize: string
skippedPercentage: number
zOrderScore: string
status: 'Optimized' | 'Compacting' | 'Needs Z-Order'
lastClustered: string
}TableDetailsinterface TableDetails {
name: string
format: string
totalSize: string
rowCount: string
statusText: string
catalog: string
storageLocation: string
lastCompacted: string
compressionCodec: string
}Files installed (2)
src/app/components/blocks/lakehouse-table-optimizer/lakehouse-table-optimizer.component.ts54 kB
// Apache Iceberg and Delta Lake table maintenance manager, compaction optimizer, snapshot // history visualizer, and time-travel SQL query helper. Port of the React block 1:1. import { ChangeDetectionStrategy, Component, Input, computed, signal } from '@angular/core' import { ArrowDownRight, Check, CheckCircle2, Clock, Copy, Database, FileCode2, FileText, FolderArchive, HardDrive, History, Layers, LucideAngularModule, RefreshCw, Search, Server, Terminal, Trash2, Workflow, Zap, } from 'lucide-angular' import { cn } from '@/lib/utils' import { UiBadgeComponent } from '@/ui/badge/badge.component' import { UiButtonComponent } from '@/ui/button/button.component' import { UiCardComponent, UiCardContentComponent, UiCardDescriptionComponent, UiCardHeaderComponent, UiCardTitleComponent, } from '@/ui/card/card.component' import { UiProgressComponent } from '@/ui/progress/progress.component' import { UiSeparatorComponent } from '@/ui/separator/separator.component' import { UiTableBodyComponent, UiTableCellComponent, UiTableComponent, UiTableHeadComponent, UiTableHeaderComponent, UiTableRowComponent, } from '@/ui/table/table.component' import { UiTabsComponent, UiTabsListComponent, UiTabsTriggerComponent } from '@/ui/tabs/tabs.component' export type SnapshotOperation = 'MERGE' | 'APPEND' | 'OPTIMIZE' | 'REPLACE_PARTITION' export type SqlSyntaxMode = 'system_time' | 'system_version' | 'pyspark' | 'trino' export interface TableSnapshot { id: string numericId: string timestamp: string relativeTime: string operation: SnapshotOperation rowsAdded: string rowsDeleted: string author: string summary: string manifestFile: string filesAdded: number filesRemoved: number byteDelta: string } export interface PartitionRecord { partitionKey: string fileCount: number avgFileSize: string totalSize: string skippedPercentage: number zOrderScore: string status: 'Optimized' | 'Compacting' | 'Needs Z-Order' lastClustered: string } export interface TableDetails { name: string format: string totalSize: string rowCount: string statusText: string catalog: string storageLocation: string lastCompacted: string compressionCodec: string } const defaultTable: TableDetails = { name: 'lakehouse.silver_customer_events', format: 'Apache Iceberg v2 · Parquet', totalSize: '4.2 TB', rowCount: '1.48B Rows', statusText: 'Table Optimized · 0 Small Files', catalog: 'AWS Glue Data Catalog (us-east-1)', storageLocation: 's3://lakehouse-data-prod/silver/customer_events/', lastCompacted: '28 mins ago', compressionCodec: 'ZSTD (Level 7)', } const defaultSnapshots: TableSnapshot[] = [ { id: 'snap_84920194821', numericId: '84920194821', timestamp: '2026-08-21 09:30:00 UTC', relativeTime: '12 mins ago', operation: 'MERGE', rowsAdded: '+142,500', rowsDeleted: '-18,200', author: 'spark-streaming-pipeline', summary: "Upserted late-arriving customer clickstream events from Kafka topic 'customer.events.prod'", manifestFile: 'snap-84920194821-1-m0.avro', filesAdded: 4, filesRemoved: 3, byteDelta: '+128 MB', }, { id: 'snap_84920183104', numericId: '84920183104', timestamp: '2026-08-21 08:00:00 UTC', relativeTime: '1 hour ago', operation: 'OPTIMIZE', rowsAdded: '+0', rowsDeleted: '-0', author: 'iceberg-compactor-daemon', summary: 'Bin-pack compaction rewrote 4,210 small files into 42 optimal 128MB Parquet data files', manifestFile: 'snap-84920183104-1-m0.avro', filesAdded: 42, filesRemoved: 4210, byteDelta: '-4.1 GB (compaction gain)', }, { id: 'snap_84920152918', numericId: '84920152918', timestamp: '2026-08-21 06:15:00 UTC', relativeTime: '3 hours ago', operation: 'APPEND', rowsAdded: '+890,200', rowsDeleted: '-0', author: 'batch-ingest-hourly', summary: 'Micro-batch append for EMEA checkout and cart telemetry session events', manifestFile: 'snap-84920152918-1-m0.avro', filesAdded: 18, filesRemoved: 0, byteDelta: '+2.2 GB', }, { id: 'snap_84920120195', numericId: '84920120195', timestamp: '2026-08-20 23:45:00 UTC', relativeTime: '9 hours ago', operation: 'REPLACE_PARTITION', rowsAdded: '+3,410,000', rowsDeleted: '-3,398,000', author: 'dbt-backfill-runner', summary: "Replaced partition 'event_date=2026-08-20' following upstream IP geolocation schema enrichment", manifestFile: 'snap-84920120195-1-m0.avro', filesAdded: 28, filesRemoved: 28, byteDelta: '+140 MB', }, { id: 'snap_84919984120', numericId: '84919984120', timestamp: '2026-08-20 18:30:00 UTC', relativeTime: '15 hours ago', operation: 'APPEND', rowsAdded: '+1,205,400', rowsDeleted: '-0', author: 'batch-ingest-hourly', summary: 'Hourly ingestion run for US-East customer checkout telemetry records', manifestFile: 'snap-84919984120-1-m0.avro', filesAdded: 24, filesRemoved: 0, byteDelta: '+3.1 GB', }, ] const defaultPartitions: PartitionRecord[] = [ { partitionKey: 'event_date=2026-08-21', fileCount: 42, avgFileSize: '128.2 MB', totalSize: '5.38 GB', skippedPercentage: 98.4, zOrderScore: '98.4%', status: 'Optimized', lastClustered: '28m ago', }, { partitionKey: 'event_date=2026-08-20', fileCount: 40, avgFileSize: '127.9 MB', totalSize: '5.12 GB', skippedPercentage: 97.8, zOrderScore: '97.8%', status: 'Optimized', lastClustered: '9h ago', }, { partitionKey: 'event_date=2026-08-19', fileCount: 38, avgFileSize: '128.0 MB', totalSize: '4.86 GB', skippedPercentage: 96.9, zOrderScore: '96.9%', status: 'Optimized', lastClustered: '1d ago', }, { partitionKey: 'event_date=2026-08-18', fileCount: 44, avgFileSize: '126.8 MB', totalSize: '5.58 GB', skippedPercentage: 95.8, zOrderScore: '95.8%', status: 'Optimized', lastClustered: '2d ago', }, { partitionKey: 'event_date=2026-08-17', fileCount: 36, avgFileSize: '128.5 MB', totalSize: '4.62 GB', skippedPercentage: 97.1, zOrderScore: '97.1%', status: 'Optimized', lastClustered: '3d ago', }, { partitionKey: 'event_date=2026-08-16', fileCount: 39, avgFileSize: '127.4 MB', totalSize: '4.97 GB', skippedPercentage: 96.5, zOrderScore: '96.5%', status: 'Optimized', lastClustered: '4d ago', }, ] const snapshotOps = ['ALL', 'MERGE', 'OPTIMIZE', 'APPEND', 'REPLACE_PARTITION'] as const @Component({ changeDetection: ChangeDetectionStrategy.Eager, selector: 'ui-lakehouse-table-optimizer, [ui-lakehouse-table-optimizer]', standalone: true, // React renders the root <div> itself: the host stays out of layout and `class` goes to the root. host: { '[attr.class]': '"contents"' }, imports: [ LucideAngularModule, UiBadgeComponent, UiButtonComponent, UiCardComponent, UiCardContentComponent, UiCardDescriptionComponent, UiCardHeaderComponent, UiCardTitleComponent, UiProgressComponent, UiSeparatorComponent, UiTableComponent, UiTableBodyComponent, UiTableCellComponent, UiTableHeadComponent, UiTableHeaderComponent, UiTableRowComponent, UiTabsComponent, UiTabsListComponent, UiTabsTriggerComponent, ], template: ` <div data-slot="lakehouse-table-optimizer" [class]="rootClass"> <!-- Header Section --> <div class="border-border bg-card rounded-xl border p-5 shadow-xs sm:p-6"> <div class="flex flex-col gap-5 lg:flex-row lg:items-center lg:justify-between"> <div class="flex items-start gap-4"> <div class="border-primary/20 bg-primary/10 text-primary flex size-11 shrink-0 items-center justify-center rounded-lg border shadow-xs" > <lucide-icon [img]="Database" class="size-6" /> </div> <div class="space-y-2"> <div class="flex flex-wrap items-center gap-2"> <h1 class="text-foreground font-mono text-lg font-bold tracking-tight break-all sm:text-xl"> {{ table().name }} </h1> <span ui-badge variant="success" class="gap-1.5 font-medium"> <span class="bg-success size-1.5 animate-pulse rounded-full"></span> {{ table().statusText }} </span> </div> <div class="text-muted-foreground flex flex-wrap items-center gap-2 text-xs"> <span ui-badge variant="outline" class="gap-1 font-mono"> <lucide-icon [img]="Layers" class="text-muted-foreground size-3" /> {{ table().format }} </span> <span ui-badge variant="secondary" class="gap-1 font-mono font-medium"> <lucide-icon [img]="HardDrive" class="text-muted-foreground size-3" /> {{ table().totalSize }} · {{ table().rowCount }} </span> <span class="hidden items-center gap-1 sm:inline-flex"> <lucide-icon [img]="Server" class="size-3" /> {{ table().catalog }} </span> </div> </div> </div> <div class="flex flex-wrap items-center gap-2.5"> <button ui-button variant="outline" size="sm" class="h-9 gap-1.5 shadow-xs" [disabled]="isExpiring()" (click)="expireSnapshots()" > @if (!isExpiring()) { <lucide-icon [img]="Clock" class="text-muted-foreground size-3.5" /> } @else { <lucide-icon [img]="RefreshCw" class="text-muted-foreground size-3.5 animate-spin" /> } <span>{{ isExpiring() ? 'Expiring Stale...' : 'Expire Snapshots' }}</span> </button> <button ui-button variant="default" size="sm" class="h-9 gap-1.5 shadow-xs" [disabled]="isCompacting()" (click)="runCompaction()" > @if (!isCompacting()) { <lucide-icon [img]="Zap" class="size-3.5" /> } @else { <lucide-icon [img]="RefreshCw" class="size-3.5 animate-spin" /> } <span>{{ isCompacting() ? 'Compacting Table...' : 'Run Compaction & Vacuum' }}</span> </button> </div> </div> @if (actionNotification()) { <div class="border-success/20 bg-success/10 text-success mt-4 flex items-center justify-between rounded-lg border p-3 text-xs" > <div class="flex items-center gap-2"> <lucide-icon [img]="CheckCircle2" class="text-success size-4 shrink-0" /> <span>{{ actionNotification() }}</span> </div> <button type="button" class="text-success/70 hover:bg-success/20 hover:text-success rounded p-1" (click)="actionNotification.set(null)" > <span class="sr-only">Dismiss</span> × </button> </div> } </div> <!-- 4 Table Health & Storage Telemetry Cards --> <div class="grid grid-cols-1 gap-4 sm:grid-cols-2 lg:grid-cols-4"> <div ui-card class="border-border bg-card shadow-xs"> <div ui-card-header class="flex flex-row items-center justify-between space-y-0 pb-2"> <h3 ui-card-title class="text-muted-foreground text-xs font-semibold tracking-wider uppercase"> Total Partitions </h3> <div class="border-border bg-muted/50 text-muted-foreground flex size-7 items-center justify-center rounded-md border" > <lucide-icon [img]="FolderArchive" class="size-3.5" /> </div> </div> <div ui-card-content class="space-y-2"> <div class="flex flex-wrap items-baseline justify-between gap-2"> <div class="text-foreground font-mono text-2xl font-bold tracking-tight tabular-nums">1,420</div> <span ui-badge variant="success" class="text-xs">100% Healthy</span> </div> <p class="text-muted-foreground text-xs">1,420 Partitions · Date Partitioned</p> <div class="space-y-1 pt-1"> <ui-progress [value]="100" class="bg-muted h-1.5" /> <div class="text-muted-foreground flex justify-between text-xs"> <span>event_date (YYYY-MM-DD)</span> <span class="font-mono tabular-nums">0 unpartitioned</span> </div> </div> </div> </div> <div ui-card class="border-border bg-card shadow-xs"> <div ui-card-header class="flex flex-row items-center justify-between space-y-0 pb-2"> <h3 ui-card-title class="text-muted-foreground text-xs font-semibold tracking-wider uppercase"> Small Files Neutralized </h3> <div class="border-border bg-muted/50 text-muted-foreground flex size-7 items-center justify-center rounded-md border" > <lucide-icon [img]="HardDrive" class="size-3.5" /> </div> </div> <div ui-card-content class="space-y-2"> <div class="flex flex-wrap items-baseline justify-between gap-2"> <div class="text-foreground font-mono text-2xl font-bold tracking-tight tabular-nums">14,280</div> <span ui-badge variant="success" class="font-mono text-xs">99.0% Reduction</span> </div> <p class="text-muted-foreground text-xs">14,280 small files compacted -> 142 optimal 128MB files</p> <div class="space-y-1 pt-1"> <ui-progress [value]="99" class="bg-muted h-1.5" /> <div class="text-muted-foreground flex justify-between text-xs"> <span>Target: 128 MB</span> <span class="font-mono tabular-nums">0 files <32MB</span> </div> </div> </div> </div> <div ui-card class="border-border bg-card shadow-xs"> <div ui-card-header class="flex flex-row items-center justify-between space-y-0 pb-2"> <h3 ui-card-title class="text-muted-foreground text-xs font-semibold tracking-wider uppercase"> Snapshot History </h3> <div class="border-border bg-muted/50 text-muted-foreground flex size-7 items-center justify-center rounded-md border" > <lucide-icon [img]="History" class="size-3.5" /> </div> </div> <div ui-card-content class="space-y-2"> <div class="flex flex-wrap items-baseline justify-between gap-2"> <div class="text-foreground font-mono text-2xl font-bold tracking-tight tabular-nums">48</div> <span ui-badge variant="info" class="font-mono text-xs">30-Day Window</span> </div> <p class="text-muted-foreground text-xs">48 Active Snapshots · 30-Day Time Travel</p> <div class="space-y-1 pt-1"> <ui-progress [value]="64" class="bg-muted h-1.5" /> <div class="text-muted-foreground flex justify-between text-xs"> <span>Oldest: Jul 22, 2026</span> <span class="font-mono tabular-nums">4.8 MB metadata</span> </div> </div> </div> </div> <div ui-card class="border-border bg-card shadow-xs"> <div ui-card-header class="flex flex-row items-center justify-between space-y-0 pb-2"> <h3 ui-card-title class="text-muted-foreground text-xs font-semibold tracking-wider uppercase"> Storage Optimization </h3> <div class="border-border bg-muted/50 text-muted-foreground flex size-7 items-center justify-center rounded-md border" > <lucide-icon [img]="Zap" class="size-3.5" /> </div> </div> <div ui-card-content class="space-y-2"> <div class="flex flex-wrap items-baseline justify-between gap-2"> <div class="text-foreground font-mono text-2xl font-bold tracking-tight tabular-nums">1.2 TB</div> <span ui-badge variant="success" class="font-mono text-xs">$240/mo Saved</span> </div> <p class="text-muted-foreground text-xs">1.2 TB freed via Vacuum · $240/mo saved</p> <div class="space-y-1 pt-1"> <ui-progress [value]="85" class="bg-muted h-1.5" /> <div class="text-muted-foreground flex justify-between text-xs"> <span>872 GB stale · 328 GB orphan</span> <span class="font-mono tabular-nums">-28.5% S3 Cost</span> </div> </div> </div> </div> </div> <!-- Main Lakehouse Inspection & Control Tabs --> <ui-tabs [value]="activeTab()" (valueChange)="setActiveTab($event)" class="w-full space-y-5"> <ui-tabs-list class="grid w-full grid-cols-2 md:grid-cols-4"> <button ui-tabs-trigger value="snapshots" class="gap-1.5 text-xs font-medium"> <lucide-icon [img]="History" class="size-3.5" /> <span>Snapshots & Time-Travel</span> </button> <button ui-tabs-trigger value="partitions" class="gap-1.5 text-xs font-medium"> <lucide-icon [img]="Layers" class="size-3.5" /> <span>Partitions & Z-Ordering</span> </button> <button ui-tabs-trigger value="maintenance" class="gap-1.5 text-xs font-medium"> <lucide-icon [img]="Zap" class="size-3.5" /> <span>Compaction & Vacuum</span> </button> <button ui-tabs-trigger value="files" class="gap-1.5 text-xs font-medium"> <lucide-icon [img]="HardDrive" class="size-3.5" /> <span>File Sizing & Distribution</span> </button> </ui-tabs-list> @if (activeTab() === 'snapshots') { <div class="space-y-5"> <div class="grid grid-cols-1 gap-6 lg:grid-cols-12"> <div class="space-y-4 lg:col-span-7"> <div ui-card class="border-border bg-card shadow-xs"> <div ui-card-header class="p-4 pb-3 sm:p-5 sm:pb-3"> <div class="flex flex-col gap-3 sm:flex-row sm:items-center sm:justify-between"> <div> <h3 ui-card-title class="text-sm font-semibold sm:text-base">Snapshot Commit History</h3> <p ui-card-description class="text-xs"> Select any snapshot to generate reproducible time-travel queries </p> </div> <div class="flex flex-wrap items-center gap-1"> @for (op of snapshotOps; track op) { <button type="button" [class]="opPillClass(op)" (click)="snapshotFilter.set(op)"> {{ op }} </button> } </div> </div> <div class="relative pt-2"> <lucide-icon [img]="Search" class="text-muted-foreground absolute top-5 left-2.5 size-3.5" /> <input [value]="snapshotSearch()" (input)="snapshotSearch.set($any($event.target).value)" type="text" placeholder="Search snapshot ID, author, summary..." class="border-border bg-background text-foreground placeholder:text-muted-foreground focus-visible:ring-ring w-full rounded-md border py-1.5 pr-3 pl-8 text-xs focus-visible:ring-2 focus-visible:outline-none" /> </div> </div> <ui-separator /> <div ui-card-content class="p-4 sm:p-5"> <div class="before:bg-border relative space-y-4 before:absolute before:top-3 before:bottom-3 before:left-[17px] before:w-0.5" > @for (snap of filteredSnapshots(); track snap.id) { <div role="button" tabindex="0" [class]="snapshotCardClass(snap.id)" (click)="selectSnapshot(snap.id)" (keydown.enter)="selectSnapshot(snap.id)" (keydown.space)="selectSnapshotKey($event, snap.id)" > <div class="border-border bg-card text-foreground group-hover:border-primary/50 relative z-10 flex size-9 shrink-0 items-center justify-center rounded-full border shadow-xs" > @switch (snap.operation) { @case ('MERGE') { <lucide-icon [img]="History" class="text-info size-4" /> } @case ('OPTIMIZE') { <lucide-icon [img]="Zap" class="text-success size-4" /> } @case ('APPEND') { <lucide-icon [img]="ArrowDownRight" class="text-chart-1 size-4" /> } @default { <lucide-icon [img]="RefreshCw" class="text-warning size-4" /> } } </div> <div class="min-w-0 flex-1 space-y-2"> <div class="flex flex-wrap items-center justify-between gap-2"> <div class="flex flex-wrap items-center gap-2"> <span class="text-foreground font-mono text-xs font-bold">{{ snap.id }}</span> <span ui-badge [variant]="operationBadgeVariant(snap.operation)" class="font-mono text-xs" > {{ snap.operation }} </span> @if (selectedSnapshotId() === snap.id) { <span ui-badge variant="default" class="text-xs">Active Time-Travel Target</span> } </div> <span class="text-muted-foreground font-mono text-xs tabular-nums">{{ snap.relativeTime }}</span> </div> <p class="text-muted-foreground text-xs">{{ snap.summary }}</p> <div class="flex flex-wrap items-center gap-3 pt-1 text-xs"> <div class="text-muted-foreground flex items-center gap-1 font-mono"> <span class="text-success font-semibold">{{ snap.rowsAdded }}</span> <span>/</span> <span class="text-destructive font-semibold">{{ snap.rowsDeleted }} rows</span> </div> <div class="text-muted-foreground flex items-center gap-1 font-mono"> <lucide-icon [img]="FileCode2" class="text-muted-foreground size-3" /> <span>{{ snap.byteDelta }}</span> </div> <div class="text-muted-foreground flex items-center gap-1 font-mono"> <span>Author:</span> <span class="text-foreground font-medium">{{ snap.author }}</span> </div> </div> </div> </div> } </div> </div> </div> </div> <div class="space-y-4 lg:col-span-5"> <div ui-card class="border-border bg-card shadow-xs"> <div ui-card-header class="p-4 pb-3 sm:p-5 sm:pb-3"> <div class="flex items-center justify-between"> <div class="space-y-1"> <h3 ui-card-title class="text-sm font-semibold sm:text-base">Time-Travel Query Picker</h3> <p ui-card-description class="text-xs"> Instant query generator for snapshot audit & reconciliation </p> </div> <span ui-badge variant="outline" class="font-mono text-xs">{{ selectedSnapshot().numericId }}</span> </div> <div class="grid grid-cols-2 gap-1 pt-3 sm:grid-cols-4"> <button type="button" [class]="syntaxClass('system_time')" (click)="sqlSyntaxMode.set('system_time')" > SYSTEM_TIME </button> <button type="button" [class]="syntaxClass('system_version')" (click)="sqlSyntaxMode.set('system_version')" > VERSION </button> <button type="button" [class]="syntaxClass('pyspark')" (click)="sqlSyntaxMode.set('pyspark')"> PySpark </button> <button type="button" [class]="syntaxClass('trino')" (click)="sqlSyntaxMode.set('trino')"> Trino SQL </button> </div> </div> <ui-separator /> <div ui-card-content class="space-y-4 p-4 sm:p-5"> <div class="border-border bg-muted/40 space-y-1.5 rounded-lg border p-3 text-xs"> <div class="text-muted-foreground flex items-center justify-between"> <span>Target Snapshot:</span> <span class="text-foreground font-mono font-semibold">{{ selectedSnapshot().id }}</span> </div> <div class="text-muted-foreground flex items-center justify-between"> <span>Point-in-Time:</span> <span class="text-foreground font-mono">{{ selectedSnapshot().timestamp }}</span> </div> <div class="text-muted-foreground flex items-center justify-between"> <span>Commit Operation:</span> <span ui-badge [variant]="operationBadgeVariant(selectedSnapshot().operation)" class="font-mono text-xs" > {{ selectedSnapshot().operation }} </span> </div> </div> <div class="border-border bg-muted/70 relative overflow-hidden rounded-lg border font-mono"> <div class="border-border bg-muted/90 text-muted-foreground flex items-center justify-between border-b px-3 py-2 text-xs" > <span class="text-foreground flex items-center gap-1.5 font-semibold"> <lucide-icon [img]="Terminal" class="text-primary size-3.5" /> Query table as of snapshot: </span> <button ui-button variant="ghost" size="sm" class="hover:bg-background hover:text-foreground h-7 gap-1 px-2 text-xs" (click)="handleCopyQuery()" > @if (copiedQuery()) { <lucide-icon [img]="Check" class="text-success size-3.5" /> } @else { <lucide-icon [img]="Copy" class="text-muted-foreground size-3.5" /> } <span>{{ copiedQuery() ? 'Copied' : 'Copy' }}</span> </button> </div> <pre class="text-foreground overflow-x-auto p-3 text-xs leading-relaxed" ><code>{{ generatedQuery() }}</code></pre> </div> <div class="border-border bg-card space-y-2 rounded-lg border p-3 text-xs"> <div class="text-muted-foreground flex items-center justify-between"> <span>Estimated Scan Overhead:</span> <span class="text-success font-mono font-semibold">42ms · 96.4% skipped</span> </div> <div class="text-muted-foreground flex items-center justify-between"> <span>Manifest Avro File:</span> <span class="text-foreground font-mono text-xs">{{ selectedSnapshot().manifestFile }}</span> </div> <div class="text-muted-foreground flex items-center justify-between"> <span>Storage Engine:</span> <span class="text-foreground">Apache Iceberg Native Parquet Reader</span> </div> </div> </div> </div> </div> </div> </div> } @if (activeTab() === 'partitions') { <div class="space-y-5"> <div ui-card class="border-border bg-card shadow-xs"> <div ui-card-header class="p-5 pb-4"> <div class="flex flex-col gap-3 md:flex-row md:items-center md:justify-between"> <div> <h3 ui-card-title class="flex items-center gap-2 text-base font-semibold"> <lucide-icon [img]="Workflow" class="text-primary size-4" /> Partition Health & Multidimensional Z-Order Clustering </h3> <p ui-card-description class="text-xs"> Z-Order clustered by customer_id, event_type to maximize Parquet min/max statistics pruning. </p> </div> <div class="relative w-full md:w-64"> <lucide-icon [img]="Search" class="text-muted-foreground absolute top-2.5 left-2.5 size-3.5" /> <input [value]="partitionSearch()" (input)="partitionSearch.set($any($event.target).value)" type="text" placeholder="Filter partitions..." class="border-border bg-background text-foreground placeholder:text-muted-foreground focus-visible:ring-ring w-full rounded-md border py-1.5 pr-3 pl-8 text-xs focus-visible:ring-2 focus-visible:outline-none" /> </div> </div> <div class="border-border bg-muted/30 mt-4 grid grid-cols-1 gap-3 rounded-lg border p-3 text-xs sm:grid-cols-2 lg:grid-cols-4" > <div> <span class="text-muted-foreground">Clustered Columns:</span> <div class="text-foreground font-mono font-semibold">customer_id, event_type</div> </div> <div> <span class="text-muted-foreground">Data Skipping Efficiency:</span> <div class="text-success font-mono font-semibold">96.4% Row-Groups Skipped</div> </div> <div> <span class="text-muted-foreground">Clustering Curve Algorithm:</span> <div class="text-foreground font-mono font-semibold">Hilbert Space-Filling (Depth 16)</div> </div> <div> <span class="text-muted-foreground">Compression Codec:</span> <div class="text-foreground font-mono font-semibold">ZSTD Level 7 · Parquet v2</div> </div> </div> </div> <ui-separator /> <div ui-card-content class="p-0"> <div class="overflow-x-auto"> <table ui-table> <thead ui-table-header> <tr ui-table-row class="hover:bg-transparent"> <th ui-table-head class="text-xs">Partition Key</th> <th ui-table-head class="text-xs">File Count</th> <th ui-table-head class="text-xs">Avg File Size</th> <th ui-table-head class="text-xs">Total Data Size</th> <th ui-table-head class="text-xs">Data Skipping</th> <th ui-table-head class="text-xs">Z-Order Status</th> <th ui-table-head class="text-right text-xs">Action</th> </tr> </thead> <tbody ui-table-body> @for (partition of filteredPartitions(); track partition.partitionKey) { <tr ui-table-row class="hover:bg-muted/50"> <td ui-table-cell class="text-foreground font-mono text-xs font-semibold"> {{ partition.partitionKey }} </td> <td ui-table-cell class="font-mono text-xs tabular-nums">{{ partition.fileCount }} files</td> <td ui-table-cell class="font-mono text-xs tabular-nums">{{ partition.avgFileSize }}</td> <td ui-table-cell class="font-mono text-xs tabular-nums">{{ partition.totalSize }}</td> <td ui-table-cell> <div class="w-32 space-y-1"> <div class="text-muted-foreground flex justify-between font-mono text-xs tabular-nums"> <span>{{ partition.skippedPercentage }}%</span> </div> <ui-progress [value]="partition.skippedPercentage" class="bg-muted h-1.5" /> </div> </td> <td ui-table-cell> <span ui-badge variant="success" class="gap-1 font-mono text-xs"> <lucide-icon [img]="CheckCircle2" class="size-3" /> {{ partition.status }} </span> </td> <td ui-table-cell class="text-right"> <button ui-button variant="ghost" size="sm" class="h-7 text-xs font-medium"> Re-cluster </button> </td> </tr> } </tbody> </table> </div> </div> </div> </div> } @if (activeTab() === 'maintenance') { <div class="space-y-5"> <div class="grid grid-cols-1 gap-4 md:grid-cols-2"> <div ui-card class="border-border bg-card shadow-xs"> <div ui-card-header class="p-4 pb-2 sm:p-5 sm:pb-2"> <div class="flex items-center justify-between"> <h3 ui-card-title class="flex items-center gap-2 text-sm font-semibold"> <lucide-icon [img]="Database" class="text-success size-4" /> Bin-Pack Data File Compaction </h3> <span ui-badge variant="success" class="text-xs">Enabled · Auto</span> </div> <p ui-card-description class="text-xs"> Merges small files into optimized 128MB Parquet chunks to prevent small file bottleneck </p> </div> <div ui-card-content class="space-y-3 p-4 pt-3 text-xs sm:p-5"> <div class="border-border bg-muted/40 space-y-2 rounded-lg border p-3"> <div class="text-muted-foreground flex justify-between"> <span>Target File Size:</span> <span class="text-foreground font-mono font-semibold">128 MB</span> </div> <div class="text-muted-foreground flex justify-between"> <span>Small File Threshold:</span> <span class="text-foreground font-mono font-semibold">< 32 MB</span> </div> <div class="text-muted-foreground flex justify-between"> <span>Execution Schedule:</span> <span class="text-foreground font-mono">Every 2 Hours (cron: 0 */2 * * *)</span> </div> <div class="text-muted-foreground flex justify-between"> <span>Engine:</span> <span class="text-foreground">Apache Spark Iceberg RewriteDataFiles Action</span> </div> </div> </div> </div> <div ui-card class="border-border bg-card shadow-xs"> <div ui-card-header class="p-4 pb-2 sm:p-5 sm:pb-2"> <div class="flex items-center justify-between"> <h3 ui-card-title class="flex items-center gap-2 text-sm font-semibold"> <lucide-icon [img]="Trash2" class="text-destructive size-4" /> Vacuum & Orphan File Purger </h3> <span ui-badge variant="success" class="text-xs">Enabled · Daily</span> </div> <p ui-card-description class="text-xs"> Hard-deletes unreferenced data files and uncommitted multi-part upload orphans in S3 </p> </div> <div ui-card-content class="space-y-3 p-4 pt-3 text-xs sm:p-5"> <div class="border-border bg-muted/40 space-y-2 rounded-lg border p-3"> <div class="text-muted-foreground flex justify-between"> <span>Retention Grace Period:</span> <span class="text-foreground font-mono font-semibold">7 Days (168 Hours)</span> </div> <div class="text-muted-foreground flex justify-between"> <span>Storage Reclaimed (30d):</span> <span class="text-success font-mono font-semibold">1.2 TB freed</span> </div> <div class="text-muted-foreground flex justify-between"> <span>Execution Schedule:</span> <span class="text-foreground font-mono">Daily at 02:00 UTC</span> </div> <div class="text-muted-foreground flex justify-between"> <span>Safety Guard:</span> <span class="text-foreground">Explicit dry-run verification check</span> </div> </div> </div> </div> <div ui-card class="border-border bg-card shadow-xs"> <div ui-card-header class="p-4 pb-2 sm:p-5 sm:pb-2"> <div class="flex items-center justify-between"> <h3 ui-card-title class="flex items-center gap-2 text-sm font-semibold"> <lucide-icon [img]="Clock" class="text-info size-4" /> Snapshot Expiration Policy </h3> <span ui-badge variant="info" class="text-xs">30-Day Window</span> </div> <p ui-card-description class="text-xs"> Controls metadata table history retention for time-travel queries and rollback safety </p> </div> <div ui-card-content class="space-y-3 p-4 pt-3 text-xs sm:p-5"> <div class="border-border bg-muted/40 space-y-2 rounded-lg border p-3"> <div class="text-muted-foreground flex justify-between"> <span>Retention Window:</span> <span class="text-foreground font-mono font-semibold">30 Days</span> </div> <div class="text-muted-foreground flex justify-between"> <span>Min Snapshots Retained:</span> <span class="text-foreground font-mono font-semibold">20 Snapshots</span> </div> <div class="text-muted-foreground flex justify-between"> <span>Clean Expired Metadata:</span> <span class="text-foreground font-mono">Yes (Purge orphan Avro manifests)</span> </div> </div> </div> </div> <div ui-card class="border-border bg-card shadow-xs"> <div ui-card-header class="p-4 pb-2 sm:p-5 sm:pb-2"> <div class="flex items-center justify-between"> <h3 ui-card-title class="flex items-center gap-2 text-sm font-semibold"> <lucide-icon [img]="FileText" class="text-chart-1 size-4" /> Manifest File Consolidation </h3> <span ui-badge variant="success" class="text-xs">Optimized</span> </div> <p ui-card-description class="text-xs"> Rewrites small manifest files to accelerate query planning and metadata scanning </p> </div> <div ui-card-content class="space-y-3 p-4 pt-3 text-xs sm:p-5"> <div class="border-border bg-muted/40 space-y-2 rounded-lg border p-3"> <div class="text-muted-foreground flex justify-between"> <span>Active Manifest Count:</span> <span class="text-foreground font-mono font-semibold">6 Manifests</span> </div> <div class="text-muted-foreground flex justify-between"> <span>Rewrite Trigger Threshold:</span> <span class="text-foreground font-mono">> 50 Small Manifests</span> </div> <div class="text-muted-foreground flex justify-between"> <span>Query Planning Overhead:</span> <span class="text-success font-mono font-semibold">< 15ms</span> </div> </div> </div> </div> </div> </div> } @if (activeTab() === 'files') { <div class="space-y-5"> <div ui-card class="border-border bg-card shadow-xs"> <div ui-card-header class="p-5 pb-3"> <h3 ui-card-title class="text-base font-semibold"> File Size Distribution & I/O Amplification Analysis </h3> <p ui-card-description class="text-xs"> Comparison of storage layout before vs. after Iceberg automatic compaction </p> </div> <ui-separator /> <div ui-card-content class="space-y-6 p-5"> <div class="grid grid-cols-1 gap-6 md:grid-cols-2"> <div class="border-border bg-muted/20 space-y-3 rounded-xl border p-4"> <div class="flex items-center justify-between"> <span class="text-destructive text-xs font-semibold">Before Compaction (Small File Problem)</span> <span ui-badge variant="destructive" class="font-mono text-xs">18.4x I/O Amplification</span> </div> <div class="space-y-2 text-xs"> <div class="text-muted-foreground flex justify-between"> <span>Total Small Files (<4MB):</span> <span class="text-foreground font-mono font-bold">14,280 Files</span> </div> <div class="text-muted-foreground flex justify-between"> <span>Average File Size:</span> <span class="text-foreground font-mono">2.4 MB</span> </div> <div class="text-muted-foreground flex justify-between"> <span>Query Scan Latency (P95):</span> <span class="text-destructive font-mono font-semibold">8.4 seconds</span> </div> <div class="text-muted-foreground flex justify-between"> <span>S3 GET Request Cost:</span> <span class="text-foreground font-mono">$380 / month</span> </div> </div> </div> <div class="border-success/30 bg-success/[0.03] space-y-3 rounded-xl border p-4"> <div class="flex items-center justify-between"> <span class="text-success text-xs font-semibold">After Compaction (Optimal Bin-Pack)</span> <span ui-badge variant="success" class="font-mono text-xs">1.02x Near-Ideal</span> </div> <div class="space-y-2 text-xs"> <div class="text-muted-foreground flex justify-between"> <span>Total Target Files (128MB):</span> <span class="text-foreground font-mono font-bold">142 Files</span> </div> <div class="text-muted-foreground flex justify-between"> <span>Average File Size:</span> <span class="text-foreground font-mono">128.2 MB</span> </div> <div class="text-muted-foreground flex justify-between"> <span>Query Scan Latency (P95):</span> <span class="text-success font-mono font-semibold">220 milliseconds (38x speedup)</span> </div> <div class="text-muted-foreground flex justify-between"> <span>S3 GET Request Cost:</span> <span class="text-foreground font-mono">$12 / month</span> </div> </div> </div> </div> </div> </div> </div> } </ui-tabs> </div> `, }) export class UiLakehouseTableOptimizerComponent { protected readonly ArrowDownRight = ArrowDownRight protected readonly Check = Check protected readonly CheckCircle2 = CheckCircle2 protected readonly Clock = Clock protected readonly Copy = Copy protected readonly Database = Database protected readonly FileCode2 = FileCode2 protected readonly FileText = FileText protected readonly FolderArchive = FolderArchive protected readonly HardDrive = HardDrive protected readonly History = History protected readonly Layers = Layers protected readonly RefreshCw = RefreshCw protected readonly Search = Search protected readonly Server = Server protected readonly Terminal = Terminal protected readonly Trash2 = Trash2 protected readonly Workflow = Workflow protected readonly Zap = Zap protected readonly snapshotOps = snapshotOps @Input() set initialTable(v: TableDetails | undefined) { if (v) this.tableOverride.set(v) } @Input() set initialSnapshots(v: TableSnapshot[] | undefined) { if (v) this.snapshots.set([...v]) } @Input() set initialPartitions(v: PartitionRecord[] | undefined) { if (v) this.partitions.set([...v]) } @Input() set initialTab(v: string) { this.activeTab.set(v ?? 'snapshots') } @Input('class') className?: string private readonly tableOverride = signal<TableDetails | undefined>(undefined) readonly table = computed(() => this.tableOverride() ?? defaultTable) readonly snapshots = signal<TableSnapshot[]>([...defaultSnapshots]) readonly partitions = signal<PartitionRecord[]>([...defaultPartitions]) readonly selectedSnapshotId = signal('snap_84920194821') readonly sqlSyntaxMode = signal<SqlSyntaxMode>('system_time') readonly snapshotFilter = signal<string>('ALL') readonly snapshotSearch = signal('') readonly partitionSearch = signal('') readonly activeTab = signal('snapshots') readonly isCompacting = signal(false) readonly isExpiring = signal(false) readonly actionNotification = signal<string | null>(null) readonly copiedQuery = signal(false) private copyTimer: ReturnType<typeof setTimeout> | undefined private actionTimer: ReturnType<typeof setTimeout> | undefined readonly selectedSnapshot = computed( () => this.snapshots().find((s) => s.id === this.selectedSnapshotId()) ?? this.snapshots()[0]!, ) readonly filteredSnapshots = computed(() => { const filter = this.snapshotFilter() const q = this.snapshotSearch().toLowerCase().trim() return this.snapshots().filter((snap) => { const matchesOp = filter === 'ALL' || snap.operation === filter const matchesSearch = q === '' || snap.id.toLowerCase().includes(q) || snap.author.toLowerCase().includes(q) || snap.summary.toLowerCase().includes(q) || snap.operation.toLowerCase().includes(q) return matchesOp && matchesSearch }) }) readonly filteredPartitions = computed(() => { const q = this.partitionSearch().toLowerCase().trim() if (!q) return this.partitions() return this.partitions().filter((p) => p.partitionKey.toLowerCase().includes(q)) }) readonly generatedQuery = computed(() => { const currentSnap = this.selectedSnapshot() const tableName = this.table().name switch (this.sqlSyntaxMode()) { case 'system_time': return `SELECT * FROM ${tableName}\nFOR SYSTEM_TIME AS OF '${currentSnap.timestamp}';` case 'system_version': return `SELECT * FROM ${tableName}\nFOR SYSTEM_VERSION AS OF ${currentSnap.numericId};` case 'pyspark': return `# PySpark Iceberg time-travel query\ndf = spark.read \\\n .format("iceberg") \\\n .option("as-of-timestamp", "${currentSnap.timestamp}") \\\n .load("${tableName}")\n\ndf.filter("customer_id = 'cust_98241'") \\\n .select("event_id", "event_type", "event_date", "payload") \\\n .show(10)` case 'trino': return `SELECT event_id, customer_id, event_type, event_date\nFROM ${tableName}\nFOR VERSION AS OF ${currentSnap.numericId}\nWHERE event_type = 'CHECKOUT_COMPLETED'\nLIMIT 50;` default: return `SELECT * FROM ${tableName}\nFOR SYSTEM_TIME AS OF '${currentSnap.timestamp}';` } }) get rootClass(): string { return cn('text-foreground w-full space-y-6', this.className) } setActiveTab(value: string): void { this.activeTab.set(value) } selectSnapshot(id: string): void { this.selectedSnapshotId.set(id) } selectSnapshotKey(e: KeyboardEvent, id: string): void { e.preventDefault() this.selectSnapshot(id) } opPillClass(op: string): string { const active = this.snapshotFilter() === op return cn( 'rounded-md px-2 py-1 font-mono text-xs transition-colors', active ? 'bg-primary text-primary-foreground font-semibold' : 'bg-muted text-muted-foreground hover:bg-accent hover:text-foreground', ) } syntaxClass(mode: SqlSyntaxMode): string { const active = this.sqlSyntaxMode() === mode return cn( 'rounded-md px-2 py-1.5 text-center font-mono text-xs transition-colors', active ? 'bg-primary text-primary-foreground font-semibold shadow-xs' : 'bg-muted text-muted-foreground hover:bg-accent hover:text-foreground', ) } snapshotCardClass(id: string): string { const active = this.selectedSnapshotId() === id return cn( 'group relative flex cursor-pointer flex-col gap-3 rounded-lg border p-3.5 transition-colors sm:flex-row sm:items-start sm:gap-4', active ? 'border-primary/60 bg-primary/[0.03] ring-primary/30 shadow-xs ring-1' : 'border-border bg-card/60 hover:border-border hover:bg-accent/40', ) } operationBadgeVariant(op: SnapshotOperation): 'default' | 'secondary' | 'outline' | 'success' | 'warning' | 'info' { switch (op) { case 'OPTIMIZE': return 'success' case 'MERGE': return 'info' case 'APPEND': return 'secondary' case 'REPLACE_PARTITION': return 'warning' default: return 'outline' } } handleCopyQuery(): void { try { void navigator.clipboard?.writeText(this.generatedQuery()) this.copiedQuery.set(true) clearTimeout(this.copyTimer) this.copyTimer = setTimeout(() => this.copiedQuery.set(false), 2000) } catch { // Clipboard fallback } } private notify(message: string): void { this.actionNotification.set(message) clearTimeout(this.actionTimer) this.actionTimer = setTimeout(() => this.actionNotification.set(null), 6000) } runCompaction(): void { if (this.isCompacting()) return this.isCompacting.set(true) this.actionNotification.set(null) setTimeout(() => { this.isCompacting.set(false) this.notify( 'Compaction & Vacuum complete: 14,280 small files compacted into 142 optimal 128MB Parquet files. Reclaimed 1.2 TB storage.', ) }, 1800) } expireSnapshots(): void { if (this.isExpiring()) return this.isExpiring.set(true) this.actionNotification.set(null) setTimeout(() => { this.isExpiring.set(false) this.notify( 'Snapshot expiration finished: Removed 12 stale snapshots older than 30-day retention window. Purged 872 GB metadata & unreferenced parquet files.', ) }, 1400) } }src/app/components/blocks/lakehouse-table-optimizer/index.ts0.2 kB
export { UiLakehouseTableOptimizerComponent, type TableSnapshot, type PartitionRecord, type TableDetails, type SnapshotOperation, type SqlSyntaxMode, } from './lakehouse-table-optimizer.component'
Raw manifest:https://uipkge.dev/r/angular/lakehouse-table-optimizer.json