{ "$schema": "https://uipkge.dev/schema/registry-item.json", "name": "spark-job-stage-visualizer", "title": "Spark Job Stage Visualizer", "type": "registry:page", "files": [ { "path": "packages/registry-angular/blocks/spark-job-stage-visualizer/spark-job-stage-visualizer.component.ts", "content": "// Databricks and Apache Spark distributed job execution DAG visualizer with stage\n// performance metrics, task skew distribution quantiles, and executor JVM GC telemetry.\n// Port of the Vue block 1:1 (multi-file Vue block consolidated into one standalone component).\nimport { ChangeDetectionStrategy, Component, Input, computed, signal } from '@angular/core'\nimport {\n ArrowRightLeft,\n Check,\n CheckCircle2,\n Clock,\n Copy,\n Cpu,\n Download,\n Flame,\n Gauge,\n LucideAngularModule,\n Server,\n ShieldAlert,\n X,\n Zap,\n} from 'lucide-angular'\nimport { cn } from '@/lib/utils'\nimport { UiBadgeComponent } from '@/ui/badge/badge.component'\nimport { UiButtonComponent } from '@/ui/button/button.component'\nimport {\n UiCardComponent,\n UiCardContentComponent,\n UiCardHeaderComponent,\n UiCardTitleComponent,\n} from '@/ui/card/card.component'\nimport { UiProgressComponent } from '@/ui/progress/progress.component'\nimport {\n UiTableBodyComponent,\n UiTableCellComponent,\n UiTableComponent,\n UiTableHeadComponent,\n UiTableHeaderComponent,\n UiTableRowComponent,\n} from '@/ui/table/table.component'\nimport { UiTabsComponent, UiTabsListComponent, UiTabsTriggerComponent } from '@/ui/tabs/tabs.component'\n\nexport type SparkTab = 'dag' | 'skew' | 'executors' | 'aqe'\n\ninterface SparkStageSummary {\n id: number\n name: string\n status: 'succeeded' | 'running' | 'pending'\n tasksTotal: number\n tasksSucceeded: number\n duration: string\n skewRatio: number\n quantiles: { min: number; p25: number; median: number; p75: number; p95: number; max: number }\n}\n\ninterface SparkExecutorSummary {\n id: string\n host: string\n role: 'driver' | 'executor'\n status: string\n cores: number\n memoryUsedGB: number\n memoryTotalGB: number\n memoryPercent: number\n gcTimeSec: number\n gcPercent: number\n tasksCompleted: number\n spillGB: number\n}\n\ninterface AqeRuleSummary {\n id: string\n name: string\n status: 'applied' | 'skipped'\n description: string\n impact: string\n}\n\nconst defaultStages: SparkStageSummary[] = [\n {\n id: 0,\n name: 'FileScan parquet',\n status: 'succeeded',\n tasksTotal: 540,\n tasksSucceeded: 540,\n duration: '18s',\n skewRatio: 1.12,\n quantiles: { min: 0.8, p25: 1.2, median: 1.6, p75: 1.8, p95: 2.0, max: 2.1 },\n },\n {\n id: 1,\n name: 'Exchange hashpartitioning',\n status: 'succeeded',\n tasksTotal: 200,\n tasksSucceeded: 200,\n duration: '42s',\n skewRatio: 1.15,\n quantiles: { min: 1.8, p25: 2.4, median: 2.9, p75: 3.2, p95: 3.4, max: 3.6 },\n },\n {\n id: 2,\n name: 'HashAggregate & Filter',\n status: 'succeeded',\n tasksTotal: 200,\n tasksSucceeded: 200,\n duration: '1m 12s',\n skewRatio: 1.18,\n quantiles: { min: 2.1, p25: 2.8, median: 3.5, p75: 3.9, p95: 4.1, max: 4.2 },\n },\n {\n id: 3,\n name: 'SortMergeJoin',\n status: 'succeeded',\n tasksTotal: 200,\n tasksSucceeded: 200,\n duration: '56s',\n skewRatio: 1.09,\n quantiles: { min: 1.9, p25: 2.5, median: 3.1, p75: 3.4, p95: 3.7, max: 3.8 },\n },\n]\n\nconst defaultExecutors: SparkExecutorSummary[] = [\n {\n id: 'driver',\n host: '10.0.12.4',\n role: 'driver',\n status: 'active',\n cores: 16,\n memoryUsedGB: 24.2,\n memoryTotalGB: 32,\n memoryPercent: 75.6,\n gcTimeSec: 4.2,\n gcPercent: 1.8,\n tasksCompleted: 285,\n spillGB: 0.36,\n },\n {\n id: 'exec-1',\n host: '10.0.12.18',\n role: 'executor',\n status: 'active',\n cores: 32,\n memoryUsedGB: 28.8,\n memoryTotalGB: 32,\n memoryPercent: 90.0,\n gcTimeSec: 5.1,\n gcPercent: 2.2,\n tasksCompleted: 284,\n spillGB: 0.42,\n },\n {\n id: 'exec-2',\n host: '10.0.12.29',\n role: 'executor',\n status: 'active',\n cores: 32,\n memoryUsedGB: 21.4,\n memoryTotalGB: 32,\n memoryPercent: 66.8,\n gcTimeSec: 4.8,\n gcPercent: 2.3,\n tasksCompleted: 286,\n spillGB: 0.34,\n },\n {\n id: 'exec-3',\n host: '10.0.12.44',\n role: 'executor',\n status: 'active',\n cores: 32,\n memoryUsedGB: 18.6,\n memoryTotalGB: 32,\n memoryPercent: 58.1,\n gcTimeSec: 4.3,\n gcPercent: 2.1,\n tasksCompleted: 285,\n spillGB: 0.36,\n },\n]\n\nconst defaultAqeRules: AqeRuleSummary[] = [\n {\n id: 'aqe-1',\n name: 'CoalesceShufflePartitions',\n status: 'applied',\n description:\n 'Coalesced 540 initial map partitions down to 200 target shuffle partitions based on target partition size (64 MB).',\n impact: 'Reduced network RPC overhead by 63% and avoided 340 small shuffle partition files.',\n },\n {\n id: 'aqe-2',\n name: 'OptimizeSkewedJoin',\n status: 'applied',\n description: 'Dynamic skew detection evaluated all 200 partition sizes against the 5x median threshold.',\n impact: 'Max partition skew ratio remained well balanced at 1.18x. No partition splitting required.',\n },\n {\n id: 'aqe-3',\n name: 'DemoteBroadcastHashJoin',\n status: 'skipped',\n description: 'Evaluated right dimension relation size (3.1 GB) against auto-broadcast threshold (10 MB).',\n impact: 'Relation exceeded broadcast threshold; preserved robust SortMergeJoin strategy.',\n },\n {\n id: 'aqe-4',\n name: 'EliminateSubqueryAliases',\n status: 'applied',\n description: 'Inlined and pruned unused redundant subquery attribute references from execution DAG.',\n impact: 'Streamlined Catalyst physical plan tree depth from 14 nodes down to 8 nodes.',\n },\n]\n\n@Component({\n changeDetection: ChangeDetectionStrategy.Eager,\n selector: 'ui-spark-job-stage-visualizer, [ui-spark-job-stage-visualizer]',\n standalone: true,\n // Vue renders the root
itself: the host stays out of layout and `class` goes to the root.\n host: { '[attr.class]': '\"contents\"' },\n imports: [\n LucideAngularModule,\n UiBadgeComponent,\n UiButtonComponent,\n UiCardComponent,\n UiCardContentComponent,\n UiCardHeaderComponent,\n UiCardTitleComponent,\n UiProgressComponent,\n UiTableBodyComponent,\n UiTableCellComponent,\n UiTableComponent,\n UiTableHeadComponent,\n UiTableHeaderComponent,\n UiTableRowComponent,\n UiTabsComponent,\n UiTabsListComponent,\n UiTabsTriggerComponent,\n ],\n template: `\n
\n \n
\n
\n \n
\n
\n \n \n
\n
\n
\n

AggregatedCustomerLTVJob

\n \n app-20260821-142850-0042\n \n \n @if (copiedAppId()) {\n \n } @else {\n \n }\n Copy Application ID\n \n
\n

Apache Spark 3.5.1 · Adaptive Query Execution (AQE) Enabled

\n
\n
\n\n \n
\n
\n \n Job Succeeded\n ·\n 03m:42s\n
\n
\n \n \n Cluster: c5d.4xlarge (4 nodes · 112 vCPUs)\n \n
\n
\n \n Started: 14:28:50 UTC\n
\n
\n
\n\n \n
\n \n @if (downloadStarted()) {\n \n } @else {\n \n }\n {{ downloadStarted() ? 'Log Exported' : 'Download EventLog' }}\n \n\n \n \n Kill Application\n \n
\n
\n\n \n @if (showKillConfirm()) {\n \n
\n \n \n This application is in finished state (SUCCEEDED). Terminating an inactive application will release log\n retention locks.\n \n
\n
\n \n \n
\n
\n }\n
\n\n \n
\n
\n
\n

Total Stages Completed

\n
\n \n
\n
\n
\n
6 / 6
\n
\n 6 / 6 Stages Succeeded\n
\n

0 failed · 0 skipped · 4 stages active in DAG

\n
\n
\n\n
\n
\n

Shuffle Read / Write Volume

\n
\n \n
\n
\n
\n
42.8 GB
\n
\n 42.8 GB Shuffle Write\n ·\n 1.48 GB Spill (Memory)\n
\n

38.6 GB shuffle read · 0 B disk spill

\n
\n
\n\n
\n
\n

Task Skew Max/Median Ratio

\n
\n \n
\n
\n
\n
1.18x
\n
\n 1.18x Ratio · Well Balanced\n
\n

\n Max task: 4.2s · Median:\n 3.5s\n

\n
\n
\n\n
\n
\n

Executor JVM GC Time

\n
\n \n
\n
\n
\n
2.1%
\n
\n 2.1% of Total CPU Time · Healthy\n
\n

18.4s GC time across 880s CPU total

\n
\n
\n
\n\n \n \n
\n \n \n \n \n \n \n\n
\n \n Active Selection:\n Stage {{ selectedStage().id }} ({{ selectedStage().name }})\n
\n
\n\n \n @if (activeTab() === 'dag') {\n
\n @for (stage of allStages(); track stage.id) {\n \n }\n
\n }\n\n \n @if (activeTab() === 'skew') {\n
\n
\n
\n

\n Task Duration Quantiles — Stage {{ selectedStage().id }} ({{ selectedStage().name }})\n

\n \n Skew {{ selectedStage().skewRatio.toFixed(2) }}x\n \n
\n
\n
\n @for (q of quantileRows(); track q.label) {\n
\n {{ q.label }}\n
\n
\n
\n {{ q.value }}s\n
\n }\n
\n @for (stage of allStages(); track stage.id) {\n \n Stage {{ stage.id }}\n \n }\n
\n
\n
\n }\n\n \n @if (activeTab() === 'executors') {\n
\n
\n
\n

\n Active Cluster Executors (4 Nodes · 112 Cores Total)\n

\n All Healthy\n
\n
\n
\n \n \n \n \n \n \n \n \n \n \n \n \n @for (exec of executors(); track exec.id) {\n \n \n \n \n \n \n \n \n }\n \n
ExecutorHostCoresMemoryGC Pause TimeTasks
{{ exec.id }}{{ exec.host }}{{ exec.cores }}\n {{ exec.memoryUsedGB.toFixed(1) }}/{{ exec.memoryTotalGB }} GB\n \n {{ exec.gcTimeSec.toFixed(1) }}s ({{ exec.gcPercent.toFixed(1) }}%)\n {{ exec.tasksCompleted }}
\n
\n
\n }\n\n \n @if (activeTab() === 'aqe') {\n
\n
\n
\n
\n

Adaptive Query Execution (AQE) Rules Engine

\n AQE Status: Active (3 Rules Applied)\n
\n

\n Catalyst optimizer rewrites evaluated at runtime against live shuffle statistics.\n

\n
\n
\n @for (rule of aqeRules(); track rule.id) {\n
\n
\n {{ rule.name }}\n \n {{ rule.status }}\n \n
\n

{{ rule.description }}

\n

\n \n {{ rule.impact }}\n

\n
\n }\n
\n }\n \n \n `,\n})\nexport class UiSparkJobStageVisualizerComponent {\n protected readonly ArrowRightLeft = ArrowRightLeft\n protected readonly Check = Check\n protected readonly CheckCircle2 = CheckCircle2\n protected readonly Clock = Clock\n protected readonly Copy = Copy\n protected readonly Cpu = Cpu\n protected readonly Download = Download\n protected readonly Flame = Flame\n protected readonly Gauge = Gauge\n protected readonly Server = Server\n protected readonly ShieldAlert = ShieldAlert\n protected readonly X = X\n protected readonly Zap = Zap\n\n @Input() set initialStageId(v: number) {\n if (v !== undefined && v !== null) this.selectedStageId.set(v)\n }\n @Input() set stages(v: SparkStageSummary[] | undefined) {\n if (v) this.stageList.set(v)\n }\n @Input('class') className?: string\n\n private readonly stageList = signal(defaultStages)\n readonly executors = signal(defaultExecutors)\n readonly aqeRules = signal(defaultAqeRules)\n\n readonly selectedStageId = signal(2)\n readonly activeTab = signal('dag')\n readonly copiedAppId = signal(false)\n readonly showKillConfirm = signal(false)\n readonly downloadStarted = signal(false)\n\n readonly allStages = computed(() => this.stageList())\n\n readonly selectedStage = computed(() => {\n const list = this.stageList()\n return list.find((s) => s.id === this.selectedStageId()) ?? list[2] ?? list[0]!\n })\n\n readonly quantileRows = computed(() => {\n const q = this.selectedStage().quantiles\n const max = q.max || 1\n const rows = [\n { label: 'min', value: q.min },\n { label: 'p25', value: q.p25 },\n { label: 'median', value: q.median },\n { label: 'p75', value: q.p75 },\n { label: 'p95', value: q.p95 },\n { label: 'max', value: q.max },\n ]\n return rows.map((r) => ({ ...r, value: r.value.toFixed(1), pct: Math.min(100, (r.value / max) * 100) }))\n })\n\n get rootClass(): string {\n return cn('text-foreground w-full space-y-6 font-sans antialiased', this.className)\n }\n\n stageCardClass(id: number): string {\n return cn(\n 'rounded-xl border p-4 text-left transition-colors cursor-pointer',\n this.selectedStageId() === id\n ? 'border-primary bg-primary/5 ring-primary shadow-xs ring-1'\n : 'border-border bg-card hover:border-primary/40',\n )\n }\n\n setActiveTab(value: string): void {\n this.activeTab.set(value as SparkTab)\n }\n\n selectStage(id: number): void {\n this.selectedStageId.set(id)\n }\n\n async copyAppId(): Promise {\n try {\n await navigator.clipboard.writeText('app-20260821-142850-0042')\n this.copiedAppId.set(true)\n setTimeout(() => this.copiedAppId.set(false), 2000)\n } catch {\n // Clipboard unavailable\n }\n }\n\n triggerDownload(): void {\n this.downloadStarted.set(true)\n setTimeout(() => this.downloadStarted.set(false), 2500)\n }\n}\n", "type": "registry:page", "target": "~/src/app/components/blocks/spark-job-stage-visualizer/spark-job-stage-visualizer.component.ts" }, { "path": "packages/registry-angular/blocks/spark-job-stage-visualizer/index.ts", "content": "export { UiSparkJobStageVisualizerComponent, type SparkTab } from './spark-job-stage-visualizer.component'\n", "type": "registry:page", "target": "~/src/app/components/blocks/spark-job-stage-visualizer/index.ts" } ], "dependencies": [ "lucide-angular" ], "devDependencies": [], "registryDependencies": [ "https://uipkge.dev/r/angular/badge.json", "https://uipkge.dev/r/angular/button.json", "https://uipkge.dev/r/angular/card.json", "https://uipkge.dev/r/angular/progress.json", "https://uipkge.dev/r/angular/table.json", "https://uipkge.dev/r/angular/tabs.json" ], "description": "Databricks and Apache Spark distributed job execution DAG visualizer with stage performance metrics, task skew distribution quantiles, and executor JVM GC telemetry.", "categories": [ "devops", "dashboard", "analytics", "data", "app" ] }