{ "$schema": "https://ui.shadcn.com/schema/registry-item.json", "name": "spark-job-stage-visualizer", "title": "Spark Job Stage Visualizer", "type": "registry:page", "files": [ { "path": "packages/registry-react/blocks/spark-job-stage-visualizer/SparkJobStageVisualizer.tsx", "content": "'use client'\n\nimport * as React from 'react'\nimport { Check, CheckCircle2, Clock, Copy, Download, Flame, Server, ShieldAlert, X } from 'lucide-react'\nimport { cn } from '@/lib/utils'\nimport { Badge } from '@/components/ui/badge'\nimport { Button } from '@/components/ui/button'\nimport { Tabs, TabsList, TabsTrigger } from '@/components/ui/tabs'\nimport { SparkJobKpis } from './SparkJobKpis'\nimport { SparkStageDagTab } from './SparkStageDagTab'\nimport { SparkStageSkewTab } from './SparkStageSkewTab'\nimport { SparkExecutorsAqe } from './SparkExecutorsAqe'\nimport type {\n AqeRule,\n ExecutorStatus,\n SparkExecutor,\n SparkStage,\n StageStatus,\n TaskBreakdownMetrics,\n TaskQuantiles,\n} from './spark-stage-types'\nimport { defaultStages } from './spark-stage-data'\n\nexport type { AqeRule, ExecutorStatus, SparkExecutor, SparkStage, StageStatus, TaskBreakdownMetrics, TaskQuantiles }\n\nexport interface SparkJobStageVisualizerProps {\n initialStageId?: number\n stages?: SparkStage[]\n className?: string\n}\n\nexport function SparkJobStageVisualizer({\n initialStageId = 2,\n stages = defaultStages,\n className,\n}: SparkJobStageVisualizerProps) {\n const [selectedStageId, setSelectedStageId] = React.useState(initialStageId)\n const [activeTab, setActiveTab] = React.useState('dag')\n const [copiedAppId, setCopiedAppId] = React.useState(false)\n const [showKillConfirm, setShowKillConfirm] = React.useState(false)\n const [downloadStarted, setDownloadStarted] = React.useState(false)\n\n const selectedStage = React.useMemo(() => {\n return stages.find((s) => s.id === selectedStageId) ?? stages[2] ?? stages[0]\n }, [stages, selectedStageId])\n\n const copyAppId = React.useCallback(() => {\n navigator.clipboard?.writeText('app-20260821-142850-0042')\n setCopiedAppId(true)\n setTimeout(() => {\n setCopiedAppId(false)\n }, 2000)\n }, [])\n\n const triggerDownload = React.useCallback(() => {\n setDownloadStarted(true)\n setTimeout(() => {\n setDownloadStarted(false)\n }, 2500)\n }, [])\n\n return (\n
\n {/* Header Section */}\n
\n
\n {/* Title & Identifiers */}\n
\n
\n
\n \n
\n
\n
\n

AggregatedCustomerLTVJob

\n \n app-20260821-142850-0042\n \n \n {copiedAppId ? : }\n Copy Application ID\n \n
\n

\n Apache Spark 3.5.1 · Adaptive Query Execution (AQE) Enabled\n

\n
\n
\n\n {/* Metadata Tags */}\n
\n
\n \n Job Succeeded\n ·\n 03m:42s\n
\n
\n \n \n Cluster: c5d.4xlarge (4 nodes · 112 vCPUs)\n \n
\n
\n \n \n Started: 14:28:50 UTC\n \n
\n
\n
\n\n {/* Action Buttons */}\n
\n \n {downloadStarted ? : }\n {downloadStarted ? 'Log Exported' : 'Download EventLog'}\n \n\n setShowKillConfirm(!showKillConfirm)}\n >\n \n Kill Application\n \n
\n
\n\n {/* Kill Confirmation Alert (Collapsible) */}\n {showKillConfirm && (\n
\n
\n \n \n This application is in finished state (SUCCEEDED). Terminating an inactive application will release log\n retention locks.\n \n
\n
\n \n \n
\n
\n )}\n
\n\n {/* 4 Spark Job Execution KPI Cards */}\n \n\n {/* Main Content Tabs */}\n \n
\n \n \n Stages DAG Flow\n \n \n Task Skew & Quantiles\n \n \n Executors (4)\n \n \n AQE Optimizer\n \n \n\n
\n \n Active Selection:\n \n Stage {selectedStage.id} ({selectedStage.name})\n \n
\n
\n\n {/* TAB 1: INTERACTIVE SPARK STAGES DAG FLOW */}\n \n\n {/* TAB 2: TASK SKEW & QUANTILES DISTRIBUTION */}\n \n\n {/* TAB 3 & 4: Executors + AQE tabs */}\n \n \n
\n )\n}\n\nexport default SparkJobStageVisualizer\n", "type": "registry:page", "target": "~/components/blocks/SparkJobStageVisualizer.tsx" }, { "path": "packages/registry-react/blocks/spark-job-stage-visualizer/SparkJobKpis.tsx", "content": "import * as React from 'react'\nimport { ArrowRightLeft, CheckCircle2, Cpu, Gauge } from 'lucide-react'\nimport { Badge } from '@/components/ui/badge'\nimport { Card, CardContent, CardHeader, CardTitle } from '@/components/ui/card'\n\nexport function SparkJobKpis() {\n return (\n
\n {/* KPI 1: Stages */}\n \n \n Total Stages Completed \n
\n \n
\n \n \n
6 / 6
\n
\n \n {' '}\n 6 / 6 Stages Succeeded{' '}\n \n
\n

0 failed · 0 skipped · 4 stages active in DAG

\n \n \n\n {/* KPI 2: Shuffle & Spill */}\n \n \n Shuffle Read / Write Volume \n
\n \n
\n \n \n
42.8 GB
\n
\n 42.8 GB Shuffle Write\n ·\n 1.48 GB Spill (Memory)\n
\n

38.6 GB shuffle read · 0 B disk spill

\n \n \n\n {/* KPI 3: Task Skew Ratio */}\n \n \n Task Skew Max/Median Ratio \n
\n \n
\n \n \n
1.18x
\n
\n \n {' '}\n 1.18x Ratio · Well Balanced{' '}\n \n
\n

\n Max task: 4.2s · Median:\n 3.5s\n

\n \n \n\n {/* KPI 4: JVM GC Overhead */}\n \n \n Executor JVM GC Time \n
\n \n
\n \n \n
2.1%
\n
\n \n {' '}\n 2.1% of Total CPU Time · Healthy{' '}\n \n
\n

18.4s GC time across 880s CPU total

\n \n \n
\n )\n}\n", "type": "registry:page", "target": "~/components/blocks/SparkJobKpis.tsx" }, { "path": "packages/registry-react/blocks/spark-job-stage-visualizer/SparkStageDagTab.tsx", "content": "import * as React from 'react'\nimport { ArrowDown, Database, Layers, Workflow } from 'lucide-react'\nimport { cn } from '@/lib/utils'\nimport { Badge } from '@/components/ui/badge'\nimport { Card, CardContent, CardDescription, CardHeader, CardTitle } from '@/components/ui/card'\nimport { TabsContent } from '@/components/ui/tabs'\nimport type { SparkStage } from './spark-stage-types'\n\nexport interface SparkStageDagTabProps {\n selectedStage: SparkStage\n selectedStageId: number\n stages: SparkStage[]\n onSelectStage: (id: number) => void\n}\n\nexport function SparkStageDagTab({\n selectedStage,\n selectedStageId,\n stages: _stages,\n onSelectStage,\n}: SparkStageDagTabProps) {\n return (\n \n \n \n
\n
\n Distributed Job Execution DAG \n \n Interactive stage graph showing task parallelization, shuffle boundaries, and operator fusion. Click any\n stage to inspect.\n \n
\n
\n \n AQE Coalesced: 540 → 200 Partitions\n \n
\n
\n \n\n \n {/* DAG Topology Container */}\n
\n {/* Two parallel branches joining together */}\n
\n {/* Left Branch: Transactions */}\n
\n
\n
\n \n \n Branch 1: Customer Transactions Pipeline\n \n
\n \n 14.8M records\n \n
\n\n {/* Stage 0 */}\n onSelectStage(0)}\n onKeyDown={(e) => {\n if (e.key === 'Enter' || e.key === ' ') {\n e.preventDefault()\n onSelectStage(0)\n }\n }}\n >\n
\n
\n
\n \n Stage 0\n \n FileScan parquet\n
\n

540 tasks · 18s · 14.2 GB read

\n
\n \n Succeeded\n \n
\n
\n Operator: FileScan parquet\n 14.82M rows\n
\n
\n\n {/* Connector Arrow */}\n
\n
\n \n 14.2 GB in-memory scan\n
\n
\n\n {/* Stage 1 */}\n onSelectStage(1)}\n onKeyDown={(e) => {\n if (e.key === 'Enter' || e.key === ' ') {\n e.preventDefault()\n onSelectStage(1)\n }\n }}\n >\n
\n
\n
\n \n Stage 1\n \n Exchange hashpartitioning\n
\n

200 tasks · 42s · 8.4 GB shuffle write

\n
\n \n Succeeded\n \n
\n
\n Key: Murmur3(customer_id)\n 200 partitions\n
\n
\n\n {/* Connector Arrow */}\n
\n
\n \n 8.4 GB shuffle transfer\n
\n
\n\n {/* Stage 2 */}\n onSelectStage(2)}\n onKeyDown={(e) => {\n if (e.key === 'Enter' || e.key === ' ') {\n e.preventDefault()\n onSelectStage(2)\n }\n }}\n >\n
\n
\n
\n \n Stage 2\n \n HashAggregate & Filter\n
\n

200 tasks · 1m 12s · 4.6 GB write

\n
\n \n Succeeded\n \n
\n
\n Aggregations: sum(amount), count(1)\n 420 MB spill\n
\n
\n \n\n {/* Right Branch: Customer Dim */}\n
\n
\n
\n \n \n Branch 2: Customer Dimension Pipeline\n \n
\n \n 2.4M records\n \n
\n\n {/* Stage 4 */}\n onSelectStage(4)}\n onKeyDown={(e) => {\n if (e.key === 'Enter' || e.key === ' ') {\n e.preventDefault()\n onSelectStage(4)\n }\n }}\n >\n
\n
\n
\n \n Stage 4\n \n FileScan customer_dim\n
\n

120 tasks · 12s · 3.1 GB read

\n
\n \n Succeeded\n \n
\n
\n Operator: FileScan parquet\n 2.40M rows\n
\n
\n\n {/* Connector Arrow */}\n
\n
\n \n 3.1 GB dimension scan\n
\n
\n\n {/* Stage 5 */}\n onSelectStage(5)}\n onKeyDown={(e) => {\n if (e.key === 'Enter' || e.key === ' ') {\n e.preventDefault()\n onSelectStage(5)\n }\n }}\n >\n
\n
\n
\n \n Stage 5\n \n Exchange customer_dim\n
\n

120 tasks · 16s · 3.1 GB shuffle write

\n
\n \n Succeeded\n \n
\n
\n Key: Murmur3(customer_id)\n 200 partitions\n
\n \n\n {/* Branch alignment spacer */}\n \n \n \n\n {/* Convergence Flow Indicator */}\n
\n
\n \n Shuffle Partition Alignment (200 Partitions × 2 Inputs)\n
\n \n
\n\n {/* Joined Final Stage: Stage 3 */}\n onSelectStage(3)}\n onKeyDown={(e) => {\n if (e.key === 'Enter' || e.key === ' ') {\n e.preventDefault()\n onSelectStage(3)\n }\n }}\n >\n
\n
\n
\n \n Stage 3\n \n

SortMergeJoin (Final Reduction)

\n \n Terminal Stage\n \n
\n

\n 200 tasks · 1m 20s · Joined 14.8M keys · 1.24M output rows\n

\n
\n
\n
\n
29.8 GB Output
\n
1.06 GB Mem Spill
\n
\n \n Succeeded\n \n
\n
\n\n
\n
\n Join Type:\n

Inner SortMergeJoin

\n
\n
\n Join Key:\n

customer_id#42 = customer_id#98

\n
\n
\n Tasks:\n

200 / 200 (100%)

\n
\n
\n Skew Max/Median:\n

1.21x (Balanced)

\n
\n
\n \n \n \n \n\n {/* Selected Stage Deep Inspector */}\n \n \n
\n
\n \n Selected Stage {selectedStage.id}\n \n {selectedStage.name} Details\n
\n
\n Duration:\n \n {selectedStage.duration}\n \n \n Completed\n \n
\n
\n \n\n \n {/* Call Site Code Box */}\n
\n Catalyst Operator Expression\n
\n {selectedStage.operator}\n
\n
\n\n {/* Stage Metrics Grid */}\n
\n
\n Tasks\n

\n {selectedStage.tasksSucceeded} / {selectedStage.tasksTotal}\n

\n
\n
\n Input Data\n

{selectedStage.inputBytes}

\n
\n
\n Shuffle Read\n

{selectedStage.shuffleReadBytes}

\n
\n
\n Shuffle Write\n

{selectedStage.shuffleWriteBytes}

\n
\n
\n Spill (Memory)\n

{selectedStage.spillMemory}

\n
\n
\n Task Skew Ratio\n

{selectedStage.skewRatio}x

\n
\n
\n\n

{selectedStage.details}

\n \n \n \n )\n}\n", "type": "registry:page", "target": "~/components/blocks/SparkStageDagTab.tsx" }, { "path": "packages/registry-react/blocks/spark-job-stage-visualizer/SparkStageSkewTab.tsx", "content": "import * as React from 'react'\nimport { Badge } from '@/components/ui/badge'\nimport { Button } from '@/components/ui/button'\nimport { Card, CardContent, CardDescription, CardHeader, CardTitle } from '@/components/ui/card'\nimport { TabsContent } from '@/components/ui/tabs'\nimport type { SparkStage } from './spark-stage-types'\nimport { getQuantilePosition } from './spark-stage-data'\n\nexport interface SparkStageSkewTabProps {\n selectedStage: SparkStage\n stages: SparkStage[]\n selectedStageId: number\n onSelectStage: (id: number) => void\n}\n\nexport function SparkStageSkewTab({ selectedStage, stages, selectedStageId, onSelectStage }: SparkStageSkewTabProps) {\n return (\n \n \n \n
\n
\n Stage Task Duration Skew & Quantiles \n \n Distribution of task execution time across executor cores for the selected stage.\n \n
\n\n {/* Quick Stage Selector Buttons */}\n
\n {stages.map((s) => (\n onSelectStage(s.id)}\n >\n Stage {s.id}\n \n ))}\n
\n
\n \n\n \n {/* Stage Title & Skew Ratio Banner */}\n
\n
\n
\n \n Stage {selectedStage.id}: {selectedStage.name}\n \n \n {selectedStage.tasksTotal} Tasks\n \n
\n

\n Duration: {selectedStage.duration} · Peak\n task memory: {selectedStage.taskBreakdown.peakMemory}\n

\n
\n\n
\n
\n Max / Median Skew:\n
{selectedStage.skewRatio}x
\n
\n \n {' '}\n Well Balanced (< 1.5x){' '}\n \n
\n
\n\n {/* Quantiles Metric Row (Min, P25, Median, P75, P95, Max) */}\n
\n
\n Min (0%)\n

\n {selectedStage.quantiles.min}s\n

\n Fastest task\n
\n\n
\n 25th Percentile\n

\n {selectedStage.quantiles.p25}s\n

\n Q1 boundary\n
\n\n
\n Median (P50)\n

\n {selectedStage.quantiles.median}s\n

\n Midpoint baseline\n
\n\n
\n 75th Percentile\n

\n {selectedStage.quantiles.p75}s\n

\n Q3 boundary\n
\n\n
\n 95th Percentile\n

\n {selectedStage.quantiles.p95}s\n

\n Tail latency\n
\n\n
\n Max (100%)\n

\n {selectedStage.quantiles.max}s\n

\n Straggler peak\n
\n
\n\n {/* Visual Distribution Bar */}\n
\n
\n Task Duration Quantiles Scale (0.0s – 5.0s)\n \n IQR: {(selectedStage.quantiles.p75 - selectedStage.quantiles.p25).toFixed(1)}s\n \n
\n\n {/* Custom Box & Whisker Horizontal Bar */}\n
\n {/* Background track */}\n
\n {/* Min to Max Range Bar */}\n \n {/* P25 to P75 Interquartile Core */}\n \n {/* Median Marker */}\n \n
\n\n {/* Markers with Pin Labels */}\n \n Min {selectedStage.quantiles.min}s\n
\n \n Med {selectedStage.quantiles.median}s\n
\n \n Max {selectedStage.quantiles.max}s\n \n \n\n
\n
\n
\n \n Full Range (Min - Max)\n
\n
\n \n IQR Core (P25 - P75)\n
\n
\n \n Median Point\n
\n
\n \n Skew status: Optimal Partitioning\n \n
\n \n\n {/* Task Internal Execution Time Breakdown */}\n
\n

\n Task Execution Phase Breakdown (Average per Task)\n

\n\n
\n
\n CPU Run Time\n

\n {{ ...selectedStage.taskBreakdown }.executorRunTime}\n

\n Active calculation\n
\n\n
\n JVM GC Overhead\n

\n {selectedStage.taskBreakdown.jvmGcTime}\n

\n Young Gen sweeps\n
\n\n
\n Shuffle Fetch Wait\n

\n {selectedStage.taskBreakdown.shuffleFetchWait}\n

\n Network remote blocks\n
\n\n
\n Deserialization\n

\n {selectedStage.taskBreakdown.deserializationTime}\n

\n Task definition\n
\n\n
\n Result Serialization\n

\n {selectedStage.taskBreakdown.serializationTime}\n

\n Map output buffers\n
\n\n
\n Peak Memory\n

\n {selectedStage.taskBreakdown.peakMemory}\n

\n Off-heap aggregate\n
\n
\n
\n \n \n \n )\n}\n", "type": "registry:page", "target": "~/components/blocks/SparkStageSkewTab.tsx" }, { "path": "packages/registry-react/blocks/spark-job-stage-visualizer/SparkExecutorsAqe.tsx", "content": "import * as React from 'react'\nimport { Badge } from '@/components/ui/badge'\nimport { Card, CardContent, CardDescription, CardHeader, CardTitle } from '@/components/ui/card'\nimport { Progress } from '@/components/ui/progress'\nimport { Table, TableBody, TableCell, TableHead, TableHeader, TableRow } from '@/components/ui/table'\nimport { TabsContent } from '@/components/ui/tabs'\nimport { Server, Zap } from 'lucide-react'\nimport type { AqeRule, SparkExecutor } from './spark-stage-types'\n\nconst defaultExecutors: SparkExecutor[] = [\n {\n id: 'driver',\n host: '10.0.12.4',\n role: 'driver',\n status: 'active',\n cores: 16,\n memoryUsedGB: 24.2,\n memoryTotalGB: 32,\n memoryPercent: 75.6,\n gcTimeSec: 4.2,\n gcPercent: 1.8,\n tasksCompleted: 285,\n tasksTotal: 285,\n tasksFailed: 0,\n shuffleReadGB: 10.8,\n shuffleWriteGB: 10.8,\n spillGB: 0.36,\n },\n {\n id: 'exec-1',\n host: '10.0.12.18',\n role: 'executor',\n status: 'active',\n cores: 32,\n memoryUsedGB: 28.8,\n memoryTotalGB: 32,\n memoryPercent: 90.0,\n gcTimeSec: 5.1,\n gcPercent: 2.2,\n tasksCompleted: 284,\n tasksTotal: 284,\n tasksFailed: 0,\n shuffleReadGB: 11.2,\n shuffleWriteGB: 11.2,\n spillGB: 0.42,\n },\n {\n id: 'exec-2',\n host: '10.0.12.29',\n role: 'executor',\n status: 'active',\n cores: 32,\n memoryUsedGB: 21.4,\n memoryTotalGB: 32,\n memoryPercent: 66.8,\n gcTimeSec: 4.8,\n gcPercent: 2.3,\n tasksCompleted: 286,\n tasksTotal: 286,\n tasksFailed: 0,\n shuffleReadGB: 10.4,\n shuffleWriteGB: 10.4,\n spillGB: 0.34,\n },\n {\n id: 'exec-3',\n host: '10.0.12.44',\n role: 'executor',\n status: 'active',\n cores: 32,\n memoryUsedGB: 18.6,\n memoryTotalGB: 32,\n memoryPercent: 58.1,\n gcTimeSec: 4.3,\n gcPercent: 2.1,\n tasksCompleted: 285,\n tasksTotal: 285,\n tasksFailed: 0,\n shuffleReadGB: 10.4,\n shuffleWriteGB: 10.4,\n spillGB: 0.36,\n },\n]\n\nconst defaultAqeRules: AqeRule[] = [\n {\n id: 'aqe-1',\n name: 'CoalesceShufflePartitions',\n status: 'applied',\n description:\n 'Coalesced 540 initial map partitions down to 200 target shuffle partitions based on target partition size (64 MB).',\n impact: 'Reduced network RPC overhead by 63% and avoided 340 small shuffle partition files.',\n },\n {\n id: 'aqe-2',\n name: 'OptimizeSkewedJoin',\n status: 'applied',\n description: 'Dynamic skew detection evaluated all 200 partition sizes against the 5x median threshold.',\n impact: 'Max partition skew ratio remained well balanced at 1.18x. No partition splitting required.',\n },\n {\n id: 'aqe-3',\n name: 'DemoteBroadcastHashJoin',\n status: 'skipped',\n description: 'Evaluated right dimension relation size (3.1 GB) against auto-broadcast threshold (10 MB).',\n impact: 'Relation exceeded broadcast threshold; preserved robust SortMergeJoin strategy.',\n },\n {\n id: 'aqe-4',\n name: 'EliminateSubqueryAliases',\n status: 'applied',\n description: 'Inlined and pruned unused redundant subquery attribute references from execution DAG.',\n impact: 'Streamlined Catalyst physical plan tree depth from 14 nodes down to 8 nodes.',\n },\n]\n\nexport function SparkExecutorsAqe() {\n return (\n <>\n {/* TAB 3: EXECUTORS TELEMETRY (4 NODES) */}\n \n \n \n
\n
\n \n Active Cluster Executors (4 Nodes · 112 Cores Total)\n \n \n Real-time worker telemetry: CPU utilization, JVM heap memory, garbage collection pauses, and task\n throughput.\n \n
\n\n
\n \n Total Memory: 93.0 / 128 GB (72.7%)\n \n \n 4/4 Nodes Active\n \n
\n
\n \n\n \n
\n \n \n \n Executor ID & Host\n Role & Status\n Cores\n Memory Used\n GC Pause Time\n Tasks Completed\n Shuffle I/O\n Memory Spill\n \n \n \n {defaultExecutors.map((exec) => (\n \n {/* Executor ID & Host */}\n \n
\n \n
\n
{exec.id}
\n
{exec.host}
\n
\n
\n \n\n {/* Role & Status */}\n \n
\n \n {exec.role}\n \n \n
\n \n\n {/* Cores */}\n {exec.cores} cores\n\n {/* Memory Used */}\n \n
\n
\n \n {exec.memoryUsedGB} / {exec.memoryTotalGB} GB\n \n 85 ? 'text-warning font-semibold' : 'text-muted-foreground'\n }\n >\n {exec.memoryPercent}%\n \n
\n \n
\n \n\n {/* GC Time */}\n \n
\n \n {exec.gcTimeSec}s ({exec.gcPercent}%)\n \n
\n \n\n {/* Tasks Completed */}\n \n {exec.tasksCompleted}\n / {exec.tasksTotal}\n (0 failed)\n \n\n {/* Shuffle I/O */}\n {exec.shuffleReadGB} GB\n\n {/* Spill */}\n {exec.spillGB} GB\n \n ))}\n \n
\n
\n \n \n \n\n {/* TAB 4: AQE OPTIMIZATION RULES */}\n \n \n \n
\n
\n Adaptive Query Execution (AQE) Rules Engine\n \n Runtime Catalyst optimizer decisions made dynamically based on actual stage map output partition\n statistics.\n \n
\n\n \n AQE Status: Active (3 Rules Applied)\n \n
\n \n\n \n
\n {defaultAqeRules.map((rule) => (\n
\n
\n
\n \n {rule.name}\n
\n \n {rule.status}\n \n
\n\n

{rule.description}

\n\n
\n Query Plan Impact: \n {rule.impact}\n
\n
\n ))}\n
\n \n \n \n \n )\n}\n", "type": "registry:page", "target": "~/components/blocks/SparkExecutorsAqe.tsx" }, { "path": "packages/registry-react/blocks/spark-job-stage-visualizer/spark-stage-data.ts", "content": "import type { SparkStage } from './spark-stage-types'\n\nexport function getQuantilePosition(val: number, max: number = 5.0): number {\n return Math.min(100, Math.max(0, (val / max) * 100))\n}\n\nexport const defaultStages: SparkStage[] = [\n {\n id: 0,\n name: 'FileScan parquet',\n callSite: 'spark.read.parquet(\"s3://warehouse/customer_transactions\")',\n operator: 'FileScan parquet default.customer_transactions [customer_id#42, amount#43, txn_date#44]',\n status: 'succeeded',\n tasksTotal: 540,\n tasksSucceeded: 540,\n tasksFailed: 0,\n duration: '18s',\n durationSec: 18,\n inputBytes: '14.2 GB',\n inputRecords: '14,820,400',\n outputBytes: '14.2 GB',\n outputRecords: '14,820,400',\n shuffleReadBytes: '0 B',\n shuffleWriteBytes: '0 B',\n spillMemory: '0 B',\n spillDisk: '0 B',\n skewRatio: 1.12,\n quantiles: { min: 0.8, p25: 1.2, median: 1.6, p75: 1.8, p95: 2.0, max: 2.1 },\n taskBreakdown: {\n executorRunTime: '1.52s',\n jvmGcTime: '32ms',\n shuffleFetchWait: '0ms',\n serializationTime: '14ms',\n deserializationTime: '28ms',\n peakMemory: '128 MB',\n },\n details:\n 'Read partitioned parquet files from AWS S3 bucket s3://warehouse/customer_transactions. Pushed filters: [IsNotNull(customer_id), GreaterThanOrEqual(txn_date, 2026-01-01)].',\n upstreamStageIds: [],\n downstreamStageIds: [1],\n branch: 'left',\n },\n {\n id: 1,\n name: 'Exchange hashpartitioning',\n callSite: 'df.repartitionByRange(200, col(\"customer_id\"))',\n operator: 'Exchange hashpartitioning(customer_id#42, 200), ENSURE_REQUIREMENTS, [plan_id=108]',\n status: 'succeeded',\n tasksTotal: 200,\n tasksSucceeded: 200,\n tasksFailed: 0,\n duration: '42s',\n durationSec: 42,\n inputBytes: '0 B',\n inputRecords: '14,820,400',\n outputBytes: '8.4 GB',\n outputRecords: '14,820,400',\n shuffleReadBytes: '14.2 GB',\n shuffleWriteBytes: '8.4 GB',\n spillMemory: '0 B',\n spillDisk: '0 B',\n skewRatio: 1.15,\n quantiles: { min: 1.8, p25: 2.4, median: 2.9, p75: 3.2, p95: 3.4, max: 3.6 },\n taskBreakdown: {\n executorRunTime: '2.74s',\n jvmGcTime: '58ms',\n shuffleFetchWait: '92ms',\n serializationTime: '18ms',\n deserializationTime: '22ms',\n peakMemory: '240 MB',\n },\n details:\n 'Shuffle exchange hash-partitioning 14.82M rows across 200 AQE-coalesced partition buckets based on Murmur3 hash of customer_id column.',\n upstreamStageIds: [0],\n downstreamStageIds: [2],\n branch: 'left',\n },\n {\n id: 2,\n name: 'HashAggregate & Filter',\n callSite: 'df.groupBy(\"customer_id\").agg(sum(\"amount\").as(\"total_spend\"), count(\"txn_id\").as(\"tx_count\"))',\n operator: 'HashAggregate(keys=[customer_id#42], functions=[sum(amount#43), count(txn_id#45)])',\n status: 'succeeded',\n tasksTotal: 200,\n tasksSucceeded: 200,\n tasksFailed: 0,\n duration: '1m 12s',\n durationSec: 72,\n inputBytes: '0 B',\n inputRecords: '14,820,400',\n outputBytes: '4.6 GB',\n outputRecords: '1,480,000',\n shuffleReadBytes: '8.4 GB',\n shuffleWriteBytes: '4.6 GB',\n spillMemory: '420 MB',\n spillDisk: '0 B',\n skewRatio: 1.18,\n quantiles: { min: 2.1, p25: 2.8, median: 3.5, p75: 3.9, p95: 4.1, max: 4.2 },\n taskBreakdown: {\n executorRunTime: '3.42s',\n jvmGcTime: '74ms',\n shuffleFetchWait: '110ms',\n serializationTime: '14ms',\n deserializationTime: '24ms',\n peakMemory: '342 MB',\n },\n details:\n 'Aggregated raw transaction events down to 1.48M unique customer lifetime metrics. In-memory hash aggregation spilled 420 MB into JVM execution memory off-heap storage.',\n upstreamStageIds: [1],\n downstreamStageIds: [3],\n branch: 'left',\n },\n {\n id: 4,\n name: 'FileScan customer_dim',\n callSite: 'spark.read.parquet(\"s3://warehouse/customer_dim\")',\n operator: 'FileScan parquet default.customer_dim [customer_id#98, full_name#99, tier#100, signup_date#101]',\n status: 'succeeded',\n tasksTotal: 120,\n tasksSucceeded: 120,\n tasksFailed: 0,\n duration: '12s',\n durationSec: 12,\n inputBytes: '3.1 GB',\n inputRecords: '2,400,000',\n outputBytes: '3.1 GB',\n outputRecords: '2,400,000',\n shuffleReadBytes: '0 B',\n shuffleWriteBytes: '0 B',\n spillMemory: '0 B',\n spillDisk: '0 B',\n skewRatio: 1.09,\n quantiles: { min: 0.6, p25: 0.9, median: 1.1, p75: 1.3, p95: 1.4, max: 1.5 },\n taskBreakdown: {\n executorRunTime: '1.04s',\n jvmGcTime: '22ms',\n shuffleFetchWait: '0ms',\n serializationTime: '10ms',\n deserializationTime: '18ms',\n peakMemory: '96 MB',\n },\n details: 'Read customer demographic and subscription tier metadata from AWS S3 parquet catalog.',\n upstreamStageIds: [],\n downstreamStageIds: [5],\n branch: 'right',\n },\n {\n id: 5,\n name: 'Exchange customer_dim',\n callSite: 'customerDim.repartition(200, col(\"customer_id\"))',\n operator: 'Exchange hashpartitioning(customer_id#98, 200), ENSURE_REQUIREMENTS, [plan_id=142]',\n status: 'succeeded',\n tasksTotal: 120,\n tasksSucceeded: 120,\n tasksFailed: 0,\n duration: '16s',\n durationSec: 16,\n inputBytes: '0 B',\n inputRecords: '2,400,000',\n outputBytes: '3.1 GB',\n outputRecords: '2,400,000',\n shuffleReadBytes: '3.1 GB',\n shuffleWriteBytes: '3.1 GB',\n spillMemory: '0 B',\n spillDisk: '0 B',\n skewRatio: 1.11,\n quantiles: { min: 0.8, p25: 1.1, median: 1.3, p75: 1.5, p95: 1.7, max: 1.8 },\n taskBreakdown: {\n executorRunTime: '1.24s',\n jvmGcTime: '26ms',\n shuffleFetchWait: '48ms',\n serializationTime: '12ms',\n deserializationTime: '16ms',\n peakMemory: '160 MB',\n },\n details:\n 'Shuffle partitioned customer dimension records across 200 partitions matching left transaction dataset partition key.',\n upstreamStageIds: [4],\n downstreamStageIds: [3],\n branch: 'right',\n },\n {\n id: 3,\n name: 'SortMergeJoin',\n callSite: 'txnAgg.join(customerDim, col(\"customer_id#42\") === col(\"customer_id#98\"), \"inner\")',\n operator: 'SortMergeJoin [customer_id#42], [customer_id#98], Inner',\n status: 'succeeded',\n tasksTotal: 200,\n tasksSucceeded: 200,\n tasksFailed: 0,\n duration: '1m 20s',\n durationSec: 80,\n inputBytes: '0 B',\n inputRecords: '1,480,000',\n outputBytes: '29.8 GB',\n outputRecords: '1,240,000 (Joined 14.8M keys)',\n shuffleReadBytes: '7.7 GB',\n shuffleWriteBytes: '29.8 GB',\n spillMemory: '1.06 GB',\n spillDisk: '0 B',\n skewRatio: 1.21,\n quantiles: { min: 2.4, p25: 3.1, median: 3.8, p75: 4.3, p95: 4.6, max: 4.8 },\n taskBreakdown: {\n executorRunTime: '3.72s',\n jvmGcTime: '82ms',\n shuffleFetchWait: '140ms',\n serializationTime: '22ms',\n deserializationTime: '32ms',\n peakMemory: '480 MB',\n },\n details:\n 'Executed distributed SortMergeJoin matching customer_id keys across left aggregated metrics and right dimension tables. AQE coalesced skewed partitions and eliminated full table broadcast risk.',\n upstreamStageIds: [2, 5],\n downstreamStageIds: [],\n branch: 'join',\n },\n]\n", "type": "registry:page", "target": "~/components/blocks/spark-stage-data.ts" }, { "path": "packages/registry-react/blocks/spark-job-stage-visualizer/spark-stage-types.ts", "content": "export type StageStatus = 'succeeded' | 'running' | 'failed' | 'pending'\nexport type ExecutorStatus = 'active' | 'dead' | 'excluded'\n\nexport interface TaskQuantiles {\n min: number\n p25: number\n median: number\n p75: number\n p95: number\n max: number\n}\n\nexport interface TaskBreakdownMetrics {\n executorRunTime: string\n jvmGcTime: string\n shuffleFetchWait: string\n serializationTime: string\n deserializationTime: string\n peakMemory: string\n}\n\nexport interface SparkStage {\n id: number\n name: string\n callSite: string\n operator: string\n status: StageStatus\n tasksTotal: number\n tasksSucceeded: number\n tasksFailed: number\n duration: string\n durationSec: number\n inputBytes: string\n inputRecords: string\n outputBytes: string\n outputRecords: string\n shuffleReadBytes: string\n shuffleWriteBytes: string\n spillMemory: string\n spillDisk: string\n skewRatio: number\n quantiles: TaskQuantiles\n taskBreakdown: TaskBreakdownMetrics\n details: string\n upstreamStageIds: number[]\n downstreamStageIds: number[]\n branch: 'left' | 'right' | 'join'\n}\n\nexport interface SparkExecutor {\n id: string\n host: string\n role: 'driver' | 'executor'\n status: ExecutorStatus\n cores: number\n memoryUsedGB: number\n memoryTotalGB: number\n memoryPercent: number\n gcTimeSec: number\n gcPercent: number\n tasksCompleted: number\n tasksTotal: number\n tasksFailed: number\n shuffleReadGB: number\n shuffleWriteGB: number\n spillGB: number\n}\n\nexport interface AqeRule {\n id: string\n name: string\n status: 'applied' | 'skipped'\n description: string\n impact: string\n}\n", "type": "registry:page", "target": "~/components/blocks/spark-stage-types.ts" } ], "dependencies": [ "lucide-react" ], "devDependencies": [], "registryDependencies": [ "https://uipkge.dev/r/react/badge.json", "https://uipkge.dev/r/react/button.json", "https://uipkge.dev/r/react/card.json", "https://uipkge.dev/r/react/progress.json", "https://uipkge.dev/r/react/table.json", "https://uipkge.dev/r/react/tabs.json" ], "description": "Databricks and Apache Spark distributed job execution DAG visualizer with stage performance metrics, task skew distribution quantiles, and executor JVM GC telemetry.", "categories": [ "devops", "dashboard", "analytics", "data", "app" ] }