Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
11 changes: 9 additions & 2 deletions README.md
Original file line number Diff line number Diff line change
Expand Up @@ -163,6 +163,13 @@ options, and a metrics table. **Raw logs** expands the captured output; a
invalid metrics show **Structured metrics unavailable** while preserving the
run's status and logs. Failed runs show their error alongside any available results.

**History** lists the instance's 20 most recent runs, newest first, with the
tool, status, duration, and metrics (or the error of a failed run). It reloads
when you start a run and when that run finishes; **Refresh** reloads it on
demand. **View** opens a run as a card with its full results and logs; runs
already shown are marked **Shown**. When history is kept only in memory, the
section says so. See [run history storage](#run-history-storage).

Use **Look up a benchmark run** to retrieve a stored run by ID. Looking up an
already displayed ID does not add a duplicate card. Records remain subject to the
configured [run history storage](#run-history-storage).
Expand Down Expand Up @@ -540,8 +547,8 @@ With the default in-memory storage, run records, including output and metrics,
are lost on restart, and completed records become unavailable one hour after
completion. Configure [PostgreSQL storage](#run-history-storage) to keep
history. Runs in progress during a restart are marked failed rather than
resumed. One backend replica is supported. The UI does not show run history yet,
and comparisons remain future work.
resumed. One backend replica is supported. The UI lists recent runs per
instance; paging beyond them, charts, and comparisons remain future work.

## License

Expand Down
142 changes: 142 additions & 0 deletions src/BenchmarkRunHistory.tsx
Original file line number Diff line number Diff line change
@@ -0,0 +1,142 @@
import { useEffect, useState } from 'react';
import {
Alert, Button, Chip, CircularProgress, Stack, Table, TableBody, TableCell, TableContainer, TableHead,
TableRow, Typography,
} from '@mui/material';
import type { PluginApi } from '@openeverest/plugin-sdk';
import { listBenchmarkRuns } from './benchmarkHistoryApi';
import type { BenchmarkRunSummary } from './benchmarkHistoryApi';
import { formatMetricValue } from './benchmarkMetricFormat';
import { STATUS_COLORS, STATUS_LABELS } from './benchmarkRunStatus';
import type { BenchmarkTarget } from './benchmarkTarget';
import { benchmarkDrivers, isBenchmarkDriverId } from './drivers';

interface BenchmarkRunHistoryProps {
pluginFetch: PluginApi['fetch'];
target: BenchmarkTarget;
/** Changing this value reloads the history, e.g. after a run finishes. */
refreshKey: number;
/** Undefined until the service status is known. */
persistentHistory: boolean | undefined;
shownRunIds: readonly string[];
onOpen: (id: string) => void;
}

type HistoryState =
| { status: 'loading'; runs?: BenchmarkRunSummary[] }
| { status: 'ready'; runs: BenchmarkRunSummary[] }
| { status: 'failed'; message: string; runs?: BenchmarkRunSummary[] };

const driverLabel = (driver: string | undefined) =>
driver === undefined ? '—' : isBenchmarkDriverId(driver) ? benchmarkDrivers[driver].label : driver;

function formatRunDuration(run: Pick<BenchmarkRunSummary, 'createdAt' | 'completedAt'>): string {
if (!run.completedAt) return '—';
const seconds = Math.max(0, Math.round((Date.parse(run.completedAt) - Date.parse(run.createdAt)) / 1000));
if (seconds < 60) return `${seconds} s`;
const minutes = Math.floor(seconds / 60);
return seconds % 60 === 0 ? `${minutes} min` : `${minutes} min ${seconds % 60} s`;
}

export function BenchmarkRunHistory({
pluginFetch, target, refreshKey, persistentHistory, shownRunIds, onOpen,
}: BenchmarkRunHistoryProps) {
const [state, setState] = useState<HistoryState>({ status: 'loading' });
const [attempt, setAttempt] = useState(0);

useEffect(() => {
const controller = new AbortController();
// Keep the previous rows visible while refreshing to avoid flicker.
setState(previous => ({ status: 'loading', runs: previous.runs }));
listBenchmarkRuns(pluginFetch, target, { signal: controller.signal }).then(
runs => { if (!controller.signal.aborted) setState({ status: 'ready', runs }); },
error => {
if (controller.signal.aborted) return;
setState(previous => ({ status: 'failed', message: error.message, runs: previous.runs }));
},
);
return () => controller.abort();
}, [pluginFetch, target.k8sCluster, target.namespace, target.instance, refreshKey, attempt]);

const runs = state.runs;
return (
<Stack spacing={1.5} component="section" aria-label="Benchmark history">
<Stack direction="row" spacing={1} sx={{ alignItems: 'center', justifyContent: 'space-between' }}>
<Typography variant="h6" component="h3">History</Typography>
<Stack direction="row" spacing={1} sx={{ alignItems: 'center' }}>
{state.status === 'loading' && <CircularProgress size={18} aria-label="Loading benchmark history" />}
<Button size="small" disabled={state.status === 'loading'} onClick={() => setAttempt(value => value + 1)}>
Refresh
</Button>
</Stack>
</Stack>
{persistentHistory === false && (
<Typography variant="body2" color="text.secondary">
History is kept in the benchmark service's memory: it is lost when the service restarts, and completed
runs are removed after one hour.
</Typography>
)}
{state.status === 'failed' && (
<Alert severity="error" action={
<Button color="inherit" onClick={() => setAttempt(value => value + 1)}>Retry</Button>
}>{state.message}</Alert>
)}
{runs && runs.length === 0 && state.status === 'ready' && (
<Typography variant="body2" color="text.secondary">
No benchmark runs have been recorded for this instance yet.
</Typography>
)}
{runs && runs.length > 0 && (
<TableContainer>
<Table size="small" aria-label="Benchmark history">
<TableHead>
<TableRow>
<TableCell>Started</TableCell>
<TableCell>Tool</TableCell>
<TableCell>Status</TableCell>
<TableCell>Duration</TableCell>
<TableCell>Results</TableCell>
<TableCell align="right">Details</TableCell>
</TableRow>
</TableHead>
<TableBody>
{runs.map(run => {
const shown = shownRunIds.includes(run.id);
return (
<TableRow key={run.id}>
<TableCell component="th" scope="row" sx={{ whiteSpace: 'nowrap' }}>
{new Date(run.createdAt).toLocaleString()}
</TableCell>
<TableCell>{driverLabel(run.driver)}</TableCell>
<TableCell>
<Chip size="small" label={STATUS_LABELS[run.status]} color={STATUS_COLORS[run.status]} />
</TableCell>
<TableCell sx={{ whiteSpace: 'nowrap' }}>{formatRunDuration(run)}</TableCell>
<TableCell sx={{ minWidth: 160 }}>
{run.metrics && run.metrics.length > 0 ? (
<Stack>
{run.metrics.map(metric => (
<Typography key={JSON.stringify([metric.name, metric.unit])} variant="body2" component="span"
sx={{ whiteSpace: 'nowrap' }}>
{`${metric.name} ${formatMetricValue(metric.value)} ${metric.unit}`}
</Typography>
))}
</Stack>
) : run.status === 'failed' ? run.error : '—'}
</TableCell>
<TableCell align="right">
<Button size="small" disabled={shown} onClick={() => onOpen(run.id)}
aria-label={shown ? `Run ${run.id} is shown` : `View run ${run.id}`}>
{shown ? 'Shown' : 'View'}
</Button>
</TableCell>
</TableRow>
);
})}
</TableBody>
</Table>
</TableContainer>
)}
</Stack>
);
}
8 changes: 1 addition & 7 deletions src/BenchmarkRunMetrics.tsx
Original file line number Diff line number Diff line change
Expand Up @@ -2,13 +2,7 @@ import {
Box, Table, TableBody, TableCell, TableContainer, TableHead, TableRow, Typography,
} from '@mui/material';
import type { BenchmarkMetric } from './benchmarkApi';

// Runners report raw tool precision (pgbench prints six decimals). Values below
// one keep significant digits so small latencies do not round to zero.
const formatMetricValue = (value: number): string =>
Math.abs(value) >= 1 || value === 0
? value.toLocaleString(undefined, { maximumFractionDigits: 2 })
: value.toLocaleString(undefined, { maximumSignificantDigits: 3 });
import { formatMetricValue } from './benchmarkMetricFormat.js';

export function BenchmarkRunMetrics({ metrics }: { metrics?: readonly BenchmarkMetric[] }) {
return (
Expand Down
14 changes: 2 additions & 12 deletions src/BenchmarkRunResult.tsx
Original file line number Diff line number Diff line change
Expand Up @@ -14,24 +14,14 @@ import {
Stack,
Typography,
} from '@mui/material';
import type { BenchmarkRun, BenchmarkRunStatus } from './benchmarkApi';
import type { BenchmarkRun } from './benchmarkApi';
import type { PollingFeedback } from './benchmarkRunPolling';
import { BenchmarkNodeAffinitySummary } from './BenchmarkNodeAffinitySummary.js';
import { BenchmarkRunMetrics } from './BenchmarkRunMetrics.js';
import { BenchmarkRunOptions } from './BenchmarkRunOptions.js';
import { STATUS_COLORS, STATUS_LABELS } from './benchmarkRunStatus.js';
import { describeRunOptions } from './drivers/index.js';

const STATUS_LABELS: Record<BenchmarkRunStatus, string> = {
running: 'Running',
succeeded: 'Succeeded',
failed: 'Failed',
};

const STATUS_COLORS: Record<BenchmarkRunStatus, 'info' | 'success' | 'error'> = {
running: 'info',
succeeded: 'success',
failed: 'error',
};

interface BenchmarkRunResultProps {
id: string;
Expand Down
32 changes: 28 additions & 4 deletions src/BenchmarkTab.tsx
Original file line number Diff line number Diff line change
Expand Up @@ -11,7 +11,8 @@ import type { BenchmarkFormValues } from './benchmarkFormValidation';
import { providerFromClusterDetailProps, targetFromClusterDetailProps } from './benchmarkTarget';
import type { BenchmarkTarget } from './benchmarkTarget';
import { getBenchmarkDriversForProvider } from './drivers';
import { getEnabledBenchmarkDrivers } from './benchmarkDriversApi';
import { getBenchmarkServiceStatus } from './benchmarkDriversApi';
import { BenchmarkRunHistory } from './BenchmarkRunHistory';

interface BenchmarkTabProps extends ClusterDetailTabProps {
pluginFetch: PluginApi['fetch'];
Expand Down Expand Up @@ -45,21 +46,30 @@ interface BenchmarkWorkflowProps {
function BenchmarkWorkflow({ pluginFetch, target, provider }: BenchmarkWorkflowProps) {
const supportedDrivers = getBenchmarkDriversForProvider(provider);
const [driverState, setDriverState] = useState<
{ status: 'loading' } | { status: 'ready'; ids: string[] } | { status: 'failed'; message: string }
| { status: 'loading' }
| { status: 'ready'; ids: string[]; persistentHistory: boolean }
| { status: 'failed'; message: string }
>({ status: 'loading' });
const [driverAttempt, setDriverAttempt] = useState(0);
const availableDrivers = driverState.status === 'ready'
? supportedDrivers.filter(id => driverState.ids.includes(id)) : [];
const [submission, setSubmission] = useState<SubmissionState>({ status: 'idle' });
const activeRequest = useRef<AbortController | null>(null);
const [runIds, setRunIds] = useState<string[]>([]);
const [historyVersion, setHistoryVersion] = useState(0);
const refreshHistory = () => setHistoryVersion(version => version + 1);
// Only runs started here refresh history when they finish; opening a past
// run from history must not trigger a reload.
const submittedRunIds = useRef(new Set<string>());

useEffect(() => {
if (getBenchmarkDriversForProvider(provider).length === 0) return;
const controller = new AbortController();
setDriverState({ status: 'loading' });
getEnabledBenchmarkDrivers(pluginFetch, { signal: controller.signal }).then(
ids => { if (!controller.signal.aborted) setDriverState({ status: 'ready', ids }); },
getBenchmarkServiceStatus(pluginFetch, { signal: controller.signal }).then(
({ drivers, persistentHistory }) => {
if (!controller.signal.aborted) setDriverState({ status: 'ready', ids: drivers, persistentHistory });
},
error => {
if (!controller.signal.aborted) setDriverState({ status: 'failed', message: error.message });
},
Expand Down Expand Up @@ -89,8 +99,10 @@ function BenchmarkWorkflow({ pluginFetch, target, provider }: BenchmarkWorkflowP
try {
const run = await createBenchmarkRun(pluginFetch, request, { signal: controller.signal });
if (activeRequest.current !== controller || controller.signal.aborted) return;
submittedRunIds.current.add(run.id);
addRun(run.id);
setSubmission({ status: 'accepted', id: run.id });
refreshHistory();
} catch (error) {
if (activeRequest.current !== controller || controller.signal.aborted) return;
setSubmission({
Expand Down Expand Up @@ -146,11 +158,23 @@ function BenchmarkWorkflow({ pluginFetch, target, provider }: BenchmarkWorkflowP
pluginFetch={pluginFetch}
id={id}
onDismiss={() => setRunIds(ids => ids.filter(runId => runId !== id))}
onSettled={submittedRunIds.current.has(id) ? refreshHistory : undefined}
/>
))}
</Box>
</Stack>
)}
{/* Like the driver check, history waits for a supported provider. */}
{supportedDrivers.length > 0 && (
<BenchmarkRunHistory
pluginFetch={pluginFetch}
target={target}
refreshKey={historyVersion}
persistentHistory={driverState.status === 'ready' ? driverState.persistentHistory : undefined}
shownRunIds={runIds}
onOpen={addRun}
/>
)}
<RunLookup onLookup={addRun} />
</Stack>
);
Expand Down
13 changes: 11 additions & 2 deletions src/TrackedBenchmarkRun.tsx
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
import { useEffect, useState } from 'react';
import { useEffect, useRef, useState } from 'react';
import type { PluginApi } from '@openeverest/plugin-sdk';
import { BenchmarkRunResult } from './BenchmarkRunResult';
import type { BenchmarkRun } from './benchmarkApi';
Expand All @@ -9,9 +9,11 @@ interface TrackedBenchmarkRunProps {
pluginFetch: PluginApi['fetch'];
id: string;
onDismiss: () => void;
/** Called once when the run is observed in a terminal status. */
onSettled?: () => void;
}

export function TrackedBenchmarkRun({ pluginFetch, id, onDismiss }: TrackedBenchmarkRunProps) {
export function TrackedBenchmarkRun({ pluginFetch, id, onDismiss, onSettled }: TrackedBenchmarkRunProps) {
const [run, setRun] = useState<BenchmarkRun | null>(null);
const [feedback, setFeedback] = useState<PollingFeedback | null>(null);
const [attempt, setAttempt] = useState(0);
Expand All @@ -25,6 +27,13 @@ export function TrackedBenchmarkRun({ pluginFetch, id, onDismiss }: TrackedBench
// Snapshots update this card without restarting its polling session.
}, [pluginFetch, id, attempt]);

const settled = run !== null && run.status !== 'running';
const onSettledRef = useRef(onSettled);
onSettledRef.current = onSettled;
useEffect(() => {
if (settled) onSettledRef.current?.();
}, [settled]);

return (
<BenchmarkRunResult
id={id}
Expand Down
11 changes: 7 additions & 4 deletions src/benchmarkApi.ts
Original file line number Diff line number Diff line change
Expand Up @@ -49,6 +49,8 @@ type BenchmarkRunCore = {
};

export type BenchmarkRun = BenchmarkRunCore & {
/** Engine reported by OpenEverest, e.g. "postgresql"; absent on older backends. */
databaseType?: string;
/** Optional for runs created before driver snapshots were available. */
driver?: string;
runnerImage?: string;
Expand Down Expand Up @@ -108,12 +110,13 @@ function isRecord(value: unknown): value is Record<string, unknown> {
return typeof value === 'object' && value !== null && !Array.isArray(value);
}

function isBenchmarkRunCore(value: unknown, id: string): value is BenchmarkRunCore & Record<string, unknown> {
// Without an expected ID (history entries), any nonblank ID is accepted.
function isBenchmarkRunCore(value: unknown, id?: string): value is BenchmarkRunCore & Record<string, unknown> {
if (!isRecord(value)) return false;
const run = value;
const timestamp = (value: unknown) => typeof value === 'string' &&
value.trim() !== '' && Number.isFinite(Date.parse(value));
return run.id === id &&
return (id === undefined ? typeof run.id === 'string' && run.id.trim() !== '' : run.id === id) &&
(run.status === 'running' || run.status === 'succeeded' || run.status === 'failed') &&
timestamp(run.createdAt) && typeof run.outputTruncated === 'boolean' &&
(run.nodeAffinity === undefined || isNodeAffinity(run.nodeAffinity)) &&
Expand All @@ -124,7 +127,7 @@ function isBenchmarkRunCore(value: unknown, id: string): value is BenchmarkRunCo
(run.status !== 'failed' || (typeof run.error === 'string' && run.error.trim() !== ''));
}

function parseBenchmarkRun(value: unknown, id: string): BenchmarkRun | undefined {
export function parseBenchmarkRun(value: unknown, id?: string): BenchmarkRun | undefined {
if (!isBenchmarkRunCore(value, id)) return undefined;
// Construct the known fields explicitly: optional metadata must be checked
// before reaching result components, even when the core run is valid.
Expand All @@ -139,7 +142,7 @@ function parseBenchmarkRun(value: unknown, id: string): BenchmarkRun | undefined
}
if (value.resources !== undefined) run.resources = value.resources;
if (value.nodeAffinity !== undefined) run.nodeAffinity = value.nodeAffinity;
for (const key of ['driver', 'runnerImage'] as const) {
for (const key of ['databaseType', 'driver', 'runnerImage'] as const) {
const field = value[key];
if (typeof field === 'string' && field.trim() !== '') run[key] = field;
}
Expand Down
Loading
Loading