From 6b95e12bfe80253267584c5d1345f085be9079de Mon Sep 17 00:00:00 2001 From: Qindeel Ishtiaq Date: Fri, 2 Oct 2026 19:30:56 -0400 Subject: [PATCH] console: add a memory by object card to the redesigned cluster page Fourth in the cluster details redesign stack. On the new page's Metrics tab, a memory by object card replaces the resource intensive objects table. It reads the same query: the largest objects on the cluster's largest replica, as shares of that replica's heap limit. One bar of the heap limit is split by object, and a legend gives each object's size and share. Objects are colored by rank with the Freshness page's palette. The design plots this over time, but per-object memory is only reported as a current value by default (mz_object_arrangement_size_history isn't collected unless arrangement_size_history_collection_interval is set), so the card shows the current snapshot. Each object links to its Maintained Objects drawer, or to its workflow graph while the maintained-objects-ui-50 flag is off. The largest replica's loading and error states move into LargestReplicaBoundary so the classic table and the card share them. The largest replica's heap limit now covers all of its processes: the per-process limit times the size's process count from mz_cluster_replica_sizes. Object sizes are replica-wide, so on a multi-process replica every share was inflated by the process count, in the classic table too. --- .../cluster/largestClusterReplica.test.sql.ts | 29 ++ .../cluster/largestClusterReplica.ts | 86 ++-- .../cluster/largestMaintainedQueries.ts | 8 +- .../ClusterPage/ClusterMetrics.test.tsx | 4 +- .../clusters/ClusterPage/ClusterMetrics.tsx | 15 +- .../ClusterPage/MemoryByObject.test.tsx | 387 ++++++++++++++++++ .../clusters/ClusterPage/MemoryByObject.tsx | 287 +++++++++++++ .../ResourceUsage/ResourceUsage.test.tsx | 4 +- .../ClusterPage/memoryByObjectModel.ts | 26 ++ .../clusters/LargestMaintainedQueries.tsx | 98 ++++- console/src/platform/clusters/queries.ts | 10 +- 11 files changed, 876 insertions(+), 78 deletions(-) create mode 100644 console/src/platform/clusters/ClusterPage/MemoryByObject.test.tsx create mode 100644 console/src/platform/clusters/ClusterPage/MemoryByObject.tsx create mode 100644 console/src/platform/clusters/ClusterPage/memoryByObjectModel.ts diff --git a/console/src/api/materialize/cluster/largestClusterReplica.test.sql.ts b/console/src/api/materialize/cluster/largestClusterReplica.test.sql.ts index c639d1c76b410..ddf182a0b88fe 100644 --- a/console/src/api/materialize/cluster/largestClusterReplica.test.sql.ts +++ b/console/src/api/materialize/cluster/largestClusterReplica.test.sql.ts @@ -40,6 +40,10 @@ describe("buildLargestClusterReplicaQuery", () => { replica_id TEXT NOT NULL, time_ns bigint ); + > CREATE TABLE mz_cluster_replica_sizes ( + size TEXT NOT NULL, + processes uint8 NOT NULL + ); `); const { @@ -85,6 +89,31 @@ describe("buildLargestClusterReplicaQuery", () => { expect(result.size).toBe("scale=1,workers=1,mem=8GiB"); expect(result.heapLimit).toBe("8589934592"); + const { + rows: [named], + } = await executeSqlHttp( + buildLargestClusterReplicaQuery(cluster.id, "r1").compile(), + { + sessionVariables: { + search_path: mockedSearchPath, + cluster: QUICKSTART_CLUSTER, + }, + }, + ); + expect(named.name).toBe("r1"); + expect(named.heapLimit).toBe("4294967296"); + + // Object sizes cover every process of a replica, so its heap limit does + // too. Four 4GiB processes outrank one 8GiB process. + await client.query(` + INSERT INTO internal_test.mz_cluster_replica_sizes VALUES + ('scale=1,workers=1,mem=4GiB', 4) + `); + const multiProcess = await fetchLargest(); + expect([r1.name, r3.name]).toContain(multiProcess.name); + expect(multiProcess.heapLimit).toBe("17179869184"); + await client.query(`DELETE FROM internal_test.mz_cluster_replica_sizes`); + // A hydrated replica is preferred over a larger, still-hydrating one // (its object is present but has no hydration time yet). await client.query(` diff --git a/console/src/api/materialize/cluster/largestClusterReplica.ts b/console/src/api/materialize/cluster/largestClusterReplica.ts index 36e863b75b96b..17e398fd9cd22 100644 --- a/console/src/api/materialize/cluster/largestClusterReplica.ts +++ b/console/src/api/materialize/cluster/largestClusterReplica.ts @@ -13,44 +13,61 @@ import { sql } from "kysely"; import { executeSqlV2, queryBuilder } from "~/api/materialize"; import { buildClusterReplicaHeapMetricsTable } from "~/api/materialize/expressionBuilders"; -export function buildLargestClusterReplicaQuery(clusterId: string) { - return ( - queryBuilder - .selectFrom("mz_cluster_replicas as cr") - .leftJoin( - buildClusterReplicaHeapMetricsTable().as("crhm"), - "crhm.replica_id", - "cr.id", - ) - // Prefer a fully-hydrated replica, whose reported sizes are stable. - // Joining mz_compute_hydration_times before aggregating lets its - // replica_id index bound the scan to this cluster's replicas, unlike an - // aggregation over the whole environment's mz_hydration_statuses. - .leftJoin("mz_compute_hydration_times as ht", (join) => - join - .onRef("ht.replica_id", "=", "cr.id") - // System introspection indexes may never report hydrated. - .on("ht.object_id", "not like", "s%"), - ) - .select([ - "cr.name", - "cr.size", - sql`crhm.heap_limit::text`.as("heapLimit"), - ]) - .where("cr.cluster_id", "=", clusterId) - .groupBy(["cr.id", "cr.name", "cr.size", "crhm.heap_limit"]) - // Prefer a replica whose compute objects are all hydrated, so its - // reported sizes are stable. time_ns is NULL until an object hydrates. A - // replica with no hydration rows, such as one just added, has bool_and - // over an all-NULL row and so sorts last. Then order by heap limit. - .orderBy(sql`bool_and(ht.time_ns IS NOT NULL) DESC`) - .orderBy(sql`crhm.heap_limit DESC NULLS LAST`) - .limit(1) - ); +// `heap_limit` is per process, but the object sizes divided by it cover every +// process of the replica. +const replicaHeapLimit = sql`(crhm.heap_limit * COALESCE(crs.processes, 1))`; + +export function buildLargestClusterReplicaQuery( + clusterId: string, + replicaName?: string, +) { + const query = queryBuilder + .selectFrom("mz_cluster_replicas as cr") + .leftJoin( + buildClusterReplicaHeapMetricsTable().as("crhm"), + "crhm.replica_id", + "cr.id", + ) + // Prefer a fully-hydrated replica, whose reported sizes are stable. + // Joining mz_compute_hydration_times before aggregating lets its + // replica_id index bound the scan to this cluster's replicas, unlike an + // aggregation over the whole environment's mz_hydration_statuses. + .leftJoin("mz_compute_hydration_times as ht", (join) => + join + .onRef("ht.replica_id", "=", "cr.id") + // System introspection indexes may never report hydrated. + .on("ht.object_id", "not like", "s%"), + ) + .leftJoin("mz_cluster_replica_sizes as crs", "crs.size", "cr.size") + .select([ + "cr.name", + "cr.size", + sql`${replicaHeapLimit}::text`.as("heapLimit"), + ]) + .where("cr.cluster_id", "=", clusterId) + .groupBy([ + "cr.id", + "cr.name", + "cr.size", + "crhm.heap_limit", + "crs.processes", + ]) + // Prefer a replica whose compute objects are all hydrated, so its + // reported sizes are stable. time_ns is NULL until an object hydrates. A + // replica with no hydration rows, such as one just added, has bool_and + // over an all-NULL row and so sorts last. Then order by heap limit. + .orderBy(sql`bool_and(ht.time_ns IS NOT NULL) DESC`) + .orderBy(sql`${replicaHeapLimit} DESC NULLS LAST`) + .limit(1); + return replicaName === undefined + ? query + : query.where("cr.name", "=", replicaName); } export type LargestClusterReplicaParams = { clusterId: string; + /** Reads this replica, with the same heap limit, instead of the largest. */ + replicaName?: string; }; /** @@ -67,6 +84,7 @@ export async function fetchLargestClusterReplica({ }) { const compiledQuery = buildLargestClusterReplicaQuery( params.clusterId, + params.replicaName, ).compile(); return executeSqlV2({ queries: compiledQuery, diff --git a/console/src/api/materialize/cluster/largestMaintainedQueries.ts b/console/src/api/materialize/cluster/largestMaintainedQueries.ts index 16a646bd7a633..81d2ae3cd0db8 100644 --- a/console/src/api/materialize/cluster/largestMaintainedQueries.ts +++ b/console/src/api/materialize/cluster/largestMaintainedQueries.ts @@ -17,7 +17,7 @@ import { } from "~/api/materialize"; export type LargestMaintainedQueriesParams = { - replicaHeapLimit: number; + replicaHeapLimit: number | null; limit: number; clusterName: string; replicaName: string; @@ -87,7 +87,9 @@ export function buildLargestMaintainedQueriesQuery({ }) // Filter out transient dataflows .where("ce.export_id", "not like", "t%") - .orderBy("memoryPercentage", sql`desc NULLS LAST`) + // Order by size: without a heap limit, memoryPercentage is NULL on every + // row. + .orderBy("size", sql`desc NULLS LAST`) .limit(() => sql.raw(limit.toString())) ); } @@ -119,7 +121,7 @@ export async function fetchLargestMaintainedQueries({ export type LargestMaintainedObjectSizesParams = { clusterId: string; replicaName: string; - replicaHeapLimit: number; + replicaHeapLimit: number | null; limit: number; }; diff --git a/console/src/platform/clusters/ClusterPage/ClusterMetrics.test.tsx b/console/src/platform/clusters/ClusterPage/ClusterMetrics.test.tsx index 0ad4ee8c4d1dc..6e9917a0b5f68 100644 --- a/console/src/platform/clusters/ClusterPage/ClusterMetrics.test.tsx +++ b/console/src/platform/clusters/ClusterPage/ClusterMetrics.test.tsx @@ -27,9 +27,7 @@ vi.mock("~/hooks/useFlags", () => ({ })); // The page's cards have their own tests and queries. vi.mock("./ResourceUsage/ResourceUsage", () => ({ ResourceUsage: () => null })); -vi.mock("~/platform/clusters/LargestMaintainedQueries", () => ({ - default: () => null, -})); +vi.mock("./MemoryByObject", () => ({ MemoryByObject: () => null })); const renderClusterMetrics = () => renderComponent( diff --git a/console/src/platform/clusters/ClusterPage/ClusterMetrics.tsx b/console/src/platform/clusters/ClusterPage/ClusterMetrics.tsx index 8cf59a642998c..cc4d8204da1a2 100644 --- a/console/src/platform/clusters/ClusterPage/ClusterMetrics.tsx +++ b/console/src/platform/clusters/ClusterPage/ClusterMetrics.tsx @@ -7,7 +7,7 @@ // the Business Source License, use of this software will be governed // by the Apache License, Version 2.0. -import { Box, HStack, Text, VStack } from "@chakra-ui/react"; +import { HStack, Text, VStack } from "@chakra-ui/react"; import { sort } from "d3"; import React from "react"; import { useParams } from "react-router-dom"; @@ -22,10 +22,10 @@ import { MainContentContainer } from "~/layouts/BaseLayout"; import { ClusterInfoBox } from "~/platform/clusters/ClusterOverview"; import { ClusterParams } from "~/platform/clusters/ClusterRoutes"; import { CLUSTERS_FETCH_ERROR_MESSAGE } from "~/platform/clusters/constants"; -import LargestMaintainedQueries from "~/platform/clusters/LargestMaintainedQueries"; import { useAllClusters } from "~/store/allClusters"; import { assert } from "~/util"; +import { MemoryByObject } from "./MemoryByObject"; import { ResourceUsage } from "./ResourceUsage/ResourceUsage"; import { useSelectedReplica } from "./useSelectedReplica"; @@ -78,12 +78,11 @@ const ClusterMetricsContent = ({ cluster }: { cluster: Cluster }) => { timePeriodMinutes={timePeriodMinutes} selectedReplicaId={selectedReplica?.id} /> - - - + ); diff --git a/console/src/platform/clusters/ClusterPage/MemoryByObject.test.tsx b/console/src/platform/clusters/ClusterPage/MemoryByObject.test.tsx new file mode 100644 index 0000000000000..91fa33d36a4e1 --- /dev/null +++ b/console/src/platform/clusters/ClusterPage/MemoryByObject.test.tsx @@ -0,0 +1,387 @@ +// Copyright Materialize, Inc. and contributors. All rights reserved. +// +// Use of this software is governed by the Business Source License +// included in the LICENSE file. +// +// As of the Change Date specified in that file, in accordance with +// the Business Source License, use of this software will be governed +// by the Apache License, Version 2.0. + +import { screen, waitFor, within } from "@testing-library/react"; +import React from "react"; + +import { ErrorCode, MzDataType } from "~/api/materialize/types"; +import { + buildColumns, + buildSqlQueryHandlerV2, + mapKyselyToTabular, + SqlHandlerOptions, +} from "~/api/mocks/buildSqlQueryHandler"; +import server from "~/api/mocks/server"; +import { getStore } from "~/jotai"; +import { clusterQueryKeys } from "~/platform/clusters/queries"; +import { + defaultRegionId, + healthyEnvironment, + renderComponent, + setFakeEnvironment, +} from "~/test/utils"; +import { parseDbVersion } from "~/version/api"; + +import { MemoryByObject } from "./MemoryByObject"; +import { segmentWidths } from "./memoryByObjectModel"; + +const flags = vi.hoisted(() => ({ values: {} as Record })); + +vi.mock("~/hooks/useFlags", () => ({ useFlags: () => flags.values })); + +const HEAP_LIMIT = 4069523456; + +const largestReplicaHandler = ( + rows: Array>, + options?: SqlHandlerOptions, + replicaName?: string, +) => + buildSqlQueryHandlerV2( + { + queryKey: clusterQueryKeys.largestClusterReplica({ + clusterId: "u1", + replicaName, + }), + results: mapKyselyToTabular({ + columns: buildColumns([ + "name", + "size", + { type_oid: MzDataType.numeric, name: "heapLimit" }, + ]), + rows, + }), + }, + options, + ); + +const largestObjectsKey = (replicaHeapLimit: number | null = HEAP_LIMIT) => + clusterQueryKeys.largestMaintainedQueries({ + clusterId: "u1", + clusterName: "quickstart", + limit: 10, + replicaName: "r1", + replicaHeapLimit, + unifiedSizes: false, + }); + +const largestObjectsHandler = ( + rows: Array> = [ + { + id: "u188", + name: "customer_view", + size: "5469140917", + memoryPercentage: "31.8345902256", + type: "materialized-view", + schemaName: "public", + databaseName: "materialize", + dataflowId: "7", + dataflowName: "Dataflow: materialize.public.customer_view", + }, + // The dataflow of a dropped object: the left join to mz_objects finds no + // row, so even its id is NULL. + { + id: null, + name: null, + size: "424919434", + memoryPercentage: "11.2686157226", + type: null, + schemaName: null, + databaseName: null, + dataflowId: "124", + dataflowName: "Dataflow: materialize.deleted_schema.orphaned_view", + }, + ], + replicaHeapLimit: number | null = HEAP_LIMIT, +) => + buildSqlQueryHandlerV2({ + queryKey: largestObjectsKey(replicaHeapLimit), + results: mapKyselyToTabular({ + columns: buildColumns([ + "id", + "name", + "size", + { type_oid: MzDataType.numeric, name: "memoryPercentage" }, + "type", + "schemaName", + "databaseName", + "dataflowId", + "dataflowName", + ]), + rows, + }), + }); + +const renderMemoryByObject = () => + renderComponent(); + +describe("MemoryByObject", () => { + beforeEach(() => { + flags.values = {}; + }); + + it("lists each object's memory on the largest replica", async () => { + flags.values = { "maintained-objects-ui-50": true }; + server.use( + largestReplicaHandler([ + { name: "r1", size: "25cc", heapLimit: String(HEAP_LIMIT) }, + ]), + largestObjectsHandler(), + ); + await renderMemoryByObject(); + + // The title shows while loading too, so wait on the subtitle. + expect(await screen.findByText("% of heap limit on r1")).toBeVisible(); + expect(screen.getByText("Objects by Memory")).toBeVisible(); + const objects = screen.getByRole("list", { name: "Objects" }); + expect( + within(objects).getByRole("link", { name: "customer_view" }), + ).toHaveAttribute( + "href", + expect.stringMatching(/\/maintained-objects\/u188$/), + ); + expect(within(objects).getByText("5.09 GB (31.8%)")).toBeVisible(); + }); + + it("shows the named replica instead of the largest", async () => { + server.use( + largestReplicaHandler( + [{ name: "r1", size: "25cc", heapLimit: String(HEAP_LIMIT) }], + undefined, + "r1", + ), + largestObjectsHandler(), + ); + await renderComponent( + , + ); + + expect(await screen.findByText("% of heap limit on r1")).toBeVisible(); + expect(screen.getByText("5.09 GB (31.8%)")).toBeVisible(); + }); + + it("links to the workflow graph while the maintained objects page is off", async () => { + server.use( + largestReplicaHandler([ + { name: "r1", size: "25cc", heapLimit: String(HEAP_LIMIT) }, + ]), + largestObjectsHandler(), + ); + await renderMemoryByObject(); + + const objects = await screen.findByRole("list", { name: "Objects" }); + expect( + within(objects).getByRole("link", { name: "customer_view" }), + ).toHaveAttribute("href", expect.stringMatching(/\/workflow$/)); + }); + + it("names a dropped object's dataflow without linking it", async () => { + server.use( + largestReplicaHandler([ + { name: "r1", size: "25cc", heapLimit: String(HEAP_LIMIT) }, + ]), + largestObjectsHandler(), + ); + await renderMemoryByObject(); + + const objects = await screen.findByRole("list", { name: "Objects" }); + expect(within(objects).getByText("orphaned_view")).toBeVisible(); + expect( + within(objects).queryByRole("link", { name: "orphaned_view" }), + ).not.toBeInTheDocument(); + }); + + it("lists sizes without a bar when the replica reports no heap limit", async () => { + server.use( + largestReplicaHandler([{ name: "r1", size: "25cc", heapLimit: null }]), + largestObjectsHandler( + [ + { + id: "u188", + name: "customer_view", + size: "5469140917", + memoryPercentage: null, + type: "materialized-view", + schemaName: "public", + databaseName: "materialize", + dataflowId: "7", + dataflowName: "Dataflow: materialize.public.customer_view", + }, + ], + null, + ), + ); + await renderMemoryByObject(); + + const objects = await screen.findByRole("list", { name: "Objects" }); + expect(within(objects).getByText("5.09 GB")).toBeVisible(); + expect(screen.getByText("Size on r1")).toBeVisible(); + expect( + screen.queryByRole("img", { name: "Memory by object" }), + ).not.toBeInTheDocument(); + }); + + it("shows the card's title while loading", async () => { + server.use(largestReplicaHandler([], { waitTimeMs: "infinite" })); + await renderMemoryByObject(); + + expect( + screen.getByRole("heading", { name: "Objects by Memory" }), + ).toBeVisible(); + expect(screen.getByTestId("loading-spinner")).toBeVisible(); + }); + + it("says when the replica has no objects", async () => { + server.use( + largestReplicaHandler([ + { name: "r1", size: "25cc", heapLimit: String(HEAP_LIMIT) }, + ]), + largestObjectsHandler([]), + ); + await renderMemoryByObject(); + + expect( + await screen.findByText("No objects are using memory on r1."), + ).toBeVisible(); + expect( + screen.getByRole("heading", { name: "Objects by Memory" }), + ).toBeVisible(); + }); + + it("shows nothing for a cluster without replicas", async () => { + server.use(largestReplicaHandler([])); + await renderMemoryByObject(); + + await waitFor(() => + expect(screen.queryByTestId("loading-spinner")).not.toBeInTheDocument(), + ); + expect(document.body.textContent).toEqual(""); + }); + + it("says the replica may be busy when the objects fail to load", async () => { + server.use( + largestReplicaHandler([ + { name: "r1", size: "25cc", heapLimit: String(HEAP_LIMIT) }, + ]), + buildSqlQueryHandlerV2({ + queryKey: largestObjectsKey(), + results: { + error: { + code: ErrorCode.INTERNAL_ERROR, + message: "largestMaintainedQueries failed", + }, + notices: [], + }, + }), + ); + await renderMemoryByObject(); + + expect( + await screen.findByText(/from replica r1, which might mean it's busy/), + ).toBeVisible(); + expect( + screen.getByRole("heading", { name: "Objects by Memory" }), + ).toBeVisible(); + }); + + it("reads sizes from mz_object_arrangement_sizes on Materialize >= v26.35", async () => { + const environment = { + ...healthyEnvironment, + status: { + ...healthyEnvironment.status, + version: parseDbVersion("v26.35.0 (ea0d129f)"), + }, + }; + // Query keys embed the environment version, so set it before building + // the handlers that match on them. + await setFakeEnvironment(getStore().set, defaultRegionId, environment); + server.use( + largestReplicaHandler([ + { name: "r1", size: "25cc", heapLimit: String(HEAP_LIMIT) }, + ]), + buildSqlQueryHandlerV2({ + queryKey: clusterQueryKeys.largestMaintainedQueries({ + clusterId: "u1", + clusterName: "quickstart", + limit: 10, + replicaName: "r1", + replicaHeapLimit: HEAP_LIMIT, + unifiedSizes: true, + }), + results: mapKyselyToTabular({ + columns: buildColumns([ + "object_id", + "size", + { type_oid: MzDataType.numeric, name: "memoryPercentage" }, + ]), + rows: [ + { + object_id: "u188", + size: "5469140917", + memoryPercentage: "31.8345902256", + }, + // Missing from the names below: a dropped object's dataflow. + { + object_id: "u999", + size: "424919434", + memoryPercentage: "11.2686157226", + }, + ], + }), + }), + buildSqlQueryHandlerV2({ + queryKey: clusterQueryKeys.maintainedObjectNames(["u188", "u999"]), + results: mapKyselyToTabular({ + columns: buildColumns([ + "id", + "name", + "type", + "schemaName", + "databaseName", + ]), + rows: [ + { + id: "u188", + name: "customer_view", + type: "materialized-view", + schemaName: "public", + databaseName: "materialize", + }, + ], + }), + }), + ); + await renderComponent( + , + { + initializeState: ({ set }) => + setFakeEnvironment(set, defaultRegionId, environment), + }, + ); + + const objects = await screen.findByRole("list", { name: "Objects" }); + expect( + within(objects).getByRole("link", { name: "customer_view" }), + ).toBeVisible(); + expect(within(objects).getByText("5.09 GB (31.8%)")).toBeVisible(); + expect(within(objects).getByText("u999")).toBeVisible(); + expect( + within(objects).queryByRole("link", { name: "u999" }), + ).not.toBeInTheDocument(); + }); +}); + +describe("segmentWidths", () => { + it("stops the bar at the heap limit", () => { + expect(segmentWidths([60, 30, 25, null])).toEqual([60, 30, 10, 0]); + }); +}); diff --git a/console/src/platform/clusters/ClusterPage/MemoryByObject.tsx b/console/src/platform/clusters/ClusterPage/MemoryByObject.tsx new file mode 100644 index 0000000000000..dcfdded570930 --- /dev/null +++ b/console/src/platform/clusters/ClusterPage/MemoryByObject.tsx @@ -0,0 +1,287 @@ +// Copyright Materialize, Inc. and contributors. All rights reserved. +// +// Use of this software is governed by the Business Source License +// included in the LICENSE file. +// +// As of the Change Date specified in that file, in accordance with +// the Business Source License, use of this software will be governed +// by the Apache License, Version 2.0. + +import { + Box, + Card, + Flex, + HStack, + Text, + Tooltip, + useTheme, + VStack, +} from "@chakra-ui/react"; +import React from "react"; +import { Link as RouterLink } from "react-router-dom"; + +import { TooltipColorSwatch } from "~/components/graphComponents"; +import TextLink from "~/components/TextLink"; +import { generateRainbowPalette } from "~/components/ThresholdLineGraph/thresholdLineGraphHelpers"; +import { useFlags } from "~/hooks/useFlags"; +import { LargestReplicaBoundary } from "~/platform/clusters/LargestMaintainedQueries"; +import { useLargestMaintainedQueries } from "~/platform/clusters/queries"; +import { + absoluteMaintainedObjectPath, + useBuildWorkflowGraphPath, +} from "~/platform/routeHelpers"; +import { useRegionSlug } from "~/store/environments"; +import { MaterializeTheme } from "~/theme"; +import { formatMemoryUsage } from "~/utils/format"; + +import { segmentWidths, TOP_OBJECT_COUNT } from "./memoryByObjectModel"; +import { PERCENT_TICKS } from "./ResourceUsage/resourceUsageStyles"; + +export interface MemoryByObjectProps { + clusterId: string; + clusterName: string; +} + +type LargestObject = NonNullable< + ReturnType["data"] +>[number]; + +/** + * The largest objects on a replica, as shares of its heap limit: the named + * replica, else the cluster's largest. + */ +export const MemoryByObject = ({ + clusterId, + clusterName, + replicaName, +}: MemoryByObjectProps & { replicaName?: string }) => ( + ( + {fallback} + )} + > + {(replica) => ( + + )} + +); + +const PolledMemoryByObject = ({ + clusterId, + clusterName, + replicaName, + replicaHeapLimit, +}: MemoryByObjectProps & { + replicaName: string; + replicaHeapLimit: number | null; +}) => { + const { data: objects } = useLargestMaintainedQueries({ + clusterId, + clusterName, + limit: TOP_OBJECT_COUNT, + replicaName, + replicaHeapLimit, + }); + return ( + + ); +}; + +/** The card's shell, which also holds its loading, error and empty states. */ +const MemoryByObjectFrame = ({ + title = "Objects by Memory", + subtitle, + children, +}: { + title?: string; + subtitle?: string; + children: React.ReactNode; +}) => { + const { colors } = useTheme(); + return ( + + + + + {title} + + {subtitle && ( + + {subtitle} + + )} + + {children} + + + ); +}; + +const MemoryByObjectCard = ({ + objects, + replicaName, + hasHeapLimit, +}: { + objects: LargestObject[]; + replicaName: string; + /** Without a heap limit there are no shares, so the card lists sizes alone. */ + hasHeapLimit: boolean; +}) => { + const { colors } = useTheme(); + // The maintained objects page only has routes while its flag is on. + const hasMaintainedObjectsPage = Boolean( + useFlags()["maintained-objects-ui-50"], + ); + const regionSlug = useRegionSlug(); + const workflowGraphPath = useBuildWorkflowGraphPath(); + + if (objects.length === 0) { + return ( + + + No objects are using memory on {replicaName}. + + + ); + } + + // Colored by rank like the Freshness page's lines, so the largest objects + // are furthest apart in hue. + const palette = generateRainbowPalette(objects.length); + const widths = segmentWidths( + objects.map((object) => object.memoryPercentage), + ); + const segments = objects.map((object, index) => { + const { id, name, databaseName, schemaName } = object; + return { + key: id ?? object.dataflowId ?? String(index), + label: name ?? object.dataflowName ?? "-", + usage: formatMemoryUsage(hasHeapLimit ? object : { size: object.size }), + color: palette[index], + widthPercent: widths[index], + path: + object.isOrphanedDataflow || !id || !name || !schemaName + ? undefined + : hasMaintainedObjectsPage + ? absoluteMaintainedObjectPath(regionSlug, id) + : workflowGraphPath({ + databaseObject: { id, name, databaseName, schemaName }, + type: object.type, + }), + }; + }); + + return ( + + {hasHeapLimit && ( + + + {segments.map( + (segment) => + segment.widthPercent > 0 && ( + + + + ), + )} + + + {PERCENT_TICKS.map((tick) => ( + + {tick}% + + ))} + + + )} + + {segments.map((segment) => ( + + + {segment.path ? ( + + {segment.label} + + ) : ( + {segment.label} + )} + + {segment.usage} + + + ))} + + + ); +}; diff --git a/console/src/platform/clusters/ClusterPage/ResourceUsage/ResourceUsage.test.tsx b/console/src/platform/clusters/ClusterPage/ResourceUsage/ResourceUsage.test.tsx index 632e33df0ef61..884e5c23c9e2d 100644 --- a/console/src/platform/clusters/ClusterPage/ResourceUsage/ResourceUsage.test.tsx +++ b/console/src/platform/clusters/ClusterPage/ResourceUsage/ResourceUsage.test.tsx @@ -54,9 +54,7 @@ import { ResourceUsage } from "./ResourceUsage"; import { CHART_MARGIN } from "./resourceUsageStyles"; // The Metrics page's other card has its own tests and queries. -vi.mock("~/platform/clusters/LargestMaintainedQueries", () => ({ - default: () => null, -})); +vi.mock("../MemoryByObject", () => ({ MemoryByObject: () => null })); // jsdom lays nothing out, so give the charts a real width. const WIDTH_PX = 900; diff --git a/console/src/platform/clusters/ClusterPage/memoryByObjectModel.ts b/console/src/platform/clusters/ClusterPage/memoryByObjectModel.ts new file mode 100644 index 0000000000000..7e178aa276e00 --- /dev/null +++ b/console/src/platform/clusters/ClusterPage/memoryByObjectModel.ts @@ -0,0 +1,26 @@ +// Copyright Materialize, Inc. and contributors. All rights reserved. +// +// Use of this software is governed by the Business Source License +// included in the LICENSE file. +// +// As of the Change Date specified in that file, in accordance with +// the Business Source License, use of this software will be governed +// by the Apache License, Version 2.0. + +import { clamp } from "~/util"; + +/** How many of the largest objects the card shows. */ +export const TOP_OBJECT_COUNT = 10; + +/** + * Bar segment widths for objects' shares of the heap limit, in order. The bar + * stops at the heap limit even if the shares add up to more. + */ +export const segmentWidths = (percentages: Array) => { + let remaining = 100; + return percentages.map((percentage) => { + const width = clamp(percentage ?? 0, 0, remaining); + remaining -= width; + return width; + }); +}; diff --git a/console/src/platform/clusters/LargestMaintainedQueries.tsx b/console/src/platform/clusters/LargestMaintainedQueries.tsx index b8989f9fb2b8e..993e67ad1486f 100644 --- a/console/src/platform/clusters/LargestMaintainedQueries.tsx +++ b/console/src/platform/clusters/LargestMaintainedQueries.tsx @@ -118,23 +118,63 @@ const LargestMaintainedQueriesError = ({ ); }; -const LargestMaintainedQueries = (props: LargestMaintainedQueriesProps) => { +export interface LargestReplica { + name: string; + /** Null while the replica reports none, as always under the process orchestrator. */ + heapLimit: number | null; +} + +export interface LargestReplicaBoundaryProps { + clusterId: string; + /** The replica to load instead of the cluster's largest. */ + replicaName?: string; + children: (replica: LargestReplica) => React.ReactNode; + /** Wraps the loading and error states, so a caller can draw them in its card. */ + wrapFallback?: (fallback: React.ReactNode) => React.ReactNode; +} + +const identity = (fallback: React.ReactNode) => fallback; + +/** + * Loads the cluster's largest replica and renders `children` with it, behind + * the loading and error states of the per-object memory insights. Renders + * nothing for a cluster without replicas. + */ +export const LargestReplicaBoundary = ({ + clusterId, + replicaName, + children, + wrapFallback = identity, +}: LargestReplicaBoundaryProps) => { return ( ( - - )} + renderFallback={({ error }) => + wrapFallback() + } > - }> - + )}> + + {children} + ); }; -const LargestReplicaLoader = (props: LargestMaintainedQueriesProps) => { +const LargestReplicaLoader = ({ + clusterId, + replicaName, + children, + wrapFallback, +}: Omit, "replicaName"> & + Pick) => { const { data: largestReplica } = useLargestClusterReplica({ - clusterId: props.clusterId, + clusterId, + replicaName, }); // Don't show anything if the cluster has no replicas @@ -142,22 +182,40 @@ const LargestReplicaLoader = (props: LargestMaintainedQueriesProps) => { return ( ( - - )} + renderFallback={({ error }) => + wrapFallback( + , + ) + } > - + {children({ + name: largestReplica.name, + heapLimit: + largestReplica.heapLimit === null + ? null + : Number(largestReplica.heapLimit), + })} ); }; +const LargestMaintainedQueries = (props: LargestMaintainedQueriesProps) => { + return ( + + {(replica) => ( + + )} + + ); +}; + const LargestMaintainedQueriesInner = ({ clusterId, clusterName, @@ -165,7 +223,7 @@ const LargestMaintainedQueriesInner = ({ replicaHeapLimit, }: LargestMaintainedQueriesProps & { replicaName: string; - replicaHeapLimit: number; + replicaHeapLimit: number | null; }) => { const { colors } = useTheme(); const navigate = useNavigate(); diff --git a/console/src/platform/clusters/queries.ts b/console/src/platform/clusters/queries.ts index 4166858773171..8a42dfab946a4 100644 --- a/console/src/platform/clusters/queries.ts +++ b/console/src/platform/clusters/queries.ts @@ -498,7 +498,8 @@ export type UseLargestMaintainedQueriesParams = { clusterName: string; limit?: number; replicaName: string | undefined; - replicaHeapLimit: number | undefined; + /** Null when the replica reports none: sizes then carry no percentage. */ + replicaHeapLimit: number | null; }; export function useLargestMaintainedQueries( params: UseLargestMaintainedQueriesParams, @@ -517,12 +518,7 @@ export function useLargestMaintainedQueries( }), queryFn: async ({ queryKey, signal }) => { const [, paramsFromKey] = queryKey; - if ( - paramsFromKey.replicaHeapLimit === null || - paramsFromKey.replicaHeapLimit === undefined || - !paramsFromKey.replicaName - ) - return null; + if (!paramsFromKey.replicaName) return null; if (paramsFromKey.unifiedSizes) { const sizes = await fetchLargestMaintainedObjectSizes({