diff --git a/torchci/components/benchmark_v3/components/common/BenchmarkShortcutCardList.tsx b/torchci/components/benchmark_v3/components/common/BenchmarkShortcutCardList.tsx new file mode 100644 index 0000000000..f8d6b1779d --- /dev/null +++ b/torchci/components/benchmark_v3/components/common/BenchmarkShortcutCardList.tsx @@ -0,0 +1,156 @@ +import ChevronLeftIcon from "@mui/icons-material/ChevronLeft"; +import ChevronRightIcon from "@mui/icons-material/ChevronRight"; +import { + Box, + Card, + CardActionArea, + CardContent, + IconButton, + Typography, +} from "@mui/material"; +import React, { useMemo, useRef } from "react"; +// import { useRouter } from "next/router"; + +export type BenchmarkShortcutItem = { + displayName: string; + fieldName: string; + value: string; + description?: string; + url?: string; +}; + +type BenchmarkShortcutCardListProps = { + benchmarkId: string; + // whatever your query params shape is + data: any; + title?: string; + /** + * Optional custom navigation (e.g. Next.js router.push). + * If not provided, falls back to window.location.href. + */ + onNavigate?: (item: BenchmarkShortcutItem) => void; +}; + +export const BenchmarkShortcutCardList: React.FC< + BenchmarkShortcutCardListProps +> = ({ benchmarkId, data, title = "Shortcuts", onNavigate }) => { + const scrollContainerRef = useRef(null); + // const router = useRouter(); + + // ---- map API response -> cardList ---- + const cardList: BenchmarkShortcutItem[] = useMemo(() => { + if (!data) return []; + return data; + }, [data]); + + // ---- scrolling helpers ---- + const handleScroll = (direction: "left" | "right") => { + const node = scrollContainerRef.current; + if (!node) return; + const delta = direction === "left" ? -320 : 320; + node.scrollBy({ left: delta, behavior: "smooth" }); + }; + + // ---- navigation ---- + const handleClickCard = (item: BenchmarkShortcutItem) => { + if (onNavigate) { + onNavigate(item); + return; + } + // Fallback: normal navigation + if (typeof window !== "undefined" && item.url) { + window.location.href = item.url; + } + }; + + if (!cardList.length) { + return null; + } + + // ---- main UI ---- + return ( + + + + {title} + + + handleScroll("left")} + > + + + handleScroll("right")} + > + + + + + + + {cardList.map((item) => ( + + handleClickCard(item)}> + + + {item.displayName} + + {item.description && ( + + {item.description} + + )} + + + + ))} + + + ); +}; diff --git a/torchci/components/benchmark_v3/components/dataRender/auto/autoComponents.tsx b/torchci/components/benchmark_v3/components/dataRender/auto/autoComponents.tsx index a5e84b7ece..8dfe914274 100644 --- a/torchci/components/benchmark_v3/components/dataRender/auto/autoComponents.tsx +++ b/torchci/components/benchmark_v3/components/dataRender/auto/autoComponents.tsx @@ -1,10 +1,11 @@ import { Alert, Typography } from "@mui/material"; -import { Grid, Stack } from "@mui/system"; +import { Box, Grid, Stack } from "@mui/system"; import { AutoComponentProps } from "components/benchmark_v3/configs/utils/autoRegistration"; import LoadingPage from "components/common/LoadingPage"; import { useBenchmarkCommittedContext, useBenchmarkTimeSeriesData, + useListBenchmarkMetadata, } from "lib/benchmark/api_helper/fe/hooks"; import { useDashboardSelector } from "lib/benchmark/store/benchmark_dashboard_provider"; import BenchmarkRawDataTable from "../components/benchmarkTimeSeries/components/BenchmarkRawDataTable"; @@ -14,6 +15,10 @@ import { UIRenderConfig } from "components/benchmark_v3/configs/config_book_type import { useRouter } from "next/router"; import { useEffect, useState } from "react"; import { BenchmarkLogSidePanelWrapper } from "../../common/BenchmarkLogViewer/BenchmarkSidePanel"; +import { + BenchmarkShortcutCardList, + BenchmarkShortcutItem, +} from "../../common/BenchmarkShortcutCardList"; import BenchmarkSingleDataTable from "../components/benchmarkTimeSeries/components/BenchmarkSingleDataTable"; import { BenchmarkSingleViewNavigation } from "../components/benchmarkTimeSeries/components/BenchmarkSingleViewNatigation"; import BenchmarkTimeSeriesChartGroup from "../components/benchmarkTimeSeries/components/BenchmarkTimeSeriesChart/BenchmarkTimeSeriesChartGroup"; @@ -326,16 +331,18 @@ export function AutoBenchmarkComparisonGithubExternalLink({ return <>; } return ( - + + + ); } @@ -675,6 +682,84 @@ export function AutoBenchmarkRawDataTable({ config }: AutoComponentProps) { ); } +export function AutoBenchmarkShortcutCardList({ config }: AutoComponentProps) { + const ctx = useBenchmarkCommittedContext(); + const update = useDashboardSelector((s) => s.update); + + const ready = !!ctx && !!ctx.committedTime; + const dataBinding = ctx?.configHandler.dataBinding; + const uiRenderConfig = config as UIRenderConfig; + const queryParams = ready + ? dataBinding.toQueryParams({ + repo: ctx.repo, + benchmarkName: ctx.benchmarkName, + timeRange: ctx.committedTime, + filters: {}, // does not rerender when filter changes, since it fetches the filter optons + }) + : null; + + const { + data: resp, + isLoading, + error, + } = useListBenchmarkMetadata(ctx.benchmarkId, queryParams); + + if (isLoading) { + return ( + + ); + } + + if (error) { + return ( + + (AutoBenchmarkShortcutCardList){error.message} + + ); + } + + // ------------------ convert to the shortcut items ------------------ + const filters = uiRenderConfig.config?.filters; + let data: BenchmarkShortcutItem[] = []; + resp?.data?.forEach((item: any) => { + const name = filters?.find((f: string) => f === item?.type); + if (name) { + const options = item?.options; + options.forEach((option: any) => { + data.push({ + displayName: option.displayName ?? "unkown", + value: option.value, + description: item?.labelName ?? "", + fieldName: name, + }); + }); + } + }); + + return ( + <> + { + const changed: Record = {}; + changed[item.fieldName] = item.value; + update({ + filters: { + ...ctx.committedFilters, + ...changed, + }, + }); + }} + /> + + ); +} + export function AutoBenchmarkSingleDataTable({ config }: AutoComponentProps) { const ctx = useBenchmarkCommittedContext(); const isWorkflowsReady = diff --git a/torchci/components/benchmark_v3/configs/teams/torchao/ao_micro_api_config.ts b/torchci/components/benchmark_v3/configs/teams/torchao/ao_micro_api_config.ts index f597fdc016..7f5be0f8bc 100644 --- a/torchci/components/benchmark_v3/configs/teams/torchao/ao_micro_api_config.ts +++ b/torchci/components/benchmark_v3/configs/teams/torchao/ao_micro_api_config.ts @@ -9,6 +9,10 @@ export const PYTORCH_AO_MICRO_API_BENCHMARK_ID = "torchao_micro_api_benchmark"; const COMPARISON_TABLE_METADATA_COLUMNS = [ ...DEFAULT_COMPARISON_TABLE_METADATA_COLUMNS, + { + field: "dtype", + displayName: "Quant Type", + }, { field: "extra_key.use_compile", displayName: "Use Compile", @@ -96,8 +100,16 @@ export const PytorcAoMicroApiBenchmarkDashoboardConfig: BenchmarkUIConfig = { }, }, renders: [ + { + type: "AutoBenchmarkShortcutCardList", + title: "Dtype Lists", + config: { + filters: ["dtype"], + }, + }, { type: "AutoBenchmarkComparisonGithubExternalLink", + title: "Github Runs", description: "See original github runs for left and right runs", config: {}, }, diff --git a/torchci/components/benchmark_v3/configs/teams/torchao/config.ts b/torchci/components/benchmark_v3/configs/teams/torchao/config.ts index 9fb2d6de89..c6db357066 100644 --- a/torchci/components/benchmark_v3/configs/teams/torchao/config.ts +++ b/torchci/components/benchmark_v3/configs/teams/torchao/config.ts @@ -150,8 +150,16 @@ export const PytorchOperatorMicroBenchmarkDashoboardConfig: BenchmarkUIConfig = }, }, renders: [ + { + type: "AutoBenchmarkShortcutCardList", + title: "Operator Lists", + config: { + filters: ["operatorName"], + }, + }, { type: "AutoBenchmarkComparisonGithubExternalLink", + title: "Github runs (external)", description: "See original github runs for left and right runs", config: {}, }, diff --git a/torchci/components/benchmark_v3/configs/utils/autoRegistration.tsx b/torchci/components/benchmark_v3/configs/utils/autoRegistration.tsx index cf7d905660..82d11642c9 100644 --- a/torchci/components/benchmark_v3/configs/utils/autoRegistration.tsx +++ b/torchci/components/benchmark_v3/configs/utils/autoRegistration.tsx @@ -3,6 +3,7 @@ import { AutoBenchmarkLogs, AutoBenchmarkPairwiseTable, AutoBenchmarkRawDataTable, + AutoBenchmarkShortcutCardList, AutoBenchmarkSingleDataTable, AutoBenchmarkSingleViewNavigation, AutoBenchmarkTimeSeriesChartGroup, @@ -70,7 +71,9 @@ export class AutoComponentRegistry { AutoBenchmarkLogs: { Component: AutoBenchmarkLogs, }, - + AutoBenchmarkShortcutCardList: { + Component: AutoBenchmarkShortcutCardList, + }, // Add your auto components here }; this.map = Object.freeze({ ...registry }); diff --git a/torchci/lib/benchmark/api_helper/backend/compilers/compiler_benchmark_data.ts b/torchci/lib/benchmark/api_helper/backend/compilers/compiler_benchmark_data.ts index f0d0279bcd..d950218cf5 100644 --- a/torchci/lib/benchmark/api_helper/backend/compilers/compiler_benchmark_data.ts +++ b/torchci/lib/benchmark/api_helper/backend/compilers/compiler_benchmark_data.ts @@ -28,6 +28,9 @@ export async function getCompilerBenchmarkTimeSeriesData( const queryParams = await getCompilerBenchmarkTimeRangeQueryParams( inputparams ); + if (!queryParams) { + return emptyTimeSeriesResponse(); + } const rows = await fetchCompilerDataFromDb(queryParams); if (rows.length === 0) { return emptyTimeSeriesResponse(); @@ -106,7 +109,7 @@ export async function getCompilerBenchmarkTimeRangeQueryParams( queryParams["workflows"] = unique_workflows; } else { console.log(`no workflow found in clickhouse using ${queryParams}`); - return []; + return undefined; } } else { console.log( diff --git a/torchci/lib/benchmark/api_helper/backend/dataFetchers/fetchers.ts b/torchci/lib/benchmark/api_helper/backend/dataFetchers/fetchers.ts index 38c8446f47..8b8622f7cb 100644 --- a/torchci/lib/benchmark/api_helper/backend/dataFetchers/fetchers.ts +++ b/torchci/lib/benchmark/api_helper/backend/dataFetchers/fetchers.ts @@ -12,6 +12,7 @@ import { import { BenchmarkMetadataQuery, PytorchOperatorMicrobenchmarkMetadataFetcher, + TorchAoMicrobApienchmarkMetadataFetcher, } from "./queryBuilderUtils/listMetadataQueryBuilder"; import { BenchmarkDataFetcher, @@ -31,6 +32,7 @@ const dataCtors: Record BenchmarkDataFetcher> = { // Register benchmark metadata fetchers. this is mainly used in list_metadata api const metaCtors: Record BenchmarkMetadataFetcher> = { pytorch_operator_microbenchmark: PytorchOperatorMicrobenchmarkMetadataFetcher, + torchao_micro_api_benchmark: TorchAoMicrobApienchmarkMetadataFetcher, default: BenchmarkMetadataQuery, }; diff --git a/torchci/lib/benchmark/api_helper/backend/dataFetchers/queryBuilderUtils/benchmarkDataQueryBuilder.ts b/torchci/lib/benchmark/api_helper/backend/dataFetchers/queryBuilderUtils/benchmarkDataQueryBuilder.ts index 8af8a8fd4b..548a8cb0ea 100644 --- a/torchci/lib/benchmark/api_helper/backend/dataFetchers/queryBuilderUtils/benchmarkDataQueryBuilder.ts +++ b/torchci/lib/benchmark/api_helper/backend/dataFetchers/queryBuilderUtils/benchmarkDataQueryBuilder.ts @@ -334,7 +334,7 @@ export class BenchmarkDataQuery extends ExecutableQueryBase { validateInputs(inputs: any) { if (!inputs.benchmarkName && !inputs.benchmarkNames) { throw new Error( - "Either benchmarkName or benchmarkNames must be provided" + "[BenchmarkDataQuery]Either benchmarkName or benchmarkNames must be provided" ); } if (!inputs.repo) { @@ -345,6 +345,8 @@ export class BenchmarkDataQuery extends ExecutableQueryBase { toQueryParams(inputs: any, id?: string): Record { this.validateInputs(inputs); + console.log("[benchmarkDatQueryBuilder] inputs:", inputs); + if (inputs.benchmarkName && !inputs.benchmarkNames) { inputs.benchmarkNames = [inputs.benchmarkName]; } diff --git a/torchci/lib/benchmark/api_helper/backend/dataFetchers/queryBuilderUtils/listCommitQueryBuilder.ts b/torchci/lib/benchmark/api_helper/backend/dataFetchers/queryBuilderUtils/listCommitQueryBuilder.ts index badae39461..46351d0108 100644 --- a/torchci/lib/benchmark/api_helper/backend/dataFetchers/queryBuilderUtils/listCommitQueryBuilder.ts +++ b/torchci/lib/benchmark/api_helper/backend/dataFetchers/queryBuilderUtils/listCommitQueryBuilder.ts @@ -142,7 +142,11 @@ export class PytorchOperatorMicroListCommitsDataFetcher } toQueryParams(inputs: any, id?: string): Record { - const params = this._data_query.toQueryParams(inputs); + const pq = { + ...inputs, + operatorName: inputs.operatorName ?? "", + }; + const params = this._data_query.toQueryParams(pq); return params; } diff --git a/torchci/lib/benchmark/api_helper/backend/dataFetchers/queryBuilderUtils/listMetadataQueryBuilder.ts b/torchci/lib/benchmark/api_helper/backend/dataFetchers/queryBuilderUtils/listMetadataQueryBuilder.ts index 8413f6c5a7..1f1b0ade5a 100644 --- a/torchci/lib/benchmark/api_helper/backend/dataFetchers/queryBuilderUtils/listMetadataQueryBuilder.ts +++ b/torchci/lib/benchmark/api_helper/backend/dataFetchers/queryBuilderUtils/listMetadataQueryBuilder.ts @@ -235,3 +235,38 @@ export class PytorchOperatorMicrobenchmarkMetadataFetcher return this._data_query.toQueryParams(params); } } + +export class TorchAoMicrobApienchmarkMetadataFetcher + extends ExecutableQueryBase + implements BenchmarkMetadataFetcher +{ + private _data_query: BenchmarkMetadataQuery; + + constructor() { + super(); + this._data_query = new BenchmarkMetadataQuery(); + } + + postProcess(data: any[]) { + let li = getDefaultBenchmarkMetadataGroup(data); + li = li.filter((item) => item.type !== BenchmarkMetadataType.DtypeName); + const customizedDtype = makeMetadataItem( + data, + "dtype", + BenchmarkMetadataType.DtypeName, + { displayName: "All Quant type", value: "" }, + "Quant Type" + ); + if (customizedDtype) { + li.push(customizedDtype); + } + return li; + } + build() { + return this._data_query.build(); + } + + toQueryParams(inputs: any) { + return this._data_query.toQueryParams(inputs); + } +} diff --git a/torchci/lib/benchmark/api_helper/backend/list_commits.ts b/torchci/lib/benchmark/api_helper/backend/list_commits.ts index 5ba36aaf17..3295c512cc 100644 --- a/torchci/lib/benchmark/api_helper/backend/list_commits.ts +++ b/torchci/lib/benchmark/api_helper/backend/list_commits.ts @@ -81,12 +81,12 @@ function getFormat(data: any, format: string = "raw") { } } -const defaultGetCommitsInputs: any = { +const defaultGeneralListCommitsInputs: any = { branches: [], models: [], backends: [], device: "", - arch: [], + arches: [], dtype: "", mode: "", }; @@ -117,7 +117,7 @@ export async function getGeneralCommits(id: string, inputparams: any) { } const queryParams = { - ...defaultGetCommitsInputs, // base defaults + ...defaultGeneralListCommitsInputs, // base defaults ...inputparams, // override with caller's values }; diff --git a/torchci/pages/api/benchmark/get_time_series.ts b/torchci/pages/api/benchmark/get_time_series.ts index adc964cfce..2d1555e0b0 100644 --- a/torchci/pages/api/benchmark/get_time_series.ts +++ b/torchci/pages/api/benchmark/get_time_series.ts @@ -1,6 +1,9 @@ import { checkAuthWithApiToken } from "lib/auth/auth"; import { CompilerQueryType } from "lib/benchmark/api_helper/backend/common/type"; -import { readApiGetParams } from "lib/benchmark/api_helper/backend/common/utils"; +import { + emptyTimeSeriesResponse, + readApiGetParams, +} from "lib/benchmark/api_helper/backend/common/utils"; import { getCompilerBenchmarkTimeSeriesData } from "lib/benchmark/api_helper/backend/compilers/compiler_benchmark_data"; import { getBenchmarkDataFetcher } from "lib/benchmark/api_helper/backend/dataFetchers/fetchers"; import { getGeneralCommits } from "lib/benchmark/api_helper/backend/list_commits"; @@ -107,6 +110,9 @@ async function getGenernalBenchmarkTimeSeries( query_params ); + if (!params) { + return emptyTimeSeriesResponse(); + } const fetcher = getBenchmarkDataFetcher(id); const result = await fetcher.applyQuery(params); return fetcher.applyFormat(result, formats); @@ -132,8 +138,8 @@ export async function getGeneralBenchmarkTimeRangeQueryParams( if (commit_results.length > 0) { queryParams["workflows"] = unique_workflows; } else { - console.log(`no workflow found in clickhouse using ${queryParams}`); - return []; + console.log("no workflow found in clickhouse using", queryParams); + return undefined; } } else { console.log(