From 0d10f3edc6df83dd91f5e561c80d73ec700bf325 Mon Sep 17 00:00:00 2001 From: Anna Mayzner Date: Mon, 24 Nov 2025 11:42:16 +0000 Subject: [PATCH 1/2] ui: Fetch distinct values for equality --- ui/src/components/widgets/data_grid/common.ts | 43 +++ .../components/widgets/data_grid/data_grid.ts | 291 +++++++++++++++++- .../data_grid/in_memory_data_source.ts | 70 +++-- .../in_memory_data_source_unittest.ts | 88 ++++++ .../widgets/data_grid/sql_data_source.ts | 41 +++ 5 files changed, 498 insertions(+), 35 deletions(-) diff --git a/ui/src/components/widgets/data_grid/common.ts b/ui/src/components/widgets/data_grid/common.ts index 24056f08431..5c76c53e73b 100644 --- a/ui/src/components/widgets/data_grid/common.ts +++ b/ui/src/components/widgets/data_grid/common.ts @@ -104,6 +104,16 @@ export interface DataGridDataSource { * Returns a promise that resolves to all filtered and sorted rows. */ exportData(): Promise; + + /** + * Get distinct values for a column with current filters applied. + * Returns up to maxValues distinct values, or 'too_many' if there are more. + * Returns 'error' if the query fails. + */ + getDistinctValues?( + column: string, + maxValues: number, + ): Promise; } /** @@ -157,3 +167,36 @@ export function areAggregatesEqual( ): boolean { return a.col === b.col && a.func === b.func; } + +/** + * Comparator function for SQL values. + * Handles strings (locale comparison), numbers, bigints, Uint8Arrays, and nulls. + */ +export function compareSqlValues(a: SqlValue, b: SqlValue): number { + // Handle null values - nulls come first + if (a === null && b === null) return 0; + if (a === null) return -1; + if (b === null) return 1; + + if (typeof a === 'number' && typeof b === 'number') { + return a - b; + } + + if (typeof a === 'bigint' && typeof b === 'bigint') { + return Number(a - b); + } + + if (typeof a === 'string' && typeof b === 'string') { + return a.localeCompare(b); + } + + if (a instanceof Uint8Array && b instanceof Uint8Array) { + // Compare by length for Uint8Arrays + return a.length - b.length; + } + + // Default comparison using string conversion + const strA = String(a); + const strB = String(b); + return strA.localeCompare(strB); +} diff --git a/ui/src/components/widgets/data_grid/data_grid.ts b/ui/src/components/widgets/data_grid/data_grid.ts index ba4ab64585a..f4b8796c5e8 100644 --- a/ui/src/components/widgets/data_grid/data_grid.ts +++ b/ui/src/components/widgets/data_grid/data_grid.ts @@ -23,7 +23,15 @@ import {Box} from '../../../widgets/box'; import {Button} from '../../../widgets/button'; import {Chip} from '../../../widgets/chip'; import {LinearProgress} from '../../../widgets/linear_progress'; -import {MenuDivider, MenuItem, MenuTitle} from '../../../widgets/menu'; +import { + MenuDivider, + MenuItem, + MenuTitle, + PopupMenu, +} from '../../../widgets/menu'; +import {PopupPosition} from '../../../widgets/popup'; +import {TextInput} from '../../../widgets/text_input'; +import {Spinner} from '../../../widgets/spinner'; import {Stack, StackAuto} from '../../../widgets/stack'; import { renderSortMenuItems, @@ -307,6 +315,174 @@ export interface DataGridApi { ): Promise; } +// Configuration constants +const MAX_DISTINCT_VALUES = 100; +const MAX_DISPLAY_STRING_LENGTH = 50; +const SEARCH_BOX_THRESHOLD = 10; + +interface DistinctValuesFilterMenuAttrs { + readonly column: ColumnDefinition; + readonly dataSource: DataGridDataSource; + readonly filters: ReadonlyArray; + readonly sorting: Sorting; + readonly onFilterAdd: OnFilterAdd; + readonly getCachedValues: () => SqlValue[] | 'too_many' | 'error' | undefined; + readonly setCachedValues: (values: SqlValue[] | 'too_many' | 'error') => void; + readonly fetchValues: () => Promise; + readonly formatValue: (value: SqlValue) => string; + readonly operator: '=' | '!='; + readonly label: string; +} + +class DistinctValuesFilterMenu + implements m.ClassComponent +{ + private distinctValues?: SqlValue[] | 'too_many' | 'error'; + private loading = false; + private searchText = ''; + + private getDistinctValuesArray(): SqlValue[] | undefined { + return Array.isArray(this.distinctValues) ? this.distinctValues : undefined; + } + + private async onPopupMount(attrs: DistinctValuesFilterMenuAttrs) { + // Check cache first + const cached = attrs.getCachedValues(); + if (cached !== undefined) { + this.distinctValues = cached; + this.loading = false; + m.redraw(); + return; + } + + // If not cached, fetch distinct values + this.loading = true; + m.redraw(); + + try { + const values = await attrs.fetchValues(); + attrs.setCachedValues(values); + this.distinctValues = values; + } finally { + this.loading = false; + m.redraw(); + } + } + + view({attrs}: m.Vnode): m.Children { + return m( + PopupMenu, + { + position: PopupPosition.RightStart, + trigger: m(MenuItem, { + label: attrs.label, + rightIcon: 'chevron_right', + closePopupOnClick: false, + }), + showArrow: false, + createNewGroup: false, + edgeOffset: 5, + onPopupMount: () => this.onPopupMount(attrs), + }, + this.renderDistinctValuesMenu(attrs), + ); + } + + private renderDistinctValuesMenu( + attrs: DistinctValuesFilterMenuAttrs, + ): m.Children { + const {column, onFilterAdd, formatValue, operator} = attrs; + + // If distinctValues not fetched yet, show loading + if (this.distinctValues === undefined) { + return m(MenuItem, { + label: 'Loading...', + disabled: true, + }); + } + + const valuesArray = this.getDistinctValuesArray(); + const isValidArray = valuesArray !== undefined && valuesArray.length > 0; + const hasNoValues = valuesArray !== undefined && valuesArray.length === 0; + + // Filter distinct values based on search text + const filteredValues = valuesArray + ? valuesArray.filter((value: SqlValue) => { + const valueStr = formatValue(value); + return valueStr.toLowerCase().includes(this.searchText.toLowerCase()); + }) + : []; + + const showSearchBox = + valuesArray && valuesArray.length > SEARCH_BOX_THRESHOLD; + + return [ + // Show search box only if we have more than threshold values + showSearchBox && + m(TextInput, { + autofocus: true, + placeholder: 'Search values...', + value: this.searchText, + oninput: (e: InputEvent) => { + if (!e.target) return; + this.searchText = (e.target as HTMLInputElement).value; + }, + }), + showSearchBox && m(MenuDivider), + // Show loading spinner + this.loading && m(Spinner), + // Show filtered distinct values + isValidArray && + filteredValues.length > 0 && + filteredValues.map((value: SqlValue) => { + const displayValue = formatValue(value); + return m(MenuItem, { + label: displayValue, + onclick: () => { + onFilterAdd({ + column: column.name, + op: operator, + value: value, + }); + }, + }); + }), + // Show "no results" message when search returns nothing + isValidArray && + filteredValues.length === 0 && + !this.loading && + m(MenuItem, { + label: 'No matching values', + disabled: true, + }), + // Show message when there are no non-null values + hasNoValues && + !this.loading && + m(MenuItem, { + label: 'No non-null values', + disabled: true, + }), + // Show message for too many distinct values + this.distinctValues === 'too_many' && + m(MenuItem, { + label: `Too many distinct values (>${MAX_DISTINCT_VALUES})`, + disabled: true, + }), + // Show message for query error + this.distinctValues === 'error' && + m(MenuItem, { + label: 'Failed to fetch values', + disabled: true, + }), + ]; + } +} + +interface CachedDistinctValues { + queryState: string; + values: SqlValue[] | 'too_many' | 'error'; +} + export class DataGrid implements m.ClassComponent { // Internal state private sorting: Sorting = {direction: 'UNSORTED'}; @@ -334,6 +510,8 @@ export class DataGrid implements m.ClassComponent { private currentDataSource?: DataGridDataSource; private currentColumns?: ReadonlyArray; private currentValueFormatter?: ValueFormatter; + // Cache for distinct values per column + private distinctValuesCache = new Map(); oninit({attrs}: m.Vnode) { if (attrs.initialSorting) { @@ -357,6 +535,67 @@ export class DataGrid implements m.ClassComponent { }); } + private getQueryState( + filters: ReadonlyArray, + sorting: Sorting, + ): string { + // Create a string representing the current query state for cache invalidation + return JSON.stringify({filters, sorting}); + } + + private getCachedDistinctValues( + column: string, + filters: ReadonlyArray, + sorting: Sorting, + ): SqlValue[] | 'too_many' | 'error' | undefined { + const queryState = this.getQueryState(filters, sorting); + const cached = this.distinctValuesCache.get(column); + if (cached && cached.queryState === queryState) { + return cached.values; + } + return undefined; + } + + private setCachedDistinctValues( + column: string, + filters: ReadonlyArray, + sorting: Sorting, + values: SqlValue[] | 'too_many' | 'error', + ): void { + const queryState = this.getQueryState(filters, sorting); + this.distinctValuesCache.set(column, { + queryState, + values, + }); + } + + private async fetchDistinctValues( + dataSource: DataGridDataSource, + column: string, + ): Promise { + if (!dataSource.getDistinctValues) { + return 'error'; + } + + return await dataSource.getDistinctValues(column, MAX_DISTINCT_VALUES); + } + + private formatValue(value: SqlValue): string { + if (value === null) { + return 'NULL'; + } + if (typeof value === 'string') { + if (value.length > MAX_DISPLAY_STRING_LENGTH) { + return value.substring(0, MAX_DISPLAY_STRING_LENGTH - 3) + '...'; + } + return value; + } + if (typeof value === 'bigint') { + return value.toString(); + } + return String(value); + } + view({attrs}: m.Vnode) { const { columns, @@ -498,6 +737,56 @@ export class DataGrid implements m.ClassComponent { }, }), ); + + // Add distinct values menu if datasource supports it + if (dataSource.getDistinctValues) { + menuItems.push( + m(DistinctValuesFilterMenu, { + column, + dataSource, + filters, + sorting, + onFilterAdd, + getCachedValues: () => + this.getCachedDistinctValues(column.name, filters, sorting), + setCachedValues: (values) => + this.setCachedDistinctValues( + column.name, + filters, + sorting, + values, + ), + fetchValues: () => + this.fetchDistinctValues(dataSource, column.name), + formatValue: (v) => this.formatValue(v), + operator: '=', + label: 'Filter equal to...', + }), + ); + menuItems.push( + m(DistinctValuesFilterMenu, { + column, + dataSource, + filters, + sorting, + onFilterAdd, + getCachedValues: () => + this.getCachedDistinctValues(column.name, filters, sorting), + setCachedValues: (values) => + this.setCachedDistinctValues( + column.name, + filters, + sorting, + values, + ), + fetchValues: () => + this.fetchDistinctValues(dataSource, column.name), + formatValue: (v) => this.formatValue(v), + operator: '!=', + label: 'Filter not equal to...', + }), + ); + } } if (Boolean(column.headerMenuItems)) { diff --git a/ui/src/components/widgets/data_grid/in_memory_data_source.ts b/ui/src/components/widgets/data_grid/in_memory_data_source.ts index 5abd5c3c988..1d802a8f2b7 100644 --- a/ui/src/components/widgets/data_grid/in_memory_data_source.ts +++ b/ui/src/components/widgets/data_grid/in_memory_data_source.ts @@ -24,6 +24,7 @@ import { AggregateSpec, areAggregateArraysEqual, DataGridFilter, + compareSqlValues, } from './common'; export class InMemoryDataSource implements DataGridDataSource { @@ -87,6 +88,39 @@ export class InMemoryDataSource implements DataGridDataSource { return this.filteredSortedData; } + /** + * Get distinct values for a column with current filters applied. + */ + async getDistinctValues( + column: string, + maxValues: number, + ): Promise { + try { + // Extract unique values from the filtered data + const uniqueValues = new Set(); + + for (const row of this.filteredSortedData) { + const value = row[column]; + if (value !== null) { + uniqueValues.add(value); + // Check if we've exceeded the limit + if (uniqueValues.size > maxValues) { + return 'too_many'; + } + } + } + + // Convert Set to Array and sort + const values = Array.from(uniqueValues); + values.sort(compareSqlValues); + + return values; + } catch (error) { + console.error('Failed to fetch distinct values:', error); + return 'error'; + } + } + private calcAggregates( results: ReadonlyArray, aggregates: ReadonlyArray, @@ -240,40 +274,8 @@ export class InMemoryDataSource implements DataGridDataSource { const valueA = a[sortColumn]; const valueB = b[sortColumn]; - // Handle null values - they come first in ascending, last in descending - if (valueA === null && valueB === null) return 0; - if (valueA === null) return sortDirection === 'ASC' ? -1 : 1; - if (valueB === null) return sortDirection === 'ASC' ? 1 : -1; - - if (typeof valueA === 'number' && typeof valueB === 'number') { - return sortDirection === 'ASC' ? valueA - valueB : valueB - valueA; - } - - if (typeof valueA === 'bigint' && typeof valueB === 'bigint') { - return sortDirection === 'ASC' - ? Number(valueA - valueB) - : Number(valueB - valueA); - } - - if (typeof valueA === 'string' && typeof valueB === 'string') { - return sortDirection === 'ASC' - ? valueA.localeCompare(valueB) - : valueB.localeCompare(valueA); - } - - if (valueA instanceof Uint8Array && valueB instanceof Uint8Array) { - // Compare by length for Uint8Arrays - return sortDirection === 'ASC' - ? valueA.length - valueB.length - : valueB.length - valueA.length; - } - - // Default comparison using string conversion - const strA = String(valueA); - const strB = String(valueB); - return sortDirection === 'ASC' - ? strA.localeCompare(strB) - : strB.localeCompare(strA); + const cmp = compareSqlValues(valueA, valueB); + return sortDirection === 'ASC' ? cmp : -cmp; }); } } diff --git a/ui/src/components/widgets/data_grid/in_memory_data_source_unittest.ts b/ui/src/components/widgets/data_grid/in_memory_data_source_unittest.ts index 9d70ab797a4..2869c2828b4 100644 --- a/ui/src/components/widgets/data_grid/in_memory_data_source_unittest.ts +++ b/ui/src/components/widgets/data_grid/in_memory_data_source_unittest.ts @@ -403,4 +403,92 @@ describe('InMemoryDataSource', () => { expect(result.aggregates.tag).toBe('C'); // Max tag alphabetically expect(result.aggregates.id).toBe(1); }); + + describe('getDistinctValues', () => { + test('returns distinct values excluding nulls', async () => { + const result = await dataSource.getDistinctValues('tag', 100); + expect(Array.isArray(result)).toBe(true); + if (Array.isArray(result)) { + expect(result).toEqual(['A', 'B', 'C']); + } + }); + + test('returns distinct numeric values', async () => { + const result = await dataSource.getDistinctValues('active', 100); + expect(Array.isArray(result)).toBe(true); + if (Array.isArray(result)) { + expect(result).toEqual([0, 1]); + } + }); + + test('excludes null values', async () => { + const result = await dataSource.getDistinctValues('value', 100); + expect(Array.isArray(result)).toBe(true); + if (Array.isArray(result)) { + // David has null value, should be excluded + expect(result).toEqual([100, 150, 200, 250n, 300n]); + expect(result).not.toContain(null); + } + }); + + test('returns "too_many" when exceeding maxValues', async () => { + const result = await dataSource.getDistinctValues('tag', 2); + expect(result).toBe('too_many'); + }); + + test('returns empty array when all values are null', async () => { + const nullData: ReadonlyArray = [ + {id: 1, col: null}, + {id: 2, col: null}, + {id: 3, col: null}, + ]; + const nullDataSource = new InMemoryDataSource(nullData); + const result = await nullDataSource.getDistinctValues('col', 100); + expect(Array.isArray(result)).toBe(true); + if (Array.isArray(result)) { + expect(result).toEqual([]); + } + }); + + test('returns sorted values', async () => { + const result = await dataSource.getDistinctValues('name', 100); + expect(Array.isArray(result)).toBe(true); + if (Array.isArray(result)) { + expect(result).toEqual([ + 'Alice', + 'Bob', + 'Charlie', + 'David', + 'Eve', + 'Mallory', + 'Trent', + ]); + } + }); + + test('respects current filters', async () => { + // Filter to only active=1 + const filters: DataGridFilter[] = [{column: 'active', op: '=', value: 1}]; + dataSource.notifyUpdate({filters}); + + const result = await dataSource.getDistinctValues('tag', 100); + expect(Array.isArray(result)).toBe(true); + if (Array.isArray(result)) { + // Only Alice, Charlie, Eve, Trent have active=1 + // Their tags are A, A, B, A + expect(result).toEqual(['A', 'B']); + expect(result).not.toContain('C'); + } + }); + + test('handles bigint values correctly', async () => { + const result = await dataSource.getDistinctValues('value', 100); + expect(Array.isArray(result)).toBe(true); + if (Array.isArray(result)) { + // Should include both numbers and bigints + expect(result).toContain(250n); + expect(result).toContain(300n); + } + }); + }); }); diff --git a/ui/src/components/widgets/data_grid/sql_data_source.ts b/ui/src/components/widgets/data_grid/sql_data_source.ts index 6121b2d1319..5679eb721b0 100644 --- a/ui/src/components/widgets/data_grid/sql_data_source.ts +++ b/ui/src/components/widgets/data_grid/sql_data_source.ts @@ -26,6 +26,7 @@ import { Pagination, AggregateSpec, areAggregateArraysEqual, + compareSqlValues, } from './common'; export class SQLDataSource implements DataGridDataSource { @@ -135,6 +136,46 @@ export class SQLDataSource implements DataGridDataSource { return result.rows; } + /** + * Get distinct values for a column with current filters applied. + */ + async getDistinctValues( + column: string, + maxValues: number, + ): Promise { + if (!this.workingQuery) { + return 'error'; + } + + try { + const query = ` + SELECT DISTINCT ${column} + FROM (${this.workingQuery}) + WHERE ${column} IS NOT NULL + LIMIT ${maxValues + 1} + `; + + const result = await this.engine.query(query); + + if (result.numRows() > maxValues) { + return 'too_many'; + } + + const values: SqlValue[] = []; + for (const it = result.iter({}); it.valid(); it.next()) { + values.push(it.get(column)); + } + + // Sort the values + values.sort(compareSqlValues); + + return values; + } catch (error) { + console.error('Failed to fetch distinct values:', error); + return 'error'; + } + } + /** * Builds a complete SQL query that defines the working dataset (ignores * pagination). From 6161e511cfd42a3fb001fde288f82f0c48cbd618 Mon Sep 17 00:00:00 2001 From: Anna Mayzner Date: Tue, 25 Nov 2025 17:21:26 +0000 Subject: [PATCH 2/2] fix --- ui/src/assets/components/data_grid.scss | 5 + ui/src/components/widgets/data_grid/common.ts | 2 + .../components/widgets/data_grid/data_grid.ts | 321 +++++++----------- .../data_grid/in_memory_data_source.ts | 26 +- .../widgets/data_grid/sql_data_source.ts | 18 + 5 files changed, 169 insertions(+), 203 deletions(-) diff --git a/ui/src/assets/components/data_grid.scss b/ui/src/assets/components/data_grid.scss index 0e07691c986..33e00dcd70b 100644 --- a/ui/src/assets/components/data_grid.scss +++ b/ui/src/assets/components/data_grid.scss @@ -52,3 +52,8 @@ text-overflow: ellipsis; } } + +.pf-distinct-values-menu { + max-height: 300px; + overflow-y: auto; +} diff --git a/ui/src/components/widgets/data_grid/common.ts b/ui/src/components/widgets/data_grid/common.ts index 5c76c53e73b..9465202e7b6 100644 --- a/ui/src/components/widgets/data_grid/common.ts +++ b/ui/src/components/widgets/data_grid/common.ts @@ -109,6 +109,8 @@ export interface DataGridDataSource { * Get distinct values for a column with current filters applied. * Returns up to maxValues distinct values, or 'too_many' if there are more. * Returns 'error' if the query fails. + * Results are cached internally by the data source and invalidated when + * filters change. */ getDistinctValues?( column: string, diff --git a/ui/src/components/widgets/data_grid/data_grid.ts b/ui/src/components/widgets/data_grid/data_grid.ts index f4b8796c5e8..6fa2a765c42 100644 --- a/ui/src/components/widgets/data_grid/data_grid.ts +++ b/ui/src/components/widgets/data_grid/data_grid.ts @@ -319,17 +319,20 @@ export interface DataGridApi { const MAX_DISTINCT_VALUES = 100; const MAX_DISPLAY_STRING_LENGTH = 50; const SEARCH_BOX_THRESHOLD = 10; +const DISTINCT_VALUES_MENU_MAX_HEIGHT = 300; + +type DistinctValuesResult = SqlValue[] | 'too_many' | 'error'; + +interface DistinctValuesMenuState { + values?: DistinctValuesResult; + loading: boolean; + searchText: string; +} interface DistinctValuesFilterMenuAttrs { readonly column: ColumnDefinition; readonly dataSource: DataGridDataSource; - readonly filters: ReadonlyArray; - readonly sorting: Sorting; readonly onFilterAdd: OnFilterAdd; - readonly getCachedValues: () => SqlValue[] | 'too_many' | 'error' | undefined; - readonly setCachedValues: (values: SqlValue[] | 'too_many' | 'error') => void; - readonly fetchValues: () => Promise; - readonly formatValue: (value: SqlValue) => string; readonly operator: '=' | '!='; readonly label: string; } @@ -337,150 +340,155 @@ interface DistinctValuesFilterMenuAttrs { class DistinctValuesFilterMenu implements m.ClassComponent { - private distinctValues?: SqlValue[] | 'too_many' | 'error'; - private loading = false; - private searchText = ''; + private state: DistinctValuesMenuState = { + values: undefined, + loading: false, + searchText: '', + }; - private getDistinctValuesArray(): SqlValue[] | undefined { - return Array.isArray(this.distinctValues) ? this.distinctValues : undefined; - } + async oninit({attrs}: m.Vnode) { + const {dataSource, column} = attrs; - private async onPopupMount(attrs: DistinctValuesFilterMenuAttrs) { - // Check cache first - const cached = attrs.getCachedValues(); - if (cached !== undefined) { - this.distinctValues = cached; - this.loading = false; - m.redraw(); + if (!dataSource.getDistinctValues) { + this.state.values = 'error'; return; } - // If not cached, fetch distinct values - this.loading = true; - m.redraw(); + this.state.loading = true; try { - const values = await attrs.fetchValues(); - attrs.setCachedValues(values); - this.distinctValues = values; + const values = await dataSource.getDistinctValues( + column.name, + MAX_DISTINCT_VALUES, + ); + this.state.values = values; } finally { - this.loading = false; - m.redraw(); + this.state.loading = false; } } view({attrs}: m.Vnode): m.Children { + const {column, onFilterAdd, operator, label} = attrs; + return m( PopupMenu, { position: PopupPosition.RightStart, trigger: m(MenuItem, { - label: attrs.label, + label, rightIcon: 'chevron_right', closePopupOnClick: false, }), showArrow: false, createNewGroup: false, edgeOffset: 5, - onPopupMount: () => this.onPopupMount(attrs), }, - this.renderDistinctValuesMenu(attrs), + this.renderMenuContent(column, onFilterAdd, operator), ); } - private renderDistinctValuesMenu( - attrs: DistinctValuesFilterMenuAttrs, + private renderMenuContent( + column: ColumnDefinition, + onFilterAdd: OnFilterAdd, + operator: '=' | '!=', ): m.Children { - const {column, onFilterAdd, formatValue, operator} = attrs; + const {values, loading, searchText} = this.state; + + // Show loading state + if (values === undefined || loading) { + return m(Spinner); + } - // If distinctValues not fetched yet, show loading - if (this.distinctValues === undefined) { + // Handle error state + if (values === 'error') { return m(MenuItem, { - label: 'Loading...', + label: 'Failed to fetch values', disabled: true, }); } - const valuesArray = this.getDistinctValuesArray(); - const isValidArray = valuesArray !== undefined && valuesArray.length > 0; - const hasNoValues = valuesArray !== undefined && valuesArray.length === 0; + // Handle too many values + if (values === 'too_many') { + return m(MenuItem, { + label: `Too many distinct values (>${MAX_DISTINCT_VALUES})`, + disabled: true, + }); + } - // Filter distinct values based on search text - const filteredValues = valuesArray - ? valuesArray.filter((value: SqlValue) => { - const valueStr = formatValue(value); - return valueStr.toLowerCase().includes(this.searchText.toLowerCase()); - }) - : []; - - const showSearchBox = - valuesArray && valuesArray.length > SEARCH_BOX_THRESHOLD; - - return [ - // Show search box only if we have more than threshold values - showSearchBox && - m(TextInput, { - autofocus: true, - placeholder: 'Search values...', - value: this.searchText, - oninput: (e: InputEvent) => { - if (!e.target) return; - this.searchText = (e.target as HTMLInputElement).value; - }, - }), - showSearchBox && m(MenuDivider), - // Show loading spinner - this.loading && m(Spinner), - // Show filtered distinct values - isValidArray && - filteredValues.length > 0 && - filteredValues.map((value: SqlValue) => { - const displayValue = formatValue(value); - return m(MenuItem, { - label: displayValue, - onclick: () => { - onFilterAdd({ - column: column.name, - op: operator, - value: value, - }); + // Handle empty values + if (values.length === 0) { + return m(MenuItem, { + label: 'No non-null values', + disabled: true, + }); + } + + // Filter values based on search text + const filteredValues = values.filter((value) => { + const valueStr = formatDisplayValue(value); + return valueStr.toLowerCase().includes(searchText.toLowerCase()); + }); + + const showSearchBox = values.length > SEARCH_BOX_THRESHOLD; + + return m( + '.pf-distinct-values-menu', + { + style: { + maxHeight: `${DISTINCT_VALUES_MENU_MAX_HEIGHT}px`, + overflowY: 'auto', + }, + }, + [ + // Search box for filtering values + showSearchBox && + m(TextInput, { + autofocus: true, + placeholder: 'Search values...', + value: searchText, + oninput: (e: InputEvent) => { + const target = e.target as HTMLInputElement; + this.state.searchText = target.value; }, - }); - }), - // Show "no results" message when search returns nothing - isValidArray && - filteredValues.length === 0 && - !this.loading && - m(MenuItem, { - label: 'No matching values', - disabled: true, - }), - // Show message when there are no non-null values - hasNoValues && - !this.loading && - m(MenuItem, { - label: 'No non-null values', - disabled: true, - }), - // Show message for too many distinct values - this.distinctValues === 'too_many' && - m(MenuItem, { - label: `Too many distinct values (>${MAX_DISTINCT_VALUES})`, - disabled: true, - }), - // Show message for query error - this.distinctValues === 'error' && - m(MenuItem, { - label: 'Failed to fetch values', - disabled: true, - }), - ]; + }), + showSearchBox && m(MenuDivider), + // Value list + filteredValues.length > 0 + ? filteredValues.map((value) => + m(MenuItem, { + label: formatDisplayValue(value), + onclick: () => { + onFilterAdd({ + column: column.name, + op: operator, + value, + }); + }, + }), + ) + : m(MenuItem, { + label: 'No matching values', + disabled: true, + }), + ], + ); } } -interface CachedDistinctValues { - queryState: string; - values: SqlValue[] | 'too_many' | 'error'; +function formatDisplayValue(value: SqlValue): string { + if (value === null) { + return 'NULL'; + } + if (typeof value === 'string') { + if (value.length > MAX_DISPLAY_STRING_LENGTH) { + return value.substring(0, MAX_DISPLAY_STRING_LENGTH - 3) + '...'; + } + return value; + } + if (typeof value === 'bigint') { + return value.toString(); + } + return String(value); } export class DataGrid implements m.ClassComponent { @@ -510,8 +518,6 @@ export class DataGrid implements m.ClassComponent { private currentDataSource?: DataGridDataSource; private currentColumns?: ReadonlyArray; private currentValueFormatter?: ValueFormatter; - // Cache for distinct values per column - private distinctValuesCache = new Map(); oninit({attrs}: m.Vnode) { if (attrs.initialSorting) { @@ -535,67 +541,6 @@ export class DataGrid implements m.ClassComponent { }); } - private getQueryState( - filters: ReadonlyArray, - sorting: Sorting, - ): string { - // Create a string representing the current query state for cache invalidation - return JSON.stringify({filters, sorting}); - } - - private getCachedDistinctValues( - column: string, - filters: ReadonlyArray, - sorting: Sorting, - ): SqlValue[] | 'too_many' | 'error' | undefined { - const queryState = this.getQueryState(filters, sorting); - const cached = this.distinctValuesCache.get(column); - if (cached && cached.queryState === queryState) { - return cached.values; - } - return undefined; - } - - private setCachedDistinctValues( - column: string, - filters: ReadonlyArray, - sorting: Sorting, - values: SqlValue[] | 'too_many' | 'error', - ): void { - const queryState = this.getQueryState(filters, sorting); - this.distinctValuesCache.set(column, { - queryState, - values, - }); - } - - private async fetchDistinctValues( - dataSource: DataGridDataSource, - column: string, - ): Promise { - if (!dataSource.getDistinctValues) { - return 'error'; - } - - return await dataSource.getDistinctValues(column, MAX_DISTINCT_VALUES); - } - - private formatValue(value: SqlValue): string { - if (value === null) { - return 'NULL'; - } - if (typeof value === 'string') { - if (value.length > MAX_DISPLAY_STRING_LENGTH) { - return value.substring(0, MAX_DISPLAY_STRING_LENGTH - 3) + '...'; - } - return value; - } - if (typeof value === 'bigint') { - return value.toString(); - } - return String(value); - } - view({attrs}: m.Vnode) { const { columns, @@ -744,21 +689,7 @@ export class DataGrid implements m.ClassComponent { m(DistinctValuesFilterMenu, { column, dataSource, - filters, - sorting, onFilterAdd, - getCachedValues: () => - this.getCachedDistinctValues(column.name, filters, sorting), - setCachedValues: (values) => - this.setCachedDistinctValues( - column.name, - filters, - sorting, - values, - ), - fetchValues: () => - this.fetchDistinctValues(dataSource, column.name), - formatValue: (v) => this.formatValue(v), operator: '=', label: 'Filter equal to...', }), @@ -767,21 +698,7 @@ export class DataGrid implements m.ClassComponent { m(DistinctValuesFilterMenu, { column, dataSource, - filters, - sorting, onFilterAdd, - getCachedValues: () => - this.getCachedDistinctValues(column.name, filters, sorting), - setCachedValues: (values) => - this.setCachedDistinctValues( - column.name, - filters, - sorting, - values, - ), - fetchValues: () => - this.fetchDistinctValues(dataSource, column.name), - formatValue: (v) => this.formatValue(v), operator: '!=', label: 'Filter not equal to...', }), diff --git a/ui/src/components/widgets/data_grid/in_memory_data_source.ts b/ui/src/components/widgets/data_grid/in_memory_data_source.ts index 1d802a8f2b7..8ac468f0cbe 100644 --- a/ui/src/components/widgets/data_grid/in_memory_data_source.ts +++ b/ui/src/components/widgets/data_grid/in_memory_data_source.ts @@ -36,6 +36,11 @@ export class InMemoryDataSource implements DataGridDataSource { private oldSorting: Sorting = {direction: 'UNSORTED'}; private oldFilters: ReadonlyArray = []; private aggregates?: ReadonlyArray; + // Cache for distinct values per column - invalidated when filters change + private distinctValuesCache = new Map< + string, + SqlValue[] | 'too_many' | 'error' + >(); constructor(data: ReadonlyArray) { this.data = data; @@ -56,15 +61,22 @@ export class InMemoryDataSource implements DataGridDataSource { filters = [], aggregates, }: DataGridModel): void { + const filtersChanged = !this.areFiltersEqual(filters, this.oldFilters); + if ( !this.isSortByEqual(sorting, this.oldSorting) || - !this.areFiltersEqual(filters, this.oldFilters) || + filtersChanged || !areAggregateArraysEqual(aggregates, this.aggregates) ) { this.oldSorting = sorting; this.oldFilters = filters; this.aggregates = aggregates; + // Clear distinct values cache when filters change + if (filtersChanged) { + this.distinctValuesCache.clear(); + } + // Apply filters let result = this.applyFilters(this.data, filters); @@ -90,11 +102,18 @@ export class InMemoryDataSource implements DataGridDataSource { /** * Get distinct values for a column with current filters applied. + * Results are cached and invalidated when filters change. */ async getDistinctValues( column: string, maxValues: number, ): Promise { + // Check cache first + const cached = this.distinctValuesCache.get(column); + if (cached !== undefined) { + return cached; + } + try { // Extract unique values from the filtered data const uniqueValues = new Set(); @@ -105,6 +124,7 @@ export class InMemoryDataSource implements DataGridDataSource { uniqueValues.add(value); // Check if we've exceeded the limit if (uniqueValues.size > maxValues) { + this.distinctValuesCache.set(column, 'too_many'); return 'too_many'; } } @@ -114,9 +134,13 @@ export class InMemoryDataSource implements DataGridDataSource { const values = Array.from(uniqueValues); values.sort(compareSqlValues); + // Cache the result + this.distinctValuesCache.set(column, values); + return values; } catch (error) { console.error('Failed to fetch distinct values:', error); + this.distinctValuesCache.set(column, 'error'); return 'error'; } } diff --git a/ui/src/components/widgets/data_grid/sql_data_source.ts b/ui/src/components/widgets/data_grid/sql_data_source.ts index 5679eb721b0..8116dcc621c 100644 --- a/ui/src/components/widgets/data_grid/sql_data_source.ts +++ b/ui/src/components/widgets/data_grid/sql_data_source.ts @@ -38,6 +38,11 @@ export class SQLDataSource implements DataGridDataSource { private aggregates?: ReadonlyArray; private cachedResult?: DataSourceResult; private isLoadingFlag = false; + // Cache for distinct values per column - invalidated when workingQuery changes + private distinctValuesCache = new Map< + string, + SqlValue[] | 'too_many' | 'error' + >(); constructor(engine: Engine, query: string) { this.engine = engine; @@ -79,6 +84,7 @@ export class SQLDataSource implements DataGridDataSource { this.cachedResult = undefined; this.pagination = undefined; this.aggregates = undefined; + this.distinctValuesCache.clear(); // Update the cache with the total row count const rowCount = await this.getRowCount(workingQuery); @@ -138,6 +144,7 @@ export class SQLDataSource implements DataGridDataSource { /** * Get distinct values for a column with current filters applied. + * Results are cached and invalidated when filters change. */ async getDistinctValues( column: string, @@ -147,6 +154,12 @@ export class SQLDataSource implements DataGridDataSource { return 'error'; } + // Check cache first + const cached = this.distinctValuesCache.get(column); + if (cached !== undefined) { + return cached; + } + try { const query = ` SELECT DISTINCT ${column} @@ -158,6 +171,7 @@ export class SQLDataSource implements DataGridDataSource { const result = await this.engine.query(query); if (result.numRows() > maxValues) { + this.distinctValuesCache.set(column, 'too_many'); return 'too_many'; } @@ -169,9 +183,13 @@ export class SQLDataSource implements DataGridDataSource { // Sort the values values.sort(compareSqlValues); + // Cache the result + this.distinctValuesCache.set(column, values); + return values; } catch (error) { console.error('Failed to fetch distinct values:', error); + this.distinctValuesCache.set(column, 'error'); return 'error'; } }