refac: move profiling queries to proto messages

web-common/src/features/canvas/stores/canvas-entity.ts

@@ -50,6 +50,9 @@ import { DEFAULT_DASHBOARD_WIDTH, namePrefixFromPath } from "../layout-util";
 import { createCustomMapStore } from "@rilldata/web-common/lib/custom-map-store";
 import type { RuntimeClient } from "@rilldata/web-common/runtime-client/v2";
 import { queryServiceConvertExpressionToMetricsSQL } from "@rilldata/web-common/runtime-client";
+import { Expression } from "@rilldata/web-common/proto/gen/rill/runtime/v1/expression_pb";
+import type { JsonValue } from "@bufbuild/protobuf";
+import { stripUndefined } from "@rilldata/web-common/runtime-client/v2/strip-undefined.ts";
 
 export const lastVisitedState = new Map<string, string>();
 
@@ -411,7 +414,11 @@ export class CanvasEntity {
         ],
         queryFn: () =>
           queryServiceConvertExpressionToMetricsSQL(this.client, {
-            expression: parsed.where as any,
+            expression: parsed.where
+              ? Expression.fromJson(
+                  stripUndefined(parsed.where) as unknown as JsonValue,
+                )
+              : undefined,
           }),
       });
     });

web-common/src/features/column-profile/column-types/NumericProfile.svelte

@@ -5,12 +5,11 @@
     INTERVALS,
     isFloat,
   } from "@rilldata/web-common/lib/duckdb-data-types";
-  import { QueryServiceColumnNumericHistogramHistogramMethod } from "@rilldata/web-common/runtime-client";
-  import { useRuntimeClient } from "@rilldata/web-common/runtime-client/v2";
   import {
     createQueryServiceColumnDescriptiveStatistics,
     createQueryServiceColumnRugHistogram,
   } from "@rilldata/web-common/runtime-client";
+  import { useRuntimeClient } from "@rilldata/web-common/runtime-client/v2";
   import { getPriorityForColumn } from "@rilldata/web-common/runtime-client/v2/request-priorities";
   import { derived } from "svelte/store";
   import ColumnProfileIcon from "../ColumnProfileIcon.svelte";
@@ -25,6 +24,8 @@
   import NumericPlot from "./details/NumericPlot.svelte";
   import NullPercentageSpark from "./sparks/NullPercentageSpark.svelte";
   import NumericSpark from "./sparks/NumericSpark.svelte";
+  import { HistogramMethod } from "@rilldata/web-common/proto/gen/rill/runtime/v1/queries_pb.ts";
+  import { getOneofValue } from "@rilldata/web-common/lib/proto-utils.ts";
 
   export let connector: string;
   export let database: string;
@@ -61,7 +62,7 @@
     databaseSchema,
     objectName,
     columnName,
-    QueryServiceColumnNumericHistogramHistogramMethod.HISTOGRAM_METHOD_DIAGNOSTIC,
+    HistogramMethod.DIAGNOSTIC,
     enableProfiling,
   );
   let fdHistogram;
@@ -73,7 +74,7 @@
       databaseSchema,
       objectName,
       columnName,
-      QueryServiceColumnNumericHistogramHistogramMethod.HISTOGRAM_METHOD_FD,
+      HistogramMethod.FD,
       enableProfiling,
     );
   }
@@ -104,7 +105,11 @@
     {
       query: {
         select($query) {
-          return $query?.numericSummary?.numericOutliers?.outliers;
+          return getOneofValue(
+            $query?.numericSummary,
+            "case",
+            "numericOutliers",
+          )?.outliers;
         },
         enabled: enableProfiling,
       },
@@ -138,7 +143,11 @@
       },
     ),
     ($query) => {
-      return $query?.data?.numericSummary?.numericStatistics;
+      return getOneofValue(
+        $query?.data?.numericSummary,
+        "case",
+        "numericStatistics",
+      );
     },
   );
 

web-common/src/features/column-profile/column-types/TimestampProfile.svelte

@@ -12,6 +12,8 @@
     isFetching,
   } from "../queries";
   import NullPercentageSpark from "./sparks/NullPercentageSpark.svelte";
+  import { FromProtoTimeGrainMap } from "@rilldata/web-common/features/dashboards/proto-state/enum-maps.ts";
+  import { TimeGrain } from "@rilldata/web-common/proto/gen/rill/runtime/v1/time_grain_pb.ts";
 
   export let connector: string;
   export let database: string;
@@ -110,8 +112,13 @@
           height={timestampDetailHeight}
           {data}
           {spark}
-          rollupTimeGrain={$timeSeries?.estimatedRollupInterval?.interval}
-          estimatedSmallestTimeGrain={$timeSeries?.smallestTimegrain}
+          rollupTimeGrain={FromProtoTimeGrainMap[
+            $timeSeries?.estimatedRollupInterval?.interval ??
+              TimeGrain.UNSPECIFIED
+          ]}
+          estimatedSmallestTimeGrain={FromProtoTimeGrainMap[
+            $timeSeries?.smallestTimegrain ?? TimeGrain.UNSPECIFIED
+          ]}
         />
       {/if}
     </div>

web-common/src/features/column-profile/column-types/details/TopK.svelte

@@ -49,6 +49,7 @@
   // Since the topk query is in a reactive statement with `enable`, `topK` can be undefined.
   // This leads to unexpected issues when paired with transition
   $: topKCopy = topK ?? topKCopy;
+  $: topKNormalised = topKCopy?.map((k) => k.toJSON());
 
   function ensureSpaces(str: string, n = 6) {
     const sanitized = DOMPurify.sanitize(str, { ALLOWED_TAGS: [] });
@@ -77,9 +78,9 @@
   /** handle LISTs and STRUCTs */
 </script>
 
-{#if topKCopy && totalRows}
+{#if topKNormalised && totalRows}
   <div transition:slide={{ duration: LIST_SLIDE_DURATION }}>
-    {#each topKCopy.slice(0, k) as item (item.value)}
+    {#each topKNormalised.slice(0, k) as item (item.value)}
       {@const negligiblePercentage = item.count / totalRows < 0.0002}
       {@const percentage = negligiblePercentage
         ? "<.01%"

web-common/src/features/column-profile/queries.ts

@@ -7,10 +7,8 @@ import {
   createQueryServiceColumnTimeSeries,
   createQueryServiceColumnTopK,
   createQueryServiceTableCardinality,
-  QueryServiceColumnNumericHistogramHistogramMethod,
   type V1ProfileColumn,
   type V1TableColumnsResponse,
-  type V1TimeSeriesValue,
 } from "@rilldata/web-common/runtime-client";
 import type { RuntimeClient } from "@rilldata/web-common/runtime-client/v2";
 import { getPriorityForColumn } from "@rilldata/web-common/runtime-client/v2/request-priorities";
@@ -19,6 +17,14 @@ import {
   type QueryObserverResult,
 } from "@tanstack/query-core";
 import { derived, type Readable, writable } from "svelte/store";
+import {
+  getOneofValue,
+  valueAsNumber,
+} from "@rilldata/web-common/lib/proto-utils.ts";
+import type {
+  HistogramMethod,
+  TimeSeriesValue,
+} from "@rilldata/web-common/proto/gen/rill/runtime/v1/queries_pb.ts";
 
 export function isFetching(...queries) {
   return queries.some((query) => query?.isFetching);
@@ -81,7 +87,11 @@ export function getSummaries(
           return {
             ...col,
             nullCount: nullValues?.data?.count,
-            cardinality: cardinality?.data?.categoricalSummary?.cardinality,
+            cardinality: getOneofValue(
+              cardinality?.data?.categoricalSummary,
+              "case",
+              "cardinality",
+            ),
             isFetching:
               profileColumnResponse.isFetching ||
               nullValues?.isFetching ||
@@ -181,7 +191,11 @@ export function getCountDistinct(
     [cardinalityQuery, totalRowsQuery],
     ([cardinality, totalRows]) => {
       return {
-        cardinality: cardinality?.data?.categoricalSummary?.cardinality,
+        cardinality: getOneofValue(
+          cardinality?.data?.categoricalSummary,
+          "case",
+          "cardinality",
+        ),
         // SAFETY: if the V1TableCardinalityResponse exists in `totalRows`,
         // then `.cardinality` should presumably always exist in the
         // cardinality query response, so we should be able to cast it to
@@ -222,15 +236,16 @@ export function getTopK(
     },
   );
   return derived(topKQuery, ($topKQuery) => {
-    return $topKQuery?.data?.categoricalSummary?.topK?.entries;
+    return getOneofValue($topKQuery?.data?.categoricalSummary, "case", "topK")
+      ?.entries;
   });
 }
 
-function convertPoint(point: V1TimeSeriesValue) {
+function convertPoint(point: TimeSeriesValue) {
   const next = {
     ...point,
-    count: point?.records?.count as number,
-    ts: point.ts ? new Date(point.ts) : new Date(0),
+    count: valueAsNumber(point?.records?.fields?.count),
+    ts: point.ts?.toDate() ?? new Date(0),
   };
   if (next.count == null || !isFinite(next.count)) {
     next.count = 0;
@@ -329,7 +344,7 @@ export function getNumericHistogram(
   databaseSchema: string,
   objectName: string,
   columnName: string,
-  histogramMethod: QueryServiceColumnNumericHistogramHistogramMethod,
+  histogramMethod: HistogramMethod,
   enabled = true,
 ) {
   return createQueryServiceColumnNumericHistogram(
@@ -346,7 +361,11 @@ export function getNumericHistogram(
     {
       query: {
         select(query) {
-          return query?.numericSummary?.numericHistogramBins?.bins;
+          return getOneofValue(
+            query?.numericSummary,
+            "case",
+            "numericHistogramBins",
+          )?.bins;
         },
         enabled,
       },

web-common/src/features/dashboards/dimension-search/useDimensionSearchResults.ts

@@ -6,6 +6,7 @@ import {
 } from "@rilldata/web-common/runtime-client";
 import type { RuntimeClient } from "@rilldata/web-common/runtime-client/v2";
 import { derived } from "svelte/store";
+import { timeStringToTimestamp } from "@rilldata/web-common/lib/proto-utils.ts";
 
 export type DimensionSearchResult = {
   dimension: string;
@@ -37,8 +38,8 @@ export function useDimensionSearchResults(
         search: searchText,
         limit: 100,
         timeRange: {
-          start: timeRangeSummary.min as any,
-          end: timeRangeSummary.max as any,
+          start: timeStringToTimestamp(timeRangeSummary.min),
+          end: timeStringToTimestamp(timeRangeSummary.max),
         },
       }),
     ),

web-common/src/features/dashboards/proto-state/enum-maps.ts

@@ -1,8 +1,13 @@
 import type { PivotTableMode } from "@rilldata/web-common/features/dashboards/pivot/types";
 import { Operation } from "@rilldata/web-common/proto/gen/rill/runtime/v1/expression_pb";
+import { ExportFormat } from "@rilldata/web-common/proto/gen/rill/runtime/v1/export_format_pb";
 import { TimeGrain } from "@rilldata/web-common/proto/gen/rill/runtime/v1/time_grain_pb";
 import { DashboardState_PivotTableMode } from "@rilldata/web-common/proto/gen/rill/ui/v1/dashboard_pb";
-import { V1Operation, V1TimeGrain } from "@rilldata/web-common/runtime-client";
+import {
+  V1ExportFormat,
+  V1Operation,
+  V1TimeGrain,
+} from "@rilldata/web-common/runtime-client";
 
 // This file should contain all the map from proto and API values.
 // TODO: we should try and find a way to merge these enums
@@ -62,3 +67,18 @@ export const FromProtoPivotTableModeMap = {} as Record<
 for (const op in ToProtoPivotTableModeMap) {
   FromProtoPivotTableModeMap[ToProtoPivotTableModeMap[op]] = op;
 }
+
+export const ToProtoExportFormatMap: Record<V1ExportFormat, ExportFormat> = {
+  [V1ExportFormat.EXPORT_FORMAT_UNSPECIFIED]: ExportFormat.UNSPECIFIED,
+  [V1ExportFormat.EXPORT_FORMAT_CSV]: ExportFormat.CSV,
+  [V1ExportFormat.EXPORT_FORMAT_XLSX]: ExportFormat.XLSX,
+  [V1ExportFormat.EXPORT_FORMAT_PARQUET]: ExportFormat.PARQUET,
+};
+
+export const FromProtoExportFormatMap = {} as Record<
+  ExportFormat,
+  V1ExportFormat
+>;
+for (const format in ToProtoExportFormatMap) {
+  FromProtoExportFormatMap[ToProtoExportFormatMap[format]] = format;
+}

web-common/src/features/dashboards/time-controls/rill-time-ranges.ts

@@ -7,6 +7,7 @@ import {
 } from "@rilldata/web-common/runtime-client";
 import type { RuntimeClient } from "@rilldata/web-common/runtime-client/v2";
 import { queryClient } from "@rilldata/web-common/lib/svelte-query/globalQueryClient";
+import { timeStringToTimestamp } from "@rilldata/web-common/lib/proto-utils.ts";
 
 export async function resolveTimeRanges(
   client: RuntimeClient,
@@ -90,7 +91,7 @@ export async function fetchTimeRanges({
     metricsViewName,
     expressions: rillTimes,
     timeZone,
-    executionTime: executionTime as any,
+    executionTime: timeStringToTimestamp(executionTime),
     timeDimension,
   };
 

web-common/src/features/dashboards/time-series/annotations-selectors.ts

@@ -10,6 +10,8 @@ import type { RuntimeClient } from "@rilldata/web-common/runtime-client/v2";
 import { DateTime, Interval } from "luxon";
 import { TIME_GRAIN } from "@rilldata/web-common/lib/time/config.ts";
 import { keepPreviousData } from "@tanstack/svelte-query";
+import { ToProtoTimeGrainMap } from "@rilldata/web-common/features/dashboards/proto-state/enum-maps.ts";
+import { timeStringToTimestamp } from "@rilldata/web-common/lib/proto-utils.ts";
 
 /**
  * Creates a query that fetches annotations for a measure and transforms
@@ -34,11 +36,13 @@ export function createAnnotationsQuery(
     {
       metricsViewName,
       timeRange: {
-        start: timeStart as any,
-        end: timeEnd as any,
+        start: timeStringToTimestamp(timeStart),
+        end: timeStringToTimestamp(timeEnd),
         timeDimension,
       },
-      timeGrain: timeGranularity as any,
+      timeGrain: timeGranularity
+        ? ToProtoTimeGrainMap[timeGranularity]
+        : undefined,
       measures: [measureName],
     },
     {

web-common/src/features/exports/ExportMenu.svelte

@@ -12,11 +12,15 @@
     type V1Query,
   } from "@rilldata/web-common/runtime-client";
   import { useRuntimeClient } from "@rilldata/web-common/runtime-client/v2";
+  import { Query } from "@rilldata/web-common/proto/gen/rill/runtime/v1/queries_pb";
+  import { ToProtoExportFormatMap } from "@rilldata/web-common/features/dashboards/proto-state/enum-maps";
+  import type { JsonValue } from "@bufbuild/protobuf";
   import { onMount } from "svelte";
   import type TScheduledReportDialog from "../scheduled-reports/ScheduledReportDialog.svelte";
   import { ResourceKind } from "@rilldata/web-common/features/entity-management/resource-selectors";
   import { eventBus } from "@rilldata/web-common/lib/event-bus/event-bus";
   import { extractErrorMessage } from "@rilldata/web-common/lib/errors";
+  import { stripUndefined } from "@rilldata/web-common/runtime-client/v2/strip-undefined.ts";
 
   const runtimeClient = useRuntimeClient();
 
@@ -50,8 +54,10 @@
     const { format, includeHeader = false } = options;
     try {
       const result = await $exportDash.mutateAsync({
-        query: exportQuery as any,
-        format: format as any,
+        query: exportQuery
+          ? Query.fromJson(stripUndefined(exportQuery) as unknown as JsonValue)
+          : undefined,
+        format: ToProtoExportFormatMap[format],
         includeHeader,
         // Include metadata for CSV/XLSX exports in Cloud context.
         ...(includeHeader &&