[ES|QL] Distinguish the functions/fields vs the values on the query level (elastic#213916)

## Summary

Closes elastic#209731

This PR is based on the change made here
elastic/elasticsearch#122459

The main difference is that:

- Functions and fields should now be added as ?? (instead of ?)
- The payload to ES is the same regardless if you send a value or a
field/function


In order to accommodate this the following changes were made:

- Now the variable name in the control form displays the ? or ?? (it
didnt display them before)
<img width="428" alt="image"
src="https://github.com/user-attachments/assets/1381ba4a-591c-47f2-af93-30d54fe7a639"
/>

- The previous created charts with the old format are bwc (this means
that they should load correctly when you checkout in this PR (a helper
function has been created to ensure it)


![meow](https://github.com/user-attachments/assets/a1863b5b-e113-494a-9231-e16386876e91)


### Release notes
Now the fields / functions variables are being described with ?? in the
query. The values variables use ? as before.

### Checklist
- [ ] [Unit or functional
tests](https://www.elastic.co/guide/en/kibana/master/development-tests.html)
were updated or added to match the most common scenarios
- [ ] The PR description includes the appropriate Release Notes section,
and the correct `release_note:*` label is applied per the
[guidelines](https://www.elastic.co/guide/en/kibana/master/contributing.html#kibana-release-notes-process)

Author: stratoula

src/platform/packages/private/kbn-esql-editor/src/esql_editor.tsx

@@ -29,6 +29,7 @@ import { useKibana } from '@kbn/kibana-react-plugin/public';
 import { ESQLLang, ESQL_LANG_ID, monaco, type ESQLCallbacks } from '@kbn/monaco';
 import memoize from 'lodash/memoize';
 import React, { memo, useCallback, useEffect, useMemo, useRef, useState } from 'react';
+import { fixESQLQueryWithVariables } from '@kbn/esql-utils';
 import { createPortal } from 'react-dom';
 import { css } from '@emotion/react';
 import { ESQLVariableType, type ESQLControlVariable } from '@kbn/esql-types';
@@ -123,9 +124,14 @@ export const ESQLEditor = memo(function ESQLEditor({
     data,
   } = kibana.services;
 
+  const fixedQuery = useMemo(
+    () => fixESQLQueryWithVariables(query.esql, esqlVariables),
+    [esqlVariables, query.esql]
+  );
+
   const variablesService = kibana.services?.esql?.variablesService;
   const histogramBarTarget = uiSettings?.get('histogram:barTarget') ?? 50;
-  const [code, setCode] = useState<string>(query.esql ?? '');
+  const [code, setCode] = useState<string>(fixedQuery ?? '');
   // To make server side errors less "sticky", register the state of the code when submitting
   const [codeWhenSubmitted, setCodeStateOnSubmission] = useState(code);
   const [editorHeight, setEditorHeight] = useState(
@@ -212,11 +218,11 @@ export const ESQLEditor = memo(function ESQLEditor({
 
   useEffect(() => {
     if (editor1.current) {
-      if (code !== query.esql) {
-        setCode(query.esql);
+      if (code !== fixedQuery) {
+        setCode(fixedQuery);
       }
     }
-  }, [code, query.esql]);
+  }, [code, fixedQuery]);
 
   // Enable the variables service if the feature is supported in the consumer app
   useEffect(() => {
@@ -285,7 +291,7 @@ export const ESQLEditor = memo(function ESQLEditor({
   monaco.editor.registerCommand('esql.control.time_literal.create', async (...args) => {
     const position = editor1.current?.getPosition();
     await triggerControl(
-      query.esql,
+      fixedQuery,
       ESQLVariableType.TIME_LITERAL,
       position,
       uiActions,
@@ -298,7 +304,7 @@ export const ESQLEditor = memo(function ESQLEditor({
   monaco.editor.registerCommand('esql.control.fields.create', async (...args) => {
     const position = editor1.current?.getPosition();
     await triggerControl(
-      query.esql,
+      fixedQuery,
       ESQLVariableType.FIELDS,
       position,
       uiActions,
@@ -311,7 +317,7 @@ export const ESQLEditor = memo(function ESQLEditor({
   monaco.editor.registerCommand('esql.control.values.create', async (...args) => {
     const position = editor1.current?.getPosition();
     await triggerControl(
-      query.esql,
+      fixedQuery,
       ESQLVariableType.VALUES,
       position,
       uiActions,
@@ -324,7 +330,7 @@ export const ESQLEditor = memo(function ESQLEditor({
   monaco.editor.registerCommand('esql.control.functions.create', async (...args) => {
     const position = editor1.current?.getPosition();
     await triggerControl(
-      query.esql,
+      fixedQuery,
       ESQLVariableType.FUNCTIONS,
       position,
       uiActions,
@@ -440,7 +446,7 @@ export const ESQLEditor = memo(function ESQLEditor({
   const esqlCallbacks: ESQLCallbacks = useMemo(() => {
     const callbacks: ESQLCallbacks = {
       getSources: async () => {
-        clearCacheWhenOld(dataSourcesCache, query.esql);
+        clearCacheWhenOld(dataSourcesCache, fixedQuery);
         const sources = await memoizedSources(dataViews, core).result;
         return sources;
       },
@@ -505,7 +511,7 @@ export const ESQLEditor = memo(function ESQLEditor({
     fieldsMetadata,
     kibana.services?.esql?.getJoinIndicesAutocomplete,
     dataSourcesCache,
-    query.esql,
+    fixedQuery,
     memoizedSources,
     dataViews,
     core,

src/platform/packages/shared/kbn-esql-utils/index.ts

@@ -39,6 +39,7 @@ export {
   mapVariableToColumn,
   getValuesFromQueryField,
   getESQLQueryVariables,
+  fixESQLQueryWithVariables,
 } from './src';
 
 export { ENABLE_ESQL, FEEDBACK_LINK } from './constants';

src/platform/packages/shared/kbn-esql-utils/src/index.ts

@@ -23,6 +23,7 @@ export {
   mapVariableToColumn,
   getValuesFromQueryField,
   getESQLQueryVariables,
+  fixESQLQueryWithVariables,
 } from './utils/query_parsing_helpers';
 export { queryCannotBeSampled } from './utils/query_cannot_be_sampled';
 export {

src/platform/packages/shared/kbn-esql-utils/src/utils/query_parsing_helpers.test.ts

@@ -20,6 +20,7 @@ import {
   getQueryColumnsFromESQLQuery,
   mapVariableToColumn,
   getValuesFromQueryField,
+  fixESQLQueryWithVariables,
 } from './query_parsing_helpers';
 import { monaco } from '@kbn/monaco';
 
@@ -581,4 +582,104 @@ describe('esql query helpers', () => {
       expect(values).toEqual('my_field');
     });
   });
+
+  describe('fixESQLQueryWithVariables', () => {
+    it('should return the query as is if no variables are given', () => {
+      const esql = 'FROM my_index | STATS COUNT(?field)';
+      const variables: ESQLControlVariable[] = [];
+      expect(fixESQLQueryWithVariables(esql, variables)).toEqual(esql);
+    });
+
+    it('should return the query as is if no fields or functions variables are given', () => {
+      const esql = 'FROM my_index | WHERE field == ?value';
+      const variables: ESQLControlVariable[] = [
+        {
+          key: 'interval',
+          value: '5 minutes',
+          type: ESQLVariableType.TIME_LITERAL,
+        },
+        {
+          key: 'agent_name',
+          value: 'go',
+          type: ESQLVariableType.VALUES,
+        },
+      ];
+      expect(fixESQLQueryWithVariables(esql, variables)).toEqual(esql);
+    });
+
+    it('should return the query as is if fields or functions variables are given but they are already used with ??', () => {
+      const esql = 'FROM my_index | STATS COUNT(??field)';
+      const variables: ESQLControlVariable[] = [
+        {
+          key: 'interval',
+          value: '5 minutes',
+          type: ESQLVariableType.TIME_LITERAL,
+        },
+        {
+          key: 'agent_name',
+          value: 'go',
+          type: ESQLVariableType.VALUES,
+        },
+        {
+          key: 'field',
+          value: 'bytes',
+          type: ESQLVariableType.FIELDS,
+        },
+      ];
+      expect(fixESQLQueryWithVariables(esql, variables)).toEqual(esql);
+    });
+
+    it('should fix the query if fields or functions variables are given and they are already used with ?', () => {
+      const esql = 'FROM my_index | STATS COUNT(?field)';
+      const expected = 'FROM my_index | STATS COUNT(??field)';
+      const variables: ESQLControlVariable[] = [
+        {
+          key: 'interval',
+          value: '5 minutes',
+          type: ESQLVariableType.TIME_LITERAL,
+        },
+        {
+          key: 'agent_name',
+          value: 'go',
+          type: ESQLVariableType.VALUES,
+        },
+        {
+          key: 'field',
+          value: 'bytes',
+          type: ESQLVariableType.FIELDS,
+        },
+      ];
+      expect(fixESQLQueryWithVariables(esql, variables)).toEqual(expected);
+    });
+
+    it('should fix a query with multiple variables', () => {
+      const esql =
+        'FROM my_index | STATS COUNT(?field) by ?breakdownField | WHERE agent.name == ?agent_name';
+      const expected =
+        'FROM my_index | STATS COUNT(??field) by ??breakdownField | WHERE agent.name == ?agent_name';
+      const variables: ESQLControlVariable[] = [
+        {
+          key: 'interval',
+          value: '5 minutes',
+          type: ESQLVariableType.TIME_LITERAL,
+        },
+        {
+          key: 'agent_name',
+          value: 'go',
+          type: ESQLVariableType.VALUES,
+        },
+        {
+          key: 'field',
+          value: 'bytes',
+          type: ESQLVariableType.FIELDS,
+        },
+        {
+          key: 'breakdownField',
+          value: 'clientip',
+          type: ESQLVariableType.FIELDS,
+        },
+      ];
+      expect(fixESQLQueryWithVariables(esql, variables)).toEqual(expected);
+    });
+  });
 });

src/platform/packages/shared/kbn-esql-utils/src/utils/query_parsing_helpers.ts

@@ -16,7 +16,7 @@ import type {
   ESQLInlineCast,
   ESQLCommandOption,
 } from '@kbn/esql-ast';
-import type { ESQLControlVariable } from '@kbn/esql-types';
+import { type ESQLControlVariable, ESQLVariableType } from '@kbn/esql-types';
 import type { DatatableColumn } from '@kbn/expressions-plugin/common';
 import { monaco } from '@kbn/monaco';
 
@@ -161,7 +161,7 @@ export const getQueryColumnsFromESQLQuery = (esql: string): string[] => {
 export const getESQLQueryVariables = (esql: string): string[] => {
   const { root } = parse(esql);
   const usedVariablesInQuery = Walker.params(root);
-  return usedVariablesInQuery.map((v) => v.text.replace('?', ''));
+  return usedVariablesInQuery.map((v) => v.text.replace(/^\?+/, ''));
 };
 
 /**
@@ -228,3 +228,36 @@ export const getValuesFromQueryField = (queryString: string, cursorPosition?: mo
     return `${column.name}`;
   }
 };
+
+// this is for backward compatibility, if the query is of fields or functions type
+// and the query is not set with ?? in the query, we should set it
+// https://github.com/elastic/elasticsearch/pull/122459
+export const fixESQLQueryWithVariables = (
+  queryString: string,
+  esqlVariables?: ESQLControlVariable[]
+) => {
+  const currentVariables = getESQLQueryVariables(queryString);
+  if (!currentVariables.length) {
+    return queryString;
+  }
+
+  // filter out the variables that are not used in the query
+  // and that they are not of type FIELDS or FUNCTIONS
+  const identifierTypeVariables = esqlVariables?.filter(
+    (variable) =>
+      currentVariables.includes(variable.key) &&
+      (variable.type === ESQLVariableType.FIELDS || variable.type === ESQLVariableType.FUNCTIONS)
+  );
+
+  // check if they are set with ?? or ? in the query
+  // replace only if there is only one ? in front of the variable
+  if (identifierTypeVariables?.length) {
+    identifierTypeVariables.forEach((variable) => {
+      const regex = new RegExp(`(?<!\\?)\\?${variable.key}`);
+      queryString = queryString.replace(regex, `??${variable.key}`);
+    });
+    return queryString;
+  }
+
+  return queryString;
+};

src/platform/packages/shared/kbn-esql-utils/src/utils/run_query.test.ts

@@ -65,7 +65,7 @@ describe('run query helpers', () => {
 
     it('should return the variables if given', () => {
       const time = { from: 'Jul 5, 2024 @ 08:03:56.849', to: 'Jul 5, 2024 @ 10:03:56.849' };
-      const query = 'FROM foo | KEEP ?field | WHERE agent.name = ?agent_name';
+      const query = 'FROM foo | KEEP ??field | WHERE agent.name = ?agent_name';
       const variables = [
         {
           key: 'field',
@@ -91,9 +91,7 @@ describe('run query helpers', () => {
       const params = getNamedParams(query, time, variables);
       expect(params).toStrictEqual([
         {
-          field: {
-            identifier: 'clientip',
-          },
+          field: 'clientip',
         },
         {
           interval: '5 minutes',
@@ -102,17 +100,15 @@ describe('run query helpers', () => {
           agent_name: 'go',
         },
         {
-          function: {
-            identifier: 'count',
-          },
+          function: 'count',
         },
       ]);
     });
 
     it('should return the variables and named params if given', () => {
       const time = { from: 'Jul 5, 2024 @ 08:03:56.849', to: 'Jul 5, 2024 @ 10:03:56.849' };
       const query =
-        'FROM foo | KEEP ?field | WHERE agent.name = ?agent_name AND time < ?_tend amd time > ?_tstart';
+        'FROM foo | KEEP ??field | WHERE agent.name = ?agent_name AND time < ?_tend amd time > ?_tstart';
       const variables = [
         {
           key: 'field',
@@ -140,9 +136,7 @@ describe('run query helpers', () => {
       expect(params[0]).toHaveProperty('_tstart');
       expect(params[1]).toHaveProperty('_tend');
       expect(params[2]).toStrictEqual({
-        field: {
-          identifier: 'clientip',
-        },
+        field: 'clientip',
       });
       expect(params[3]).toStrictEqual({
         interval: '5 minutes',
@@ -152,9 +146,7 @@ describe('run query helpers', () => {
       });
 
       expect(params[5]).toStrictEqual({
-        function: {
-          identifier: 'count',
-        },
+        function: 'count',
       });
     });
   });

src/platform/packages/shared/kbn-esql-utils/src/utils/run_query.ts

@@ -16,7 +16,6 @@ import { esFieldTypeToKibanaFieldType } from '@kbn/field-types';
 import type { ESQLColumn, ESQLSearchResponse, ESQLSearchParams } from '@kbn/es-types';
 import { lastValueFrom } from 'rxjs';
 import { type ESQLControlVariable } from '@kbn/esql-types';
-import { ESQLVariableType } from '@kbn/esql-types';
 
 export const hasStartEndParams = (query: string) => /\?_tstart|\?_tend/i.test(query);
 
@@ -47,12 +46,8 @@ export const getNamedParams = (
 ) => {
   const namedParams: ESQLSearchParams['params'] = getStartEndParams(query, timeRange);
   if (variables?.length) {
-    variables?.forEach(({ key, value, type }) => {
-      if (type === ESQLVariableType.FIELDS || type === ESQLVariableType.FUNCTIONS) {
-        namedParams.push({ [key]: { identifier: value } });
-      } else {
-        namedParams.push({ [key]: value });
-      }
+    variables?.forEach(({ key, value }) => {
+      namedParams.push({ [key]: value });
     });
   }
   return namedParams;