"use strict";

Object.defineProperty(exports, "__esModule", {
  value: true
});
exports.useFetchDataComparisonResult = exports.useDataSearch = exports.getDataComparisonType = exports.fetchInParallelChunks = void 0;
var _lodash = require("lodash");
var _react = require("react");
var _rxjs = require("rxjs");
var _mlIsPopulatedObject = require("@kbn/ml-is-populated-object");
var _mlQueryUtils = require("@kbn/ml-query-utils");
var _i18n = require("@kbn/i18n");
var _mlErrorUtils = require("@kbn/ml-error-utils");
var _mlIsDefined = require("@kbn/ml-is-defined");
var _mlChi2test = require("@kbn/ml-chi2test");
var _public = require("@kbn/data-plugin/public");
var _saved_search_utils = require("../index_data_visualizer/utils/saved_search_utils");
var _kibana_context = require("../kibana_context");
var _use_state_manager = require("./use_state_manager");
var _constants = require("./constants");
var _types = require("./types");
/*
 * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
 * or more contributor license agreements. Licensed under the Elastic License
 * 2.0; you may not use this file except in compliance with the Elastic License
 * 2.0.
 */

const getDataComparisonType = kibanaType => {
  switch (kibanaType) {
    case 'number':
      return _constants.DATA_COMPARISON_TYPE.NUMERIC;
    case 'boolean':
    case 'string':
      return _constants.DATA_COMPARISON_TYPE.CATEGORICAL;
    default:
      return _constants.DATA_COMPARISON_TYPE.UNSUPPORTED;
  }
};
exports.getDataComparisonType = getDataComparisonType;
const computeDomain = comparisonDistribution => {
  const domain = {
    x: {
      min: 0,
      max: 0
    },
    percentage: {
      min: 0,
      max: 0
    },
    doc_count: {
      min: 0,
      max: 0
    }
  };
  comparisonDistribution.forEach(dist => {
    if ((0, _mlIsDefined.isDefined)(dist.percentage)) {
      if (dist.percentage >= domain.percentage.max) {
        domain.percentage.max = dist.percentage;
      } else {
        domain.percentage.min = dist.percentage;
      }
    }
    if ((0, _mlIsDefined.isDefined)(dist.doc_count)) {
      if (dist.doc_count >= domain.doc_count.max) {
        domain.doc_count.max = dist.doc_count;
      } else {
        domain.doc_count.min = dist.doc_count;
      }
    }
    const parsedKey = typeof dist.key === 'number' ? dist.key : parseFloat(dist.key);
    if (!isNaN(parsedKey)) {
      if (parsedKey >= domain.x.max) {
        domain.x.max = parsedKey;
      } else {
        domain.x.min = parsedKey;
      }
    }
  });
  return domain;
};
const useDataSearch = () => {
  const {
    data
  } = (0, _kibana_context.useDataVisualizerKibana)().services;
  return (0, _react.useCallback)(async (esSearchRequestParams, abortSignal) => {
    try {
      const {
        rawResponse: resp
      } = await (0, _rxjs.lastValueFrom)(data.search.search({
        params: esSearchRequestParams
      }, {
        abortSignal
      }));
      return resp;
    } catch (error) {
      if (error.name === 'AbortError') {
        // ignore abort errors
      } else {
        throw Error(error);
      }
    }
  }, [data]);
};
exports.useDataSearch = useDataSearch;
const percents = [5, 10, 15, 20, 25, 30, 35, 40, 45, 50, 55, 60, 65, 70, 75, 80, 85, 90, 95];
const normalizeHistogram = histogram => {
  // Compute a total doc_count for all terms
  const totalDocCount = histogram.reduce((acc, term) => acc + term.doc_count, 0);
  // Iterate over the original array and update the doc_count of each term in the new array
  return histogram.map(term => ({
    ...term,
    percentage: totalDocCount > 0 ? term.doc_count / totalDocCount : 0
  }));
};
const normalizeTerms = (terms, keys, totalDocCount) => {
  // Create a new array of terms with the same keys as the given array
  const normalizedTerms = keys.map(term => ({
    ...term,
    doc_count: 0,
    percentage: 0
  }));

  // Iterate over the original array and update the doc_count of each term in the new array
  terms.forEach(term => {
    const index = keys.findIndex(k => k.key === term.key.toString());
    if (index !== -1) {
      normalizedTerms[index].doc_count = term.doc_count;
      normalizedTerms[index].percentage = term.doc_count / totalDocCount;
    }
  });
  return {
    normalizedTerms,
    totalDocCount
  };
};
const processDataComparisonResult = result => {
  return Object.entries(result).map(([featureName, data]) => {
    if ((0, _types.isNumericDriftData)(data)) {
      // normalize data.referenceHistogram and data.comparisonHistogram to use frequencies instead of counts
      const referenceHistogram = normalizeHistogram(data.referenceHistogram);
      const comparisonHistogram = normalizeHistogram(data.comparisonHistogram);
      const comparisonDistribution = [...referenceHistogram.map(h => ({
        ...h,
        g: _constants.REFERENCE_LABEL
      })), ...comparisonHistogram.map(h => ({
        ...h,
        g: _constants.COMPARISON_LABEL
      }))];
      const domain = computeDomain(comparisonHistogram);
      return {
        featureName,
        secondaryType: data.secondaryType,
        fieldType: _constants.DATA_COMPARISON_TYPE.NUMERIC,
        driftDetected: data.pValue < _constants.DRIFT_P_VALUE_THRESHOLD,
        similarityTestPValue: data.pValue,
        referenceHistogram: referenceHistogram !== null && referenceHistogram !== void 0 ? referenceHistogram : [],
        comparisonHistogram: comparisonHistogram !== null && comparisonHistogram !== void 0 ? comparisonHistogram : [],
        comparisonDistribution,
        domain
      };
    }

    // normalize data.baselineTerms and data.driftedTerms to have same keys
    // Get all unique keys from both arrays
    const allKeys = Array.from(new Set([...data.baselineTerms.map(term => term.key.toString()), ...data.driftedTerms.map(term => term.key.toString())]));

    // Compute a total doc_count for all terms
    const referenceTotalDocCount = data.baselineTerms.reduce((acc, term) => acc + term.doc_count, data.baselineSumOtherDocCount);
    const comparisonTotalDocCount = data.driftedTerms.reduce((acc, term) => acc + term.doc_count, data.driftedSumOtherDocCount);

    // Sort the categories (allKeys) by the following metric: Math.abs(comparisonDocCount-referenceDocCount)/referenceDocCount
    const sortedKeys = allKeys.map(k => {
      const key = k.toString();
      const baselineTerm = data.baselineTerms.find(t => t.key === key);
      const driftedTerm = data.driftedTerms.find(t => t.key === key);
      if (baselineTerm && driftedTerm) {
        const referencePercentage = baselineTerm.doc_count / referenceTotalDocCount;
        const comparisonPercentage = driftedTerm.doc_count / comparisonTotalDocCount;
        return {
          key,
          relative_drift: Math.abs(comparisonPercentage - referencePercentage) / referencePercentage
        };
      }
      return {
        key,
        relative_drift: 0
      };
    }).sort((s1, s2) => s2.relative_drift - s1.relative_drift);

    // Normalize the baseline and drifted terms arrays
    const {
      normalizedTerms: normalizedBaselineTerms
    } = normalizeTerms(data.baselineTerms, sortedKeys, referenceTotalDocCount);
    const {
      normalizedTerms: normalizedDriftedTerms
    } = normalizeTerms(data.driftedTerms, sortedKeys, comparisonTotalDocCount);
    const pValue = (0, _mlChi2test.computeChi2PValue)(normalizedBaselineTerms, normalizedDriftedTerms);
    const comparisonDistribution = [...normalizedBaselineTerms.map(h => ({
      ...h,
      g: _constants.REFERENCE_LABEL
    })), ...normalizedDriftedTerms.map(h => ({
      ...h,
      g: _constants.COMPARISON_LABEL
    }))];
    return {
      featureName,
      secondaryType: data.secondaryType,
      fieldType: _constants.DATA_COMPARISON_TYPE.CATEGORICAL,
      driftDetected: pValue < _constants.DRIFT_P_VALUE_THRESHOLD,
      similarityTestPValue: pValue,
      referenceHistogram: normalizedBaselineTerms !== null && normalizedBaselineTerms !== void 0 ? normalizedBaselineTerms : [],
      comparisonHistogram: normalizedDriftedTerms !== null && normalizedDriftedTerms !== void 0 ? normalizedDriftedTerms : [],
      comparisonDistribution,
      domain: computeDomain(comparisonDistribution)
    };
  });
};
const getDataComparisonQuery = ({
  runtimeFields,
  searchQuery,
  datetimeField,
  timeRange
}) => {
  let rangeFilter;
  if (timeRange && datetimeField !== undefined && (0, _mlIsPopulatedObject.isPopulatedObject)(timeRange, ['start', 'end'])) {
    rangeFilter = {
      range: {
        [datetimeField]: {
          gte: timeRange.start,
          lte: timeRange.end,
          format: 'epoch_millis'
        }
      }
    };
  }
  const query = (0, _lodash.cloneDeep)(!searchQuery || (0, _mlIsPopulatedObject.isPopulatedObject)(searchQuery, ['match_all']) ? (0, _mlQueryUtils.getDefaultDSLQuery)() : searchQuery);
  if (rangeFilter && (0, _mlIsPopulatedObject.isPopulatedObject)(query, ['bool'])) {
    if (Array.isArray(query.bool.filter)) {
      // @ts-expect-error gte and lte can be numeric
      query.bool.filter.push(rangeFilter);
    } else {
      // @ts-expect-error gte and lte can be numeric
      query.bool.filter = [rangeFilter];
    }
  }
  const queryAndRuntimeMappings = {
    query
  };
  if (runtimeFields) {
    queryAndRuntimeMappings.runtime_mappings = runtimeFields;
  }
  return queryAndRuntimeMappings;
};
const fetchReferenceBaselineData = async ({
  baseRequest,
  fields,
  randomSamplerWrapper,
  dataSearch,
  signal
}) => {
  const baselineRequest = {
    ...baseRequest
  };
  const baselineRequestAggs = {};

  // for each field with type "numeric", add a percentiles agg to the request
  for (const {
    field,
    type
  } of fields) {
    // if the field is numeric, add a percentiles and stats aggregations to the request
    if (type === _constants.DATA_COMPARISON_TYPE.NUMERIC) {
      baselineRequestAggs[`${field}_percentiles`] = {
        percentiles: {
          field,
          percents
        }
      };
      baselineRequestAggs[`${field}_stats`] = {
        stats: {
          field
        }
      };
    }
    // if the field is categorical, add a terms aggregation to the request
    if (type === _constants.DATA_COMPARISON_TYPE.CATEGORICAL) {
      baselineRequestAggs[`${field}_terms`] = {
        terms: {
          field,
          size: 100 // also DFA can potentially handle problems with 100 categories, for visualization purposes we will use top 10
        }
      };
    }
  }

  const baselineResponse = await dataSearch({
    ...baselineRequest,
    body: {
      ...baselineRequest.body,
      aggs: randomSamplerWrapper.wrap(baselineRequestAggs)
    }
  }, signal);
  return baselineResponse;
};
const fetchComparisonDriftedData = async ({
  dataSearch,
  fields,
  baselineResponseAggs,
  baseRequest,
  baselineRequest,
  randomSamplerWrapper,
  signal
}) => {
  var _rangesResp$aggregati;
  const driftedRequest = {
    ...baseRequest
  };
  const driftedRequestAggs = {};

  // Since aggregation is not able to split the values into distinct 5% intervals,
  // this breaks our assumption of uniform distributed fractions in the`ks_test`.
  // So, to fix this in the general case, we need to run an additional ranges agg to get the doc count for the ranges
  // that we get from the percentiles aggregation
  // and use it in the bucket_count_ks_test
  const rangesRequestAggs = {};
  for (const {
    field,
    type
  } of fields) {
    if ((0, _mlIsPopulatedObject.isPopulatedObject)(baselineResponseAggs, [`${field}_percentiles`]) && type === _constants.DATA_COMPARISON_TYPE.NUMERIC) {
      // create ranges based on percentiles
      const percentiles = Object.values(baselineResponseAggs[`${field}_percentiles`].values);
      const ranges = [];
      percentiles.forEach((val, idx) => {
        if (idx === 0) {
          ranges.push({
            to: val
          });
        } else if (idx === percentiles.length - 1) {
          ranges.push({
            from: val
          });
        } else {
          ranges.push({
            from: percentiles[idx - 1],
            to: val
          });
        }
      });
      const rangeAggs = {
        range: {
          field,
          ranges
        }
      };
      // add range and bucket_count_ks_test to the request
      rangesRequestAggs[`${field}_ranges`] = rangeAggs;
      driftedRequestAggs[`${field}_ranges`] = rangeAggs;

      // add stats aggregation to the request
      driftedRequestAggs[`${field}_stats`] = {
        stats: {
          field
        }
      };
    }
    // if feature is categoric perform terms aggregation
    if (type === _constants.DATA_COMPARISON_TYPE.CATEGORICAL) {
      driftedRequestAggs[`${field}_terms`] = {
        terms: {
          field,
          size: 100 // also DFA can potentially handle problems with 100 categories, for visualization purposes we will use top 10
        }
      };
    }
  }

  // Compute fractions based on results of ranges
  const rangesResp = await dataSearch({
    ...baselineRequest,
    body: {
      ...baselineRequest.body,
      aggs: randomSamplerWrapper.wrap(rangesRequestAggs)
    }
  }, signal);
  const fieldsWithNoOverlap = new Set();
  const rangesAggs = rangesResp !== null && rangesResp !== void 0 && (_rangesResp$aggregati = rangesResp.aggregations) !== null && _rangesResp$aggregati !== void 0 && _rangesResp$aggregati.sample ? rangesResp.aggregations.sample : rangesResp === null || rangesResp === void 0 ? void 0 : rangesResp.aggregations;
  for (const {
    field
  } of fields) {
    if ((0, _mlIsPopulatedObject.isPopulatedObject)(rangesAggs, [`${field}_ranges`])) {
      const buckets = rangesAggs[`${field}_ranges`].buckets;
      if (Array.isArray(buckets)) {
        const totalSumOfAllBuckets = buckets.reduce((acc, bucket) => acc + bucket.doc_count, 0);
        const fractions = buckets.map(bucket => ({
          ...bucket,
          fraction: bucket.doc_count / totalSumOfAllBuckets
        }));
        if (totalSumOfAllBuckets > 0) {
          driftedRequestAggs[`${field}_ks_test`] = {
            bucket_count_ks_test: {
              buckets_path: `${field}_ranges > _count`,
              alternative: ['two_sided'],
              ...(totalSumOfAllBuckets > 0 ? {
                fractions: fractions.map(bucket => Number(bucket.fraction.toFixed(3)))
              } : {})
            }
          };
        } else {
          // If all doc_counts are 0, that means there's no overlap whatsoever
          // in which case we don't need to make the ks test agg, because it defaults to astronomically small value
          fieldsWithNoOverlap.add(field);
        }
      }
    }
  }
  const driftedResp = await dataSearch({
    ...driftedRequest,
    body: {
      ...driftedRequest.body,
      aggs: randomSamplerWrapper.wrap(driftedRequestAggs)
    }
  }, signal);
  fieldsWithNoOverlap.forEach(field => {
    if (driftedResp.aggregations) {
      driftedResp.aggregations[`${field}_ks_test`] = {
        // Setting -Infinity to represent astronomically small number
        // which would be represented as < 0.000001 in table
        two_sided: -Infinity
      };
    }
  });
  return driftedResp;
};
const fetchHistogramData = async ({
  dataSearch,
  fields,
  driftedRespAggs,
  baselineResponseAggs,
  baseRequest,
  randomSamplerWrapper,
  signal
}) => {
  const histogramRequestAggs = {};
  const fieldRange = {};
  for (const {
    field,
    type
  } of fields) {
    // add histogram aggregation with min and max from baseline
    if (type === _constants.DATA_COMPARISON_TYPE.NUMERIC && baselineResponseAggs[`${field}_stats`] && driftedRespAggs[`${field}_stats`]) {
      const numBins = 10;
      const min = Math.min(baselineResponseAggs[`${field}_stats`].min, driftedRespAggs[`${field}_stats`].min);
      const max = Math.max(baselineResponseAggs[`${field}_stats`].max, driftedRespAggs[`${field}_stats`].max);
      const interval = (max - min) / numBins;
      if (interval === 0) {
        continue;
      }
      const offset = min;
      fieldRange[field] = {
        min,
        max,
        interval
      };
      histogramRequestAggs[`${field}_histogram`] = {
        histogram: {
          field,
          interval,
          offset,
          extended_bounds: {
            min,
            max
          }
        }
      };
    }
  }
  if ((0, _mlIsPopulatedObject.isPopulatedObject)(histogramRequestAggs)) {
    const histogramRequest = {
      ...baseRequest,
      body: {
        ...baseRequest.body,
        aggs: randomSamplerWrapper.wrap(histogramRequestAggs)
      }
    };
    return dataSearch(histogramRequest, signal);
  }
};
const isFulfilled = input => input.status === 'fulfilled';
const isRejected = input => input.status === 'rejected';
function isReturnedError(arg) {
  return (0, _mlIsPopulatedObject.isPopulatedObject)(arg, ['error']);
}

/**
 * Help split one big request into multiple requests (with max of 30 fields/request)
 * to avoid too big of a data payload
 * Returns a merged
 * @param fields - list of fields to split
 * @param randomSamplerWrapper - helper from randomSampler to pack and unpack 'sample' path from esResponse.aggregations
 * @param asyncFetchFn - callback function with the divided fields
 */
const fetchInParallelChunks = async ({
  fields,
  randomSamplerWrapper,
  asyncFetchFn,
  errorMsg
}) => {
  const {
    unwrap
  } = randomSamplerWrapper;
  const results = await Promise.allSettled((0, _lodash.chunk)(fields, 30).map(chunkedFields => asyncFetchFn(chunkedFields)));
  const mergedResults = results.filter(isFulfilled).filter(r => r.value).map(r => {
    try {
      return unwrap(r === null || r === void 0 ? void 0 : r.value.aggregations);
    } catch (e) {
      return undefined;
    }
  }).filter(_mlIsDefined.isDefined);
  if (mergedResults.length === 0) {
    const error = results.find(isRejected);
    if (error) {
      // eslint-disable-next-line no-console
      console.error(error);
      return {
        error: errorMsg !== null && errorMsg !== void 0 ? errorMsg : 'An error occurred fetching data drift data',
        errorBody: error.reason.message
      };
    }
  }
  const baselineResponseAggs = (0, _lodash.flatten)(mergedResults).reduce((prev, acc) => ({
    ...acc,
    ...prev
  }), {});
  return baselineResponseAggs;
};
exports.fetchInParallelChunks = fetchInParallelChunks;
const initialState = {
  data: undefined,
  status: _types.FETCH_STATUS.NOT_INITIATED,
  error: undefined,
  errorBody: undefined
};
const useFetchDataComparisonResult = ({
  fields,
  initialSettings,
  currentDataView,
  timeRanges,
  searchString,
  searchQueryLanguage,
  lastRefresh
} = {
  lastRefresh: 0
}) => {
  const dataSearch = useDataSearch();
  const [result, setResult] = (0, _react.useState)(initialState);
  const [loaded, setLoaded] = (0, _react.useState)(0);
  const [progressMessage, setProgressMessage] = (0, _react.useState)();
  const abortController = (0, _react.useRef)(new AbortController());
  const {
    uiSettings,
    data: {
      query: queryManager
    }
  } = (0, _kibana_context.useDataVisualizerKibana)().services;
  const {
    reference: referenceStateManager,
    comparison: comparisonStateManager
  } = (0, _use_state_manager.useDataDriftStateManagerContext)();
  const cancelRequest = (0, _react.useCallback)(() => {
    abortController.current.abort();
    abortController.current = new AbortController();
    setResult(initialState);
    setProgressMessage(undefined);
    setLoaded(0);
  }, []);
  (0, _react.useEffect)(() => {
    const doFetchEsRequest = async function () {
      var _referenceStateManage;
      const randomSampler = referenceStateManager.randomSampler;
      const randomSamplerProd = comparisonStateManager.randomSampler;
      if (!randomSampler || !randomSamplerProd) return;
      const randomSamplerWrapper = randomSampler.createRandomSamplerWrapper();
      const prodRandomSamplerWrapper = randomSamplerProd.createRandomSamplerWrapper();
      setLoaded(0);
      setResult({
        data: undefined,
        status: _types.FETCH_STATUS.NOT_INITIATED,
        error: undefined
      });
      setProgressMessage(_i18n.i18n.translate('xpack.dataVisualizer.dataDrift.progress.started', {
        defaultMessage: `Ready to fetch data for comparison.`
      }));
      const signal = abortController.current.signal;
      if (!fields || !currentDataView) return;
      setResult({
        data: undefined,
        status: _types.FETCH_STATUS.LOADING,
        error: undefined
      });

      // Placeholder for when there might be difference data views in the future
      const referenceIndex = initialSettings ? initialSettings.reference : currentDataView === null || currentDataView === void 0 ? void 0 : currentDataView.getIndexPattern();
      const comparisonIndex = initialSettings ? initialSettings.comparison : referenceIndex;
      const runtimeFields = currentDataView === null || currentDataView === void 0 ? void 0 : currentDataView.getRuntimeMappings();
      setProgressMessage(_i18n.i18n.translate('xpack.dataVisualizer.dataDrift.progress.loadedFields', {
        defaultMessage: `Loaded fields from index '{referenceIndex}' to analyze.`,
        values: {
          referenceIndex
        }
      }));
      const kqlQuery = searchString !== undefined && searchQueryLanguage !== undefined ? {
        query: searchString,
        language: searchQueryLanguage
      } : undefined;
      const refDataQuery = getDataComparisonQuery({
        searchQuery: (0, _saved_search_utils.createMergedEsQuery)(kqlQuery, (0, _public.mapAndFlattenFilters)([...queryManager.filterManager.getFilters(), ...((_referenceStateManage = referenceStateManager.filters) !== null && _referenceStateManage !== void 0 ? _referenceStateManage : [])]), currentDataView, uiSettings),
        datetimeField: currentDataView === null || currentDataView === void 0 ? void 0 : currentDataView.timeFieldName,
        runtimeFields,
        timeRange: timeRanges === null || timeRanges === void 0 ? void 0 : timeRanges.reference
      });
      try {
        var _comparisonStateManag;
        const fieldsCount = fields.length;
        setProgressMessage(_i18n.i18n.translate('xpack.dataVisualizer.dataDrift.progress.loadingReference', {
          defaultMessage: `Loading reference data for {fieldsCount} fields.`,
          values: {
            fieldsCount
          }
        }));
        const baselineRequest = {
          index: referenceIndex,
          body: {
            size: 0,
            aggs: {},
            ...refDataQuery
          }
        };
        const baselineResponseAggs = await fetchInParallelChunks({
          fields,
          randomSamplerWrapper,
          asyncFetchFn: chunkedFields => fetchReferenceBaselineData({
            dataSearch,
            baseRequest: baselineRequest,
            fields: chunkedFields,
            randomSamplerWrapper,
            signal
          })
        });
        if (isReturnedError(baselineResponseAggs)) {
          setResult({
            data: undefined,
            status: _types.FETCH_STATUS.FAILURE,
            error: baselineResponseAggs.error,
            errorBody: baselineResponseAggs.errorBody
          });
          return;
        }
        setProgressMessage(_i18n.i18n.translate('xpack.dataVisualizer.dataDrift.progress.loadedReference', {
          defaultMessage: `Loaded reference data.`
        }));
        setLoaded(0.25);
        const prodDataQuery = getDataComparisonQuery({
          searchQuery: (0, _saved_search_utils.createMergedEsQuery)(kqlQuery, (0, _public.mapAndFlattenFilters)([...queryManager.filterManager.getFilters(), ...((_comparisonStateManag = comparisonStateManager.filters) !== null && _comparisonStateManag !== void 0 ? _comparisonStateManag : [])]), currentDataView, uiSettings),
          datetimeField: currentDataView === null || currentDataView === void 0 ? void 0 : currentDataView.timeFieldName,
          runtimeFields,
          timeRange: timeRanges === null || timeRanges === void 0 ? void 0 : timeRanges.comparison
        });
        setProgressMessage(_i18n.i18n.translate('xpack.dataVisualizer.dataDrift.progress.loadingComparison', {
          defaultMessage: `Loading comparison data for {fieldsCount} fields.`,
          values: {
            fieldsCount
          }
        }));
        const driftedRequest = {
          index: comparisonIndex,
          body: {
            size: 0,
            aggs: {},
            ...prodDataQuery
          }
        };
        const driftedRespAggs = await fetchInParallelChunks({
          fields,
          randomSamplerWrapper: prodRandomSamplerWrapper,
          asyncFetchFn: chunkedFields => fetchComparisonDriftedData({
            dataSearch,
            baseRequest: driftedRequest,
            baselineRequest,
            baselineResponseAggs,
            fields: chunkedFields,
            randomSamplerWrapper: prodRandomSamplerWrapper,
            signal
          })
        });
        if (isReturnedError(driftedRespAggs)) {
          setResult({
            data: undefined,
            status: _types.FETCH_STATUS.FAILURE,
            error: driftedRespAggs.error,
            errorBody: driftedRespAggs.errorBody
          });
          return;
        }
        setLoaded(0.5);
        setProgressMessage(_i18n.i18n.translate('xpack.dataVisualizer.dataDrift.progress.loadedComparison', {
          defaultMessage: `Loaded comparison data. Now loading histogram data.`
        }));
        const referenceHistogramRequest = {
          index: referenceIndex,
          body: {
            size: 0,
            aggs: {},
            ...refDataQuery
          }
        };
        const referenceHistogramRespAggs = await fetchInParallelChunks({
          fields,
          randomSamplerWrapper,
          asyncFetchFn: chunkedFields => fetchHistogramData({
            dataSearch,
            baseRequest: referenceHistogramRequest,
            baselineResponseAggs,
            driftedRespAggs,
            fields: chunkedFields,
            randomSamplerWrapper,
            signal
          })
        });
        if (isReturnedError(referenceHistogramRespAggs)) {
          setResult({
            data: undefined,
            status: _types.FETCH_STATUS.FAILURE,
            error: referenceHistogramRespAggs.error,
            errorBody: referenceHistogramRespAggs.errorBody
          });
          return;
        }
        setLoaded(0.75);
        setProgressMessage(_i18n.i18n.translate('xpack.dataVisualizer.dataDrift.progress.loadedReferenceHistogram', {
          defaultMessage: `Loaded histogram data for reference data set.`
        }));
        const comparisonHistogramRequest = {
          index: comparisonIndex,
          body: {
            size: 0,
            aggs: {},
            ...prodDataQuery
          }
        };
        const comparisonHistogramRespAggs = await fetchInParallelChunks({
          fields,
          randomSamplerWrapper,
          asyncFetchFn: chunkedFields => fetchHistogramData({
            dataSearch,
            baseRequest: comparisonHistogramRequest,
            baselineResponseAggs,
            driftedRespAggs,
            fields: chunkedFields,
            randomSamplerWrapper,
            signal
          })
        });
        if (isReturnedError(comparisonHistogramRespAggs)) {
          setResult({
            data: undefined,
            status: _types.FETCH_STATUS.FAILURE,
            error: comparisonHistogramRespAggs.error,
            errorBody: comparisonHistogramRespAggs.errorBody
          });
          return;
        }
        const data = {};
        for (const {
          field,
          type,
          secondaryType
        } of fields) {
          if (type === _constants.DATA_COMPARISON_TYPE.NUMERIC && driftedRespAggs[`${field}_ks_test`] && referenceHistogramRespAggs[`${field}_histogram`] && comparisonHistogramRespAggs[`${field}_histogram`]) {
            data[field] = {
              secondaryType,
              type: _constants.DATA_COMPARISON_TYPE.NUMERIC,
              pValue: driftedRespAggs[`${field}_ks_test`].two_sided,
              referenceHistogram: referenceHistogramRespAggs[`${field}_histogram`].buckets,
              comparisonHistogram: comparisonHistogramRespAggs[`${field}_histogram`].buckets
            };
          }
          if (type === _constants.DATA_COMPARISON_TYPE.CATEGORICAL && driftedRespAggs[`${field}_terms`] && baselineResponseAggs[`${field}_terms`]) {
            var _driftedRespAggs$buck, _baselineResponseAggs;
            data[field] = {
              secondaryType,
              type: _constants.DATA_COMPARISON_TYPE.CATEGORICAL,
              driftedTerms: (_driftedRespAggs$buck = driftedRespAggs[`${field}_terms`].buckets) !== null && _driftedRespAggs$buck !== void 0 ? _driftedRespAggs$buck : [],
              driftedSumOtherDocCount: driftedRespAggs[`${field}_terms`].sum_other_doc_count,
              baselineTerms: (_baselineResponseAggs = baselineResponseAggs[`${field}_terms`].buckets) !== null && _baselineResponseAggs !== void 0 ? _baselineResponseAggs : [],
              baselineSumOtherDocCount: baselineResponseAggs[`${field}_terms`].sum_other_doc_count
            };
          }
        }
        setProgressMessage(_i18n.i18n.translate('xpack.dataVisualizer.dataDrift.progress.loadedHistogramData', {
          defaultMessage: `Loaded histogram data for comparison data set.`
        }));
        setResult({
          data: processDataComparisonResult(data),
          status: _types.FETCH_STATUS.SUCCESS
        });
        setLoaded(1);
      } catch (e) {
        // eslint-disable-next-line no-console
        console.error(e);
        setResult({
          data: undefined,
          status: _types.FETCH_STATUS.FAILURE,
          error: 'An error occurred while fetching data drift data',
          errorBody: (0, _mlErrorUtils.extractErrorMessage)(e)
        });
      }
    };
    doFetchEsRequest();
  },
  // eslint-disable-next-line react-hooks/exhaustive-deps
  [referenceStateManager, comparisonStateManager, dataSearch,
  // eslint-disable-next-line react-hooks/exhaustive-deps
  JSON.stringify({
    fields,
    timeRanges,
    currentDataView: currentDataView === null || currentDataView === void 0 ? void 0 : currentDataView.id,
    searchString,
    lastRefresh
  })]);
  const dataComparisonResult = (0, _react.useMemo)(() => ({
    result: {
      ...result,
      loaded,
      progressMessage
    },
    cancelRequest
  }), [result, loaded, progressMessage, cancelRequest]);
  return dataComparisonResult;
};
exports.useFetchDataComparisonResult = useFetchDataComparisonResult;