openobserve/tests/ui-testing/playwright-tests/utils/data-ingestion.js

const testLogger = require('./test-logger.js');
const logsdata = require("../../../test-data/logs_data.json");
const { getAuthHeaders, getOrgIdentifier } = require('./cloud-auth.js');
const { ingestBase } = require('./ingest-url.js');

// Rows are stamped at ingest, so a 5-min refresh keeps them inside every 15-min test window.
const SHARED_STREAM = 'e2e_automate';
const SHARED_STREAM_REFRESH_MS = 5 * 60 * 1000;
const lastSharedIngest = new Map();

/**
 * Ingest test data into a stream via API using Node.js context (secure)
 * @param {import('@playwright/test').Page} page - Playwright page object
 * @param {string} streamName - Name of the stream to ingest data into (default: "e2e_automate")
 * @param {{ force?: boolean }} [options] - `force` posts even if the shared stream was refreshed recently (use after a stream settings change)
 * @returns {Promise<object>} - API response, or `{ skipped: true }` when this worker refreshed the shared stream within the last 5 min
 */
async function ingestTestData(page, streamName = "e2e_automate", { force = false } = {}) {
  const orgId = getOrgIdentifier();
  const headers = getHeaders();
  const baseUrl = process.env.INGESTION_URL.endsWith('/')
    ? process.env.INGESTION_URL.slice(0, -1)
    : process.env.INGESTION_URL;

  const sharedKey = `${orgId}:${streamName}`;
  const lastIngest = streamName === SHARED_STREAM ? lastSharedIngest.get(sharedKey) : undefined;
  if (!force && lastIngest && Date.now() - lastIngest < SHARED_STREAM_REFRESH_MS) {
    testLogger.info('Test data ingestion skipped — this worker refreshed the shared stream recently', { streamName, ageMs: Date.now() - lastIngest });
    return { skipped: true };
  }

  try {
    const response = await page.request.post(`${ingestBase(baseUrl)}/api/${orgId}/${streamName}/_json`, {
      headers: headers,
      data: logsdata
    });
    if (response.ok() && streamName === SHARED_STREAM) lastSharedIngest.set(sharedKey, Date.now());

    const responseData = await response.json().catch(() => ({ error: 'Failed to parse JSON' }));
    testLogger.debug('Test data ingestion response', { response: responseData, streamName });
    return responseData;
  } catch (e) {
    testLogger.debug('Test data ingestion error', { error: e.message, streamName });
    return { error: e.message };
  }
}

/**
 * Generates authentication headers for API requests.
 * Delegates to cloud-auth.js which handles both cloud (cookie-based) and
 * self-hosted (Basic Auth) environments.
 *
 * @returns {Object} Headers object with appropriate auth for the environment
 */
function getHeaders() {
  return getAuthHeaders();
}

/**
 * Constructs the ingestion URL for a specific stream.
 * @param {string} orgId - The organization ID
 * @param {string} streamName - The name of the stream to ingest data into
 * @returns {string} The complete ingestion URL
 * @example
 * const url = getIngestionUrl("default", "my_stream");
 * // Returns: "http://localhost:5080/api/default/my_stream/_json"
 */
function getIngestionUrl(orgId, streamName) {
  return `${ingestBase(process.env.INGESTION_URL)}/api/${orgId}/${streamName}/_json`;
}

/**
 * Sends an ingestion request using page.request API (secure, keeps credentials in Node.js context).
 * @param {Page} page - Playwright page object
 * @param {string} url - The ingestion URL
 * @param {Object} payload - The data payload to ingest
 * @param {Object} headers - Request headers including authentication
 * @returns {Promise<Object>} The API response
 * @example
 * const response = await sendRequest(page, url, { log: "test" }, headers);
 * // Returns: { code: 200, status: [{ name: "stream", successful: 1 }] }
 */
async function sendRequest(page, url, payload, headers) {
  try {
    const response = await page.request.post(url, {
      headers: headers,
      data: payload
    });
    return await response.json().catch(() => ({ error: 'Failed to parse JSON' }));
  } catch (e) {
    testLogger.debug('sendRequest error', { error: e.message, url });
    return { error: e.message };
  }
}

/**
 * Ingest custom data into a stream via API using Node.js context (secure)
 * @param {import('@playwright/test').Page} page - Playwright page object
 * @param {string} streamName - Name of the stream to ingest data into
 * @param {Array} data - Array of log records to ingest
 * @returns {Promise<object>} - API response with status and data
 */
async function ingestCustomData(page, streamName, data) {
  const orgId = getOrgIdentifier();
  const headers = getHeaders();
  const baseUrl = process.env.INGESTION_URL.endsWith('/')
    ? process.env.INGESTION_URL.slice(0, -1)
    : process.env.INGESTION_URL;

  try {
    // Use Playwright's request API to keep credentials in Node.js context
    const response = await page.request.post(`${ingestBase(baseUrl)}/api/${orgId}/${streamName}/_json`, {
      headers: headers,
      data: data
    });

    let responseData = null;
    try {
      responseData = await response.json();
    } catch (e) {
      responseData = { error: 'Failed to parse JSON response' };
    }

    testLogger.debug('Custom data ingestion response', { status: response.status(), streamName });
    return {
      status: response.status(),
      data: responseData
    };
  } catch (e) {
    testLogger.debug('Custom data ingestion error', { error: e.message, streamName });
    return {
      status: 500,
      data: { error: e.message }
    };
  }
}

/**
 * Enable log patterns extraction on a stream via API using Node.js context (secure)
 * @param {import('@playwright/test').Page} page - Playwright page object
 * @param {string} streamName - Name of the stream
 * @returns {Promise<object>} - API response with status and data
 */
async function enableLogPatternsExtraction(page, streamName) {
  const orgId = getOrgIdentifier();
  const headers = getHeaders();
  const baseUrl = process.env.INGESTION_URL.endsWith('/')
    ? process.env.INGESTION_URL.slice(0, -1)
    : process.env.INGESTION_URL;

  const settingsPayload = {
    enable_log_patterns_extraction: true
  };

  try {
    // Use Playwright's request API to keep credentials in Node.js context
    const response = await page.request.put(`${baseUrl}/api/${orgId}/streams/${streamName}/settings?type=logs`, {
      headers: headers,
      data: settingsPayload
    });

    let responseData = null;
    try {
      responseData = await response.json();
    } catch (e) {
      responseData = { error: 'Failed to parse JSON response' };
    }

    testLogger.debug('Enable log patterns extraction response', { status: response.status(), streamName, data: responseData });
    return {
      status: response.status(),
      data: responseData
    };
  } catch (e) {
    testLogger.debug('Enable log patterns extraction error', { error: e.message, streamName });
    return {
      status: 500,
      data: { error: e.message }
    };
  }
}

/**
 * Wait for stream data to be indexed by polling the search API
 * Replaces arbitrary waitForTimeout with explicit condition check
 * @param {import('@playwright/test').Page} page - Playwright page object
 * @param {string} streamName - Name of the stream to check
 * @param {number} expectedMinCount - Minimum expected record count (default: 1)
 * @param {number} maxWaitMs - Maximum wait time in ms (default: 30000)
 * @param {number} pollIntervalMs - Polling interval in ms (default: 2000)
 * @returns {Promise<boolean>} - True if data found, false if timed out
 */
async function waitForStreamData(page, streamName, expectedMinCount = 1, maxWaitMs = 30000, pollIntervalMs = 2000) {
  const orgId = getOrgIdentifier();
  const headers = getHeaders();
  const baseUrl = process.env.INGESTION_URL.endsWith('/')
    ? process.env.INGESTION_URL.slice(0, -1)
    : process.env.INGESTION_URL;

  const startTime = Date.now();
  const endTime = Date.now() * 1000; // microseconds
  const startTimeUs = (Date.now() - 3600000) * 1000; // 1 hour ago in microseconds

  const searchPayload = {
    query: {
      sql: `SELECT COUNT(*) as count FROM "${streamName}"`,
      start_time: startTimeUs,
      end_time: endTime,
      from: 0,
      size: 1
    }
  };

  while (Date.now() - startTime < maxWaitMs) {
    try {
      const response = await page.request.post(`${baseUrl}/api/${orgId}/_search?type=logs`, {
        headers: headers,
        data: searchPayload
      });

      if (response.status() === 200) {
        const data = await response.json().catch(() => null);
        const count = data?.hits?.[0]?.count || data?.total || 0;

        if (count >= expectedMinCount) {
          testLogger.debug('Stream data indexed', { streamName, count, waitedMs: Date.now() - startTime });
          return true;
        }
        testLogger.debug('Polling for stream data', { streamName, currentCount: count, expectedMinCount });
      }
    } catch (e) {
      testLogger.debug('Stream data poll error', { error: e.message, streamName });
    }

    // Wait before next poll
    await new Promise(resolve => setTimeout(resolve, pollIntervalMs));
  }

  testLogger.warn('Stream data poll timed out', { streamName, maxWaitMs });
  return false;
}

/**
 * Polls the search API until a specific field value is searchable in a stream.
 * Use this as a readiness gate after ingesting a unique value into a SHARED stream
 * (e.g. e2e_automate) before driving the UI: freshly-ingested data is not queryable
 * immediately (WAL -> index lag), so the field-values panel can return stale/other
 * values and miss the just-ingested one. Waiting on the actual value (not just a row
 * count) guarantees the value is present before the UI assertion runs.
 * @param {import('@playwright/test').Page} page - Playwright page object
 * @param {string} streamName - Name of the stream to check
 * @param {string} fieldName - Field to filter on (e.g. "service_name")
 * @param {string} fieldValue - Exact value that must be searchable
 * @param {number} maxWaitMs - Maximum wait time in ms (default: 30000)
 * @param {number} pollIntervalMs - Polling interval in ms (default: 2000)
 * @returns {Promise<boolean>} - True if the value became searchable, false if timed out
 */
async function waitForFieldValueSearchable(page, streamName, fieldName, fieldValue, maxWaitMs = 30000, pollIntervalMs = 2000) {
  // fieldName is interpolated unescaped into the SQL identifier position, so restrict it
  // to a plain column identifier (callers pass hardcoded names like "service_name").
  if (!/^[a-zA-Z_][a-zA-Z0-9_]*$/.test(fieldName)) {
    throw new Error(`waitForFieldValueSearchable: invalid fieldName "${fieldName}" (expected a column identifier)`);
  }
  const orgId = getOrgIdentifier();
  const headers = getHeaders();
  const baseUrl = process.env.INGESTION_URL.endsWith('/')
    ? process.env.INGESTION_URL.slice(0, -1)
    : process.env.INGESTION_URL;

  const startTime = Date.now();
  // Escape single quotes to keep the SQL literal well-formed.
  const safeValue = String(fieldValue).replace(/'/g, "''");
  const searchPayload = {
    query: {
      sql: `SELECT COUNT(*) as count FROM "${streamName}" WHERE ${fieldName} = '${safeValue}'`,
      start_time: (Date.now() - 3600000) * 1000, // 1 hour ago in microseconds
      end_time: Date.now() * 1000,
      from: 0,
      size: 1
    }
  };

  while (Date.now() - startTime < maxWaitMs) {
    try {
      // end_time must advance each poll so newly-ingested records fall inside the window.
      searchPayload.query.end_time = Date.now() * 1000;
      const response = await page.request.post(`${baseUrl}/api/${orgId}/_search?type=logs`, {
        headers: headers,
        data: searchPayload
      });

      if (response.status() === 200) {
        const data = await response.json().catch(() => null);
        const count = data?.hits?.[0]?.count || 0;
        if (count >= 1) {
          testLogger.debug('Field value searchable', { streamName, fieldName, fieldValue, count, waitedMs: Date.now() - startTime });
          return true;
        }
        testLogger.debug('Polling for field value', { streamName, fieldName, fieldValue });
      } else {
        // Surface non-200s (400/401/403/5xx) so a persistent failure is visible in logs
        // rather than silently burning the full timeout as "not found yet".
        testLogger.warn('Field value poll got non-200', { status: response.status(), streamName, fieldName });
      }
    } catch (e) {
      testLogger.warn('Field value poll error', { error: e.message, streamName, fieldName });
    }

    await new Promise(resolve => setTimeout(resolve, pollIntervalMs));
  }

  testLogger.warn('Field value poll timed out', { streamName, fieldName, fieldValue, maxWaitMs });
  return false;
}

/** Polls /streams until the stream is enumerated — distinct from waitForStreamData (which polls _search), because a new stream is queryable before it is listed and every stream PICKER is built from this list, not from search. */
async function waitForStreamListed(page, streamName, streamType = 'logs', maxWaitMs = 90000, pollIntervalMs = 2000) {
  const orgId = getOrgIdentifier();
  const headers = getHeaders();
  // INGESTION_URL first, then ZO_BASE_URL: on cloud the two hosts differ and only the former feeds the picker's list.
  const rawBase = process.env.INGESTION_URL || process.env.ZO_BASE_URL || '';
  const baseUrl = rawBase.endsWith('/') ? rawBase.slice(0, -1) : rawBase;

  const startTime = Date.now();
  while (Date.now() - startTime < maxWaitMs) {
    try {
      const response = await page.request.get(
        `${baseUrl}/api/${orgId}/streams?type=${encodeURIComponent(streamType)}`,
        { headers }
      );
      if (response.status() === 200) {
        const data = await response.json().catch(() => null);
        const list = Array.isArray(data?.list) ? data.list : [];
        if (list.some((s) => s?.name === streamName)) {
          testLogger.debug('Stream listed', { streamName, waitedMs: Date.now() - startTime });
          return true;
        }
      } else {
        testLogger.warn('Stream list poll got non-200', { status: response.status(), streamName });
      }
    } catch (e) {
      testLogger.warn('Stream list poll error', { error: e.message, streamName });
    }
    await new Promise((resolve) => setTimeout(resolve, pollIntervalMs));
  }

  testLogger.warn('Stream list poll timed out', { streamName, streamType, maxWaitMs });
  return false;
}

module.exports = {
  ingestTestData,
  getHeaders,
  getIngestionUrl,
  sendRequest,
  ingestCustomData,
  enableLogPatternsExtraction,
  waitForStreamData,
  waitForFieldValueSearchable,
  waitForStreamListed
};