openobserve/tests/ui-testing/playwright-tests/SDR/multipleSDRPatterns.spec.js

const { test, navigateToBase } = require('../utils/enhanced-baseFixtures.js');
const testLogger = require('../utils/test-logger.js');
const PageManager = require('../../pages/page-manager.js');
const { getAuthHeaders, getOrgIdentifier } = require('../utils/cloud-auth.js');

// Helper function to ingest a SINGLE log entry with retry logic
async function ingestSingleLog(page, streamName, fieldName, fieldValue, maxRetries = 5) {
  const orgId = getOrgIdentifier();
  const headers = getAuthHeaders();

  // Unique marker so verification can query the search API for THIS exact record
  // (WHERE sdr_test_id = '<marker>') instead of guessing which rendered row is latest.
  const marker = `sdr-${require('crypto').randomUUID()}`;

  const logEntry = {
    level: "info",
    [fieldName]: fieldValue,
    log: `Test log with ${fieldName} = ${fieldValue}`,
    sdr_test_id: marker,
    _timestamp: Date.now() * 1000
  };

  testLogger.info(`Ingesting single log with ${fieldName}: ${fieldValue}`);

  //Retry ingestion with exponential backoff for "stream being deleted" errors
  for (let attempt = 1; attempt <= maxRetries; attempt++) {
    const response = await page.evaluate(async ({ url, headers, orgId, streamName, logData }) => {
      const fetchResponse = await fetch(`${url}/api/${orgId}/${streamName}/_json`, {
        method: 'POST',
        headers: headers,
        body: JSON.stringify([logData])
      });
      const responseJson = await fetchResponse.json();
      return {
        status: fetchResponse.status,
        statusText: fetchResponse.statusText,
        body: responseJson
      };
    }, {
      url: process.env.INGESTION_URL,
      headers: headers,
      orgId: orgId,
      streamName: streamName,
      logData: logEntry
    });

    testLogger.info(`Ingestion API response (attempt ${attempt}/${maxRetries}) - Status: ${response.status}, Body:`, response.body);

    if (response.status === 200) {
      testLogger.info('Ingestion successful, waiting for stream to be indexed...');
      await page.waitForTimeout(5000);
      return marker;
    }

    // Check for "stream being deleted" error - retry with backoff
    const errorMessage = response.body?.message || JSON.stringify(response.body);
    if (errorMessage.includes('being deleted') && attempt < maxRetries) {
      const waitTime = attempt * 5000;
      testLogger.info(`Stream is being deleted, waiting ${waitTime/1000}s before retry...`);
      await page.waitForTimeout(waitTime);
      continue;
    }

    testLogger.error(`Ingestion failed! Status: ${response.status}, Response:`, response.body);
    throw new Error(`Ingestion failed with status ${response.status}: ${JSON.stringify(response.body)}`);
  }
}

// Wrap a page.request API call with retry tolerance for transient cloud-network
// failures (e.g. "apiRequestContext.post: Connection timeout" / ECONNRESET on the
// shared cloud gateway). Only retries genuine connection/timeout errors — a real
// HTTP error response still surfaces via the caller's res.ok() check. Prevents a
// single network blip from failing an expensive @slow flow (previously only
// recovered by a full, minutes-long test-level retry).
async function requestWithRetry(page, action, label, maxRetries = 3) {
  for (let attempt = 1; attempt <= maxRetries; attempt++) {
    try {
      return await action();
    } catch (err) {
      const msg = (err && err.message) || String(err);
      const retriable = /Connection timeout|ECONNRESET|ECONNREFUSED|socket hang up|Request timed out|net::|timeout/i.test(msg);
      if (!retriable || attempt === maxRetries) throw err;
      testLogger.warn(`${label} failed (attempt ${attempt}/${maxRetries}): ${msg} — retrying`);
      await page.waitForTimeout(attempt * 2000);
    }
  }
}

test.describe("Multiple Patterns on One Field", { tag: '@enterprise' }, () => {
  test.describe.configure({ mode: 'serial' });
  let pm;

  // Generate unique test run ID for isolation - ensures parallel test runs don't conflict
  const testRunId = Date.now().toString(36) + Math.random().toString(36).substring(2, 6);

  // Query-time patterns (2) - tested first, no re-ingestion needed
  // Each pattern name includes testRunId for isolation
  const queryTimePatterns = [
    { name: `log_filename_${testRunId}`, description: 'Log file name format', pattern: '^\\w+\\.log$', value: 'application.log', action: 'drop', timeType: 'query', scenario: 'query_drop' },
    { name: `time_hh_mm_ss_${testRunId}`, description: 'Time in HH:MM:SS format', pattern: '^\\d{2}:\\d{2}:\\d{2}$', value: '14:30:45', action: 'redact', timeType: 'query', scenario: 'query_redact' }
  ];

  // Ingestion-time patterns (2) - tested second, requires re-ingestion
  const ingestionTimePatterns = [
    { name: `ifsc_code_${testRunId}`, description: 'IFSC code format validation', pattern: '^[A-Z]{2}\\d{2}[A-Z]{4}\\d{10}$', value: 'AB12CDEF1234567890', action: 'drop', timeType: 'ingestion', scenario: 'ingestion_drop' },
    { name: `date_dd_mm_yyyy_${testRunId}`, description: 'Date in DD/MM/YYYY format', pattern: '^\\d{2}/\\d{2}/\\d{4}$', value: '25/12/2024', action: 'redact', timeType: 'ingestion', scenario: 'ingestion_redact' }
  ];

  const fieldName = "multi_data"; // Single field for all patterns
  // Use unique stream name to avoid "stream being deleted" conflicts
  const testStreamName = `sdr_poc_multi_${testRunId}`;

  test.beforeEach(async ({ page }, testInfo) => {
    testLogger.testStart(testInfo.title, testInfo.file);
    await navigateToBase(page);
    pm = new PageManager(page);
    await page.waitForLoadState('networkidle', { timeout: 10000 }).catch(() => {});
    testLogger.info('Import POC test setup completed');
  });

  // Setup: Create patterns via browser-context API (sends cookies for cloud auth)
  test('setup: create patterns', {
    tag: ['@sdr', '@setup', '@sdrMultiPattern']
  }, async ({ page }) => {
    testLogger.info(`=== SETUP: Creating 4 patterns via page.request, testRunId: ${testRunId} ===`);
    const baseUrl = (process.env.INGESTION_URL || process.env.ZO_BASE_URL || 'http://localhost:5080').replace(/\/$/, '');
    const org = getOrgIdentifier();
    const headers = getAuthHeaders();
    const allPatternsToCreate = [...queryTimePatterns, ...ingestionTimePatterns];

    for (const patternDef of allPatternsToCreate) {
      const res = await requestWithRetry(
        page,
        () => page.request.post(`${baseUrl}/api/${org}/re_patterns`, {
          headers,
          data: { name: patternDef.name, description: patternDef.description, pattern: patternDef.pattern }
        }),
        `create pattern ${patternDef.name}`
      );
      if (!res.ok()) {
        const body = await res.text();
        throw new Error(`Failed to create pattern ${patternDef.name}: ${res.status()} ${body}`);
      }
      testLogger.info(`Pattern ${patternDef.name} created via API`);
    }

    // Verify all patterns exist
    const listRes = await requestWithRetry(
      page,
      () => page.request.get(`${baseUrl}/api/${org}/re_patterns`, { headers }),
      'list patterns'
    );
    const listData = await listRes.json();
    const existingNames = new Set((listData.patterns || []).map(p => p.name));
    for (const p of allPatternsToCreate) {
      if (!existingNames.has(p.name)) {
        throw new Error(`Pattern ${p.name} not found after creation`);
      }
    }
    testLogger.info('=== SETUP COMPLETE: All 4 patterns created via API ===');
  });

  // Multiple patterns on ONE field (SEQUENTIAL FLOW)
  // This test has 10 steps with multiple navigations, so needs extended timeout
  test('should link 4 patterns to one field and verify with sequential ingestion', {
    tag: ['@sdr', '@poc', '@sdrMultiPattern', '@slow']
  }, async ({ page }, testInfo) => {
    // Extend timeout to 7 minutes - 10 sequential steps with navigation + CI overhead
    test.setTimeout(420000);
    testLogger.info('=== Testing multiple patterns on ONE field - SEQUENTIAL FLOW ===');
    testLogger.info(`Field: ${fieldName}`);
    testLogger.info('Strategy: Ingest one log → verify → ingest next → verify (repeating pattern)');

    // ==================== STEP 1: Ingest log #1 for query-time drop pattern ====================
    testLogger.info('========== STEP 1: Ingest log with value "application.log" ==========');
    const markerLog1 = await ingestSingleLog(page, testStreamName, fieldName, queryTimePatterns[0].value); // "application.log"
    await pm.sdrVerificationPage.verifySingleFieldInLatestLog(pm.logsPage, testStreamName, fieldName, false, false, markerLog1);
    testLogger.info('STEP 1 PASSED: Log #1 ingested, field visible (no patterns yet)');

    // ==================== STEP 2: Link query-time DROP pattern ====================
    testLogger.info('========== STEP 2: Link query-time DROP pattern (log_filename) ==========');
    await pm.streamAssociationPage.associatePatternWithStream(
      testStreamName,
      queryTimePatterns[0].name, // 'log_filename'
      queryTimePatterns[0].action, // 'drop'
      queryTimePatterns[0].timeType, // 'query'
      fieldName
    );
    testLogger.info('STEP 2 PASSED: Query-time DROP pattern linked');

    // ==================== STEP 3: Verify query-time DROP on existing log ====================
    // No re-ingestion: query-time drop transforms log #1 at search time. Reuse its marker.
    testLogger.info('========== STEP 3: Verify query-time DROP works on EXISTING log ==========');
    await pm.sdrVerificationPage.verifySingleFieldInLatestLog(pm.logsPage, testStreamName, fieldName, true, false, markerLog1);
    testLogger.info('STEP 3 PASSED: Field DROPPED at query time (no re-ingestion needed)');

    // ==================== STEP 4: Ingest log #2 for query-time redact pattern ====================
    testLogger.info('========== STEP 4: Ingest log with value "14:30:45" ==========');
    const markerLog2 = await ingestSingleLog(page, testStreamName, fieldName, queryTimePatterns[1].value); // "14:30:45"
    await pm.sdrVerificationPage.verifySingleFieldInLatestLog(pm.logsPage, testStreamName, fieldName, false, false, markerLog2);
    testLogger.info('STEP 4 PASSED: Log #2 ingested, field visible (pattern not yet linked)');

    // ==================== STEP 5: Link query-time REDACT pattern ====================
    testLogger.info('========== STEP 5: Link query-time REDACT pattern (time_hh_mm_ss) ==========');
    await pm.streamAssociationPage.addAdditionalPatternToField(
      testStreamName,
      fieldName,
      queryTimePatterns[1].name, // 'time_hh_mm_ss'
      queryTimePatterns[1].action, // 'redact'
      queryTimePatterns[1].timeType // 'query'
    );
    testLogger.info('STEP 5 PASSED: Query-time REDACT pattern linked (now 2 patterns total)');

    // ==================== STEP 6: Verify query-time REDACT on existing log ====================
    // No re-ingestion: query-time redact transforms log #2 at search time. Reuse its marker.
    testLogger.info('========== STEP 6: Verify query-time REDACT works on EXISTING log ==========');
    await pm.sdrVerificationPage.verifySingleFieldInLatestLog(pm.logsPage, testStreamName, fieldName, false, true, markerLog2);
    testLogger.info('STEP 6 PASSED: Field REDACTED at query time (no re-ingestion needed)');

    // ==================== STEP 7: Link ingestion-time DROP pattern ====================
    testLogger.info('========== STEP 7: Link ingestion-time DROP pattern (ifsc_code) ==========');
    await pm.streamAssociationPage.addAdditionalPatternToField(
      testStreamName,
      fieldName,
      ingestionTimePatterns[0].name, // 'ifsc_code'
      ingestionTimePatterns[0].action, // 'drop'
      ingestionTimePatterns[0].timeType // 'ingestion'
    );
    testLogger.info('STEP 7 PASSED: Ingestion-time DROP pattern linked (now 3 patterns total)');

    // ==================== STEP 8: Ingest log #3 and verify ingestion-time DROP ====================
    testLogger.info('========== STEP 8: Ingest log with value "AB12CDEF1234567890" ==========');
    const markerLog3 = await ingestSingleLog(page, testStreamName, fieldName, ingestionTimePatterns[0].value); // IFSC code
    await pm.sdrVerificationPage.verifySingleFieldInLatestLog(pm.logsPage, testStreamName, fieldName, true, false, markerLog3);
    testLogger.info('STEP 8 PASSED: Field DROPPED at ingestion time');

    // ==================== STEP 9: Link ingestion-time REDACT pattern ====================
    testLogger.info('========== STEP 9: Link ingestion-time REDACT pattern (date_dd_mm_yyyy) ==========');
    await pm.streamAssociationPage.addAdditionalPatternToField(
      testStreamName,
      fieldName,
      ingestionTimePatterns[1].name, // 'date_dd_mm_yyyy'
      ingestionTimePatterns[1].action, // 'redact'
      ingestionTimePatterns[1].timeType // 'ingestion'
    );
    testLogger.info('STEP 9 PASSED: Ingestion-time REDACT pattern linked (now 4 patterns total on ONE field)');

    // ==================== STEP 10: Ingest log #4 and verify ingestion-time REDACT ====================
    testLogger.info('========== STEP 10: Ingest log with value "25/12/2024" ==========');
    const markerLog4 = await ingestSingleLog(page, testStreamName, fieldName, ingestionTimePatterns[1].value); // Date
    await pm.sdrVerificationPage.verifySingleFieldInLatestLog(pm.logsPage, testStreamName, fieldName, false, true, markerLog4);
    testLogger.info('STEP 10 PASSED: Field REDACTED at ingestion time');

    testLogger.info('=== ALL 4 PATTERNS ON ONE FIELD TEST COMPLETED SUCCESSFULLY ===');
    testLogger.info('  - Field: multi_data has 4 patterns');
    testLogger.info('  - Query-time: 1 drop + 1 redact (work on existing data)');
    testLogger.info('  - Ingestion-time: 1 drop + 1 redact (require re-ingestion)');
    testLogger.info('  - Each pattern tested independently with sequential ingestion');
  });
});