handleLazyLoadingSite method

Implementation

Future<HeadlessBrowserResult> handleLazyLoadingSite(
  String url, {
  Map<String, String>? headers,
  Map<String, String>? selectors,
  Map<String, String>? attributes,
  int? timeoutMillis,
  int scrollCount = 5,
  int scrollDelay = 1000,
}) async {
  _logger.info('Handling lazy loading site: $url');

  final result = await _service.scrapeUrl(
    url,
    headers: headers,
    timeoutMillis: timeoutMillis,
  );

  if (!result.success) {
    return result;
  }

  // Scroll down to trigger lazy loading
  for (int i = 0; i < scrollCount; i++) {
    await _service.executeScript('''
      window.scrollTo({
        top: document.body.scrollHeight * ${(i + 1) / scrollCount},
        behavior: 'smooth'
      });
    ''');

    await Future.delayed(Duration(milliseconds: scrollDelay));
  }

  // Scroll back to top
  await _service.executeScript('window.scrollTo(0, 0);');

  // Extract data if selectors provided
  Map<String, dynamic>? extractedData;
  if (selectors != null && selectors.isNotEmpty) {
    extractedData = await _service.executeScript('''
      (function() {
        const result = {};
        ${selectors.entries.map((entry) {
      final key = entry.key;
      final selector = entry.value;
      final attribute = attributes?[key];

      if (attribute != null) {
        return '''
              result["$key"] = Array.from(document.querySelectorAll('$selector'))
                .map(el => el.getAttribute('$attribute'))
                .filter(val => val !== null);
            ''';
      } else {
        return '''
              result["$key"] = Array.from(document.querySelectorAll('$selector'))
                .map(el => el.textContent.trim())
                .filter(val => val !== "");
            ''';
      }
    }).join('\n')}
        return result;
      })();
    ''');
  }

  // Get updated HTML
  final html = await _service.executeScript(
    'document.documentElement.outerHTML',
  );

  return HeadlessBrowserResult.success(
    html: html,
    data: extractedData,
    elapsedMillis: result.elapsedMillis,
  );
}