scrapeWithChunking<T> method

Future<T> scrapeWithChunking<T>({
  1. required String url,
  2. required DataChunker dataChunker,
  3. required FutureOr<T> processor(
    1. String chunk,
    2. T? previousResult
    ),
  4. Map<String, String>? headers,
  5. int? timeout,
  6. int? retries,
  7. T? initialResult,
})

Scrapes a URL with chunked processing for large HTML documents

Implementation

Future<T> scrapeWithChunking<T>({
  required String url,
  required DataChunker dataChunker,
  required FutureOr<T> Function(String chunk, T? previousResult) processor,
  Map<String, String>? headers,
  int? timeout,
  int? retries,
  T? initialResult,
}) async {
  // Fetch the HTML
  final html = await fetchHtml(
    url: url,
    headers: headers,
    timeout: timeout,
    retries: retries,
  );

  // Process the HTML in chunks
  return await dataChunker.processStringInChunks<T>(
    data: html,
    processor: processor,
    initialResult: initialResult,
  );
}