createTaskScheduler method

TaskScheduler createTaskScheduler({
  1. TaskSchedulerConfig? config,
  2. ResourceMonitor? resourceMonitor,
  3. Logger? logger,
})

Creates a task scheduler for parallel scraping

Implementation

TaskScheduler createTaskScheduler({
  TaskSchedulerConfig? config,
  ResourceMonitor? resourceMonitor,
  Logger? logger,
}) {
  // Create a resource monitor if not provided
  final monitor = resourceMonitor ?? ResourceMonitor(logger: logger);

  // Start the resource monitor
  monitor.start();

  // Create the task scheduler
  final scheduler = TaskScheduler(
    rateLimiter: rateLimiter,
    resourceMonitor: monitor,
    config: config ?? TaskSchedulerConfig(),
    logger: logger,
  );

  // Start the scheduler
  scheduler.start();

  return scheduler;
}