create static method

Future<ConcurrentWebScraper> create({
  1. required ProxyManager proxyManager,
  2. int maxConcurrentTasks = 5,
  3. int defaultTimeout = 30000,
  4. int maxRetries = 3,
  5. bool respectRobotsTxt = true,
})

Factory constructor to create a ConcurrentWebScraper with default components

Implementation

static Future<ConcurrentWebScraper> create({
  required ProxyManager proxyManager,
  int maxConcurrentTasks = 5,
  int defaultTimeout = 30000,
  int maxRetries = 3,
  bool respectRobotsTxt = true,
}) async {
  final logger = ScrapingLogger();
  final robotsTxtHandler = RobotsTxtHandler(
    proxyManager: proxyManager,
    logger: logger,
    respectRobotsTxt: respectRobotsTxt,
  );
  final streamingParser = StreamingHtmlParser(logger: logger);

  return ConcurrentWebScraper(
    proxyManager: proxyManager,
    maxConcurrentTasks: maxConcurrentTasks,
    defaultTimeout: defaultTimeout,
    maxRetries: maxRetries,
    logger: logger,
    robotsTxtHandler: robotsTxtHandler,
    streamingParser: streamingParser,
    respectRobotsTxt: respectRobotsTxt,
  );
}