AdaptiveScrapingStrategy constructor

AdaptiveScrapingStrategy({
  1. SiteReputationTracker? reputationTracker,
  2. ScrapingStrategy? defaultStrategy,
})

Creates a new AdaptiveScrapingStrategy

Implementation

AdaptiveScrapingStrategy({
  SiteReputationTracker? reputationTracker,
  ScrapingStrategy? defaultStrategy,
}) : _reputationTracker = reputationTracker ?? SiteReputationTracker(),
     _defaultStrategy =
         defaultStrategy ??
         ScrapingStrategy(
           retries: 3,
           timeout: 30000,
           headers: {
             'User-Agent':
                 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.124 Safari/537.36',
             'Accept':
                 'text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8',
           },
           initialBackoff: 1000,
           backoffMultiplier: 1.5,
           maxBackoff: 10000,
           useRandomUserAgent: false,
           rotateProxiesOnRetry: true,
           validateProxies: true,
         );