EnhancedRateLimiter constructor
EnhancedRateLimiter({
- int defaultDelayMs = 1000,
- Map<
String, int> ? domainDelays, - int maxRetries = 3,
- int initialBackoffMs = 1000,
- int maxBackoffMs = 60000,
- double backoffMultiplier = 2.0,
- RobotsTxtHandler? robotsTxtHandler,
- ScrapingLogger? logger,
Creates a new EnhancedRateLimiter with the given parameters
defaultDelayMs
is the default delay between requests to the same domain
domainDelays
is a map of domains to their custom delays
maxRetries
is the maximum number of retry attempts for rate-limited requests
initialBackoffMs
is the initial backoff time in milliseconds
maxBackoffMs
is the maximum backoff time in milliseconds
backoffMultiplier
is the backoff multiplier for exponential backoff
robotsTxtHandler
is the robots.txt handler for getting crawl delays
logger
is the logger for logging operations
Implementation
EnhancedRateLimiter({
int defaultDelayMs = 1000,
Map<String, int>? domainDelays,
int maxRetries = 3,
int initialBackoffMs = 1000,
int maxBackoffMs = 60000,
double backoffMultiplier = 2.0,
RobotsTxtHandler? robotsTxtHandler,
ScrapingLogger? logger,
}) : _defaultDelayMs = defaultDelayMs,
_domainDelays = domainDelays ?? {},
_maxRetries = maxRetries,
_initialBackoffMs = initialBackoffMs,
_maxBackoffMs = maxBackoffMs,
_backoffMultiplier = backoffMultiplier,
_robotsTxtHandler = robotsTxtHandler,
_logger = logger ?? ScrapingLogger();