getCrawlDelay method

int? getCrawlDelay(
  1. String userAgent
)

Gets the crawl delay for a user agent in milliseconds

Implementation

int? getCrawlDelay(String userAgent) {
  final userAgentLower = userAgent.toLowerCase();
  final specificAgent = _findSpecificUserAgent(userAgentLower);

  if (specificAgent != null && _crawlDelays.containsKey(specificAgent)) {
    return _crawlDelays[specificAgent]! * 1000; // Convert to milliseconds
  }

  // Check wildcard
  if (_crawlDelays.containsKey('*')) {
    return _crawlDelays['*']! * 1000; // Convert to milliseconds
  }

  return null;
}