getRules method

Future<RobotsTxtRules?> getRules(
  1. String domain
)

Gets the robots.txt rules for a domain

domain is the domain to get rules for

Implementation

Future<RobotsTxtRules?> getRules(String domain) async {
  if (!_respectRobotsTxt) {
    return RobotsTxtRules.empty();
  }

  // Check if we need to fetch the robots.txt file
  if (!_cachedRules.containsKey(domain) ||
      _cachedRules[domain]!.isExpired(_cacheExpirationMs)) {
    try {
      await _fetchRobotsTxt(domain);
    } catch (e) {
      _logger.error('Error fetching robots.txt for $domain: $e');
      // If we can't fetch the robots.txt file, return null
      return null;
    }
  }

  return _cachedRules[domain];
}