getOptimalHeaders method

Map<String, String> getOptimalHeaders(
  1. String url,
  2. Map<String, String> defaultHeaders
)

Gets the optimal headers for the given URL based on past success

Implementation

Map<String, String> getOptimalHeaders(
  String url,
  Map<String, String> defaultHeaders,
) {
  final domain = _extractDomain(url);
  if (domain == null) return defaultHeaders;

  if (_siteReputations.containsKey(domain)) {
    final reputation = _siteReputations[domain]!;

    // If the site has a low success rate, add more browser-like headers
    if (reputation.successRate < 0.7) {
      final enhancedHeaders = Map<String, String>.from(defaultHeaders);
      enhancedHeaders['Accept'] =
          'text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8';
      enhancedHeaders['Accept-Language'] = 'en-US,en;q=0.5';
      enhancedHeaders['Connection'] = 'keep-alive';
      enhancedHeaders['Upgrade-Insecure-Requests'] = '1';
      enhancedHeaders['Cache-Control'] = 'max-age=0';

      // Add more headers based on the specific error patterns
      if (reputation.hasErrorPattern('timeout')) {
        enhancedHeaders['Keep-Alive'] = 'timeout=15, max=100';
      }

      if (reputation.hasErrorPattern('ssl') ||
          reputation.hasErrorPattern('certificate')) {
        enhancedHeaders['Sec-Fetch-Dest'] = 'document';
        enhancedHeaders['Sec-Fetch-Mode'] = 'navigate';
        enhancedHeaders['Sec-Fetch-Site'] = 'none';
        enhancedHeaders['Sec-Fetch-User'] = '?1';
      }

      return enhancedHeaders;
    }
  }

  return defaultHeaders;
}