getOptimalHeaders method
Gets the optimal headers for the given URL based on past success
Implementation
Map<String, String> getOptimalHeaders(
String url,
Map<String, String> defaultHeaders,
) {
final domain = _extractDomain(url);
if (domain == null) return defaultHeaders;
if (_siteReputations.containsKey(domain)) {
final reputation = _siteReputations[domain]!;
// If the site has a low success rate, add more browser-like headers
if (reputation.successRate < 0.7) {
final enhancedHeaders = Map<String, String>.from(defaultHeaders);
enhancedHeaders['Accept'] =
'text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8';
enhancedHeaders['Accept-Language'] = 'en-US,en;q=0.5';
enhancedHeaders['Connection'] = 'keep-alive';
enhancedHeaders['Upgrade-Insecure-Requests'] = '1';
enhancedHeaders['Cache-Control'] = 'max-age=0';
// Add more headers based on the specific error patterns
if (reputation.hasErrorPattern('timeout')) {
enhancedHeaders['Keep-Alive'] = 'timeout=15, max=100';
}
if (reputation.hasErrorPattern('ssl') ||
reputation.hasErrorPattern('certificate')) {
enhancedHeaders['Sec-Fetch-Dest'] = 'document';
enhancedHeaders['Sec-Fetch-Mode'] = 'navigate';
enhancedHeaders['Sec-Fetch-Site'] = 'none';
enhancedHeaders['Sec-Fetch-User'] = '?1';
}
return enhancedHeaders;
}
}
return defaultHeaders;
}