You signed in with another tab or window. Reload to refresh your session.You signed out in another tab or window. Reload to refresh your session.You switched accounts on another tab or window. Reload to refresh your session.Dismiss alert
I'm trying to use the impersonation with a set of private ja3 / akamai / headers. But I think from my understanding of curl_cffi and curl-impersonate that when using ja3 and akamai parameter the header logic is not used.
I'd like to know how we can set the order of the headers with the ja3/ akamai parameter ( including automatic headers like host)
Edit, i tried the following code but it is not working at all, i'm getting flagged by the site i'm trying to scrape.
CHROME_HEADER_ORDER = [
'Host',
'Connection',
'Content-Length',
'sec-ch-ua',
'sec-ch-ua-mobile',
'sec-ch-ua-platform',
'Upgrade-Insecure-Requests',
'User-Agent',
'Accept',
'Sec-Fetch-Site',
'Sec-Fetch-Mode',
'Sec-Fetch-User',
'Sec-Fetch-Dest',
'Accept-Encoding',
'Accept-Language',
'Cookie',
'Referer',
'Origin',
'Content-Type',
'Cache-Control',
'Pragma'
]
# Safari's standard header order
SAFARI_HEADER_ORDER = [
'Host',
'Accept',
'User-Agent',
'Accept-Language',
'Accept-Encoding',
'Connection',
'Cookie',
'Referer',
'Upgrade-Insecure-Requests',
'Content-Type',
'Content-Length',
'Origin',
'Cache-Control',
'Pragma'
]
# Edge's standard header order (based on Chromium but with some differences)
EDGE_HEADER_ORDER = [
'Host',
'Connection',
'Content-Length',
'sec-ch-ua',
'sec-ch-ua-mobile',
'sec-ch-ua-platform',
'User-Agent',
'Accept',
'Accept-Encoding',
'Accept-Language',
'Sec-Fetch-Site',
'Sec-Fetch-Mode',
'Sec-Fetch-User',
'Sec-Fetch-Dest',
'Upgrade-Insecure-Requests',
'Cookie',
'Referer',
'Origin',
'Content-Type',
'Cache-Control'
]
def _reorder_headers(self, headers: Dict[str, str], browser_type: str = 'chrome') -> Dict[str, str]:
"""
Reorder headers according to the specified browser's standard order
Args:
headers: Dictionary of headers to reorder
browser_type: Type of browser to use for header ordering ('chrome', 'safari', 'edge')
Returns:
Dict[str, str]: Reordered headers dictionary
"""
# Get the header order for the specified browser (default to Chrome if not found)
header_order = BROWSER_HEADER_ORDERS.get(browser_type.lower(), CHROME_HEADER_ORDER)
# Create a new ordered dictionary for the result
ordered_headers = {}
# First add headers that exist in our order list
for header_name in header_order:
if header_name.lower() in [k.lower() for k in headers.keys()]:
# Find the actual case-sensitive key from the original headers
original_key = next(k for k in headers.keys() if k.lower() == header_name.lower())
ordered_headers[original_key] = headers[original_key]
# Then add any remaining headers that weren't in our order list
for key, value in headers.items():
if key not in ordered_headers:
ordered_headers[key] = value
return ordered_headers
fingerprint = random.choice(browserFingerprints)
headers = fingerprint['headers']
headers['Accept-Language'] = random.choice(accept_languages)
headers['Referer'] = referer
# Determine browser type from User-Agent
browser_type = 'chrome' # Default to Chrome
ua_lower = current_user_agent.lower()
if 'safari' in ua_lower and 'chrome' not in ua_lower:
browser_type = 'safari'
elif 'edg/' in ua_lower:
browser_type = 'edge'
headers = self._reorder_headers(headers, browser_type)
ja3 = fingerprint['ja3']
akamai = fingerprint['akamai']
extra_fp = {
'tls_permute_extensions': fingerprint['support_randomization'],
'tls_grease': fingerprint['support_randomization']
}
if method == 'GET':
response = await current_session.get(url,ja3=ja3,akamai=akamai, **kwargs,
headers=headers,extra_fp=extra_fp,default_headers=False)
else:
response = await current_session.post(url,ja3=ja3,akamai=akamai, **kwargs,
headers=headers,extra_fp=extra_fp,default_headers=False)
Thanks in advance.
The text was updated successfully, but these errors were encountered:
I'm trying to use the impersonation with a set of private ja3 / akamai / headers. But I think from my understanding of curl_cffi and curl-impersonate that when using ja3 and akamai parameter the header logic is not used.
I'd like to know how we can set the order of the headers with the ja3/ akamai parameter ( including automatic headers like host)
Edit, i tried the following code but it is not working at all, i'm getting flagged by the site i'm trying to scrape.
Thanks in advance.
The text was updated successfully, but these errors were encountered: