selenium3.0不用代理的情况下，获取异步请求的数据

大家好，又见面了，我是你们的朋友全栈君。如果您正在找激活码,请点击查看最新教程,关注关注公众号 “全栈程序员社区” 获取激活教程,可能之前旧版本教程已经失效.最新Idea2022.1教程亲测有效,一键激活。

Jetbrains全系列IDE使用 1年只要46元 售后保障 童叟无欺

最近爬取一个网站的时候，反爬比较厉害，各种弹窗，各种验证码，无限debugger，关键数据是ajax请求异步加载的。使用代理绕过前面几种反爬后，获取ajax的request和response成了头疼的问题，最终使用selenium的network日志分析来解决。为了方便以后使用，

目录

工具类：

使用方法：

控制台：

​使用代理：

工具类：

import json
from selenium import webdriver
from selenium.webdriver import DesiredCapabilities


def get_xhr_logs(chrome):
    log_xhr_array = []
    for typelog in chrome.log_types:
        perfs = chrome.get_log(typelog)
        for row in perfs:
            log_data = row
            message_ = log_data['message']
            try:
                log_json = json.loads(message_)
                log = log_json['message']
                if log['method'] == 'Network.responseReceived':
                    # 去掉静态js、css等，仅保留xhr请求
                    type_ = log['params']['type']
                    if type_ == "XHR":
                        log_xhr_array.append(log)
            except:
                pass
    return log_xhr_array


def get_log_options():
    option = webdriver.ChromeOptions()
    option.add_argument('--no-sandbox')
    option.add_argument('--headless')
    option.add_argument("--disable-extensions")
    option.add_argument("--allow-running-insecure-content")
    option.add_argument("--ignore-certificate-errors")
    option.add_argument("--disable-single-click-autofill")
    option.add_argument("--disable-autofill-keyboard-accessory-view[8]")
    option.add_argument("--disable-full-form-autofill-ios")
    option.add_argument('user-agent=Mozilla/5.0 (Macintosh; Intel Mac OS X 10.14; rv:55.0) Gecko/20100101 Firefox/55.0')
    option.add_experimental_option('w3c', False)
    option.add_experimental_option('perfLoggingPrefs', {
        'enableNetwork': True,
        'enablePage': False,
    })
    return option


def get_caps():
    caps = DesiredCapabilities.CHROME
    caps['loggingPrefs'] = {
        'browser': 'ALL',
        'performance': 'ALL',
    }
    caps['perfLoggingPrefs'] = {
        'enableNetwork': True,
        'enablePage': False,
        'enableTimeline': False
    }
    return caps

使用方法：

以访问百度一下，你就知道为例：

import json
import os
import time

# 导入工具类
from myscrapy.network_log_option import *

if __name__ == '__main__':
    # 使用工具类来获取options配置，而不是平时的webdriver.ChromeOptions()方法
    options = get_log_options()
    # 使用工具类来获取caps
    desired_capabilities = get_caps()
    # 这里也可以对options和caps加入其他的参数，比如代理参数等
    chrome = webdriver.Chrome(options=options, desired_capabilities=desired_capabilities)
    chrome.get("https://www.baidu.com/")
    chrome.maximize_window()
    # 用工具类来获取ajax请求日志
    logs = get_xhr_logs(chrome)
    for log in logs:
        print(log)
    chrome.quit()

控制台：

结果对应了百度的两条ajax请求：

C:\Users\Lenovo\AppData\Local\Programs\Python\Python39\python.exe D:/pythonwork/myscrapy/myscrapy/myscrapy/test002.py
{'method': 'Network.responseReceived', 'params': {'frameId': 'E6B2F71F5881A0C139C6405650B9B287', 'loaderId': 'AC357BD1718137E4CDBE450B8C470D1B', 'requestId': '39148.67', 'response': {'connectionId': 76, 'connectionReused': True, 'encodedDataLength': 117, 'fromDiskCache': False, 'fromPrefetchCache': False, 'fromServiceWorker': False, 'headers': {'Content-Length': '53', 'Content-Type': 'text/plain; charset=UTF-8', 'Date': 'Wed, 23 Jun 2021 02:35:04 GMT'}, 'headersText': 'HTTP/1.1 200 OK\r\nContent-Length: 53\r\nContent-Type: text/plain; charset=UTF-8\r\nDate: Wed, 23 Jun 2021 02:35:04 GMT\r\n\r\n', 'mimeType': 'text/plain', 'protocol': 'http/1.1', 'remoteIPAddress': '112.80.248.75', 'remotePort': 443, 'requestHeaders': {'Accept': 'application/json, text/javascript, */*; q=0.01', 'Accept-Encoding': 'gzip, deflate, br', 'Accept-Language': 'zh-CN', 'Connection': 'keep-alive', 'Cookie': 'BIDUPSID=5B2A6601F8BFCE8B33606C7DE9B03925; PSTM=1624415703; BAIDUID=5B2A6601F8BFCE8B846CF739704B0464:FG=1; BD_HOME=1; H_PS_PSSID=34132_34099_33970_31254_34004_33607_34106_34135_34119_26350; BD_UPN=13314652; BA_HECTOR=0ta4ah0l4l008lc5361gd57eo0q', 'Host': 'www.baidu.com', 'Referer': 'https://www.baidu.com/', 'Sec-Fetch-Dest': 'empty', 'Sec-Fetch-Mode': 'cors', 'Sec-Fetch-Site': 'same-origin', 'User-Agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10.14; rv:55.0) Gecko/20100101 Firefox/55.0', 'sec-ch-ua': '', 'sec-ch-ua-mobile': '?0'}, 'requestHeadersText': 'GET /sugrec?prod=pc_his&from=pc_web&json=1&sid=34132_34099_33970_31254_34004_33607_34106_34135_34119_26350&hisdata=&_t=1624415704263&csor=0 HTTP/1.1\r\nHost: www.baidu.com\r\nConnection: keep-alive\r\nsec-ch-ua:\r\nAccept: application/json, text/javascript, */*; q=0.01\r\nsec-ch-ua-mobile: ?0\r\nUser-Agent: Mozilla/5.0 (Macintosh; Intel Mac OS X 10.14; rv:55.0) Gecko/20100101 Firefox/55.0\r\nSec-Fetch-Site: same-origin\r\nSec-Fetch-Mode: cors\r\nSec-Fetch-Dest: empty\r\nReferer: https://www.baidu.com/\r\nAccept-Encoding: gzip, deflate, br\r\nAccept-Language: zh-CN\r\nCookie: BIDUPSID=5B2A6601F8BFCE8B33606C7DE9B03925; PSTM=1624415703; BAIDUID=5B2A6601F8BFCE8B846CF739704B0464:FG=1; BD_HOME=1; H_PS_PSSID=34132_34099_33970_31254_34004_33607_34106_34135_34119_26350; BD_UPN=13314652; BA_HECTOR=0ta4ah0l4l008lc5361gd57eo0q\r\n', 'responseTime': 1624415704296.042, 'securityDetails': {'certificateId': 0, 'certificateTransparencyCompliance': 'unknown', 'cipher': 'AES_128_GCM', 'issuer': 'GlobalSign Organization Validation CA - SHA256 - G2', 'keyExchange': 'ECDHE_RSA', 'keyExchangeGroup': 'P-256', 'protocol': 'TLS 1.2', 'sanList': ['baidu.com', 'baifubao.com', 'www.baidu.cn', 'www.baidu.com.cn', 'mct.y.nuomi.com', 'apollo.auto', 'dwz.cn', '*.baidu.com', '*.baifubao.com', '*.baidustatic.com', '*.bdstatic.com', '*.bdimg.com', '*.hao123.com', '*.nuomi.com', '*.chuanke.com', '*.trustgo.com', '*.bce.baidu.com', '*.eyun.baidu.com', '*.map.baidu.com', '*.mbd.baidu.com', '*.fanyi.baidu.com', '*.baidubce.com', '*.mipcdn.com', '*.news.baidu.com', '*.baidupcs.com', '*.aipage.com', '*.aipage.cn', '*.bcehost.com', '*.safe.baidu.com', '*.im.baidu.com', '*.baiducontent.com', '*.dlnel.com', '*.dlnel.org', '*.dueros.baidu.com', '*.su.baidu.com', '*.91.com', '*.hao123.baidu.com', '*.apollo.auto', '*.xueshu.baidu.com', '*.bj.baidubce.com', '*.gz.baidubce.com', '*.smartapps.cn', '*.bdtjrcv.com', '*.hao222.com', '*.haokan.com', '*.pae.baidu.com', '*.vd.bdstatic.com', 'click.hm.baidu.com', 'log.hm.baidu.com', 'cm.pos.baidu.com', 'wn.pos.baidu.com', 'update.pan.baidu.com'], 'signedCertificateTimestampList': [], 'subjectName': 'baidu.com', 'validFrom': 1585811098, 'validTo': 1627277462}, 'securityState': 'secure', 'status': 200, 'statusText': 'OK', 'timing': {'connectEnd': -1, 'connectStart': -1, 'dnsEnd': -1, 'dnsStart': -1, 'proxyEnd': 0.455, 'proxyStart': 0.185, 'pushEnd': 0, 'pushStart': 0, 'receiveHeadersEnd': 31.722, 'requestTime': 325463.079675, 'sendEnd': 0.705, 'sendStart': 0.523, 'sslEnd': -1, 'sslStart': -1, 'workerFetchStart': -1, 'workerReady': -1, 'workerRespondWithSettled': -1, 'workerStart': -1}, 'url': 'https://www.baidu.com/sugrec?prod=pc_his&from=pc_web&json=1&sid=34132_34099_33970_31254_34004_33607_34106_34135_34119_26350&hisdata=&_t=1624415704263&csor=0'}, 'timestamp': 325463.112169, 'type': 'XHR'}}
{'method': 'Network.responseReceived', 'params': {'frameId': 'E6B2F71F5881A0C139C6405650B9B287', 'loaderId': 'AC357BD1718137E4CDBE450B8C470D1B', 'requestId': '39148.68', 'response': {'connectionId': 15, 'connectionReused': True, 'encodedDataLength': 360, 'fromDiskCache': False, 'fromPrefetchCache': False, 'fromServiceWorker': False, 'headers': {'Accept-Ranges': 'bytes', 'Cache-Control': 'max-age=315360000', 'Content-Encoding': 'gzip', 'Content-Length': '7112', 'Content-Type': 'application/javascript', 'Date': 'Wed, 23 Jun 2021 02:35:04 GMT', 'Etag': '"4451-4fdbd6734c340"', 'Expires': 'Sat, 21 Jun 2031 02:35:04 GMT', 'Last-Modified': 'Wed, 09 Jul 2014 07:10:29 GMT', 'Server': 'Apache', 'Vary': 'Accept-Encoding,User-Agent'}, 'headersText': 'HTTP/1.1 200 OK\r\nAccept-Ranges: bytes\r\nCache-Control: max-age=315360000\r\nContent-Encoding: gzip\r\nContent-Length: 7112\r\nContent-Type: application/javascript\r\nDate: Wed, 23 Jun 2021 02:35:04 GMT\r\nEtag: "4451-4fdbd6734c340"\r\nExpires: Sat, 21 Jun 2031 02:35:04 GMT\r\nLast-Modified: Wed, 09 Jul 2014 07:10:29 GMT\r\nServer: Apache\r\nVary: Accept-Encoding,User-Agent\r\n\r\n', 'mimeType': 'application/javascript', 'protocol': 'http/1.1', 'remoteIPAddress': '112.80.248.75', 'remotePort': 443, 'requestHeaders': {'Accept': 'text/javascript, application/javascript, application/ecmascript, application/x-ecmascript, */*; q=0.01', 'Accept-Encoding': 'gzip, deflate, br', 'Accept-Language': 'zh-CN', 'Connection': 'keep-alive', 'Cookie': 'BIDUPSID=5B2A6601F8BFCE8B33606C7DE9B03925; PSTM=1624415703; BAIDUID=5B2A6601F8BFCE8B846CF739704B0464:FG=1; BD_HOME=1; H_PS_PSSID=34132_34099_33970_31254_34004_33607_34106_34135_34119_26350; BD_UPN=13314652; BA_HECTOR=0ta4ah0l4l008lc5361gd57eo0q', 'Host': 'www.baidu.com', 'Referer': 'https://www.baidu.com/', 'Sec-Fetch-Dest': 'empty', 'Sec-Fetch-Mode': 'cors', 'Sec-Fetch-Site': 'same-origin', 'User-Agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10.14; rv:55.0) Gecko/20100101 Firefox/55.0', 'X-Requested-With': 'XMLHttpRequest', 'sec-ch-ua': '', 'sec-ch-ua-mobile': '?0'}, 'requestHeadersText': 'GET /cache/fpid/chromelib_1_1.js?_=1624415704024 HTTP/1.1\r\nHost: www.baidu.com\r\nConnection: keep-alive\r\nsec-ch-ua:\r\nAccept: text/javascript, application/javascript, application/ecmascript, application/x-ecmascript, */*; q=0.01\r\nX-Requested-With: XMLHttpRequest\r\nsec-ch-ua-mobile: ?0\r\nUser-Agent: Mozilla/5.0 (Macintosh; Intel Mac OS X 10.14; rv:55.0) Gecko/20100101 Firefox/55.0\r\nSec-Fetch-Site: same-origin\r\nSec-Fetch-Mode: cors\r\nSec-Fetch-Dest: empty\r\nReferer: https://www.baidu.com/\r\nAccept-Encoding: gzip, deflate, br\r\nAccept-Language: zh-CN\r\nCookie: BIDUPSID=5B2A6601F8BFCE8B33606C7DE9B03925; PSTM=1624415703; BAIDUID=5B2A6601F8BFCE8B846CF739704B0464:FG=1; BD_HOME=1; H_PS_PSSID=34132_34099_33970_31254_34004_33607_34106_34135_34119_26350; BD_UPN=13314652; BA_HECTOR=0ta4ah0l4l008lc5361gd57eo0q\r\n', 'responseTime': 1624415704513.065, 'securityDetails': {'certificateId': 0, 'certificateTransparencyCompliance': 'unknown', 'cipher': 'AES_128_GCM', 'issuer': 'GlobalSign Organization Validation CA - SHA256 - G2', 'keyExchange': 'ECDHE_RSA', 'keyExchangeGroup': 'P-256', 'protocol': 'TLS 1.2', 'sanList': ['baidu.com', 'baifubao.com', 'www.baidu.cn', 'www.baidu.com.cn', 'mct.y.nuomi.com', 'apollo.auto', 'dwz.cn', '*.baidu.com', '*.baifubao.com', '*.baidustatic.com', '*.bdstatic.com', '*.bdimg.com', '*.hao123.com', '*.nuomi.com', '*.chuanke.com', '*.trustgo.com', '*.bce.baidu.com', '*.eyun.baidu.com', '*.map.baidu.com', '*.mbd.baidu.com', '*.fanyi.baidu.com', '*.baidubce.com', '*.mipcdn.com', '*.news.baidu.com', '*.baidupcs.com', '*.aipage.com', '*.aipage.cn', '*.bcehost.com', '*.safe.baidu.com', '*.im.baidu.com', '*.baiducontent.com', '*.dlnel.com', '*.dlnel.org', '*.dueros.baidu.com', '*.su.baidu.com', '*.91.com', '*.hao123.baidu.com', '*.apollo.auto', '*.xueshu.baidu.com', '*.bj.baidubce.com', '*.gz.baidubce.com', '*.smartapps.cn', '*.bdtjrcv.com', '*.hao222.com', '*.haokan.com', '*.pae.baidu.com', '*.vd.bdstatic.com', 'click.hm.baidu.com', 'log.hm.baidu.com', 'cm.pos.baidu.com', 'wn.pos.baidu.com', 'update.pan.baidu.com'], 'signedCertificateTimestampList': [], 'subjectName': 'baidu.com', 'validFrom': 1585811098, 'validTo': 1627277462}, 'securityState': 'secure', 'status': 200, 'statusText': 'OK', 'timing': {'connectEnd': -1, 'connectStart': -1, 'dnsEnd': -1, 'dnsStart': -1, 'proxyEnd': 1, 'proxyStart': 0.65, 'pushEnd': 0, 'pushStart': 0, 'receiveHeadersEnd': 53.016, 'requestTime': 325463.276445, 'sendEnd': 1.227, 'sendStart': 1.08, 'sslEnd': -1, 'sslStart': -1, 'workerFetchStart': -1, 'workerReady': -1, 'workerRespondWithSettled': -1, 'workerStart': -1}, 'url': 'https://www.baidu.com/cache/fpid/chromelib_1_1.js?_=1624415704024'}, 'timestamp': 325463.33218, 'type': 'XHR'}}

使用代理：

比如，使用chrom的远程调试代理：

import json
import os
import time
from myscrapy.network_log_option import *
if __name__ == '__main__':
# 启动代理命令,打开代理浏览器窗口
cd_dir = r'cd C:\Users\Lenovo\AppData\Local\Google\Chrome\Application'
start_proxy = r'chrome.exe --remote-debugging-port=9222 --user-data-dir="C:\selenum\AutomationProfile"'
os.system(cd_dir)
time.sleep(0.5)
os.system(start_proxy)
# 使用工具类来获取options配置，而不是平时的webdriver.ChromeOptions()方法
options = get_log_options()
# 配置代理参数
options.add_experimental_option('debuggerAddress', 'localhost:90222')
# 使用工具类来获取caps
desired_capabilities = get_caps()
# 这里也可以对options和caps加入其他的参数，比如代理参数等
chrome = webdriver.Chrome(options=options, desired_capabilities=desired_capabilities)
chrome.get("https://www.baidu.com/")
chrome.maximize_window()
# 用工具类来获取ajax请求日志
logs = get_xhr_logs(chrome)
for log in logs:
print(log)
chrome.quit()