diff --git a/run_spider.sh b/run_spider.sh index d22be7d..113ab70 100755 --- a/run_spider.sh +++ b/run_spider.sh @@ -13,7 +13,7 @@ ############################################## export export TERM=linux source /home/hadoop/.zshrc -v env0 +v env1 ############################################## ## additional files list diff --git a/spider/mspider/ghostdriver.log b/spider/mspider/ghostdriver.log index c2becb3..e0c0c37 100644 --- a/spider/mspider/ghostdriver.log +++ b/spider/mspider/ghostdriver.log @@ -1,617 +1,379 @@ -2015-03-30T15:39:09 [DEBUG] CookieJar - Created but will not store cookies (use option '--cookies-file=' to enable persisten cookie storage) -2015-03-30T15:39:09 [DEBUG] Phantom - execute: Configuration -2015-03-30T15:39:09 [DEBUG] 0 objectName : "" -2015-03-30T15:39:09 [DEBUG] 1 cookiesFile : "" -2015-03-30T15:39:09 [DEBUG] 2 diskCacheEnabled : "false" -2015-03-30T15:39:09 [DEBUG] 3 maxDiskCacheSize : "-1" -2015-03-30T15:39:09 [DEBUG] 4 ignoreSslErrors : "false" -2015-03-30T15:39:09 [DEBUG] 5 localToRemoteUrlAccessEnabled : "false" -2015-03-30T15:39:09 [DEBUG] 6 outputEncoding : "UTF-8" -2015-03-30T15:39:09 [DEBUG] 7 proxyType : "http" -2015-03-30T15:39:09 [DEBUG] 8 proxy : ":8" -2015-03-30T15:39:09 [DEBUG] 9 proxyAuth : ":" -2015-03-30T15:39:09 [DEBUG] 10 scriptEncoding : "UTF-8" -2015-03-30T15:39:09 [DEBUG] 11 webSecurityEnabled : "true" -2015-03-30T15:39:09 [DEBUG] 12 offlineStoragePath : "" -2015-03-30T15:39:09 [DEBUG] 13 offlineStorageDefaultQuota : "-1" -2015-03-30T15:39:09 [DEBUG] 14 printDebugMessages : "true" -2015-03-30T15:39:09 [DEBUG] 15 javascriptCanOpenWindows : "true" -2015-03-30T15:39:09 [DEBUG] 16 javascriptCanCloseWindows : "true" -2015-03-30T15:39:09 [DEBUG] 17 sslProtocol : "sslv3" -2015-03-30T15:39:09 [DEBUG] 18 sslCertificatesPath : "" -2015-03-30T15:39:09 [DEBUG] 19 webdriver : ":58200" -2015-03-30T15:39:09 [DEBUG] 20 webdriverLogFile : "" -2015-03-30T15:39:09 [DEBUG] 21 webdriverLogLevel : "debug" -2015-03-30T15:39:09 [DEBUG] 22 webdriverSeleniumGridHub : "" -2015-03-30T15:39:09 [DEBUG] Phantom - execute: Script & Arguments -2015-03-30T15:39:09 [DEBUG] script: "main.js" -2015-03-30T15:39:09 [DEBUG] 0 arg: "--ip=" -2015-03-30T15:39:09 [DEBUG] 1 arg: "--port=58200" -2015-03-30T15:39:09 [DEBUG] 2 arg: "--logLevel=debug" -2015-03-30T15:39:09 [DEBUG] Phantom - execute: Starting Remote WebDriver mode +2015-03-30T16:18:22 [DEBUG] CookieJar - Created but will not store cookies (use option '--cookies-file=' to enable persisten cookie storage) +2015-03-30T16:18:22 [DEBUG] Phantom - execute: Configuration +2015-03-30T16:18:22 [DEBUG] 0 objectName : "" +2015-03-30T16:18:22 [DEBUG] 1 cookiesFile : "" +2015-03-30T16:18:22 [DEBUG] 2 diskCacheEnabled : "false" +2015-03-30T16:18:22 [DEBUG] 3 maxDiskCacheSize : "-1" +2015-03-30T16:18:22 [DEBUG] 4 ignoreSslErrors : "false" +2015-03-30T16:18:22 [DEBUG] 5 localToRemoteUrlAccessEnabled : "false" +2015-03-30T16:18:22 [DEBUG] 6 outputEncoding : "UTF-8" +2015-03-30T16:18:22 [DEBUG] 7 proxyType : "http" +2015-03-30T16:18:22 [DEBUG] 8 proxy : ":1080" +2015-03-30T16:18:22 [DEBUG] 9 proxyAuth : ":" +2015-03-30T16:18:22 [DEBUG] 10 scriptEncoding : "UTF-8" +2015-03-30T16:18:22 [DEBUG] 11 webSecurityEnabled : "true" +2015-03-30T16:18:22 [DEBUG] 12 offlineStoragePath : "" +2015-03-30T16:18:22 [DEBUG] 13 offlineStorageDefaultQuota : "-1" +2015-03-30T16:18:22 [DEBUG] 14 printDebugMessages : "true" +2015-03-30T16:18:22 [DEBUG] 15 javascriptCanOpenWindows : "true" +2015-03-30T16:18:22 [DEBUG] 16 javascriptCanCloseWindows : "true" +2015-03-30T16:18:22 [DEBUG] 17 sslProtocol : "tlsv1" +2015-03-30T16:18:22 [DEBUG] 18 sslCertificatesPath : "" +2015-03-30T16:18:22 [DEBUG] 19 webdriver : ":44021" +2015-03-30T16:18:22 [DEBUG] 20 webdriverLogFile : "" +2015-03-30T16:18:22 [DEBUG] 21 webdriverLogLevel : "debug" +2015-03-30T16:18:22 [DEBUG] 22 webdriverSeleniumGridHub : "" +2015-03-30T16:18:22 [DEBUG] Phantom - execute: Script & Arguments +2015-03-30T16:18:22 [DEBUG] script: "main.js" +2015-03-30T16:18:22 [DEBUG] 0 arg: "--ip=" +2015-03-30T16:18:22 [DEBUG] 1 arg: "--port=44021" +2015-03-30T16:18:22 [DEBUG] 2 arg: "--logLevel=debug" +2015-03-30T16:18:22 [DEBUG] Phantom - execute: Starting Remote WebDriver mode PhantomJS is launching GhostDriver... -2015-03-30T15:39:09 [DEBUG] WebPage - setupFrame "" -2015-03-30T15:39:09 [DEBUG] FileSystem - _open: ":/modules/fs.js" QMap(("mode", QVariant(QString, "r") ) ) -2015-03-30T15:39:09 [DEBUG] FileSystem - _open: ":/modules/system.js" QMap(("mode", QVariant(QString, "r") ) ) -2015-03-30T15:39:09 [DEBUG] FileSystem - _open: ":/modules/webpage.js" QMap(("mode", QVariant(QString, "r") ) ) -2015-03-30T15:39:09 [DEBUG] FileSystem - _open: ":/ghostdriver/./hub_register.js" QMap(("mode", QVariant(QString, "r") ) ) -2015-03-30T15:39:09 [DEBUG] FileSystem - _open: ":/ghostdriver/./logger.js" QMap(("mode", QVariant(QString, "r") ) ) -2015-03-30T15:39:09 [DEBUG] FileSystem - _open: ":/ghostdriver/./third_party/console++.js" QMap(("mode", QVariant(QString, "r") ) ) -2015-03-30T15:39:09 [DEBUG] FileSystem - _open: ":/modules/webserver.js" QMap(("mode", QVariant(QString, "r") ) ) -2015-03-30T15:39:09 [DEBUG] FileSystem - _open: ":/ghostdriver/./third_party/parseuri.js" QMap(("mode", QVariant(QString, "r") ) ) -2015-03-30T15:39:09 [DEBUG] FileSystem - _open: ":/ghostdriver/./config.js" QMap(("mode", QVariant(QString, "r") ) ) -[DEBUG - 2015-03-30T07:39:09.689Z] Config - init - Configuration => {"ip":"127.0.0.1","port":"58200","hub":null,"logFile":null,"logLevel":"DEBUG","logColor":false} -2015-03-30T15:39:09 [DEBUG] Phantom - injectJs: "session.js" -2015-03-30T15:39:09 [DEBUG] Phantom - injectJs: prepending ":/ghostdriver/" -2015-03-30T15:39:09 [DEBUG] Phantom - injectJs: "inputs.js" -2015-03-30T15:39:09 [DEBUG] Phantom - injectJs: prepending ":/ghostdriver/" -2015-03-30T15:39:09 [DEBUG] Phantom - injectJs: "request_handlers/request_handler.js" -2015-03-30T15:39:09 [DEBUG] Phantom - injectJs: prepending ":/ghostdriver/" -2015-03-30T15:39:09 [DEBUG] Phantom - injectJs: "request_handlers/status_request_handler.js" -2015-03-30T15:39:09 [DEBUG] Phantom - injectJs: prepending ":/ghostdriver/" -2015-03-30T15:39:09 [DEBUG] FileSystem - _open: ":/ghostdriver/./errors.js" QMap(("mode", QVariant(QString, "r") ) ) -2015-03-30T15:39:09 [DEBUG] Phantom - injectJs: "request_handlers/shutdown_request_handler.js" -2015-03-30T15:39:09 [DEBUG] Phantom - injectJs: prepending ":/ghostdriver/" -2015-03-30T15:39:09 [DEBUG] Phantom - injectJs: "request_handlers/session_manager_request_handler.js" -2015-03-30T15:39:09 [DEBUG] Phantom - injectJs: prepending ":/ghostdriver/" -2015-03-30T15:39:09 [DEBUG] Phantom - injectJs: "request_handlers/session_request_handler.js" -2015-03-30T15:39:09 [DEBUG] Phantom - injectJs: prepending ":/ghostdriver/" -2015-03-30T15:39:09 [DEBUG] Phantom - injectJs: "request_handlers/webelement_request_handler.js" -2015-03-30T15:39:09 [DEBUG] Phantom - injectJs: prepending ":/ghostdriver/" -2015-03-30T15:39:09 [DEBUG] Phantom - injectJs: "request_handlers/router_request_handler.js" -2015-03-30T15:39:09 [DEBUG] Phantom - injectJs: prepending ":/ghostdriver/" -2015-03-30T15:39:09 [DEBUG] Phantom - injectJs: "webelementlocator.js" -2015-03-30T15:39:09 [DEBUG] Phantom - injectJs: prepending ":/ghostdriver/" -[INFO - 2015-03-30T07:39:09.700Z] GhostDriver - Main - running on port 58200 -2015-03-30T15:39:10 [DEBUG] HTTP Request - URI /wd/hub/session -2015-03-30T15:39:10 [DEBUG] HTTP Request - Method POST -2015-03-30T15:39:10 [DEBUG] HTTP Request - HTTP Version 1.1 -2015-03-30T15:39:10 [DEBUG] HTTP Request - Query String -2015-03-30T15:39:10 [DEBUG] HTTP Request - Receiving Header "Accept-Encoding" = "identity" -2015-03-30T15:39:10 [DEBUG] HTTP Request - Receiving Header "Content-Length" = "100" -2015-03-30T15:39:10 [DEBUG] HTTP Request - Receiving Header "Host" = "127.0.0.1:58200" -2015-03-30T15:39:10 [DEBUG] HTTP Request - Receiving Header "Accept" = "application/json" -2015-03-30T15:39:10 [DEBUG] HTTP Request - Receiving Header "User-Agent" = "Python-urllib/2.7" -2015-03-30T15:39:10 [DEBUG] HTTP Request - Receiving Header "Connection" = "close" -2015-03-30T15:39:10 [DEBUG] HTTP Request - Receiving Header "Content-Type" = "application/json;charset=UTF-8" -2015-03-30T15:39:10 [DEBUG] HTTP Request - Method POST/PUT -2015-03-30T15:39:10 [DEBUG] HTTP Request - Content Body: {"desiredCapabilities": {"phantomjs.page.settings.userAgent": "Scrapy/0.24.4 (+http://scrapy.org)"}} -[DEBUG - 2015-03-30T07:39:10.504Z] RouterReqHand - _handle - Request => { - "headers": { - "Accept": "application/json", - "Accept-Encoding": "identity", - "Connection": "close", - "Content-Length": "100", - "Content-Type": "application/json;charset=UTF-8", - "Host": "127.0.0.1:58200", - "User-Agent": "Python-urllib/2.7" - }, - "httpVersion": "1.1", - "method": "POST", - "post": "{\"desiredCapabilities\": {\"phantomjs.page.settings.userAgent\": \"Scrapy/0.24.4 (+http://scrapy.org)\"}}", - "url": "/wd/hub/session", - "urlParsed": { - "anchor": "", - "query": "", - "file": "session", - "directory": "/wd/hub/", - "path": "/wd/hub/session", - "relative": "/wd/hub/session", - "port": "", - "host": "", - "password": "", - "user": "", - "userInfo": "", - "authority": "", - "protocol": "", - "source": "/wd/hub/session", - "queryKey": {}, - "chunks": [ - "wd", - "hub", - "session" - ] - } -} -2015-03-30T15:39:10 [DEBUG] FileSystem - _open: ":/ghostdriver/./third_party/uuid.js" QMap(("mode", QVariant(QString, "r") ) ) -2015-03-30T15:39:10 [DEBUG] WebpageCallbacks - getFilePickerCallback -2015-03-30T15:39:10 [DEBUG] WebpageCallbacks - getGenericCallback -[DEBUG - 2015-03-30T07:39:10.517Z] Session [dab84dd0-d6af-11e4-95ed-a71961c71b3a] - _decorateNewWindow - page.settings: {"XSSAuditingEnabled":false,"javascriptCanCloseWindows":true,"javascriptCanOpenWindows":true,"javascriptEnabled":true,"loadImages":false,"localToRemoteUrlAccessEnabled":false,"userAgent":"Scrapy/0.24.4 (+http://scrapy.org)","webSecurityEnabled":true} -[INFO - 2015-03-30T07:39:10.517Z] Session [dab84dd0-d6af-11e4-95ed-a71961c71b3a] - CONSTRUCTOR - Desired Capabilities: {"phantomjs.page.settings.userAgent":"Scrapy/0.24.4 (+http://scrapy.org)"} -[INFO - 2015-03-30T07:39:10.517Z] Session [dab84dd0-d6af-11e4-95ed-a71961c71b3a] - CONSTRUCTOR - Negotiated Capabilities: {"browserName":"phantomjs","version":"1.9.0","driverName":"ghostdriver","driverVersion":"1.0.3","platform":"linux-unknown-64bit","javascriptEnabled":true,"takesScreenshot":true,"handlesAlerts":false,"databaseEnabled":false,"locationContextEnabled":false,"applicationCacheEnabled":false,"browserConnectionEnabled":false,"cssSelectorsEnabled":true,"webStorageEnabled":false,"rotatable":false,"acceptSslCerts":false,"nativeEvents":true,"proxy":{"proxyType":"direct"},"phantomjs.page.settings.userAgent":"Scrapy/0.24.4 (+http://scrapy.org)"} -[INFO - 2015-03-30T07:39:10.518Z] SessionManagerReqHand - _postNewSessionCommand - New Session Created: dab84dd0-d6af-11e4-95ed-a71961c71b3a -2015-03-30T15:39:10 [DEBUG] HTTP Response - Status Code 303 See Other -2015-03-30T15:39:10 [DEBUG] HTTP Response - Sending Header "Cache" = "no-cache" -2015-03-30T15:39:10 [DEBUG] HTTP Response - Sending Header "Content-Type" = "application/json;charset=UTF-8" -2015-03-30T15:39:10 [DEBUG] HTTP Response - Sending Header "Location" = "http://127.0.0.1:58200/wd/hub/session/dab84dd0-d6af-11e4-95ed-a71961c71b3a" -2015-03-30T15:39:10 [DEBUG] HTTP Request - URI /wd/hub/session/dab84dd0-d6af-11e4-95ed-a71961c71b3a -2015-03-30T15:39:10 [DEBUG] HTTP Request - Method GET -2015-03-30T15:39:10 [DEBUG] HTTP Request - HTTP Version 1.1 -2015-03-30T15:39:10 [DEBUG] HTTP Request - Query String -2015-03-30T15:39:10 [DEBUG] HTTP Request - Receiving Header "Accept-Encoding" = "identity" -2015-03-30T15:39:10 [DEBUG] HTTP Request - Receiving Header "Host" = "127.0.0.1:58200" -2015-03-30T15:39:10 [DEBUG] HTTP Request - Receiving Header "Connection" = "close" -2015-03-30T15:39:10 [DEBUG] HTTP Request - Receiving Header "Accept" = "application/json" -2015-03-30T15:39:10 [DEBUG] HTTP Request - Receiving Header "User-Agent" = "Python-urllib/2.7" -[DEBUG - 2015-03-30T07:39:10.520Z] RouterReqHand - _handle - Request => { - "headers": { - "Accept": "application/json", - "Accept-Encoding": "identity", - "Connection": "close", - "Host": "127.0.0.1:58200", - "User-Agent": "Python-urllib/2.7" - }, - "httpVersion": "1.1", - "method": "GET", - "url": "/wd/hub/session/dab84dd0-d6af-11e4-95ed-a71961c71b3a", - "urlParsed": { - "anchor": "", - "query": "", - "file": "dab84dd0-d6af-11e4-95ed-a71961c71b3a", - "directory": "/wd/hub/session/", - "path": "/wd/hub/session/dab84dd0-d6af-11e4-95ed-a71961c71b3a", - "relative": "/wd/hub/session/dab84dd0-d6af-11e4-95ed-a71961c71b3a", - "port": "", - "host": "", - "password": "", - "user": "", - "userInfo": "", - "authority": "", - "protocol": "", - "source": "/wd/hub/session/dab84dd0-d6af-11e4-95ed-a71961c71b3a", - "queryKey": {}, - "chunks": [ - "wd", - "hub", - "session", - "dab84dd0-d6af-11e4-95ed-a71961c71b3a" - ] - } -} -2015-03-30T15:39:10 [DEBUG] HTTP Response - Status Code 200 OK -2015-03-30T15:39:10 [DEBUG] HTTP Response - Sending Header "Cache" = "no-cache" -2015-03-30T15:39:10 [DEBUG] HTTP Response - Sending Header "Content-Length" = "609" -2015-03-30T15:39:10 [DEBUG] HTTP Response - Sending Header "Content-Type" = "application/json;charset=UTF-8" -2015-03-30T15:39:10 [DEBUG] HTTP Request - URI /wd/hub/session/dab84dd0-d6af-11e4-95ed-a71961c71b3a/url -2015-03-30T15:39:10 [DEBUG] HTTP Request - Method POST -2015-03-30T15:39:10 [DEBUG] HTTP Request - HTTP Version 1.1 -2015-03-30T15:39:10 [DEBUG] HTTP Request - Query String -2015-03-30T15:39:10 [DEBUG] HTTP Request - Receiving Header "Accept-Encoding" = "identity" -2015-03-30T15:39:10 [DEBUG] HTTP Request - Receiving Header "Content-Length" = "87" -2015-03-30T15:39:10 [DEBUG] HTTP Request - Receiving Header "Host" = "127.0.0.1:58200" -2015-03-30T15:39:10 [DEBUG] HTTP Request - Receiving Header "Accept" = "application/json" -2015-03-30T15:39:10 [DEBUG] HTTP Request - Receiving Header "User-Agent" = "Python-urllib/2.7" -2015-03-30T15:39:10 [DEBUG] HTTP Request - Receiving Header "Connection" = "close" -2015-03-30T15:39:10 [DEBUG] HTTP Request - Receiving Header "Content-Type" = "application/json;charset=UTF-8" -2015-03-30T15:39:10 [DEBUG] HTTP Request - Method POST/PUT -2015-03-30T15:39:10 [DEBUG] HTTP Request - Content Body: {"url": "http://image.baidu.com/", "sessionId": "dab84dd0-d6af-11e4-95ed-a71961c71b3a"} -[DEBUG - 2015-03-30T07:39:10.534Z] RouterReqHand - _handle - Request => { - "headers": { - "Accept": "application/json", - "Accept-Encoding": "identity", - "Connection": "close", - "Content-Length": "87", - "Content-Type": "application/json;charset=UTF-8", - "Host": "127.0.0.1:58200", - "User-Agent": "Python-urllib/2.7" - }, - "httpVersion": "1.1", - "method": "POST", - "post": "{\"url\": \"http://image.baidu.com/\", \"sessionId\": \"dab84dd0-d6af-11e4-95ed-a71961c71b3a\"}", - "url": "/wd/hub/session/dab84dd0-d6af-11e4-95ed-a71961c71b3a/url", - "urlParsed": { - "anchor": "", - "query": "", - "file": "url", - "directory": "/wd/hub/session/dab84dd0-d6af-11e4-95ed-a71961c71b3a/", - "path": "/wd/hub/session/dab84dd0-d6af-11e4-95ed-a71961c71b3a/url", - "relative": "/wd/hub/session/dab84dd0-d6af-11e4-95ed-a71961c71b3a/url", - "port": "", - "host": "", - "password": "", - "user": "", - "userInfo": "", - "authority": "", - "protocol": "", - "source": "/wd/hub/session/dab84dd0-d6af-11e4-95ed-a71961c71b3a/url", - "queryKey": {}, - "chunks": [ - "wd", - "hub", - "session", - "dab84dd0-d6af-11e4-95ed-a71961c71b3a", - "url" - ] - } -} -[DEBUG - 2015-03-30T07:39:10.536Z] SessionReqHand - _postUrlCommand - Session 'dab84dd0-d6af-11e4-95ed-a71961c71b3a' is about to load URL: http://image.baidu.com/ -2015-03-30T15:39:10 [DEBUG] WebPage - updateLoadingProgress: 10 -[DEBUG - 2015-03-30T07:39:10.548Z] Session [dab84dd0-d6af-11e4-95ed-a71961c71b3a] - _execFuncAndWaitForLoadDecorator - Page Loading in Session: true -[DEBUG - 2015-03-30T07:39:10.649Z] Session [dab84dd0-d6af-11e4-95ed-a71961c71b3a] - _execFuncAndWaitForLoadDecorator - Page Loading in Session: true -2015-03-30T15:39:10 [DEBUG] CookieJar - Saved "BAIDUID=24E8436499A3BAAA38734F19C8270F34:FG=1; expires=Tue, 29-Mar-2016 07:39:10 GMT; domain=.baidu.com; path=/" -2015-03-30T15:39:10 [DEBUG] WebPage - setupFrame "" -2015-03-30T15:39:10 [DEBUG] WebPage - updateLoadingProgress: 24 -[DEBUG - 2015-03-30T07:39:10.749Z] Session [dab84dd0-d6af-11e4-95ed-a71961c71b3a] - _execFuncAndWaitForLoadDecorator - Page Loading in Session: true -[DEBUG - 2015-03-30T07:39:10.849Z] Session [dab84dd0-d6af-11e4-95ed-a71961c71b3a] - _execFuncAndWaitForLoadDecorator - Page Loading in Session: true -[DEBUG - 2015-03-30T07:39:10.949Z] Session [dab84dd0-d6af-11e4-95ed-a71961c71b3a] - _execFuncAndWaitForLoadDecorator - Page Loading in Session: true -2015-03-30T15:39:11 [DEBUG] WebPage - updateLoadingProgress: 25 -2015-03-30T15:39:11 [DEBUG] WebPage - updateLoadingProgress: 34 -[DEBUG - 2015-03-30T07:39:11.049Z] Session [dab84dd0-d6af-11e4-95ed-a71961c71b3a] - _execFuncAndWaitForLoadDecorator - Page Loading in Session: true -[DEBUG - 2015-03-30T07:39:11.150Z] Session [dab84dd0-d6af-11e4-95ed-a71961c71b3a] - _execFuncAndWaitForLoadDecorator - Page Loading in Session: true -[DEBUG - 2015-03-30T07:39:11.250Z] Session [dab84dd0-d6af-11e4-95ed-a71961c71b3a] - _execFuncAndWaitForLoadDecorator - Page Loading in Session: true -[DEBUG - 2015-03-30T07:39:11.350Z] Session [dab84dd0-d6af-11e4-95ed-a71961c71b3a] - _execFuncAndWaitForLoadDecorator - Page Loading in Session: true -[DEBUG - 2015-03-30T07:39:11.450Z] Session [dab84dd0-d6af-11e4-95ed-a71961c71b3a] - _execFuncAndWaitForLoadDecorator - Page Loading in Session: true -[DEBUG - 2015-03-30T07:39:11.550Z] Session [dab84dd0-d6af-11e4-95ed-a71961c71b3a] - _execFuncAndWaitForLoadDecorator - Page Loading in Session: true -[DEBUG - 2015-03-30T07:39:11.651Z] Session [dab84dd0-d6af-11e4-95ed-a71961c71b3a] - _execFuncAndWaitForLoadDecorator - Page Loading in Session: true -[DEBUG - 2015-03-30T07:39:11.751Z] Session [dab84dd0-d6af-11e4-95ed-a71961c71b3a] - _execFuncAndWaitForLoadDecorator - Page Loading in Session: true -[DEBUG - 2015-03-30T07:39:11.851Z] Session [dab84dd0-d6af-11e4-95ed-a71961c71b3a] - _execFuncAndWaitForLoadDecorator - Page Loading in Session: true -[DEBUG - 2015-03-30T07:39:11.951Z] Session [dab84dd0-d6af-11e4-95ed-a71961c71b3a] - _execFuncAndWaitForLoadDecorator - Page Loading in Session: true -2015-03-30T15:39:12 [DEBUG] WebPage - updateLoadingProgress: 42 -[DEBUG - 2015-03-30T07:39:12.117Z] Session [dab84dd0-d6af-11e4-95ed-a71961c71b3a] - _execFuncAndWaitForLoadDecorator - Page Loading in Session: true -2015-03-30T15:39:12 [DEBUG] WebPage - updateLoadingProgress: 46 -2015-03-30T15:39:12 [DEBUG] WebPage - updateLoadingProgress: 60 -2015-03-30T15:39:12 [DEBUG] WebPage - updateLoadingProgress: 68 -2015-03-30T15:39:12 [DEBUG] WebPage - updateLoadingProgress: 71 -2015-03-30T15:39:12 [DEBUG] WebPage - updateLoadingProgress: 76 -[DEBUG - 2015-03-30T07:39:12.238Z] Session [dab84dd0-d6af-11e4-95ed-a71961c71b3a] - _execFuncAndWaitForLoadDecorator - Page Loading in Session: true -[DEBUG - 2015-03-30T07:39:12.338Z] Session [dab84dd0-d6af-11e4-95ed-a71961c71b3a] - _execFuncAndWaitForLoadDecorator - Page Loading in Session: true -[DEBUG - 2015-03-30T07:39:12.438Z] Session [dab84dd0-d6af-11e4-95ed-a71961c71b3a] - _execFuncAndWaitForLoadDecorator - Page Loading in Session: true -2015-03-30T15:39:12 [DEBUG] CookieJar - Saved "BAIDUID=24E8436499A3BAAA38734F19C8270F34:FG=1; expires=Tue, 29-Mar-2016 07:39:10 GMT; domain=.baidu.com; path=/" -2015-03-30T15:39:12 [DEBUG] CookieJar - Saved "HMVT=737dbb498415dd39d8abf5bc2404b290|1427701152|; domain=.hm.baidu.com; path=/" -2015-03-30T15:39:12 [DEBUG] CookieJar - Saved "HMACCOUNT=8E373D7C3072E1C3; expires=Mon, 18-Jan-2038 00:00:00 GMT; domain=.hm.baidu.com; path=/" -2015-03-30T15:39:12 [DEBUG] WebPage - updateLoadingProgress: 81 -2015-03-30T15:39:12 [DEBUG] WebPage - updateLoadingProgress: 100 -[DEBUG - 2015-03-30T07:39:12.446Z] Session [dab84dd0-d6af-11e4-95ed-a71961c71b3a] - _oneShotCallback - onLoadFinished -[DEBUG - 2015-03-30T07:39:12.446Z] Session [dab84dd0-d6af-11e4-95ed-a71961c71b3a] - _execFuncAndWaitForLoadDecorator - onLoadFinished: success -2015-03-30T15:39:12 [DEBUG] WebPage - setupFrame "" -2015-03-30T15:39:12 [DEBUG] CookieJar - Saved "BAIDUID=24E8436499A3BAAA38734F19C8270F34:FG=1; expires=Tue, 29-Mar-2016 07:39:10 GMT; domain=.baidu.com; path=/" -2015-03-30T15:39:12 [DEBUG] CookieJar - Saved "HMVT=737dbb498415dd39d8abf5bc2404b290|1427701152|; domain=.hm.baidu.com; path=/" -2015-03-30T15:39:12 [DEBUG] CookieJar - Saved "HMACCOUNT=8E373D7C3072E1C3; expires=Mon, 18-Jan-2038 00:00:00 GMT; domain=.hm.baidu.com; path=/" -2015-03-30T15:39:12 [DEBUG] CookieJar - Saved "Hm_lvt_737dbb498415dd39d8abf5bc2404b290=1427700229,1427700537,1427700716,1427701152; expires=Tue, 29-Mar-2016 07:39:12 GMT; domain=.image.baidu.com; path=/" -2015-03-30T15:39:12 [DEBUG] CookieJar - Saved "BAIDUID=24E8436499A3BAAA38734F19C8270F34:FG=1; expires=Tue, 29-Mar-2016 07:39:10 GMT; domain=.baidu.com; path=/" -2015-03-30T15:39:12 [DEBUG] CookieJar - Saved "HMVT=737dbb498415dd39d8abf5bc2404b290|1427701152|; domain=.hm.baidu.com; path=/" -2015-03-30T15:39:12 [DEBUG] CookieJar - Saved "HMACCOUNT=8E373D7C3072E1C3; expires=Mon, 18-Jan-2038 00:00:00 GMT; domain=.hm.baidu.com; path=/" -2015-03-30T15:39:12 [DEBUG] CookieJar - Saved "Hm_lvt_737dbb498415dd39d8abf5bc2404b290=1427700229,1427700537,1427700716,1427701152; expires=Tue, 29-Mar-2016 07:39:12 GMT; domain=.image.baidu.com; path=/" -2015-03-30T15:39:12 [DEBUG] CookieJar - Saved "Hm_lpvt_737dbb498415dd39d8abf5bc2404b290=1427701152; domain=.image.baidu.com; path=/" -[DEBUG - 2015-03-30T07:39:12.538Z] Session [dab84dd0-d6af-11e4-95ed-a71961c71b3a] - _execFuncAndWaitForLoadDecorator - Page Loading in Session: false -[DEBUG - 2015-03-30T07:39:12.538Z] Session [dab84dd0-d6af-11e4-95ed-a71961c71b3a] - _resetOneShotCallbacksDecorator -[DEBUG - 2015-03-30T07:39:12.539Z] SessionReqHand - _SuccessHandler - status: success -2015-03-30T15:39:12 [DEBUG] HTTP Response - Status Code 200 OK -2015-03-30T15:39:12 [DEBUG] HTTP Response - Sending Header "Cache" = "no-cache" -2015-03-30T15:39:12 [DEBUG] HTTP Response - Sending Header "Content-Length" = "74" -2015-03-30T15:39:12 [DEBUG] HTTP Response - Sending Header "Content-Type" = "application/json;charset=UTF-8" -2015-03-30T15:39:12 [DEBUG] HTTP Request - URI /wd/hub/session/dab84dd0-d6af-11e4-95ed-a71961c71b3a/source -2015-03-30T15:39:12 [DEBUG] HTTP Request - Method GET -2015-03-30T15:39:12 [DEBUG] HTTP Request - HTTP Version 1.1 -2015-03-30T15:39:12 [DEBUG] HTTP Request - Query String -2015-03-30T15:39:12 [DEBUG] HTTP Request - Receiving Header "Accept-Encoding" = "identity" -2015-03-30T15:39:12 [DEBUG] HTTP Request - Receiving Header "Host" = "127.0.0.1:58200" -2015-03-30T15:39:12 [DEBUG] HTTP Request - Receiving Header "Content-Type" = "application/json;charset=UTF-8" -2015-03-30T15:39:12 [DEBUG] HTTP Request - Receiving Header "Connection" = "close" -2015-03-30T15:39:12 [DEBUG] HTTP Request - Receiving Header "Accept" = "application/json" -2015-03-30T15:39:12 [DEBUG] HTTP Request - Receiving Header "User-Agent" = "Python-urllib/2.7" -[DEBUG - 2015-03-30T07:39:12.542Z] RouterReqHand - _handle - Request => { - "headers": { - "Accept": "application/json", - "Accept-Encoding": "identity", - "Connection": "close", - "Content-Type": "application/json;charset=UTF-8", - "Host": "127.0.0.1:58200", - "User-Agent": "Python-urllib/2.7" - }, - "httpVersion": "1.1", - "method": "GET", - "url": "/wd/hub/session/dab84dd0-d6af-11e4-95ed-a71961c71b3a/source", - "urlParsed": { - "anchor": "", - "query": "", - "file": "source", - "directory": "/wd/hub/session/dab84dd0-d6af-11e4-95ed-a71961c71b3a/", - "path": "/wd/hub/session/dab84dd0-d6af-11e4-95ed-a71961c71b3a/source", - "relative": "/wd/hub/session/dab84dd0-d6af-11e4-95ed-a71961c71b3a/source", - "port": "", - "host": "", - "password": "", - "user": "", - "userInfo": "", - "authority": "", - "protocol": "", - "source": "/wd/hub/session/dab84dd0-d6af-11e4-95ed-a71961c71b3a/source", - "queryKey": {}, - "chunks": [ - "wd", - "hub", - "session", - "dab84dd0-d6af-11e4-95ed-a71961c71b3a", - "source" - ] - } -} -2015-03-30T15:39:12 [DEBUG] HTTP Response - Status Code 200 OK -2015-03-30T15:39:12 [DEBUG] HTTP Response - Sending Header "Cache" = "no-cache" -2015-03-30T15:39:12 [DEBUG] HTTP Response - Sending Header "Content-Length" = "94212" -2015-03-30T15:39:12 [DEBUG] HTTP Response - Sending Header "Content-Type" = "application/json;charset=UTF-8" -2015-03-30T15:39:15 [DEBUG] HTTP Request - URI /wd/hub/session/dab84dd0-d6af-11e4-95ed-a71961c71b3a/url -2015-03-30T15:39:15 [DEBUG] HTTP Request - Method POST -2015-03-30T15:39:15 [DEBUG] HTTP Request - HTTP Version 1.1 -2015-03-30T15:39:15 [DEBUG] HTTP Request - Query String -2015-03-30T15:39:15 [DEBUG] HTTP Request - Receiving Header "Accept-Encoding" = "identity" -2015-03-30T15:39:15 [DEBUG] HTTP Request - Receiving Header "Content-Length" = "83" -2015-03-30T15:39:15 [DEBUG] HTTP Request - Receiving Header "Host" = "127.0.0.1:58200" -2015-03-30T15:39:15 [DEBUG] HTTP Request - Receiving Header "Accept" = "application/json" -2015-03-30T15:39:15 [DEBUG] HTTP Request - Receiving Header "User-Agent" = "Python-urllib/2.7" -2015-03-30T15:39:15 [DEBUG] HTTP Request - Receiving Header "Connection" = "close" -2015-03-30T15:39:15 [DEBUG] HTTP Request - Receiving Header "Content-Type" = "application/json;charset=UTF-8" -2015-03-30T15:39:15 [DEBUG] HTTP Request - Method POST/PUT -2015-03-30T15:39:15 [DEBUG] HTTP Request - Content Body: {"url": "http://v.baidu.com/", "sessionId": "dab84dd0-d6af-11e4-95ed-a71961c71b3a"} -[DEBUG - 2015-03-30T07:39:15.021Z] RouterReqHand - _handle - Request => { - "headers": { - "Accept": "application/json", - "Accept-Encoding": "identity", - "Connection": "close", - "Content-Length": "83", - "Content-Type": "application/json;charset=UTF-8", - "Host": "127.0.0.1:58200", - "User-Agent": "Python-urllib/2.7" - }, - "httpVersion": "1.1", - "method": "POST", - "post": "{\"url\": \"http://v.baidu.com/\", \"sessionId\": \"dab84dd0-d6af-11e4-95ed-a71961c71b3a\"}", - "url": "/wd/hub/session/dab84dd0-d6af-11e4-95ed-a71961c71b3a/url", - "urlParsed": { - "anchor": "", - "query": "", - "file": "url", - "directory": "/wd/hub/session/dab84dd0-d6af-11e4-95ed-a71961c71b3a/", - "path": "/wd/hub/session/dab84dd0-d6af-11e4-95ed-a71961c71b3a/url", - "relative": "/wd/hub/session/dab84dd0-d6af-11e4-95ed-a71961c71b3a/url", - "port": "", - "host": "", - "password": "", - "user": "", - "userInfo": "", - "authority": "", - "protocol": "", - "source": "/wd/hub/session/dab84dd0-d6af-11e4-95ed-a71961c71b3a/url", - "queryKey": {}, - "chunks": [ - "wd", - "hub", - "session", - "dab84dd0-d6af-11e4-95ed-a71961c71b3a", - "url" - ] - } -} -[DEBUG - 2015-03-30T07:39:15.022Z] SessionReqHand - _postUrlCommand - Session 'dab84dd0-d6af-11e4-95ed-a71961c71b3a' is about to load URL: http://v.baidu.com/ -2015-03-30T15:39:15 [DEBUG] WebPage - updateLoadingProgress: 10 -[DEBUG - 2015-03-30T07:39:15.032Z] Session [dab84dd0-d6af-11e4-95ed-a71961c71b3a] - _execFuncAndWaitForLoadDecorator - Page Loading in Session: true -[DEBUG - 2015-03-30T07:39:15.133Z] Session [dab84dd0-d6af-11e4-95ed-a71961c71b3a] - _execFuncAndWaitForLoadDecorator - Page Loading in Session: true -[DEBUG - 2015-03-30T07:39:15.233Z] Session [dab84dd0-d6af-11e4-95ed-a71961c71b3a] - _execFuncAndWaitForLoadDecorator - Page Loading in Session: true -[DEBUG - 2015-03-30T07:39:15.333Z] Session [dab84dd0-d6af-11e4-95ed-a71961c71b3a] - _execFuncAndWaitForLoadDecorator - Page Loading in Session: true -[DEBUG - 2015-03-30T07:39:15.433Z] Session [dab84dd0-d6af-11e4-95ed-a71961c71b3a] - _execFuncAndWaitForLoadDecorator - Page Loading in Session: true -[DEBUG - 2015-03-30T07:39:15.533Z] Session [dab84dd0-d6af-11e4-95ed-a71961c71b3a] - _execFuncAndWaitForLoadDecorator - Page Loading in Session: true -[DEBUG - 2015-03-30T07:39:15.633Z] Session [dab84dd0-d6af-11e4-95ed-a71961c71b3a] - _execFuncAndWaitForLoadDecorator - Page Loading in Session: true -[DEBUG - 2015-03-30T07:39:15.733Z] Session [dab84dd0-d6af-11e4-95ed-a71961c71b3a] - _execFuncAndWaitForLoadDecorator - Page Loading in Session: true -[DEBUG - 2015-03-30T07:39:15.834Z] Session [dab84dd0-d6af-11e4-95ed-a71961c71b3a] - _execFuncAndWaitForLoadDecorator - Page Loading in Session: true -[DEBUG - 2015-03-30T07:39:15.934Z] Session [dab84dd0-d6af-11e4-95ed-a71961c71b3a] - _execFuncAndWaitForLoadDecorator - Page Loading in Session: true -2015-03-30T15:39:15 [DEBUG] CookieJar - Saved "HMVT=737dbb498415dd39d8abf5bc2404b290|1427701152|; domain=.hm.baidu.com; path=/" -2015-03-30T15:39:15 [DEBUG] CookieJar - Saved "HMACCOUNT=8E373D7C3072E1C3; expires=Mon, 18-Jan-2038 00:00:00 GMT; domain=.hm.baidu.com; path=/" -2015-03-30T15:39:15 [DEBUG] CookieJar - Saved "Hm_lvt_737dbb498415dd39d8abf5bc2404b290=1427700229,1427700537,1427700716,1427701152; expires=Tue, 29-Mar-2016 07:39:12 GMT; domain=.image.baidu.com; path=/" -2015-03-30T15:39:15 [DEBUG] CookieJar - Saved "Hm_lpvt_737dbb498415dd39d8abf5bc2404b290=1427701152; domain=.image.baidu.com; path=/" -2015-03-30T15:39:15 [DEBUG] CookieJar - Saved "BAIDUID=383833C1F0BD08B9311CD7DD32FB6879:FG=1; expires=Tue, 29-Mar-2016 07:39:15 GMT; domain=.baidu.com; path=/" -[DEBUG - 2015-03-30T07:39:16.034Z] Session [dab84dd0-d6af-11e4-95ed-a71961c71b3a] - _execFuncAndWaitForLoadDecorator - Page Loading in Session: true -[DEBUG - 2015-03-30T07:39:16.134Z] Session [dab84dd0-d6af-11e4-95ed-a71961c71b3a] - _execFuncAndWaitForLoadDecorator - Page Loading in Session: true -[DEBUG - 2015-03-30T07:39:16.234Z] Session [dab84dd0-d6af-11e4-95ed-a71961c71b3a] - _execFuncAndWaitForLoadDecorator - Page Loading in Session: true -[DEBUG - 2015-03-30T07:39:16.334Z] Session [dab84dd0-d6af-11e4-95ed-a71961c71b3a] - _execFuncAndWaitForLoadDecorator - Page Loading in Session: true -[DEBUG - 2015-03-30T07:39:16.435Z] Session [dab84dd0-d6af-11e4-95ed-a71961c71b3a] - _execFuncAndWaitForLoadDecorator - Page Loading in Session: true -[DEBUG - 2015-03-30T07:39:16.535Z] Session [dab84dd0-d6af-11e4-95ed-a71961c71b3a] - _execFuncAndWaitForLoadDecorator - Page Loading in Session: true -2015-03-30T15:39:16 [DEBUG] WebPage - setupFrame "" -2015-03-30T15:39:16 [DEBUG] WebPage - updateLoadingProgress: 39 -2015-03-30T15:39:16 [DEBUG] WebPage - updateLoadingProgress: 53 -[DEBUG - 2015-03-30T07:39:16.635Z] Session [dab84dd0-d6af-11e4-95ed-a71961c71b3a] - _execFuncAndWaitForLoadDecorator - Page Loading in Session: true -2015-03-30T15:39:16 [DEBUG] Network - Resource request error: 5 ( "Operation canceled" ) URL: "http://hm.baidu.com/hm.gif?cc=1&ck=1&cl=32-bit&ds=1024x768&ep=4145,4145&et=3&ja=0&ln=en-US&lo=0<=1427701152&nv=0&rnd=752059763&si=737dbb498415dd39d8abf5bc2404b290&st=4&v=1.0.75&lv=3&tt=百度图片—全球最大中文图片库" -[DEBUG - 2015-03-30T07:39:16.736Z] Session [dab84dd0-d6af-11e4-95ed-a71961c71b3a] - _execFuncAndWaitForLoadDecorator - Page Loading in Session: true -[DEBUG - 2015-03-30T07:39:16.836Z] Session [dab84dd0-d6af-11e4-95ed-a71961c71b3a] - _execFuncAndWaitForLoadDecorator - Page Loading in Session: true -2015-03-30T15:39:16 [DEBUG] WebPage - updateLoadingProgress: 54 -2015-03-30T15:39:16 [DEBUG] WebPage - updateLoadingProgress: 57 -2015-03-30T15:39:16 [DEBUG] WebPage - updateLoadingProgress: 60 -2015-03-30T15:39:16 [DEBUG] WebPage - updateLoadingProgress: 66 -2015-03-30T15:39:16 [DEBUG] WebPage - updateLoadingProgress: 72 -2015-03-30T15:39:16 [DEBUG] WebPage - updateLoadingProgress: 76 -2015-03-30T15:39:16 [DEBUG] WebPage - updateLoadingProgress: 79 -2015-03-30T15:39:16 [DEBUG] WebPage - updateLoadingProgress: 82 -2015-03-30T15:39:16 [DEBUG] WebPage - updateLoadingProgress: 85 -2015-03-30T15:39:16 [DEBUG] WebPage - updateLoadingProgress: 88 -2015-03-30T15:39:16 [DEBUG] Cache hit level 1 -2015-03-30T15:39:16 [DEBUG] Cache hit level 1 -2015-03-30T15:39:16 [DEBUG] Cache hit level 1 -2015-03-30T15:39:16 [DEBUG] Cache hit level 1 -2015-03-30T15:39:16 [DEBUG] Cache hit level 1 -2015-03-30T15:39:16 [DEBUG] Cache hit level 1 -2015-03-30T15:39:16 [DEBUG] Cache hit level 1 -2015-03-30T15:39:16 [DEBUG] Cache hit level 1 -2015-03-30T15:39:16 [DEBUG] Cache hit level 1 -2015-03-30T15:39:16 [DEBUG] Cache hit level 1 -2015-03-30T15:39:16 [DEBUG] Cache hit level 1 -2015-03-30T15:39:16 [DEBUG] Cache hit level 1 -2015-03-30T15:39:16 [DEBUG] Cache hit level 1 -2015-03-30T15:39:16 [DEBUG] Cache hit level 1 -[DEBUG - 2015-03-30T07:39:16.992Z] Session [dab84dd0-d6af-11e4-95ed-a71961c71b3a] - _execFuncAndWaitForLoadDecorator - Page Loading in Session: true -2015-03-30T15:39:17 [DEBUG] Cache hit level 1 -2015-03-30T15:39:17 [DEBUG] Cache hit level 1 -2015-03-30T15:39:17 [DEBUG] Cache hit level 1 -2015-03-30T15:39:17 [DEBUG] Cache hit level 1 -2015-03-30T15:39:17 [DEBUG] Cache hit level 1 -2015-03-30T15:39:17 [DEBUG] Cache hit level 1 -2015-03-30T15:39:17 [DEBUG] Cache hit level 1 -[DEBUG - 2015-03-30T07:39:17.159Z] Session [dab84dd0-d6af-11e4-95ed-a71961c71b3a] - _execFuncAndWaitForLoadDecorator - Page Loading in Session: true -2015-03-30T15:39:17 [DEBUG] WebPage - updateLoadingProgress: 88 -[DEBUG - 2015-03-30T07:39:17.377Z] Session [dab84dd0-d6af-11e4-95ed-a71961c71b3a] - _execFuncAndWaitForLoadDecorator - Page Loading in Session: true -[DEBUG - 2015-03-30T07:39:17.477Z] Session [dab84dd0-d6af-11e4-95ed-a71961c71b3a] - _execFuncAndWaitForLoadDecorator - Page Loading in Session: true -[DEBUG - 2015-03-30T07:39:17.577Z] Session [dab84dd0-d6af-11e4-95ed-a71961c71b3a] - _execFuncAndWaitForLoadDecorator - Page Loading in Session: true -2015-03-30T15:39:17 [DEBUG] WebPage - updateLoadingProgress: 88 -[DEBUG - 2015-03-30T07:39:17.678Z] Session [dab84dd0-d6af-11e4-95ed-a71961c71b3a] - _execFuncAndWaitForLoadDecorator - Page Loading in Session: true -2015-03-30T15:39:17 [DEBUG] CookieJar - Saved "HMVT=737dbb498415dd39d8abf5bc2404b290|1427701152|; domain=.hm.baidu.com; path=/" -2015-03-30T15:39:17 [DEBUG] CookieJar - Saved "HMACCOUNT=8E373D7C3072E1C3; expires=Mon, 18-Jan-2038 00:00:00 GMT; domain=.hm.baidu.com; path=/" -2015-03-30T15:39:17 [DEBUG] CookieJar - Saved "Hm_lvt_737dbb498415dd39d8abf5bc2404b290=1427700229,1427700537,1427700716,1427701152; expires=Tue, 29-Mar-2016 07:39:12 GMT; domain=.image.baidu.com; path=/" -2015-03-30T15:39:17 [DEBUG] CookieJar - Saved "Hm_lpvt_737dbb498415dd39d8abf5bc2404b290=1427701152; domain=.image.baidu.com; path=/" -2015-03-30T15:39:17 [DEBUG] CookieJar - Saved "BAIDUID=383833C1F0BD08B9311CD7DD32FB6879:FG=1; expires=Tue, 29-Mar-2016 07:39:15 GMT; domain=.baidu.com; path=/" -2015-03-30T15:39:17 [DEBUG] CookieJar - Saved "BDV_TRACE=1427701157734:; expires=Mon, 06-Apr-2015 07:39:17 GMT; domain=v.baidu.com; path=/" -2015-03-30T15:39:17 [DEBUG] WebPage - updateLoadingProgress: 88 -[DEBUG - 2015-03-30T07:39:17.778Z] Session [dab84dd0-d6af-11e4-95ed-a71961c71b3a] - _execFuncAndWaitForLoadDecorator - Page Loading in Session: true -[DEBUG - 2015-03-30T07:39:17.878Z] Session [dab84dd0-d6af-11e4-95ed-a71961c71b3a] - _execFuncAndWaitForLoadDecorator - Page Loading in Session: true -2015-03-30T15:39:17 [DEBUG] WebPage - updateLoadingProgress: 89 -[DEBUG - 2015-03-30T07:39:17.978Z] Session [dab84dd0-d6af-11e4-95ed-a71961c71b3a] - _execFuncAndWaitForLoadDecorator - Page Loading in Session: true -2015-03-30T15:39:18 [DEBUG] WebPage - updateLoadingProgress: 90 -[DEBUG - 2015-03-30T07:39:18.078Z] Session [dab84dd0-d6af-11e4-95ed-a71961c71b3a] - _execFuncAndWaitForLoadDecorator - Page Loading in Session: true -2015-03-30T15:39:18 [DEBUG] CookieJar - Saved "HMVT=737dbb498415dd39d8abf5bc2404b290|1427701152|; domain=.hm.baidu.com; path=/" -2015-03-30T15:39:18 [DEBUG] CookieJar - Saved "HMACCOUNT=8E373D7C3072E1C3; expires=Mon, 18-Jan-2038 00:00:00 GMT; domain=.hm.baidu.com; path=/" -2015-03-30T15:39:18 [DEBUG] CookieJar - Saved "Hm_lvt_737dbb498415dd39d8abf5bc2404b290=1427700229,1427700537,1427700716,1427701152; expires=Tue, 29-Mar-2016 07:39:12 GMT; domain=.image.baidu.com; path=/" -2015-03-30T15:39:18 [DEBUG] CookieJar - Saved "Hm_lpvt_737dbb498415dd39d8abf5bc2404b290=1427701152; domain=.image.baidu.com; path=/" -2015-03-30T15:39:18 [DEBUG] CookieJar - Saved "BAIDUID=383833C1F0BD08B9311CD7DD32FB6879:FG=1; expires=Tue, 29-Mar-2016 07:39:15 GMT; domain=.baidu.com; path=/" -2015-03-30T15:39:18 [DEBUG] CookieJar - Saved "BDV_TRACE=1427701157734:; expires=Mon, 06-Apr-2015 07:39:17 GMT; domain=v.baidu.com; path=/" -2015-03-30T15:39:18 [DEBUG] CookieJar - Saved "mc_2015-03-30="827@1"; expires=Mon, 30-Mar-2015 08:39:18 GMT; domain=bubble.baidu.com; path=/rs/message/" -2015-03-30T15:39:18 [DEBUG] CookieJar - Saved "PUSH_TIMESTAMP=1427701158092; expires=Mon, 30-Mar-2015 08:39:18 GMT; domain=bubble.baidu.com; path=/rs/message/" -2015-03-30T15:39:18 [DEBUG] WebPage - updateLoadingProgress: 90 -2015-03-30T15:39:18 [DEBUG] WebPage - setupFrame "-->" -2015-03-30T15:39:18 [DEBUG] WebPage - updateLoadingProgress: 90 -[DEBUG - 2015-03-30T07:39:18.179Z] Session [dab84dd0-d6af-11e4-95ed-a71961c71b3a] - _execFuncAndWaitForLoadDecorator - Page Loading in Session: true -[DEBUG - 2015-03-30T07:39:18.314Z] Session [dab84dd0-d6af-11e4-95ed-a71961c71b3a] - _execFuncAndWaitForLoadDecorator - Page Loading in Session: true -2015-03-30T15:39:18 [DEBUG] WebPage - updateLoadingProgress: 90 -2015-03-30T15:39:18 [DEBUG] WebPage - setupFrame "-->" -2015-03-30T15:39:18 [DEBUG] WebPage - updateLoadingProgress: 90 -2015-03-30T15:39:18 [DEBUG] WebPage - setupFrame "baidu_clb_slot_iframe_1051937_0" -2015-03-30T15:39:18 [DEBUG] WebPage - updateLoadingProgress: 90 -2015-03-30T15:39:18 [DEBUG] WebPage - setupFrame "adframe_wrap_64593" -2015-03-30T15:39:18 [DEBUG] WebPage - updateLoadingProgress: 90 -2015-03-30T15:39:18 [DEBUG] WebPage - setupFrame "cproIframe_639667_1" -2015-03-30T15:39:18 [DEBUG] WebPage - updateLoadingProgress: 90 -2015-03-30T15:39:18 [DEBUG] WebPage - setupFrame "cproIframe_1040005_1" -2015-03-30T15:39:18 [DEBUG] WebPage - updateLoadingProgress: 90 -[DEBUG - 2015-03-30T07:39:18.415Z] Session [dab84dd0-d6af-11e4-95ed-a71961c71b3a] - _execFuncAndWaitForLoadDecorator - Page Loading in Session: true -2015-03-30T15:39:18 [DEBUG] WebPage - setupFrame "-->" -2015-03-30T15:39:18 [DEBUG] WebPage - updateLoadingProgress: 90 -2015-03-30T15:39:18 [DEBUG] WebPage - setupFrame "ifr39928" -2015-03-30T15:39:18 [DEBUG] WebPage - updateLoadingProgress: 90 -2015-03-30T15:39:18 [DEBUG] CookieJar - Saved "HMVT=737dbb498415dd39d8abf5bc2404b290|1427701152|; domain=.hm.baidu.com; path=/" -2015-03-30T15:39:18 [DEBUG] CookieJar - Saved "HMACCOUNT=8E373D7C3072E1C3; expires=Mon, 18-Jan-2038 00:00:00 GMT; domain=.hm.baidu.com; path=/" -2015-03-30T15:39:18 [DEBUG] CookieJar - Saved "Hm_lvt_737dbb498415dd39d8abf5bc2404b290=1427700229,1427700537,1427700716,1427701152; expires=Tue, 29-Mar-2016 07:39:12 GMT; domain=.image.baidu.com; path=/" -2015-03-30T15:39:18 [DEBUG] CookieJar - Saved "Hm_lpvt_737dbb498415dd39d8abf5bc2404b290=1427701152; domain=.image.baidu.com; path=/" -2015-03-30T15:39:18 [DEBUG] CookieJar - Saved "BAIDUID=383833C1F0BD08B9311CD7DD32FB6879:FG=1; expires=Tue, 29-Mar-2016 07:39:15 GMT; domain=.baidu.com; path=/" -2015-03-30T15:39:18 [DEBUG] CookieJar - Saved "BDV_TRACE=1427701157734:; expires=Mon, 06-Apr-2015 07:39:17 GMT; domain=v.baidu.com; path=/" -2015-03-30T15:39:18 [DEBUG] CookieJar - Saved "mc_2015-03-30="827@1"; expires=Mon, 30-Mar-2015 08:39:18 GMT; domain=bubble.baidu.com; path=/rs/message/" -2015-03-30T15:39:18 [DEBUG] CookieJar - Saved "PUSH_TIMESTAMP=1427701158092; expires=Mon, 30-Mar-2015 08:39:18 GMT; domain=bubble.baidu.com; path=/rs/message/" -2015-03-30T15:39:18 [DEBUG] CookieJar - Saved "ISBID=383833C1F0BD08B9311CD7DD32FB6879:FG=1; expires=Tue, 31-Mar-2015 07:39:18 GMT; domain=cb.baidu.com; path=/" -2015-03-30T15:39:18 [DEBUG] CookieJar - Saved "HMVT=737dbb498415dd39d8abf5bc2404b290|1427701152|; domain=.hm.baidu.com; path=/" -2015-03-30T15:39:18 [DEBUG] CookieJar - Saved "HMACCOUNT=8E373D7C3072E1C3; expires=Mon, 18-Jan-2038 00:00:00 GMT; domain=.hm.baidu.com; path=/" -2015-03-30T15:39:18 [DEBUG] CookieJar - Saved "Hm_lvt_737dbb498415dd39d8abf5bc2404b290=1427700229,1427700537,1427700716,1427701152; expires=Tue, 29-Mar-2016 07:39:12 GMT; domain=.image.baidu.com; path=/" -2015-03-30T15:39:18 [DEBUG] CookieJar - Saved "Hm_lpvt_737dbb498415dd39d8abf5bc2404b290=1427701152; domain=.image.baidu.com; path=/" -2015-03-30T15:39:18 [DEBUG] CookieJar - Saved "BAIDUID=383833C1F0BD08B9311CD7DD32FB6879:FG=1; expires=Tue, 29-Mar-2016 07:39:15 GMT; domain=.baidu.com; path=/" -2015-03-30T15:39:18 [DEBUG] CookieJar - Saved "BDV_TRACE=1427701157734:; expires=Mon, 06-Apr-2015 07:39:17 GMT; domain=v.baidu.com; path=/" -2015-03-30T15:39:18 [DEBUG] CookieJar - Saved "mc_2015-03-30="827@1"; expires=Mon, 30-Mar-2015 08:39:18 GMT; domain=bubble.baidu.com; path=/rs/message/" -2015-03-30T15:39:18 [DEBUG] CookieJar - Saved "PUSH_TIMESTAMP=1427701158092; expires=Mon, 30-Mar-2015 08:39:18 GMT; domain=bubble.baidu.com; path=/rs/message/" -2015-03-30T15:39:18 [DEBUG] CookieJar - Saved "ISBID=383833C1F0BD08B9311CD7DD32FB6879:FG=1; expires=Tue, 31-Mar-2015 07:39:18 GMT; domain=cb.baidu.com; path=/" -2015-03-30T15:39:18 [DEBUG] CookieJar - Saved "ISUS=1; expires=Tue, 31-Mar-2015 07:39:18 GMT; domain=cb.baidu.com; path=/" -2015-03-30T15:39:18 [DEBUG] WebPage - updateLoadingProgress: 90 -[DEBUG - 2015-03-30T07:39:18.517Z] Session [dab84dd0-d6af-11e4-95ed-a71961c71b3a] - _execFuncAndWaitForLoadDecorator - Page Loading in Session: true -2015-03-30T15:39:18 [DEBUG] WebPage - setupFrame "-->" -2015-03-30T15:39:18 [DEBUG] WebPage - updateLoadingProgress: 90 -2015-03-30T15:39:18 [DEBUG] WebPage - setupFrame "ifr15222" -2015-03-30T15:39:18 [DEBUG] WebPage - updateLoadingProgress: 90 -[DEBUG - 2015-03-30T07:39:18.619Z] Session [dab84dd0-d6af-11e4-95ed-a71961c71b3a] - _execFuncAndWaitForLoadDecorator - Page Loading in Session: true -2015-03-30T15:39:18 [DEBUG] WebPage - updateLoadingProgress: 90 -[DEBUG - 2015-03-30T07:39:18.720Z] Session [dab84dd0-d6af-11e4-95ed-a71961c71b3a] - _execFuncAndWaitForLoadDecorator - Page Loading in Session: true -[DEBUG - 2015-03-30T07:39:18.821Z] Session [dab84dd0-d6af-11e4-95ed-a71961c71b3a] - _execFuncAndWaitForLoadDecorator - Page Loading in Session: true -[DEBUG - 2015-03-30T07:39:18.921Z] Session [dab84dd0-d6af-11e4-95ed-a71961c71b3a] - _execFuncAndWaitForLoadDecorator - Page Loading in Session: true -[DEBUG - 2015-03-30T07:39:19.021Z] Session [dab84dd0-d6af-11e4-95ed-a71961c71b3a] - _execFuncAndWaitForLoadDecorator - Page Loading in Session: true -[DEBUG - 2015-03-30T07:39:19.122Z] Session [dab84dd0-d6af-11e4-95ed-a71961c71b3a] - _execFuncAndWaitForLoadDecorator - Page Loading in Session: true -2015-03-30T15:39:19 [DEBUG] CookieJar - Saved "HMVT=737dbb498415dd39d8abf5bc2404b290|1427701152|; domain=.hm.baidu.com; path=/" -2015-03-30T15:39:19 [DEBUG] CookieJar - Saved "HMACCOUNT=8E373D7C3072E1C3; expires=Mon, 18-Jan-2038 00:00:00 GMT; domain=.hm.baidu.com; path=/" -2015-03-30T15:39:19 [DEBUG] CookieJar - Saved "Hm_lvt_737dbb498415dd39d8abf5bc2404b290=1427700229,1427700537,1427700716,1427701152; expires=Tue, 29-Mar-2016 07:39:12 GMT; domain=.image.baidu.com; path=/" -2015-03-30T15:39:19 [DEBUG] CookieJar - Saved "Hm_lpvt_737dbb498415dd39d8abf5bc2404b290=1427701152; domain=.image.baidu.com; path=/" -2015-03-30T15:39:19 [DEBUG] CookieJar - Saved "BAIDUID=383833C1F0BD08B9311CD7DD32FB6879:FG=1; expires=Tue, 29-Mar-2016 07:39:15 GMT; domain=.baidu.com; path=/" -2015-03-30T15:39:19 [DEBUG] CookieJar - Saved "BDV_TRACE=1427701157734:; expires=Mon, 06-Apr-2015 07:39:17 GMT; domain=v.baidu.com; path=/" -2015-03-30T15:39:19 [DEBUG] CookieJar - Saved "mc_2015-03-30="827@1"; expires=Mon, 30-Mar-2015 08:39:18 GMT; domain=bubble.baidu.com; path=/rs/message/" -2015-03-30T15:39:19 [DEBUG] CookieJar - Saved "PUSH_TIMESTAMP=1427701158092; expires=Mon, 30-Mar-2015 08:39:18 GMT; domain=bubble.baidu.com; path=/rs/message/" -2015-03-30T15:39:19 [DEBUG] CookieJar - Saved "ISBID=383833C1F0BD08B9311CD7DD32FB6879:FG=1; expires=Tue, 31-Mar-2015 07:39:18 GMT; domain=cb.baidu.com; path=/" -2015-03-30T15:39:19 [DEBUG] CookieJar - Saved "ISUS=1; expires=Tue, 31-Mar-2015 07:39:18 GMT; domain=cb.baidu.com; path=/" -[DEBUG - 2015-03-30T07:39:19.222Z] Session [dab84dd0-d6af-11e4-95ed-a71961c71b3a] - _execFuncAndWaitForLoadDecorator - Page Loading in Session: true -[DEBUG - 2015-03-30T07:39:19.322Z] Session [dab84dd0-d6af-11e4-95ed-a71961c71b3a] - _execFuncAndWaitForLoadDecorator - Page Loading in Session: true -[DEBUG - 2015-03-30T07:39:19.422Z] Session [dab84dd0-d6af-11e4-95ed-a71961c71b3a] - _execFuncAndWaitForLoadDecorator - Page Loading in Session: true -2015-03-30T15:39:19 [DEBUG] WebPage - updateLoadingProgress: 90 -2015-03-30T15:39:19 [DEBUG] WebPage - updateLoadingProgress: 100 -[DEBUG - 2015-03-30T07:39:19.476Z] Session [dab84dd0-d6af-11e4-95ed-a71961c71b3a] - _oneShotCallback - onLoadFinished -[DEBUG - 2015-03-30T07:39:19.476Z] Session [dab84dd0-d6af-11e4-95ed-a71961c71b3a] - _execFuncAndWaitForLoadDecorator - onLoadFinished: success -2015-03-30T15:39:19 [DEBUG] WebPage - setupFrame "" -2015-03-30T15:39:19 [DEBUG] WebPage - setupFrame "BAIDU_DUP_fp_iframe" -2015-03-30T15:39:19 [DEBUG] WebPage - updateLoadingProgress: 10 -2015-03-30T15:39:19 [DEBUG] WebPage - updateLoadingProgress: 30 -[DEBUG - 2015-03-30T07:39:19.523Z] Session [dab84dd0-d6af-11e4-95ed-a71961c71b3a] - _execFuncAndWaitForLoadDecorator - Page Loading in Session: true -[DEBUG - 2015-03-30T07:39:19.623Z] Session [dab84dd0-d6af-11e4-95ed-a71961c71b3a] - _execFuncAndWaitForLoadDecorator - Page Loading in Session: true -2015-03-30T15:39:19 [DEBUG] WebPage - updateLoadingProgress: 31 -2015-03-30T15:39:19 [DEBUG] WebPage - updateLoadingProgress: 40 -2015-03-30T15:39:19 [DEBUG] WebPage - setupFrame "-->" -2015-03-30T15:39:19 [DEBUG] WebPage - updateLoadingProgress: 40 -[DEBUG - 2015-03-30T07:39:19.724Z] Session [dab84dd0-d6af-11e4-95ed-a71961c71b3a] - _execFuncAndWaitForLoadDecorator - Page Loading in Session: true -2015-03-30T15:39:19 [DEBUG] WebPage - updateLoadingProgress: 42 -[DEBUG - 2015-03-30T07:39:19.824Z] Session [dab84dd0-d6af-11e4-95ed-a71961c71b3a] - _execFuncAndWaitForLoadDecorator - Page Loading in Session: true -2015-03-30T15:39:19 [DEBUG] WebPage - updateLoadingProgress: 100 -[DEBUG - 2015-03-30T07:39:19.925Z] Session [dab84dd0-d6af-11e4-95ed-a71961c71b3a] - _execFuncAndWaitForLoadDecorator - Page Loading in Session: false -[DEBUG - 2015-03-30T07:39:19.926Z] Session [dab84dd0-d6af-11e4-95ed-a71961c71b3a] - _resetOneShotCallbacksDecorator -[DEBUG - 2015-03-30T07:39:19.927Z] SessionReqHand - _SuccessHandler - status: success -2015-03-30T15:39:19 [DEBUG] HTTP Response - Status Code 200 OK -2015-03-30T15:39:19 [DEBUG] HTTP Response - Sending Header "Cache" = "no-cache" -2015-03-30T15:39:19 [DEBUG] HTTP Response - Sending Header "Content-Length" = "74" -2015-03-30T15:39:19 [DEBUG] HTTP Response - Sending Header "Content-Type" = "application/json;charset=UTF-8" -2015-03-30T15:39:19 [DEBUG] HTTP Request - URI /wd/hub/session/dab84dd0-d6af-11e4-95ed-a71961c71b3a/source -2015-03-30T15:39:19 [DEBUG] HTTP Request - Method GET -2015-03-30T15:39:19 [DEBUG] HTTP Request - HTTP Version 1.1 -2015-03-30T15:39:19 [DEBUG] HTTP Request - Query String -2015-03-30T15:39:19 [DEBUG] HTTP Request - Receiving Header "Accept-Encoding" = "identity" -2015-03-30T15:39:19 [DEBUG] HTTP Request - Receiving Header "Host" = "127.0.0.1:58200" -2015-03-30T15:39:19 [DEBUG] HTTP Request - Receiving Header "Content-Type" = "application/json;charset=UTF-8" -2015-03-30T15:39:19 [DEBUG] HTTP Request - Receiving Header "Connection" = "close" -2015-03-30T15:39:19 [DEBUG] HTTP Request - Receiving Header "Accept" = "application/json" -2015-03-30T15:39:19 [DEBUG] HTTP Request - Receiving Header "User-Agent" = "Python-urllib/2.7" -[DEBUG - 2015-03-30T07:39:19.933Z] RouterReqHand - _handle - Request => { - "headers": { - "Accept": "application/json", - "Accept-Encoding": "identity", - "Connection": "close", - "Content-Type": "application/json;charset=UTF-8", - "Host": "127.0.0.1:58200", - "User-Agent": "Python-urllib/2.7" - }, - "httpVersion": "1.1", - "method": "GET", - "url": "/wd/hub/session/dab84dd0-d6af-11e4-95ed-a71961c71b3a/source", - "urlParsed": { - "anchor": "", - "query": "", - "file": "source", - "directory": "/wd/hub/session/dab84dd0-d6af-11e4-95ed-a71961c71b3a/", - "path": "/wd/hub/session/dab84dd0-d6af-11e4-95ed-a71961c71b3a/source", - "relative": "/wd/hub/session/dab84dd0-d6af-11e4-95ed-a71961c71b3a/source", - "port": "", - "host": "", - "password": "", - "user": "", - "userInfo": "", - "authority": "", - "protocol": "", - "source": "/wd/hub/session/dab84dd0-d6af-11e4-95ed-a71961c71b3a/source", - "queryKey": {}, - "chunks": [ - "wd", - "hub", - "session", - "dab84dd0-d6af-11e4-95ed-a71961c71b3a", - "source" - ] - } -} -2015-03-30T15:39:19 [DEBUG] HTTP Response - Status Code 200 OK -2015-03-30T15:39:19 [DEBUG] HTTP Response - Sending Header "Cache" = "no-cache" -2015-03-30T15:39:19 [DEBUG] HTTP Response - Sending Header "Content-Length" = "321607" -2015-03-30T15:39:19 [DEBUG] HTTP Response - Sending Header "Content-Type" = "application/json;charset=UTF-8" +2015-03-30T16:18:22 [DEBUG] WebPage - setupFrame "" +2015-03-30T16:18:22 [DEBUG] FileSystem - _open: ":/modules/fs.js" QMap(("mode", QVariant(QString, "r") ) ) +2015-03-30T16:18:22 [DEBUG] FileSystem - _open: ":/modules/system.js" QMap(("mode", QVariant(QString, "r") ) ) +2015-03-30T16:18:22 [DEBUG] FileSystem - _open: ":/modules/webpage.js" QMap(("mode", QVariant(QString, "r") ) ) +2015-03-30T16:18:22 [DEBUG] FileSystem - _open: ":/modules/webserver.js" QMap(("mode", QVariant(QString, "r") ) ) +2015-03-30T16:18:22 [DEBUG] FileSystem - _open: ":/ghostdriver/./hub_register.js" QMap(("mode", QVariant(QString, "r") ) ) +2015-03-30T16:18:22 [DEBUG] FileSystem - _open: ":/ghostdriver/./logger.js" QMap(("mode", QVariant(QString, "r") ) ) +2015-03-30T16:18:22 [DEBUG] FileSystem - _open: ":/ghostdriver/./third_party/console++.js" QMap(("mode", QVariant(QString, "r") ) ) +2015-03-30T16:18:22 [DEBUG] FileSystem - _open: ":/ghostdriver/./config.js" QMap(("mode", QVariant(QString, "r") ) ) +[DEBUG - 2015-03-30T08:18:22.186Z] Config - config.init - {"ip":"127.0.0.1","port":"44021","hub":null,"logFile":null,"logLevel":"DEBUG","logColor":false} +2015-03-30T16:18:22 [DEBUG] FileSystem - _open: ":/ghostdriver/./third_party/parseuri.js" QMap(("mode", QVariant(QString, "r") ) ) +2015-03-30T16:18:22 [DEBUG] Phantom - injectJs: "session.js" +2015-03-30T16:18:22 [DEBUG] Phantom - injectJs: prepending ":/ghostdriver/" +2015-03-30T16:18:22 [DEBUG] Phantom - injectJs: "inputs.js" +2015-03-30T16:18:22 [DEBUG] Phantom - injectJs: prepending ":/ghostdriver/" +2015-03-30T16:18:22 [DEBUG] Phantom - injectJs: "request_handlers/request_handler.js" +2015-03-30T16:18:22 [DEBUG] Phantom - injectJs: prepending ":/ghostdriver/" +2015-03-30T16:18:22 [DEBUG] Phantom - injectJs: "request_handlers/status_request_handler.js" +2015-03-30T16:18:22 [DEBUG] Phantom - injectJs: prepending ":/ghostdriver/" +2015-03-30T16:18:22 [DEBUG] FileSystem - _open: ":/ghostdriver/./errors.js" QMap(("mode", QVariant(QString, "r") ) ) +2015-03-30T16:18:22 [DEBUG] Phantom - injectJs: "request_handlers/shutdown_request_handler.js" +2015-03-30T16:18:22 [DEBUG] Phantom - injectJs: prepending ":/ghostdriver/" +2015-03-30T16:18:22 [DEBUG] Phantom - injectJs: "request_handlers/session_manager_request_handler.js" +2015-03-30T16:18:22 [DEBUG] Phantom - injectJs: prepending ":/ghostdriver/" +2015-03-30T16:18:22 [DEBUG] Phantom - injectJs: "request_handlers/session_request_handler.js" +2015-03-30T16:18:22 [DEBUG] Phantom - injectJs: prepending ":/ghostdriver/" +2015-03-30T16:18:22 [DEBUG] Phantom - injectJs: "request_handlers/webelement_request_handler.js" +2015-03-30T16:18:22 [DEBUG] Phantom - injectJs: prepending ":/ghostdriver/" +2015-03-30T16:18:22 [DEBUG] Phantom - injectJs: "request_handlers/router_request_handler.js" +2015-03-30T16:18:22 [DEBUG] Phantom - injectJs: prepending ":/ghostdriver/" +2015-03-30T16:18:22 [DEBUG] Phantom - injectJs: "webelementlocator.js" +2015-03-30T16:18:22 [DEBUG] Phantom - injectJs: prepending ":/ghostdriver/" +[INFO - 2015-03-30T08:18:22.233Z] GhostDriver - Main - running on port 44021 +2015-03-30T16:18:23 [DEBUG] HTTP Request - URI /wd/hub/session +2015-03-30T16:18:23 [DEBUG] HTTP Request - Method POST +2015-03-30T16:18:23 [DEBUG] HTTP Request - HTTP Version 1.1 +2015-03-30T16:18:23 [DEBUG] HTTP Request - Query String +2015-03-30T16:18:23 [DEBUG] HTTP Request - Receiving Header "Accept-Encoding" = "identity" +2015-03-30T16:18:23 [DEBUG] HTTP Request - Receiving Header "Content-Length" = "100" +2015-03-30T16:18:23 [DEBUG] HTTP Request - Receiving Header "Host" = "127.0.0.1:44021" +2015-03-30T16:18:23 [DEBUG] HTTP Request - Receiving Header "Accept" = "application/json" +2015-03-30T16:18:23 [DEBUG] HTTP Request - Receiving Header "User-Agent" = "Python-urllib/2.7" +2015-03-30T16:18:23 [DEBUG] HTTP Request - Receiving Header "Connection" = "close" +2015-03-30T16:18:23 [DEBUG] HTTP Request - Receiving Header "Content-Type" = "application/json;charset=UTF-8" +2015-03-30T16:18:23 [DEBUG] HTTP Request - Method POST/PUT +2015-03-30T16:18:23 [DEBUG] HTTP Request - Content Body: {"desiredCapabilities": {"phantomjs.page.settings.userAgent": "Scrapy/0.24.5 (+http://scrapy.org)"}} +[DEBUG - 2015-03-30T08:18:23.125Z] RouterReqHand - _handle - {"headers":{"Accept":"application/json","Accept-Encoding":"identity","Connection":"close","Content-Length":"100","Content-Type":"application/json;charset=UTF-8","Host":"127.0.0.1:44021","User-Agent":"Python-urllib/2.7"},"httpVersion":"1.1","method":"POST","post":"{\"desiredCapabilities\": {\"phantomjs.page.settings.userAgent\": \"Scrapy/0.24.5 (+http://scrapy.org)\"}}","url":"/session","urlParsed":{"anchor":"","query":"","file":"session","directory":"/","path":"/session","relative":"/session","port":"","host":"","password":"","user":"","userInfo":"","authority":"","protocol":"","source":"/session","queryKey":{},"chunks":["session"]}} +2015-03-30T16:18:23 [DEBUG] FileSystem - _open: ":/ghostdriver/./third_party/uuid.js" QMap(("mode", QVariant(QString, "r") ) ) +2015-03-30T16:18:23 [DEBUG] WebpageCallbacks - getFilePickerCallback +2015-03-30T16:18:23 [DEBUG] WebpageCallbacks - getGenericCallback +[INFO - 2015-03-30T08:18:23.130Z] Session [54fd5270-d6b5-11e4-bf64-67a72685a843] - page.settings - {"XSSAuditingEnabled":false,"javascriptCanCloseWindows":true,"javascriptCanOpenWindows":true,"javascriptEnabled":true,"loadImages":false,"localToRemoteUrlAccessEnabled":false,"userAgent":"Scrapy/0.24.5 (+http://scrapy.org)","webSecurityEnabled":true} +[INFO - 2015-03-30T08:18:23.130Z] Session [54fd5270-d6b5-11e4-bf64-67a72685a843] - page.customHeaders: - {} +[DEBUG - 2015-03-30T08:18:23.130Z] Session [54fd5270-d6b5-11e4-bf64-67a72685a843] - Session.desiredCapabilities - {"phantomjs.page.settings.userAgent":"Scrapy/0.24.5 (+http://scrapy.org)"} +[INFO - 2015-03-30T08:18:23.130Z] Session [54fd5270-d6b5-11e4-bf64-67a72685a843] - Session.negotiatedCapabilities - {"browserName":"phantomjs","version":"1.9.8","driverName":"ghostdriver","driverVersion":"1.1.0","platform":"linux-unknown-64bit","javascriptEnabled":true,"takesScreenshot":true,"handlesAlerts":false,"databaseEnabled":false,"locationContextEnabled":false,"applicationCacheEnabled":false,"browserConnectionEnabled":false,"cssSelectorsEnabled":true,"webStorageEnabled":false,"rotatable":false,"acceptSslCerts":false,"nativeEvents":true,"proxy":{"proxyType":"direct"},"phantomjs.page.settings.userAgent":"Scrapy/0.24.5 (+http://scrapy.org)"} +[INFO - 2015-03-30T08:18:23.130Z] SessionManagerReqHand - _postNewSessionCommand - New Session Created: 54fd5270-d6b5-11e4-bf64-67a72685a843 +2015-03-30T16:18:23 [DEBUG] HTTP Response - Status Code 200 OK +2015-03-30T16:18:23 [DEBUG] HTTP Response - Sending Header "Cache" = "no-cache" +2015-03-30T16:18:23 [DEBUG] HTTP Response - Sending Header "Content-Length" = "609" +2015-03-30T16:18:23 [DEBUG] HTTP Response - Sending Header "Content-Type" = "application/json;charset=UTF-8" +2015-03-30T16:18:23 [DEBUG] HTTP Request - URI /wd/hub/session/54fd5270-d6b5-11e4-bf64-67a72685a843/url +2015-03-30T16:18:23 [DEBUG] HTTP Request - Method POST +2015-03-30T16:18:23 [DEBUG] HTTP Request - HTTP Version 1.1 +2015-03-30T16:18:23 [DEBUG] HTTP Request - Query String +2015-03-30T16:18:23 [DEBUG] HTTP Request - Receiving Header "Accept-Encoding" = "identity" +2015-03-30T16:18:23 [DEBUG] HTTP Request - Receiving Header "Content-Length" = "87" +2015-03-30T16:18:23 [DEBUG] HTTP Request - Receiving Header "Host" = "127.0.0.1:44021" +2015-03-30T16:18:23 [DEBUG] HTTP Request - Receiving Header "Accept" = "application/json" +2015-03-30T16:18:23 [DEBUG] HTTP Request - Receiving Header "User-Agent" = "Python-urllib/2.7" +2015-03-30T16:18:23 [DEBUG] HTTP Request - Receiving Header "Connection" = "close" +2015-03-30T16:18:23 [DEBUG] HTTP Request - Receiving Header "Content-Type" = "application/json;charset=UTF-8" +2015-03-30T16:18:23 [DEBUG] HTTP Request - Method POST/PUT +2015-03-30T16:18:23 [DEBUG] HTTP Request - Content Body: {"url": "http://image.baidu.com/", "sessionId": "54fd5270-d6b5-11e4-bf64-67a72685a843"} +[DEBUG - 2015-03-30T08:18:23.136Z] RouterReqHand - _handle - {"headers":{"Accept":"application/json","Accept-Encoding":"identity","Connection":"close","Content-Length":"87","Content-Type":"application/json;charset=UTF-8","Host":"127.0.0.1:44021","User-Agent":"Python-urllib/2.7"},"httpVersion":"1.1","method":"POST","post":"{\"url\": \"http://image.baidu.com/\", \"sessionId\": \"54fd5270-d6b5-11e4-bf64-67a72685a843\"}","url":"/session/54fd5270-d6b5-11e4-bf64-67a72685a843/url","urlParsed":{"anchor":"","query":"","file":"url","directory":"/session/54fd5270-d6b5-11e4-bf64-67a72685a843/","path":"/session/54fd5270-d6b5-11e4-bf64-67a72685a843/url","relative":"/session/54fd5270-d6b5-11e4-bf64-67a72685a843/url","port":"","host":"","password":"","user":"","userInfo":"","authority":"","protocol":"","source":"/session/54fd5270-d6b5-11e4-bf64-67a72685a843/url","queryKey":{},"chunks":["session","54fd5270-d6b5-11e4-bf64-67a72685a843","url"]}} +[DEBUG - 2015-03-30T08:18:23.137Z] SessionReqHand - _postUrlCommand - Session '54fd5270-d6b5-11e4-bf64-67a72685a843' is about to load URL: http://image.baidu.com/ +2015-03-30T16:18:23 [DEBUG] WebPage - updateLoadingProgress: 10 +[DEBUG - 2015-03-30T08:18:23.137Z] Session [54fd5270-d6b5-11e4-bf64-67a72685a843] - page.onResourceRequested - {"headers":[{"name":"User-Agent","value":"Scrapy/0.24.5 (+http://scrapy.org)"},{"name":"Accept","value":"text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8"}],"id":1,"method":"GET","time":"2015-03-30T08:18:23.137Z","url":"http://image.baidu.com/"} +[DEBUG - 2015-03-30T08:18:23.138Z] Session [54fd5270-d6b5-11e4-bf64-67a72685a843] - _oneShotCallback - onLoadStarted +[DEBUG - 2015-03-30T08:18:23.148Z] Session [54fd5270-d6b5-11e4-bf64-67a72685a843] - _execFuncAndWaitForLoadDecorator - Page Loading in Session: true +2015-03-30T16:18:23 [DEBUG] CookieJar - Saved "BAIDUID=DB1D387F3D1BF6229139D2C032ACF8DF:FG=1; expires=Tue, 29-Mar-2016 08:18:23 GMT; domain=.baidu.com; path=/" +[DEBUG - 2015-03-30T08:18:23.235Z] Session [54fd5270-d6b5-11e4-bf64-67a72685a843] - page.onResourceReceived - {"bodySize":85955,"contentType":"text/html","headers":[{"name":"Content-Encoding","value":"gzip"},{"name":"Content-Type","value":"text/html"},{"name":"Date","value":"Mon, 30 Mar 2015 08:18:23 GMT"},{"name":"P3p","value":"CP=\" OTI DSP COR IVA OUR IND COM \""},{"name":"Server","value":"Apache"},{"name":"Set-Cookie","value":"BAIDUID=DB1D387F3D1BF6229139D2C032ACF8DF:FG=1; max-age=31536000; expires=Tue, 29-Mar-16 08:18:23 GMT; domain=.baidu.com; path=/; version=1\nuserid=deleted; expires=Sun, 30-Mar-2014 08:18:22 GMT; path=/"},{"name":"Vary","value":"Accept-Encoding"},{"name":"Transfer-Encoding","value":"chunked"}],"id":1,"redirectURL":null,"stage":"start","status":200,"statusText":"OK","time":"2015-03-30T08:18:23.235Z","url":"http://image.baidu.com/"} +2015-03-30T16:18:23 [DEBUG] WebPage - setupFrame "" +[DEBUG - 2015-03-30T08:18:23.238Z] Session [54fd5270-d6b5-11e4-bf64-67a72685a843] - page.onResourceRequested - {"headers":[{"name":"User-Agent","value":"Scrapy/0.24.5 (+http://scrapy.org)"},{"name":"Accept","value":"*/*"},{"name":"Referer","value":"http://image.baidu.com/"}],"id":2,"method":"GET","time":"2015-03-30T08:18:23.238Z","url":"http://img0.bdstatic.com/static/common/fw_ff73dfc.js"} +[DEBUG - 2015-03-30T08:18:23.238Z] Session [54fd5270-d6b5-11e4-bf64-67a72685a843] - page.onResourceRequested - {"headers":[{"name":"User-Agent","value":"Scrapy/0.24.5 (+http://scrapy.org)"},{"name":"Accept","value":"text/css,*/*;q=0.1"},{"name":"Referer","value":"http://image.baidu.com/"}],"id":3,"method":"GET","time":"2015-03-30T08:18:23.238Z","url":"http://img1.bdstatic.com/static/common/pkg/base_b1e571e.css"} +[DEBUG - 2015-03-30T08:18:23.238Z] Session [54fd5270-d6b5-11e4-bf64-67a72685a843] - page.onResourceRequested - {"headers":[{"name":"User-Agent","value":"Scrapy/0.24.5 (+http://scrapy.org)"},{"name":"Accept","value":"text/css,*/*;q=0.1"},{"name":"Referer","value":"http://image.baidu.com/"}],"id":4,"method":"GET","time":"2015-03-30T08:18:23.238Z","url":"http://img2.bdstatic.com/static/common/pkg/page_a8fc491.css"} +[DEBUG - 2015-03-30T08:18:23.239Z] Session [54fd5270-d6b5-11e4-bf64-67a72685a843] - page.onResourceRequested - {"headers":[{"name":"User-Agent","value":"Scrapy/0.24.5 (+http://scrapy.org)"},{"name":"Accept","value":"text/css,*/*;q=0.1"},{"name":"Referer","value":"http://image.baidu.com/"}],"id":5,"method":"GET","time":"2015-03-30T08:18:23.239Z","url":"http://img0.bdstatic.com/static/home/pkg/default_44cee3f.css"} +2015-03-30T16:18:23 [DEBUG] WebPage - updateLoadingProgress: 24 +[DEBUG - 2015-03-30T08:18:23.239Z] Session [54fd5270-d6b5-11e4-bf64-67a72685a843] - page.onResourceReceived - {"contentType":"text/html","headers":[{"name":"Content-Encoding","value":"gzip"},{"name":"Content-Type","value":"text/html"},{"name":"Date","value":"Mon, 30 Mar 2015 08:18:23 GMT"},{"name":"P3p","value":"CP=\" OTI DSP COR IVA OUR IND COM \""},{"name":"Server","value":"Apache"},{"name":"Set-Cookie","value":"BAIDUID=DB1D387F3D1BF6229139D2C032ACF8DF:FG=1; max-age=31536000; expires=Tue, 29-Mar-16 08:18:23 GMT; domain=.baidu.com; path=/; version=1\nuserid=deleted; expires=Sun, 30-Mar-2014 08:18:22 GMT; path=/"},{"name":"Vary","value":"Accept-Encoding"},{"name":"Transfer-Encoding","value":"chunked"}],"id":1,"redirectURL":null,"stage":"end","status":200,"statusText":"OK","time":"2015-03-30T08:18:23.239Z","url":"http://image.baidu.com/"} +[DEBUG - 2015-03-30T08:18:23.248Z] Session [54fd5270-d6b5-11e4-bf64-67a72685a843] - _execFuncAndWaitForLoadDecorator - Page Loading in Session: true +[DEBUG - 2015-03-30T08:18:23.348Z] Session [54fd5270-d6b5-11e4-bf64-67a72685a843] - _execFuncAndWaitForLoadDecorator - Page Loading in Session: true +[DEBUG - 2015-03-30T08:18:23.448Z] Session [54fd5270-d6b5-11e4-bf64-67a72685a843] - _execFuncAndWaitForLoadDecorator - Page Loading in Session: true +[DEBUG - 2015-03-30T08:18:23.548Z] Session [54fd5270-d6b5-11e4-bf64-67a72685a843] - _execFuncAndWaitForLoadDecorator - Page Loading in Session: true +[DEBUG - 2015-03-30T08:18:23.648Z] Session [54fd5270-d6b5-11e4-bf64-67a72685a843] - _execFuncAndWaitForLoadDecorator - Page Loading in Session: true +[DEBUG - 2015-03-30T08:18:23.748Z] Session [54fd5270-d6b5-11e4-bf64-67a72685a843] - _execFuncAndWaitForLoadDecorator - Page Loading in Session: true +[DEBUG - 2015-03-30T08:18:23.848Z] Session [54fd5270-d6b5-11e4-bf64-67a72685a843] - _execFuncAndWaitForLoadDecorator - Page Loading in Session: true +[DEBUG - 2015-03-30T08:18:23.948Z] Session [54fd5270-d6b5-11e4-bf64-67a72685a843] - _execFuncAndWaitForLoadDecorator - Page Loading in Session: true +[DEBUG - 2015-03-30T08:18:24.048Z] Session [54fd5270-d6b5-11e4-bf64-67a72685a843] - _execFuncAndWaitForLoadDecorator - Page Loading in Session: true +[DEBUG - 2015-03-30T08:18:24.148Z] Session [54fd5270-d6b5-11e4-bf64-67a72685a843] - _execFuncAndWaitForLoadDecorator - Page Loading in Session: true +[DEBUG - 2015-03-30T08:18:24.248Z] Session [54fd5270-d6b5-11e4-bf64-67a72685a843] - _execFuncAndWaitForLoadDecorator - Page Loading in Session: true +[DEBUG - 2015-03-30T08:18:24.348Z] Session [54fd5270-d6b5-11e4-bf64-67a72685a843] - _execFuncAndWaitForLoadDecorator - Page Loading in Session: true +[DEBUG - 2015-03-30T08:18:24.449Z] Session [54fd5270-d6b5-11e4-bf64-67a72685a843] - _execFuncAndWaitForLoadDecorator - Page Loading in Session: true +[DEBUG - 2015-03-30T08:18:24.549Z] Session [54fd5270-d6b5-11e4-bf64-67a72685a843] - _execFuncAndWaitForLoadDecorator - Page Loading in Session: true +[DEBUG - 2015-03-30T08:18:24.649Z] Session [54fd5270-d6b5-11e4-bf64-67a72685a843] - _execFuncAndWaitForLoadDecorator - Page Loading in Session: true +[DEBUG - 2015-03-30T08:18:24.749Z] Session [54fd5270-d6b5-11e4-bf64-67a72685a843] - _execFuncAndWaitForLoadDecorator - Page Loading in Session: true +[DEBUG - 2015-03-30T08:18:24.849Z] Session [54fd5270-d6b5-11e4-bf64-67a72685a843] - _execFuncAndWaitForLoadDecorator - Page Loading in Session: true +[DEBUG - 2015-03-30T08:18:24.949Z] Session [54fd5270-d6b5-11e4-bf64-67a72685a843] - _execFuncAndWaitForLoadDecorator - Page Loading in Session: true +[DEBUG - 2015-03-30T08:18:25.049Z] Session [54fd5270-d6b5-11e4-bf64-67a72685a843] - _execFuncAndWaitForLoadDecorator - Page Loading in Session: true +[DEBUG - 2015-03-30T08:18:25.149Z] Session [54fd5270-d6b5-11e4-bf64-67a72685a843] - _execFuncAndWaitForLoadDecorator - Page Loading in Session: true +[DEBUG - 2015-03-30T08:18:25.249Z] Session [54fd5270-d6b5-11e4-bf64-67a72685a843] - _execFuncAndWaitForLoadDecorator - Page Loading in Session: true +[DEBUG - 2015-03-30T08:18:25.349Z] Session [54fd5270-d6b5-11e4-bf64-67a72685a843] - _execFuncAndWaitForLoadDecorator - Page Loading in Session: true +[DEBUG - 2015-03-30T08:18:25.449Z] Session [54fd5270-d6b5-11e4-bf64-67a72685a843] - _execFuncAndWaitForLoadDecorator - Page Loading in Session: true +[DEBUG - 2015-03-30T08:18:25.549Z] Session [54fd5270-d6b5-11e4-bf64-67a72685a843] - _execFuncAndWaitForLoadDecorator - Page Loading in Session: true +[DEBUG - 2015-03-30T08:18:25.649Z] Session [54fd5270-d6b5-11e4-bf64-67a72685a843] - _execFuncAndWaitForLoadDecorator - Page Loading in Session: true +[DEBUG - 2015-03-30T08:18:25.749Z] Session [54fd5270-d6b5-11e4-bf64-67a72685a843] - _execFuncAndWaitForLoadDecorator - Page Loading in Session: true +[DEBUG - 2015-03-30T08:18:25.849Z] Session [54fd5270-d6b5-11e4-bf64-67a72685a843] - _execFuncAndWaitForLoadDecorator - Page Loading in Session: true +[DEBUG - 2015-03-30T08:18:25.949Z] Session [54fd5270-d6b5-11e4-bf64-67a72685a843] - _execFuncAndWaitForLoadDecorator - Page Loading in Session: true +[DEBUG - 2015-03-30T08:18:26.049Z] Session [54fd5270-d6b5-11e4-bf64-67a72685a843] - _execFuncAndWaitForLoadDecorator - Page Loading in Session: true +[DEBUG - 2015-03-30T08:18:26.149Z] Session [54fd5270-d6b5-11e4-bf64-67a72685a843] - _execFuncAndWaitForLoadDecorator - Page Loading in Session: true +[DEBUG - 2015-03-30T08:18:26.249Z] Session [54fd5270-d6b5-11e4-bf64-67a72685a843] - _execFuncAndWaitForLoadDecorator - Page Loading in Session: true +[DEBUG - 2015-03-30T08:18:26.349Z] Session [54fd5270-d6b5-11e4-bf64-67a72685a843] - _execFuncAndWaitForLoadDecorator - Page Loading in Session: true +[DEBUG - 2015-03-30T08:18:26.449Z] Session [54fd5270-d6b5-11e4-bf64-67a72685a843] - _execFuncAndWaitForLoadDecorator - Page Loading in Session: true +[DEBUG - 2015-03-30T08:18:26.549Z] Session [54fd5270-d6b5-11e4-bf64-67a72685a843] - _execFuncAndWaitForLoadDecorator - Page Loading in Session: true +[DEBUG - 2015-03-30T08:18:26.649Z] Session [54fd5270-d6b5-11e4-bf64-67a72685a843] - _execFuncAndWaitForLoadDecorator - Page Loading in Session: true +[DEBUG - 2015-03-30T08:18:26.750Z] Session [54fd5270-d6b5-11e4-bf64-67a72685a843] - _execFuncAndWaitForLoadDecorator - Page Loading in Session: true +[DEBUG - 2015-03-30T08:18:26.850Z] Session [54fd5270-d6b5-11e4-bf64-67a72685a843] - _execFuncAndWaitForLoadDecorator - Page Loading in Session: true +[DEBUG - 2015-03-30T08:18:26.950Z] Session [54fd5270-d6b5-11e4-bf64-67a72685a843] - _execFuncAndWaitForLoadDecorator - Page Loading in Session: true +[DEBUG - 2015-03-30T08:18:27.050Z] Session [54fd5270-d6b5-11e4-bf64-67a72685a843] - _execFuncAndWaitForLoadDecorator - Page Loading in Session: true +[DEBUG - 2015-03-30T08:18:27.150Z] Session [54fd5270-d6b5-11e4-bf64-67a72685a843] - _execFuncAndWaitForLoadDecorator - Page Loading in Session: true +[DEBUG - 2015-03-30T08:18:27.250Z] Session [54fd5270-d6b5-11e4-bf64-67a72685a843] - _execFuncAndWaitForLoadDecorator - Page Loading in Session: true +[DEBUG - 2015-03-30T08:18:27.350Z] Session [54fd5270-d6b5-11e4-bf64-67a72685a843] - _execFuncAndWaitForLoadDecorator - Page Loading in Session: true +[DEBUG - 2015-03-30T08:18:27.450Z] Session [54fd5270-d6b5-11e4-bf64-67a72685a843] - _execFuncAndWaitForLoadDecorator - Page Loading in Session: true +[DEBUG - 2015-03-30T08:18:27.550Z] Session [54fd5270-d6b5-11e4-bf64-67a72685a843] - _execFuncAndWaitForLoadDecorator - Page Loading in Session: true +[DEBUG - 2015-03-30T08:18:27.650Z] Session [54fd5270-d6b5-11e4-bf64-67a72685a843] - _execFuncAndWaitForLoadDecorator - Page Loading in Session: true +[DEBUG - 2015-03-30T08:18:27.750Z] Session [54fd5270-d6b5-11e4-bf64-67a72685a843] - _execFuncAndWaitForLoadDecorator - Page Loading in Session: true +[DEBUG - 2015-03-30T08:18:27.850Z] Session [54fd5270-d6b5-11e4-bf64-67a72685a843] - _execFuncAndWaitForLoadDecorator - Page Loading in Session: true +[DEBUG - 2015-03-30T08:18:27.950Z] Session [54fd5270-d6b5-11e4-bf64-67a72685a843] - _execFuncAndWaitForLoadDecorator - Page Loading in Session: true +[DEBUG - 2015-03-30T08:18:28.050Z] Session [54fd5270-d6b5-11e4-bf64-67a72685a843] - _execFuncAndWaitForLoadDecorator - Page Loading in Session: true +[DEBUG - 2015-03-30T08:18:28.150Z] Session [54fd5270-d6b5-11e4-bf64-67a72685a843] - _execFuncAndWaitForLoadDecorator - Page Loading in Session: true +[DEBUG - 2015-03-30T08:18:28.250Z] Session [54fd5270-d6b5-11e4-bf64-67a72685a843] - _execFuncAndWaitForLoadDecorator - Page Loading in Session: true +[DEBUG - 2015-03-30T08:18:28.350Z] Session [54fd5270-d6b5-11e4-bf64-67a72685a843] - _execFuncAndWaitForLoadDecorator - Page Loading in Session: true +[DEBUG - 2015-03-30T08:18:28.450Z] Session [54fd5270-d6b5-11e4-bf64-67a72685a843] - _execFuncAndWaitForLoadDecorator - Page Loading in Session: true +[DEBUG - 2015-03-30T08:18:28.523Z] Session [54fd5270-d6b5-11e4-bf64-67a72685a843] - page.onResourceReceived - {"bodySize":99052,"contentType":"text/css","headers":[{"name":"Server","value":"JSP3/2.0.6"},{"name":"Date","value":"Mon, 30 Mar 2015 08:18:28 GMT"},{"name":"Content-Type","value":"text/css"},{"name":"Connection","value":"keep-alive"},{"name":"ETag","value":"\"1904792173\""},{"name":"Last-Modified","value":"Tue, 24 Mar 2015 03:28:45 GMT"},{"name":"Expires","value":"Tue, 31 Mar 2015 03:40:09 GMT"},{"name":"Age","value":"535099"},{"name":"Cache-Control","value":"max-age=604800"},{"name":"Accept-Ranges","value":"bytes"},{"name":"Content-Encoding","value":"gzip"},{"name":"Vary","value":"Accept-Encoding"}],"id":4,"redirectURL":null,"stage":"start","status":200,"statusText":"OK","time":"2015-03-30T08:18:28.523Z","url":"http://img2.bdstatic.com/static/common/pkg/page_a8fc491.css"} +2015-03-30T16:18:28 [DEBUG] WebPage - updateLoadingProgress: 34 +[DEBUG - 2015-03-30T08:18:28.524Z] Session [54fd5270-d6b5-11e4-bf64-67a72685a843] - page.onResourceReceived - {"contentType":"text/css","headers":[{"name":"Server","value":"JSP3/2.0.6"},{"name":"Date","value":"Mon, 30 Mar 2015 08:18:28 GMT"},{"name":"Content-Type","value":"text/css"},{"name":"Connection","value":"keep-alive"},{"name":"ETag","value":"\"1904792173\""},{"name":"Last-Modified","value":"Tue, 24 Mar 2015 03:28:45 GMT"},{"name":"Expires","value":"Tue, 31 Mar 2015 03:40:09 GMT"},{"name":"Age","value":"535099"},{"name":"Cache-Control","value":"max-age=604800"},{"name":"Accept-Ranges","value":"bytes"},{"name":"Content-Encoding","value":"gzip"},{"name":"Vary","value":"Accept-Encoding"}],"id":4,"redirectURL":null,"stage":"end","status":200,"statusText":"OK","time":"2015-03-30T08:18:28.524Z","url":"http://img2.bdstatic.com/static/common/pkg/page_a8fc491.css"} +[DEBUG - 2015-03-30T08:18:28.530Z] Session [54fd5270-d6b5-11e4-bf64-67a72685a843] - page.onResourceReceived - {"bodySize":1053,"contentType":"text/css","headers":[{"name":"Server","value":"JSP3/2.0.6"},{"name":"Date","value":"Mon, 30 Mar 2015 08:18:28 GMT"},{"name":"Content-Type","value":"text/css"},{"name":"Connection","value":"keep-alive"},{"name":"ETag","value":"\"50909773\""},{"name":"Last-Modified","value":"Tue, 24 Mar 2015 03:28:45 GMT"},{"name":"Expires","value":"Wed, 01 Apr 2015 03:47:27 GMT"},{"name":"Age","value":"448261"},{"name":"Cache-Control","value":"max-age=604800"},{"name":"Accept-Ranges","value":"bytes"},{"name":"Content-Encoding","value":"gzip"},{"name":"Vary","value":"Accept-Encoding"}],"id":3,"redirectURL":null,"stage":"start","status":200,"statusText":"OK","time":"2015-03-30T08:18:28.530Z","url":"http://img1.bdstatic.com/static/common/pkg/base_b1e571e.css"} +[DEBUG - 2015-03-30T08:18:28.531Z] Session [54fd5270-d6b5-11e4-bf64-67a72685a843] - page.onResourceReceived - {"bodySize":38471,"contentType":"text/css","headers":[{"name":"Server","value":"JSP3/2.0.6"},{"name":"Date","value":"Mon, 30 Mar 2015 08:18:28 GMT"},{"name":"Content-Type","value":"text/css"},{"name":"Connection","value":"keep-alive"},{"name":"ETag","value":"\"1704647181\""},{"name":"Last-Modified","value":"Thu, 26 Mar 2015 06:38:01 GMT"},{"name":"Expires","value":"Thu, 02 Apr 2015 07:49:41 GMT"},{"name":"Age","value":"347327"},{"name":"Cache-Control","value":"max-age=604800"},{"name":"Accept-Ranges","value":"bytes"},{"name":"Content-Encoding","value":"gzip"},{"name":"Vary","value":"Accept-Encoding"}],"id":5,"redirectURL":null,"stage":"start","status":200,"statusText":"OK","time":"2015-03-30T08:18:28.531Z","url":"http://img0.bdstatic.com/static/home/pkg/default_44cee3f.css"} +2015-03-30T16:18:28 [DEBUG] WebPage - updateLoadingProgress: 39 +[DEBUG - 2015-03-30T08:18:28.531Z] Session [54fd5270-d6b5-11e4-bf64-67a72685a843] - page.onResourceReceived - {"bodySize":2335,"contentType":"text/javascript","headers":[{"name":"Server","value":"JSP3/2.0.6"},{"name":"Date","value":"Mon, 30 Mar 2015 08:18:28 GMT"},{"name":"Content-Type","value":"text/javascript"},{"name":"Connection","value":"keep-alive"},{"name":"ETag","value":"\"54086285\""},{"name":"Last-Modified","value":"Tue, 24 Mar 2015 03:28:45 GMT"},{"name":"Expires","value":"Thu, 02 Apr 2015 02:47:25 GMT"},{"name":"Age","value":"365463"},{"name":"Cache-Control","value":"max-age=604800"},{"name":"Accept-Ranges","value":"bytes"},{"name":"Content-Encoding","value":"gzip"},{"name":"Vary","value":"Accept-Encoding"}],"id":2,"redirectURL":null,"stage":"start","status":200,"statusText":"OK","time":"2015-03-30T08:18:28.531Z","url":"http://img0.bdstatic.com/static/common/fw_ff73dfc.js"} +[DEBUG - 2015-03-30T08:18:28.531Z] Session [54fd5270-d6b5-11e4-bf64-67a72685a843] - page.onResourceReceived - {"contentType":"text/css","headers":[{"name":"Server","value":"JSP3/2.0.6"},{"name":"Date","value":"Mon, 30 Mar 2015 08:18:28 GMT"},{"name":"Content-Type","value":"text/css"},{"name":"Connection","value":"keep-alive"},{"name":"ETag","value":"\"50909773\""},{"name":"Last-Modified","value":"Tue, 24 Mar 2015 03:28:45 GMT"},{"name":"Expires","value":"Wed, 01 Apr 2015 03:47:27 GMT"},{"name":"Age","value":"448261"},{"name":"Cache-Control","value":"max-age=604800"},{"name":"Accept-Ranges","value":"bytes"},{"name":"Content-Encoding","value":"gzip"},{"name":"Vary","value":"Accept-Encoding"}],"id":3,"redirectURL":null,"stage":"end","status":200,"statusText":"OK","time":"2015-03-30T08:18:28.531Z","url":"http://img1.bdstatic.com/static/common/pkg/base_b1e571e.css"} +[DEBUG - 2015-03-30T08:18:28.531Z] Session [54fd5270-d6b5-11e4-bf64-67a72685a843] - page.onResourceReceived - {"contentType":"text/css","headers":[{"name":"Server","value":"JSP3/2.0.6"},{"name":"Date","value":"Mon, 30 Mar 2015 08:18:28 GMT"},{"name":"Content-Type","value":"text/css"},{"name":"Connection","value":"keep-alive"},{"name":"ETag","value":"\"1704647181\""},{"name":"Last-Modified","value":"Thu, 26 Mar 2015 06:38:01 GMT"},{"name":"Expires","value":"Thu, 02 Apr 2015 07:49:41 GMT"},{"name":"Age","value":"347327"},{"name":"Cache-Control","value":"max-age=604800"},{"name":"Accept-Ranges","value":"bytes"},{"name":"Content-Encoding","value":"gzip"},{"name":"Vary","value":"Accept-Encoding"}],"id":5,"redirectURL":null,"stage":"end","status":200,"statusText":"OK","time":"2015-03-30T08:18:28.531Z","url":"http://img0.bdstatic.com/static/home/pkg/default_44cee3f.css"} +[DEBUG - 2015-03-30T08:18:28.531Z] Session [54fd5270-d6b5-11e4-bf64-67a72685a843] - page.onResourceReceived - {"contentType":"text/javascript","headers":[{"name":"Server","value":"JSP3/2.0.6"},{"name":"Date","value":"Mon, 30 Mar 2015 08:18:28 GMT"},{"name":"Content-Type","value":"text/javascript"},{"name":"Connection","value":"keep-alive"},{"name":"ETag","value":"\"54086285\""},{"name":"Last-Modified","value":"Tue, 24 Mar 2015 03:28:45 GMT"},{"name":"Expires","value":"Thu, 02 Apr 2015 02:47:25 GMT"},{"name":"Age","value":"365463"},{"name":"Cache-Control","value":"max-age=604800"},{"name":"Accept-Ranges","value":"bytes"},{"name":"Content-Encoding","value":"gzip"},{"name":"Vary","value":"Accept-Encoding"}],"id":2,"redirectURL":null,"stage":"end","status":200,"statusText":"OK","time":"2015-03-30T08:18:28.531Z","url":"http://img0.bdstatic.com/static/common/fw_ff73dfc.js"} +[DEBUG - 2015-03-30T08:18:28.563Z] Session [54fd5270-d6b5-11e4-bf64-67a72685a843] - _execFuncAndWaitForLoadDecorator - Page Loading in Session: true +[DEBUG - 2015-03-30T08:18:28.566Z] Session [54fd5270-d6b5-11e4-bf64-67a72685a843] - page.onResourceRequested - {"headers":[{"name":"User-Agent","value":"Scrapy/0.24.5 (+http://scrapy.org)"},{"name":"Accept","value":"*/*"},{"name":"Referer","value":"http://image.baidu.com/"}],"id":6,"method":"GET","time":"2015-03-30T08:18:28.566Z","url":"http://img.baidu.com/hunter/image.js?st=-16525"} +[DEBUG - 2015-03-30T08:18:28.567Z] Session [54fd5270-d6b5-11e4-bf64-67a72685a843] - page.onResourceRequested - {"headers":[{"name":"User-Agent","value":"Scrapy/0.24.5 (+http://scrapy.org)"},{"name":"Accept","value":"*/*"},{"name":"Referer","value":"http://image.baidu.com/"}],"id":7,"method":"GET","time":"2015-03-30T08:18:28.567Z","url":"http://hm.baidu.com/h.js?737dbb498415dd39d8abf5bc2404b290"} +[DEBUG - 2015-03-30T08:18:28.569Z] Session [54fd5270-d6b5-11e4-bf64-67a72685a843] - page.onResourceRequested - {"headers":[{"name":"User-Agent","value":"Scrapy/0.24.5 (+http://scrapy.org)"},{"name":"Accept","value":"*/*"},{"name":"Referer","value":"http://image.baidu.com/"}],"id":8,"method":"GET","time":"2015-03-30T08:18:28.569Z","url":"http://img0.bdstatic.com/static/common/pkg/base_9584ab6.js"} +[DEBUG - 2015-03-30T08:18:28.569Z] Session [54fd5270-d6b5-11e4-bf64-67a72685a843] - page.onResourceRequested - {"headers":[{"name":"User-Agent","value":"Scrapy/0.24.5 (+http://scrapy.org)"},{"name":"Accept","value":"*/*"},{"name":"Referer","value":"http://image.baidu.com/"}],"id":9,"method":"GET","time":"2015-03-30T08:18:28.569Z","url":"http://img2.bdstatic.com/static/common/pkg/com_bd5e594.js"} +[DEBUG - 2015-03-30T08:18:28.570Z] Session [54fd5270-d6b5-11e4-bf64-67a72685a843] - page.onResourceRequested - {"headers":[{"name":"User-Agent","value":"Scrapy/0.24.5 (+http://scrapy.org)"},{"name":"Accept","value":"*/*"},{"name":"Referer","value":"http://image.baidu.com/"}],"id":10,"method":"GET","time":"2015-03-30T08:18:28.570Z","url":"http://img2.bdstatic.com/static/common/pkg/other_5dfc0a1.js"} +[DEBUG - 2015-03-30T08:18:28.570Z] Session [54fd5270-d6b5-11e4-bf64-67a72685a843] - page.onResourceRequested - {"headers":[{"name":"User-Agent","value":"Scrapy/0.24.5 (+http://scrapy.org)"},{"name":"Accept","value":"*/*"},{"name":"Referer","value":"http://image.baidu.com/"}],"id":11,"method":"GET","time":"2015-03-30T08:18:28.570Z","url":"http://img2.bdstatic.com/static/home/pkg/default_ad84cc1.js"} +[DEBUG - 2015-03-30T08:18:28.576Z] Session [54fd5270-d6b5-11e4-bf64-67a72685a843] - page.onResourceReceived - {"bodySize":11312,"contentType":"text/javascript","headers":[{"name":"Server","value":"JSP3/2.0.6"},{"name":"Date","value":"Mon, 30 Mar 2015 08:18:28 GMT"},{"name":"Content-Type","value":"text/javascript"},{"name":"Connection","value":"keep-alive"},{"name":"ETag","value":"\"1627996781\""},{"name":"Last-Modified","value":"Fri, 27 Mar 2015 03:53:28 GMT"},{"name":"Expires","value":"Fri, 03 Apr 2015 04:04:36 GMT"},{"name":"Age","value":"274432"},{"name":"Cache-Control","value":"max-age=604800"},{"name":"Accept-Ranges","value":"bytes"},{"name":"Content-Encoding","value":"gzip"},{"name":"Vary","value":"Accept-Encoding"}],"id":11,"redirectURL":null,"stage":"start","status":200,"statusText":"OK","time":"2015-03-30T08:18:28.576Z","url":"http://img2.bdstatic.com/static/home/pkg/default_ad84cc1.js"} +2015-03-30T16:18:28 [DEBUG] WebPage - updateLoadingProgress: 44 +[DEBUG - 2015-03-30T08:18:28.576Z] Session [54fd5270-d6b5-11e4-bf64-67a72685a843] - page.onResourceReceived - {"contentType":"text/javascript","headers":[{"name":"Server","value":"JSP3/2.0.6"},{"name":"Date","value":"Mon, 30 Mar 2015 08:18:28 GMT"},{"name":"Content-Type","value":"text/javascript"},{"name":"Connection","value":"keep-alive"},{"name":"ETag","value":"\"1627996781\""},{"name":"Last-Modified","value":"Fri, 27 Mar 2015 03:53:28 GMT"},{"name":"Expires","value":"Fri, 03 Apr 2015 04:04:36 GMT"},{"name":"Age","value":"274432"},{"name":"Cache-Control","value":"max-age=604800"},{"name":"Accept-Ranges","value":"bytes"},{"name":"Content-Encoding","value":"gzip"},{"name":"Vary","value":"Accept-Encoding"}],"id":11,"redirectURL":null,"stage":"end","status":200,"statusText":"OK","time":"2015-03-30T08:18:28.576Z","url":"http://img2.bdstatic.com/static/home/pkg/default_ad84cc1.js"} +[DEBUG - 2015-03-30T08:18:28.579Z] Session [54fd5270-d6b5-11e4-bf64-67a72685a843] - page.onResourceReceived - {"bodySize":59907,"contentType":"text/javascript","headers":[{"name":"Server","value":"JSP3/2.0.6"},{"name":"Date","value":"Mon, 30 Mar 2015 08:18:28 GMT"},{"name":"Content-Type","value":"text/javascript"},{"name":"Connection","value":"keep-alive"},{"name":"ETag","value":"\"1936212525\""},{"name":"Last-Modified","value":"Tue, 24 Mar 2015 03:28:45 GMT"},{"name":"Expires","value":"Wed, 01 Apr 2015 11:15:58 GMT"},{"name":"Age","value":"421350"},{"name":"Cache-Control","value":"max-age=604800"},{"name":"Accept-Ranges","value":"bytes"},{"name":"Content-Encoding","value":"gzip"},{"name":"Vary","value":"Accept-Encoding"}],"id":9,"redirectURL":null,"stage":"start","status":200,"statusText":"OK","time":"2015-03-30T08:18:28.579Z","url":"http://img2.bdstatic.com/static/common/pkg/com_bd5e594.js"} +2015-03-30T16:18:28 [DEBUG] WebPage - updateLoadingProgress: 58 +2015-03-30T16:18:28 [DEBUG] WebPage - updateLoadingProgress: 64 +[DEBUG - 2015-03-30T08:18:28.580Z] Session [54fd5270-d6b5-11e4-bf64-67a72685a843] - page.onResourceReceived - {"contentType":"text/javascript","headers":[{"name":"Server","value":"JSP3/2.0.6"},{"name":"Date","value":"Mon, 30 Mar 2015 08:18:28 GMT"},{"name":"Content-Type","value":"text/javascript"},{"name":"Connection","value":"keep-alive"},{"name":"ETag","value":"\"1936212525\""},{"name":"Last-Modified","value":"Tue, 24 Mar 2015 03:28:45 GMT"},{"name":"Expires","value":"Wed, 01 Apr 2015 11:15:58 GMT"},{"name":"Age","value":"421350"},{"name":"Cache-Control","value":"max-age=604800"},{"name":"Accept-Ranges","value":"bytes"},{"name":"Content-Encoding","value":"gzip"},{"name":"Vary","value":"Accept-Encoding"}],"id":9,"redirectURL":null,"stage":"end","status":200,"statusText":"OK","time":"2015-03-30T08:18:28.580Z","url":"http://img2.bdstatic.com/static/common/pkg/com_bd5e594.js"} +[DEBUG - 2015-03-30T08:18:28.583Z] Session [54fd5270-d6b5-11e4-bf64-67a72685a843] - page.onResourceReceived - {"bodySize":52796,"contentType":"text/javascript","headers":[{"name":"Server","value":"JSP3/2.0.6"},{"name":"Date","value":"Mon, 30 Mar 2015 08:18:28 GMT"},{"name":"Content-Type","value":"text/javascript"},{"name":"Connection","value":"keep-alive"},{"name":"ETag","value":"\"1765499629\""},{"name":"Last-Modified","value":"Fri, 20 Mar 2015 11:03:30 GMT"},{"name":"Expires","value":"Mon, 30 Mar 2015 09:17:15 GMT"},{"name":"Age","value":"601273"},{"name":"Cache-Control","value":"max-age=604800"},{"name":"Accept-Ranges","value":"bytes"},{"name":"Content-Encoding","value":"gzip"},{"name":"Vary","value":"Accept-Encoding"}],"id":10,"redirectURL":null,"stage":"start","status":200,"statusText":"OK","time":"2015-03-30T08:18:28.583Z","url":"http://img2.bdstatic.com/static/common/pkg/other_5dfc0a1.js"} +2015-03-30T16:18:28 [DEBUG] WebPage - updateLoadingProgress: 72 +[DEBUG - 2015-03-30T08:18:28.583Z] Session [54fd5270-d6b5-11e4-bf64-67a72685a843] - page.onResourceReceived - {"bodySize":98066,"contentType":"text/javascript","headers":[{"name":"Server","value":"JSP3/2.0.6"},{"name":"Date","value":"Mon, 30 Mar 2015 08:18:28 GMT"},{"name":"Content-Type","value":"text/javascript"},{"name":"Connection","value":"keep-alive"},{"name":"ETag","value":"\"1905843949\""},{"name":"Last-Modified","value":"Tue, 24 Mar 2015 03:28:45 GMT"},{"name":"Expires","value":"Thu, 02 Apr 2015 13:47:28 GMT"},{"name":"Age","value":"325860"},{"name":"Cache-Control","value":"max-age=604800"},{"name":"Accept-Ranges","value":"bytes"},{"name":"Content-Encoding","value":"gzip"},{"name":"Vary","value":"Accept-Encoding"}],"id":8,"redirectURL":null,"stage":"start","status":200,"statusText":"OK","time":"2015-03-30T08:18:28.583Z","url":"http://img0.bdstatic.com/static/common/pkg/base_9584ab6.js"} +2015-03-30T16:18:28 [DEBUG] WebPage - updateLoadingProgress: 78 +[DEBUG - 2015-03-30T08:18:28.583Z] Session [54fd5270-d6b5-11e4-bf64-67a72685a843] - page.onResourceReceived - {"contentType":"text/javascript","headers":[{"name":"Server","value":"JSP3/2.0.6"},{"name":"Date","value":"Mon, 30 Mar 2015 08:18:28 GMT"},{"name":"Content-Type","value":"text/javascript"},{"name":"Connection","value":"keep-alive"},{"name":"ETag","value":"\"1765499629\""},{"name":"Last-Modified","value":"Fri, 20 Mar 2015 11:03:30 GMT"},{"name":"Expires","value":"Mon, 30 Mar 2015 09:17:15 GMT"},{"name":"Age","value":"601273"},{"name":"Cache-Control","value":"max-age=604800"},{"name":"Accept-Ranges","value":"bytes"},{"name":"Content-Encoding","value":"gzip"},{"name":"Vary","value":"Accept-Encoding"}],"id":10,"redirectURL":null,"stage":"end","status":200,"statusText":"OK","time":"2015-03-30T08:18:28.583Z","url":"http://img2.bdstatic.com/static/common/pkg/other_5dfc0a1.js"} +[DEBUG - 2015-03-30T08:18:28.583Z] Session [54fd5270-d6b5-11e4-bf64-67a72685a843] - page.onResourceReceived - {"contentType":"text/javascript","headers":[{"name":"Server","value":"JSP3/2.0.6"},{"name":"Date","value":"Mon, 30 Mar 2015 08:18:28 GMT"},{"name":"Content-Type","value":"text/javascript"},{"name":"Connection","value":"keep-alive"},{"name":"ETag","value":"\"1905843949\""},{"name":"Last-Modified","value":"Tue, 24 Mar 2015 03:28:45 GMT"},{"name":"Expires","value":"Thu, 02 Apr 2015 13:47:28 GMT"},{"name":"Age","value":"325860"},{"name":"Cache-Control","value":"max-age=604800"},{"name":"Accept-Ranges","value":"bytes"},{"name":"Content-Encoding","value":"gzip"},{"name":"Vary","value":"Accept-Encoding"}],"id":8,"redirectURL":null,"stage":"end","status":200,"statusText":"OK","time":"2015-03-30T08:18:28.583Z","url":"http://img0.bdstatic.com/static/common/pkg/base_9584ab6.js"} +2015-03-30T16:18:28 [DEBUG] CookieJar - Saved "BAIDUID=DB1D387F3D1BF6229139D2C032ACF8DF:FG=1; expires=Tue, 29-Mar-2016 08:18:23 GMT; domain=.baidu.com; path=/" +2015-03-30T16:18:28 [DEBUG] CookieJar - Saved "HMVT=737dbb498415dd39d8abf5bc2404b290|1427703508|; domain=.hm.baidu.com; path=/" +2015-03-30T16:18:28 [DEBUG] CookieJar - Saved "HMACCOUNT=1333D269EFC6101A; expires=Mon, 18-Jan-2038 00:00:00 GMT; domain=.hm.baidu.com; path=/" +[DEBUG - 2015-03-30T08:18:28.649Z] Session [54fd5270-d6b5-11e4-bf64-67a72685a843] - page.onResourceReceived - {"contentType":"text/javascript","headers":[{"name":"Accept-Ranges","value":"bytes"},{"name":"Cache-Control","value":"max-age=311040000"},{"name":"Content-Length","value":"0"},{"name":"Content-Type","value":"text/javascript"},{"name":"Date","value":"Mon, 30 Mar 2015 08:18:28 GMT"},{"name":"Etag","value":"\"1778109334\""},{"name":"Expires","value":"Wed, 05 Feb 2025 08:18:28 GMT"},{"name":"Last-Modified","value":"Fri, 19 Dec 2014 01:30:10 GMT"},{"name":"Server","value":"BWS/1.0"}],"id":6,"redirectURL":null,"stage":"end","status":200,"statusText":"OK","time":"2015-03-30T08:18:28.649Z","url":"http://img.baidu.com/hunter/image.js?st=-16525"} +[DEBUG - 2015-03-30T08:18:28.649Z] Session [54fd5270-d6b5-11e4-bf64-67a72685a843] - page.onResourceReceived - {"bodySize":18438,"contentType":"application/javascript","headers":[{"name":"Cache-Control","value":"max-age=0, must-revalidate"},{"name":"Content-Encoding","value":"gzip"},{"name":"Content-Type","value":"application/javascript"},{"name":"Date","value":"Mon, 30 Mar 2015 08:18:28 GMT"},{"name":"Etag","value":"85653a379913e46ab4726791a313cd37"},{"name":"P3p","value":"CP=\"CURa ADMa DEVa PSAo PSDo OUR BUS UNI PUR INT DEM STA PRE COM NAV OTC NOI DSP COR\""},{"name":"Server","value":"apache"},{"name":"Set-Cookie","value":"HMVT=737dbb498415dd39d8abf5bc2404b290|1427703508|; Path=/; Domain=hm.baidu.com\nHMACCOUNT=1333D269EFC6101A; Path=/; Domain=hm.baidu.com; Expires=Sun, 18 Jan 2038 00:00:00 GMT"}],"id":7,"redirectURL":null,"stage":"start","status":200,"statusText":"OK","time":"2015-03-30T08:18:28.649Z","url":"http://hm.baidu.com/h.js?737dbb498415dd39d8abf5bc2404b290"} +2015-03-30T16:18:28 [DEBUG] WebPage - updateLoadingProgress: 82 +[DEBUG - 2015-03-30T08:18:28.649Z] Session [54fd5270-d6b5-11e4-bf64-67a72685a843] - page.onResourceReceived - {"contentType":"application/javascript","headers":[{"name":"Cache-Control","value":"max-age=0, must-revalidate"},{"name":"Content-Encoding","value":"gzip"},{"name":"Content-Type","value":"application/javascript"},{"name":"Date","value":"Mon, 30 Mar 2015 08:18:28 GMT"},{"name":"Etag","value":"85653a379913e46ab4726791a313cd37"},{"name":"P3p","value":"CP=\"CURa ADMa DEVa PSAo PSDo OUR BUS UNI PUR INT DEM STA PRE COM NAV OTC NOI DSP COR\""},{"name":"Server","value":"apache"},{"name":"Set-Cookie","value":"HMVT=737dbb498415dd39d8abf5bc2404b290|1427703508|; Path=/; Domain=hm.baidu.com\nHMACCOUNT=1333D269EFC6101A; Path=/; Domain=hm.baidu.com; Expires=Sun, 18 Jan 2038 00:00:00 GMT"}],"id":7,"redirectURL":null,"stage":"end","status":200,"statusText":"OK","time":"2015-03-30T08:18:28.649Z","url":"http://hm.baidu.com/h.js?737dbb498415dd39d8abf5bc2404b290"} +2015-03-30T16:18:28 [DEBUG] WebPage - updateLoadingProgress: 100 +2015-03-30T16:18:28 [DEBUG] CookieJar - Saved "BAIDUID=DB1D387F3D1BF6229139D2C032ACF8DF:FG=1; expires=Tue, 29-Mar-2016 08:18:23 GMT; domain=.baidu.com; path=/" +2015-03-30T16:18:28 [DEBUG] CookieJar - Saved "HMVT=737dbb498415dd39d8abf5bc2404b290|1427703508|; domain=.hm.baidu.com; path=/" +2015-03-30T16:18:28 [DEBUG] CookieJar - Saved "HMACCOUNT=1333D269EFC6101A; expires=Mon, 18-Jan-2038 00:00:00 GMT; domain=.hm.baidu.com; path=/" +2015-03-30T16:18:28 [DEBUG] CookieJar - Saved "Hm_lvt_737dbb498415dd39d8abf5bc2404b290=1427702229,1427703125,1427703224,1427703509; expires=Tue, 29-Mar-2016 08:18:28 GMT; domain=.image.baidu.com; path=/" +2015-03-30T16:18:28 [DEBUG] CookieJar - Saved "BAIDUID=DB1D387F3D1BF6229139D2C032ACF8DF:FG=1; expires=Tue, 29-Mar-2016 08:18:23 GMT; domain=.baidu.com; path=/" +2015-03-30T16:18:28 [DEBUG] CookieJar - Saved "HMVT=737dbb498415dd39d8abf5bc2404b290|1427703508|; domain=.hm.baidu.com; path=/" +2015-03-30T16:18:28 [DEBUG] CookieJar - Saved "HMACCOUNT=1333D269EFC6101A; expires=Mon, 18-Jan-2038 00:00:00 GMT; domain=.hm.baidu.com; path=/" +2015-03-30T16:18:28 [DEBUG] CookieJar - Saved "Hm_lvt_737dbb498415dd39d8abf5bc2404b290=1427702229,1427703125,1427703224,1427703509; expires=Tue, 29-Mar-2016 08:18:28 GMT; domain=.image.baidu.com; path=/" +2015-03-30T16:18:28 [DEBUG] CookieJar - Saved "Hm_lpvt_737dbb498415dd39d8abf5bc2404b290=1427703509; domain=.image.baidu.com; path=/" +[DEBUG - 2015-03-30T08:18:28.676Z] Session [54fd5270-d6b5-11e4-bf64-67a72685a843] - _oneShotCallback - onLoadFinished +[DEBUG - 2015-03-30T08:18:28.676Z] Session [54fd5270-d6b5-11e4-bf64-67a72685a843] - _execFuncAndWaitForLoadDecorator - onLoadFinished: success +2015-03-30T16:18:28 [DEBUG] WebPage - setupFrame "" +[DEBUG - 2015-03-30T08:18:28.677Z] Session [54fd5270-d6b5-11e4-bf64-67a72685a843] - _execFuncAndWaitForLoadDecorator - Page Loading in Session: false +[DEBUG - 2015-03-30T08:18:28.677Z] SessionReqHand - _SuccessHandler - status: success +2015-03-30T16:18:28 [DEBUG] HTTP Response - Status Code 200 OK +2015-03-30T16:18:28 [DEBUG] HTTP Response - Sending Header "Cache" = "no-cache" +2015-03-30T16:18:28 [DEBUG] HTTP Response - Sending Header "Content-Length" = "74" +2015-03-30T16:18:28 [DEBUG] HTTP Response - Sending Header "Content-Type" = "application/json;charset=UTF-8" +2015-03-30T16:18:28 [DEBUG] HTTP Request - URI /wd/hub/session/54fd5270-d6b5-11e4-bf64-67a72685a843/source +2015-03-30T16:18:28 [DEBUG] HTTP Request - Method GET +2015-03-30T16:18:28 [DEBUG] HTTP Request - HTTP Version 1.1 +2015-03-30T16:18:28 [DEBUG] HTTP Request - Query String +2015-03-30T16:18:28 [DEBUG] HTTP Request - Receiving Header "Accept-Encoding" = "identity" +2015-03-30T16:18:28 [DEBUG] HTTP Request - Receiving Header "Host" = "127.0.0.1:44021" +2015-03-30T16:18:28 [DEBUG] HTTP Request - Receiving Header "Content-Type" = "application/json;charset=UTF-8" +2015-03-30T16:18:28 [DEBUG] HTTP Request - Receiving Header "Connection" = "close" +2015-03-30T16:18:28 [DEBUG] HTTP Request - Receiving Header "Accept" = "application/json" +2015-03-30T16:18:28 [DEBUG] HTTP Request - Receiving Header "User-Agent" = "Python-urllib/2.7" +[DEBUG - 2015-03-30T08:18:28.684Z] Session [54fd5270-d6b5-11e4-bf64-67a72685a843] - page.onResourceRequested - {"headers":[{"name":"User-Agent","value":"Scrapy/0.24.5 (+http://scrapy.org)"},{"name":"Accept","value":"*/*"},{"name":"Referer","value":"http://image.baidu.com/"}],"id":12,"method":"GET","time":"2015-03-30T08:18:28.684Z","url":"http://img1.bdstatic.com/static/common/pkg/onload_647184b.js"} +[DEBUG - 2015-03-30T08:18:28.685Z] RouterReqHand - _handle - {"headers":{"Accept":"application/json","Accept-Encoding":"identity","Connection":"close","Content-Type":"application/json;charset=UTF-8","Host":"127.0.0.1:44021","User-Agent":"Python-urllib/2.7"},"httpVersion":"1.1","method":"GET","url":"/session/54fd5270-d6b5-11e4-bf64-67a72685a843/source","urlParsed":{"anchor":"","query":"","file":"source","directory":"/session/54fd5270-d6b5-11e4-bf64-67a72685a843/","path":"/session/54fd5270-d6b5-11e4-bf64-67a72685a843/source","relative":"/session/54fd5270-d6b5-11e4-bf64-67a72685a843/source","port":"","host":"","password":"","user":"","userInfo":"","authority":"","protocol":"","source":"/session/54fd5270-d6b5-11e4-bf64-67a72685a843/source","queryKey":{},"chunks":["session","54fd5270-d6b5-11e4-bf64-67a72685a843","source"]}} +2015-03-30T16:18:28 [DEBUG] HTTP Response - Status Code 200 OK +2015-03-30T16:18:28 [DEBUG] HTTP Response - Sending Header "Cache" = "no-cache" +2015-03-30T16:18:28 [DEBUG] HTTP Response - Sending Header "Content-Length" = "94668" +2015-03-30T16:18:28 [DEBUG] HTTP Response - Sending Header "Content-Type" = "application/json;charset=UTF-8" +[DEBUG - 2015-03-30T08:18:28.703Z] Session [54fd5270-d6b5-11e4-bf64-67a72685a843] - page.onResourceReceived - {"bodySize":66029,"contentType":"text/javascript","headers":[{"name":"Server","value":"JSP3/2.0.6"},{"name":"Date","value":"Mon, 30 Mar 2015 08:18:28 GMT"},{"name":"Content-Type","value":"text/javascript"},{"name":"Connection","value":"keep-alive"},{"name":"ETag","value":"\"591126017\""},{"name":"Last-Modified","value":"Tue, 24 Mar 2015 03:28:45 GMT"},{"name":"Expires","value":"Sat, 04 Apr 2015 16:17:40 GMT"},{"name":"Age","value":"144048"},{"name":"Cache-Control","value":"max-age=604800"},{"name":"Accept-Ranges","value":"bytes"},{"name":"Content-Encoding","value":"gzip"},{"name":"Vary","value":"Accept-Encoding"}],"id":12,"redirectURL":null,"stage":"start","status":200,"statusText":"OK","time":"2015-03-30T08:18:28.702Z","url":"http://img1.bdstatic.com/static/common/pkg/onload_647184b.js"} +[DEBUG - 2015-03-30T08:18:28.703Z] Session [54fd5270-d6b5-11e4-bf64-67a72685a843] - page.onResourceReceived - {"contentType":"text/javascript","headers":[{"name":"Server","value":"JSP3/2.0.6"},{"name":"Date","value":"Mon, 30 Mar 2015 08:18:28 GMT"},{"name":"Content-Type","value":"text/javascript"},{"name":"Connection","value":"keep-alive"},{"name":"ETag","value":"\"591126017\""},{"name":"Last-Modified","value":"Tue, 24 Mar 2015 03:28:45 GMT"},{"name":"Expires","value":"Sat, 04 Apr 2015 16:17:40 GMT"},{"name":"Age","value":"144048"},{"name":"Cache-Control","value":"max-age=604800"},{"name":"Accept-Ranges","value":"bytes"},{"name":"Content-Encoding","value":"gzip"},{"name":"Vary","value":"Accept-Encoding"}],"id":12,"redirectURL":null,"stage":"end","status":200,"statusText":"OK","time":"2015-03-30T08:18:28.703Z","url":"http://img1.bdstatic.com/static/common/pkg/onload_647184b.js"} +[DEBUG - 2015-03-30T08:18:28.710Z] Session [54fd5270-d6b5-11e4-bf64-67a72685a843] - page.onResourceRequested - {"headers":[{"name":"X-Requested-With","value":"XMLHttpRequest"},{"name":"User-Agent","value":"Scrapy/0.24.5 (+http://scrapy.org)"},{"name":"Accept","value":"application/json, text/javascript, */*; q=0.01"},{"name":"Referer","value":"http://image.baidu.com/"}],"id":13,"method":"GET","time":"2015-03-30T08:18:28.710Z","url":"http://image.baidu.com/user/msg?t=1427703508708"} +[DEBUG - 2015-03-30T08:18:28.711Z] Session [54fd5270-d6b5-11e4-bf64-67a72685a843] - page.onResourceRequested - {"headers":[{"name":"X-Requested-With","value":"XMLHttpRequest"},{"name":"User-Agent","value":"Scrapy/0.24.5 (+http://scrapy.org)"},{"name":"Accept","value":"application/json, text/javascript, */*; q=0.01"},{"name":"Referer","value":"http://image.baidu.com/"}],"id":14,"method":"GET","time":"2015-03-30T08:18:28.711Z","url":"http://image.baidu.com/user/medal?uid=0&t=1427703508710"} +[DEBUG - 2015-03-30T08:18:28.736Z] Session [54fd5270-d6b5-11e4-bf64-67a72685a843] - page.onResourceReceived - {"bodySize":98,"contentType":"application/json; charset=utf-8;","headers":[{"name":"Content-Encoding","value":"gzip"},{"name":"Content-Type","value":"application/json; charset=utf-8;"},{"name":"Date","value":"Mon, 30 Mar 2015 08:18:28 GMT"},{"name":"Server","value":"Apache"},{"name":"Vary","value":"Accept-Encoding"}],"id":14,"redirectURL":null,"stage":"start","status":200,"statusText":"OK","time":"2015-03-30T08:18:28.736Z","url":"http://image.baidu.com/user/medal?uid=0&t=1427703508710"} +[DEBUG - 2015-03-30T08:18:28.737Z] Session [54fd5270-d6b5-11e4-bf64-67a72685a843] - page.onResourceReceived - {"bodySize":72,"contentType":"application/json; charset=utf-8;","headers":[{"name":"Content-Encoding","value":"gzip"},{"name":"Content-Type","value":"application/json; charset=utf-8;"},{"name":"Date","value":"Mon, 30 Mar 2015 08:18:28 GMT"},{"name":"Server","value":"Apache"},{"name":"Vary","value":"Accept-Encoding"}],"id":13,"redirectURL":null,"stage":"start","status":200,"statusText":"OK","time":"2015-03-30T08:18:28.737Z","url":"http://image.baidu.com/user/msg?t=1427703508708"} +[DEBUG - 2015-03-30T08:18:28.737Z] Session [54fd5270-d6b5-11e4-bf64-67a72685a843] - page.onResourceReceived - {"contentType":"application/json; charset=utf-8;","headers":[{"name":"Content-Encoding","value":"gzip"},{"name":"Content-Type","value":"application/json; charset=utf-8;"},{"name":"Date","value":"Mon, 30 Mar 2015 08:18:28 GMT"},{"name":"Server","value":"Apache"},{"name":"Vary","value":"Accept-Encoding"}],"id":14,"redirectURL":null,"stage":"end","status":200,"statusText":"OK","time":"2015-03-30T08:18:28.737Z","url":"http://image.baidu.com/user/medal?uid=0&t=1427703508710"} +[DEBUG - 2015-03-30T08:18:28.738Z] Session [54fd5270-d6b5-11e4-bf64-67a72685a843] - page.onResourceReceived - {"contentType":"application/json; charset=utf-8;","headers":[{"name":"Content-Encoding","value":"gzip"},{"name":"Content-Type","value":"application/json; charset=utf-8;"},{"name":"Date","value":"Mon, 30 Mar 2015 08:18:28 GMT"},{"name":"Server","value":"Apache"},{"name":"Vary","value":"Accept-Encoding"}],"id":13,"redirectURL":null,"stage":"end","status":200,"statusText":"OK","time":"2015-03-30T08:18:28.737Z","url":"http://image.baidu.com/user/msg?t=1427703508708"} +2015-03-30T16:18:28 [DEBUG] HTTP Request - URI /wd/hub/session/54fd5270-d6b5-11e4-bf64-67a72685a843/url +2015-03-30T16:18:28 [DEBUG] HTTP Request - Method POST +2015-03-30T16:18:28 [DEBUG] HTTP Request - HTTP Version 1.1 +2015-03-30T16:18:28 [DEBUG] HTTP Request - Query String +2015-03-30T16:18:28 [DEBUG] HTTP Request - Receiving Header "Accept-Encoding" = "identity" +2015-03-30T16:18:28 [DEBUG] HTTP Request - Receiving Header "Content-Length" = "83" +2015-03-30T16:18:28 [DEBUG] HTTP Request - Receiving Header "Host" = "127.0.0.1:44021" +2015-03-30T16:18:28 [DEBUG] HTTP Request - Receiving Header "Accept" = "application/json" +2015-03-30T16:18:28 [DEBUG] HTTP Request - Receiving Header "User-Agent" = "Python-urllib/2.7" +2015-03-30T16:18:28 [DEBUG] HTTP Request - Receiving Header "Connection" = "close" +2015-03-30T16:18:28 [DEBUG] HTTP Request - Receiving Header "Content-Type" = "application/json;charset=UTF-8" +2015-03-30T16:18:28 [DEBUG] HTTP Request - Method POST/PUT +2015-03-30T16:18:28 [DEBUG] HTTP Request - Content Body: {"url": "http://v.baidu.com/", "sessionId": "54fd5270-d6b5-11e4-bf64-67a72685a843"} +[DEBUG - 2015-03-30T08:18:28.998Z] RouterReqHand - _handle - {"headers":{"Accept":"application/json","Accept-Encoding":"identity","Connection":"close","Content-Length":"83","Content-Type":"application/json;charset=UTF-8","Host":"127.0.0.1:44021","User-Agent":"Python-urllib/2.7"},"httpVersion":"1.1","method":"POST","post":"{\"url\": \"http://v.baidu.com/\", \"sessionId\": \"54fd5270-d6b5-11e4-bf64-67a72685a843\"}","url":"/session/54fd5270-d6b5-11e4-bf64-67a72685a843/url","urlParsed":{"anchor":"","query":"","file":"url","directory":"/session/54fd5270-d6b5-11e4-bf64-67a72685a843/","path":"/session/54fd5270-d6b5-11e4-bf64-67a72685a843/url","relative":"/session/54fd5270-d6b5-11e4-bf64-67a72685a843/url","port":"","host":"","password":"","user":"","userInfo":"","authority":"","protocol":"","source":"/session/54fd5270-d6b5-11e4-bf64-67a72685a843/url","queryKey":{},"chunks":["session","54fd5270-d6b5-11e4-bf64-67a72685a843","url"]}} +[DEBUG - 2015-03-30T08:18:28.998Z] SessionReqHand - _postUrlCommand - Session '54fd5270-d6b5-11e4-bf64-67a72685a843' is about to load URL: http://v.baidu.com/ +2015-03-30T16:18:28 [DEBUG] WebPage - updateLoadingProgress: 10 +[DEBUG - 2015-03-30T08:18:28.999Z] Session [54fd5270-d6b5-11e4-bf64-67a72685a843] - page.onResourceRequested - {"headers":[{"name":"User-Agent","value":"Scrapy/0.24.5 (+http://scrapy.org)"},{"name":"Accept","value":"text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8"}],"id":15,"method":"GET","time":"2015-03-30T08:18:28.999Z","url":"http://v.baidu.com/"} +[DEBUG - 2015-03-30T08:18:29.009Z] Session [54fd5270-d6b5-11e4-bf64-67a72685a843] - _execFuncAndWaitForLoadDecorator - Page Loading in Session: true +[DEBUG - 2015-03-30T08:18:29.109Z] Session [54fd5270-d6b5-11e4-bf64-67a72685a843] - _execFuncAndWaitForLoadDecorator - Page Loading in Session: true +2015-03-30T16:18:29 [DEBUG] CookieJar - Saved "HMVT=737dbb498415dd39d8abf5bc2404b290|1427703508|; domain=.hm.baidu.com; path=/" +2015-03-30T16:18:29 [DEBUG] CookieJar - Saved "HMACCOUNT=1333D269EFC6101A; expires=Mon, 18-Jan-2038 00:00:00 GMT; domain=.hm.baidu.com; path=/" +2015-03-30T16:18:29 [DEBUG] CookieJar - Saved "Hm_lvt_737dbb498415dd39d8abf5bc2404b290=1427702229,1427703125,1427703224,1427703509; expires=Tue, 29-Mar-2016 08:18:28 GMT; domain=.image.baidu.com; path=/" +2015-03-30T16:18:29 [DEBUG] CookieJar - Saved "Hm_lpvt_737dbb498415dd39d8abf5bc2404b290=1427703509; domain=.image.baidu.com; path=/" +2015-03-30T16:18:29 [DEBUG] CookieJar - Saved "BAIDUID=94628AFA622ED102CC73E486CEB1CC9B:FG=1; expires=Tue, 29-Mar-2016 08:18:29 GMT; domain=.baidu.com; path=/" +[DEBUG - 2015-03-30T08:18:29.197Z] Session [54fd5270-d6b5-11e4-bf64-67a72685a843] - page.onResourceReceived - {"bodySize":88297,"contentType":"text/html; charset=UTF-8","headers":[{"name":"Content-Encoding","value":"gzip"},{"name":"Content-Type","value":"text/html; charset=UTF-8"},{"name":"Date","value":"Mon, 30 Mar 2015 08:18:29 GMT"},{"name":"P3p","value":"CP=\" OTI DSP COR IVA OUR IND COM \""},{"name":"Server","value":"Apache"},{"name":"Set-Cookie","value":"BAIDUID=94628AFA622ED102CC73E486CEB1CC9B:FG=1; expires=Tue, 29-Mar-16 08:18:29 GMT; max-age=31536000; path=/; domain=.baidu.com; version=1"},{"name":"Tracecode","value":"11090845760282143242033016"},{"name":"X-Powered-By","value":"HPHP"},{"name":"Transfer-Encoding","value":"chunked"}],"id":15,"redirectURL":null,"stage":"start","status":200,"statusText":"OK","time":"2015-03-30T08:18:29.197Z","url":"http://v.baidu.com/"} +[DEBUG - 2015-03-30T08:18:29.198Z] Session [54fd5270-d6b5-11e4-bf64-67a72685a843] - page.onResourceRequested - {"headers":[{"name":"Cache-Control","value":"max-age=0"},{"name":"User-Agent","value":"Scrapy/0.24.5 (+http://scrapy.org)"},{"name":"Referer","value":"http://image.baidu.com/"},{"name":"Accept","value":"*/*"}],"id":16,"method":"GET","time":"2015-03-30T08:18:29.198Z","url":"http://hm.baidu.com/hm.gif?cc=1&ck=1&cl=32-bit&ds=1024x768&ep=547%2C547&et=3&ja=0&ln=en-US&lo=0<=1427703509&nv=0&rnd=1761342466&si=737dbb498415dd39d8abf5bc2404b290&st=4&v=1.0.75&lv=3&tt=%E7%99%BE%E5%BA%A6%E5%9B%BE%E7%89%87%E2%80%94%E5%85%A8%E7%90%83%E6%9C%80%E5%A4%A7%E4%B8%AD%E6%96%87%E5%9B%BE%E7%89%87%E5%BA%93"} +2015-03-30T16:18:29 [DEBUG] WebPage - setupFrame "" +[DEBUG - 2015-03-30T08:18:29.199Z] Session [54fd5270-d6b5-11e4-bf64-67a72685a843] - page.onResourceRequested - {"headers":[{"name":"User-Agent","value":"Scrapy/0.24.5 (+http://scrapy.org)"},{"name":"Accept","value":"*/*"},{"name":"Referer","value":"http://v.baidu.com/"}],"id":17,"method":"GET","time":"2015-03-30T08:18:29.199Z","url":"http://img.baidu.com/hunter/alog/alog.min.js"} +[DEBUG - 2015-03-30T08:18:29.200Z] Session [54fd5270-d6b5-11e4-bf64-67a72685a843] - page.onResourceRequested - {"headers":[{"name":"User-Agent","value":"Scrapy/0.24.5 (+http://scrapy.org)"},{"name":"Accept","value":"text/css,*/*;q=0.1"},{"name":"Referer","value":"http://v.baidu.com/"}],"id":18,"method":"GET","time":"2015-03-30T08:18:29.200Z","url":"http://vs5.bdstatic.com/browse_static/v3/common/pkg/common_sync_css_0_f2a7259b.css"} +[DEBUG - 2015-03-30T08:18:29.200Z] Session [54fd5270-d6b5-11e4-bf64-67a72685a843] - page.onResourceRequested - {"headers":[{"name":"User-Agent","value":"Scrapy/0.24.5 (+http://scrapy.org)"},{"name":"Accept","value":"text/css,*/*;q=0.1"},{"name":"Referer","value":"http://v.baidu.com/"}],"id":19,"method":"GET","time":"2015-03-30T08:18:29.200Z","url":"http://vs6.bdstatic.com/browse_static/v3/index/pkg/index_sync_css_1_e546a06.css"} +[DEBUG - 2015-03-30T08:18:29.200Z] Session [54fd5270-d6b5-11e4-bf64-67a72685a843] - page.onResourceRequested - {"headers":[{"name":"User-Agent","value":"Scrapy/0.24.5 (+http://scrapy.org)"},{"name":"Accept","value":"text/css,*/*;q=0.1"},{"name":"Referer","value":"http://v.baidu.com/"}],"id":20,"method":"GET","time":"2015-03-30T08:18:29.200Z","url":"http://vs5.bdstatic.com/browse_static/v3/common/pkg/common_ui_video_89908f9b.css"} +[DEBUG - 2015-03-30T08:18:29.200Z] Session [54fd5270-d6b5-11e4-bf64-67a72685a843] - page.onResourceRequested - {"headers":[{"name":"User-Agent","value":"Scrapy/0.24.5 (+http://scrapy.org)"},{"name":"Accept","value":"text/css,*/*;q=0.1"},{"name":"Referer","value":"http://v.baidu.com/"}],"id":21,"method":"GET","time":"2015-03-30T08:18:29.200Z","url":"http://vs5.bdstatic.com/browse_static/v3/index/pkg/index_sync_css_2_601ed6e.css"} +[DEBUG - 2015-03-30T08:18:29.200Z] Session [54fd5270-d6b5-11e4-bf64-67a72685a843] - page.onResourceRequested - {"headers":[{"name":"User-Agent","value":"Scrapy/0.24.5 (+http://scrapy.org)"},{"name":"Accept","value":"text/css,*/*;q=0.1"},{"name":"Referer","value":"http://v.baidu.com/"}],"id":22,"method":"GET","time":"2015-03-30T08:18:29.200Z","url":"http://vs6.bdstatic.com/browse_static/v3/index/pkg/home_50935db.css"} +[DEBUG - 2015-03-30T08:18:29.201Z] Session [54fd5270-d6b5-11e4-bf64-67a72685a843] - page.onResourceRequested - {"headers":[{"name":"User-Agent","value":"Scrapy/0.24.5 (+http://scrapy.org)"},{"name":"Accept","value":"text/css,*/*;q=0.1"},{"name":"Referer","value":"http://v.baidu.com/"}],"id":23,"method":"GET","time":"2015-03-30T08:18:29.201Z","url":"http://vs6.bdstatic.com/browse_static/v3/index/pkg/index_sync_css_0_6f95ddb.css"} +2015-03-30T16:18:29 [DEBUG] WebPage - updateLoadingProgress: 34 +2015-03-30T16:18:29 [DEBUG] WebPage - updateLoadingProgress: 50 +[DEBUG - 2015-03-30T08:18:29.209Z] Session [54fd5270-d6b5-11e4-bf64-67a72685a843] - _execFuncAndWaitForLoadDecorator - Page Loading in Session: true +[DEBUG - 2015-03-30T08:18:29.229Z] Session [54fd5270-d6b5-11e4-bf64-67a72685a843] - page.onResourceRequested - {"headers":[{"name":"User-Agent","value":"Scrapy/0.24.5 (+http://scrapy.org)"},{"name":"Accept","value":"*/*"},{"name":"Referer","value":"http://v.baidu.com/"}],"id":24,"method":"GET","time":"2015-03-30T08:18:29.229Z","url":"http://vs5.bdstatic.com/browse_static/v3/common/pkg/common_sync_js_0_6f4d000b.js"} +[DEBUG - 2015-03-30T08:18:29.229Z] Session [54fd5270-d6b5-11e4-bf64-67a72685a843] - page.onResourceRequested - {"headers":[{"name":"User-Agent","value":"Scrapy/0.24.5 (+http://scrapy.org)"},{"name":"Accept","value":"*/*"},{"name":"Referer","value":"http://v.baidu.com/"}],"id":25,"method":"GET","time":"2015-03-30T08:18:29.229Z","url":"http://vs5.bdstatic.com/browse_static/v3/common/pkg/common_sync_js_3_b08eb3ab.js"} +[DEBUG - 2015-03-30T08:18:29.229Z] Session [54fd5270-d6b5-11e4-bf64-67a72685a843] - page.onResourceRequested - {"headers":[{"name":"User-Agent","value":"Scrapy/0.24.5 (+http://scrapy.org)"},{"name":"Accept","value":"*/*"},{"name":"Referer","value":"http://v.baidu.com/"}],"id":26,"method":"GET","time":"2015-03-30T08:18:29.229Z","url":"http://vs6.bdstatic.com/browse_static/v3/common/pkg/common_sync_js_1_2627975b.js"} +[DEBUG - 2015-03-30T08:18:29.230Z] Session [54fd5270-d6b5-11e4-bf64-67a72685a843] - page.onResourceRequested - {"headers":[{"name":"User-Agent","value":"Scrapy/0.24.5 (+http://scrapy.org)"},{"name":"Accept","value":"*/*"},{"name":"Referer","value":"http://v.baidu.com/"}],"id":27,"method":"GET","time":"2015-03-30T08:18:29.230Z","url":"http://vs6.bdstatic.com/browse_static/v3/common/pkg/common_sync_js_2_0d2c104f.js"} +[DEBUG - 2015-03-30T08:18:29.230Z] Session [54fd5270-d6b5-11e4-bf64-67a72685a843] - page.onResourceRequested - {"headers":[{"name":"User-Agent","value":"Scrapy/0.24.5 (+http://scrapy.org)"},{"name":"Accept","value":"*/*"},{"name":"Referer","value":"http://v.baidu.com/"}],"id":28,"method":"GET","time":"2015-03-30T08:18:29.230Z","url":"http://vs6.bdstatic.com/browse_static/v3/common/pkg/common_video_aef024c5.js"} +[DEBUG - 2015-03-30T08:18:29.230Z] Session [54fd5270-d6b5-11e4-bf64-67a72685a843] - page.onResourceRequested - {"headers":[{"name":"User-Agent","value":"Scrapy/0.24.5 (+http://scrapy.org)"},{"name":"Accept","value":"*/*"},{"name":"Referer","value":"http://v.baidu.com/"}],"id":29,"method":"GET","time":"2015-03-30T08:18:29.230Z","url":"http://vs5.bdstatic.com/browse_static/v3/index/pkg/home_video_fc81955.js"} +[DEBUG - 2015-03-30T08:18:29.230Z] Session [54fd5270-d6b5-11e4-bf64-67a72685a843] - page.onResourceRequested - {"headers":[{"name":"User-Agent","value":"Scrapy/0.24.5 (+http://scrapy.org)"},{"name":"Accept","value":"*/*"},{"name":"Referer","value":"http://v.baidu.com/"}],"id":30,"method":"GET","time":"2015-03-30T08:18:29.230Z","url":"http://vs5.bdstatic.com/browse_static/v3/index/pkg/index_sync_js_0_a6301a9.js"} +2015-03-30T16:18:29 [DEBUG] WebPage - updateLoadingProgress: 53 +[DEBUG - 2015-03-30T08:18:29.230Z] Session [54fd5270-d6b5-11e4-bf64-67a72685a843] - page.onResourceReceived - {"contentType":"text/html; charset=UTF-8","headers":[{"name":"Content-Encoding","value":"gzip"},{"name":"Content-Type","value":"text/html; charset=UTF-8"},{"name":"Date","value":"Mon, 30 Mar 2015 08:18:29 GMT"},{"name":"P3p","value":"CP=\" OTI DSP COR IVA OUR IND COM \""},{"name":"Server","value":"Apache"},{"name":"Set-Cookie","value":"BAIDUID=94628AFA622ED102CC73E486CEB1CC9B:FG=1; expires=Tue, 29-Mar-16 08:18:29 GMT; max-age=31536000; path=/; domain=.baidu.com; version=1"},{"name":"Tracecode","value":"11090845760282143242033016"},{"name":"X-Powered-By","value":"HPHP"},{"name":"Transfer-Encoding","value":"chunked"}],"id":15,"redirectURL":null,"stage":"end","status":200,"statusText":"OK","time":"2015-03-30T08:18:29.230Z","url":"http://v.baidu.com/"} +[DEBUG - 2015-03-30T08:18:29.241Z] Session [54fd5270-d6b5-11e4-bf64-67a72685a843] - page.onResourceReceived - {"bodySize":4620,"contentType":"text/javascript","headers":[{"name":"Accept-Ranges","value":"bytes"},{"name":"Cache-Control","value":"max-age=311040000"},{"name":"Content-Encoding","value":"gzip"},{"name":"Content-Type","value":"text/javascript"},{"name":"Date","value":"Mon, 30 Mar 2015 08:18:29 GMT"},{"name":"Etag","value":"\"1267227191\""},{"name":"Expires","value":"Wed, 05 Feb 2025 08:18:29 GMT"},{"name":"Last-Modified","value":"Fri, 23 May 2014 06:06:14 GMT"},{"name":"Server","value":"BWS/1.0"},{"name":"Vary","value":"Accept-Encoding"}],"id":17,"redirectURL":null,"stage":"start","status":200,"statusText":"OK","time":"2015-03-30T08:18:29.241Z","url":"http://img.baidu.com/hunter/alog/alog.min.js"} +[DEBUG - 2015-03-30T08:18:29.241Z] Session [54fd5270-d6b5-11e4-bf64-67a72685a843] - page.onResourceReceived - {"contentType":"text/javascript","headers":[{"name":"Accept-Ranges","value":"bytes"},{"name":"Cache-Control","value":"max-age=311040000"},{"name":"Content-Encoding","value":"gzip"},{"name":"Content-Type","value":"text/javascript"},{"name":"Date","value":"Mon, 30 Mar 2015 08:18:29 GMT"},{"name":"Etag","value":"\"1267227191\""},{"name":"Expires","value":"Wed, 05 Feb 2025 08:18:29 GMT"},{"name":"Last-Modified","value":"Fri, 23 May 2014 06:06:14 GMT"},{"name":"Server","value":"BWS/1.0"},{"name":"Vary","value":"Accept-Encoding"}],"id":17,"redirectURL":null,"stage":"end","status":200,"statusText":"OK","time":"2015-03-30T08:18:29.241Z","url":"http://img.baidu.com/hunter/alog/alog.min.js"} +[DEBUG - 2015-03-30T08:18:29.242Z] Session [54fd5270-d6b5-11e4-bf64-67a72685a843] - page.onResourceReceived - {"bodySize":43,"contentType":"image/gif","headers":[{"name":"Cache-Control","value":"private, max-age=0, no-cache"},{"name":"Content-Length","value":"43"},{"name":"Content-Type","value":"image/gif"},{"name":"Date","value":"Mon, 30 Mar 2015 08:18:29 GMT"},{"name":"Pragma","value":"no-cache"},{"name":"Server","value":"apache"},{"name":"X-Content-Type-Options","value":"nosniff"}],"id":16,"redirectURL":null,"stage":"start","status":200,"statusText":"OK","time":"2015-03-30T08:18:29.242Z","url":"http://hm.baidu.com/hm.gif?cc=1&ck=1&cl=32-bit&ds=1024x768&ep=547%2C547&et=3&ja=0&ln=en-US&lo=0<=1427703509&nv=0&rnd=1761342466&si=737dbb498415dd39d8abf5bc2404b290&st=4&v=1.0.75&lv=3&tt=%E7%99%BE%E5%BA%A6%E5%9B%BE%E7%89%87%E2%80%94%E5%85%A8%E7%90%83%E6%9C%80%E5%A4%A7%E4%B8%AD%E6%96%87%E5%9B%BE%E7%89%87%E5%BA%93"} +2015-03-30T16:18:29 [DEBUG] Network - Resource request error: 5 ( "Operation canceled" ) URL: "http://hm.baidu.com/hm.gif?cc=1&ck=1&cl=32-bit&ds=1024x768&ep=547,547&et=3&ja=0&ln=en-US&lo=0<=1427703509&nv=0&rnd=1761342466&si=737dbb498415dd39d8abf5bc2404b290&st=4&v=1.0.75&lv=3&tt=百度图片—全球最大中文图片库" +[DEBUG - 2015-03-30T08:18:29.242Z] Session [54fd5270-d6b5-11e4-bf64-67a72685a843] - page.onResourceError - {"errorCode":5,"errorString":"Operation canceled","id":16,"url":"http://hm.baidu.com/hm.gif?cc=1&ck=1&cl=32-bit&ds=1024x768&ep=547,547&et=3&ja=0&ln=en-US&lo=0<=1427703509&nv=0&rnd=1761342466&si=737dbb498415dd39d8abf5bc2404b290&st=4&v=1.0.75&lv=3&tt=百度图片—全球最大中文图片库"} +[DEBUG - 2015-03-30T08:18:29.242Z] Session [54fd5270-d6b5-11e4-bf64-67a72685a843] - page.onResourceReceived - {"contentType":"image/gif","headers":[{"name":"Cache-Control","value":"private, max-age=0, no-cache"},{"name":"Content-Length","value":"43"},{"name":"Content-Type","value":"image/gif"},{"name":"Date","value":"Mon, 30 Mar 2015 08:18:29 GMT"},{"name":"Pragma","value":"no-cache"},{"name":"Server","value":"apache"},{"name":"X-Content-Type-Options","value":"nosniff"}],"id":16,"redirectURL":null,"stage":"end","status":200,"statusText":"OK","time":"2015-03-30T08:18:29.242Z","url":"http://hm.baidu.com/hm.gif?cc=1&ck=1&cl=32-bit&ds=1024x768&ep=547%2C547&et=3&ja=0&ln=en-US&lo=0<=1427703509&nv=0&rnd=1761342466&si=737dbb498415dd39d8abf5bc2404b290&st=4&v=1.0.75&lv=3&tt=%E7%99%BE%E5%BA%A6%E5%9B%BE%E7%89%87%E2%80%94%E5%85%A8%E7%90%83%E6%9C%80%E5%A4%A7%E4%B8%AD%E6%96%87%E5%9B%BE%E7%89%87%E5%BA%93"} +[DEBUG - 2015-03-30T08:18:29.252Z] Session [54fd5270-d6b5-11e4-bf64-67a72685a843] - page.onResourceReceived - {"bodySize":5283,"contentType":"text/css","headers":[{"name":"Date","value":"Mon, 30 Mar 2015 08:18:29 GMT"},{"name":"Content-Type","value":"text/css"},{"name":"Last-Modified","value":"Wed, 25 Mar 2015 02:51:38 GMT"},{"name":"Transfer-Encoding","value":"chunked"},{"name":"Connection","value":"keep-alive"},{"name":"Vary","value":"Accept-Encoding"},{"name":"Server","value":"Apache"},{"name":"Expires","value":"Wed, 29 Apr 2015 08:18:29 GMT"},{"name":"Cache-Control","value":"max-age=2592000"},{"name":"Access-Control-Allow-Origin","value":"http://*.baidu.com"},{"name":"Content-Encoding","value":"gzip"}],"id":19,"redirectURL":null,"stage":"start","status":200,"statusText":"OK","time":"2015-03-30T08:18:29.252Z","url":"http://vs6.bdstatic.com/browse_static/v3/index/pkg/index_sync_css_1_e546a06.css"} +[DEBUG - 2015-03-30T08:18:29.252Z] Session [54fd5270-d6b5-11e4-bf64-67a72685a843] - page.onResourceReceived - {"contentType":"text/css","headers":[{"name":"Date","value":"Mon, 30 Mar 2015 08:18:29 GMT"},{"name":"Content-Type","value":"text/css"},{"name":"Last-Modified","value":"Wed, 25 Mar 2015 02:51:38 GMT"},{"name":"Transfer-Encoding","value":"chunked"},{"name":"Connection","value":"keep-alive"},{"name":"Vary","value":"Accept-Encoding"},{"name":"Server","value":"Apache"},{"name":"Expires","value":"Wed, 29 Apr 2015 08:18:29 GMT"},{"name":"Cache-Control","value":"max-age=2592000"},{"name":"Access-Control-Allow-Origin","value":"http://*.baidu.com"},{"name":"Content-Encoding","value":"gzip"}],"id":19,"redirectURL":null,"stage":"end","status":200,"statusText":"OK","time":"2015-03-30T08:18:29.252Z","url":"http://vs6.bdstatic.com/browse_static/v3/index/pkg/index_sync_css_1_e546a06.css"} +[DEBUG - 2015-03-30T08:18:29.254Z] Session [54fd5270-d6b5-11e4-bf64-67a72685a843] - page.onResourceReceived - {"bodySize":21112,"contentType":"application/x-javascript","headers":[{"name":"Date","value":"Mon, 30 Mar 2015 08:18:29 GMT"},{"name":"Content-Type","value":"application/x-javascript"},{"name":"Last-Modified","value":"Fri, 27 Mar 2015 09:05:49 GMT"},{"name":"Transfer-Encoding","value":"chunked"},{"name":"Connection","value":"keep-alive"},{"name":"Vary","value":"Accept-Encoding"},{"name":"Server","value":"Apache"},{"name":"Expires","value":"Wed, 29 Apr 2015 08:18:29 GMT"},{"name":"Cache-Control","value":"max-age=2592000"},{"name":"Access-Control-Allow-Origin","value":"http://*.baidu.com"},{"name":"Content-Encoding","value":"gzip"}],"id":28,"redirectURL":null,"stage":"start","status":200,"statusText":"OK","time":"2015-03-30T08:18:29.254Z","url":"http://vs6.bdstatic.com/browse_static/v3/common/pkg/common_video_aef024c5.js"} +2015-03-30T16:18:29 [DEBUG] WebPage - updateLoadingProgress: 58 +[DEBUG - 2015-03-30T08:18:29.254Z] Session [54fd5270-d6b5-11e4-bf64-67a72685a843] - page.onResourceReceived - {"contentType":"application/x-javascript","headers":[{"name":"Date","value":"Mon, 30 Mar 2015 08:18:29 GMT"},{"name":"Content-Type","value":"application/x-javascript"},{"name":"Last-Modified","value":"Fri, 27 Mar 2015 09:05:49 GMT"},{"name":"Transfer-Encoding","value":"chunked"},{"name":"Connection","value":"keep-alive"},{"name":"Vary","value":"Accept-Encoding"},{"name":"Server","value":"Apache"},{"name":"Expires","value":"Wed, 29 Apr 2015 08:18:29 GMT"},{"name":"Cache-Control","value":"max-age=2592000"},{"name":"Access-Control-Allow-Origin","value":"http://*.baidu.com"},{"name":"Content-Encoding","value":"gzip"}],"id":28,"redirectURL":null,"stage":"end","status":200,"statusText":"OK","time":"2015-03-30T08:18:29.254Z","url":"http://vs6.bdstatic.com/browse_static/v3/common/pkg/common_video_aef024c5.js"} +[DEBUG - 2015-03-30T08:18:29.255Z] Session [54fd5270-d6b5-11e4-bf64-67a72685a843] - page.onResourceReceived - {"bodySize":20381,"contentType":"text/css","headers":[{"name":"Date","value":"Mon, 30 Mar 2015 08:18:29 GMT"},{"name":"Content-Type","value":"text/css"},{"name":"Last-Modified","value":"Wed, 25 Mar 2015 02:51:38 GMT"},{"name":"Transfer-Encoding","value":"chunked"},{"name":"Connection","value":"keep-alive"},{"name":"Vary","value":"Accept-Encoding"},{"name":"Server","value":"Apache"},{"name":"Expires","value":"Wed, 29 Apr 2015 08:18:29 GMT"},{"name":"Cache-Control","value":"max-age=2592000"},{"name":"Access-Control-Allow-Origin","value":"http://*.baidu.com"},{"name":"Content-Encoding","value":"gzip"}],"id":23,"redirectURL":null,"stage":"start","status":200,"statusText":"OK","time":"2015-03-30T08:18:29.255Z","url":"http://vs6.bdstatic.com/browse_static/v3/index/pkg/index_sync_css_0_6f95ddb.css"} +2015-03-30T16:18:29 [DEBUG] WebPage - updateLoadingProgress: 61 +[DEBUG - 2015-03-30T08:18:29.255Z] Session [54fd5270-d6b5-11e4-bf64-67a72685a843] - page.onResourceReceived - {"contentType":"text/css","headers":[{"name":"Date","value":"Mon, 30 Mar 2015 08:18:29 GMT"},{"name":"Content-Type","value":"text/css"},{"name":"Last-Modified","value":"Wed, 25 Mar 2015 02:51:38 GMT"},{"name":"Transfer-Encoding","value":"chunked"},{"name":"Connection","value":"keep-alive"},{"name":"Vary","value":"Accept-Encoding"},{"name":"Server","value":"Apache"},{"name":"Expires","value":"Wed, 29 Apr 2015 08:18:29 GMT"},{"name":"Cache-Control","value":"max-age=2592000"},{"name":"Access-Control-Allow-Origin","value":"http://*.baidu.com"},{"name":"Content-Encoding","value":"gzip"}],"id":23,"redirectURL":null,"stage":"end","status":200,"statusText":"OK","time":"2015-03-30T08:18:29.255Z","url":"http://vs6.bdstatic.com/browse_static/v3/index/pkg/index_sync_css_0_6f95ddb.css"} +[DEBUG - 2015-03-30T08:18:29.272Z] Session [54fd5270-d6b5-11e4-bf64-67a72685a843] - page.onResourceReceived - {"bodySize":81681,"contentType":"text/css","headers":[{"name":"Date","value":"Mon, 30 Mar 2015 08:18:29 GMT"},{"name":"Content-Type","value":"text/css"},{"name":"Last-Modified","value":"Wed, 25 Mar 2015 02:51:38 GMT"},{"name":"Transfer-Encoding","value":"chunked"},{"name":"Connection","value":"keep-alive"},{"name":"Vary","value":"Accept-Encoding"},{"name":"Server","value":"Apache"},{"name":"Expires","value":"Wed, 29 Apr 2015 08:18:29 GMT"},{"name":"Cache-Control","value":"max-age=2592000"},{"name":"Access-Control-Allow-Origin","value":"http://*.baidu.com"},{"name":"Content-Encoding","value":"gzip"}],"id":22,"redirectURL":null,"stage":"start","status":200,"statusText":"OK","time":"2015-03-30T08:18:29.272Z","url":"http://vs6.bdstatic.com/browse_static/v3/index/pkg/home_50935db.css"} +2015-03-30T16:18:29 [DEBUG] WebPage - updateLoadingProgress: 68 +[DEBUG - 2015-03-30T08:18:29.272Z] Session [54fd5270-d6b5-11e4-bf64-67a72685a843] - page.onResourceReceived - {"contentType":"text/css","headers":[{"name":"Date","value":"Mon, 30 Mar 2015 08:18:29 GMT"},{"name":"Content-Type","value":"text/css"},{"name":"Last-Modified","value":"Wed, 25 Mar 2015 02:51:38 GMT"},{"name":"Transfer-Encoding","value":"chunked"},{"name":"Connection","value":"keep-alive"},{"name":"Vary","value":"Accept-Encoding"},{"name":"Server","value":"Apache"},{"name":"Expires","value":"Wed, 29 Apr 2015 08:18:29 GMT"},{"name":"Cache-Control","value":"max-age=2592000"},{"name":"Access-Control-Allow-Origin","value":"http://*.baidu.com"},{"name":"Content-Encoding","value":"gzip"}],"id":22,"redirectURL":null,"stage":"end","status":200,"statusText":"OK","time":"2015-03-30T08:18:29.272Z","url":"http://vs6.bdstatic.com/browse_static/v3/index/pkg/home_50935db.css"} +[DEBUG - 2015-03-30T08:18:29.277Z] Session [54fd5270-d6b5-11e4-bf64-67a72685a843] - page.onResourceReceived - {"bodySize":65132,"contentType":"application/x-javascript","headers":[{"name":"Date","value":"Mon, 30 Mar 2015 08:18:29 GMT"},{"name":"Content-Type","value":"application/x-javascript"},{"name":"Last-Modified","value":"Fri, 27 Mar 2015 09:05:49 GMT"},{"name":"Transfer-Encoding","value":"chunked"},{"name":"Connection","value":"keep-alive"},{"name":"Vary","value":"Accept-Encoding"},{"name":"Server","value":"Apache"},{"name":"Expires","value":"Wed, 29 Apr 2015 08:18:29 GMT"},{"name":"Cache-Control","value":"max-age=2592000"},{"name":"Access-Control-Allow-Origin","value":"http://*.baidu.com"},{"name":"Content-Encoding","value":"gzip"}],"id":26,"redirectURL":null,"stage":"start","status":200,"statusText":"OK","time":"2015-03-30T08:18:29.276Z","url":"http://vs6.bdstatic.com/browse_static/v3/common/pkg/common_sync_js_1_2627975b.js"} +2015-03-30T16:18:29 [DEBUG] WebPage - updateLoadingProgress: 73 +[DEBUG - 2015-03-30T08:18:29.277Z] Session [54fd5270-d6b5-11e4-bf64-67a72685a843] - page.onResourceReceived - {"contentType":"application/x-javascript","headers":[{"name":"Date","value":"Mon, 30 Mar 2015 08:18:29 GMT"},{"name":"Content-Type","value":"application/x-javascript"},{"name":"Last-Modified","value":"Fri, 27 Mar 2015 09:05:49 GMT"},{"name":"Transfer-Encoding","value":"chunked"},{"name":"Connection","value":"keep-alive"},{"name":"Vary","value":"Accept-Encoding"},{"name":"Server","value":"Apache"},{"name":"Expires","value":"Wed, 29 Apr 2015 08:18:29 GMT"},{"name":"Cache-Control","value":"max-age=2592000"},{"name":"Access-Control-Allow-Origin","value":"http://*.baidu.com"},{"name":"Content-Encoding","value":"gzip"}],"id":26,"redirectURL":null,"stage":"end","status":200,"statusText":"OK","time":"2015-03-30T08:18:29.277Z","url":"http://vs6.bdstatic.com/browse_static/v3/common/pkg/common_sync_js_1_2627975b.js"} +[DEBUG - 2015-03-30T08:18:29.277Z] Session [54fd5270-d6b5-11e4-bf64-67a72685a843] - page.onResourceReceived - {"bodySize":43046,"contentType":"application/x-javascript","headers":[{"name":"Date","value":"Mon, 30 Mar 2015 08:18:29 GMT"},{"name":"Content-Type","value":"application/x-javascript"},{"name":"Last-Modified","value":"Fri, 27 Mar 2015 09:05:49 GMT"},{"name":"Transfer-Encoding","value":"chunked"},{"name":"Connection","value":"keep-alive"},{"name":"Vary","value":"Accept-Encoding"},{"name":"Server","value":"Apache"},{"name":"Expires","value":"Wed, 29 Apr 2015 08:18:29 GMT"},{"name":"Cache-Control","value":"max-age=2592000"},{"name":"Access-Control-Allow-Origin","value":"http://*.baidu.com"},{"name":"Content-Encoding","value":"gzip"}],"id":27,"redirectURL":null,"stage":"start","status":200,"statusText":"OK","time":"2015-03-30T08:18:29.277Z","url":"http://vs6.bdstatic.com/browse_static/v3/common/pkg/common_sync_js_2_0d2c104f.js"} +2015-03-30T16:18:29 [DEBUG] WebPage - updateLoadingProgress: 77 +[DEBUG - 2015-03-30T08:18:29.277Z] Session [54fd5270-d6b5-11e4-bf64-67a72685a843] - page.onResourceReceived - {"contentType":"application/x-javascript","headers":[{"name":"Date","value":"Mon, 30 Mar 2015 08:18:29 GMT"},{"name":"Content-Type","value":"application/x-javascript"},{"name":"Last-Modified","value":"Fri, 27 Mar 2015 09:05:49 GMT"},{"name":"Transfer-Encoding","value":"chunked"},{"name":"Connection","value":"keep-alive"},{"name":"Vary","value":"Accept-Encoding"},{"name":"Server","value":"Apache"},{"name":"Expires","value":"Wed, 29 Apr 2015 08:18:29 GMT"},{"name":"Cache-Control","value":"max-age=2592000"},{"name":"Access-Control-Allow-Origin","value":"http://*.baidu.com"},{"name":"Content-Encoding","value":"gzip"}],"id":27,"redirectURL":null,"stage":"end","status":200,"statusText":"OK","time":"2015-03-30T08:18:29.277Z","url":"http://vs6.bdstatic.com/browse_static/v3/common/pkg/common_sync_js_2_0d2c104f.js"} +[DEBUG - 2015-03-30T08:18:29.309Z] Session [54fd5270-d6b5-11e4-bf64-67a72685a843] - _execFuncAndWaitForLoadDecorator - Page Loading in Session: true +[DEBUG - 2015-03-30T08:18:29.409Z] Session [54fd5270-d6b5-11e4-bf64-67a72685a843] - _execFuncAndWaitForLoadDecorator - Page Loading in Session: true +[DEBUG - 2015-03-30T08:18:29.510Z] Session [54fd5270-d6b5-11e4-bf64-67a72685a843] - _execFuncAndWaitForLoadDecorator - Page Loading in Session: true +[DEBUG - 2015-03-30T08:18:29.610Z] Session [54fd5270-d6b5-11e4-bf64-67a72685a843] - _execFuncAndWaitForLoadDecorator - Page Loading in Session: true +[DEBUG - 2015-03-30T08:18:29.710Z] Session [54fd5270-d6b5-11e4-bf64-67a72685a843] - _execFuncAndWaitForLoadDecorator - Page Loading in Session: true +[DEBUG - 2015-03-30T08:18:29.810Z] Session [54fd5270-d6b5-11e4-bf64-67a72685a843] - _execFuncAndWaitForLoadDecorator - Page Loading in Session: true +[DEBUG - 2015-03-30T08:18:29.910Z] Session [54fd5270-d6b5-11e4-bf64-67a72685a843] - _execFuncAndWaitForLoadDecorator - Page Loading in Session: true +[DEBUG - 2015-03-30T08:18:30.010Z] Session [54fd5270-d6b5-11e4-bf64-67a72685a843] - _execFuncAndWaitForLoadDecorator - Page Loading in Session: true +[DEBUG - 2015-03-30T08:18:30.110Z] Session [54fd5270-d6b5-11e4-bf64-67a72685a843] - _execFuncAndWaitForLoadDecorator - Page Loading in Session: true +[DEBUG - 2015-03-30T08:18:30.210Z] Session [54fd5270-d6b5-11e4-bf64-67a72685a843] - _execFuncAndWaitForLoadDecorator - Page Loading in Session: true +[DEBUG - 2015-03-30T08:18:30.310Z] Session [54fd5270-d6b5-11e4-bf64-67a72685a843] - _execFuncAndWaitForLoadDecorator - Page Loading in Session: true +[DEBUG - 2015-03-30T08:18:30.410Z] Session [54fd5270-d6b5-11e4-bf64-67a72685a843] - _execFuncAndWaitForLoadDecorator - Page Loading in Session: true +[DEBUG - 2015-03-30T08:18:30.510Z] Session [54fd5270-d6b5-11e4-bf64-67a72685a843] - _execFuncAndWaitForLoadDecorator - Page Loading in Session: true +[DEBUG - 2015-03-30T08:18:30.610Z] Session [54fd5270-d6b5-11e4-bf64-67a72685a843] - _execFuncAndWaitForLoadDecorator - Page Loading in Session: true +[DEBUG - 2015-03-30T08:18:30.710Z] Session [54fd5270-d6b5-11e4-bf64-67a72685a843] - _execFuncAndWaitForLoadDecorator - Page Loading in Session: true +[DEBUG - 2015-03-30T08:18:30.810Z] Session [54fd5270-d6b5-11e4-bf64-67a72685a843] - _execFuncAndWaitForLoadDecorator - Page Loading in Session: true +[DEBUG - 2015-03-30T08:18:30.910Z] Session [54fd5270-d6b5-11e4-bf64-67a72685a843] - _execFuncAndWaitForLoadDecorator - Page Loading in Session: true +[DEBUG - 2015-03-30T08:18:31.010Z] Session [54fd5270-d6b5-11e4-bf64-67a72685a843] - _execFuncAndWaitForLoadDecorator - Page Loading in Session: true +[DEBUG - 2015-03-30T08:18:31.110Z] Session [54fd5270-d6b5-11e4-bf64-67a72685a843] - _execFuncAndWaitForLoadDecorator - Page Loading in Session: true +[DEBUG - 2015-03-30T08:18:31.210Z] Session [54fd5270-d6b5-11e4-bf64-67a72685a843] - _execFuncAndWaitForLoadDecorator - Page Loading in Session: true +[DEBUG - 2015-03-30T08:18:31.310Z] Session [54fd5270-d6b5-11e4-bf64-67a72685a843] - _execFuncAndWaitForLoadDecorator - Page Loading in Session: true +[DEBUG - 2015-03-30T08:18:31.410Z] Session [54fd5270-d6b5-11e4-bf64-67a72685a843] - _execFuncAndWaitForLoadDecorator - Page Loading in Session: true +[DEBUG - 2015-03-30T08:18:31.510Z] Session [54fd5270-d6b5-11e4-bf64-67a72685a843] - _execFuncAndWaitForLoadDecorator - Page Loading in Session: true +[DEBUG - 2015-03-30T08:18:31.610Z] Session [54fd5270-d6b5-11e4-bf64-67a72685a843] - _execFuncAndWaitForLoadDecorator - Page Loading in Session: true +[DEBUG - 2015-03-30T08:18:31.710Z] Session [54fd5270-d6b5-11e4-bf64-67a72685a843] - _execFuncAndWaitForLoadDecorator - Page Loading in Session: true +[DEBUG - 2015-03-30T08:18:31.811Z] Session [54fd5270-d6b5-11e4-bf64-67a72685a843] - _execFuncAndWaitForLoadDecorator - Page Loading in Session: true +[DEBUG - 2015-03-30T08:18:31.911Z] Session [54fd5270-d6b5-11e4-bf64-67a72685a843] - _execFuncAndWaitForLoadDecorator - Page Loading in Session: true +[DEBUG - 2015-03-30T08:18:32.011Z] Session [54fd5270-d6b5-11e4-bf64-67a72685a843] - _execFuncAndWaitForLoadDecorator - Page Loading in Session: true +[DEBUG - 2015-03-30T08:18:32.111Z] Session [54fd5270-d6b5-11e4-bf64-67a72685a843] - _execFuncAndWaitForLoadDecorator - Page Loading in Session: true +[DEBUG - 2015-03-30T08:18:32.211Z] Session [54fd5270-d6b5-11e4-bf64-67a72685a843] - _execFuncAndWaitForLoadDecorator - Page Loading in Session: true +[DEBUG - 2015-03-30T08:18:32.311Z] Session [54fd5270-d6b5-11e4-bf64-67a72685a843] - _execFuncAndWaitForLoadDecorator - Page Loading in Session: true +[DEBUG - 2015-03-30T08:18:32.411Z] Session [54fd5270-d6b5-11e4-bf64-67a72685a843] - _execFuncAndWaitForLoadDecorator - Page Loading in Session: true +[DEBUG - 2015-03-30T08:18:32.511Z] Session [54fd5270-d6b5-11e4-bf64-67a72685a843] - _execFuncAndWaitForLoadDecorator - Page Loading in Session: true +[DEBUG - 2015-03-30T08:18:32.611Z] Session [54fd5270-d6b5-11e4-bf64-67a72685a843] - _execFuncAndWaitForLoadDecorator - Page Loading in Session: true +[DEBUG - 2015-03-30T08:18:32.711Z] Session [54fd5270-d6b5-11e4-bf64-67a72685a843] - _execFuncAndWaitForLoadDecorator - Page Loading in Session: true +[DEBUG - 2015-03-30T08:18:32.811Z] Session [54fd5270-d6b5-11e4-bf64-67a72685a843] - _execFuncAndWaitForLoadDecorator - Page Loading in Session: true +[DEBUG - 2015-03-30T08:18:32.911Z] Session [54fd5270-d6b5-11e4-bf64-67a72685a843] - _execFuncAndWaitForLoadDecorator - Page Loading in Session: true +[DEBUG - 2015-03-30T08:18:33.011Z] Session [54fd5270-d6b5-11e4-bf64-67a72685a843] - _execFuncAndWaitForLoadDecorator - Page Loading in Session: true +[DEBUG - 2015-03-30T08:18:33.111Z] Session [54fd5270-d6b5-11e4-bf64-67a72685a843] - _execFuncAndWaitForLoadDecorator - Page Loading in Session: true +[DEBUG - 2015-03-30T08:18:33.211Z] Session [54fd5270-d6b5-11e4-bf64-67a72685a843] - _execFuncAndWaitForLoadDecorator - Page Loading in Session: true +[DEBUG - 2015-03-30T08:18:33.311Z] Session [54fd5270-d6b5-11e4-bf64-67a72685a843] - _execFuncAndWaitForLoadDecorator - Page Loading in Session: true +[DEBUG - 2015-03-30T08:18:33.411Z] Session [54fd5270-d6b5-11e4-bf64-67a72685a843] - _execFuncAndWaitForLoadDecorator - Page Loading in Session: true +[DEBUG - 2015-03-30T08:18:33.511Z] Session [54fd5270-d6b5-11e4-bf64-67a72685a843] - _execFuncAndWaitForLoadDecorator - Page Loading in Session: true +[DEBUG - 2015-03-30T08:18:33.611Z] Session [54fd5270-d6b5-11e4-bf64-67a72685a843] - _execFuncAndWaitForLoadDecorator - Page Loading in Session: true +[DEBUG - 2015-03-30T08:18:33.711Z] Session [54fd5270-d6b5-11e4-bf64-67a72685a843] - _execFuncAndWaitForLoadDecorator - Page Loading in Session: true +[DEBUG - 2015-03-30T08:18:33.811Z] Session [54fd5270-d6b5-11e4-bf64-67a72685a843] - _execFuncAndWaitForLoadDecorator - Page Loading in Session: true +[DEBUG - 2015-03-30T08:18:33.911Z] Session [54fd5270-d6b5-11e4-bf64-67a72685a843] - _execFuncAndWaitForLoadDecorator - Page Loading in Session: true +[DEBUG - 2015-03-30T08:18:34.011Z] Session [54fd5270-d6b5-11e4-bf64-67a72685a843] - _execFuncAndWaitForLoadDecorator - Page Loading in Session: true +[DEBUG - 2015-03-30T08:18:34.112Z] Session [54fd5270-d6b5-11e4-bf64-67a72685a843] - _execFuncAndWaitForLoadDecorator - Page Loading in Session: true +[DEBUG - 2015-03-30T08:18:34.212Z] Session [54fd5270-d6b5-11e4-bf64-67a72685a843] - _execFuncAndWaitForLoadDecorator - Page Loading in Session: true +[DEBUG - 2015-03-30T08:18:34.312Z] Session [54fd5270-d6b5-11e4-bf64-67a72685a843] - _execFuncAndWaitForLoadDecorator - Page Loading in Session: true +[DEBUG - 2015-03-30T08:18:34.412Z] Session [54fd5270-d6b5-11e4-bf64-67a72685a843] - _execFuncAndWaitForLoadDecorator - Page Loading in Session: true +[DEBUG - 2015-03-30T08:18:34.512Z] Session [54fd5270-d6b5-11e4-bf64-67a72685a843] - _execFuncAndWaitForLoadDecorator - Page Loading in Session: true +[DEBUG - 2015-03-30T08:18:34.612Z] Session [54fd5270-d6b5-11e4-bf64-67a72685a843] - _execFuncAndWaitForLoadDecorator - Page Loading in Session: true +[DEBUG - 2015-03-30T08:18:34.712Z] Session [54fd5270-d6b5-11e4-bf64-67a72685a843] - _execFuncAndWaitForLoadDecorator - Page Loading in Session: true +[DEBUG - 2015-03-30T08:18:34.812Z] Session [54fd5270-d6b5-11e4-bf64-67a72685a843] - _execFuncAndWaitForLoadDecorator - Page Loading in Session: true +[DEBUG - 2015-03-30T08:18:34.912Z] Session [54fd5270-d6b5-11e4-bf64-67a72685a843] - _execFuncAndWaitForLoadDecorator - Page Loading in Session: true +[DEBUG - 2015-03-30T08:18:35.012Z] Session [54fd5270-d6b5-11e4-bf64-67a72685a843] - _execFuncAndWaitForLoadDecorator - Page Loading in Session: true diff --git a/spider/mspider/mspider/__init__.pyc b/spider/mspider/mspider/__init__.pyc index 93ce4b9..3299343 100644 Binary files a/spider/mspider/mspider/__init__.pyc and b/spider/mspider/mspider/__init__.pyc differ diff --git a/spider/mspider/mspider/items.pyc b/spider/mspider/mspider/items.pyc index 04f2329..2a49ac4 100644 Binary files a/spider/mspider/mspider/items.pyc and b/spider/mspider/mspider/items.pyc differ diff --git a/spider/mspider/mspider/pipelines.py b/spider/mspider/mspider/pipelines.py index c10831a..d3d6ff4 100644 --- a/spider/mspider/mspider/pipelines.py +++ b/spider/mspider/mspider/pipelines.py @@ -5,7 +5,7 @@ # Don't forget to add your pipeline to the ITEM_PIPELINES setting # See: http://doc.scrapy.org/en/latest/topics/item-pipeline.html -from .items import DoubanItem +from .items import * from hashlib import md5 import happybase @@ -69,7 +69,7 @@ class HbaseDumper(object): class MspiderPipeline(object): def __init__(self): - self.hbasedumper = HbaseDumper(DoubanItem) + self.hbasedumper = HbaseDumper(LarvaeItem) def process_item(self, item, spider): try: diff --git a/spider/mspider/mspider/pipelines.pyc b/spider/mspider/mspider/pipelines.pyc index 243dfbf..a1fb67b 100644 Binary files a/spider/mspider/mspider/pipelines.pyc and b/spider/mspider/mspider/pipelines.pyc differ diff --git a/spider/mspider/mspider/settings.py b/spider/mspider/mspider/settings.py index 13a446d..b3a59c6 100644 --- a/spider/mspider/mspider/settings.py +++ b/spider/mspider/mspider/settings.py @@ -21,6 +21,9 @@ NEWSPIDER_MODULE = 'mspider.spiders' ##Custom for Chunk +LOG_LEVEL = 'WARNING' + + # anti-banning COOKIES_ENABLED = True # USER_AGENT = 'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/40.0.2214.111 Safari/537.36' @@ -54,5 +57,14 @@ ITEM_PIPELINES = { } # DOWNLOAD_DELAY = 1 -# ITEM_PIPELINES = {'mspider.pipelines.MspiderPipeline': 300, } + +ITEM_PIPELINES = {'mspider.pipelines.MspiderPipeline': 300, } + +CLOSESPIDER_TIMEOUT = 1800 +CLOSESPIDER_ITEMCOUNT = 10000 +CLOSESPIDER_ERRORCOUNT = 1 + + + + diff --git a/spider/mspider/mspider/settings.pyc b/spider/mspider/mspider/settings.pyc index 65b5da0..a993d3f 100644 Binary files a/spider/mspider/mspider/settings.pyc and b/spider/mspider/mspider/settings.pyc differ diff --git a/spider/mspider/mspider/spiders/__init__.pyc b/spider/mspider/mspider/spiders/__init__.pyc index 44aef51..09b85ce 100644 Binary files a/spider/mspider/mspider/spiders/__init__.pyc and b/spider/mspider/mspider/spiders/__init__.pyc differ diff --git a/spider/mspider/mspider/spiders/larvae.py b/spider/mspider/mspider/spiders/larvae.py index a5af4ab..9d94c4c 100644 --- a/spider/mspider/mspider/spiders/larvae.py +++ b/spider/mspider/mspider/spiders/larvae.py @@ -27,11 +27,11 @@ class Larvae(Spider): xpaths = json.loads(parser.get('custom', 'xpaths')) def start_requests(self): - print self.start_urls - print self.allowed_domains - print self.cookies - print self.regexes - print self.xpaths + print "start urls:", self.start_urls + print "allowed domains:", self.allowed_domains + print "cookies:", self.cookies + print "regexes:", self.regexes + print "xpaths:", self.xpaths self.cookies = json.loads(parser.get("custom", "cookies")) for url in self.start_urls: @@ -59,4 +59,4 @@ class Larvae(Spider): item['data'] = requests.get(img).content item['id'] = md5(item['data']).hexdigest() - yield item \ No newline at end of file + yield item diff --git a/spider/mspider/mspider/spiders/larvae.pyc b/spider/mspider/mspider/spiders/larvae.pyc index 8c9f864..a3fb527 100644 Binary files a/spider/mspider/mspider/spiders/larvae.pyc and b/spider/mspider/mspider/spiders/larvae.pyc differ diff --git a/spider/mspider/mspider/spiders/test000.pyc b/spider/mspider/mspider/spiders/test000.pyc index 4ab0e24..788c68b 100644 Binary files a/spider/mspider/mspider/spiders/test000.pyc and b/spider/mspider/mspider/spiders/test000.pyc differ diff --git a/spider/mspider/mymidlleware.pyc b/spider/mspider/mymidlleware.pyc index 003f23b..79c80ce 100644 Binary files a/spider/mspider/mymidlleware.pyc and b/spider/mspider/mymidlleware.pyc differ -- libgit2 0.21.2