Webdriver - Web Scraping

There are speed comparison between call Py-script and RSelenium through Webdriver https://github.com/englianhu/WebDriver-DynamicWebpage-Scrapping.

#### Using rPython to call py script

library(rPython)
#'python.exec('from selenium import webdriver')
#'python.exec('browser = webdriver.Chrome()')
#'python.exec('browser.set_window_size(1015, 600)')
#'python.exec('browser.set_window_position(0, 200)')

#'python.load('C:/Users/Scibrokes Trading/Documents/GitHub/englianhu/WebDriver-DynamicWebpage-Scrapping/7M.py')
#'python.load('C:/Users/Scibrokes Trading/Documents/Github/englianhu/WebDriver-DynamicWebpage-Scrapping/NowGoal.py')
python.load('C:/Users/Scibrokes Trading/Documents/Github/englianhu/WebDriver-DynamicWebpage-Scrapping/Pymodel.py')
url0910 = 'http://data2.7m.cn/history_Matches_Data/2009-2010/92/en/index.shtml'

python.assign('url0910', url0910)
system.time(python.exec('eng0910 = get_7M_matches(url0910)'))

##    user  system elapsed 
##    2.33    0.07   28.36

library(RSelenium)
lnk <- 'http://data2.7m.cn/history_Matches_Data/2009-2010/92/en/index.shtml'

remDr <- remoteDriver(browserName = "chrome")
#'remDr$setImplicitWaitTimeout(3000)
remDr$open()

## [1] "Connecting to remote server"
## $applicationCacheEnabled
## [1] FALSE
## 
## $rotatable
## [1] FALSE
## 
## $mobileEmulationEnabled
## [1] FALSE
## 
## $chrome
## $chrome$userDataDir
## [1] "C:\\Users\\SCIBRO~1\\AppData\\Local\\Temp\\scoped_dir2280_595"
## 
## 
## $takesHeapSnapshot
## [1] TRUE
## 
## $databaseEnabled
## [1] FALSE
## 
## $handlesAlerts
## [1] TRUE
## 
## $version
## [1] "39.0.2171.95"
## 
## $platform
## [1] "WIN8"
## 
## $browserConnectionEnabled
## [1] FALSE
## 
## $nativeEvents
## [1] TRUE
## 
## $acceptSslCerts
## [1] TRUE
## 
## $webdriver.remote.sessionid
## [1] "11145f33-da0e-42ff-b6f4-95b658f9705d"
## 
## $locationContextEnabled
## [1] TRUE
## 
## $webStorageEnabled
## [1] TRUE
## 
## $browserName
## [1] "chrome"
## 
## $takesScreenshot
## [1] TRUE
## 
## $javascriptEnabled
## [1] TRUE
## 
## $cssSelectorsEnabled
## [1] TRUE
## 
## $id
## [1] "11145f33-da0e-42ff-b6f4-95b658f9705d"

remDr$navigate(lnk)

tableElem <- remDr$findElement("id", "e_run_tb")
xData <- tableElem$getElementAttribute("outerHTML")[[1]]
xData <- htmlParse(xData, encoding = "UTF-8")
mRnd <- readHTMLTable(xData)$e_run_tb
rm(tableElem, xData)

system.time(lapply(as.list(seq(nrow(mRnd))), function(i){
  lapply(as.list(seq(ncol(mRnd))), function(j){
    webElem <- remDr$findElement(using = 'xpath', paste0('//*[@id="e_run_tb"]/tbody/tr[',i,']/td[',j,']'))
    webElem$clickElement()

    tableElem <- remDr$findElement("id", "Match_Table")
    xData <- tableElem$getElementAttribute("outerHTML")[[1]]
    xData <- htmlParse(xData, encoding = "UTF-8")
    readHTMLTable(xData)})}))

##    user  system elapsed 
##    2.68    0.03    7.53