RevoscaleR Reading and Transforming Data
This notebook follows the tutorial at
https://docs.microsoft.com/en-us/machine-learning-server/r/tutorial-revoscaler-data-import-transform
library(readr)
library(RevoScaleR)
list.files(rxGetOption("sampleDataDir"))## [1] "AirlineDemo1kNoMissing.csv" "AirlineDemoSmall.csv"
## [3] "AirlineDemoSmall.xdf" "AirlineDemoSmallComposite"
## [5] "AirlineDemoSmallOrc" "AirlineDemoSmallParquet"
## [7] "AirlineDemoSmallSplit" "AirlineDemoSmallUC.xdf"
## [9] "ccFraudScoreSmall.csv" "ccFraudSmall.csv"
## [11] "CensusWorkers.xdf" "claims.dat"
## [13] "claims.sas7bdat" "claims.sav"
## [15] "claims.sd7" "claims.sqlite"
## [17] "claims.sts" "claims.txt"
## [19] "claims.xdf" "claims_.txt"
## [21] "claims4blocks.xdf" "claimsExtra.txt"
## [23] "claimsParquet" "claimsQuote.txt"
## [25] "claimsTab.txt" "claimsTxt"
## [27] "claimsXdf" "CustomerSurvey.xdf"
## [29] "DJIAdaily.xdf" "fourthgraders.xdf"
## [31] "hyphens.txt" "Kyphosis.xdf"
## [33] "mortDefaultSmall.xdf" "mortDefaultSmall2000.csv"
## [35] "mortDefaultSmall2001.csv" "mortDefaultSmall2002.csv"
## [37] "mortDefaultSmall2003.csv" "mortDefaultSmall2004.csv"
## [39] "mortDefaultSmall2005.csv" "mortDefaultSmall2006.csv"
## [41] "mortDefaultSmall2007.csv" "mortDefaultSmall2008.csv"
## [43] "mortDefaultSmall2009.csv" "mrsDebugParquet"
## [45] "README" "testAvro4.bin"
## [47] "Utf16leDb.sqlite"Where is my working directory?
getwd()## [1] "D:/Dropbox/Documents/SMU/CSC 360/Fall 2017 MSSA" mysource <- file.path(rxGetOption("sampleDataDir"), "AirlineDemoSmall.csv")
airXdfData <- rxImport(inData=mysource)## Rows Read: 500000, Total Rows Processed: 500000, Total Chunk Time: 0.989 seconds
## Rows Read: 100000, Total Rows Processed: 600000, Total Chunk Time: 0.270 secondsNote that the data was read in in chunks.
let’s also import this data using RStudio’s import facility.
AirlineDemoSmall_df <- read_csv("C:/Program Files/Microsoft/ML Server/R_SERVER/library/RevoScaleR/SampleData/AirlineDemoSmall.csv",
col_types = cols(ArrDelay = col_integer()))## Warning in rbind(names(probs), probs_f): number of columns of result is not
## a multiple of vector length (arg 1)## Warning: 17372 parsing failures.
## row # A tibble: 5 x 5 col row col expected actual expected <int> <chr> <chr> <chr> actual 1 99 ArrDelay an integer M file 2 312 ArrDelay an integer M row 3 325 ArrDelay an integer M col 4 611 ArrDelay an integer M expected 5 781 ArrDelay an integer M actual # ... with 1 more variables: file <chr>
## ... ................. ... .................................. ........ .................................. ...... .................................. .... .................................. ... .................................. ... .................................. ........ .................................. ...... .......................................
## See problems(...) for more details.Do a summary of the dataframe
summary(AirlineDemoSmall_df)## ArrDelay CRSDepTime DayOfWeek
## Min. : -86.00 Min. : 0.01667 Length:600000
## 1st Qu.: -9.00 1st Qu.: 9.41667 Class :character
## Median : 0.00 Median :13.41667 Mode :character
## Mean : 11.32 Mean :13.48227
## 3rd Qu.: 16.00 3rd Qu.:17.33333
## Max. :1490.00 Max. :23.98333
## NA's :17372Make a copy of the xdf file on disk
airXdfData <- rxImport(inData=mysource, outFile="airExample.xdf",overwrite = TRUE)## Rows Read: 500000, Total Rows Processed: 500000, Total Chunk Time: 1.003 seconds
## Rows Read: 100000, Total Rows Processed: 600000, Total Chunk Time: 0.270 secondsWhat is airXdfData?
str(airXdfData)## Formal class 'RxXdfData' [package "RevoScaleR"] with 19 slots
## ..@ fileSystem :List of 1
## .. ..$ fileSystemType: chr "native"
## .. ..- attr(*, "class")= chr [1:2] "RxNativeFileSystem" "RxFileSystem"
## ..@ createCompositeSet : NULL
## ..@ createPartitionSet : NULL
## ..@ blocksPerCompositeFile: int 3
## ..@ readByBlock : logi TRUE
## ..@ xdfUuid : chr "005759F5D17B4E48BF0D832E394664F7"
## ..@ cache : logi FALSE
## ..@ dfName : chr "df-9BFD30DC8CAE4247AF2481278DE0C463"
## ..@ dfType : chr "xdf"
## ..@ dfSource : chr "airExample.xdf"
## ..@ file : chr "airExample.xdf"
## ..@ colNames : chr ""
## ..@ id :<externalptr>
## ..@ colClasses : NULL
## ..@ colInfo : NULL
## ..@ returnDataFrame : logi TRUE
## ..@ stringsAsFactors : logi FALSE
## ..@ rowsOrBlocksPerRead : int 1
## ..@ compatibilityRequest :Classes 'CompatibilityRequest', 'R6' <CompatibilityRequest>
## Public:
## assertServerCapability: function (capability, notSupported, notKnown)
## clone: function (deep = FALSE)
## deferredAssertServerCapability: function (capability, notSupported, notKnown)
## getRequestedCapabilities: function ()
## initialize: function (server, notSupported = capabilityNotSupported, notKnown = serverNotKnown)
## merge: function (request)
## requestCapability: function (capability)
## runDeferredAssertions: function (server)
## serialize: function (file)
## Private:
## deferredRequests: list
## notKnown: function (server, capability, warningMessage)
## notSupported: function (server, capability, errorMessage)
## requestedCapabilities:
## runCallback: function (type, server, capability, userHandler)
## server: ServerDefinition, AbstractServerDefinitionIt’s definitely not a dataframe!
We do have a utility function that looks a lot like summary.
rxGetInfo(airXdfData, getVarInfo = TRUE)## File name: D:\Dropbox\Documents\SMU\CSC 360\Fall 2017 MSSA\airExample.xdf
## Number of observations: 6e+05
## Number of variables: 3
## Number of blocks: 2
## Compression type: zlib
## Variable information:
## Var 1: ArrDelay, Type: character
## Var 2: CRSDepTime, Type: numeric, Storage: float32, Low/High: (0.0167, 23.9833)
## Var 3: DayOfWeek, Type: characterWhat happens if we try to run summary?
summary(airXdfData)## Call:
## rxSummary(formula = form, data = object, byTerm = TRUE, reportProgress = 0L)
##
## Summary Statistics Results for: ~ArrDelay + CRSDepTime + DayOfWeek
## Data: object (RxXdfData Data Source)
## File name: airExample.xdf
## Number of valid observations: 6e+05
##
## Name Mean StdDev Min Max ValidObs MissingObs
## CRSDepTime 13.48227 4.697566 0.016667 23.98333 6e+05 0Note that this is different from what we would get from running summary on a dataframe. Note that summary() is more than one function. It looks at its input object and finds a summary particular to that type of object.
Also note that the median and quantiles are missing.
It is probably more convenient to use a different rx funcion if we just want the variable information.
rxGetVarInfo(airXdfData) ## Var 1: ArrDelay, Type: character
## Var 2: CRSDepTime, Type: numeric, Storage: float32, Low/High: (0.0167, 23.9833)
## Var 3: DayOfWeek, Type: characterReimport the data using diferent parameters.
airXdfData <- rxImport(inData=mysource, outFile="airExample.xdf",
stringsAsFactors=TRUE, missingValueString="M", rowsPerRead=200000,
overwrite=TRUE)## Rows Read: 200000, Total Rows Processed: 200000, Total Chunk Time: 0.377 seconds
## Rows Read: 200000, Total Rows Processed: 400000, Total Chunk Time: 0.355 seconds
## Rows Read: 200000, Total Rows Processed: 600000, Total Chunk Time: 0.362 secondsLook at the results again.
rxGetInfo(airXdfData)## File name: D:\Dropbox\Documents\SMU\CSC 360\Fall 2017 MSSA\airExample.xdf
## Number of observations: 6e+05
## Number of variables: 3
## Number of blocks: 3
## Compression type: zlibThis is different from the output in the Microsoft document. Use instead.
rxGetVarInfo(airXdfData)## Var 1: ArrDelay, Type: integer, Low/High: (-86, 1490)
## Var 2: CRSDepTime, Type: numeric, Storage: float32, Low/High: (0.0167, 23.9833)
## Var 3: DayOfWeek
## 7 factor levels: Monday Tuesday Wednesday Thursday Friday Saturday SundayGet descriptive statistics for ArrDelay
rxSummary(~ ArrDelay, data = airXdfData)## Rows Read: 200000, Total Rows Processed: 200000, Total Chunk Time: 0.004 seconds
## Rows Read: 200000, Total Rows Processed: 400000, Total Chunk Time: 0.005 seconds
## Rows Read: 200000, Total Rows Processed: 600000, Total Chunk Time: 0.004 seconds
## Computation time: 0.016 seconds.## Call:
## rxSummary(formula = ~ArrDelay, data = airXdfData)
##
## Summary Statistics Results for: ~ArrDelay
## Data: airXdfData (RxXdfData Data Source)
## File name: airExample.xdf
## Number of valid observations: 6e+05
##
## Name Mean StdDev Min Max ValidObs MissingObs
## ArrDelay 11.31794 40.68854 -86 1490 582628 17372Visualize the data
rxHistogram(~ArrDelay|DayOfWeek, data = airXdfData)## Rows Read: 200000, Total Rows Processed: 200000, Total Chunk Time: 0.080 secondsRows Read: 200000, Total Rows Processed: 400000, Total Chunk Time: 0.077 secondsRows Read: 200000, Total Rows Processed: 600000, Total Chunk Time: 0.082 seconds
## Computation time: 0.250 seconds.