library(rhdfs)
## Loading required package: rJava
##
## HADOOP_CMD=/usr/bin/hadoop
##
## Be sure to run hdfs.init()
library(rmr2)
## Warning: S3 methods 'gorder.default', 'gorder.factor', 'gorder.data.frame',
## 'gorder.matrix', 'gorder.raw' were declared in NAMESPACE but not found
## Please review your hadoop settings. See help(hadoop.settings)
library(plyrmr)
## Loading required package: reshape2
## Loading required package: dplyr
##
## Attaching package: 'dplyr'
##
## The following object is masked from 'package:stats':
##
## filter
##
## The following objects are masked from 'package:base':
##
## intersect, setdiff, setequal, union
##
##
## Attaching package: 'plyrmr'
##
## The following objects are masked from 'package:dplyr':
##
## count, intersect, select, transmute, ungroup, union
##
## The following object is masked from 'package:reshape2':
##
## dcast
##
## The following object is masked from 'package:rmr2':
##
## gather
##
## The following objects are masked from 'package:base':
##
## intersect, ncol, nrow, rbind, sample, union
hdfs.init()
rmr.options(backend="local")
## NULL
plyrmr.options(backend="local")
## list()
file.remove('/tmp/mtcars')
## [1] TRUE
cardata = to.dfs(mtcars, output = '/tmp/mtcars')
bind.cols(input(cardata), carb.per.cyl = carb/cyl)
## mpg cyl disp hp drat wt qsec vs am gear carb carb.per.cyl
## 1 21.0 6 160.0 110 3.90 2.620 16.46 0 1 4 4 0.6666667
## 2 21.0 6 160.0 110 3.90 2.875 17.02 0 1 4 4 0.6666667
## 3 22.8 4 108.0 93 3.85 2.320 18.61 1 1 4 1 0.2500000
## 4 21.4 6 258.0 110 3.08 3.215 19.44 1 0 3 1 0.1666667
## 5 18.7 8 360.0 175 3.15 3.440 17.02 0 0 3 2 0.2500000
## 6 18.1 6 225.0 105 2.76 3.460 20.22 1 0 3 1 0.1666667
## 7 14.3 8 360.0 245 3.21 3.570 15.84 0 0 3 4 0.5000000
## 8 24.4 4 146.7 62 3.69 3.190 20.00 1 0 4 2 0.5000000
## 9 22.8 4 140.8 95 3.92 3.150 22.90 1 0 4 2 0.5000000
## 10 19.2 6 167.6 123 3.92 3.440 18.30 1 0 4 4 0.6666667
## 11 17.8 6 167.6 123 3.92 3.440 18.90 1 0 4 4 0.6666667
## 12 16.4 8 275.8 180 3.07 4.070 17.40 0 0 3 3 0.3750000
## 13 17.3 8 275.8 180 3.07 3.730 17.60 0 0 3 3 0.3750000
## 14 15.2 8 275.8 180 3.07 3.780 18.00 0 0 3 3 0.3750000
## 15 10.4 8 472.0 205 2.93 5.250 17.98 0 0 3 4 0.5000000
## 16 10.4 8 460.0 215 3.00 5.424 17.82 0 0 3 4 0.5000000
## 17 14.7 8 440.0 230 3.23 5.345 17.42 0 0 3 4 0.5000000
## 18 32.4 4 78.7 66 4.08 2.200 19.47 1 1 4 1 0.2500000
## 19 30.4 4 75.7 52 4.93 1.615 18.52 1 1 4 2 0.5000000
## 20 33.9 4 71.1 65 4.22 1.835 19.90 1 1 4 1 0.2500000
## 21 21.5 4 120.1 97 3.70 2.465 20.01 1 0 3 1 0.2500000
## 22 15.5 8 318.0 150 2.76 3.520 16.87 0 0 3 2 0.2500000
## 23 15.2 8 304.0 150 3.15 3.435 17.30 0 0 3 2 0.2500000
## 24 13.3 8 350.0 245 3.73 3.840 15.41 0 0 3 4 0.5000000
## 25 19.2 8 400.0 175 3.08 3.845 17.05 0 0 3 2 0.2500000
## 26 27.3 4 79.0 66 4.08 1.935 18.90 1 1 4 1 0.2500000
## 27 26.0 4 120.3 91 4.43 2.140 16.70 0 1 5 2 0.5000000
## 28 30.4 4 95.1 113 3.77 1.513 16.90 1 1 5 2 0.5000000
## 29 15.8 8 351.0 264 4.22 3.170 14.50 0 1 5 4 0.5000000
## 30 19.7 6 145.0 175 3.62 2.770 15.50 0 1 5 6 1.0000000
## 31 15.0 8 301.0 335 3.54 3.570 14.60 0 1 5 8 1.0000000
## 32 21.4 4 121.0 109 4.11 2.780 18.60 1 1 4 2 0.5000000
data(Titanic)
titanic = data.frame(Titanic)
where(
titanic,
Freq >=100)
## Class Sex Age Survived Freq
## 9 1st Male Adult No 118
## 10 2nd Male Adult No 154
## 11 3rd Male Adult No 387
## 12 Crew Male Adult No 670
## 28 Crew Male Adult Yes 192
## 29 1st Female Adult Yes 140
titanic %|% where(Freq >=100)
## Class Sex Age Survived Freq
## 9 1st Male Adult No 118
## 10 2nd Male Adult No 154
## 11 3rd Male Adult No 387
## 12 Crew Male Adult No 670
## 28 Crew Male Adult Yes 192
## 29 1st Female Adult Yes 140
file.remove('/tmp/titanic')
## [1] TRUE
tidata = to.dfs(data.frame(Titanic), output =
'/tmp/titanic')
input(tidata) %|% transmute(sum(Freq))
## sum.Freq.
## 1 2201
input(tidata) %|% group(Sex) %|%
transmute(sum(Freq))
## Sex sum.Freq.
## 1 Male 1731
## 2 Female 470
as.data.frame(count(input(tidata),Sex))
## Sex.Sex Sex.freq
## 1 Female 16
## 2 Male 16
sample(input(tidata), n=10)
## Class Sex Age Survived Freq
## 1 1st Male Child No 0
## 2 2nd Male Child No 0
## 3 3rd Male Child No 35
## 4 Crew Male Child No 0
## 5 1st Female Child No 0
## 6 2nd Female Child No 0
## 7 3rd Female Child No 17
## 8 Crew Female Child No 0
## 9 1st Male Adult No 118
## 10 2nd Male Adult No 154
as.data.frame(top.k(input(tidata), .k=5, Freq))
## Class Sex Age Survived Freq
## 1 Crew Male Adult No 670
## 2 3rd Male Adult No 387
## 3 Crew Male Adult Yes 192
## 4 2nd Male Adult No 154
## 5 1st Female Adult Yes 140
as.data.frame(bottom.k(input(tidata), .k=5, Freq))
## Class Sex Age Survived Freq
## 1 1st Male Child No 0
## 2 2nd Male Child No 0
## 3 Crew Male Child No 0
## 4 1st Female Child No 0
## 5 2nd Female Child No 0
convert_tb = data.frame(Label=c("No","Yes"),
Symbol=c(0,1))
ctb = to.dfs(convert_tb, output = 'convert')
as.data.frame(plyrmr::merge(input(tidata), input(ctb),
by.x="Survived", by.y="Label"))
## Survived Class Sex Age Freq Symbol
## 1 No 1st Male Child 0 0
## 2 No 2nd Male Child 0 0
## 3 No 3rd Male Child 35 0
## 4 No Crew Male Child 0 0
## 5 No 1st Female Child 0 0
## 6 No 2nd Female Child 0 0
## 7 No 3rd Female Child 17 0
## 8 No Crew Female Child 0 0
## 9 No 1st Male Adult 118 0
## 10 No 2nd Male Adult 154 0
## 11 No 3rd Male Adult 387 0
## 12 No Crew Male Adult 670 0
## 13 No 1st Female Adult 4 0
## 14 No 2nd Female Adult 13 0
## 15 No 3rd Female Adult 89 0
## 16 No Crew Female Adult 3 0
## 17 Yes 1st Male Child 5 1
## 18 Yes 2nd Male Child 11 1
## 19 Yes 3rd Male Child 13 1
## 20 Yes Crew Male Child 0 1
## 21 Yes 1st Female Child 1 1
## 22 Yes 2nd Female Child 13 1
## 23 Yes 3rd Female Child 14 1
## 24 Yes Crew Female Child 0 1
## 25 Yes 1st Male Adult 57 1
## 26 Yes 2nd Male Adult 14 1
## 27 Yes 3rd Male Adult 75 1
## 28 Yes Crew Male Adult 192 1
## 29 Yes 1st Female Adult 140 1
## 30 Yes 2nd Female Adult 80 1
## 31 Yes 3rd Female Adult 76 1
## 32 Yes Crew Female Adult 20 1
file.remove('convert')
## [1] TRUE
file.remove('/tmp/tempreture')
## [1] TRUE
tempreture = read.table("~/rhadoopiii/Data/tempreture.tab", head=TRUE)
tempdata = to.dfs(tempreture, output =
'/tmp/tempreture')
res = input("/tmp/tempreture") %|%
group(STN...) %|%
group(YEARMODA) %|%
transmute(mean.temp = mean(TEMP))
temp = data.frame(res)
head(temp)
## STN... YEARMODA mean.temp
## 1 8403 20140101 85.8
## 2 8403 20140102 86.3
## 3 8403 20140103 85.9
## 4 8403 20140104 85.6
## 5 8403 20140105 84.8
## 6 8403 20140106 86.8
dim(temp)
## [1] 57 3
temp[,2] <- as.integer(temp[,2])
colnames(temp) <- c("station", "yearMonth",
"mean.temp")
mean( table(temp[,1]) )
## [1] 57
mean( table(temp[,2]) )
## [1] 1
library(ggplot2)
ggplot(temp, aes(yearMonth, mean.temp, group=station, colour=station)) + geom_line() + labs(x="Date", y="Temperature in F", title="Changes in Average Temperature") + theme(legend.position = "none") + scale_x_continuous(breaks=20140101:20140226) + stat_summary(fun.y = mean, colour = "red", geom="line", aes(group = 1))
