Spark Example

 # Import MySQL database to R
library(RMySQL)
# Connect to the MySQL database
conn <- dbConnect(
  RMySQL::MySQL(), 
  dbname='global-warming',
  host='127.0.0.1',
  port= 3306,
  user='root', 
  password='TempPassword')
# Importtable into R data.frames
df <- dbGetQuery(conn, "select * from co2;")
# Disconnect from the database
dbDisconnect(conn)

[1] TRUE

# Install and import Spark
library(sparklyr) # Connect R to Spark
spark_install(version='2.0.2') # Install spark (if not already installed)

# Create local spark cluster
sc <- spark_connect(master = "local", spark_home=spark_home_dir(version = "2.0.2"))

# Copy data frames into local Spark cluster tables 
library(dplyr)
df_tbl <- copy_to(sc, df, overwrite = TRUE)

# Display the data structures of the data stored in native R vs Spark
class(df) # intial data imported into R

[1] "data.frame"

class(df_tbl) # data converted to tbl_spark

[1] "tbl_spark" "tbl_sql"   "tbl_lazy"  "tbl"      

# Display the Spark Table
head(df_tbl)

# Summarise the Spark table to display total_co2 and count of co2 records per country
df_tbl %>%
  group_by(country_name) %>%
  summarize(total_co2 = sum(co2_emissions), count = n())

Missing values are always removed in SQL.
Use `SUM(x, na.rm = TRUE)` to silence this warningMissing values are always removed in SQL.
Use `SUM(x, na.rm = TRUE)` to silence this warningMissing values are always removed in SQL.
Use `SUM(x, na.rm = TRUE)` to silence this warningMissing values are always removed in SQL.
Use `SUM(x, na.rm = TRUE)` to silence this warningMissing values are always removed in SQL.
Use `SUM(x, na.rm = TRUE)` to silence this warningMissing values are always removed in SQL.
Use `SUM(x, na.rm = TRUE)` to silence this warning

# Disconnect from spark
spark_disconnect(sc)