library(knitr)
# A Prefix nulling hook.

# Make sure to keep the default for normal processing.
default_output_hook <- knitr::knit_hooks$get("output")

# Output hooks handle normal R console output.
knitr::knit_hooks$set( output = function(x, options) {

  comment <- knitr::opts_current$get("comment")
  if( is.na(comment) ) comment <- ""
  can_null <- grepl( paste0( comment, "\\s*\\[\\d?\\]" ),
                     x, perl = TRUE)
  do_null <- isTRUE( knitr::opts_current$get("null_prefix") )
  if( can_null && do_null ) {
    # By default R print output aligns at the right brace.
    align_index <- regexpr( "\\]", x )[1] - 1
    # Two cases: start or newline
    re <- paste0( "^.{", align_index, "}\\]")
    rep <- comment
    x <- gsub( re, rep,  x )
    re <- paste0( "\\\n.{", align_index, "}\\]")
    rep <- paste0( "\n", comment )
    x <- gsub( re, rep,  x )
  }

  default_output_hook( x, options )

})

knitr::opts_template$set("kill_prefix"=list(comment=NA, null_prefix=TRUE))

Principles of Data Visualization and Introduction to ggplot2

I have provided you with data about the 5,000 fastest growing companies in the US, as compiled by Inc. magazine. lets read this in:

inc <- read.csv("https://raw.githubusercontent.com/charleyferrari/CUNY_DATA_608/master/module1/Data/inc5000_data.csv", header= TRUE)

And lets preview this data:

head(inc)
  Rank                         Name Growth_Rate   Revenue
1    1                         Fuhu      421.48 1.179e+08
2    2        FederalConference.com      248.31 4.960e+07
3    3                The HCI Group      245.45 2.550e+07
4    4                      Bridger      233.08 1.900e+09
5    5                       DataXu      213.37 8.700e+07
6    6 MileStone Community Builders      179.38 4.570e+07
                      Industry Employees         City State
1 Consumer Products & Services       104   El Segundo    CA
2          Government Services        51     Dumfries    VA
3                       Health       132 Jacksonville    FL
4                       Energy        50      Addison    TX
5      Advertising & Marketing       220       Boston    MA
6                  Real Estate        63       Austin    TX
summary(inc)
      Rank                          Name       Growth_Rate     
 Min.   :   1   (Add)ventures         :   1   Min.   :  0.340  
 1st Qu.:1252   @Properties           :   1   1st Qu.:  0.770  
 Median :2502   1-Stop Translation USA:   1   Median :  1.420  
 Mean   :2502   110 Consulting        :   1   Mean   :  4.612  
 3rd Qu.:3751   11thStreetCoffee.com  :   1   3rd Qu.:  3.290  
 Max.   :5000   123 Exteriors         :   1   Max.   :421.480  
                (Other)               :4995                    
    Revenue                                  Industry      Employees      
 Min.   :2.000e+06   IT Services                 : 733   Min.   :    1.0  
 1st Qu.:5.100e+06   Business Products & Services: 482   1st Qu.:   25.0  
 Median :1.090e+07   Advertising & Marketing     : 471   Median :   53.0  
 Mean   :4.822e+07   Health                      : 355   Mean   :  232.7  
 3rd Qu.:2.860e+07   Software                    : 342   3rd Qu.:  132.0  
 Max.   :1.010e+10   Financial Services          : 260   Max.   :66803.0  
                     (Other)                     :2358   NA's   :12       
            City          State     
 New York     : 160   CA     : 701  
 Chicago      :  90   TX     : 387  
 Austin       :  88   NY     : 311  
 Houston      :  76   VA     : 283  
 San Francisco:  75   FL     : 282  
 Atlanta      :  74   IL     : 273  
 (Other)      :4438   (Other):2764

Think a bit on what these summaries mean. Use the space below to add some more relevant non-visual exploratory information you think helps you understand this data:

First let’s start with some aggregate counts of the data:

library(dplyr)
library(kableExtra)

temp <- inc %>%
  select(Industry) %>%
  unique() %>%
  tally() %>%
  rename(Industries = n)

temp <- inc %>%
  select(City) %>%
  unique() %>%
  tally() %>%
  rename(Cities = n) %>%
  bind_cols(temp)

inc %>%
  select(State) %>%
  unique() %>%
  tally() %>%
  rename(States = n) %>%
  bind_cols(temp) %>%
  kable() %>%
  kable_styling()
States Cities Industries
52 1519 25

Hmmm. There are 52 states in the data set. I’m guessing there is DC but I wonder if there is an error in the data. I will take a closer look at this:

unique(inc$State)
 CA VA FL TX MA TN UT RI SC DC NJ OH WA ME NY CO GA IL AZ NC MD MN OK
 PA CT IN MS WI WY MI MO KS OR NE AL HI NV IA KY ID AK LA DE AR NH VT
 NM SD ND PR MT WV
52 Levels: AK AL AR AZ CA CO CT DC DE FL GA HI IA ID IL IN KS KY LA ... WY

Ahh! There’s Puerto Rico. My hunch about DC was correct. No need to clean that up. Great!

I am wondering how some of the variables differ the industry. I will look at the size of the work force and the revenue. I will also construct an output per worker to see which industry is the most productive in economic terms:

inc %>%
  group_by(Industry) %>%
  mutate(`Output per Worker` = Revenue / Employees) %>%
  summarise(`Avg Employees` = round(mean(Employees, na.rm = TRUE), 0),
            `Avg Revenue` = mean(Revenue, na.rm = TRUE),
            `Avg Output per Worker` = round(mean(`Output per Worker`, na.rm = TRUE), 0),
            Count = n()) %>%
  arrange(desc(`Avg Output per Worker`)) %>%
  kable() %>%
  kable_styling()
Industry Avg Employees Avg Revenue Avg Output per Worker Count
Energy 243 126344954 1554656 109
Computer Hardware 221 270129545 817702 44
Logistics & Transportation 260 95745161 794811 155
Food & Beverage 511 98559542 618383 131
Insurance 147 46758000 474966 50
Consumer Products & Services 224 73676847 466068 203
Construction 156 70450802 465682 187
Manufacturing 172 49546875 453524 256
Telecommunications 243 56855814 449260 129
Real Estate 199 30892708 434516 96
Travel & Hospitality 372 47283871 414788 62
Retail 183 50529064 412555 203
Human Resources 1158 47173980 395972 196
Financial Services 183 50580385 394231 260
Business Products & Services 244 54705187 359097 482
Health 233 50319437 325199 355
Media 177 32266667 307144 54
Advertising & Marketing 84 16528662 306036 471
Education 93 13726506 296454 83
Environmental Services 199 51741176 283607 51
Security 562 52230137 283391 73
IT Services 140 28214598 270494 733
Government Services 130 29748020 243596 202
Software 150 23802924 225989 342
Engineering 276 34222973 201120 74

This is not supprising that the top industries are the most “productive.” They are very capital intensive.

Question 1

Create a graph that shows the distribution of companies in the dataset by State (ie how many are in each state). There are a lot of States, so consider which axis you should use. This visualization is ultimately going to be consumed on a ‘portrait’ oriented screen (ie taller than wide), which should further guide your layout choices.

library(ggplot2)

inc %>%
  group_by(State) %>%
  tally() %>%
  ggplot(aes(x = n, y=reorder(State, n))) +
  geom_point(color = "dodger blue") +
  labs(title = "Number of Fastest Growing Companies by State",
       y = element_blank(),
       x = element_blank()) +
  theme(panel.background = element_blank(),
        panel.grid.major = element_line(color = "gray95"),
        panel.grid.major.x = element_line(color = "gray95", linetype="dashed"))

Question 2

Lets dig in on the state with the 3rd most companies in the data set. Imagine you work for the state and are interested in how many people are employed by companies in different industries. Create a plot that shows the average and/or median employment by industry for companies in this state (only use cases with full data, use R’s complete.cases() function.) In addition to this, your graph should show how variable the ranges are, and you should deal with outliers.

inc %>%
  filter(State == "NY") %>%
  select(Industry, Employees) %>%
  na.omit() %>%
  group_by(Industry) %>%
  mutate(Median = median(Employees)) %>%
  ggplot(aes(x = reorder(Industry, Median), y = Employees)) +
  ylim(0, 1000) +
  labs(title = "Workforce Size by Industry",
       subtitle = "(employment over 1,000 not shown)",
       y = element_blank(),
       x = element_blank()) +
  theme(panel.background = element_blank(),
        panel.grid.major = element_line(color = "gray95"),
        panel.grid.major.x = element_line(color = "gray95", linetype="dashed")) +
  geom_boxplot(outlier.colour="gray", outlier.shape = 8, outlier.size = 0.9) +
  coord_flip()

Question 3

Now imagine you work for an investor and want to see which industries generate the most revenue per employee. Create a chart that makes this information clear. Once again, the distribution per industry should be shown.

Funny! I guess I think like this investor.

inc %>%
  select(Industry, Revenue, Employees) %>%
  na.omit() %>%
  mutate(`Revenue Per Employee` = Revenue / Employees / 1000000) %>%
  group_by(Industry) %>%
  mutate(Median = median(`Revenue Per Employee`, na.rm = TRUE)) %>%
  ggplot(aes(x = reorder(Industry, Median), y = `Revenue Per Employee`)) +
  ylim(0, 1) +
  labs(title = "Revenue per Employee by Industry", 
       subtitle = "(revenue over $1MM not shown)",
       y = "Millions of Dollars",
       x = element_blank()) +
  theme(panel.background = element_blank(),
        panel.grid.major = element_line(color = "gray95"),
        panel.grid.major.x = element_line(color = "gray95", linetype="dashed")) +
  geom_boxplot(outlier.colour="gray", outlier.shape = 8, outlier.size = 0.9) +
  coord_flip()

Available at: https://github.com/mikeasilva/CUNY-SPS/blob/master/DATA608/hw1.rmd