survey_tib <- survey_tib %>% 
 filter((`Q3_In which country do you currently reside?`=='United States of America')&(`Q6_Select the title most similar to your current role (or most recent title if retired): - Selected Choice`=='Data Scientist'))
#write.csv(survey_tib,"C:/Users/newma/OneDrive/Desktop/MSDS Fall 2021/DATA 607 - Data Acquisition and Mgt/Project 3/surv.csv", row.names = FALSE)

surv_dataset <- survey_tib %>% select(starts_with(c("Q1_","Q2_","Q3_","Q7_","Q33")),-contains("OTHER_TEXT")) %>%
pivot_longer(starts_with("Q33"),names_to="Dataset",values_to="Dataset_type")

surv_dataset_tib<-surv_dataset %>% 
  count(surv_dataset$Dataset_type) %>%
  rename("DataSource" = "surv_dataset$Dataset_type","Count"="n")%>%
  arrange(desc(Count))

#write.csv(surv_dataset,"C:/Users/newma/OneDrive/Desktop/MSDS Fall 2021/DATA 607 - Data Acquisition and Mgt/Project 3/tibviz.csv", row.names = FALSE)

surv_dataset_tib<-surv_dataset_tib[-c(1),]
surv_dataset_tib2<-surv_dataset_tib %>% 
  mutate(proportion = round((surv_dataset_tib$Count /sum(surv_dataset_tib$Count))*100,2))

surv_dataset_tib2 %>% ggplot(aes(reorder(DataSource,proportion),proportion)) +
  geom_col(fill="#06272C")+coord_flip()+geom_text(aes(label=proportion),color="red") +labs(x="Data Sources")+theme_bw()

kbl(surv_dataset_tib2)%>%
  kable_styling() %>% kable_paper("hover", full_width = F)

#US count only
#Is there a correlation between salary and highest degree earned?
survey_ds_US<-filter(survey_ds,country=="United States of America")
#raw plot with all values
ggplot(survey_ds_US%>%count(salary,highest_degree),
  aes(salary,n,fill=highest_degree))+
  geom_bar(stat="identity")+
  ggtitle("Salary and Highest Degree in the US")+
  xlab("salary K ($)")+ylab("count")

mean(survey_ds_US$salary,na.rm=TRUE)

#removing null data and looking at data bettween 0-100k salary
survey_f1<-filter(survey_ds_US,salary<250)
survey_f1<-filter(survey_f1,highest_degree!="I prefer not to answer")
#further filter out non-degree inputs
survey_f2<-filter(survey_f1,highest_degree=="Master’s degree"|highest_degree=="Bachelor’s degree"|highest_degree=="Doctoral degree")
ggplot(survey_f2%>%count(salary,highest_degree),
  aes(salary,n,fill=highest_degree))+
  geom_bar(stat="identity")+
  ggtitle("Salary and Highest Degree in the US")+
  xlab("salary K ($)")+ylab("count")

mean(survey_f2$salary,na.rm=TRUE)

#summarizing with proportions for the US
subset1<-subset(survey_f2,select=c("highest_degree","salary"))
MS<-filter(subset1,highest_degree=="Master’s degree")
BA<-filter(subset1,highest_degree=="Bachelor’s degree")
PHD<-filter(subset1,highest_degree=="Doctoral degree")
MS_count<-count(MS,salary)
BA_count<-count(BA,salary)
PHD_count<-count(PHD,salary)
PHD_count<-PHD_count%>%add_row(salary=50,n=0,.before=5)
deg_freq1<-data.frame(BA_count$n,MS_count$n,PHD_count$n)
row_sum<-rowSums(deg_freq1)
deg_freq1<-data.frame(BA_count$salary,BA_count$n,MS_count$n,PHD_count$n,row_sum)
BA_pct<-round(deg_freq1$BA_count.n/deg_freq1$row_sum*100)
BA_pct<-data.frame(BA[1:length(BA_pct),1],BA_pct,BA_count$salary)
colnames(BA_pct)<-c('highest degree','percent','salary')
MS_pct<-round(deg_freq1$MS_count.n/deg_freq1$row_sum*100)
MS_pct<-data.frame(MS[1:length(MS_pct),1],MS_pct,BA_count$salary)
colnames(MS_pct)<-c('highest degree','percent','salary')
PHD_pct<-round(deg_freq1$PHD_count.n/deg_freq1$row_sum*100)
PHD_pct<-data.frame(PHD[1:length(PHD_pct),1],PHD_pct,BA_count$salary)
colnames(PHD_pct)<-c('highest degree','percent','salary')
combined<-rbind(BA_pct,MS_pct,PHD_pct)

combined<-combined%>%
  group_by(salary)%>%
  arrange(salary,desc(`highest degree`))%>%
  mutate(lab_ypos=cumsum(percent)-0.5*percent)
ggplot(combined,aes(x=salary,y=percent))+
  geom_col(aes(fill=`highest degree`),width=8)+
  xlab("salary K ($)")+ylab("percent %")+
  ggtitle("Degree and Salary in the US <$250K")+
  geom_text(aes(y=lab_ypos,label=percent,group=`highest degree`),color="white")

#For salary greater than 200k
survey_f3<-filter(survey_ds_US,salary>250)
survey_f3<-filter(survey_f3,highest_degree!="I prefer not to answer")
survey_f3<-filter(survey_f3,highest_degree=="Master’s degree"|highest_degree=="Bachelor’s degree"|highest_degree=="Doctoral degree")
ggplot(survey_f3%>%count(salary,highest_degree),
  aes(salary,n,fill=highest_degree))+
  geom_bar(stat="identity")+
  ggtitle("Salary and Highest Degree Globally >$250K")+
  xlab("salary K ($)")+ylab("count")

#summarizing with proportions for the US
subset3<-subset(survey_f3,select=c("highest_degree","salary"))
MS<-filter(subset3,highest_degree=="Master’s degree")
BA<-filter(subset3,highest_degree=="Bachelor’s degree")
PHD<-filter(subset3,highest_degree=="Doctoral degree")
MS_count<-count(MS,salary)
BA_count<-count(BA,salary)
PHD_count<-count(PHD,salary)
BA_count<-BA_count%>%add_row(salary=500,n=0)
deg_freq1<-data.frame(BA_count$n,MS_count$n,PHD_count$n)
row_sum<-rowSums(deg_freq1)
deg_freq1<-data.frame(BA_count$salary,BA_count$n,MS_count$n,PHD_count$n,row_sum)
BA_pct<-round(deg_freq1$BA_count.n/deg_freq1$row_sum*100)
BA_pct<-data.frame(BA[1:length(BA_pct),1],BA_pct,BA_count$salary)
colnames(BA_pct)<-c('highest degree','percent','salary')
MS_pct<-round(deg_freq1$MS_count.n/deg_freq1$row_sum*100)
MS_pct<-data.frame(MS[1:length(MS_pct),1],MS_pct,BA_count$salary)
colnames(MS_pct)<-c('highest degree','percent','salary')
PHD_pct<-round(deg_freq1$PHD_count.n/deg_freq1$row_sum*100)
PHD_pct<-data.frame(PHD[1:length(PHD_pct),1],PHD_pct,BA_count$salary)
colnames(PHD_pct)<-c('highest degree','percent','salary')
combined<-rbind(BA_pct,MS_pct,PHD_pct)
combined<-combined%>%
  group_by(salary)%>%
  arrange(salary,desc(`highest degree`))%>%
  mutate(lab_ypos=cumsum(percent)-0.5*percent)
ggplot(combined,aes(x=salary,y=percent))+
  geom_col(aes(fill=`highest degree`),width=50)+
  xlab("salary K ($)")+ylab("percent %")+
  ggtitle("Degree and Salary in the US <$250K")+
  geom_text(aes(y=lab_ypos,label=percent,group=`highest degree`),color="white")

surv_viz_tool<- survey_tib %>% select(starts_with(c("Q1_","Q2_","Q3_","Q7_","Q21")),-contains("OTHER_TEXT")) %>%
pivot_longer(starts_with("Q21"),names_to="ToolName",values_to="Tool")

surv_viz_tool_tib <- surv_viz_tool %>%
  count(surv_viz_tool$Tool) %>% rename("VisualTools" = "surv_viz_tool$Tool","Count"="n")%>%
  arrange(desc(Count))

surv_viz_tool_tib <- surv_viz_tool_tib[-c(1),]
surv_viz_tool_tib2<-surv_viz_tool_tib %>%
  mutate(proportion = round((Count /sum(Count))*100,2))%>% arrange(desc(proportion))
  
surv_viz_tool_tib2 %>% ggplot(aes(reorder(VisualTools,proportion),proportion)) +
  geom_col()+coord_flip()+geom_col(fill="#A7ADBE")+
  geom_text(aes(label=proportion),color="red") +labs(x="Visual Tools")+theme_bw()

kbl(surv_viz_tool_tib2) %>%
  kable_styling() %>% kable_paper("hover", full_width = F)

ide_tool<- survey_tib %>% 
  select(starts_with(c("Q1_","Q2_","Q3_","Q7_","Q13")),-contains("OTHER_TEXT")) %>%
pivot_longer(starts_with("Q13"),names_to="IDEName",values_to="IDEUsed")
#write.csv(surv_viz_tool,"C:/Users/newma/OneDrive/Desktop/MSDS Fall 2021/DATA 607 - Data Acquisition and Mgt/Project 3/survviz.csv", row.names = FALSE)

ide_tool_tib <- ide_tool %>%
  count(ide_tool$IDEUsed) %>% rename("IDE" = "ide_tool$IDEUsed","Count"="n")%>%
  arrange(desc(Count))

ide_tool_tib<-ide_tool_tib[-c(1),]
ide_tool_tib2<-ide_tool_tib %>%
  mutate(proportion = round((Count /sum(Count))*100,2))%>%arrange(desc(proportion))

ide_tool_tib2 %>% ggplot(aes(reorder(IDE,proportion),proportion)) +
  geom_col(fill="#18213A")+coord_flip()+
  geom_text(aes(label=proportion),color="red")+
  labs(x="IDE")+theme_bw()

kbl(ide_tool_tib2)%>%
  kable_styling() %>% kable_paper("hover", full_width = F)

lang_tool<- survey_tib %>% 
  select(starts_with(c("Q1_","Q2_","Q3_","Q7_","Q16")),-contains("OTHER_TEXT")) %>%
pivot_longer(starts_with("Q16"),names_to="LangName",values_to="LangUsed")

Lang_tool_tib <- lang_tool %>%
  count(lang_tool$LangUsed) %>% rename("Language" = "lang_tool$LangUsed","Count"="n")%>%
  arrange(desc(Count))

Lang_tool_tib<-Lang_tool_tib[-c(1),]
Lang_tool_tib2<-Lang_tool_tib %>%
  mutate(proportion = round((Count /sum(Count))*100,2))%>%arrange(desc(proportion))

Lang_tool_tib2 %>% ggplot(aes(reorder(Language,proportion),proportion)) +
  geom_col(fill='#633974')+coord_flip()+
  geom_text(aes(label=proportion),color="green")+ labs(x="Language")+theme_bw()

kbl(Lang_tool_tib2) %>%
  kable_styling() %>% kable_paper("hover", full_width = F)

# declare lists that will be used to preprocess data
x <- c(10000, 20000, 125000, 150000, 200000, 30000, 250000, 300000, 40000, 400000, 50000, 500000, 60000, 600000, 70000, 80000, 90000, 100000)
level <- c("0-10,000", "10-20,000", "100-125,000", "125-150,000", "150-200,000", "20-30,000", "200-250,000", "250-300,000", "30-40,000", "300-400,000", "40-50,000", "400-500,000", "50-60,000", "500,000+", "60-70,000", "70-80,000", "80-90,000", "90-100,000")


# select desired filters


a = survey_raw %>%
  filter(Q6=="Data Scientist")
a = a %>%
  filter(Q3 == "United States of America")
# Choose necessary variables
b <- a[,c(1:13,66:81,83,111:121,123,266:276)]
# remove irrelevant data
b <- b[b$Q9 != "I do not wish to disclose my approximate yearly compensation", ]  
b <- b[b$Q9 != "", ] 
# replace character data with numeric data
for (i in 1:length(level)) {
  b$Q9[b$Q9 == level[i]] <- x[i] 
}
b$Q9 <- as.numeric(b$Q9)

# change empty strings to NA for easier count
b$Q16_Part_1[b$Q16_Part_1 == ""] <- NA
b$Q16_Part_2[b$Q16_Part_2 == ""] <- NA
b$Q16_Part_3[b$Q16_Part_3 == ""] <- NA
b$Q16_Part_4[b$Q16_Part_4 == ""] <- NA
b$Q16_Part_5[b$Q16_Part_5 == ""] <- NA
b$Q16_Part_6[b$Q16_Part_6 == ""] <- NA
b$Q16_Part_7[b$Q16_Part_7 == ""] <- NA
b$Q16_Part_8[b$Q16_Part_8 == ""] <- NA
b$Q16_Part_9[b$Q16_Part_9 == ""] <- NA
b$Q16_Part_10[b$Q16_Part_10 == ""] <- NA
b$Q16_Part_11[b$Q16_Part_11 == ""] <- NA
b$Q16_Part_12[b$Q16_Part_12 == ""] <- NA
b$Q16_Part_13[b$Q16_Part_13 == ""] <- NA
b$Q16_Part_14[b$Q16_Part_14 == ""] <- NA
b$Q16_Part_15[b$Q16_Part_15 == ""] <- NA
b$Q16_Part_16[b$Q16_Part_16 == ""] <- NA
b$Q16_Part_18[b$Q16_Part_18 == ""] <- NA

# calculate count
c = 0
c <- b %>% rowwise() %>% mutate(Count=sum(!is.na(c_across(14:30))))
d <- count(c,as.factor(Count))

# scatterplot
ggplot(c, aes(x=Count, y=Q9)) +
  geom_point(position = position_jitter(width = .07, height = .07)) +
  scale_color_gradient(low='red', high='green') +
  ggtitle("Salary vs Number of Programming Languages used") +
  xlab("Programming Language by Count") +
  ylab("Salary") +
  stat_smooth(method = "lm", col = "red")

# linear regression to get the exact slope
lm(data=c,Q9~Count)

# Change empty string to NA
b$Q21_Part_1[b$Q21_Part_1 == ""] <- NA
b$Q21_Part_2[b$Q21_Part_2 == ""] <- NA
b$Q21_Part_3[b$Q21_Part_3 == ""] <- NA
b$Q21_Part_4[b$Q21_Part_4 == ""] <- NA
b$Q21_Part_5[b$Q21_Part_5 == ""] <- NA
b$Q21_Part_6[b$Q21_Part_6 == ""] <- NA
b$Q21_Part_7[b$Q21_Part_7 == ""] <- NA
b$Q21_Part_8[b$Q21_Part_8 == ""] <- NA
b$Q21_Part_9[b$Q21_Part_9 == ""] <- NA
b$Q21_Part_10[b$Q21_Part_10 == ""] <- NA
b$Q21_Part_11[b$Q21_Part_11 == ""] <- NA
b$Q21_Part_13[b$Q21_Part_13 == ""] <- NA
# Calculate count
c = 0
c <- b %>% rowwise() %>% mutate(Count=sum(!is.na(c_across(31:42))))
d <- c %>% count(Count)
# scatterplot
ggplot(c, aes(x=Count, y=Q9)) +
  geom_point(position = position_jitter(width = .07, height = .07)) +
  ggtitle("Salary vs Number of Visualization Tools used") + 
  xlab("Visualization Tools") +
  ylab("Salary") +
  stat_smooth(method = "lm", col = "red")

# linear regression to get the exact slope
lm(data=c,Q9~Count)

# Change empty string to NA
b$Q33_Part_1[b$Q33_Part_1 == ""] <- NA
b$Q33_Part_2[b$Q33_Part_2 == ""] <- NA
b$Q33_Part_3[b$Q33_Part_3 == ""] <- NA
b$Q33_Part_4[b$Q33_Part_4 == ""] <- NA
b$Q33_Part_5[b$Q33_Part_5 == ""] <- NA
b$Q33_Part_6[b$Q33_Part_6 == ""] <- NA
b$Q33_Part_7[b$Q33_Part_7 == ""] <- NA
b$Q33_Part_8[b$Q33_Part_8 == ""] <- NA
b$Q33_Part_9[b$Q33_Part_9 == ""] <- NA
b$Q33_Part_10[b$Q33_Part_10 == ""] <- NA
b$Q33_Part_11[b$Q33_Part_11 == ""] <- NA
# Calculate count
c = 0
c <- b %>% rowwise() %>% mutate(Count=sum(!is.na(c_across(43:53))))
# scatterplot
ggplot(c, aes(x=Count, y=Q9)) +
  geom_point(position = position_jitter(width = .07, height = .07)) +
  ggtitle("Salary vs Number of Data Acquistion Methods used") +
  xlab("Data Acquistion Methods by Count") +
  ylab("Salary") +
  stat_smooth(method = "lm", col = "red")

# linear regression to get the exact slope
lm(data=c,Q9~Count) 

age_df <- country_data %>% rename(value = age) %>% select(region, value)

country_choropleth(age_df,
                 title      = "Data Scientist Age by Country",
                 legend     = "Average Age",
                 num_colors = 9) + scale_fill_brewer(palette = "BuGn")

income_df <- country_data %>% rename(value = income) %>% select(region, value)

country_choropleth(income_df,
                 title      = "Data Scientist Income by Country",
                 legend     = "Average Income",
                 num_colors = 9) + scale_fill_brewer(palette = "YlOrRd")

experience_df <- country_data %>% rename(value = experience) %>% select(region, value)

country_choropleth(experience_df,
                 title      = "Data Scientist Years Experience by Country",
                 legend     = "Average Years Experience",
                 num_colors = 9) + scale_fill_brewer(palette = "Greys")

lang_df <- lang_country_data %>% rename(value = avg_lang) %>% select(region, value)

country_choropleth(experience_df,
                 title      = "Data Scientist Number of Programming Languages by Country",
                 legend     = "Average Number of Languages",
                 num_colors = 9) + scale_fill_brewer(palette = "Blues")