packages <- c("tidyverse", "infer", "fst", "modelsummary", "effects", "survey", "performance", "flextable", "broom", "scales", "effects") 

new_packages <- packages[!(packages %in% installed.packages()[,"Package"])]
if(length(new_packages)) install.packages(new_packages)

lapply(packages, library, character.only = TRUE)
## ── Attaching core tidyverse packages ──────────────────────── tidyverse 2.0.0 ──
## ✔ dplyr     1.1.4     ✔ readr     2.1.5
## ✔ forcats   1.0.0     ✔ stringr   1.5.1
## ✔ ggplot2   3.4.4     ✔ tibble    3.2.1
## ✔ lubridate 1.9.3     ✔ tidyr     1.3.0
## ✔ purrr     1.0.2     
## ── Conflicts ────────────────────────────────────────── tidyverse_conflicts() ──
## ✖ dplyr::filter() masks stats::filter()
## ✖ dplyr::lag()    masks stats::lag()
## ℹ Use the conflicted package (<http://conflicted.r-lib.org/>) to force all conflicts to become errors
## Loading required package: carData
## 
## lattice theme set by effectsTheme()
## See ?effectsTheme for details.
## 
## Loading required package: grid
## 
## Loading required package: Matrix
## 
## 
## Attaching package: 'Matrix'
## 
## 
## The following objects are masked from 'package:tidyr':
## 
##     expand, pack, unpack
## 
## 
## Loading required package: survival
## 
## 
## Attaching package: 'survey'
## 
## 
## The following object is masked from 'package:graphics':
## 
##     dotchart
## 
## 
## 
## Attaching package: 'flextable'
## 
## 
## The following object is masked from 'package:purrr':
## 
##     compose
## 
## 
## 
## Attaching package: 'scales'
## 
## 
## The following object is masked from 'package:purrr':
## 
##     discard
## 
## 
## The following object is masked from 'package:readr':
## 
##     col_factor
## [[1]]
##  [1] "lubridate" "forcats"   "stringr"   "dplyr"     "purrr"     "readr"    
##  [7] "tidyr"     "tibble"    "ggplot2"   "tidyverse" "stats"     "graphics" 
## [13] "grDevices" "utils"     "datasets"  "methods"   "base"     
## 
## [[2]]
##  [1] "infer"     "lubridate" "forcats"   "stringr"   "dplyr"     "purrr"    
##  [7] "readr"     "tidyr"     "tibble"    "ggplot2"   "tidyverse" "stats"    
## [13] "graphics"  "grDevices" "utils"     "datasets"  "methods"   "base"     
## 
## [[3]]
##  [1] "fst"       "infer"     "lubridate" "forcats"   "stringr"   "dplyr"    
##  [7] "purrr"     "readr"     "tidyr"     "tibble"    "ggplot2"   "tidyverse"
## [13] "stats"     "graphics"  "grDevices" "utils"     "datasets"  "methods"  
## [19] "base"     
## 
## [[4]]
##  [1] "modelsummary" "fst"          "infer"        "lubridate"    "forcats"     
##  [6] "stringr"      "dplyr"        "purrr"        "readr"        "tidyr"       
## [11] "tibble"       "ggplot2"      "tidyverse"    "stats"        "graphics"    
## [16] "grDevices"    "utils"        "datasets"     "methods"      "base"        
## 
## [[5]]
##  [1] "effects"      "carData"      "modelsummary" "fst"          "infer"       
##  [6] "lubridate"    "forcats"      "stringr"      "dplyr"        "purrr"       
## [11] "readr"        "tidyr"        "tibble"       "ggplot2"      "tidyverse"   
## [16] "stats"        "graphics"     "grDevices"    "utils"        "datasets"    
## [21] "methods"      "base"        
## 
## [[6]]
##  [1] "survey"       "survival"     "Matrix"       "grid"         "effects"     
##  [6] "carData"      "modelsummary" "fst"          "infer"        "lubridate"   
## [11] "forcats"      "stringr"      "dplyr"        "purrr"        "readr"       
## [16] "tidyr"        "tibble"       "ggplot2"      "tidyverse"    "stats"       
## [21] "graphics"     "grDevices"    "utils"        "datasets"     "methods"     
## [26] "base"        
## 
## [[7]]
##  [1] "performance"  "survey"       "survival"     "Matrix"       "grid"        
##  [6] "effects"      "carData"      "modelsummary" "fst"          "infer"       
## [11] "lubridate"    "forcats"      "stringr"      "dplyr"        "purrr"       
## [16] "readr"        "tidyr"        "tibble"       "ggplot2"      "tidyverse"   
## [21] "stats"        "graphics"     "grDevices"    "utils"        "datasets"    
## [26] "methods"      "base"        
## 
## [[8]]
##  [1] "flextable"    "performance"  "survey"       "survival"     "Matrix"      
##  [6] "grid"         "effects"      "carData"      "modelsummary" "fst"         
## [11] "infer"        "lubridate"    "forcats"      "stringr"      "dplyr"       
## [16] "purrr"        "readr"        "tidyr"        "tibble"       "ggplot2"     
## [21] "tidyverse"    "stats"        "graphics"     "grDevices"    "utils"       
## [26] "datasets"     "methods"      "base"        
## 
## [[9]]
##  [1] "broom"        "flextable"    "performance"  "survey"       "survival"    
##  [6] "Matrix"       "grid"         "effects"      "carData"      "modelsummary"
## [11] "fst"          "infer"        "lubridate"    "forcats"      "stringr"     
## [16] "dplyr"        "purrr"        "readr"        "tidyr"        "tibble"      
## [21] "ggplot2"      "tidyverse"    "stats"        "graphics"     "grDevices"   
## [26] "utils"        "datasets"     "methods"      "base"        
## 
## [[10]]
##  [1] "scales"       "broom"        "flextable"    "performance"  "survey"      
##  [6] "survival"     "Matrix"       "grid"         "effects"      "carData"     
## [11] "modelsummary" "fst"          "infer"        "lubridate"    "forcats"     
## [16] "stringr"      "dplyr"        "purrr"        "readr"        "tidyr"       
## [21] "tibble"       "ggplot2"      "tidyverse"    "stats"        "graphics"    
## [26] "grDevices"    "utils"        "datasets"     "methods"      "base"        
## 
## [[11]]
##  [1] "scales"       "broom"        "flextable"    "performance"  "survey"      
##  [6] "survival"     "Matrix"       "grid"         "effects"      "carData"     
## [11] "modelsummary" "fst"          "infer"        "lubridate"    "forcats"     
## [16] "stringr"      "dplyr"        "purrr"        "readr"        "tidyr"       
## [21] "tibble"       "ggplot2"      "tidyverse"    "stats"        "graphics"    
## [26] "grDevices"    "utils"        "datasets"     "methods"      "base"

##1. provide the visual generated by the infer package that includes the null distribution with confidence intervals along with your calculated test statistic

norway_data <- read.fst("norway_data.fst")

i found the code in homework 3- ANSWER KEY an tutorial 6

norway_task1 <- norway_data %>%
  mutate(
    ID = case_when(
      happy %in% 0:4 ~ "unhappy",                     
      happy %in% 6:10 ~ "happy",                  
      happy %in% c(77, 88) ~ NA_character_,    
      TRUE ~ NA_character_                           
    ),
    educ.ba = case_when(
      essround < 5 & edulvla == 5 ~ "BA or more",
      essround >= 5 & edulvlb > 600 ~ "BA or more",
      TRUE ~ "No BA"
    ),
    edulvla = ifelse(edulvla %in% c(77, 88, 99), NA_integer_, edulvla),
    edulvlb = ifelse(edulvlb %in% c(5555, 7777, 8888), NA_integer_, edulvlb),
    educ.ba = factor(educ.ba, levels = c("No BA", "BA or more"))  
  ) %>%
  filter(!is.na(ID), !is.na(educ.ba))  
mytab <- table(norway_task1$ID, norway_task1$educ.ba)
mytab
##          
##           No BA BA or more
##   happy    9425       5384
##   unhappy   359        145
rsums <- rowSums(mytab)
csums <- colSums(mytab)
N <- sum(mytab)
expected <- outer(rsums, csums) / N
expected
##             No BA BA or more
## happy   9461.9771  5347.0229
## unhappy  322.0229   181.9771
alpha <- 0.05
df <- (nrow(mytab) - 1) * (ncol(mytab) - 1)
critical_value <- qchisq(1 - alpha, df)

observed <- mytab
chi_squared <- sum((observed - expected)^2 / expected)

p_value <- pchisq(chi_squared, df, lower.tail = FALSE)

cat("Critical Value:", critical_value, "\n")
## Critical Value: 3.841459
cat("Chi-Squared Statistic:", chi_squared, "\n")
## Chi-Squared Statistic: 12.15985
cat("P-Value:", p_value, "\n")
## P-Value: 0.0004882927
edu_stat <- norway_task1 %>%
  specify(explanatory = ID, 
          response = educ.ba, 
          success = "BA or more") %>% 
 hypothesize(null = "independence") %>%
  calculate(stat = "chisq") 
## Warning: The statistic is based on a difference or ratio; by default, for
## difference-based statistics, the explanatory variable is subtracted in the
## order "happy" - "unhappy", or divided in the order "happy" / "unhappy" for
## ratio-based statistics. To specify this order yourself, supply `order =
## c("happy", "unhappy")` to the calculate() function.
print(edu_stat$stat)
## X-squared 
##  11.83323
null_edu <- norway_task1 %>%
  specify(explanatory = ID,
          response = educ.ba,
          success = "BA or more") %>% 
  hypothesize(null = "independence") %>%
  generate(reps = 1000, type = "permute") %>% 
  calculate(stat = "chisq")
## Warning: The statistic is based on a difference or ratio; by default, for
## difference-based statistics, the explanatory variable is subtracted in the
## order "happy" - "unhappy", or divided in the order "happy" / "unhappy" for
## ratio-based statistics. To specify this order yourself, supply `order =
## c("happy", "unhappy")` to the calculate() function.
null_edu
## Response: educ.ba (factor)
## Explanatory: ID (factor)
## Null Hypothesis: independence
## # A tibble: 1,000 × 2
##    replicate    stat
##        <int>   <dbl>
##  1         1 0.503  
##  2         2 1.17   
##  3         3 1.62   
##  4         4 4.12   
##  5         5 1.18   
##  6         6 0.00202
##  7         7 1.38   
##  8         8 2.43   
##  9         9 0.0194 
## 10        10 0.00202
## # ℹ 990 more rows
p_val <- null_edu %>% 
  get_pvalue(obs_stat = edu_stat, direction = "greater") 

p_val
## # A tibble: 1 × 1
##   p_value
##     <dbl>
## 1   0.001
conf_int <- null_edu%>%
  get_confidence_interval(level = 0.95, type = "percentile")

null_edu %>%
  visualize(data, bins = 15, method = "simulation", dens_color = "black") +
  shade_p_value(obs_stat = edu_stat, direction = "greater") +
  shade_confidence_interval(endpoints = conf_int)

edu_stat <- 1000

conf_int <- null_edu %>%
  get_confidence_interval(level = 0.95, type = "percentile")

null_edu %>%
  visualize(bins = 10, method = "simulation", dens_color = "black") +
  geom_vline(aes(xintercept = edu_stat), color = "red", linetype = "dashed", size = 1) +
  shade_p_value(obs_stat = edu_stat, direction = "greater") +
  shade_confidence_interval(endpoints = conf_int) +
  coord_cartesian(xlim = c(0, max(null_edu$stat) * 1.1)) + 
  labs(title = "Null Distribution for Norway",
       x = "Chi-Square",
       y = "Count")
## Warning: Using `size` aesthetic for lines was deprecated in ggplot2 3.4.0.
## ℹ Please use `linewidth` instead.
## This warning is displayed once every 8 hours.
## Call `lifecycle::last_lifecycle_warnings()` to see where this warning was
## generated.
## Warning in min(diff(unique_loc)): no non-missing arguments to min; returning
## Inf

In the visualization of the null distibution the red line indicates evidence under the hypothesis testing to reject the null.

Critical Value (3.841459):

Chi-Squared Statistic (12.15985):

The Chi squared statistic value is greater than the critical value which indicates a deviation from what would be expected is there was no association between the variables. This also indicates that there is some evidence to reject the null hypothesis.

P-Value (0.0004882927)

This P value would provide evidence to reject the null hypothesis. This value is smaller than the significance level of (α = 0.05) which provides evidence to reject the null and support the alternative hypothesis