library(readr)
library(dplyr)
## 
## Attaching package: 'dplyr'
## The following objects are masked from 'package:stats':
## 
##     filter, lag
## The following objects are masked from 'package:base':
## 
##     intersect, setdiff, setequal, union
library(ggplot2)
mydata <- read_delim("C:\\Users\\abhir\\OneDrive\\Documents\\luddy\\H510-Stats,R\\adult.csv", delim = ",")
## Rows: 32561 Columns: 15
## ── Column specification ────────────────────────────────────────────────────────
## Delimiter: ","
## chr (9): Work-Class, Education, Marital Status, Occupation, Relationship, Ra...
## dbl (6): Age, Fnl_Wgt, Education_Level, Capital_Gain, Capital_Loss, hrs_week
## 
## ℹ Use `spec()` to retrieve the full column specification for this data.
## ℹ Specify the column types or set `show_col_types = FALSE` to quiet this message.
mydata
## # A tibble: 32,561 × 15
##      Age `Work-Class`        Fnl_Wgt Education  Education_Level `Marital Status`
##    <dbl> <chr>                 <dbl> <chr>                <dbl> <chr>           
##  1    39 " State-gov"          77516 " Bachelo…              13 " Never-married"
##  2    50 " Self-emp-not-inc"   83311 " Bachelo…              13 " Married-civ-s…
##  3    38 " Private"           215646 " HS-grad"               9 " Divorced"     
##  4    53 " Private"           234721 " 11th"                  7 " Married-civ-s…
##  5    28 " Private"           338409 " Bachelo…              13 " Married-civ-s…
##  6    37 " Private"           284582 " Masters"              14 " Married-civ-s…
##  7    49 " Private"           160187 " 9th"                   5 " Married-spous…
##  8    52 " Self-emp-not-inc"  209642 " HS-grad"               9 " Married-civ-s…
##  9    31 " Private"            45781 " Masters"              14 " Never-married"
## 10    42 " Private"           159449 " Bachelo…              13 " Married-civ-s…
## # ℹ 32,551 more rows
## # ℹ 9 more variables: Occupation <chr>, Relationship <chr>, Race <chr>,
## #   Sex <chr>, Capital_Gain <dbl>, Capital_Loss <dbl>, hrs_week <dbl>,
## #   Country <chr>, Annual_pay <chr>
mydata$Sex <- as.character(mydata$Sex)

# Check for leading and trailing spaces in the "Sex" column
leading_spaces <- sum(startsWith(mydata$Sex, " "))
trailing_spaces <- sum(endsWith(mydata$Sex, " "))

# Print the counts of leading and trailing spaces
print(leading_spaces)
## [1] 32561
print(trailing_spaces)
## [1] 0
# Remove leading and trailing spaces from the "Sex" column
mydata$Sex <- trimws(mydata$Sex)
missing_sex <- sum(is.na(mydata$Sex))

# Print the number of missing values
print(missing_sex)
## [1] 0
# Display the structure of the dataset
str(mydata)
## spc_tbl_ [32,561 × 15] (S3: spec_tbl_df/tbl_df/tbl/data.frame)
##  $ Age            : num [1:32561] 39 50 38 53 28 37 49 52 31 42 ...
##  $ Work-Class     : chr [1:32561] " State-gov" " Self-emp-not-inc" " Private" " Private" ...
##  $ Fnl_Wgt        : num [1:32561] 77516 83311 215646 234721 338409 ...
##  $ Education      : chr [1:32561] " Bachelors" " Bachelors" " HS-grad" " 11th" ...
##  $ Education_Level: num [1:32561] 13 13 9 7 13 14 5 9 14 13 ...
##  $ Marital Status : chr [1:32561] " Never-married" " Married-civ-spouse" " Divorced" " Married-civ-spouse" ...
##  $ Occupation     : chr [1:32561] " Adm-clerical" " Exec-managerial" " Handlers-cleaners" " Handlers-cleaners" ...
##  $ Relationship   : chr [1:32561] " Not-in-family" " Husband" " Not-in-family" " Husband" ...
##  $ Race           : chr [1:32561] " White" " White" " White" " Black" ...
##  $ Sex            : chr [1:32561] "Male" "Male" "Male" "Male" ...
##  $ Capital_Gain   : num [1:32561] 2174 0 0 0 0 ...
##  $ Capital_Loss   : num [1:32561] 0 0 0 0 0 0 0 0 0 0 ...
##  $ hrs_week       : num [1:32561] 40 13 40 40 40 40 16 45 50 40 ...
##  $ Country        : chr [1:32561] " United-States" " United-States" " United-States" " United-States" ...
##  $ Annual_pay     : chr [1:32561] " <=50K" " <=50K" " <=50K" " <=50K" ...
##  - attr(*, "spec")=
##   .. cols(
##   ..   Age = col_double(),
##   ..   `Work-Class` = col_character(),
##   ..   Fnl_Wgt = col_double(),
##   ..   Education = col_character(),
##   ..   Education_Level = col_double(),
##   ..   `Marital Status` = col_character(),
##   ..   Occupation = col_character(),
##   ..   Relationship = col_character(),
##   ..   Race = col_character(),
##   ..   Sex = col_character(),
##   ..   Capital_Gain = col_double(),
##   ..   Capital_Loss = col_double(),
##   ..   hrs_week = col_double(),
##   ..   Country = col_character(),
##   ..   Annual_pay = col_character()
##   .. )
##  - attr(*, "problems")=<externalptr>
# Summary of the Sex variable
table(mydata$Sex)
## 
## Female   Male 
##  10771  21790
# Extract the number of observations in each group
n_Male <- sum(mydata$Sex == "Male")
n_Female <- sum(mydata$Sex == "Female")

cat("Number of observations in Male group:", n_Male, "\n")
## Number of observations in Male group: 21790
cat("Number of observations in Female group:", n_Female, "\n")
## Number of observations in Female group: 10771
unique_sex <- unique(mydata$Sex)
unique_sex
## [1] "Male"   "Female"
# Filter data for Male and Female observations
Male_hrs <- mydata[mydata$Sex == "Male", "hrs_week"]
Female_hrs <- mydata[mydata$Sex == "Female", "hrs_week"]

# Check the number of observations for Male and Female
n_Male <- sum(mydata$Sex == "Male")
n_Female <- sum(mydata$Sex == "Female")

cat("Number of Male observations:", n_Male, "\n")
## Number of Male observations: 21790
cat("Number of Female observations:", n_Female, "\n")
## Number of Female observations: 10771

Null Hypothesis H0 : There is no significant difference in the hours in a week worked by Male and Female

Alternative Hypothesis H1: There is a significant difference between the hours in a worked by Male and Female

Alpha Level (0.05): The alpha level, typically set at 0.05, represents the significance threshold used to determine whether to reject the null hypothesis. A significance level of 0.05 means that there is a 5% chance of incorrectly rejecting the null hypothesis when it is true. This level of significance is widely accepted in many scientific fields and provides a balance between sensitivity (detecting true effects) and specificity (minimizing false positives).

Power Level (0.80): Power is the probability of correctly rejecting a false null hypothesis (i.e., the probability of detecting a true effect when it exists). A power level of 0.80 indicates that the study has an 80% chance of detecting a true effect if one actually exists. This level of power is commonly considered acceptable in many fields, as it provides a reasonable balance between the risk of Type I and Type II errors. A higher power level would require a larger sample size, which may not always be feasible due to practical constraints.

Minimum Effect Size (0.3): The minimum effect size represents the smallest meaningful difference or effect that the study aims to detect. A minimum effect size of 0.3 suggests that the study is designed to detect relatively moderate effects. Choosing an effect size of 0.3 indicates that detecting effects that are neither too small to be practically significant nor too large to be easily detectable. This value is often based on prior research, clinical relevance, or expert judgment.

Male_hrs <- mydata[mydata$Sex == "Male", "hrs_week"]
Female_hrs <- mydata[mydata$Sex == "Female", "hrs_week"]

# Perform two-sample t-test
t_test_result <- t.test(Male_hrs, Female_hrs)

# Neyman-Pearson test
alpha <- 0.05
if (abs(t_test_result$statistic) > qt(1 - alpha/2, df = t_test_result$parameter)) {
  cat("Reject the null hypothesis at the", alpha, "level of significance.\n")
} else {
  cat("Fail to reject the null hypothesis at the", alpha, "level of significance.\n")
}
## Reject the null hypothesis at the 0.05 level of significance.

The result indicates that you reject the null hypothesis at the 0.05 level of significance, suggesting that there is a significant difference in the average hours per week between males and females.

# Perform two-sample t-test
t_test_result <- t.test(Male_hrs, Female_hrs)

# Output the p-value
cat("P-value:", t_test_result$p.value, "\n")
## P-value: 0

The p-value of 0 suggests that there is a highly significant difference in the average hours per week between males and females. This indicates strong evidence against the null hypothesis, indicating that the average hours per week for males and females are different

# Convert to vectors
Male_hrs <- unlist(Male_hrs)
Female_hrs <- unlist(Female_hrs)

# Perform the variance test
fisher_result <- var.test(Male_hrs, Female_hrs)

# Interpret the p-value
cat("Fisher's style test p-value:", fisher_result$p.value, "\n")
## Fisher's style test p-value: 0.00204127

the p-value is approximately 0.00204127. This indicates that there is a significant difference in the variances between the two groups (male and female)

library(ggplot2)

# Subset data for California and New York
Male_data <- mydata[mydata$Sex == "Male", ]
Female_data <- mydata[mydata$Sex == "Female", ]

# Create a boxplot
ggplot(mapping = aes(x = factor(Sex), y = hrs_week, fill = Sex)) +
  geom_boxplot(data = rbind(Male_data, Female_data), width = 0.6) +
  labs(title = "Comparison of working hours between Male and Female",
       x = "Sex", y = "hours_week") +
  theme_minimal() +
  theme(axis.text.x = element_text(angle = 45, hjust = 1))

The visualization show that the hours per week worked by Male’s is significantly higher than Female ,the hours worked by Female’s is between 25 to 40 where as Males is 35 to 50.

mean_female_hrs <- mean(Female_hrs)

# Print the mean
print(mean_female_hrs)
## [1] 36.41036

#HYPOTHESIS 2 Null Hypothesis h0: The hours per week worked by females is less than 36.41 Alternate Hypothesis h1: The hours per week worked by females is greatert than 36.41

Alpha Level (0.05): The alpha level, typically set at 0.05, represents the significance threshold used to determine whether to reject the null hypothesis. A significance level of 0.05 means that there is a 5% chance of incorrectly rejecting the null hypothesis when it is true. This level of significance is widely accepted in many scientific fields and provides a balance between sensitivity (detecting true effects) and specificity (minimizing false positives).

Power Level (0.80): Power is the probability of correctly rejecting a false null hypothesis (i.e., the probability of detecting a true effect when it exists). A power level of 0.80 indicates that the study has an 80% chance of detecting a true effect if one actually exists. This level of power is commonly considered acceptable in many fields, as it provides a reasonable balance between the risk of Type I and Type II errors. A higher power level would require a larger sample size, which may not always be feasible due to practical constraints.

Minimum Effect Size (0.3): The minimum effect size represents the smallest meaningful difference or effect that the study aims to detect. A minimum effect size of 0.3 suggests that the study is designed to detect relatively moderate effects. Choosing an effect size of 0.3 indicates that the effects that are neither too small to be practically significant nor too large to be easily detectable. This value is often based on prior research, clinical relevance, or expert judgment.

mean_female_hrs=36.41
m=mean_female_hrs

Female_hrs_week <- mydata[mydata$Sex == "Female", "hrs_week"]

# Neyman-Pearson test using t.test
alpha <- 0.05
t_test_result <- t.test(Female_hrs_week,mu=m)

# Interpret the results
if (abs(t_test_result$statistic) > qt(1 - alpha/2, df = t_test_result$parameter)) {
  cat("Reject the null hypothesis (t.test for Hours per week) is less than 36.41", alpha, "level of significance.\n")
} else {
  cat("Fail to reject the null hypothesis (t.test for Hours per week) is more than 36.41", alpha, "level of significance.\n")
}
## Fail to reject the null hypothesis (t.test for Hours per week) is more than 36.41 0.05 level of significance.

The test statistic represents the result of a statistical test, often indicating the degree of difference or relationship between groups or variables in a dataset. In this case, the test statistic value of 21443.89 is specific to the test you conducted

cat("P-value (t.test for hrs_week):", t_test_result$p.value, "\n")
## P-value (t.test for hrs_week): 0.9974681

The p-value of approximately 0.9974681 obtained from the t-test for hrs_week suggests that there is insufficient evidence to reject the null hypothesis.

library(ggplot2)

# Creating a histogram for the Hourly Wage distribution in Texas
ggplot(mydata[mydata$Sex == "Female", ], aes(x = `hrs_week`)) +
  geom_histogram(binwidth = 2, fill = "red", color = "black", alpha = 0.7) +
  labs(title = "Hours per week distribution for Female",
       x = "Hours per week", y = "Frequency") +
  theme_minimal()

The above plot indicates that more than 4500 Female’s work for more than 35hours per week.