Engineering Psyc
2025-04-13
#import the data
library(readr)
data <- read_csv("/Users/jyxu/Downloads/AI Animatic Storyboard - Data Collection (Group 4) - SUBMIT.csv")## New names:
## Rows: 6 Columns: 27
## ── Column specification
## ──────────────────────────────────────────────────────── Delimiter: "," chr
## (8): Date, Start Time, End Time, Factor 1: Prompt Engineering, Factor 2... dbl
## (19): Participant ID, Visual Clarity (0/1), Unnatural Discrepancy (0/1),...
## ℹ Use `spec()` to retrieve the full column specification for this data. ℹ
## Specify the column types or set `show_col_types = FALSE` to quiet this message.
## • `NASA Aggregate` -> `NASA Aggregate...25`
## • `NASA Aggregate` -> `NASA Aggregate...26`#clean the data
# Clean the data
data_clean <- data |>
select(-matches("\\(0-20\\)")) |>
select(-`NASA Aggregate...25`) |>
rename(`NASA Aggregate` = `NASA Aggregate...26`,
`Time to Completion` = `Time to Completion (mins)`,
`Visual Clarity` = `Visual Clarity (0/1)`,
`Unnatural Discrepancy` = `Unnatural Discrepancy (0/1)`,
`NASA TLX - Mental Demand` = `NASA TLX - Mental Demand (10)`,
`NASA TLX - Physical Demand` = `NASA TLX - Physical Demand (10)`,
`NASA TLX - Temporal Demand` = `NASA TLX - Temporal Demand (10)`,
`NASA TLX - Performance` = `NASA TLX - Performance (10)`,
`NASA TLX - Effort` = `NASA TLX - Effort (10)`,
`NASA TLX - Frustration` = `NASA TLX - Frustration (10)`)
# Clean up the time to complete column
data_clean <- data_clean |>
mutate(`Time to Completion` = as.numeric(gsub(" mins?| MINS?", "", `Time to Completion`, ignore.case = TRUE)))
#mutate to be numeric
# Clean up and convert types directly in-place
data_clean <- data_clean |>
mutate(
`Time to Completion` = as.numeric(`Time to Completion`),
`Visual Clarity` = as.numeric(`Visual Clarity`),
`Unnatural Discrepancy` = as.numeric(`Unnatural Discrepancy`),
`Number of Revisions` = as.numeric(`Number of Revisions`),
`NASA TLX - Mental Demand` = as.numeric(`NASA TLX - Mental Demand`),
`NASA TLX - Physical Demand` = as.numeric(`NASA TLX - Physical Demand`),
`NASA TLX - Temporal Demand` = as.numeric(`NASA TLX - Temporal Demand`),
`NASA TLX - Performance` = as.numeric(`NASA TLX - Performance`),
`NASA TLX - Effort` = as.numeric(`NASA TLX - Effort`),
`NASA TLX - Frustration` = as.numeric(`NASA TLX - Frustration`),
`NASA Aggregate` = as.numeric(`NASA Aggregate`)
)##Descriptive stats & Report
descriptive_stats <- data_clean |>
summarise(
MentalDemand.Mean = mean(`NASA TLX - Mental Demand`, na.rm = TRUE),
MentalDemand.SD = sd(`NASA TLX - Mental Demand`, na.rm = TRUE),
PhysicalDemand.Mean = mean(`NASA TLX - Physical Demand`, na.rm = TRUE),
PhysicalDemand.SD = sd(`NASA TLX - Physical Demand`, na.rm = TRUE),
TemporalDemand.Mean = mean(`NASA TLX - Temporal Demand`, na.rm = TRUE),
TemporalDemand.SD = sd(`NASA TLX - Temporal Demand`, na.rm = TRUE),
Performance.Mean = mean(`NASA TLX - Performance`, na.rm = TRUE),
Performance.SD = sd(`NASA TLX - Performance`, na.rm = TRUE),
Effort.Mean = mean(`NASA TLX - Effort`, na.rm = TRUE),
Effort.SD = sd(`NASA TLX - Effort`, na.rm = TRUE),
Frustration.Mean = mean(`NASA TLX - Frustration`, na.rm = TRUE),
Frustration.SD = sd(`NASA TLX - Frustration`, na.rm = TRUE),
NASAaggregate.Mean = mean(`NASA Aggregate`, na.rm = TRUE),
NASAaggregate.SD = sd(`NASA Aggregate`, na.rm = TRUE),
CompletionTime.Mean = mean(`Time to Completion`, na.rm = TRUE),
CompletionTime.SD = sd(`Time to Completion`, na.rm = TRUE),
VisualClarity.Mean = mean(`Visual Clarity`, na.rm = TRUE),
VisualClarity.SD = sd(`Visual Clarity`, na.rm = TRUE),
UnnaturalDiscrepancy.Mean = mean(`Unnatural Discrepancy`, na.rm = TRUE),
UnnaturalDiscrepancy.SD = sd(`Unnatural Discrepancy`, na.rm = TRUE),
Revisions.Mean = mean(`Number of Revisions`, na.rm = TRUE),
Revisions.SD = sd(`Number of Revisions`, na.rm = TRUE)
)
# View result
print(descriptive_stats)## # A tibble: 1 × 22
## MentalDemand.Mean MentalDemand.SD PhysicalDemand.Mean PhysicalDemand.SD
## <dbl> <dbl> <dbl> <dbl>
## 1 5 2 1.33 0.516
## # ℹ 18 more variables: TemporalDemand.Mean <dbl>, TemporalDemand.SD <dbl>,
## # Performance.Mean <dbl>, Performance.SD <dbl>, Effort.Mean <dbl>,
## # Effort.SD <dbl>, Frustration.Mean <dbl>, Frustration.SD <dbl>,
## # NASAaggregate.Mean <dbl>, NASAaggregate.SD <dbl>,
## # CompletionTime.Mean <dbl>, CompletionTime.SD <dbl>,
## # VisualClarity.Mean <dbl>, VisualClarity.SD <dbl>,
## # UnnaturalDiscrepancy.Mean <dbl>, UnnaturalDiscrepancy.SD <dbl>, …report(descriptive_stats)## The data contains 1 observations of the following 22 variables:
##
## - MentalDemand.Mean: n = 1, Mean = 5.00, SD = , Median = 5.00, MAD = 0.00,
## range: [5, 5], Skewness = , Kurtosis = , 0 missing
## - MentalDemand.SD: n = 1, Mean = 2.00, SD = , Median = 2.00, MAD = 0.00, range:
## [2, 2], Skewness = , Kurtosis = , 0 missing
## - PhysicalDemand.Mean: n = 1, Mean = 1.33, SD = , Median = 1.33, MAD = 0.00,
## range: [1.33, 1.33], Skewness = , Kurtosis = , 0 missing
## - PhysicalDemand.SD: n = 1, Mean = 0.52, SD = , Median = 0.52, MAD = 0.00,
## range: [0.52, 0.52], Skewness = , Kurtosis = , 0 missing
## - TemporalDemand.Mean: n = 1, Mean = 6.50, SD = , Median = 6.50, MAD = 0.00,
## range: [6.50, 6.50], Skewness = , Kurtosis = , 0 missing
## - TemporalDemand.SD: n = 1, Mean = 1.38, SD = , Median = 1.38, MAD = 0.00,
## range: [1.38, 1.38], Skewness = , Kurtosis = , 0 missing
## - Performance.Mean: n = 1, Mean = 7.50, SD = , Median = 7.50, MAD = 0.00,
## range: [7.50, 7.50], Skewness = , Kurtosis = , 0 missing
## - Performance.SD: n = 1, Mean = 1.52, SD = , Median = 1.52, MAD = 0.00, range:
## [1.52, 1.52], Skewness = , Kurtosis = , 0 missing
## - Effort.Mean: n = 1, Mean = 4.67, SD = , Median = 4.67, MAD = 0.00, range:
## [4.67, 4.67], Skewness = , Kurtosis = , 0 missing
## - Effort.SD: n = 1, Mean = 2.25, SD = , Median = 2.25, MAD = 0.00, range:
## [2.25, 2.25], Skewness = , Kurtosis = , 0 missing
## - Frustration.Mean: n = 1, Mean = 6.17, SD = , Median = 6.17, MAD = 0.00,
## range: [6.17, 6.17], Skewness = , Kurtosis = , 0 missing
## - Frustration.SD: n = 1, Mean = 3.31, SD = , Median = 3.31, MAD = 0.00, range:
## [3.31, 3.31], Skewness = , Kurtosis = , 0 missing
## - NASAaggregate.Mean: n = 1, Mean = 30.83, SD = , Median = 30.83, MAD = 0.00,
## range: [30.83, 30.83], Skewness = , Kurtosis = , 0 missing
## - NASAaggregate.SD: n = 1, Mean = 6.43, SD = , Median = 6.43, MAD = 0.00,
## range: [6.43, 6.43], Skewness = , Kurtosis = , 0 missing
## - CompletionTime.Mean: n = 1, Mean = 10.67, SD = , Median = 10.67, MAD = 0.00,
## range: [10.67, 10.67], Skewness = , Kurtosis = , 0 missing
## - CompletionTime.SD: n = 1, Mean = 1.97, SD = , Median = 1.97, MAD = 0.00,
## range: [1.97, 1.97], Skewness = , Kurtosis = , 0 missing
## - VisualClarity.Mean: n = 1, Mean = 0.42, SD = , Median = 0.42, MAD = 0.00,
## range: [0.42, 0.42], Skewness = , Kurtosis = , 0 missing
## - VisualClarity.SD: n = 1, Mean = 0.38, SD = , Median = 0.38, MAD = 0.00,
## range: [0.38, 0.38], Skewness = , Kurtosis = , 0 missing
## - UnnaturalDiscrepancy.Mean: n = 1, Mean = 0.50, SD = , Median = 0.50, MAD =
## 0.00, range: [0.50, 0.50], Skewness = , Kurtosis = , 0 missing
## - UnnaturalDiscrepancy.SD: n = 1, Mean = 0.32, SD = , Median = 0.32, MAD =
## 0.00, range: [0.32, 0.32], Skewness = , Kurtosis = , 0 missing
## - Revisions.Mean: n = 1, Mean = 0.50, SD = , Median = 0.50, MAD = 0.00, range:
## [0.50, 0.50], Skewness = , Kurtosis = , 0 missing
## - Revisions.SD: n = 1, Mean = 0.55, SD = , Median = 0.55, MAD = 0.00, range:
## [0.55, 0.55], Skewness = , Kurtosis = , 0 missingNASA TLX - Mental Demand: M = 5.00, SD = 2.00, Min = 2, Max = 7
NASA TLX - Physical Demand: M = 1.33, SD = 0.52, Min = 1, Max = 2
NASA TLX - Temporal Demand: M = 6.50, SD = 1.38, Min = 5, Max = 8
NASA TLX - Performance: M = 7.50, SD = 1.52, Min = 6, Max = 10
NASA TLX - Effort: M = 4.67, SD = 2.25, Min = 1, Max = 7
NASA TLX - Frustration: M = 6.17, SD = 3.31, Min = 1, Max = 10
NASA Aggregate: M = 30.83, SD = 6.43, Min = 21, Max = 37
Time to Completion (mins): M = 10.67, SD = 1.97, Min = 8, Max = 13
Visual Clarity: M = 0.42, SD = 0.38, Min = 0, Max = 1
Unnatural Discrepancy: M = 0.50, SD = 0.32, Min = 0, Max = 1
Number of Revisions: M = 0.50, SD = 0.55, Min = 0, Max = 1
##ANOVA
#clean the data:
data_clean <- data_clean |>
rename(
`Factor 1` = `Factor 1: Prompt Engineering`,
`Factor 2` = `Factor 2: Storyboard Production`
) |>
mutate(
`Factor 1` = as.factor(`Factor 1`),
`Factor 2` = as.factor(`Factor 2`)
)#two way anova:
aov_mental <- aov(`NASA TLX - Mental Demand` ~ `Factor 1` * `Factor 2`, data = data_clean)
summary(aov_mental)## Df Sum Sq Mean Sq F value Pr(>F)
## `Factor 1` 1 0.0 0.000 0.000 1.000
## `Factor 2` 1 1.5 1.500 0.243 0.656
## Residuals 3 18.5 6.167aov_physical <- aov(`NASA TLX - Physical Demand` ~ `Factor 1` * `Factor 2`, data = data_clean)
summary(aov_physical)## Df Sum Sq Mean Sq F value Pr(>F)
## `Factor 1` 1 0.0000 0.0000 0 1.000
## `Factor 2` 1 0.6667 0.6667 3 0.182
## Residuals 3 0.6667 0.2222aov_temporal <- aov(`NASA TLX - Temporal Demand` ~ `Factor 1` * `Factor 2`, data = data_clean)
summary(aov_temporal)## Df Sum Sq Mean Sq F value Pr(>F)
## `Factor 1` 1 1.5 1.500 0.692 0.466
## `Factor 2` 1 1.5 1.500 0.692 0.466
## Residuals 3 6.5 2.167aov_performance <- aov(`NASA TLX - Performance` ~ `Factor 1` * `Factor 2`, data = data_clean)
summary(aov_performance)## Df Sum Sq Mean Sq F value Pr(>F)
## `Factor 1` 1 1.5 1.500 0.45 0.55
## `Factor 2` 1 0.0 0.000 0.00 1.00
## Residuals 3 10.0 3.333aov_effort <- aov(`NASA TLX - Effort` ~ `Factor 1` * `Factor 2`, data = data_clean)
summary(aov_effort)## Df Sum Sq Mean Sq F value Pr(>F)
## `Factor 1` 1 0.667 0.667 0.098 0.775
## `Factor 2` 1 4.167 4.167 0.610 0.492
## Residuals 3 20.500 6.833aov_frustration <- aov(`NASA TLX - Frustration` ~ `Factor 1` * `Factor 2`, data = data_clean)
summary(aov_frustration)## Df Sum Sq Mean Sq F value Pr(>F)
## `Factor 1` 1 37.50 37.50 8.544 0.0613 .
## `Factor 2` 1 4.17 4.17 0.949 0.4017
## Residuals 3 13.17 4.39
## ---
## Signif. codes: 0 '***' 0.001 '**' 0.01 '*' 0.05 '.' 0.1 ' ' 1aov_NASA <- aov(`NASA Aggregate` ~ `Factor 1` * `Factor 2`, data = data_clean)
summary(aov_NASA)## Df Sum Sq Mean Sq F value Pr(>F)
## `Factor 1` 1 73.50 73.50 1.798 0.272
## `Factor 2` 1 10.67 10.67 0.261 0.645
## Residuals 3 122.67 40.89aov_time <- aov(`Time to Completion` ~ `Factor 1` * `Factor 2`, data = data_clean)
summary(aov_time)## Df Sum Sq Mean Sq F value Pr(>F)
## `Factor 1` 1 0.667 0.667 0.117 0.755
## `Factor 2` 1 1.500 1.500 0.262 0.644
## Residuals 3 17.167 5.722low degrees of freedom (df = 3 for residuals For mental: no significant main effect of Prompt Engineering, F(1, 3) = 0.00, p = 1.00, nor of Storyboard Production, F(1, 3) = 0.24, p = .656
For physical: no significant main effect of Prompt Engineering, F(1, 3) = 0.00, p = 1.00, nor of Storyboard Production, F(1, 3) = 3.00, p = .18.
#Kruskal-Wallis Tests
# Mental Demand
kruskal_mental <- kruskal.test(`NASA TLX - Mental Demand` ~ `Aggregate success score`, data = data_clean)
print(kruskal_mental)##
## Kruskal-Wallis rank sum test
##
## data: NASA TLX - Mental Demand by Aggregate success score
## Kruskal-Wallis chi-squared = 1.8182, df = 3, p-value = 0.611# Physical Demand
kruskal_physical <- kruskal.test(`NASA TLX - Physical Demand` ~ `Aggregate success score`, data = data_clean)
print(kruskal_physical)##
## Kruskal-Wallis rank sum test
##
## data: NASA TLX - Physical Demand by Aggregate success score
## Kruskal-Wallis chi-squared = 2.5, df = 3, p-value = 0.4753# Temporal Demand
kruskal_temporal <- kruskal.test(`NASA TLX - Temporal Demand` ~ `Aggregate success score`, data = data_clean)
print(kruskal_temporal)##
## Kruskal-Wallis rank sum test
##
## data: NASA TLX - Temporal Demand by Aggregate success score
## Kruskal-Wallis chi-squared = 2.5253, df = 3, p-value = 0.4707# Performance
kruskal_performance <- kruskal.test(`NASA TLX - Performance` ~ `Aggregate success score`, data = data_clean)
print(kruskal_performance)##
## Kruskal-Wallis rank sum test
##
## data: NASA TLX - Performance by Aggregate success score
## Kruskal-Wallis chi-squared = 0.90909, df = 3, p-value = 0.8232# Effort
kruskal_effort <- kruskal.test(`NASA TLX - Effort` ~ `Aggregate success score`, data = data_clean)
print(kruskal_effort)##
## Kruskal-Wallis rank sum test
##
## data: NASA TLX - Effort by Aggregate success score
## Kruskal-Wallis chi-squared = 3.0882, df = 3, p-value = 0.3782# Frustration
kruskal_frustration <- kruskal.test(`NASA TLX - Frustration` ~ `Aggregate success score`, data = data_clean)
print(kruskal_frustration)##
## Kruskal-Wallis rank sum test
##
## data: NASA TLX - Frustration by Aggregate success score
## Kruskal-Wallis chi-squared = 1, df = 3, p-value = 0.8013# NASA Aggregate
kruskal_aggregate <- kruskal.test(`NASA Aggregate` ~ `Aggregate success score`, data = data_clean)
print(kruskal_aggregate)##
## Kruskal-Wallis rank sum test
##
## data: NASA Aggregate by Aggregate success score
## Kruskal-Wallis chi-squared = 3.6667, df = 3, p-value = 0.2998For mental: test statistic is 1.82, df = 4. the p-value is 0.611, so no statistically significant difference in mental demand across the different levels of aggregate success score.
Visual
#Violin plot for factor 1
ggplot(data_clean, aes(x = `Factor 1`, y = `NASA Aggregate`, fill = `Factor 1` )) +
geom_violin(trim = FALSE, alpha = 0.6) +
geom_jitter(position = position_jitter(0.2))
labs(title = "NASA Aggregate by Factor 1") +
theme_minimal()## NULL#Violin plot for factor 1
ggplot(data_clean, aes(x = `Factor 2`, y = `NASA Aggregate`, fill = `Factor 2`)) +
geom_violin(trim = FALSE, alpha = 0.6) +
geom_jitter(position = position_jitter(0.2))
labs(title = "NASA Aggregate by Factor 2") +
theme_minimal()## NULL