Data generation and analysis modules for standardized questionnaire UMUX, Usability Metric for User Experience (2010, Kraig Finstad).

This notebook is made by Paul Amat, amat-design.com.

UMUX

About

Name: UMUX (Usability Metric for User Experience)

Author(s)-Date: Kraig Finstad, 2010

Licence: No license involved. Free to use (Kraig Finstad, email from April 2023)

Principal article: Finstad, Kraig. (2010). The Usability Metric for User Experience. Interacting with Computers. 22. 323-327. 10.1016/j.intcom.2010.04.004.

Link: https://www.researchgate.net/publication/220054775_The_Usability_Metric_for_User_Experience

Other sources:

Kraig Finstad, The Usability Metric for User Experience, Interacting with Computers, Volume 22, Issue 5, September 2010, Pages 323–327, https://doi.org/10.1016/j.intcom.2010.04.004

Items

7-points Likert scales, Strongly disagree - Strongly agree:

This software capabilities meet my requirements.
Using this software is a frustrating experience.
This software prototype is easy to use.
I have to spend too much time correcting things with this software.

Sources: Qualaroo

Generate

UMUX <- list()

n <- 200

for (i in 1:4){
  len <- length(UMUX)
  UMUX[[len+1]] <- numeric(0)
  names(UMUX)[len+1] <- paste("UMUX", i, sep = "")
}

for (i in 1:4) {
  if (i==1 | i==3) {
    UMUX[[i]] <- sample(1:7, n, replace = T, prob = c(1, 1, 1, 1, 1, 3, 2))
  } else if (i==2 | i==4) {
    UMUX[[i]] <- sample(1:7, n, replace = T, prob = c(2, 3, 1, 1, 1, 1, 1))
  }
}

UMUX <- as.data.frame(UMUX)

head(UMUX)

##   UMUX1 UMUX2 UMUX3 UMUX4
## 1     1     4     2     2
## 2     5     2     6     3
## 3     4     7     7     3
## 4     6     1     6     4
## 5     2     2     1     2
## 6     6     7     1     7

# display 6 first rows
knitr::kable(head(UMUX), caption = "Raw UMUX data (6 first rows)")

Raw UMUX data (6 first rows)
UMUX1	UMUX2	UMUX3	UMUX4
1	4	2	2
5	2	6	3
4	7	7	3
6	1	6	4
2	2	1	2
6	7	1	7

Transform

Odd items are scored as [user score - 1]. Even items are scored as [5 - user score].
Add up these differences and divide the sum by 24 (the highest possible score).
Multiply your quotient by 100.
Average your results across users.

Sources: Qualaroo

UMUX.tr <- UMUX

for (i in 1:ncol(UMUX.tr)){
  if (i %% 2 == 0) { #even number
    UMUX.tr[ ,i] <- 5 - UMUX.tr[ ,i]
  } else { #odd
    UMUX.tr[ ,i] <- UMUX.tr[ ,i] - 1
  }
}

UMUX.tr$Score <- rep(0, nrow(UMUX.tr))

for (i in 1:nrow(UMUX.tr)) {
  UMUX.tr[i, "Score"] <- (sum(as.numeric(UMUX.tr[i, ])) / 24) * 100
}

head(UMUX.tr)

##   UMUX1 UMUX2 UMUX3 UMUX4     Score
## 1     0     1     1     3 20.833333
## 2     4     3     5     2 58.333333
## 3     3    -2     6     2 37.500000
## 4     5     4     5     1 62.500000
## 5     1     3     0     3 29.166667
## 6     5    -2     0    -2  4.166667

# display 6 first rows
knitr::kable(head(UMUX.tr), caption = "Transformed UMUX data (6 first rows)")

Transformed UMUX data (6 first rows)
UMUX1	UMUX2	UMUX3	UMUX4	Score
0	1	1	3	20.833333
4	3	5	2	58.333333
3	-2	6	2	37.500000
5	4	5	1	62.500000
1	3	0	3	29.166667
5	-2	0	-2	4.166667

Infere

# initializing bootstrap
table.S <- numeric(1000)

# loop to generate means from original data
for(i in 1:1000) {
  table.S[i] <- mean(sample(UMUX.tr$Score, 10, replace=T))
}

# sort generated means
table.S.sorted <- sort(table.S)

# catch conf int by selecting heads and tails
UMUX.ci <- c(table.S.sorted[25], table.S.sorted[975])

print(paste("Total mean of Score:", round(mean(UMUX.tr$Score), 1)))

## [1] "Total mean of Score: 46.3"

print(paste("95% CI for SUS Score mean (bootstrap):", round(UMUX.ci, 1)[1], round(UMUX.ci, 1)[2]))

## [1] "95% CI for SUS Score mean (bootstrap): 35 57.5"

Visualize

Visualizing raw data

if (!require(ggplot2)) install.packages("ggplot2")

## Loading required package: ggplot2

library(ggplot2)

ggplot(data = UMUX, aes(x = UMUX1, fill = after_stat(count))) +
  geom_bar(width = 0.2) +
  scale_color_discrete() +
  theme_minimal() +
  guides(fill = "none") +
  labs(x = "UMUX1", y = "Count", title ="Distribution of UMUX1", subtitle = paste("Distribution of raw answers from UMUX question 1, coded UMUX1. N=", nrow(UMUX), sep = ""))

Visualizing scores

vlines <- data.frame("Mean" = mean(UMUX.tr$Score))

CIrect <- data.frame("Low" = UMUX.ci[1], "High" = UMUX.ci[2])

limits <- c(0, 100)

ggplot() +
  geom_histogram(data = UMUX.tr, aes(Score, fill = after_stat(x)), binwidth = 8) +
  coord_cartesian(xlim = limits) +
  scale_fill_gradient(name = "Score", low = "red", high = "green", limits = limits) +
  geom_vline(data = vlines, aes(xintercept = Mean), colour = "DodgerBlue") +
  geom_text(data = vlines, aes(x = Mean + 3, label="Mean/CI", y = 4.5), colour = "DodgerBlue", angle = 90) +
  geom_rect(aes(xmin = CIrect$Low, xmax = CIrect$High), ymin = -200, ymax = 200, fill = "DodgerBlue", alpha = 0.15) +
  labs(x = "UMUX scores", y = "Density", title ="Distribution of UMUX scores", subtitle = paste("N=", nrow(UMUX), sep = ""), caption = "95% confidence interval computed by bootstrap")

Conclusion

print(paste("Total UMUX score:", round(mean(UMUX.tr$Score), 1)))

## [1] "Total UMUX score: 46.3"

UMUX-LITE

About

Name: UMUX-LITE (Usability Metric for User Experience)

Author(s)-Date: Lewis, Utesch & Maher, 2013

Principal article: Lewis, James R. et al. “UMUX-LITE: when there’s no time for the SUS.” Proceedings of the SIGCHI Conference on Human Factors in Computing Systems (2013): n. pag.

Link: https://www.semanticscholar.org/paper/UMUX-LITE%3A-when-there's-no-time-for-the-SUS-Lewis-Utesch/33995b2a7d85d2247ba1cd5ac5777da9248e82e8

Other sources: Lewis, J.R., Utesch, B.S., Maher, D.E. (2015). Investigating the Correspondence Between UMUX-LITE and SUS Scores. In: Marcus, A. (eds) Design, User Experience, and Usability: Design Discourse. Lecture Notes in Computer Science(), vol 9186. Springer, Cham. https://doi.org/10.1007/978-3-319-20886-2_20

Items

5 or 7-points Likert scales, Disagree-Agree:

This software capabilities meet my requirements.
This software prototype is easy to use.

note: 7-points Likert scales allows to converting to SUS score.

Sources: Qualaroo

Generate

UMUXlite <- list()

for (i in 1:2){
  len <- length(UMUXlite)
  UMUXlite[[len+1]] <- numeric(0)
  names(UMUXlite)[len+1] <- paste("UMUXlite", i, sep = "")
}

UMUXlite[[1]] <- sample(1:7, n, replace = T, prob = c(0.1, 0.2, 0.5, 1, 3, 8, 4))
UMUXlite[[2]] <- sample(1:7, n, replace = T, prob = c(0.1, 0.2, 0.5, 1, 3, 8, 4))

UMUXlite <- as.data.frame(UMUXlite)

head(UMUXlite)

##   UMUXlite1 UMUXlite2
## 1         5         7
## 2         6         5
## 3         7         6
## 4         7         5
## 5         5         4
## 6         6         6

Transform

Items are scored by subtracting one from the user response: [user score - 1]
Add the two adjusted scores and divide the sum by 12 (the highest possible score).
Multiply your quotient by 100.
Average your results across users.

Sources: Qualaroo

UMUXlite.tr <- UMUXlite

UMUXlite.tr <- UMUXlite.tr - 1

UMUXlite.tr$Score <- rep(0, nrow(UMUXlite.tr))

for (i in 1:nrow(UMUXlite.tr)) {
  UMUXlite.tr[i, "Score"] <- (sum(as.numeric(UMUXlite.tr[i, ])) / 12) * 100
}

head(UMUXlite.tr)

##   UMUXlite1 UMUXlite2    Score
## 1         4         6 83.33333
## 2         5         4 75.00000
## 3         6         5 91.66667
## 4         6         4 83.33333
## 5         4         3 58.33333
## 6         5         5 83.33333

Infere

# initializing bootstrap
table.S <- numeric(1000)

# loop to generate means from original data
for(i in 1:1000) {
  table.S[i] <- mean(sample(UMUXlite.tr$Score, 10, replace=T))
}

# sort generated means
table.S.sorted <- sort(table.S)

# catch conf int by selecting heads and tails
UMUXlite.ci <- c(table.S.sorted[25], table.S.sorted[975])

print(paste("Total mean of Score:", round(mean(UMUXlite.tr$Score), 1)))

## [1] "Total mean of Score: 80.6"

print(paste("95% CI for SUS Score mean (bootstrap):", round(UMUXlite.ci, 1)[1], round(UMUXlite.ci, 1)[2]))

## [1] "95% CI for SUS Score mean (bootstrap): 72.5 88.3"

SUS score prediction

UMUXlite.tr$SUScomp <- rep(0, nrow(UMUXlite.tr))

for (i in 1:nrow(UMUXlite.tr)) {
  UMUXlite.tr[i, "SUScomp"] <- 
    0.65 * ((UMUXlite.tr[i, 1] + UMUXlite.tr[i, 2] - 2) * (100/12)) + 22.9
}

print(paste("SUS comparison score:", round(mean(UMUXlite.tr$SUScomp), 1)))

## [1] "SUS comparison score: 64.5"

Visualize

vlines <- data.frame("Mean" = mean(UMUXlite.tr$Score))

CIrect <- data.frame("Low" = UMUXlite.ci[1], "High" = UMUXlite.ci[2])

limits <- c(0, 100)

ggplot() +
  geom_histogram(data = UMUXlite.tr, aes(Score, fill = ..x..), binwidth = 8) +
  coord_cartesian(xlim = limits) +
  scale_fill_gradient(name = "Score", low = "red", high = "green", limits = limits) +
  geom_vline(data = vlines, aes(xintercept = Mean), colour = "DodgerBlue") +
  geom_text(data = vlines, aes(x = Mean + 3, label="Mean/CI", y = 4.5), colour = "DodgerBlue", angle = 90) +
  geom_rect(aes(xmin = CIrect$Low, xmax = CIrect$High), ymin = -200, ymax = 200, fill = "DodgerBlue", alpha = 0.15) +
  labs(x = "UMUX Lite scores", y = "Density", title ="Distribution of UMUX Lite scores")

## Warning: The dot-dot notation (`..x..`) was deprecated in ggplot2 3.4.0.
## ℹ Please use `after_stat(x)` instead.
## This warning is displayed once every 8 hours.
## Call `lifecycle::last_lifecycle_warnings()` to see where this warning was
## generated.

Conclusion

print(paste("Total UMUX-Lite score:", round(mean(UMUXlite.tr$Score), 1)))

## [1] "Total UMUX-Lite score: 80.6"

print(paste("SUS comparison score:", round(mean(UMUXlite.tr$SUScomp), 1)))

## [1] "SUS comparison score: 64.5"

UMUX & UMUX-Lite Notebook

Paul Amat

June 25, 2022

UMUX

About

Items

Generate

Transform

Infere

Visualize

Conclusion

UMUX-LITE

About

Items

Generate

Transform

Infere

SUS score prediction

Visualize

Conclusion