Organic Chemistry Survey: Disciplinary Self-Efficacy

Prepare Data

Import surveys, combine into single data frame, delete identifying information, assign IDs, and separate out by scale for item examination.

# https://hansjoerg.me/2018/04/23/rasch-in-r-tutorial/

knitr::knit_hooks$set(
   error = function(x, options) {
     paste('\n\n<div class="alert alert-danger">',
           gsub('##', '\n', gsub('^##\ Error', '**Error**', x)),
           '</div>', sep = '\n')
   },
   warning = function(x, options) {
     paste('\n\n<div class="alert alert-warning">',
           gsub('##', '\n', gsub('^##\ Warning:', '**Warning**', x)),
           '</div>', sep = '\n')
   },
   message = function(x, options) {
     paste('\n\n<div class="alert alert-info">',
           gsub('##', '\n', x),
           '</div>', sep = '\n')
   }
)

# load libraries ----------------------------------------------------------
library(stringi)
library(psych)
library(DT)
library(naniar)
library(UpSetR)
library(nFactors)
library(lavaan)
library(corrplot)
library(tidyr)

library(ggplot2)
library(dplyr)
library("eRm")
library("ltm")
library("difR")
library("psych")

# load data ---------------------------------------------------------------
# alt <- read.csv(file="UBelong Post-Survey Pitt OChem Spring 2022 Alternative Scales_April 28, 2022_12.34.csv", header=T)
# alt <- alt[-c(1,2),]
# alt$scale <- "alt"
# 
# orig <- read.csv(file="UBelong Post-Survey Pitt OChem Spring 2022 Original Scales_April 28, 2022_12.35.csv", header=T)
# orig <- orig[-c(1,2),]
# orig$scale <- "orig"
# 
# df <- rbind.data.frame(alt, orig)
# df <- subset(df, select = -c(1:19))
# names(df)
# myFun <- function(n) {
#   a <- do.call(paste0, replicate(5, sample(LETTERS, n, TRUE), FALSE))
#   paste0(a, sprintf("%04d", sample(9999, n, TRUE)), sample(LETTERS, n, TRUE))
# }
# df$id <- myFun(nrow(df))
# write.csv(df, file="imported_anonymized.csv", row.names = F)

df <- read.csv(file="imported_anonymized.csv", header=T)

# extract items -----------------------------------------------------------
# new items
EEochem <- subset(df, select=c(scale,grep("EEochem", colnames(df)))) # entry expectations
CCdisc <- subset(df, select=c(scale,grep("CCdisc", colnames(df)))) # classroom climate
IDochem <- cbind.data.frame(subset(df, select=c(scale,grep("IDochem", colnames(df)))), subset(df, select=grep("FASochem", colnames(df)))) # identity
CSochem <- subset(df, select=grep("CSochem", colnames(df))) # career satisfaction

# established scales
MSchem <- subset(df, select=c(scale,grep("MSchem", colnames(df)))) # discipline growth mindset (chemistry)
IPchem <- subset(df, select=c(scale,grep("IPchem", colnames(df)))) # instructor growth mindset (chemistry)
SEchem <- subset(df, select=c(scale, grep("SEchem", colnames(df)))) # disciplinary self-efficacy (chemistry)
MSochem <- subset(df, select=c(scale, grep("MSochem", colnames(df)))) # disciplinary growth mindset (organic chemistry)
IPochem <- subset(df, select=grep("IPochem", colnames(df))) # instructor growth mindset (organic chemistry)
SEochem <- subset(df, select=grep("SEochem", colnames(df))) # disciplinary self-efficacy (organic chemistry)
CNEBochem_class <- cbind.data.frame(subset(subset(df, select=grep("CNEBochem", colnames(df))), select=c(1:3))) # entity norms and beliefs
CNEBochem_self <- cbind.data.frame(subset(subset(df, select=grep("CNEBochem", colnames(df))), select=c(4:6))) # entity norms and beliefs
CNHSochem_others <- cbind.data.frame(subset(subset(df, select=grep("CNHSochem", colnames(df))), select=c(1:3))) # help seeking
CNHSochem_self <- cbind.data.frame(subset(subset(df, select=grep("CNHSochem", colnames(df))), select=c(4:6))) # help seeking
CNSWochem <- subset(df, select=grep("CNSWochem", colnames(df))) # help seeking
FCochem <- subset(df, select=grep("FCochem", colnames(df))) # faculty caring

SEchem$SEchem03_rc[SEchem$SEchem03 == 1] <- 4
SEchem$SEchem03_rc[SEchem$SEchem03 == 2] <- 3
SEchem$SEchem03_rc[SEchem$SEchem03 == 3] <- 2
SEchem$SEchem03_rc[SEchem$SEchem03 == 4] <- 1
SEchem$SEchem05_rc[SEchem$SEchem05 == 1] <- 4
SEchem$SEchem05_rc[SEchem$SEchem05 == 2] <- 3
SEchem$SEchem05_rc[SEchem$SEchem05 == 3] <- 2
SEchem$SEchem05_rc[SEchem$SEchem05 == 4] <- 1
SEchem$SEchem03 <- SEchem$SEchem03_rc
SEchem$SEchem05 <- SEchem$SEchem05_rc
SEchem <- subset(SEchem, select=-c(SEchem05_rc, SEchem03_rc))

Disciplinary Self-Efficacy

Items

If I study, I will do well on a test in an chemistry course.
I am able to help my classmates with chemistry coursework.
I get a sinking feeling when I think of trying to tackle tough chemistry problems.
I always understand concepts I am studying in chemistry.
Other people understand more than I do about what is going on in my chemistry courses.

Stats - Original

Univariate Stats

d <- subset(SEchem, scale == "orig", select=-c(scale))
SEchem_desc <- data.frame(describe(d))
datatable(subset(SEchem_desc, select=-c(n, trimmed, mad))) %>%
  formatRound(1:10) %>%
  formatStyle(8:9, color = styleInterval(c(-2, 2), c('red', 'black', 'red')))

Missingness

vis_miss(d)

# gg_miss_upset(EEochem)

Histograms

ggplot(gather(d), aes(value)) + 
  geom_histogram(bins = 4) + 
  facet_wrap(~key)

Warning Removed 6 rows containing non-finite values (stat_bin).

Item Correlations

corr <- corr.test(d, adjust = "holm")

rval <- corr$r
rval[lower.tri(corr$r, diag = T)] <- NA

datatable(rval) %>%
  formatRound(1:ncol(rval)) %>%
  formatStyle(1:ncol(rval), color = styleInterval(c(-.7, .7), c('red', 'black', 'red')))

corrplot(corr$r)

EFA

d <- na.omit(d)
ev <- eigen(cor(d))
ap <- parallel(subject=nrow(d),var=ncol(d),rep=100,cent=.05)
nS <- nScree(x=ev$values, aparallel=ap$eigen$qevpea)
plotnScree(nS)

EFA <- factanal(d, factors = 1, rotation = "promax", cutoff = 0.3)
print(EFA, digits=3, cutoff=.1, sort=TRUE)

## 
## Call:
## factanal(x = d, factors = 1, rotation = "promax", cutoff = 0.3)
## 
## Uniquenesses:
## SEchem01 SEchem02 SEchem03 SEchem04 SEchem05 
##    0.500    0.565    0.766    0.866    0.872 
## 
## Loadings:
## [1] 0.707 0.660 0.483 0.367 0.357
## 
##                Factor1
## SS loadings      1.431
## Proportion Var   0.286
## 
## Test of the hypothesis that 1 factor is sufficient.
## The chi square statistic is 7.67 on 5 degrees of freedom.
## The p-value is 0.176

Stats - Alternative

Univariate Stats

d <- subset(SEchem, scale == "alt", select=-c(scale))
SEchem_desc <- data.frame(describe(d))
datatable(subset(SEchem_desc, select=-c(n, trimmed, mad))) %>%
  formatRound(1:10) %>%
  formatStyle(8:9, color = styleInterval(c(-2, 2), c('red', 'black', 'red')))

Missingness

vis_miss(d)

# gg_miss_upset(EEochem)

Histograms

ggplot(gather(d), aes(value)) + 
  geom_histogram(bins = 4) + 
  facet_wrap(~key)

Warning Removed 5 rows containing non-finite values (stat_bin).

Item Correlations

corr <- corr.test(d, adjust = "holm")

rval <- corr$r
rval[lower.tri(corr$r, diag = T)] <- NA

datatable(rval) %>%
  formatRound(1:ncol(rval)) %>%
  formatStyle(1:ncol(rval), color = styleInterval(c(-.7, .7), c('red', 'black', 'red')))

corrplot(corr$r)

EFA

d <- na.omit(d)
ev <- eigen(cor(d))
ap <- parallel(subject=nrow(d),var=ncol(d),rep=100,cent=.05)
nS <- nScree(x=ev$values, aparallel=ap$eigen$qevpea)
plotnScree(nS)

EFA <- factanal(d, factors = 1, rotation = "promax", cutoff = 0.3)
print(EFA, digits=3, cutoff=.4, sort=TRUE)

## 
## Call:
## factanal(x = d, factors = 1, rotation = "promax", cutoff = 0.3)
## 
## Uniquenesses:
## SEchem01 SEchem02 SEchem03 SEchem04 SEchem05 
##    0.650    0.602    0.633    0.548    0.488 
## 
## Loadings:
## [1] 0.592 0.631 0.606 0.672 0.716
## 
##                Factor1
## SS loadings      2.079
## Proportion Var   0.416
## 
## Test of the hypothesis that 1 factor is sufficient.
## The chi square statistic is 4.04 on 5 degrees of freedom.
## The p-value is 0.544

1PL Model - Orig (All)

Summary & Fit

d <- subset(SEchem, scale == "orig", select=-c(scale))
d <- d %>%
  mutate_at(vars(1:ncol(d)), recode, `1` = 0, `2` = 0, `3` = 1, `4` = 1)

mod <- rasch(d)

Warning in rasch(d): Hessian matrix at convergence is not positive definite; unstable solution.

summary(mod)

Warning in sqrt(diag(new.covar)): NaNs produced

Warning in sqrt(Var[n.ind + 1, n.ind + 1]): NaNs produced

## 
## Call:
## rasch(data = d)
## 
## Model Summary:
##    log.Lik      AIC      BIC
##  -239.8847 491.7694 506.1361
## 
## Coefficients:
##                   value std.err  z.vals
## Dffclt.SEchem01 -0.9707  0.1701 -5.7055
## Dffclt.SEchem02 -0.6623  0.2259 -2.9312
## Dffclt.SEchem03 -0.9764  0.1656 -5.8955
## Dffclt.SEchem04  1.9294     NaN     NaN
## Dffclt.SEchem05 -1.1096  0.1219 -9.1020
## Dscrmn           0.9970     NaN     NaN
## 
## Integration:
## method: Gauss-Hermite
## quadrature points: 21 
## 
## Optimization:
## Convergence: 0 
## max(|grad|): 13 
## quasi-Newton: BFGS

item.fit(mod, simulate.p.value=T)

Warning in rasch(data = X.new): Hessian matrix at convergence is not positive definite; unstable solution.

## 
## Item-Fit Statistics and P-values
## 
## Call:
## rasch(data = d)
## 
## Alternative: Items do not fit the model
## Ability Categories: 10
## Monte Carlo samples: 100 
## 
##             X^2 Pr(>X^2)
## SEchem01 6.4975    0.901
## SEchem02 9.6174   0.6733
## SEchem03 2.6261   0.9901
## SEchem04 8.9029   0.9604
## SEchem05 4.7580   0.9406

ICC

plot(mod, type="ICC", cex = .7, legend = F, col = 1)

IIC

plot(mod, type="IIC", cex = .7, legend = F, col = 1)

Individual ICC Plots

items <- colnames(d)
n <- 1
for (i in 1:ncol(d)) {
  plot(mod, type = 'ICC', auto.key = FALSE, items = n, main = items[n], annot = F)
  n <- n + 1
}

Warning in plot.window(…): “auto.key” is not a graphical parameter

Warning in plot.xy(xy, type, …): “auto.key” is not a graphical parameter

Warning in axis(side = side, at = at, labels = labels, …): “auto.key” is not a

graphical parameter

Warning in axis(side = side, at = at, labels = labels, …): “auto.key” is not a

graphical parameter

Warning in box(…): “auto.key” is not a graphical parameter

Warning in title(…): “auto.key” is not a graphical parameter

Warning in plot.xy(xy.coords(x, y), type = type, …): “auto.key” is not a

graphical parameter

Warning in plot.window(…): “auto.key” is not a graphical parameter

Warning in plot.xy(xy, type, …): “auto.key” is not a graphical parameter

Warning in axis(side = side, at = at, labels = labels, …): “auto.key” is not a

graphical parameter

Warning in axis(side = side, at = at, labels = labels, …): “auto.key” is not a

graphical parameter

Warning in box(…): “auto.key” is not a graphical parameter

Warning in title(…): “auto.key” is not a graphical parameter

Warning in plot.xy(xy.coords(x, y), type = type, …): “auto.key” is not a

graphical parameter

Warning in plot.window(…): “auto.key” is not a graphical parameter

Warning in plot.xy(xy, type, …): “auto.key” is not a graphical parameter

Warning in axis(side = side, at = at, labels = labels, …): “auto.key” is not a

graphical parameter

Warning in axis(side = side, at = at, labels = labels, …): “auto.key” is not a

graphical parameter

Warning in box(…): “auto.key” is not a graphical parameter

Warning in title(…): “auto.key” is not a graphical parameter

Warning in plot.xy(xy.coords(x, y), type = type, …): “auto.key” is not a

graphical parameter

Warning in plot.window(…): “auto.key” is not a graphical parameter

Warning in plot.xy(xy, type, …): “auto.key” is not a graphical parameter

Warning in axis(side = side, at = at, labels = labels, …): “auto.key” is not a

graphical parameter

Warning in axis(side = side, at = at, labels = labels, …): “auto.key” is not a

graphical parameter

Warning in box(…): “auto.key” is not a graphical parameter

Warning in title(…): “auto.key” is not a graphical parameter

Warning in plot.xy(xy.coords(x, y), type = type, …): “auto.key” is not a

graphical parameter

Warning in plot.window(…): “auto.key” is not a graphical parameter

Warning in plot.xy(xy, type, …): “auto.key” is not a graphical parameter

Warning in axis(side = side, at = at, labels = labels, …): “auto.key” is not a

graphical parameter

Warning in axis(side = side, at = at, labels = labels, …): “auto.key” is not a

graphical parameter

Warning in box(…): “auto.key” is not a graphical parameter

Warning in title(…): “auto.key” is not a graphical parameter

Warning in plot.xy(xy.coords(x, y), type = type, …): “auto.key” is not a

graphical parameter

Test Information Function

plot(mod, type=c("IIC"), items=c(0), ylim=c(0,1.1))

1PL Model - Orig (1F)

Summary & Fit

d1 <- subset(d, select=c(SEchem01,SEchem02,SEchem03))

d <- d1

mod <- rasch(d)
summary(mod)

## 
## Call:
## rasch(data = d)
## 
## Model Summary:
##    log.Lik      AIC      BIC
##  -153.0036 314.0073 323.5851
## 
## Coefficients:
##                      value      std.err z.vals
## Dffclt.SEchem01 -19567.153 1.048857e+09      0
## Dffclt.SEchem02  -6385.386 3.422755e+08      0
## Dffclt.SEchem03 -21528.406 1.153986e+09      0
## Dscrmn               0.000 2.109600e+00      0
## 
## Integration:
## method: Gauss-Hermite
## quadrature points: 21 
## 
## Optimization:
## Convergence: 0 
## max(|grad|): 0.00022 
## quasi-Newton: BFGS

item.fit(mod, simulate.p.value=T)

Warning in rasch(data = X.new): Hessian matrix at convergence is not positive definite; unstable solution.

## 
## Item-Fit Statistics and P-values
## 
## Call:
## rasch(data = d)
## 
## Alternative: Items do not fit the model
## Ability Categories: 10
## Monte Carlo samples: 100 
## 
##              X^2 Pr(>X^2)
## SEchem01 37.9915   0.0297
## SEchem02 38.0803   0.0297
## SEchem03 18.8865   0.5743

ICC

plot(mod, type="ICC", cex = .7, legend = F, col = 1)

IIC

plot(mod, type="IIC", cex = .7, legend = F, col = 1)

Individual ICC Plots

items <- colnames(d)
n <- 1
for (i in 1:ncol(d)) {
  plot(mod, type = 'ICC', auto.key = FALSE, items = n, main = items[n], annot = F)
  n <- n + 1
}

Warning in plot.window(…): “auto.key” is not a graphical parameter

Warning in plot.xy(xy, type, …): “auto.key” is not a graphical parameter

Warning in axis(side = side, at = at, labels = labels, …): “auto.key” is not a

graphical parameter

Warning in axis(side = side, at = at, labels = labels, …): “auto.key” is not a

graphical parameter

Warning in box(…): “auto.key” is not a graphical parameter

Warning in title(…): “auto.key” is not a graphical parameter

Warning in plot.xy(xy.coords(x, y), type = type, …): “auto.key” is not a

graphical parameter

Warning in plot.window(…): “auto.key” is not a graphical parameter

Warning in plot.xy(xy, type, …): “auto.key” is not a graphical parameter

Warning in axis(side = side, at = at, labels = labels, …): “auto.key” is not a

graphical parameter

Warning in axis(side = side, at = at, labels = labels, …): “auto.key” is not a

graphical parameter

Warning in box(…): “auto.key” is not a graphical parameter

Warning in title(…): “auto.key” is not a graphical parameter

Warning in plot.xy(xy.coords(x, y), type = type, …): “auto.key” is not a

graphical parameter

Warning in plot.window(…): “auto.key” is not a graphical parameter

Warning in plot.xy(xy, type, …): “auto.key” is not a graphical parameter

Warning in axis(side = side, at = at, labels = labels, …): “auto.key” is not a

graphical parameter

Warning in axis(side = side, at = at, labels = labels, …): “auto.key” is not a

graphical parameter

Warning in box(…): “auto.key” is not a graphical parameter

Warning in title(…): “auto.key” is not a graphical parameter

Warning in plot.xy(xy.coords(x, y), type = type, …): “auto.key” is not a

graphical parameter

Test Information Function

plot(mod, type=c("IIC"), items=c(0), ylim=c(0,1.1))

1PL Model - Alt (All)

Summary & Fit

d <- subset(SEchem, scale == "alt", select=-c(scale))
d <- d %>%
  mutate_at(vars(1:ncol(d)), recode, `1` = 0, `2` = 0, `3` = 1, `4` = 1)

mod <- rasch(d)

Warning in rasch(d): Hessian matrix at convergence is not positive definite; unstable solution.

summary(mod)

Warning in sqrt(diag(new.covar)): NaNs produced

Warning in sqrt(Var[n.ind + 1, n.ind + 1]): NaNs produced

## 
## Call:
## rasch(data = d)
## 
## Model Summary:
##  log.Lik      AIC      BIC
##  -299.36 610.7201 626.4699
## 
## Coefficients:
##                    value std.err z.vals
## Dffclt.SEchem01 -40.5678     NaN    NaN
## Dffclt.SEchem02  -3.4604     NaN    NaN
## Dffclt.SEchem03 -46.6089     NaN    NaN
## Dffclt.SEchem04  53.7920     NaN    NaN
## Dffclt.SEchem05 -28.8565     NaN    NaN
## Dscrmn            0.0251     NaN    NaN
## 
## Integration:
## method: Gauss-Hermite
## quadrature points: 21 
## 
## Optimization:
## Convergence: 0 
## max(|grad|): 0.47 
## quasi-Newton: BFGS

item.fit(mod, simulate.p.value=T)

Warning in rasch(data = X.new): Hessian matrix at convergence is not positive definite; unstable solution.

## 
## Item-Fit Statistics and P-values
## 
## Call:
## rasch(data = d)
## 
## Alternative: Items do not fit the model
## Ability Categories: 10
## Monte Carlo samples: 100 
## 
##              X^2 Pr(>X^2)
## SEchem01 36.9332   0.0099
## SEchem02 26.9178   0.3762
## SEchem03 14.9327   0.5644
## SEchem04 18.8706   0.6436
## SEchem05  9.1756   0.9505

ICC

plot(mod, type="ICC", cex = .7, legend = F, col = 1)

IIC

plot(mod, type="IIC", cex = .7, legend = F, col = 1)

Individual ICC Plots

items <- colnames(d)
n <- 1
for (i in 1:ncol(d)) {
  plot(mod, type = 'ICC', auto.key = FALSE, items = n, main = items[n], annot = F)
  n <- n + 1
}

Warning in plot.window(…): “auto.key” is not a graphical parameter

Warning in plot.xy(xy, type, …): “auto.key” is not a graphical parameter

Warning in axis(side = side, at = at, labels = labels, …): “auto.key” is not a

graphical parameter

Warning in axis(side = side, at = at, labels = labels, …): “auto.key” is not a

graphical parameter

Warning in box(…): “auto.key” is not a graphical parameter

Warning in title(…): “auto.key” is not a graphical parameter

Warning in plot.xy(xy.coords(x, y), type = type, …): “auto.key” is not a

graphical parameter

Warning in plot.window(…): “auto.key” is not a graphical parameter

Warning in plot.xy(xy, type, …): “auto.key” is not a graphical parameter

Warning in axis(side = side, at = at, labels = labels, …): “auto.key” is not a

graphical parameter

Warning in axis(side = side, at = at, labels = labels, …): “auto.key” is not a

graphical parameter

Warning in box(…): “auto.key” is not a graphical parameter

Warning in title(…): “auto.key” is not a graphical parameter

Warning in plot.xy(xy.coords(x, y), type = type, …): “auto.key” is not a

graphical parameter

Warning in plot.window(…): “auto.key” is not a graphical parameter

Warning in plot.xy(xy, type, …): “auto.key” is not a graphical parameter

Warning in axis(side = side, at = at, labels = labels, …): “auto.key” is not a

graphical parameter

Warning in axis(side = side, at = at, labels = labels, …): “auto.key” is not a

graphical parameter

Warning in box(…): “auto.key” is not a graphical parameter

Warning in title(…): “auto.key” is not a graphical parameter

Warning in plot.xy(xy.coords(x, y), type = type, …): “auto.key” is not a

graphical parameter

Warning in plot.window(…): “auto.key” is not a graphical parameter

Warning in plot.xy(xy, type, …): “auto.key” is not a graphical parameter

Warning in axis(side = side, at = at, labels = labels, …): “auto.key” is not a

graphical parameter

Warning in axis(side = side, at = at, labels = labels, …): “auto.key” is not a

graphical parameter

Warning in box(…): “auto.key” is not a graphical parameter

Warning in title(…): “auto.key” is not a graphical parameter

Warning in plot.xy(xy.coords(x, y), type = type, …): “auto.key” is not a

graphical parameter

Warning in plot.window(…): “auto.key” is not a graphical parameter

Warning in plot.xy(xy, type, …): “auto.key” is not a graphical parameter

Warning in axis(side = side, at = at, labels = labels, …): “auto.key” is not a

graphical parameter

Warning in axis(side = side, at = at, labels = labels, …): “auto.key” is not a

graphical parameter

Warning in box(…): “auto.key” is not a graphical parameter

Warning in title(…): “auto.key” is not a graphical parameter

Warning in plot.xy(xy.coords(x, y), type = type, …): “auto.key” is not a

graphical parameter

Test Information Function

plot(mod, type=c("IIC"), items=c(0), ylim=c(0,1.1))

Organic Chemistry Survey: Disciplinary Self-Efficacy

Heather Perkins

5/13/2022

Prepare Data

Disciplinary Self-Efficacy

Items

Stats - Original

Univariate Stats

Missingness

Histograms

Item Correlations

EFA

Stats - Alternative

Univariate Stats

Missingness

Histograms

Item Correlations

EFA

1PL Model - Orig (All)

Summary & Fit

ICC

IIC

Individual ICC Plots

Test Information Function

1PL Model - Orig (1F)

Summary & Fit

ICC

IIC

Individual ICC Plots

Test Information Function

1PL Model - Alt (All)

Summary & Fit

ICC

IIC

Individual ICC Plots

Test Information Function