Organic Chemistry Survey: Disciplinary Growth Mindset, Chem vs oChem

Prepare Data

Import surveys, combine into single data frame, delete identifying information, assign IDs, and separate out by scale for item examination.

# https://hansjoerg.me/2018/04/23/rasch-in-r-tutorial/

knitr::knit_hooks$set(
   error = function(x, options) {
     paste('\n\n<div class="alert alert-danger">',
           gsub('##', '\n', gsub('^##\ Error', '**Error**', x)),
           '</div>', sep = '\n')
   },
   warning = function(x, options) {
     paste('\n\n<div class="alert alert-warning">',
           gsub('##', '\n', gsub('^##\ Warning:', '**Warning**', x)),
           '</div>', sep = '\n')
   },
   message = function(x, options) {
     paste('\n\n<div class="alert alert-info">',
           gsub('##', '\n', x),
           '</div>', sep = '\n')
   }
)

# load libraries ----------------------------------------------------------
library(stringi)
library(psych)
library(DT)
library(naniar)
library(UpSetR)
library(nFactors)
library(lavaan)
library(corrplot)
library(tidyr)

library(ggplot2)
library(dplyr)
library("eRm")
library("ltm")
library("difR")
library("psych")

# load data ---------------------------------------------------------------
# alt <- read.csv(file="UBelong Post-Survey Pitt OChem Spring 2022 Alternative Scales_April 28, 2022_12.34.csv", header=T)
# alt <- alt[-c(1,2),]
# alt$scale <- "alt"
# 
# orig <- read.csv(file="UBelong Post-Survey Pitt OChem Spring 2022 Original Scales_April 28, 2022_12.35.csv", header=T)
# orig <- orig[-c(1,2),]
# orig$scale <- "orig"
# 
# df <- rbind.data.frame(alt, orig)
# df <- subset(df, select = -c(1:19))
# names(df)
# myFun <- function(n) {
#   a <- do.call(paste0, replicate(5, sample(LETTERS, n, TRUE), FALSE))
#   paste0(a, sprintf("%04d", sample(9999, n, TRUE)), sample(LETTERS, n, TRUE))
# }
# df$id <- myFun(nrow(df))
# write.csv(df, file="imported_anonymized.csv", row.names = F)

df <- read.csv(file="imported_anonymized.csv", header=T)

# extract items -----------------------------------------------------------
# new items
EEochem <- subset(df, select=c(scale,grep("EEochem", colnames(df)))) # entry expectations
CCdisc <- subset(df, select=grep("CCdisc", colnames(df))) # classroom climate
IDochem <- cbind.data.frame(subset(df, select=c(scale,grep("IDochem", colnames(df)))), subset(df, select=grep("FASochem", colnames(df)))) # identity
CSochem <- subset(df, select=grep("CSochem", colnames(df))) # career satisfaction

# established scales
MSchem <- subset(df, select=c(scale,grep("MSchem", colnames(df)))) # discipline growth mindset (chemistry)
IPchem <- subset(df, select=grep("IPchem", colnames(df))) # instructor growth mindset (chemistry)
SEchem <- subset(df, select=grep("SEchem", colnames(df))) # disciplinary self-efficacy (chemistry)
MSochem <- subset(df, select=c(scale, grep("MSochem", colnames(df)))) # disciplinary growth mindset (organic chemistry)
IPochem <- subset(df, select=grep("IPochem", colnames(df))) # instructor growth mindset (organic chemistry)
SEochem <- subset(df, select=grep("SEochem", colnames(df))) # disciplinary self-efficacy (organic chemistry)
CNEBochem_class <- cbind.data.frame(subset(subset(df, select=grep("CNEBochem", colnames(df))), select=c(1:3))) # entity norms and beliefs
CNEBochem_self <- cbind.data.frame(subset(subset(df, select=grep("CNEBochem", colnames(df))), select=c(4:6))) # entity norms and beliefs
CNHSochem_others <- cbind.data.frame(subset(subset(df, select=grep("CNHSochem", colnames(df))), select=c(1:3))) # help seeking
CNHSochem_self <- cbind.data.frame(subset(subset(df, select=grep("CNHSochem", colnames(df))), select=c(4:6))) # help seeking
CNSWochem <- subset(df, select=grep("CNSWochem", colnames(df))) # help seeking
FCochem <- subset(df, select=grep("FCochem", colnames(df))) # faculty caring

items <- paste("item",1:7,sep="")
colnames(MSchem)[2:8] <- items
colnames(MSochem)[2:8] <- items
MSchem$domain <- "chem"
MSochem$domain <- "ochem"
MSall <- rbind.data.frame(MSchem, MSochem)

Disciplinary Growth Mindset (C)

Items

Chemistry

Even if I were to spend a lot of time working on difficult chemistry problems, I cannot develop my intelligence in chemistry further.
I won’t get better at chemistry if I try harder.
I could never excel in chemistry because I do not have what it takes to be a chemistry person.
I could never become really good at chemistry even if I were to work hard because I don’t have natural ability.
I can become even better at solving chemistry problems through hard work.
I am capable of really understanding chemistry if I work hard.
I can change my intelligence in chemistry quite a lot by working hard.

Organic Chemistry

Even if I were to spend a lot of time working on difficult organic chemistry problems, I cannot develop my intelligence in organic chemistry further.
I won’t get better at organic chemistry if I try harder.
I could never excel in organic chemistry because I do not have what it takes to be an organic chemistry person.
I could never become really good at organic chemistry even if I were to work hard because I don’t have natural ability.
I can become even better at solving organic chemistry problems through hard work.
I am capable of really understanding organic chemistry if I work hard.
I can change my intelligence in organic chemistry quite a lot by working hard.

Stats - Original

Univariate Stats

d <- subset(MSall, scale == "orig", select=-c(scale))
do <- subset(MSall, scale == "orig" & domain == "ochem", select=-c(scale, domain))
dc <- subset(MSall, scale == "orig" & domain == "chem", select=-c(scale, domain))
describeBy(d, group = "domain")

## 
##  Descriptive statistics by group 
## domain: chem
##         vars  n mean   sd median trimmed  mad min max range  skew kurtosis   se
## item1      1 81 1.96 0.86      2    1.88 1.48   1   4     3  0.66    -0.19 0.10
## item2      2 80 1.82 0.74      2    1.73 0.00   1   4     3  0.83     0.83 0.08
## item3      3 81 1.89 0.81      2    1.82 1.48   1   4     3  0.62    -0.17 0.09
## item4      4 80 1.88 0.88      2    1.78 1.48   1   4     3  0.68    -0.41 0.10
## item5      5 81 3.36 0.69      3    3.45 1.48   1   4     3 -1.04     1.36 0.08
## item6      6 81 3.23 0.73      3    3.32 1.48   1   4     3 -0.76     0.46 0.08
## item7      7 81 3.16 0.81      3    3.26 1.48   1   4     3 -0.84     0.34 0.09
## domain*    8 81 1.00 0.00      1    1.00 0.00   1   1     0   NaN      NaN 0.00
## ------------------------------------------------------------ 
## domain: ochem
##         vars  n mean   sd median trimmed  mad min max range  skew kurtosis   se
## item1      1 80 2.08 0.87      2    2.02 1.48   1   4     3  0.43    -0.55 0.10
## item2      2 80 1.93 0.82      2    1.86 1.48   1   4     3  0.54    -0.40 0.09
## item3      3 80 1.96 0.88      2    1.89 1.48   1   4     3  0.51    -0.63 0.10
## item4      4 80 2.00 0.87      2    1.94 1.48   1   4     3  0.45    -0.66 0.10
## item5      5 80 3.17 0.71      3    3.25 0.00   1   4     3 -0.68     0.60 0.08
## item6      6 80 3.10 0.72      3    3.16 0.00   1   4     3 -0.55     0.22 0.08
## item7      7 80 3.11 0.78      3    3.20 0.00   1   4     3 -0.83     0.64 0.09
## domain*    8 81 1.00 0.00      1    1.00 0.00   1   1     0   NaN      NaN 0.00

Missingness (oChem)

vis_miss(do)

# gg_miss_upset(EEochem)

Histograms (oChem)

ggplot(gather(do), aes(value)) + 
  geom_histogram(bins = 4) + 
  facet_wrap(~key)

Warning Removed 7 rows containing non-finite values (stat_bin).

Item Correlations (oChem)

corr <- corr.test(do, adjust = "holm")

rval <- corr$r
rval[lower.tri(corr$r, diag = T)] <- NA

datatable(rval) %>%
  formatRound(1:ncol(rval)) %>%
  formatStyle(1:ncol(rval), color = styleInterval(c(-.7, .7), c('red', 'black', 'red')))

corrplot(corr$r)

EFA (oChem) (1fac)

do <- na.omit(do)
ev <- eigen(cor(do))
ap <- parallel(subject=nrow(do),var=ncol(do),rep=100,cent=.05)
nS <- nScree(x=ev$values, aparallel=ap$eigen$qevpea)
plotnScree(nS)

EFA <- factanal(do, factors = 1, rotation = "promax", cutoff = 0.3)
print(EFA, digits=3, cutoff=.4, sort=TRUE)

## 
## Call:
## factanal(x = do, factors = 1, rotation = "promax", cutoff = 0.3)
## 
## Uniquenesses:
## item1 item2 item3 item4 item5 item6 item7 
## 0.359 0.284 0.225 0.152 0.335 0.221 0.440 
## 
## Loadings:
## [1]  0.801  0.846  0.880  0.921 -0.815 -0.882 -0.748
## 
##                Factor1
## SS loadings      4.982
## Proportion Var   0.712
## 
## Test of the hypothesis that 1 factor is sufficient.
## The chi square statistic is 54.14 on 14 degrees of freedom.
## The p-value is 1.22e-06

EFA (oChem) (2fac)

do <- na.omit(do)
ev <- eigen(cor(do))
ap <- parallel(subject=nrow(do),var=ncol(do),rep=100,cent=.05)
nS <- nScree(x=ev$values, aparallel=ap$eigen$qevpea)
plotnScree(nS)

EFA <- factanal(do, factors = 2, rotation = "promax", cutoff = 0.3)
print(EFA, digits=3, cutoff=.4, sort=TRUE)

## 
## Call:
## factanal(x = do, factors = 2, rotation = "promax", cutoff = 0.3)
## 
## Uniquenesses:
## item1 item2 item3 item4 item5 item6 item7 
## 0.295 0.278 0.166 0.109 0.333 0.237 0.086 
## 
## Loadings:
##       Factor1 Factor2
## item2  0.792         
## item3  1.031         
## item4  0.986         
## item6 -0.620         
## item1         -0.565 
## item7          1.099 
## item5 -0.460   0.400 
## 
##                Factor1 Factor2
## SS loadings      3.393   1.806
## Proportion Var   0.485   0.258
## Cumulative Var   0.485   0.743
## 
## Factor Correlations:
##         Factor1 Factor2
## Factor1     1.0    -0.8
## Factor2    -0.8     1.0
## 
## Test of the hypothesis that 2 factors are sufficient.
## The chi square statistic is 20.19 on 8 degrees of freedom.
## The p-value is 0.00963

Missingness (Chem)

vis_miss(dc)

# gg_miss_upset(EEochem)

Histograms (Chem)

ggplot(gather(dc), aes(value)) + 
  geom_histogram(bins = 4) + 
  facet_wrap(~key)

Warning Removed 2 rows containing non-finite values (stat_bin).

Item Correlations (Chem)

corr <- corr.test(dc, adjust = "holm")

rval <- corr$r
rval[lower.tri(corr$r, diag = T)] <- NA

datatable(rval) %>%
  formatRound(1:ncol(rval)) %>%
  formatStyle(1:ncol(rval), color = styleInterval(c(-.7, .7), c('red', 'black', 'red')))

corrplot(corr$r)

EFA (Chem) (1fac)

dc <- na.omit(dc)
ev <- eigen(cor(dc))
ap <- parallel(subject=nrow(dc),var=ncol(dc),rep=100,cent=.05)
nS <- nScree(x=ev$values, aparallel=ap$eigen$qevpea)
plotnScree(nS)

EFA <- factanal(dc, factors = 1, rotation = "promax", cutoff = 0.3)
print(EFA, digits=3, cutoff=.4, sort=TRUE)

## 
## Call:
## factanal(x = dc, factors = 1, rotation = "promax", cutoff = 0.3)
## 
## Uniquenesses:
## item1 item2 item3 item4 item5 item6 item7 
## 0.436 0.281 0.154 0.112 0.706 0.528 0.642 
## 
## Loadings:
## [1]  0.751  0.848  0.920  0.942 -0.543 -0.687 -0.598
## 
##                Factor1
## SS loadings      4.141
## Proportion Var   0.592
## 
## Test of the hypothesis that 1 factor is sufficient.
## The chi square statistic is 158.34 on 14 degrees of freedom.
## The p-value is 1.53e-26

EFA (Chem) (2fac)

dc <- na.omit(dc)
ev <- eigen(cor(dc))
ap <- parallel(subject=nrow(dc),var=ncol(dc),rep=100,cent=.05)
nS <- nScree(x=ev$values, aparallel=ap$eigen$qevpea)
plotnScree(nS)

EFA <- factanal(dc, factors = 2, rotation = "promax", cutoff = 0.3)
print(EFA, digits=3, cutoff=.4, sort=TRUE)

## 
## Call:
## factanal(x = dc, factors = 2, rotation = "promax", cutoff = 0.3)
## 
## Uniquenesses:
## item1 item2 item3 item4 item5 item6 item7 
## 0.446 0.250 0.130 0.111 0.181 0.148 0.370 
## 
## Loadings:
##       Factor1 Factor2
## item1  0.708         
## item2  0.918         
## item3  0.936         
## item4  0.894         
## item5          0.974 
## item6          0.849 
## item7          0.737 
## 
##                Factor1 Factor2
## SS loadings      3.056   2.229
## Proportion Var   0.437   0.318
## Cumulative Var   0.437   0.755
## 
## Factor Correlations:
##         Factor1 Factor2
## Factor1    1.00   -0.61
## Factor2   -0.61    1.00
## 
## Test of the hypothesis that 2 factors are sufficient.
## The chi square statistic is 58.7 on 8 degrees of freedom.
## The p-value is 8.36e-10

Stats - Alternative

Univariate Stats

d <- subset(MSall, scale == "alt", select=-c(scale))
do <- subset(MSall, scale == "alt" & domain == "ochem", select=-c(scale, domain))
dc <- subset(MSall, scale == "alt" & domain == "chem", select=-c(scale, domain))
describeBy(d, group = "domain")

## 
##  Descriptive statistics by group 
## domain: chem
##         vars   n mean   sd median trimmed  mad min max range  skew kurtosis
## item1      1 101 1.89 0.76      2    1.83 0.00   1   4     3  0.59     0.03
## item2      2 101 1.74 0.72      2    1.65 1.48   1   4     3  0.74     0.36
## item3      3 101 1.82 0.74      2    1.75 1.48   1   4     3  0.58    -0.08
## item4      4 101 1.83 0.79      2    1.74 1.48   1   4     3  0.79     0.30
## item5      5 101 3.37 0.64      3    3.43 1.48   1   4     3 -0.72     0.48
## item6      6 101 3.28 0.67      3    3.36 0.00   1   4     3 -0.57     0.13
## item7      7 101 3.28 0.68      3    3.35 1.48   2   4     2 -0.40    -0.87
## domain*    8 102 1.00 0.00      1    1.00 0.00   1   1     0   NaN      NaN
##           se
## item1   0.08
## item2   0.07
## item3   0.07
## item4   0.08
## item5   0.06
## item6   0.07
## item7   0.07
## domain* 0.00
## ------------------------------------------------------------ 
## domain: ochem
##         vars   n mean   sd median trimmed  mad min max range  skew kurtosis
## item1      1 101 1.95 0.71      2    1.91 0.00   1   4     3  0.40    -0.04
## item2      2 101 1.85 0.74      2    1.79 1.48   1   4     3  0.53    -0.14
## item3      3 101 2.00 0.81      2    1.94 1.48   1   4     3  0.55    -0.14
## item4      4 101 1.95 0.82      2    1.89 1.48   1   4     3  0.53    -0.34
## item5      5 101 3.31 0.60      3    3.35 0.00   2   4     2 -0.22    -0.67
## item6      6 101 3.21 0.64      3    3.26 0.00   2   4     2 -0.20    -0.68
## item7      7 101 3.22 0.61      3    3.27 0.00   2   4     2 -0.15    -0.56
## domain*    8 102 1.00 0.00      1    1.00 0.00   1   1     0   NaN      NaN
##           se
## item1   0.07
## item2   0.07
## item3   0.08
## item4   0.08
## item5   0.06
## item6   0.06
## item7   0.06
## domain* 0.00

Missingness (oChem)

vis_miss(do)

# gg_miss_upset(EEochem)

Histograms (oChem)

ggplot(gather(do), aes(value)) + 
  geom_histogram(bins = 4) + 
  facet_wrap(~key)

Warning Removed 7 rows containing non-finite values (stat_bin).

Item Correlations (oChem)

corr <- corr.test(do, adjust = "holm")

rval <- corr$r
rval[lower.tri(corr$r, diag = T)] <- NA

datatable(rval) %>%
  formatRound(1:ncol(rval)) %>%
  formatStyle(1:ncol(rval), color = styleInterval(c(-.7, .7), c('red', 'black', 'red')))

corrplot(corr$r)

EFA (oChem) (1fac)

do <- na.omit(do)
ev <- eigen(cor(do))
ap <- parallel(subject=nrow(do),var=ncol(do),rep=100,cent=.05)
nS <- nScree(x=ev$values, aparallel=ap$eigen$qevpea)
plotnScree(nS)

EFA <- factanal(do, factors = 1, rotation = "promax", cutoff = 0.3)
print(EFA, digits=3, cutoff=.4, sort=TRUE)

## 
## Call:
## factanal(x = do, factors = 1, rotation = "promax", cutoff = 0.3)
## 
## Uniquenesses:
## item1 item2 item3 item4 item5 item6 item7 
## 0.217 0.228 0.249 0.207 0.724 0.488 0.630 
## 
## Loadings:
## [1]  0.885  0.878  0.866  0.891 -0.525 -0.715 -0.609
## 
##                Factor1
## SS loadings      4.256
## Proportion Var   0.608
## 
## Test of the hypothesis that 1 factor is sufficient.
## The chi square statistic is 102.43 on 14 degrees of freedom.
## The p-value is 1.62e-15

EFA (oChem) (2fac)

do <- na.omit(do)
ev <- eigen(cor(do))
ap <- parallel(subject=nrow(do),var=ncol(do),rep=100,cent=.05)
nS <- nScree(x=ev$values, aparallel=ap$eigen$qevpea)
plotnScree(nS)

EFA <- factanal(do, factors = 2, rotation = "promax", cutoff = 0.3)
print(EFA, digits=3, cutoff=.4, sort=TRUE)

## 
## Call:
## factanal(x = do, factors = 2, rotation = "promax", cutoff = 0.3)
## 
## Uniquenesses:
## item1 item2 item3 item4 item5 item6 item7 
## 0.244 0.247 0.178 0.175 0.479 0.249 0.267 
## 
## Loadings:
##       Factor1 Factor2
## item1  0.789         
## item2  0.769         
## item3  0.963         
## item4  0.898         
## item5          0.752 
## item6          0.754 
## item7          0.899 
## 
##                Factor1 Factor2
## SS loadings      2.979   1.983
## Proportion Var   0.426   0.283
## Cumulative Var   0.426   0.709
## 
## Factor Correlations:
##         Factor1 Factor2
## Factor1   1.000  -0.665
## Factor2  -0.665   1.000
## 
## Test of the hypothesis that 2 factors are sufficient.
## The chi square statistic is 33 on 8 degrees of freedom.
## The p-value is 6.15e-05

Missingness (Chem)

vis_miss(dc)

# gg_miss_upset(EEochem)

Histograms (Chem)

ggplot(gather(dc), aes(value)) + 
  geom_histogram(bins = 4) + 
  facet_wrap(~key)

Warning Removed 7 rows containing non-finite values (stat_bin).

Item Correlations (Chem)

corr <- corr.test(dc, adjust = "holm")

rval <- corr$r
rval[lower.tri(corr$r, diag = T)] <- NA

datatable(rval) %>%
  formatRound(1:ncol(rval)) %>%
  formatStyle(1:ncol(rval), color = styleInterval(c(-.7, .7), c('red', 'black', 'red')))

corrplot(corr$r)

EFA (Chem) (1fac)

dc <- na.omit(dc)
ev <- eigen(cor(dc))
ap <- parallel(subject=nrow(dc),var=ncol(dc),rep=100,cent=.05)
nS <- nScree(x=ev$values, aparallel=ap$eigen$qevpea)
plotnScree(nS)

EFA <- factanal(dc, factors = 1, rotation = "promax", cutoff = 0.3)
print(EFA, digits=3, cutoff=.4, sort=TRUE)

## 
## Call:
## factanal(x = dc, factors = 1, rotation = "promax", cutoff = 0.3)
## 
## Uniquenesses:
## item1 item2 item3 item4 item5 item6 item7 
## 0.582 0.420 0.410 0.295 0.658 0.351 0.411 
## 
## Loadings:
## [1]  0.646  0.762  0.768  0.840 -0.585 -0.805 -0.768
## 
##                Factor1
## SS loadings      3.872
## Proportion Var   0.553
## 
## Test of the hypothesis that 1 factor is sufficient.
## The chi square statistic is 87.64 on 14 degrees of freedom.
## The p-value is 1.06e-12

EFA (Chem) (2fac)

dc <- na.omit(dc)
ev <- eigen(cor(dc))
ap <- parallel(subject=nrow(dc),var=ncol(dc),rep=100,cent=.05)
nS <- nScree(x=ev$values, aparallel=ap$eigen$qevpea)
plotnScree(nS)

EFA <- factanal(dc, factors = 2, rotation = "promax", cutoff = 0.3)
print(EFA, digits=3, cutoff=.4, sort=TRUE)

## 
## Call:
## factanal(x = dc, factors = 2, rotation = "promax", cutoff = 0.3)
## 
## Uniquenesses:
## item1 item2 item3 item4 item5 item6 item7 
## 0.630 0.465 0.316 0.005 0.580 0.234 0.249 
## 
## Loadings:
##       Factor1 Factor2
## item5  0.756         
## item6  0.867         
## item7  0.899         
## item3          0.802 
## item4          1.058 
## item1                
## item2 -0.441         
## 
##                Factor1 Factor2
## SS loadings      2.440   2.016
## Proportion Var   0.349   0.288
## Cumulative Var   0.349   0.637
## 
## Factor Correlations:
##         Factor1 Factor2
## Factor1    1.00   -0.74
## Factor2   -0.74    1.00
## 
## Test of the hypothesis that 2 factors are sufficient.
## The chi square statistic is 28.43 on 8 degrees of freedom.
## The p-value is 0.000399

1PL Model - (oChem) Orig (All)

Summary & Fit

d <- na.omit(subset(MSochem, scale == "orig", select=-c(scale, domain)))
d <- d %>%
  mutate_at(vars(1:ncol(d)), recode, `1` = 0, `2` = 0, `3` = 1, `4` = 1)

mod <- rasch(d)

Warning in rasch(d): Hessian matrix at convergence is not positive definite; unstable solution.

summary(mod)

Warning in sqrt(diag(new.covar)): NaNs produced

Warning in sqrt(Var[n.ind + 1, n.ind + 1]): NaNs produced

## 
## Call:
## rasch(data = d)
## 
## Model Summary:
##    log.Lik      AIC      BIC
##  -287.3612 590.7224 609.7786
## 
## Coefficients:
##                value std.err z.vals
## Dffclt.item1  2.5280     NaN    NaN
## Dffclt.item2  3.4925     NaN    NaN
## Dffclt.item3  2.1731     NaN    NaN
## Dffclt.item4  2.1327     NaN    NaN
## Dffclt.item5 -4.4847     NaN    NaN
## Dffclt.item6 -3.2966     NaN    NaN
## Dffclt.item7 -3.7358     NaN    NaN
## Dscrmn        0.3739     NaN    NaN
## 
## Integration:
## method: Gauss-Hermite
## quadrature points: 21 
## 
## Optimization:
## Convergence: 0 
## max(|grad|): 9.3 
## quasi-Newton: BFGS

item.fit(mod, simulate.p.value=T)

Warning in rasch(data = X.new): Hessian matrix at convergence is not positive definite; unstable solution.

## 
## Item-Fit Statistics and P-values
## 
## Call:
## rasch(data = d)
## 
## Alternative: Items do not fit the model
## Ability Categories: 10
## Monte Carlo samples: 100 
## 
##           X^2 Pr(>X^2)
## item1 38.8371   0.0099
## item2 32.2261   0.0099
## item3 42.0373   0.0099
## item4 44.5833   0.0099
## item5 24.0376   0.0198
## item6 30.7196   0.0198
## item7 20.5930   0.1089

ICC

plot(mod, type="ICC", cex = .7, legend = F, col = 1)

IIC

plot(mod, type="IIC", cex = .7, legend = F, col = 1)

Individual ICC Plots

items <- colnames(d)
n <- 1
for (i in 1:ncol(d)) {
  plot(mod, type = 'ICC', auto.key = FALSE, items = n, main = items[n], annot = F)
  n <- n + 1
}

Warning in plot.window(…): “auto.key” is not a graphical parameter

Warning in plot.xy(xy, type, …): “auto.key” is not a graphical parameter

Warning in axis(side = side, at = at, labels = labels, …): “auto.key” is not a

graphical parameter

Warning in axis(side = side, at = at, labels = labels, …): “auto.key” is not a

graphical parameter

Warning in box(…): “auto.key” is not a graphical parameter

Warning in title(…): “auto.key” is not a graphical parameter

Warning in plot.xy(xy.coords(x, y), type = type, …): “auto.key” is not a

graphical parameter

Warning in plot.window(…): “auto.key” is not a graphical parameter

Warning in plot.xy(xy, type, …): “auto.key” is not a graphical parameter

Warning in axis(side = side, at = at, labels = labels, …): “auto.key” is not a

graphical parameter

Warning in axis(side = side, at = at, labels = labels, …): “auto.key” is not a

graphical parameter

Warning in box(…): “auto.key” is not a graphical parameter

Warning in title(…): “auto.key” is not a graphical parameter

Warning in plot.xy(xy.coords(x, y), type = type, …): “auto.key” is not a

graphical parameter

Warning in plot.window(…): “auto.key” is not a graphical parameter

Warning in plot.xy(xy, type, …): “auto.key” is not a graphical parameter

Warning in axis(side = side, at = at, labels = labels, …): “auto.key” is not a

graphical parameter

Warning in axis(side = side, at = at, labels = labels, …): “auto.key” is not a

graphical parameter

Warning in box(…): “auto.key” is not a graphical parameter

Warning in title(…): “auto.key” is not a graphical parameter

Warning in plot.xy(xy.coords(x, y), type = type, …): “auto.key” is not a

graphical parameter

Warning in plot.window(…): “auto.key” is not a graphical parameter

Warning in plot.xy(xy, type, …): “auto.key” is not a graphical parameter

Warning in axis(side = side, at = at, labels = labels, …): “auto.key” is not a

graphical parameter

Warning in axis(side = side, at = at, labels = labels, …): “auto.key” is not a

graphical parameter

Warning in box(…): “auto.key” is not a graphical parameter

Warning in title(…): “auto.key” is not a graphical parameter

Warning in plot.xy(xy.coords(x, y), type = type, …): “auto.key” is not a

graphical parameter

Warning in plot.window(…): “auto.key” is not a graphical parameter

Warning in plot.xy(xy, type, …): “auto.key” is not a graphical parameter

Warning in axis(side = side, at = at, labels = labels, …): “auto.key” is not a

graphical parameter

Warning in axis(side = side, at = at, labels = labels, …): “auto.key” is not a

graphical parameter

Warning in box(…): “auto.key” is not a graphical parameter

Warning in title(…): “auto.key” is not a graphical parameter

Warning in plot.xy(xy.coords(x, y), type = type, …): “auto.key” is not a

graphical parameter

Warning in plot.window(…): “auto.key” is not a graphical parameter

Warning in plot.xy(xy, type, …): “auto.key” is not a graphical parameter

Warning in axis(side = side, at = at, labels = labels, …): “auto.key” is not a

graphical parameter

Warning in axis(side = side, at = at, labels = labels, …): “auto.key” is not a

graphical parameter

Warning in box(…): “auto.key” is not a graphical parameter

Warning in title(…): “auto.key” is not a graphical parameter

Warning in plot.xy(xy.coords(x, y), type = type, …): “auto.key” is not a

graphical parameter

Warning in plot.window(…): “auto.key” is not a graphical parameter

Warning in plot.xy(xy, type, …): “auto.key” is not a graphical parameter

Warning in axis(side = side, at = at, labels = labels, …): “auto.key” is not a

graphical parameter

Warning in axis(side = side, at = at, labels = labels, …): “auto.key” is not a

graphical parameter

Warning in box(…): “auto.key” is not a graphical parameter

Warning in title(…): “auto.key” is not a graphical parameter

Warning in plot.xy(xy.coords(x, y), type = type, …): “auto.key” is not a

graphical parameter

Test Information Function

plot(mod, type=c("IIC"), items=c(0))

1PL Model - (oChem) Orig (1fac)

Summary & Fit

d1 <- subset(d, select=c(item1, item2, item3, item4))
d2 <- subset(d, select=c(item5, item6, item7))

mod <- rasch(d1)
summary(mod)

## 
## Call:
## rasch(data = d1)
## 
## Model Summary:
##    log.Lik      AIC      BIC
##  -128.8986 267.7971 279.7073
## 
## Coefficients:
##               value std.err z.vals
## Dffclt.item1 0.5177  0.1703 3.0395
## Dffclt.item2 0.7232  0.1855 3.8980
## Dffclt.item3 0.5980  0.1741 3.4337
## Dffclt.item4 0.5576  0.1720 3.2413
## Dscrmn       4.2405  0.9283 4.5678
## 
## Integration:
## method: Gauss-Hermite
## quadrature points: 21 
## 
## Optimization:
## Convergence: 0 
## max(|grad|): 0.0019 
## quasi-Newton: BFGS

item.fit(mod, simulate.p.value=T)

Warning in rasch(data = X.new): Hessian matrix at convergence is not positive definite; unstable solution.

## 
## Item-Fit Statistics and P-values
## 
## Call:
## rasch(data = d1)
## 
## Alternative: Items do not fit the model
## Ability Categories: 10
## Monte Carlo samples: 100 
## 
##           X^2 Pr(>X^2)
## item1 16.4232   0.0396
## item2  1.8297    0.901
## item3  2.2430   0.8515
## item4  2.6200   0.7723

ICC

plot(mod, type="ICC", cex = .7, legend = F, col = 1)

IIC

plot(mod, type="IIC", cex = .7, legend = F, col = 1)

Individual ICC Plots

# items <- colnames(d)
# n <- 1
# for (i in 1:ncol(d)) {
#   plot(mod, type = 'ICC', auto.key = FALSE, items = n, main = items[n], annot = F)
#   n <- n + 1
# }

Test Information Function

plot(mod, type=c("IIC"), items=c(0))

1PL Model - (oChem) Orig (2fac)

Summary & Fit

mod <- rasch(d2)
summary(mod)

## 
## Call:
## rasch(data = d2)
## 
## Model Summary:
##    log.Lik      AIC      BIC
##  -69.90423 147.8085 157.3366
## 
## Coefficients:
##                value std.err  z.vals
## Dffclt.item5 -1.2168  0.1813 -6.7109
## Dffclt.item6 -1.0278  0.1730 -5.9405
## Dffclt.item7 -1.0875  0.1783 -6.1003
## Dscrmn        4.9752  1.1579  4.2967
## 
## Integration:
## method: Gauss-Hermite
## quadrature points: 21 
## 
## Optimization:
## Convergence: 0 
## max(|grad|): 2e-05 
## quasi-Newton: BFGS

item.fit(mod, simulate.p.value=T)

Warning in rasch(data = X.new): Hessian matrix at convergence is not positive definite; unstable solution.

## 
## Item-Fit Statistics and P-values
## 
## Call:
## rasch(data = d2)
## 
## Alternative: Items do not fit the model
## Ability Categories: 10
## Monte Carlo samples: 100 
## 
##          X^2 Pr(>X^2)
## item5 1.4435   0.6436
## item6 1.4461   0.6139
## item7 1.2561   0.6535

ICC

plot(mod, type="ICC", cex = .7, legend = F, col = 1)

IIC

plot(mod, type="IIC", cex = .7, legend = F, col = 1)

Individual ICC Plots

# items <- colnames(d)
# n <- 1
# for (i in 1:ncol(d)) {
#   plot(mod, type = 'ICC', auto.key = FALSE, items = n, main = items[n], annot = F)
#   n <- n + 1
# }

Test Information Function

plot(mod, type=c("IIC"), items=c(0))

1PL Model - (oChem) Alt (1fac)

Summary & Fit

d <- na.omit(subset(MSochem, scale == "alt", select=-c(scale, domain)))
d <- d %>%
  mutate_at(vars(1:ncol(d)), recode, `1` = 0, `2` = 0, `3` = 1, `4` = 1)

d1 <- subset(d, select=c(item1, item2, item3, item4))
d2 <- subset(d, select=c(item5, item6, item7))

mod <- rasch(d1)
summary(mod)

## 
## Call:
## rasch(data = d1)
## 
## Model Summary:
##    log.Lik      AIC     BIC
##  -136.7717 283.5434 296.619
## 
## Coefficients:
##               value std.err z.vals
## Dffclt.item1 0.8025  0.1239 6.4753
## Dffclt.item2 0.8703  0.1462 5.9538
## Dffclt.item3 0.6856  0.0975 7.0306
## Dffclt.item4 0.6856  0.0975 7.0306
## Dscrmn       5.8815  2.2872 2.5715
## 
## Integration:
## method: Gauss-Hermite
## quadrature points: 21 
## 
## Optimization:
## Convergence: 0 
## max(|grad|): 0.00028 
## quasi-Newton: BFGS

item.fit(mod, simulate.p.value=T)

Warning in rasch(data = X.new): Hessian matrix at convergence is not positive definite; unstable solution.

## 
## Item-Fit Statistics and P-values
## 
## Call:
## rasch(data = d1)
## 
## Alternative: Items do not fit the model
## Ability Categories: 10
## Monte Carlo samples: 100 
## 
##           X^2 Pr(>X^2)
## item1  2.4302   0.8218
## item2 16.9066   0.1782
## item3 10.0230   0.4059
## item4  4.7953   0.6337

ICC

plot(mod, type="ICC", cex = .7, legend = F, col = 1)

IIC

plot(mod, type="IIC", cex = .7, legend = F, col = 1)

Individual ICC Plots

# items <- colnames(d)
# n <- 1
# for (i in 1:ncol(d)) {
#   plot(mod, type = 'ICC', auto.key = FALSE, items = n, main = items[n], annot = F)
#   n <- n + 1
# }

Test Information Function

plot(mod, type=c("IIC"), items=c(0))

1PL Model - (oChem) Alt (2fac)

Summary & Fit

mod <- rasch(d2)
summary(mod)

## 
## Call:
## rasch(data = d2)
## 
## Model Summary:
##    log.Lik      AIC      BIC
##  -75.85734 159.7147 170.1752
## 
## Coefficients:
##                value std.err  z.vals
## Dffclt.item5 -1.6501  0.2410 -6.8469
## Dffclt.item6 -1.3159  0.1944 -6.7701
## Dffclt.item7 -1.4342  0.2066 -6.9420
## Dscrmn        3.7259  1.0979  3.3936
## 
## Integration:
## method: Gauss-Hermite
## quadrature points: 21 
## 
## Optimization:
## Convergence: 0 
## max(|grad|): 2.3e-05 
## quasi-Newton: BFGS

item.fit(mod, simulate.p.value=T)

Warning in rasch(data = X.new): Hessian matrix at convergence is not positive definite; unstable solution.

## 
## Item-Fit Statistics and P-values
## 
## Call:
## rasch(data = d2)
## 
## Alternative: Items do not fit the model
## Ability Categories: 10
## Monte Carlo samples: 100 
## 
##          X^2 Pr(>X^2)
## item5 7.5460   0.0792
## item6 3.4757   0.4356
## item7 4.1284   0.2574

ICC

plot(mod, type="ICC", cex = .7, legend = F, col = 1)

IIC

plot(mod, type="IIC", cex = .7, legend = F, col = 1)

Individual ICC Plots

# items <- colnames(d)
# n <- 1
# for (i in 1:ncol(d)) {
#   plot(mod, type = 'ICC', auto.key = FALSE, items = n, main = items[n], annot = F)
#   n <- n + 1
# }

Test Information Function

plot(mod, type=c("IIC"), items=c(0))

Chem vs oChem

Correlations

chem <- subset(MSall, domain == "chem" & scale == "orig")
ochem <- subset(MSall, domain == "ochem" & scale == "alt")
chem$f1 <- (chem$item1 + chem$item2 + chem$item3 + chem$item4)/4
chem$f2 <- (chem$item5 + chem$item6 + chem$item7)/3
ochem$f1 <- (ochem$item1 + ochem$item2 + ochem$item3 + ochem$item4)/4
ochem$f2 <- (ochem$item5 + ochem$item6 + ochem$item7)/3

final1 <- cbind.data.frame(f1=chem$f1, f2=chem$f2, domain=chem$domain)
final2 <- cbind.data.frame(f1=ochem$f1, f2=ochem$f2, domain=ochem$domain)
final <- rbind.data.frame(final1, final2)

corr.test(final1[1:2], adjust = "holm")

## Call:corr.test(x = final1[1:2], adjust = "holm")
## Correlation matrix 
##       f1    f2
## f1  1.00 -0.61
## f2 -0.61  1.00
## Sample Size 
##    f1 f2
## f1 79 79
## f2 79 81
## Probability values (Entries above the diagonal are adjusted for multiple tests.) 
##    f1 f2
## f1  0  0
## f2  0  0
## 
##  To see confidence intervals of the correlations, print with the short=FALSE option

corr.test(final2[1:2], adjust = "holm")

## Call:corr.test(x = final2[1:2], adjust = "holm")
## Correlation matrix 
##       f1    f2
## f1  1.00 -0.63
## f2 -0.63  1.00
## Sample Size 
## [1] 101
## Probability values (Entries above the diagonal are adjusted for multiple tests.) 
##    f1 f2
## f1  0  0
## f2  0  0
## 
##  To see confidence intervals of the correlations, print with the short=FALSE option

corr.test(final[1:2], adjust = "holm")

## Call:corr.test(x = final[1:2], adjust = "holm")
## Correlation matrix 
##       f1    f2
## f1  1.00 -0.61
## f2 -0.61  1.00
## Sample Size 
##     f1  f2
## f1 180 180
## f2 180 182
## Probability values (Entries above the diagonal are adjusted for multiple tests.) 
##    f1 f2
## f1  0  0
## f2  0  0
## 
##  To see confidence intervals of the correlations, print with the short=FALSE option

t-Tests

t.test(final$f1~final$domain)

## 
##  Welch Two Sample t-test
## 
## data:  final$f1 by final$domain
## t = -0.50425, df = 163.01, p-value = 0.6148
## alternative hypothesis: true difference in means between group chem and group ochem is not equal to 0
## 95 percent confidence interval:
##  -0.2713978  0.1609830
## sample estimates:
##  mean in group chem mean in group ochem 
##            1.882911            1.938119

t.test(final$f2~final$domain)

## 
##  Welch Two Sample t-test
## 
## data:  final$f2 by final$domain
## t = 0.073708, df = 151.03, p-value = 0.9413
## alternative hypothesis: true difference in means between group chem and group ochem is not equal to 0
## 95 percent confidence interval:
##  -0.1755931  0.1892019
## sample estimates:
##  mean in group chem mean in group ochem 
##            3.251029            3.244224

Organic Chemistry Survey: Disciplinary Growth Mindset, Chem vs oChem

Heather Perkins

5/13/2022

Prepare Data

Disciplinary Growth Mindset (C)

Items

Chemistry

Organic Chemistry

Stats - Original

Univariate Stats

Missingness (oChem)

Histograms (oChem)

Item Correlations (oChem)

EFA (oChem) (1fac)

EFA (oChem) (2fac)

Missingness (Chem)

Histograms (Chem)

Item Correlations (Chem)

EFA (Chem) (1fac)

EFA (Chem) (2fac)

Stats - Alternative

Univariate Stats

Missingness (oChem)

Histograms (oChem)

Item Correlations (oChem)

EFA (oChem) (1fac)

EFA (oChem) (2fac)

Missingness (Chem)

Histograms (Chem)

Item Correlations (Chem)

EFA (Chem) (1fac)

EFA (Chem) (2fac)

1PL Model - (oChem) Orig (All)

Summary & Fit

ICC

IIC

Individual ICC Plots

Test Information Function

1PL Model - (oChem) Orig (1fac)

Summary & Fit

ICC

IIC

Individual ICC Plots

Test Information Function

1PL Model - (oChem) Orig (2fac)

Summary & Fit

ICC

IIC

Individual ICC Plots

Test Information Function

1PL Model - (oChem) Alt (1fac)

Summary & Fit

ICC

IIC

Individual ICC Plots

Test Information Function

1PL Model - (oChem) Alt (2fac)

Summary & Fit

ICC

IIC

Individual ICC Plots

Test Information Function

Chem vs oChem

Correlations

t-Tests