irt_first_pass

Goals of these analyses: Start fitting IRT models to large datasets of 2AFC picture matching tasks Are kids and adults similar in their item parameters? Which items are good and should be included going forward? Does estimated theta (i.e., latent ability) from the models track with age?

To do based on meeting: 1. Get rid of bad items 2. Get rid of items without data 3. Get rid of people who are low performing (Slightly overfit this dataset)

Look at model fit statistics for 2PL and PC2PL Further item refinement –

Then look at thetas in the subpopulations

(next steps) Any other sociodemographics that we have? (what do we have) Are there DIF items for ELLs? Do we see the different distractor items (from DAVIT or CLIP) ordering in terms of difficulty in their item parameters?

Setup

library(tidyverse)
library(here)
library(lubridate)
library(ggthemes)
library(langcog)
library(mirt)
library(knitr)

Load and merge data

Load preporcessed data

pilot_data_cdm <- read_csv(file = here::here('data/cdm_data/cdm-trials-022123.csv')) %>%
  mutate(cohort = 'cdm')

demo_data <- read_csv(file = here::here('data/demo_data/demo-trials-012223.csv')) %>%
  mutate(cohort = 'demo')

school_data <- read_csv(file = here::here('data/rocketship_data/school-trials-022223.csv')) %>%
  mutate(cohort = 'school')

prolific_data <- read_csv(file = here::here('data/adult_prolific_pilot_preprocessed/prolific_data_preprocessed.csv')) %>%
  mutate(cohort = 'prolific') %>%
  mutate(pid = sub_id) %>% 
  mutate(age = 'Adult') %>%
  mutate(response = 3) # for merging correctly later since preprocessed

User metadata from firebase for school/demo/cdm

users <- read_csv(file = here::here('data/cdm_data/prod_roar-prod_tasks_vocab_users.csv'))  %>%
  select(age,id,ell) 

Join together datasets, merge in metadata

pilot_data <- pilot_data_cdm %>%
  full_join(demo_data) %>%
  full_join(school_data) %>%
  left_join(users %>% select(age, id, ell), by=c('pid' = 'id'))  %>%
  full_join(prolific_data) 

Filter kids who don’t pass 2/3 catch trials

catch_exclude <- pilot_data %>%
  filter(response>0) %>%
  filter(wordPairing == 'catch') %>%
  group_by(pid, start_time) %>%
  summarize(percent_correct = mean(correct), num_completed = n()) %>%
  filter(percent_correct<.5)

Demographics of filtered subs based on catch trials

catch_exclude_filtered <- pilot_data %>%
  filter(pid %in% catch_exclude$pid) %>%
  distinct(age,pid,cohort) %>%
  group_by(cohort,age) %>%
  summarize(num_subs = length(pid))

catch_exclude_filtered %>%
  kable()

cohort	age	num_subs
cdm	2	21
cdm	3	8
cdm	4	2
cdm	5	3
cdm	6	2
cdm	8	2
cdm	Adult	3
demo	3	1
demo	5	1
demo	Adult	6
school	NA	36

Histograph of participants at school

rocketship_by_kid <- school_data %>%
  group_by(pid) %>%
  summarize(prop_correct = mean(correct))


hist(rocketship_by_kid$prop_correct)

Filter to eliminate low accuracy participants

low_acc_exclude <- pilot_data %>%
  filter(response>0) %>%
  group_by(pid) %>%
  filter(!wordPairing %in% c('catch','practice')) %>%
  summarize(percent_correct = mean(correct)) %>%
  filter(percent_correct<.5)

high_acc_exclude <- pilot_data %>%
  filter(response>0) %>%
  group_by(pid) %>%
  filter(!wordPairing %in% c('catch','practice')) %>%
  summarize(percent_correct = mean(correct)) %>%
  filter(percent_correct==1)

low_acc_filtered_subs <- pilot_data %>%
  filter(pid %in% low_acc_exclude$pid) %>%
  distinct(age,pid,cohort) %>%
  group_by(cohort,age) %>%
  summarize(num_subs = length(pid))

Demographics of subs who had below chance accuracy

low_acc_filtered_subs %>%
  kable()

cohort	age	num_subs
cdm	2	16
cdm	3	5
cdm	4	2
cdm	5	1
cdm	6	4
cdm	7	1
cdm	8	1
cdm	Adult	3
demo	3	1
demo	5	1
demo	Adult	3
school	NA	26

Filter to actual responses from test trials

pilot_data_filtered <- pilot_data %>%
  filter(response>0) %>%
  filter(!wordPairing %in% c('catch','practice')) %>%
  filter(!pid %in% catch_exclude$pid) %>%
  filter(!pid %in% low_acc_exclude$pid)

Make dataset with only school/proflic data (experiental contexts)

pilot_data_experimental <- school_data %>%
  mutate(age = '9') %>%
  # left_join(users %>% select(age, id, ell), by=c('pid' = 'id'))  %>%
  full_join(prolific_data)

pilot_data_experimental_filtered <- pilot_data_experimental %>%
  filter(response>0) %>%
  filter(!wordPairing %in% c('catch','practice')) %>%
  filter(!pid %in% catch_exclude$pid) %>%
  filter(!pid %in% low_acc_exclude$pid)

Add inextra fields useful for anlaysis

Item-pair - word1/word2 Basic adult vs. kid classification for initial models

pilot_data_filtered <- pilot_data_filtered %>%
  mutate(kid_or_adult = case_when(age == 'Adult' ~  'Adult',
                                  age != 'Adult' ~ 'Child',
                                  is.na(age) == TRUE ~ 'Child')) %>%
  mutate(item_pair = paste0(word1,'_', word2)) %>%
  mutate(sub_id = pid) 

pilot_data_experimental_filtered <- pilot_data_experimental_filtered %>%
  mutate(kid_or_adult = case_when(age == 'Adult' ~  'Adult',
                                  age != 'Adult' ~ 'Child',
                                  is.na(age) == TRUE ~ 'Child')) %>%
  mutate(item_pair = paste0(word1,'_', word2)) %>%
  mutate(sub_id = pid) 

Check # of subjects/trials by cohort

pilot_data_filtered %>%
  group_by(cohort) %>%
  summarize(num_subs = length(unique(pid)),total_trials = sum(length(trialId))) %>%
  kable()

cohort	num_subs	total_trials
cdm	301	9417
demo	719	29745
prolific	191	25785
school	1610	74661

Number of subjects who are adult in each cohort

pilot_data_filtered %>%
  filter(age == 'Adult') %>%
  group_by(cohort) %>%
  summarize(num_subs = length(unique(pid)),total_trials = sum(length(trialId))) %>%
  kable()

cohort	num_subs	total_trials
cdm	22	555
demo	679	28063
prolific	191	25785

Number of subjecst, age, and avg trials by cohort for kids

pilot_data_filtered %>%
  # filter(kid_or_adult == 'Child') %>%
  mutate(age_numeric = as.numeric(age)) %>%
  group_by(cohort,age) %>%
  summarize(num_subs = length(unique(pid)),avg_age = mean(age_numeric, na.rm=TRUE), total_trials = sum(length(trialId))) %>%
  kable()

cohort	age	num_subs	avg_age	total_trials
cdm	10+	16	NaN	513
cdm	2	51	2	1277
cdm	3	55	3	1867
cdm	4	48	4	1480
cdm	5	39	5	1368
cdm	6	32	6	1039
cdm	7	21	7	648
cdm	8	10	8	377
cdm	9	6	9	251
cdm	Adult	22	NaN	555
cdm	NA	1	NaN	42
demo	10+	14	NaN	630
demo	5	8	5	360
demo	6	1	6	45
demo	7	3	7	135
demo	8	2	8	60
demo	9	1	9	45
demo	Adult	679	NaN	28063
demo	NA	11	NaN	407
prolific	Adult	191	NaN	25785
school	NA	1610	NaN	74661

Only within school/prolifi

by_cohort_experimental <- pilot_data_experimental_filtered %>%
  group_by(cohort) %>%
  summarize(num_subs = length(unique(pid)),total_trials = sum(length(trialId)))

kid_vs_adult <- pilot_data_filtered %>%
  group_by(kid_or_adult) %>%
  summarize(num_subs = length(unique(pid)), total_trials = sum(length(trialId)))

Examine basic demographics in plots

Plot individual kids performance

by_kid <- pilot_data_filtered %>%
  group_by(pid, cohort) %>%
  summarize(prop_correct = mean(correct))

ggplot(data=by_kid, aes(x=cohort, y=prop_correct, color=cohort)) +
  geom_jitter(width=.2, height=.02, alpha=.4) +
  theme_few() +
  geom_hline(yintercept=.5)

Get filtered datasets for IRT modeling

Calculate which items are at ceiling for kicking out of models

Compute pc, # responses by each cohort

item_by_cohort <- pilot_data_filtered %>%
  group_by(kid_or_adult, item_pair) %>%
  summarize(num_responses = length(correct), pc = mean(correct))  %>%
  group_by(item_pair) %>%
  mutate(both_kids_and_adults = length(unique(kid_or_adult)))

Compute pc/num responses overall

item_all_responses <- pilot_data_filtered %>%
  group_by(item_pair) %>%
  summarize(num_responses = length(correct), pc = mean(correct)) 

Construct set of ‘bad items’ that we want to kick out in order to be able to compare item responses across cohorts

ceiling = item_by_cohort %>%
  filter(pc==1) 

not_enough_data = item_by_cohort %>%
  filter(both_kids_and_adults==1)  

not_enough_responses = item_by_cohort %>%
  filter(num_responses<50)  

bad_items <- ceiling %>%
  full_join(not_enough_responses %>% select(item_pair)) %>%
  full_join(not_enough_data %>% select(item_pair)) %>%
  distinct(item_pair)

There are 101 items that were eliminated because adults were at ceiling.

There are 0 items that were eliminated because we didn’t have data in both cohorts.

There are 123 items that were eliminated because we had less than 50 data point in a given cohort.

This led to a total of 157 UNIQUE items that were eliminated.

Set of included items for cohort comparison

items_included <- item_by_cohort %>%
  ungroup() %>%
  filter(!item_pair %in% bad_items$item_pair) %>%
  distinct(item_pair)
 

length(unique(items_included$item_pair))

## [1] 236

There are 236 items included across both cohorts that are not at ceiling.

Get data for IRT models

pilot_data_irt_subset <- pilot_data_filtered %>%
  ungroup() %>%
  filter(!item_pair %in% bad_items$item_pair) 

length(unique(pilot_data_irt_subset$item_pair))

## [1] 236

Construct matrixes for mdoeling

For kids with subset of items

d_wide_kid<- pilot_data_irt_subset %>%
  ungroup() %>%
  filter(kid_or_adult == 'Child') %>%
  select(sub_id, item_pair, correct) %>%
  arrange(item_pair) %>%
  ungroup() %>%
  pivot_wider(names_from=item_pair, values_from=correct, values_fn = ~mean(.x)) %>%
  ungroup()

d_mat_kid <- d_wide_kid %>%
  select(-sub_id) %>%
  data.frame %>%
  data.matrix 

rownames(d_mat_kid) <- d_wide_kid$sub_id

assertthat::assert_that(dim(d_mat_kid)[2]==length(items_included$item_pair))

## [1] TRUE

For adults with subset of items

d_wide_adult<- pilot_data_irt_subset %>%
  ungroup() %>%
  filter(kid_or_adult == 'Adult') %>%
  select(sub_id, item_pair, correct) %>%
  arrange(item_pair) %>%
  ungroup() %>%
  pivot_wider(names_from=item_pair, values_from=correct, values_fn = ~mean(.x)) %>%
  ungroup()

d_mat_adult <- d_wide_adult %>%
  select(-sub_id) %>%
  data.frame %>%
  data.matrix 

rownames(d_mat_adult) <- d_wide_adult$sub_id

assertthat::assert_that(dim(d_mat_adult)[2]==length(items_included$item_pair))

## [1] TRUE

For all data and items

d_wide_all_data_and_subs <- pilot_data_filtered %>%
  ungroup() %>%
  select(sub_id, item_pair, correct) %>%
  arrange(item_pair) %>%
  ungroup() %>%
  pivot_wider(names_from=item_pair, values_from=correct, values_fn = ~mean(.x)) %>%
  ungroup()

d_mat_all_data_and_subs <- d_wide_all_data_and_subs %>%
  select(-sub_id) %>%
  data.frame %>%
  data.matrix 

rownames(d_mat_all_data_and_subs) <- d_wide_all_data_and_subs$sub_id

# assertthat::assert_that(dim(d_mat_adult)[2]==length(items_included$item_pair))

IRT Models

Fit a Rasch model first to kids vs adults

mod_1pl_kid <- mirt::mirt(d_mat_kid, 1, itemtype='Rasch',guess=.5,  verbose=TRUE)

## 
Iteration: 1, Log-Lik: -28889.377, Max-Change: 9.37609
Iteration: 2, Log-Lik: -22000.348, Max-Change: 2.32462
Iteration: 3, Log-Lik: -21682.691, Max-Change: 1.70471
Iteration: 4, Log-Lik: -21549.511, Max-Change: 1.63326
Iteration: 5, Log-Lik: -21459.356, Max-Change: 1.36244
Iteration: 6, Log-Lik: -21388.191, Max-Change: 0.84180
Iteration: 7, Log-Lik: -21329.389, Max-Change: 0.41738
Iteration: 8, Log-Lik: -21280.613, Max-Change: 0.41190
Iteration: 9, Log-Lik: -21240.867, Max-Change: 0.57415
Iteration: 10, Log-Lik: -21209.313, Max-Change: 0.39874
Iteration: 11, Log-Lik: -21185.042, Max-Change: 0.64918
Iteration: 12, Log-Lik: -21166.870, Max-Change: 0.23597
Iteration: 13, Log-Lik: -21153.710, Max-Change: 0.82459
Iteration: 14, Log-Lik: -21144.441, Max-Change: 0.36367
Iteration: 15, Log-Lik: -21138.115, Max-Change: 0.51702
Iteration: 16, Log-Lik: -21133.895, Max-Change: 0.11862
Iteration: 17, Log-Lik: -21131.148, Max-Change: 0.25402
Iteration: 18, Log-Lik: -21129.395, Max-Change: 0.04464
Iteration: 19, Log-Lik: -21128.331, Max-Change: 1.24544
Iteration: 20, Log-Lik: -21127.638, Max-Change: 0.08023
Iteration: 21, Log-Lik: -21127.220, Max-Change: 0.05459
Iteration: 22, Log-Lik: -21126.688, Max-Change: 0.02901
Iteration: 23, Log-Lik: -21126.648, Max-Change: 0.01282
Iteration: 24, Log-Lik: -21126.644, Max-Change: 0.00879
Iteration: 25, Log-Lik: -21126.643, Max-Change: 0.01182
Iteration: 26, Log-Lik: -21126.649, Max-Change: 0.00420
Iteration: 27, Log-Lik: -21126.656, Max-Change: 0.00286
Iteration: 28, Log-Lik: -21126.661, Max-Change: 0.00276
Iteration: 29, Log-Lik: -21126.666, Max-Change: 0.00133
Iteration: 30, Log-Lik: -21126.668, Max-Change: 0.00084
Iteration: 31, Log-Lik: -21126.670, Max-Change: 0.00123
Iteration: 32, Log-Lik: -21126.673, Max-Change: 0.00044
Iteration: 33, Log-Lik: -21126.673, Max-Change: 0.00029
Iteration: 34, Log-Lik: -21126.674, Max-Change: 0.00022
Iteration: 35, Log-Lik: -21126.675, Max-Change: 0.00009

mod_1pl_adult <- mirt::mirt(d_mat_adult, 1, itemtype='Rasch',guess=.5,  verbose=TRUE)

## 
Iteration: 1, Log-Lik: -7387.302, Max-Change: 5.59298
Iteration: 2, Log-Lik: -6394.091, Max-Change: 3.76600
Iteration: 3, Log-Lik: -6353.737, Max-Change: 4.95714
Iteration: 4, Log-Lik: -6344.553, Max-Change: 0.32909
Iteration: 5, Log-Lik: -6340.239, Max-Change: 0.51562
Iteration: 6, Log-Lik: -6337.310, Max-Change: 0.09977
Iteration: 7, Log-Lik: -6333.911, Max-Change: 0.13192
Iteration: 8, Log-Lik: -6332.304, Max-Change: 0.10465
Iteration: 9, Log-Lik: -6330.934, Max-Change: 0.09068
Iteration: 10, Log-Lik: -6327.137, Max-Change: 0.25364
Iteration: 11, Log-Lik: -6326.141, Max-Change: 0.11176
Iteration: 12, Log-Lik: -6325.984, Max-Change: 0.06547
Iteration: 13, Log-Lik: -6325.720, Max-Change: 0.07460
Iteration: 14, Log-Lik: -6325.723, Max-Change: 0.03726
Iteration: 15, Log-Lik: -6325.747, Max-Change: 0.02517
Iteration: 16, Log-Lik: -6325.678, Max-Change: 0.05004
Iteration: 17, Log-Lik: -6325.750, Max-Change: 0.02196
Iteration: 18, Log-Lik: -6325.807, Max-Change: 0.01296
Iteration: 19, Log-Lik: -6325.838, Max-Change: 0.01236
Iteration: 20, Log-Lik: -6325.875, Max-Change: 0.00696
Iteration: 21, Log-Lik: -6325.897, Max-Change: 0.00494
Iteration: 22, Log-Lik: -6325.909, Max-Change: 0.00999
Iteration: 23, Log-Lik: -6325.942, Max-Change: 0.00445
Iteration: 24, Log-Lik: -6325.958, Max-Change: 0.00260
Iteration: 25, Log-Lik: -6325.967, Max-Change: 0.00230
Iteration: 26, Log-Lik: -6325.975, Max-Change: 0.00132
Iteration: 27, Log-Lik: -6325.980, Max-Change: 0.00100
Iteration: 28, Log-Lik: -6325.984, Max-Change: 0.00199
Iteration: 29, Log-Lik: -6325.991, Max-Change: 0.00082
Iteration: 30, Log-Lik: -6325.994, Max-Change: 0.00056
Iteration: 31, Log-Lik: -6325.996, Max-Change: 0.00045
Iteration: 32, Log-Lik: -6325.998, Max-Change: 0.00047
Iteration: 33, Log-Lik: -6325.999, Max-Change: 0.00023
Iteration: 34, Log-Lik: -6326.000, Max-Change: 0.00028
Iteration: 35, Log-Lik: -6326.001, Max-Change: 0.00017
Iteration: 36, Log-Lik: -6326.002, Max-Change: 0.00020
Iteration: 37, Log-Lik: -6326.002, Max-Change: 0.00023
Iteration: 38, Log-Lik: -6326.002, Max-Change: 0.00012
Iteration: 39, Log-Lik: -6326.003, Max-Change: 0.00011
Iteration: 40, Log-Lik: -6326.003, Max-Change: 0.00010

coefs_rasch_kid <- as_data_frame(coef(mod_1pl_kid, simplify = TRUE)$items) %>%
  mutate(item_pair = rownames(coef(mod_1pl_kid, simplify = TRUE)$items))

coefs_rasch_adults <- as_data_frame(coef(mod_1pl_adult, simplify = TRUE)$items) %>%
  mutate(item_pair = rownames(coef(mod_1pl_adult, simplify = TRUE)$items))

coefs <- coefs_rasch_adults  %>% 
  select(d, item_pair) %>%
  mutate(cohort = 'adults') %>%
  full_join(coefs_rasch_kid %>% select(d, item_pair) %>% mutate(cohort = 'kid')) %>%
  pivot_wider(names_from = cohort, values_from = d) %>%
  mutate(differences = adults-kid)

ggplot(data = coefs, aes(x=adults, y=kid)) +
  geom_point(alpha=.8) +
  geom_smooth(method='lm', color='grey') +
  theme_few() +
  ggrepel::geom_label_repel(aes(label = item_pair))

Compute correlation for difficulty – 1PL model

cor.test(coefs$adults, coefs$kid)

## 
##  Pearson's product-moment correlation
## 
## data:  coefs$adults and coefs$kid
## t = 15.328, df = 234, p-value < 2.2e-16
## alternative hypothesis: true correlation is not equal to 0
## 95 percent confidence interval:
##  0.6377760 0.7662657
## sample estimates:
##       cor 
## 0.7078279

Fit a 2PL model with some basic priors on difficulty/slopes

These models don’t converge without specifying some priors, not sure about these!

start.dim = length(colnames(d_wide_kid))-1
mm = (
  'F = 1-%d,
PRIOR = (1-%d, a1, norm, .2, 1),
PRIOR = (1-%d, d, norm, 0, 2)'
)
mm = mirt.model(sprintf(mm,start.dim,start.dim,start.dim))

mod_2pl_priors_kid <- mirt::mirt(d_mat_kid, mm, itemtype='2PL',guess=.5,  upper=1, verbose=TRUE)

## 
Iteration: 1, Log-Lik: -27726.870, Max-Change: 4.00859
Iteration: 2, Log-Lik: -22390.923, Max-Change: 1.25315
Iteration: 3, Log-Lik: -21984.948, Max-Change: 0.59355
Iteration: 4, Log-Lik: -21897.720, Max-Change: 0.28450
Iteration: 5, Log-Lik: -21879.529, Max-Change: 0.14795
Iteration: 6, Log-Lik: -21875.311, Max-Change: 0.08107
Iteration: 7, Log-Lik: -21874.225, Max-Change: 0.04673
Iteration: 8, Log-Lik: -21873.913, Max-Change: 0.03159
Iteration: 9, Log-Lik: -21873.816, Max-Change: 0.02214
Iteration: 10, Log-Lik: -21873.769, Max-Change: 0.00885
Iteration: 11, Log-Lik: -21873.760, Max-Change: 0.00413
Iteration: 12, Log-Lik: -21873.757, Max-Change: 0.00300
Iteration: 13, Log-Lik: -21873.755, Max-Change: 0.00056
Iteration: 14, Log-Lik: -21873.754, Max-Change: 0.00055
Iteration: 15, Log-Lik: -21873.754, Max-Change: 0.00261
Iteration: 16, Log-Lik: -21873.754, Max-Change: 0.00174
Iteration: 17, Log-Lik: -21873.754, Max-Change: 0.00026
Iteration: 18, Log-Lik: -21873.754, Max-Change: 0.00131
Iteration: 19, Log-Lik: -21873.754, Max-Change: 0.00021
Iteration: 20, Log-Lik: -21873.754, Max-Change: 0.00018
Iteration: 21, Log-Lik: -21873.754, Max-Change: 0.00085
Iteration: 22, Log-Lik: -21873.754, Max-Change: 0.00011
Iteration: 23, Log-Lik: -21873.754, Max-Change: 0.00056
Iteration: 24, Log-Lik: -21873.754, Max-Change: 0.00010

mod_2pl_priors_adult <- mirt::mirt(d_mat_adult, mm, itemtype='2PL',guess=.5,  upper=1, verbose=TRUE)

## 
Iteration: 1, Log-Lik: -15034.924, Max-Change: 10.76019
Iteration: 2, Log-Lik: -7800.818, Max-Change: 6.61617
Iteration: 3, Log-Lik: -7424.745, Max-Change: 0.73782
Iteration: 4, Log-Lik: -7340.682, Max-Change: 0.29459
Iteration: 5, Log-Lik: -7314.637, Max-Change: 0.19030
Iteration: 6, Log-Lik: -7304.752, Max-Change: 0.12414
Iteration: 7, Log-Lik: -7300.532, Max-Change: 0.07922
Iteration: 8, Log-Lik: -7298.616, Max-Change: 0.05502
Iteration: 9, Log-Lik: -7297.709, Max-Change: 0.03993
Iteration: 10, Log-Lik: -7296.980, Max-Change: 0.01975
Iteration: 11, Log-Lik: -7296.917, Max-Change: 0.01342
Iteration: 12, Log-Lik: -7296.886, Max-Change: 0.00819
Iteration: 13, Log-Lik: -7296.858, Max-Change: 0.00593
Iteration: 14, Log-Lik: -7296.857, Max-Change: 0.00092
Iteration: 15, Log-Lik: -7296.856, Max-Change: 0.00069
Iteration: 16, Log-Lik: -7296.856, Max-Change: 0.00054
Iteration: 17, Log-Lik: -7296.856, Max-Change: 0.00038
Iteration: 18, Log-Lik: -7296.856, Max-Change: 0.00031
Iteration: 19, Log-Lik: -7296.856, Max-Change: 0.00090
Iteration: 20, Log-Lik: -7296.856, Max-Change: 0.00022
Iteration: 21, Log-Lik: -7296.856, Max-Change: 0.00065
Iteration: 22, Log-Lik: -7296.856, Max-Change: 0.00030
Iteration: 23, Log-Lik: -7296.856, Max-Change: 0.00064
Iteration: 24, Log-Lik: -7296.856, Max-Change: 0.00022
Iteration: 25, Log-Lik: -7296.856, Max-Change: 0.00019
Iteration: 26, Log-Lik: -7296.856, Max-Change: 0.00007

Munge coefficeints for plotting

coefs_2pl_kid <- as_data_frame(coef(mod_2pl_priors_kid, simplify = TRUE)$items) %>%
  mutate(item_pair = rownames(coef(mod_2pl_priors_kid, simplify = TRUE)$items))

coefs_2pl_adults <- as_data_frame(coef(mod_2pl_priors_adult, simplify = TRUE)$items) %>%
  mutate(item_pair = rownames(coef(mod_2pl_priors_adult, simplify = TRUE)$items))

coefs_2pl <- coefs_2pl_adults  %>% 
  select(d, item_pair) %>%
  mutate(cohort = 'adults') %>%
  full_join(coefs_2pl_kid %>% select(d, item_pair) %>% mutate(cohort = 'kid')) %>%
  pivot_wider(names_from = cohort, values_from = d) %>%
  mutate(differences = adults-kid)

coefs_a1_2pl <- coefs_2pl_adults  %>% 
  select(a1, item_pair) %>%
  mutate(cohort = 'adults') %>%
  full_join(coefs_2pl_kid %>% select(a1, item_pair) %>% mutate(cohort = 'kid')) %>%
  pivot_wider(names_from = cohort, values_from = a1) %>%
  mutate(differences = adults-kid)

Plot adult vs. kids 2pl model: difficulty

ggplot(data = coefs_2pl, aes(x=adults, y=kid)) +
  geom_point(alpha=.8) +
  geom_smooth(method='lm', color='grey') +
  theme_few() +
  ggtitle('Adult vs. Kid - 2pl IRT model coefficients - difficulty') +
  ggrepel::geom_label_repel(aes(label = item_pair), max.overlaps=20)

### Compute correlation for difficulty

cor.test(coefs_2pl_adults$d, coefs_2pl_kid$d)

## 
##  Pearson's product-moment correlation
## 
## data:  coefs_2pl_adults$d and coefs_2pl_kid$d
## t = 12.337, df = 234, p-value < 2.2e-16
## alternative hypothesis: true correlation is not equal to 0
## 95 percent confidence interval:
##  0.5436467 0.6993978
## sample estimates:
##       cor 
## 0.6277652

Plot adult vs. kids 2pl model: discriminability

ggplot(data = coefs_a1_2pl, aes(x=adults, y=kid)) +
  geom_point(alpha=.8) +
  geom_smooth(method='lm', color='grey') +
  theme_few() +
  ggtitle('Adult vs. Kid - 2pl IRT model coefficients - discriminability (slopes)') +
  ggrepel::geom_label_repel(aes(label = item_pair), max.overlaps = 15)

Low discrim items for adults

ggplot(data = coefs_a1_2pl %>% filter(adults < 1), aes(x=adults, y=kid)) +
  geom_point(alpha=.8) +
  geom_smooth(method='lm', color='grey') +
  theme_few() +
  ggtitle('Low discrim items for adults)') +
  ggrepel::geom_label_repel(aes(label = item_pair), max.overlaps = 12)

ggplot(data = coefs_a1_2pl %>% filter(kid < 1), aes(x=adults, y=kid)) +
  geom_point(alpha=.8) +
  geom_smooth(method='lm', color='grey') +
  theme_few() +
  ggtitle('Low discrim items for kids') +
  ggrepel::geom_label_repel(aes(label = item_pair), max.overlaps = 12)

Compute correlation for discriminability

cor.test(coefs_2pl_adults$a1, coefs_2pl_kid$a1)

## 
##  Pearson's product-moment correlation
## 
## data:  coefs_2pl_adults$a1 and coefs_2pl_kid$a1
## t = 0.53428, df = 234, p-value = 0.5937
## alternative hypothesis: true correlation is not equal to 0
## 95 percent confidence interval:
##  -0.09321055  0.16188431
## sample estimates:
##        cor 
## 0.03490542

Look at difficulty vs. slopes in kids only

ggplot(data = coefs_2pl_kid, aes(d, a1)) + 
  geom_point(alpha=.5) + 
  ggrepel::geom_label_repel(aes(label = item_pair), max.overlaps=20, size=2) +
  theme_few(base_size=10) + 
  xlab("Kid IRT difficulty") +
  ylab("Kid IRT discriminability (slope)")

Look at difficulty vs. slopes in adults only

ggplot(data = coefs_2pl_adults, aes(d, a1)) + 
  geom_point(alpha=.5) + 
  ggrepel::geom_label_repel(aes(label = item_pair), max.overlaps=20, size=2) +
  theme_few(base_size=10) + 
  xlab("Adult IRT difficulty") +
  ylab("Adult IRT discriminability (slope)")

Histogram of difficult parameters in adults vs. kids

hist(coefs_2pl_adults$a1)

coefs_2pl_adults %>%
  filter(a1 < quantile(a1, .1)) %>%
  arrange(a1) %>%
  kable()

a1	d	g	u	item_pair
-0.3580626	4.0506084	0.5	1	hamster_rabbit
-0.1417250	2.3645302	0.5	1	blower_buggy
-0.1351852	3.4616448	0.5	1	otter_goose
-0.0720198	3.9495034	0.5	1	sandbag_valve
-0.0200448	3.8965188	0.5	1	milkshake_robot
-0.0069976	3.8679752	0.5	1	shower_toe
0.0117655	3.4648557	0.5	1	candlestick_egg
0.0402581	3.4716923	0.5	1	watermelon_strawberry
0.0426134	-3.7683261	0.5	1	gazelle_antelope
0.0499397	3.1729882	0.5	1	modem_scanner
0.0600082	3.4180637	0.5	1	screw_saw
0.0604040	3.9405595	0.5	1	locker_cabinet
0.1426777	2.7703056	0.5	1	turtle_frog
0.1733058	-0.9266288	0.5	1	biscuit_cookie
0.1844999	2.0274021	0.5	1	freezer_cooler
0.1919244	3.8984655	0.5	1	spinach_buckle
0.2040434	2.5951069	0.5	1	buffet_counter
0.2533247	2.4168397	0.5	1	towel_blanket
0.2659141	3.5108527	0.5	1	fence_railing
0.2713606	3.5597997	0.5	1	squirrel_eagle
0.3003819	3.1952968	0.5	1	coffee_vase
0.3106121	1.3245648	0.5	1	papaya_mango
0.3121030	3.4659911	0.5	1	corset_mantle
0.3385644	3.0331051	0.5	1	bamboo_lumber

coefs_2pl_kid %>%
  filter(a1 < quantile(a1, .1)) %>%
  arrange(a1) %>%
  kable()

a1	d	g	u	item_pair
-0.8916197	-0.8075449	0.5	1	kayak_canoe
-0.6968886	-2.1131271	0.5	1	gazelle_antelope
-0.2456996	-2.9373355	0.5	1	grate_crate
-0.1437426	-1.3546279	0.5	1	flan_amplifier
-0.1196014	-4.3253240	0.5	1	mulch_compost
-0.0474579	-2.4341081	0.5	1	tuxedo_suit
-0.0139838	-4.1472365	0.5	1	bobsled_sidecar
0.0702731	-0.6957066	0.5	1	pajamas_pants
0.1270106	-1.0531937	0.5	1	biscuit_cookie
0.2835022	-0.6774742	0.5	1	carousel_carriage
0.3173912	0.8907790	0.5	1	scoop_sauce
0.3298989	-0.2586631	0.5	1	gondola_trolley
0.3585657	-0.0517293	0.5	1	taillight_bumper
0.3681523	0.1039327	0.5	1	modem_baklava
0.3984940	-3.7095692	0.5	1	candlestick_candle
0.4174189	-0.0866390	0.5	1	artichoke_leek
0.4401009	1.1922746	0.5	1	cymbal_mallet
0.4415978	0.5740577	0.5	1	sorbet_palette
0.4584462	0.2034705	0.5	1	sorbet_tamale
0.4590157	3.7874154	0.5	1	fox_reindeer
0.4647155	-1.8293478	0.5	1	sauerkraut_potpourri
0.4667314	1.2687341	0.5	1	blower_buggy
0.4861686	0.1486342	0.5	1	modem_scanner
0.5012720	1.9398378	0.5	1	pajamas_cage

hist(coefs_2pl_kid$a1)

# Fit a model to all data – adults and kids – examine items

d_long_all<- pilot_data_filtered %>%
  ungroup() %>%
  select(sub_id, item_pair, correct) %>%
  arrange(item_pair)

d_wide_all<- d_long_all %>%
  ungroup() %>%
  pivot_wider(names_from=item_pair, values_from=correct, values_fn = ~mean(.x))

d_mat <- d_wide_all %>%
  select(-sub_id) %>%
  data.frame %>%
  data.matrix

rownames(d_mat) <- d_wide_all$sub_id

num_items = length(colnames(d_mat_all_data_and_subs))-1
mm = (
  'F = 1-%d
PRIOR = (1-%d, a1, norm, .2, 1),
PRIOR = (1-%d, d, norm, 0, 2)'
)
mm = mirt.model(sprintf(mm,num_items,num_items,num_items))

mod_2pl_priors_all <- mirt::mirt(d_mat_all_data_and_subs, mm, itemtype='2PL',guess=.5,  upper=1, verbose=TRUE)

## 
Iteration: 1, Log-Lik: -59920.360, Max-Change: 22.07329
Iteration: 2, Log-Lik: -44792.378, Max-Change: 12.10760
Iteration: 3, Log-Lik: -43101.621, Max-Change: 5.31141
Iteration: 4, Log-Lik: -42822.542, Max-Change: 2.92943
Iteration: 5, Log-Lik: -42726.146, Max-Change: 0.35862
Iteration: 6, Log-Lik: -42685.508, Max-Change: 0.17923
Iteration: 7, Log-Lik: -42665.606, Max-Change: 0.18411
Iteration: 8, Log-Lik: -42654.069, Max-Change: 0.12319
Iteration: 9, Log-Lik: -42646.951, Max-Change: 0.12744
Iteration: 10, Log-Lik: -42642.415, Max-Change: 0.10373
Iteration: 11, Log-Lik: -42639.483, Max-Change: 0.10473
Iteration: 12, Log-Lik: -42637.546, Max-Change: 0.09453
Iteration: 13, Log-Lik: -42634.118, Max-Change: 0.01896
Iteration: 14, Log-Lik: -42634.043, Max-Change: 0.00624
Iteration: 15, Log-Lik: -42634.000, Max-Change: 0.00598
Iteration: 16, Log-Lik: -42633.928, Max-Change: 0.00213
Iteration: 17, Log-Lik: -42633.924, Max-Change: 0.00193
Iteration: 18, Log-Lik: -42633.923, Max-Change: 0.00162
Iteration: 19, Log-Lik: -42633.922, Max-Change: 0.00121
Iteration: 20, Log-Lik: -42633.921, Max-Change: 0.00103
Iteration: 21, Log-Lik: -42633.921, Max-Change: 0.00076
Iteration: 22, Log-Lik: -42633.920, Max-Change: 0.00149
Iteration: 23, Log-Lik: -42633.920, Max-Change: 0.00101
Iteration: 24, Log-Lik: -42633.919, Max-Change: 0.00083
Iteration: 25, Log-Lik: -42633.919, Max-Change: 0.00081
Iteration: 26, Log-Lik: -42633.919, Max-Change: 0.00072
Iteration: 27, Log-Lik: -42633.919, Max-Change: 0.00071
Iteration: 28, Log-Lik: -42633.919, Max-Change: 0.00081
Iteration: 29, Log-Lik: -42633.919, Max-Change: 0.00069
Iteration: 30, Log-Lik: -42633.919, Max-Change: 0.00341
Iteration: 31, Log-Lik: -42633.918, Max-Change: 0.00391
Iteration: 32, Log-Lik: -42633.918, Max-Change: 0.00289
Iteration: 33, Log-Lik: -42633.918, Max-Change: 0.00212
Iteration: 34, Log-Lik: -42633.918, Max-Change: 0.00061
Iteration: 35, Log-Lik: -42633.918, Max-Change: 0.00302
Iteration: 36, Log-Lik: -42633.918, Max-Change: 0.00256
Iteration: 37, Log-Lik: -42633.918, Max-Change: 0.00056
Iteration: 38, Log-Lik: -42633.918, Max-Change: 0.00275
Iteration: 39, Log-Lik: -42633.918, Max-Change: 0.00194
Iteration: 40, Log-Lik: -42633.918, Max-Change: 0.00051
Iteration: 41, Log-Lik: -42633.918, Max-Change: 0.00250
Iteration: 42, Log-Lik: -42633.918, Max-Change: 0.00155
Iteration: 43, Log-Lik: -42633.918, Max-Change: 0.00046
Iteration: 44, Log-Lik: -42633.918, Max-Change: 0.00226
Iteration: 45, Log-Lik: -42633.918, Max-Change: 0.00132
Iteration: 46, Log-Lik: -42633.918, Max-Change: 0.00042
Iteration: 47, Log-Lik: -42633.918, Max-Change: 0.00205
Iteration: 48, Log-Lik: -42633.918, Max-Change: 0.00113
Iteration: 49, Log-Lik: -42633.918, Max-Change: 0.00038
Iteration: 50, Log-Lik: -42633.918, Max-Change: 0.00185
Iteration: 51, Log-Lik: -42633.918, Max-Change: 0.00100
Iteration: 52, Log-Lik: -42633.918, Max-Change: 0.00034
Iteration: 53, Log-Lik: -42633.918, Max-Change: 0.00167
Iteration: 54, Log-Lik: -42633.918, Max-Change: 0.00088
Iteration: 55, Log-Lik: -42633.918, Max-Change: 0.00031
Iteration: 56, Log-Lik: -42633.918, Max-Change: 0.00151
Iteration: 57, Log-Lik: -42633.918, Max-Change: 0.00078
Iteration: 58, Log-Lik: -42633.918, Max-Change: 0.00028
Iteration: 59, Log-Lik: -42633.918, Max-Change: 0.00136
Iteration: 60, Log-Lik: -42633.918, Max-Change: 0.00069
Iteration: 61, Log-Lik: -42633.918, Max-Change: 0.00025
Iteration: 62, Log-Lik: -42633.918, Max-Change: 0.00123
Iteration: 63, Log-Lik: -42633.918, Max-Change: 0.00062
Iteration: 64, Log-Lik: -42633.918, Max-Change: 0.00023
Iteration: 65, Log-Lik: -42633.918, Max-Change: 0.00111
Iteration: 66, Log-Lik: -42633.918, Max-Change: 0.00055
Iteration: 67, Log-Lik: -42633.918, Max-Change: 0.00020
Iteration: 68, Log-Lik: -42633.918, Max-Change: 0.00100
Iteration: 69, Log-Lik: -42633.918, Max-Change: 0.00049
Iteration: 70, Log-Lik: -42633.918, Max-Change: 0.00018
Iteration: 71, Log-Lik: -42633.918, Max-Change: 0.00090
Iteration: 72, Log-Lik: -42633.918, Max-Change: 0.00044
Iteration: 73, Log-Lik: -42633.918, Max-Change: 0.00017
Iteration: 74, Log-Lik: -42633.918, Max-Change: 0.00081
Iteration: 75, Log-Lik: -42633.918, Max-Change: 0.00039
Iteration: 76, Log-Lik: -42633.918, Max-Change: 0.00015
Iteration: 77, Log-Lik: -42633.918, Max-Change: 0.00073
Iteration: 78, Log-Lik: -42633.918, Max-Change: 0.00035
Iteration: 79, Log-Lik: -42633.918, Max-Change: 0.00013
Iteration: 80, Log-Lik: -42633.918, Max-Change: 0.00066
Iteration: 81, Log-Lik: -42633.918, Max-Change: 0.00032
Iteration: 82, Log-Lik: -42633.918, Max-Change: 0.00012
Iteration: 83, Log-Lik: -42633.918, Max-Change: 0.00059
Iteration: 84, Log-Lik: -42633.918, Max-Change: 0.00028
Iteration: 85, Log-Lik: -42633.918, Max-Change: 0.00011
Iteration: 86, Log-Lik: -42633.918, Max-Change: 0.00053
Iteration: 87, Log-Lik: -42633.918, Max-Change: 0.00026
Iteration: 88, Log-Lik: -42633.918, Max-Change: 0.00010

coefs_2pl_all <- as_data_frame(coef(mod_2pl_priors_all, simplify = TRUE)$items) %>%
  mutate(item_pair = rownames(coef(mod_2pl_priors_all, simplify = TRUE)$items))

Difficulty vs. slope in all

ggplot(data = coefs_2pl_all, aes(d, a1)) + 
  geom_point(alpha=.5) + 
  ggrepel::geom_label_repel(aes(label = item_pair), max.overlaps=20, size=2) +
  theme_few(base_size=10) + 
  xlab("All IRT difficulty") +
  ylab("All IRT discriminability (slope)")

Histogram of least informative items overall

##Items that are too easy

coefs_2pl_all %>%
  filter(d > quantile(d, .8)) %>%
  arrange(d) %>%
  kable()

a1	d	g	u	item_pair
1.1203001	3.039186	0.5	1	carrot_phone
1.0205581	3.062549	0.5	1	map_marker
2.1467630	3.073309	0.5	1	fence_mask
1.5448845	3.111613	0.5	1	squirrel_eagle
1.2415037	3.115967	0.5	1	whistle_wheelbarrow
1.7268206	3.118892	0.5	1	telescope_tripod
1.7444241	3.134305	0.5	1	net_tee
1.6959516	3.194275	0.5	1	clothespin_volleyball
1.5551431	3.199342	0.5	1	camp_to.steal
1.9647643	3.206000	0.5	1	spatula_lava
1.6070718	3.216424	0.5	1	swing_foot
1.3987770	3.229957	0.5	1	seagull_lobster
1.7347603	3.242474	0.5	1	oil_grain
1.2258122	3.283059	0.5	1	coffee_drink
1.3309650	3.293280	0.5	1	spinach_buckle
1.4450092	3.298177	0.5	1	sink_toilet
0.9217709	3.305032	0.5	1	cake_refrigerator
1.3510312	3.307258	0.5	1	shower_toe
1.6677665	3.317932	0.5	1	locker_basket
1.4533975	3.327922	0.5	1	fan_boulder
1.3161654	3.330067	0.5	1	teabag_fingerprint
1.6750615	3.349312	0.5	1	buffet_crib
1.5866861	3.367724	0.5	1	shirt_uniform
1.0626674	3.369071	0.5	1	lollipop_doorbell
0.9764287	3.371203	0.5	1	locker_cabinet
1.9597976	3.377810	0.5	1	wrench_swimsuit
1.3639858	3.389018	0.5	1	teapot_brake
1.3288054	3.406742	0.5	1	rice_dice
1.3291047	3.422468	0.5	1	blender_mailbox
1.1513720	3.429997	0.5	1	shirt_salad
1.1750671	3.453451	0.5	1	carrot_vegetable
1.0491233	3.460033	0.5	1	footbath_trough
0.7160676	3.477152	0.5	1	watermelon_strawberry
1.3828659	3.490081	0.5	1	milkshake_robot
1.0277711	3.530527	0.5	1	sunflower_pineapple
1.0065883	3.536066	0.5	1	footbath_vest
1.5639530	3.547836	0.5	1	hedgehog_owl
1.4778208	3.554809	0.5	1	fence_railing
1.3001878	3.563968	0.5	1	fan_album
1.6168012	3.581453	0.5	1	seaweed_mousetrap
1.5930106	3.585259	0.5	1	trumpet_violin
0.2188665	3.594940	0.5	1	fox_reindeer
1.9296478	3.599647	0.5	1	swordfish_leopard
1.3595483	3.601908	0.5	1	turkey_swan
1.2924996	3.608499	0.5	1	typewriter_sunglasses
1.3253560	3.642301	0.5	1	potato_pot
1.5271683	3.643151	0.5	1	acorn_coconut
0.7870748	3.675282	0.5	1	koala_bee
1.0731518	3.678426	0.5	1	rice_box
1.0402096	3.688370	0.5	1	freezer_nest
1.6416021	3.723482	0.5	1	hamster_tadpole
2.1143866	3.784440	0.5	1	honey_baby
1.0990422	3.793780	0.5	1	sunflower_rim
1.3109265	3.815503	0.5	1	telescope_windshield
1.3319211	3.817023	0.5	1	oatmeal_collar
1.3402391	3.817379	0.5	1	potato_glasses
1.9762571	3.829459	0.5	1	ski_suitcase
1.1913219	3.832402	0.5	1	cake_dessert
1.2518804	3.856287	0.5	1	triangle_diamond
1.4197672	3.870667	0.5	1	sprinkler_flower
1.1054382	3.925262	0.5	1	snail_cow
1.7991806	3.931746	0.5	1	chandelier_lightbulb
1.4139813	3.931813	0.5	1	cornbread_wreath
1.8130951	3.938239	0.5	1	turtle_horse
1.8128913	3.940805	0.5	1	net_domino
1.1234225	3.970497	0.5	1	ship_nose
1.6990732	3.998613	0.5	1	lollipop_candy
1.8233502	4.002080	0.5	1	treasure_rope
1.3561816	4.032068	0.5	1	marshmallow_dryer
1.8903201	4.046975	0.5	1	sink_stair
1.5242990	4.051177	0.5	1	acorn_key
1.9342543	4.082466	0.5	1	tongue_lipstick
1.3385574	4.100723	0.5	1	cheese_mud
1.1227112	4.103345	0.5	1	tongue_envelope
1.4884980	4.127945	0.5	1	pie_flashlight
1.8660351	4.386784	0.5	1	turkey_goat
1.6089964	4.408481	0.5	1	triangle_lighter
1.6823316	4.519456	0.5	1	elbow_bed
1.4439446	4.612319	0.5	1	watermelon_screwdriver

Items that don’t discriminate

bad_discrim <- coefs_2pl_all %>%
  filter(a1 < quantile(a1, .1)) %>%
  arrange(a1)

bad_discrim %>%
  kable() 

a1	d	g	u	item_pair
-0.8699345	-2.0904604	0.5	1	aversion_neutral
-0.7206162	-2.9797699	0.5	1	gazelle_antelope
-0.1492750	-0.6611290	0.5	1	resuscitation_pipetting
0.0000000	1.8425318	0.5	1	wrench_wreck
0.0717524	-3.0730188	0.5	1	skimmer_strainer
0.1213419	-0.4663131	0.5	1	tourniquet_scalpel
0.1651480	-0.3751178	0.5	1	concentric_on.each.other
0.1745380	0.2004710	0.5	1	arcade_high.voltage
0.2064152	-1.0469667	0.5	1	biscuit_cookie
0.2118420	-0.4727689	0.5	1	saffron_clove
0.2187734	0.2744293	0.5	1	incensed_smile
0.2188665	3.5949395	0.5	1	fox_reindeer
0.2419179	-1.2077390	0.5	1	tumble_scythes
0.2790509	-0.4993120	0.5	1	incensed_tired
0.2817110	-3.1278124	0.5	1	bust_monument
0.2818925	2.5903401	0.5	1	camp_Paint
0.2966475	-0.0508415	0.5	1	resuscitation_measure.volts
0.3518980	1.4966166	0.5	1	milkshake_milk
0.3925259	1.2368085	0.5	1	figurehead_re.cover.plate
0.4309190	-3.4201064	0.5	1	urban_village
0.4765097	-4.0369300	0.5	1	precarious_taiqi
0.4797841	-3.6592079	0.5	1	urban_rural
0.5159196	1.0069757	0.5	1	papaya_mango
0.5208491	0.5185959	0.5	1	percussion_cello
0.5386545	2.4925473	0.5	1	camp_march
0.5521554	2.4729046	0.5	1	marshmallow_snowball
0.5588526	-0.1093249	0.5	1	concentric_one.above.the.other
0.5757777	-3.8916502	0.5	1	colony_herd
0.5805519	-3.6535689	0.5	1	colony_pack
0.5985945	1.3012308	0.5	1	squash_pumpkin
0.6146388	2.1709332	0.5	1	turtle_frog
0.6218005	0.2426835	0.5	1	skimmer_spatula
0.6230445	1.6685440	0.5	1	blower_buggy
0.6240067	-2.0771457	0.5	1	saffron_star.anise
0.6262345	-1.6480280	0.5	1	timid_excited
0.6406467	-0.9816086	0.5	1	triad_quartet
0.6536960	1.6647980	0.5	1	pie_pizza
0.6650648	0.5702578	0.5	1	beret_safari.hat
0.6662257	-0.2261989	0.5	1	tourniquet_wiper
0.6702478	2.0115457	0.5	1	elbow_arm

high_discrim <- coefs_2pl_all %>%
  filter(a1 > quantile(a1, .9)) %>%
  arrange(a1)

high_discrim %>%
  kable() 

a1	d	g	u	item_pair
2.056851	0.0665910	0.5	1	gutter_filter
2.072449	1.1653786	0.5	1	silverware_trophy
2.074499	0.9043242	0.5	1	antenna_stem
2.076460	0.6196242	0.5	1	kimono_turntable
2.092832	-1.0290070	0.5	1	sauerkraut_potpourri
2.114387	3.7844402	0.5	1	honey_baby
2.123313	-0.1235945	0.5	1	artifact_crystal
2.128908	1.6355601	0.5	1	saddle_handle
2.129147	1.9367702	0.5	1	stew_mixer
2.145904	-1.0070078	0.5	1	freezer_cooler
2.146763	3.0733088	0.5	1	fence_mask
2.160760	1.5120460	0.5	1	foam_float
2.170701	-0.8520996	0.5	1	turbine_generator
2.188677	0.7727513	0.5	1	scaffolding_veil
2.193455	1.9722460	0.5	1	stump_bookshelf
2.218671	2.1491016	0.5	1	bark_skin
2.225930	-0.7077772	0.5	1	stew_soup
2.252611	1.5846038	0.5	1	thermos_firewood
2.293179	-0.9080598	0.5	1	cheese_butter
2.296870	2.7299832	0.5	1	turbine_tab
2.298384	0.2660658	0.5	1	scrabble_poker
2.349732	0.7352558	0.5	1	parsley_tiara
2.355767	-0.8850925	0.5	1	flan_fuse
2.372047	0.2943802	0.5	1	stump_log
2.520730	0.1721156	0.5	1	bouquet_bandanna
2.537951	-1.3023605	0.5	1	coaster_painting
2.568831	-1.3692457	0.5	1	tuxedo_suit
2.610815	0.0632070	0.5	1	grate_reel
2.648662	-0.8351806	0.5	1	flan_amplifier
2.724403	-1.0475319	0.5	1	corset_harness
2.726947	-0.0796049	0.5	1	bouquet_centerpiece
2.728923	0.6562987	0.5	1	prism_radar
2.818669	-0.7557641	0.5	1	mulch_clarinet
2.879336	-2.4651821	0.5	1	bobsled_sidecar
2.917648	-0.5656942	0.5	1	scrabble_whisk
2.988333	-1.2470131	0.5	1	thermos_oilcan
3.030575	-0.8767999	0.5	1	aloe_bracket
3.210406	-0.8993710	0.5	1	pitcher_batter
3.314044	-1.6248238	0.5	1	pitcher_tumbleweed
3.315042	-0.5283748	0.5	1	scaffolding_satellite

Look at individual item curves

thetas <- seq(-6,6,.1)

irt2pl <- function(a, d, theta = seq(-6,6,.1)) {
  p = boot::inv.logit(a * (theta + d))
  return(p)
}

Plot items with best discrimination

iccs <- coefs_2pl_all %>%
  filter(a1 > quantile(a1, .9)) %>%
  split(.$item_pair) %>%
  map_df(function(d) {
    return(data_frame(item_pair = d$item_pair,
                      theta = thetas, 
                      p = irt2pl(d$a1, d$d, thetas)))
  })

ggplot(iccs,  
       aes(x = theta, y = p)) + 
  geom_line() + 
  facet_wrap(~item_pair) + 
  xlab("Ability") + 
  ylab("Probability of comprehension") +
  ggtitle('Items with best discrimination (all data)')

Plot items with worst discrimination

iccs <- coefs_2pl_all %>%
  filter(a1 < quantile(a1, .1)) %>%
  split(.$item_pair) %>%
  map_df(function(d) {
    return(data_frame(item_pair = d$item_pair,
                      theta = thetas, 
                      p = irt2pl(d$a1, d$d, thetas)))
  })

ggplot(iccs,  
       aes(x = theta, y = p)) + 
  geom_line() + 
  facet_wrap(~item_pair) + 
  xlab("Ability") + 
  ylab("Probability of comprehension") +
  ggtitle('Items with worst discrimination (all data)')

Plot items with highest difficulty

iccs <- coefs_2pl_all %>%
  filter(d < quantile(d, .1)) %>%
  split(.$item_pair) %>%
  map_df(function(d) {
    return(data_frame(item_pair = d$item_pair,
                      theta = thetas, 
                      p = irt2pl(d$a1, d$d, thetas)))
  })

ggplot(iccs,  
       aes(x = theta, y = p)) + 
  geom_line() + 
  facet_wrap(~item_pair) + 
  xlab("Ability") + 
  ylab("Probability of comprehension") +
  ggtitle('Most difficult items (all data)')

Plot items with lowest difficulty

iccs <- coefs_2pl_all %>%
  filter(d > quantile(d, .9)) %>%
  split(.$item_pair) %>%
  map_df(function(d) {
    return(data_frame(item_pair = d$item_pair,
                      theta = thetas, 
                      p = irt2pl(d$a1, d$d, thetas)))
  })

ggplot(iccs,  
       aes(x = theta, y = p)) + 
  geom_line() + 
  facet_wrap(~item_pair) + 
  xlab("Ability") + 
  ylab("Probability of comprehension") +
  ggtitle('Easiest items (all data)')

Item fit statistics

Tried computing many of these, but they don’t finish in <1 hour, so moving on to new techniques for now..

mod_1pl_all <- mirt::mirt(d_mat_all_data_and_subs, 1, itemtype='Rasch',guess=.5,  verbose=TRUE)

## 
Iteration: 1, Log-Lik: -55138.394, Max-Change: 6.08098
Iteration: 2, Log-Lik: -42500.365, Max-Change: 2.72005
Iteration: 3, Log-Lik: -42064.219, Max-Change: 2.00271
Iteration: 4, Log-Lik: -41922.360, Max-Change: 1.11186
Iteration: 5, Log-Lik: -41831.802, Max-Change: 0.59453
Iteration: 6, Log-Lik: -41758.739, Max-Change: 0.46886
Iteration: 7, Log-Lik: -41695.557, Max-Change: 0.33224
Iteration: 8, Log-Lik: -41639.772, Max-Change: 0.30415
Iteration: 9, Log-Lik: -41590.325, Max-Change: 0.27279
Iteration: 10, Log-Lik: -41546.842, Max-Change: 0.24187
Iteration: 11, Log-Lik: -41508.912, Max-Change: 0.21156
Iteration: 12, Log-Lik: -41476.210, Max-Change: 0.18383
Iteration: 13, Log-Lik: -41448.346, Max-Change: 0.15871
Iteration: 14, Log-Lik: -41424.929, Max-Change: 0.13499
Iteration: 15, Log-Lik: -41405.470, Max-Change: 0.11432
Iteration: 16, Log-Lik: -41389.531, Max-Change: 0.09576
Iteration: 17, Log-Lik: -41376.614, Max-Change: 0.07956
Iteration: 18, Log-Lik: -41366.265, Max-Change: 0.06515
Iteration: 19, Log-Lik: -41358.057, Max-Change: 0.08740
Iteration: 20, Log-Lik: -41351.615, Max-Change: 0.06423
Iteration: 21, Log-Lik: -41346.582, Max-Change: 0.09052
Iteration: 22, Log-Lik: -41342.683, Max-Change: 0.05802
Iteration: 23, Log-Lik: -41339.688, Max-Change: 0.04770
Iteration: 24, Log-Lik: -41337.394, Max-Change: 0.06835
Iteration: 25, Log-Lik: -41330.152, Max-Change: 0.03736
Iteration: 26, Log-Lik: -41330.322, Max-Change: 0.02022
Iteration: 27, Log-Lik: -41330.338, Max-Change: 0.02056
Iteration: 28, Log-Lik: -41330.327, Max-Change: 2.51042
Iteration: 29, Log-Lik: -41330.305, Max-Change: 0.01924
Iteration: 30, Log-Lik: -41330.291, Max-Change: 0.00296
Iteration: 31, Log-Lik: -41330.259, Max-Change: 0.00194
Iteration: 32, Log-Lik: -41330.256, Max-Change: 0.00294
Iteration: 33, Log-Lik: -41330.251, Max-Change: 0.00126
Iteration: 34, Log-Lik: -41330.235, Max-Change: 0.00137
Iteration: 35, Log-Lik: -41330.235, Max-Change: 0.00079
Iteration: 36, Log-Lik: -41330.234, Max-Change: 0.00092
Iteration: 37, Log-Lik: -41330.231, Max-Change: 0.00024
Iteration: 38, Log-Lik: -41330.231, Max-Change: 0.00022
Iteration: 39, Log-Lik: -41330.231, Max-Change: 0.00022
Iteration: 40, Log-Lik: -41330.231, Max-Change: 0.00028
Iteration: 41, Log-Lik: -41330.231, Max-Change: 0.00010
Iteration: 42, Log-Lik: -41330.230, Max-Change: 0.00022
Iteration: 43, Log-Lik: -41330.230, Max-Change: 0.00019
Iteration: 44, Log-Lik: -41330.231, Max-Change: 0.00012
Iteration: 45, Log-Lik: -41330.231, Max-Change: 0.00012
Iteration: 46, Log-Lik: -41330.230, Max-Change: 0.00000

thetas <- fscores(mod_1pl_all, method = 'ML')
itemfit = itemfit(mod_1pl_all, fit_stats = 'X2',  Theta=thetas)

itemfit$X2

##   [1]     2.122096     2.074947    28.352481     8.811092    11.885934
##   [6]    11.393951    12.231098   169.397168    67.587278    16.326175
##  [11]     6.194394    14.873031     6.159223     5.354200   137.753259
##  [16]   149.578820    20.891082    15.058188     6.590762     4.323156
##  [21]    10.661175    10.262713     6.533685     4.387604     5.673304
##  [26]     6.460858     2.367236    25.116336     3.818333   310.732038
##  [31] 64341.830255    14.571343     2.744659   302.079652     4.668664
##  [36]     7.683020    31.924059    11.379727    26.952682     9.750262
##  [41]     4.203866     3.969692     3.914226    70.374739    36.111702
##  [46]    15.156828     4.793786     8.557571     6.710239    13.461854
##  [51]     3.270946    54.310341    15.061703     8.795150     5.490768
##  [56]    26.095003     7.063954     5.315550     2.433910     4.277679
##  [61]     2.854680     4.301741     5.268698     6.069480     3.092666
##  [66]     9.921766     8.987431     7.290132     8.988921    12.568101
##  [71]    18.941980     8.913473    10.102898    67.477318    12.950927
##  [76]    13.341221    40.634376    18.462334     4.143532  8471.678375
##  [81]    27.828045    85.864152     1.682123     8.329425     6.308595
##  [86]    79.342375    47.635180     6.447017     2.990803     2.825690
##  [91]    12.809659     2.930994    28.673769     6.428003    49.514119
##  [96]    12.023278    17.223434     5.364765  7744.229374     3.444129
## [101]    87.756451    71.312838     2.614695   223.495071   116.946236
## [106]    35.991559    12.559925     8.664302     8.427482     5.986889
## [111]    55.885965    29.963301    48.059037    18.179691     6.360001
## [116]     7.320297    15.024601     4.533598     5.778255    44.513071
## [121]    12.595663    15.115490     7.809380     4.468007     8.372026
## [126]    37.220359     5.320351     6.483413    12.374303     4.995082
## [131]     6.703915     6.440270    12.345771     8.853811     4.707423
## [136]     6.087913    18.279852    96.835617   824.344046  2761.418693
## [141]     3.511675     3.181513    11.327945    15.152437     3.966397
## [146]     6.681607     3.410950     3.141864     3.346565    15.679370
## [151]    14.118051   119.013394     2.759963    20.740517   121.666236
## [156]    51.678809     5.944359   191.034930    34.242307    50.581679
## [161]     5.678054     3.643137     5.958762    11.714372    10.794789
## [166]     1.867047     8.027195     6.375145    14.700498   194.014457
## [171]    82.711327     9.305921     2.431495     4.287210     4.818196
## [176]     5.847159    15.747678    16.826850     2.212213    72.654701
## [181]     6.039565    18.017707    20.943339    61.945149     9.567278
## [186]     9.591138    15.215888   172.997560     8.789619     5.882990
## [191]    12.927365    42.289659   765.866656     5.642937     4.104802
## [196]     2.392690    17.987047     3.775431     9.954242     2.563166
## [201]    13.423429    13.214853    51.415414    48.068321    16.634186
## [206]    12.569562     4.488021   117.873341    23.694735    11.417285
## [211]     7.912573     9.241557     5.841957   255.533967   461.126596
## [216]     5.359092    34.854182    20.210709    14.157829    20.630264
## [221]    32.510883     6.116607    24.643292    87.350258     2.747519
## [226]     4.843826    33.027063    36.724368    68.979631    28.122764
## [231]    23.708779     5.940187    15.959374     3.324476     6.523051
## [236]     6.668542     4.592877     3.506067     7.582451     3.432709
## [241]    13.287679    16.633163    47.325472    25.592615    16.507085
## [246]    13.649386     7.695213     7.634685   157.956799     7.596612
## [251]     8.295404     5.887937     4.051964    11.677740    79.353709
## [256]   427.234641   486.779890    16.026964     3.055802     8.308069
## [261]    59.526517    76.647211    73.923948    28.778486     6.102484
## [266]    10.777564     4.467430     7.093745    12.565023    40.540311
## [271]     7.986180     1.402318   552.869884     3.122629    11.047102
## [276]    15.356881    40.018739     5.088270  5035.967077     3.675719
## [281]     8.633687     3.820147     3.955670     5.095675     5.858978
## [286]    11.192403     4.624221    12.423953     3.312057    13.544600
## [291]     8.205291     2.344692   266.523551     9.741386     7.722953
## [296]    60.564099    17.407595     6.154674     5.380762    25.876299
## [301]     1.608552     6.926346     9.834860    12.321894     4.262087
## [306]    38.074974     3.092335     5.264781     1.983663    25.444393
## [311]     5.819172    62.105919     7.592905   410.245194    16.361289
## [316]     8.257281   401.793194     7.485002     6.768996     4.409889
## [321]     2.880642     9.131532     7.093383    10.959981     8.954943
## [326]     5.531176     5.923968     1.655589    26.802676    39.857108
## [331]     3.635188    14.145116     3.548260    41.072647     6.646177
## [336]     8.600885     5.922302     3.553813     8.093463     4.561042
## [341]    11.185965     3.505933     1.844214     6.319642    33.912246
## [346]     3.136027    69.755886   273.719319    14.588769     3.577993
## [351]     9.394422   137.469383  2137.161392   306.859083     5.321720
## [356]    10.413903     3.936144 30807.839570    22.658591   102.216611
## [361]     1.376914     1.625236     6.688565     8.089121     4.724926
## [366]    16.434710     4.107461   218.663294    17.051923    12.514884
## [371]     5.657884     3.079879     6.010105    40.560044     2.036050
## [376]     3.786751    12.150778     5.155424     1.935079    91.747736
## [381]    22.311254    18.513903     2.030868    39.605179     4.483405
## [386]     3.816822    10.404416     6.748160     3.735073    40.161532
## [391]     3.269217     5.302776    25.982237

bad_items <- itemfit %>%
  filter(X2>quantile(X2,.9)) %>%
  arrange(-X2)

Plot items with high X2 values

iccs <- coefs_2pl_all %>%
  filter(item_pair %in% bad_items$item) %>%
  split(.$item_pair) %>%
  map_df(function(d) {
    return(data_frame(item_pair = d$item_pair,
                      theta = thetas, 
                      p = irt2pl(d$a1, d$d, thetas)))
  })

ggplot(iccs,  
       aes(x = theta, y = p)) + 
  geom_line() + 
  facet_wrap(~item_pair) + 
  xlab("Ability") + 
  ylab("Probability of comprehension") +
  ggtitle('Easiest items (all data)')

Try iteratively fitting 2PL model

Fit 2 parameter IRT model with guess rate of 0.5

outliers <- TRUE
maxiter <- 20
iteration <- 0
df_good <- d_mat
aminmax <- c(.7, Inf) # Keep items with positive slopes between .7 and 1

# Model priors specified in loop

while (outliers > 0 & iteration<maxiter){
  iteration <- iteration + 1;
  start.dim <- dim(df_good)[2]-1
  mm = (
  'F = 1-%d,
  PRIOR = (1-%d, a1, norm, .2, 1),
  PRIOR = (1-%d, d, norm, 0, 2)'
  )
  mm = mirt.model(sprintf(mm,start.dim,start.dim,start.dim))
  m <- mirt(df_good, model = mm,itemtype = '2PL',guess=0.5) # 2AFC. Guess Rate = 0.5
  co <- coef(m,simplify=TRUE, IRTpars = TRUE) # Get coeeficients
  co <- tibble::rownames_to_column(as.data.frame(co$items),'words')
  ggplot(co, aes(a, b)) + geom_point(size=3)
  ggsave(sprintf('2PL-ModelParams_%d.png',iteration))
  # Remove items with low or extreme slope and refit
  df_good <- d_mat[,which(co$a>aminmax[1] & co$a<aminmax[2])]
  end.dim <- dim(df_good)[2]-1
  outliers <- sum(!(co$a>aminmax[1] & co$a<aminmax[2]))
  print(sprintf('2PL ITERATION %d. STARTED WITH %d ITEMS. %d OUTLIERS REMOVED. %d ITEMS RETAINED.',iteration,start.dim,outliers,end.dim))
}

## 
Iteration: 1, Log-Lik: -59920.360, Max-Change: 22.07329
Iteration: 2, Log-Lik: -44792.378, Max-Change: 12.10760
Iteration: 3, Log-Lik: -43101.621, Max-Change: 5.31141
Iteration: 4, Log-Lik: -42822.542, Max-Change: 2.92943
Iteration: 5, Log-Lik: -42726.146, Max-Change: 0.35862
Iteration: 6, Log-Lik: -42685.508, Max-Change: 0.17923
Iteration: 7, Log-Lik: -42665.606, Max-Change: 0.18411
Iteration: 8, Log-Lik: -42654.069, Max-Change: 0.12319
Iteration: 9, Log-Lik: -42646.951, Max-Change: 0.12744
Iteration: 10, Log-Lik: -42642.415, Max-Change: 0.10373
Iteration: 11, Log-Lik: -42639.483, Max-Change: 0.10473
Iteration: 12, Log-Lik: -42637.546, Max-Change: 0.09453
Iteration: 13, Log-Lik: -42634.118, Max-Change: 0.01896
Iteration: 14, Log-Lik: -42634.043, Max-Change: 0.00624
Iteration: 15, Log-Lik: -42634.000, Max-Change: 0.00598
Iteration: 16, Log-Lik: -42633.928, Max-Change: 0.00213
Iteration: 17, Log-Lik: -42633.924, Max-Change: 0.00193
Iteration: 18, Log-Lik: -42633.923, Max-Change: 0.00162
Iteration: 19, Log-Lik: -42633.922, Max-Change: 0.00121
Iteration: 20, Log-Lik: -42633.921, Max-Change: 0.00103
Iteration: 21, Log-Lik: -42633.921, Max-Change: 0.00076
Iteration: 22, Log-Lik: -42633.920, Max-Change: 0.00149
Iteration: 23, Log-Lik: -42633.920, Max-Change: 0.00101
Iteration: 24, Log-Lik: -42633.919, Max-Change: 0.00083
Iteration: 25, Log-Lik: -42633.919, Max-Change: 0.00081
Iteration: 26, Log-Lik: -42633.919, Max-Change: 0.00072
Iteration: 27, Log-Lik: -42633.919, Max-Change: 0.00071
Iteration: 28, Log-Lik: -42633.919, Max-Change: 0.00081
Iteration: 29, Log-Lik: -42633.919, Max-Change: 0.00069
Iteration: 30, Log-Lik: -42633.919, Max-Change: 0.00341
Iteration: 31, Log-Lik: -42633.918, Max-Change: 0.00391
Iteration: 32, Log-Lik: -42633.918, Max-Change: 0.00289
Iteration: 33, Log-Lik: -42633.918, Max-Change: 0.00212
Iteration: 34, Log-Lik: -42633.918, Max-Change: 0.00061
Iteration: 35, Log-Lik: -42633.918, Max-Change: 0.00302
Iteration: 36, Log-Lik: -42633.918, Max-Change: 0.00256
Iteration: 37, Log-Lik: -42633.918, Max-Change: 0.00056
Iteration: 38, Log-Lik: -42633.918, Max-Change: 0.00275
Iteration: 39, Log-Lik: -42633.918, Max-Change: 0.00194
Iteration: 40, Log-Lik: -42633.918, Max-Change: 0.00051
Iteration: 41, Log-Lik: -42633.918, Max-Change: 0.00250
Iteration: 42, Log-Lik: -42633.918, Max-Change: 0.00155
Iteration: 43, Log-Lik: -42633.918, Max-Change: 0.00046
Iteration: 44, Log-Lik: -42633.918, Max-Change: 0.00226
Iteration: 45, Log-Lik: -42633.918, Max-Change: 0.00132
Iteration: 46, Log-Lik: -42633.918, Max-Change: 0.00042
Iteration: 47, Log-Lik: -42633.918, Max-Change: 0.00205
Iteration: 48, Log-Lik: -42633.918, Max-Change: 0.00113
Iteration: 49, Log-Lik: -42633.918, Max-Change: 0.00038
Iteration: 50, Log-Lik: -42633.918, Max-Change: 0.00185
Iteration: 51, Log-Lik: -42633.918, Max-Change: 0.00100
Iteration: 52, Log-Lik: -42633.918, Max-Change: 0.00034
Iteration: 53, Log-Lik: -42633.918, Max-Change: 0.00167
Iteration: 54, Log-Lik: -42633.918, Max-Change: 0.00088
Iteration: 55, Log-Lik: -42633.918, Max-Change: 0.00031
Iteration: 56, Log-Lik: -42633.918, Max-Change: 0.00151
Iteration: 57, Log-Lik: -42633.918, Max-Change: 0.00078
Iteration: 58, Log-Lik: -42633.918, Max-Change: 0.00028
Iteration: 59, Log-Lik: -42633.918, Max-Change: 0.00136
Iteration: 60, Log-Lik: -42633.918, Max-Change: 0.00069
Iteration: 61, Log-Lik: -42633.918, Max-Change: 0.00025
Iteration: 62, Log-Lik: -42633.918, Max-Change: 0.00123
Iteration: 63, Log-Lik: -42633.918, Max-Change: 0.00062
Iteration: 64, Log-Lik: -42633.918, Max-Change: 0.00023
Iteration: 65, Log-Lik: -42633.918, Max-Change: 0.00111
Iteration: 66, Log-Lik: -42633.918, Max-Change: 0.00055
Iteration: 67, Log-Lik: -42633.918, Max-Change: 0.00020
Iteration: 68, Log-Lik: -42633.918, Max-Change: 0.00100
Iteration: 69, Log-Lik: -42633.918, Max-Change: 0.00049
Iteration: 70, Log-Lik: -42633.918, Max-Change: 0.00018
Iteration: 71, Log-Lik: -42633.918, Max-Change: 0.00090
Iteration: 72, Log-Lik: -42633.918, Max-Change: 0.00044
Iteration: 73, Log-Lik: -42633.918, Max-Change: 0.00017
Iteration: 74, Log-Lik: -42633.918, Max-Change: 0.00081
Iteration: 75, Log-Lik: -42633.918, Max-Change: 0.00039
Iteration: 76, Log-Lik: -42633.918, Max-Change: 0.00015
Iteration: 77, Log-Lik: -42633.918, Max-Change: 0.00073
Iteration: 78, Log-Lik: -42633.918, Max-Change: 0.00035
Iteration: 79, Log-Lik: -42633.918, Max-Change: 0.00013
Iteration: 80, Log-Lik: -42633.918, Max-Change: 0.00066
Iteration: 81, Log-Lik: -42633.918, Max-Change: 0.00032
Iteration: 82, Log-Lik: -42633.918, Max-Change: 0.00012
Iteration: 83, Log-Lik: -42633.918, Max-Change: 0.00059
Iteration: 84, Log-Lik: -42633.918, Max-Change: 0.00028
Iteration: 85, Log-Lik: -42633.918, Max-Change: 0.00011
Iteration: 86, Log-Lik: -42633.918, Max-Change: 0.00053
Iteration: 87, Log-Lik: -42633.918, Max-Change: 0.00026
Iteration: 88, Log-Lik: -42633.918, Max-Change: 0.00010

## [1] "2PL ITERATION 1. STARTED WITH 392 ITEMS. 43 OUTLIERS REMOVED. 349 ITEMS RETAINED."
## 
Iteration: 1, Log-Lik: -54215.740, Max-Change: 18.28001
Iteration: 2, Log-Lik: -40003.600, Max-Change: 10.15031
Iteration: 3, Log-Lik: -38449.876, Max-Change: 4.84932
Iteration: 4, Log-Lik: -38197.898, Max-Change: 2.19555
Iteration: 5, Log-Lik: -38109.435, Max-Change: 0.30561
Iteration: 6, Log-Lik: -38070.002, Max-Change: 0.22197
Iteration: 7, Log-Lik: -38049.375, Max-Change: 0.16645
Iteration: 8, Log-Lik: -38037.219, Max-Change: 0.09486
Iteration: 9, Log-Lik: -38029.701, Max-Change: 0.08154
Iteration: 10, Log-Lik: -38024.926, Max-Change: 0.07825
Iteration: 11, Log-Lik: -38021.817, Max-Change: 0.06263
Iteration: 12, Log-Lik: -38019.814, Max-Change: 0.04775
Iteration: 13, Log-Lik: -38016.324, Max-Change: 0.00778
Iteration: 14, Log-Lik: -38016.257, Max-Change: 0.00680
Iteration: 15, Log-Lik: -38016.215, Max-Change: 0.00536
Iteration: 16, Log-Lik: -38016.144, Max-Change: 0.00230
Iteration: 17, Log-Lik: -38016.141, Max-Change: 0.00142
Iteration: 18, Log-Lik: -38016.140, Max-Change: 0.00079
Iteration: 19, Log-Lik: -38016.139, Max-Change: 0.00052
Iteration: 20, Log-Lik: -38016.139, Max-Change: 0.00047
Iteration: 21, Log-Lik: -38016.138, Max-Change: 0.00042
Iteration: 22, Log-Lik: -38016.137, Max-Change: 0.00102
Iteration: 23, Log-Lik: -38016.137, Max-Change: 0.00082
Iteration: 24, Log-Lik: -38016.137, Max-Change: 0.00055
Iteration: 25, Log-Lik: -38016.137, Max-Change: 0.00043
Iteration: 26, Log-Lik: -38016.137, Max-Change: 0.00186
Iteration: 27, Log-Lik: -38016.137, Max-Change: 0.00230
Iteration: 28, Log-Lik: -38016.137, Max-Change: 0.00069
Iteration: 29, Log-Lik: -38016.137, Max-Change: 0.00049
Iteration: 30, Log-Lik: -38016.137, Max-Change: 0.00038
Iteration: 31, Log-Lik: -38016.137, Max-Change: 0.00159
Iteration: 32, Log-Lik: -38016.137, Max-Change: 0.00115
Iteration: 33, Log-Lik: -38016.137, Max-Change: 0.00085
Iteration: 34, Log-Lik: -38016.137, Max-Change: 0.00139
Iteration: 35, Log-Lik: -38016.137, Max-Change: 0.00118
Iteration: 36, Log-Lik: -38016.137, Max-Change: 0.00087
Iteration: 37, Log-Lik: -38016.137, Max-Change: 0.00024
Iteration: 38, Log-Lik: -38016.137, Max-Change: 0.00024
Iteration: 39, Log-Lik: -38016.137, Max-Change: 0.00116
Iteration: 40, Log-Lik: -38016.137, Max-Change: 0.00094
Iteration: 41, Log-Lik: -38016.137, Max-Change: 0.00069
Iteration: 42, Log-Lik: -38016.137, Max-Change: 0.00051
Iteration: 43, Log-Lik: -38016.137, Max-Change: 0.00097
Iteration: 44, Log-Lik: -38016.137, Max-Change: 0.00078
Iteration: 45, Log-Lik: -38016.137, Max-Change: 0.00057
Iteration: 46, Log-Lik: -38016.137, Max-Change: 0.00017
Iteration: 47, Log-Lik: -38016.137, Max-Change: 0.00016
Iteration: 48, Log-Lik: -38016.137, Max-Change: 0.00080
Iteration: 49, Log-Lik: -38016.137, Max-Change: 0.00063
Iteration: 50, Log-Lik: -38016.137, Max-Change: 0.00046
Iteration: 51, Log-Lik: -38016.137, Max-Change: 0.00034
Iteration: 52, Log-Lik: -38016.137, Max-Change: 0.00013
Iteration: 53, Log-Lik: -38016.137, Max-Change: 0.00065
Iteration: 54, Log-Lik: -38016.137, Max-Change: 0.00040
Iteration: 55, Log-Lik: -38016.137, Max-Change: 0.00012
Iteration: 56, Log-Lik: -38016.137, Max-Change: 0.00056
Iteration: 57, Log-Lik: -38016.137, Max-Change: 0.00035
Iteration: 58, Log-Lik: -38016.137, Max-Change: 0.00010

## [1] "2PL ITERATION 2. STARTED WITH 349 ITEMS. 2 OUTLIERS REMOVED. 347 ITEMS RETAINED."
## 
Iteration: 1, Log-Lik: -53593.338, Max-Change: 20.91328
Iteration: 2, Log-Lik: -40342.021, Max-Change: 10.94145
Iteration: 3, Log-Lik: -39073.081, Max-Change: 5.00637
Iteration: 4, Log-Lik: -38826.621, Max-Change: 2.23976
Iteration: 5, Log-Lik: -38751.040, Max-Change: 0.27475
Iteration: 6, Log-Lik: -38720.523, Max-Change: 0.12551
Iteration: 7, Log-Lik: -38706.191, Max-Change: 0.09824
Iteration: 8, Log-Lik: -38698.363, Max-Change: 0.10030
Iteration: 9, Log-Lik: -38693.745, Max-Change: 0.09730
Iteration: 10, Log-Lik: -38690.917, Max-Change: 0.07916
Iteration: 11, Log-Lik: -38689.135, Max-Change: 0.05918
Iteration: 12, Log-Lik: -38688.018, Max-Change: 0.04206
Iteration: 13, Log-Lik: -38686.226, Max-Change: 0.00758
Iteration: 14, Log-Lik: -38686.180, Max-Change: 0.00576
Iteration: 15, Log-Lik: -38686.153, Max-Change: 0.00466
Iteration: 16, Log-Lik: -38686.123, Max-Change: 0.00227
Iteration: 17, Log-Lik: -38686.117, Max-Change: 0.00192
Iteration: 18, Log-Lik: -38686.113, Max-Change: 0.00166
Iteration: 19, Log-Lik: -38686.104, Max-Change: 0.00180
Iteration: 20, Log-Lik: -38686.104, Max-Change: 0.00144
Iteration: 21, Log-Lik: -38686.104, Max-Change: 0.00096
Iteration: 22, Log-Lik: -38686.104, Max-Change: 0.00177
Iteration: 23, Log-Lik: -38686.104, Max-Change: 0.00131
Iteration: 24, Log-Lik: -38686.104, Max-Change: 0.00092
Iteration: 25, Log-Lik: -38686.104, Max-Change: 0.00150
Iteration: 26, Log-Lik: -38686.104, Max-Change: 0.00090
Iteration: 27, Log-Lik: -38686.104, Max-Change: 0.00066
Iteration: 28, Log-Lik: -38686.104, Max-Change: 0.00128
Iteration: 29, Log-Lik: -38686.104, Max-Change: 0.00057
Iteration: 30, Log-Lik: -38686.104, Max-Change: 0.00042
Iteration: 31, Log-Lik: -38686.104, Max-Change: 0.00544
Iteration: 32, Log-Lik: -38686.104, Max-Change: 0.00066
Iteration: 33, Log-Lik: -38686.104, Max-Change: 0.00048
Iteration: 34, Log-Lik: -38686.104, Max-Change: 0.00012
Iteration: 35, Log-Lik: -38686.104, Max-Change: 0.00009

## [1] "2PL ITERATION 3. STARTED WITH 347 ITEMS. 36 OUTLIERS REMOVED. 311 ITEMS RETAINED."
## 
Iteration: 1, Log-Lik: -49250.023, Max-Change: 23.10124
Iteration: 2, Log-Lik: -36680.589, Max-Change: 8.36575
Iteration: 3, Log-Lik: -35527.964, Max-Change: 4.69536
Iteration: 4, Log-Lik: -35313.843, Max-Change: 2.21978
Iteration: 5, Log-Lik: -35243.621, Max-Change: 0.28247
Iteration: 6, Log-Lik: -35214.697, Max-Change: 0.12617
Iteration: 7, Log-Lik: -35201.139, Max-Change: 0.09567
Iteration: 8, Log-Lik: -35193.775, Max-Change: 0.07567
Iteration: 9, Log-Lik: -35189.469, Max-Change: 0.06126
Iteration: 10, Log-Lik: -35186.836, Max-Change: 0.04838
Iteration: 11, Log-Lik: -35185.216, Max-Change: 0.03993
Iteration: 12, Log-Lik: -35184.208, Max-Change: 0.02735
Iteration: 13, Log-Lik: -35182.647, Max-Change: 0.00813
Iteration: 14, Log-Lik: -35182.602, Max-Change: 0.00586
Iteration: 15, Log-Lik: -35182.574, Max-Change: 0.00479
Iteration: 16, Log-Lik: -35182.544, Max-Change: 0.00234
Iteration: 17, Log-Lik: -35182.538, Max-Change: 0.00194
Iteration: 18, Log-Lik: -35182.534, Max-Change: 0.00195
Iteration: 19, Log-Lik: -35182.526, Max-Change: 0.00158
Iteration: 20, Log-Lik: -35182.526, Max-Change: 0.00123
Iteration: 21, Log-Lik: -35182.526, Max-Change: 0.00084
Iteration: 22, Log-Lik: -35182.526, Max-Change: 0.00068
Iteration: 23, Log-Lik: -35182.526, Max-Change: 0.00336
Iteration: 24, Log-Lik: -35182.526, Max-Change: 0.00207
Iteration: 25, Log-Lik: -35182.526, Max-Change: 0.00073
Iteration: 26, Log-Lik: -35182.526, Max-Change: 0.00060
Iteration: 27, Log-Lik: -35182.526, Max-Change: 0.00059
Iteration: 28, Log-Lik: -35182.526, Max-Change: 0.00292
Iteration: 29, Log-Lik: -35182.526, Max-Change: 0.00113
Iteration: 30, Log-Lik: -35182.526, Max-Change: 0.00083
Iteration: 31, Log-Lik: -35182.526, Max-Change: 0.00262
Iteration: 32, Log-Lik: -35182.526, Max-Change: 0.00084
Iteration: 33, Log-Lik: -35182.526, Max-Change: 0.00062
Iteration: 34, Log-Lik: -35182.526, Max-Change: 0.01168
Iteration: 35, Log-Lik: -35182.525, Max-Change: 0.00250
Iteration: 36, Log-Lik: -35182.525, Max-Change: 0.00183
Iteration: 37, Log-Lik: -35182.525, Max-Change: 0.00048
Iteration: 38, Log-Lik: -35182.525, Max-Change: 0.00035
Iteration: 39, Log-Lik: -35182.525, Max-Change: 0.00026
Iteration: 40, Log-Lik: -35182.525, Max-Change: 0.00624
Iteration: 41, Log-Lik: -35182.525, Max-Change: 0.00057
Iteration: 42, Log-Lik: -35182.525, Max-Change: 0.00042
Iteration: 43, Log-Lik: -35182.525, Max-Change: 0.00013
Iteration: 44, Log-Lik: -35182.525, Max-Change: 0.00013
Iteration: 45, Log-Lik: -35182.525, Max-Change: 0.00013
Iteration: 46, Log-Lik: -35182.525, Max-Change: 0.00062
Iteration: 47, Log-Lik: -35182.525, Max-Change: 0.00026
Iteration: 48, Log-Lik: -35182.525, Max-Change: 0.00019
Iteration: 49, Log-Lik: -35182.525, Max-Change: 0.00055
Iteration: 50, Log-Lik: -35182.525, Max-Change: 0.00027
Iteration: 51, Log-Lik: -35182.525, Max-Change: 0.00020
Iteration: 52, Log-Lik: -35182.525, Max-Change: 0.00010

## [1] "2PL ITERATION 4. STARTED WITH 311 ITEMS. 12 OUTLIERS REMOVED. 299 ITEMS RETAINED."
## 
Iteration: 1, Log-Lik: -45898.322, Max-Change: 18.63583
Iteration: 2, Log-Lik: -34587.399, Max-Change: 9.43726
Iteration: 3, Log-Lik: -33639.496, Max-Change: 4.83061
Iteration: 4, Log-Lik: -33439.361, Max-Change: 1.32920
Iteration: 5, Log-Lik: -33381.550, Max-Change: 0.18192
Iteration: 6, Log-Lik: -33362.763, Max-Change: 0.09880
Iteration: 7, Log-Lik: -33355.052, Max-Change: 0.07450
Iteration: 8, Log-Lik: -33351.324, Max-Change: 0.05658
Iteration: 9, Log-Lik: -33349.318, Max-Change: 0.04249
Iteration: 10, Log-Lik: -33348.189, Max-Change: 0.03162
Iteration: 11, Log-Lik: -33347.536, Max-Change: 0.02423
Iteration: 12, Log-Lik: -33347.153, Max-Change: 0.01966
Iteration: 13, Log-Lik: -33346.649, Max-Change: 0.00519
Iteration: 14, Log-Lik: -33346.627, Max-Change: 0.00383
Iteration: 15, Log-Lik: -33346.616, Max-Change: 0.00314
Iteration: 16, Log-Lik: -33346.599, Max-Change: 0.00398
Iteration: 17, Log-Lik: -33346.596, Max-Change: 0.00106
Iteration: 18, Log-Lik: -33346.596, Max-Change: 0.00389
Iteration: 19, Log-Lik: -33346.596, Max-Change: 0.00454
Iteration: 20, Log-Lik: -33346.595, Max-Change: 0.00292
Iteration: 21, Log-Lik: -33346.595, Max-Change: 0.00224
Iteration: 22, Log-Lik: -33346.595, Max-Change: 0.00331
Iteration: 23, Log-Lik: -33346.595, Max-Change: 0.00347
Iteration: 24, Log-Lik: -33346.595, Max-Change: 0.00237
Iteration: 25, Log-Lik: -33346.595, Max-Change: 0.00060
Iteration: 26, Log-Lik: -33346.595, Max-Change: 0.00059
Iteration: 27, Log-Lik: -33346.595, Max-Change: 0.00292
Iteration: 28, Log-Lik: -33346.595, Max-Change: 0.00265
Iteration: 29, Log-Lik: -33346.595, Max-Change: 0.00188
Iteration: 30, Log-Lik: -33346.595, Max-Change: 0.00131
Iteration: 31, Log-Lik: -33346.595, Max-Change: 0.00257
Iteration: 32, Log-Lik: -33346.595, Max-Change: 0.00206
Iteration: 33, Log-Lik: -33346.595, Max-Change: 0.00145
Iteration: 34, Log-Lik: -33346.595, Max-Change: 0.00046
Iteration: 35, Log-Lik: -33346.595, Max-Change: 0.00228
Iteration: 36, Log-Lik: -33346.595, Max-Change: 0.00184
Iteration: 37, Log-Lik: -33346.594, Max-Change: 0.00042
Iteration: 38, Log-Lik: -33346.594, Max-Change: 0.00206
Iteration: 39, Log-Lik: -33346.594, Max-Change: 0.00154
Iteration: 40, Log-Lik: -33346.594, Max-Change: 0.00038
Iteration: 41, Log-Lik: -33346.594, Max-Change: 0.00185
Iteration: 42, Log-Lik: -33346.594, Max-Change: 0.00137
Iteration: 43, Log-Lik: -33346.594, Max-Change: 0.00034
Iteration: 44, Log-Lik: -33346.594, Max-Change: 0.00167
Iteration: 45, Log-Lik: -33346.594, Max-Change: 0.00119
Iteration: 46, Log-Lik: -33346.594, Max-Change: 0.00030
Iteration: 47, Log-Lik: -33346.594, Max-Change: 0.00150
Iteration: 48, Log-Lik: -33346.594, Max-Change: 0.00105
Iteration: 49, Log-Lik: -33346.594, Max-Change: 0.00027
Iteration: 50, Log-Lik: -33346.594, Max-Change: 0.00135
Iteration: 51, Log-Lik: -33346.594, Max-Change: 0.00093
Iteration: 52, Log-Lik: -33346.594, Max-Change: 0.00025
Iteration: 53, Log-Lik: -33346.594, Max-Change: 0.00121
Iteration: 54, Log-Lik: -33346.594, Max-Change: 0.00082
Iteration: 55, Log-Lik: -33346.594, Max-Change: 0.00022
Iteration: 56, Log-Lik: -33346.594, Max-Change: 0.00109
Iteration: 57, Log-Lik: -33346.594, Max-Change: 0.00072
Iteration: 58, Log-Lik: -33346.594, Max-Change: 0.00020
Iteration: 59, Log-Lik: -33346.594, Max-Change: 0.00098
Iteration: 60, Log-Lik: -33346.594, Max-Change: 0.00064
Iteration: 61, Log-Lik: -33346.594, Max-Change: 0.00018
Iteration: 62, Log-Lik: -33346.594, Max-Change: 0.00088
Iteration: 63, Log-Lik: -33346.594, Max-Change: 0.00057
Iteration: 64, Log-Lik: -33346.594, Max-Change: 0.00016
Iteration: 65, Log-Lik: -33346.594, Max-Change: 0.00079
Iteration: 66, Log-Lik: -33346.594, Max-Change: 0.00050
Iteration: 67, Log-Lik: -33346.594, Max-Change: 0.00015
Iteration: 68, Log-Lik: -33346.594, Max-Change: 0.00071
Iteration: 69, Log-Lik: -33346.594, Max-Change: 0.00045
Iteration: 70, Log-Lik: -33346.594, Max-Change: 0.00013
Iteration: 71, Log-Lik: -33346.594, Max-Change: 0.00064
Iteration: 72, Log-Lik: -33346.594, Max-Change: 0.00040
Iteration: 73, Log-Lik: -33346.594, Max-Change: 0.00012
Iteration: 74, Log-Lik: -33346.594, Max-Change: 0.00058
Iteration: 75, Log-Lik: -33346.594, Max-Change: 0.00035
Iteration: 76, Log-Lik: -33346.594, Max-Change: 0.00011
Iteration: 77, Log-Lik: -33346.594, Max-Change: 0.00052
Iteration: 78, Log-Lik: -33346.594, Max-Change: 0.00031
Iteration: 79, Log-Lik: -33346.594, Max-Change: 0.00009

## [1] "2PL ITERATION 5. STARTED WITH 299 ITEMS. 38 OUTLIERS REMOVED. 261 ITEMS RETAINED."
## 
Iteration: 1, Log-Lik: -41075.352, Max-Change: 16.00120
Iteration: 2, Log-Lik: -31118.119, Max-Change: 7.78021
Iteration: 3, Log-Lik: -30296.033, Max-Change: 4.91813
Iteration: 4, Log-Lik: -30114.710, Max-Change: 0.44479
Iteration: 5, Log-Lik: -30059.835, Max-Change: 0.19410
Iteration: 6, Log-Lik: -30043.974, Max-Change: 0.09844
Iteration: 7, Log-Lik: -30038.017, Max-Change: 0.06436
Iteration: 8, Log-Lik: -30035.372, Max-Change: 0.04762
Iteration: 9, Log-Lik: -30034.070, Max-Change: 0.03449
Iteration: 10, Log-Lik: -30033.382, Max-Change: 0.02594
Iteration: 11, Log-Lik: -30033.004, Max-Change: 0.02013
Iteration: 12, Log-Lik: -30032.788, Max-Change: 0.01364
Iteration: 13, Log-Lik: -30032.553, Max-Change: 0.00631
Iteration: 14, Log-Lik: -30032.536, Max-Change: 0.00431
Iteration: 15, Log-Lik: -30032.526, Max-Change: 0.00328
Iteration: 16, Log-Lik: -30032.511, Max-Change: 0.00219
Iteration: 17, Log-Lik: -30032.510, Max-Change: 0.00448
Iteration: 18, Log-Lik: -30032.510, Max-Change: 0.00365
Iteration: 19, Log-Lik: -30032.509, Max-Change: 0.00105
Iteration: 20, Log-Lik: -30032.509, Max-Change: 0.00057
Iteration: 21, Log-Lik: -30032.509, Max-Change: 0.00267
Iteration: 22, Log-Lik: -30032.509, Max-Change: 0.00350
Iteration: 23, Log-Lik: -30032.509, Max-Change: 0.00223
Iteration: 24, Log-Lik: -30032.509, Max-Change: 0.00160
Iteration: 25, Log-Lik: -30032.509, Max-Change: 0.00045
Iteration: 26, Log-Lik: -30032.509, Max-Change: 0.00218
Iteration: 27, Log-Lik: -30032.509, Max-Change: 0.00190
Iteration: 28, Log-Lik: -30032.509, Max-Change: 0.00043
Iteration: 29, Log-Lik: -30032.509, Max-Change: 0.00038
Iteration: 30, Log-Lik: -30032.509, Max-Change: 0.00184
Iteration: 31, Log-Lik: -30032.509, Max-Change: 0.00174
Iteration: 32, Log-Lik: -30032.508, Max-Change: 0.00119
Iteration: 33, Log-Lik: -30032.508, Max-Change: 0.00080
Iteration: 34, Log-Lik: -30032.508, Max-Change: 0.00031
Iteration: 35, Log-Lik: -30032.508, Max-Change: 0.00150
Iteration: 36, Log-Lik: -30032.508, Max-Change: 0.00101
Iteration: 37, Log-Lik: -30032.508, Max-Change: 0.00026
Iteration: 38, Log-Lik: -30032.508, Max-Change: 0.00026
Iteration: 39, Log-Lik: -30032.508, Max-Change: 0.00126
Iteration: 40, Log-Lik: -30032.508, Max-Change: 0.00087
Iteration: 41, Log-Lik: -30032.508, Max-Change: 0.00059
Iteration: 42, Log-Lik: -30032.508, Max-Change: 0.00040
Iteration: 43, Log-Lik: -30032.508, Max-Change: 0.00104
Iteration: 44, Log-Lik: -30032.508, Max-Change: 0.00064
Iteration: 45, Log-Lik: -30032.508, Max-Change: 0.00044
Iteration: 46, Log-Lik: -30032.508, Max-Change: 0.00018
Iteration: 47, Log-Lik: -30032.508, Max-Change: 0.00087
Iteration: 48, Log-Lik: -30032.508, Max-Change: 0.00057
Iteration: 49, Log-Lik: -30032.508, Max-Change: 0.00015
Iteration: 50, Log-Lik: -30032.508, Max-Change: 0.00075
Iteration: 51, Log-Lik: -30032.508, Max-Change: 0.00054
Iteration: 52, Log-Lik: -30032.508, Max-Change: 0.00013
Iteration: 53, Log-Lik: -30032.508, Max-Change: 0.00064
Iteration: 54, Log-Lik: -30032.508, Max-Change: 0.00043
Iteration: 55, Log-Lik: -30032.508, Max-Change: 0.00011
Iteration: 56, Log-Lik: -30032.508, Max-Change: 0.00055
Iteration: 57, Log-Lik: -30032.508, Max-Change: 0.00037
Iteration: 58, Log-Lik: -30032.508, Max-Change: 0.00010

## [1] "2PL ITERATION 6. STARTED WITH 261 ITEMS. 31 OUTLIERS REMOVED. 230 ITEMS RETAINED."
## 
Iteration: 1, Log-Lik: -35164.863, Max-Change: 17.99941
Iteration: 2, Log-Lik: -26234.609, Max-Change: 5.83621
Iteration: 3, Log-Lik: -25598.581, Max-Change: 3.51457
Iteration: 4, Log-Lik: -25456.251, Max-Change: 0.47900
Iteration: 5, Log-Lik: -25422.707, Max-Change: 0.19078
Iteration: 6, Log-Lik: -25413.320, Max-Change: 0.10501
Iteration: 7, Log-Lik: -25410.231, Max-Change: 0.06105
Iteration: 8, Log-Lik: -25409.087, Max-Change: 0.03879
Iteration: 9, Log-Lik: -25408.606, Max-Change: 0.02264
Iteration: 10, Log-Lik: -25408.307, Max-Change: 0.00782
Iteration: 11, Log-Lik: -25408.257, Max-Change: 0.00551
Iteration: 12, Log-Lik: -25408.232, Max-Change: 0.00444
Iteration: 13, Log-Lik: -25408.206, Max-Change: 0.00516
Iteration: 14, Log-Lik: -25408.205, Max-Change: 0.00055
Iteration: 15, Log-Lik: -25408.205, Max-Change: 0.00037
Iteration: 16, Log-Lik: -25408.205, Max-Change: 0.00035
Iteration: 17, Log-Lik: -25408.205, Max-Change: 0.00170
Iteration: 18, Log-Lik: -25408.204, Max-Change: 0.00120
Iteration: 19, Log-Lik: -25408.204, Max-Change: 0.00071
Iteration: 20, Log-Lik: -25408.204, Max-Change: 0.00042
Iteration: 21, Log-Lik: -25408.204, Max-Change: 0.00133
Iteration: 22, Log-Lik: -25408.204, Max-Change: 0.00210
Iteration: 23, Log-Lik: -25408.204, Max-Change: 0.00129
Iteration: 24, Log-Lik: -25408.204, Max-Change: 0.00079
Iteration: 25, Log-Lik: -25408.204, Max-Change: 0.00025
Iteration: 26, Log-Lik: -25408.204, Max-Change: 0.00097
Iteration: 27, Log-Lik: -25408.204, Max-Change: 0.00108
Iteration: 28, Log-Lik: -25408.204, Max-Change: 0.00027
Iteration: 29, Log-Lik: -25408.204, Max-Change: 0.00017
Iteration: 30, Log-Lik: -25408.204, Max-Change: 0.00074
Iteration: 31, Log-Lik: -25408.204, Max-Change: 0.00083
Iteration: 32, Log-Lik: -25408.204, Max-Change: 0.00051
Iteration: 33, Log-Lik: -25408.204, Max-Change: 0.00031
Iteration: 34, Log-Lik: -25408.204, Max-Change: 0.00011
Iteration: 35, Log-Lik: -25408.204, Max-Change: 0.00053
Iteration: 36, Log-Lik: -25408.204, Max-Change: 0.00039
Iteration: 37, Log-Lik: -25408.204, Max-Change: 0.00012
Iteration: 38, Log-Lik: -25408.204, Max-Change: 0.00008

## [1] "2PL ITERATION 7. STARTED WITH 230 ITEMS. 39 OUTLIERS REMOVED. 191 ITEMS RETAINED."
## 
Iteration: 1, Log-Lik: -30451.398, Max-Change: 11.27658
Iteration: 2, Log-Lik: -22914.663, Max-Change: 5.28589
Iteration: 3, Log-Lik: -22353.638, Max-Change: 2.49154
Iteration: 4, Log-Lik: -22209.422, Max-Change: 0.40625
Iteration: 5, Log-Lik: -22173.918, Max-Change: 0.20508
Iteration: 6, Log-Lik: -22163.675, Max-Change: 0.09765
Iteration: 7, Log-Lik: -22160.478, Max-Change: 0.05748
Iteration: 8, Log-Lik: -22159.325, Max-Change: 0.03732
Iteration: 9, Log-Lik: -22158.882, Max-Change: 0.02109
Iteration: 10, Log-Lik: -22158.638, Max-Change: 0.01078
Iteration: 11, Log-Lik: -22158.580, Max-Change: 0.00752
Iteration: 12, Log-Lik: -22158.552, Max-Change: 0.00650
Iteration: 13, Log-Lik: -22158.526, Max-Change: 0.00201
Iteration: 14, Log-Lik: -22158.525, Max-Change: 0.00068
Iteration: 15, Log-Lik: -22158.524, Max-Change: 0.00052
Iteration: 16, Log-Lik: -22158.524, Max-Change: 0.00038
Iteration: 17, Log-Lik: -22158.524, Max-Change: 0.00036
Iteration: 18, Log-Lik: -22158.524, Max-Change: 0.00034
Iteration: 19, Log-Lik: -22158.524, Max-Change: 0.00036
Iteration: 20, Log-Lik: -22158.524, Max-Change: 0.00138
Iteration: 21, Log-Lik: -22158.524, Max-Change: 0.00117
Iteration: 22, Log-Lik: -22158.524, Max-Change: 0.00057
Iteration: 23, Log-Lik: -22158.524, Max-Change: 0.00029
Iteration: 24, Log-Lik: -22158.524, Max-Change: 0.00091
Iteration: 25, Log-Lik: -22158.524, Max-Change: 0.00106
Iteration: 26, Log-Lik: -22158.524, Max-Change: 0.00053
Iteration: 27, Log-Lik: -22158.524, Max-Change: 0.00027
Iteration: 28, Log-Lik: -22158.524, Max-Change: 0.00057
Iteration: 29, Log-Lik: -22158.524, Max-Change: 0.00061
Iteration: 30, Log-Lik: -22158.524, Max-Change: 0.00031
Iteration: 31, Log-Lik: -22158.524, Max-Change: 0.00013
Iteration: 32, Log-Lik: -22158.524, Max-Change: 0.00038
Iteration: 33, Log-Lik: -22158.524, Max-Change: 0.00044
Iteration: 34, Log-Lik: -22158.524, Max-Change: 0.00014
Iteration: 35, Log-Lik: -22158.524, Max-Change: 0.00007

## [1] "2PL ITERATION 8. STARTED WITH 191 ITEMS. 28 OUTLIERS REMOVED. 163 ITEMS RETAINED."
## 
Iteration: 1, Log-Lik: -25220.018, Max-Change: 7.13564
Iteration: 2, Log-Lik: -19090.890, Max-Change: 3.11739
Iteration: 3, Log-Lik: -18712.306, Max-Change: 0.80648
Iteration: 4, Log-Lik: -18596.003, Max-Change: 0.40503
Iteration: 5, Log-Lik: -18564.436, Max-Change: 0.26498
Iteration: 6, Log-Lik: -18555.016, Max-Change: 0.16870
Iteration: 7, Log-Lik: -18552.149, Max-Change: 0.07010
Iteration: 8, Log-Lik: -18551.343, Max-Change: 0.04138
Iteration: 9, Log-Lik: -18551.066, Max-Change: 0.02556
Iteration: 10, Log-Lik: -18550.943, Max-Change: 0.01113
Iteration: 11, Log-Lik: -18550.920, Max-Change: 0.00828
Iteration: 12, Log-Lik: -18550.910, Max-Change: 0.00661
Iteration: 13, Log-Lik: -18550.902, Max-Change: 0.00226
Iteration: 14, Log-Lik: -18550.901, Max-Change: 0.00128
Iteration: 15, Log-Lik: -18550.901, Max-Change: 0.00066
Iteration: 16, Log-Lik: -18550.901, Max-Change: 0.00321
Iteration: 17, Log-Lik: -18550.901, Max-Change: 0.00054
Iteration: 18, Log-Lik: -18550.901, Max-Change: 0.00260
Iteration: 19, Log-Lik: -18550.901, Max-Change: 0.00141
Iteration: 20, Log-Lik: -18550.901, Max-Change: 0.00068
Iteration: 21, Log-Lik: -18550.901, Max-Change: 0.00037
Iteration: 22, Log-Lik: -18550.901, Max-Change: 0.00181
Iteration: 23, Log-Lik: -18550.901, Max-Change: 0.00148
Iteration: 24, Log-Lik: -18550.901, Max-Change: 0.00115
Iteration: 25, Log-Lik: -18550.901, Max-Change: 0.00039
Iteration: 26, Log-Lik: -18550.901, Max-Change: 0.00023
Iteration: 27, Log-Lik: -18550.901, Max-Change: 0.00109
Iteration: 28, Log-Lik: -18550.901, Max-Change: 0.00073
Iteration: 29, Log-Lik: -18550.901, Max-Change: 0.00036
Iteration: 30, Log-Lik: -18550.901, Max-Change: 0.00018
Iteration: 31, Log-Lik: -18550.901, Max-Change: 0.00090
Iteration: 32, Log-Lik: -18550.901, Max-Change: 0.00021
Iteration: 33, Log-Lik: -18550.901, Max-Change: 0.00016
Iteration: 34, Log-Lik: -18550.901, Max-Change: 0.00396
Iteration: 35, Log-Lik: -18550.901, Max-Change: 0.00015
Iteration: 36, Log-Lik: -18550.901, Max-Change: 0.00008

## [1] "2PL ITERATION 9. STARTED WITH 163 ITEMS. 32 OUTLIERS REMOVED. 131 ITEMS RETAINED."
## 
Iteration: 1, Log-Lik: -20755.431, Max-Change: 7.42563
Iteration: 2, Log-Lik: -15734.121, Max-Change: 3.62409
Iteration: 3, Log-Lik: -15430.653, Max-Change: 0.56921
Iteration: 4, Log-Lik: -15310.520, Max-Change: 0.41213
Iteration: 5, Log-Lik: -15274.828, Max-Change: 0.28214
Iteration: 6, Log-Lik: -15265.083, Max-Change: 0.16768
Iteration: 7, Log-Lik: -15262.306, Max-Change: 0.08792
Iteration: 8, Log-Lik: -15261.448, Max-Change: 0.05184
Iteration: 9, Log-Lik: -15261.183, Max-Change: 0.03241
Iteration: 10, Log-Lik: -15261.077, Max-Change: 0.01020
Iteration: 11, Log-Lik: -15261.059, Max-Change: 0.00716
Iteration: 12, Log-Lik: -15261.051, Max-Change: 0.00481
Iteration: 13, Log-Lik: -15261.047, Max-Change: 0.00189
Iteration: 14, Log-Lik: -15261.046, Max-Change: 0.00155
Iteration: 15, Log-Lik: -15261.046, Max-Change: 0.00079
Iteration: 16, Log-Lik: -15261.046, Max-Change: 0.00058
Iteration: 17, Log-Lik: -15261.046, Max-Change: 0.00250
Iteration: 18, Log-Lik: -15261.046, Max-Change: 0.00045
Iteration: 19, Log-Lik: -15261.046, Max-Change: 0.00035
Iteration: 20, Log-Lik: -15261.046, Max-Change: 0.00138
Iteration: 21, Log-Lik: -15261.046, Max-Change: 0.00118
Iteration: 22, Log-Lik: -15261.046, Max-Change: 0.00058
Iteration: 23, Log-Lik: -15261.046, Max-Change: 0.00029
Iteration: 24, Log-Lik: -15261.046, Max-Change: 0.00088
Iteration: 25, Log-Lik: -15261.046, Max-Change: 0.00109
Iteration: 26, Log-Lik: -15261.046, Max-Change: 0.00055
Iteration: 27, Log-Lik: -15261.046, Max-Change: 0.00028
Iteration: 28, Log-Lik: -15261.046, Max-Change: 0.00012
Iteration: 29, Log-Lik: -15261.046, Max-Change: 0.00055
Iteration: 30, Log-Lik: -15261.046, Max-Change: 0.00036
Iteration: 31, Log-Lik: -15261.046, Max-Change: 0.00015
Iteration: 32, Log-Lik: -15261.046, Max-Change: 0.00008

## [1] "2PL ITERATION 10. STARTED WITH 131 ITEMS. 20 OUTLIERS REMOVED. 111 ITEMS RETAINED."
## 
Iteration: 1, Log-Lik: -16819.832, Max-Change: 4.17858
Iteration: 2, Log-Lik: -12669.272, Max-Change: 0.69354
Iteration: 3, Log-Lik: -12476.925, Max-Change: 0.52033
Iteration: 4, Log-Lik: -12377.972, Max-Change: 0.38544
Iteration: 5, Log-Lik: -12343.679, Max-Change: 0.28241
Iteration: 6, Log-Lik: -12332.346, Max-Change: 0.18793
Iteration: 7, Log-Lik: -12328.726, Max-Change: 0.10345
Iteration: 8, Log-Lik: -12327.576, Max-Change: 0.07174
Iteration: 9, Log-Lik: -12327.171, Max-Change: 0.04023
Iteration: 10, Log-Lik: -12327.077, Max-Change: 0.03081
Iteration: 11, Log-Lik: -12327.018, Max-Change: 0.01605
Iteration: 12, Log-Lik: -12326.998, Max-Change: 0.00897
Iteration: 13, Log-Lik: -12326.990, Max-Change: 0.00548
Iteration: 14, Log-Lik: -12326.987, Max-Change: 0.00345
Iteration: 15, Log-Lik: -12326.986, Max-Change: 0.00264
Iteration: 16, Log-Lik: -12326.985, Max-Change: 0.00070
Iteration: 17, Log-Lik: -12326.985, Max-Change: 0.00050
Iteration: 18, Log-Lik: -12326.985, Max-Change: 0.00203
Iteration: 19, Log-Lik: -12326.985, Max-Change: 0.00021
Iteration: 20, Log-Lik: -12326.985, Max-Change: 0.00010

## [1] "2PL ITERATION 11. STARTED WITH 111 ITEMS. 29 OUTLIERS REMOVED. 82 ITEMS RETAINED."
## 
Iteration: 1, Log-Lik: -12921.114, Max-Change: 4.76466
Iteration: 2, Log-Lik: -9751.355, Max-Change: 0.49754
Iteration: 3, Log-Lik: -9646.140, Max-Change: 0.46677
Iteration: 4, Log-Lik: -9560.923, Max-Change: 0.42047
Iteration: 5, Log-Lik: -9521.365, Max-Change: 0.33729
Iteration: 6, Log-Lik: -9506.878, Max-Change: 0.21098
Iteration: 7, Log-Lik: -9501.494, Max-Change: 0.13953
Iteration: 8, Log-Lik: -9499.511, Max-Change: 0.08745
Iteration: 9, Log-Lik: -9498.799, Max-Change: 0.05349
Iteration: 10, Log-Lik: -9498.444, Max-Change: 0.02447
Iteration: 11, Log-Lik: -9498.392, Max-Change: 0.01637
Iteration: 12, Log-Lik: -9498.372, Max-Change: 0.01307
Iteration: 13, Log-Lik: -9498.360, Max-Change: 0.00755
Iteration: 14, Log-Lik: -9498.358, Max-Change: 0.00520
Iteration: 15, Log-Lik: -9498.357, Max-Change: 0.00370
Iteration: 16, Log-Lik: -9498.356, Max-Change: 0.00121
Iteration: 17, Log-Lik: -9498.356, Max-Change: 0.00069
Iteration: 18, Log-Lik: -9498.355, Max-Change: 0.00252
Iteration: 19, Log-Lik: -9498.355, Max-Change: 0.00019
Iteration: 20, Log-Lik: -9498.355, Max-Change: 0.00057
Iteration: 21, Log-Lik: -9498.355, Max-Change: 0.00007

## [1] "2PL ITERATION 12. STARTED WITH 82 ITEMS. 18 OUTLIERS REMOVED. 64 ITEMS RETAINED."
## 
Iteration: 1, Log-Lik: -10009.069, Max-Change: 4.18687
Iteration: 2, Log-Lik: -7688.959, Max-Change: 0.39223
Iteration: 3, Log-Lik: -7634.001, Max-Change: 0.45997
Iteration: 4, Log-Lik: -7576.194, Max-Change: 0.39127
Iteration: 5, Log-Lik: -7539.885, Max-Change: 0.30838
Iteration: 6, Log-Lik: -7522.320, Max-Change: 0.24503
Iteration: 7, Log-Lik: -7514.883, Max-Change: 0.18984
Iteration: 8, Log-Lik: -7511.654, Max-Change: 0.12354
Iteration: 9, Log-Lik: -7510.376, Max-Change: 0.07190
Iteration: 10, Log-Lik: -7509.656, Max-Change: 0.03624
Iteration: 11, Log-Lik: -7509.562, Max-Change: 0.02630
Iteration: 12, Log-Lik: -7509.517, Max-Change: 0.01971
Iteration: 13, Log-Lik: -7509.489, Max-Change: 0.00663
Iteration: 14, Log-Lik: -7509.485, Max-Change: 0.00523
Iteration: 15, Log-Lik: -7509.483, Max-Change: 0.00398
Iteration: 16, Log-Lik: -7509.481, Max-Change: 0.00065
Iteration: 17, Log-Lik: -7509.481, Max-Change: 0.00056
Iteration: 18, Log-Lik: -7509.480, Max-Change: 0.00049
Iteration: 19, Log-Lik: -7509.480, Max-Change: 0.00078
Iteration: 20, Log-Lik: -7509.480, Max-Change: 0.00014
Iteration: 21, Log-Lik: -7509.480, Max-Change: 0.00007

## [1] "2PL ITERATION 13. STARTED WITH 64 ITEMS. 19 OUTLIERS REMOVED. 45 ITEMS RETAINED."
## 
Iteration: 1, Log-Lik: -7135.176, Max-Change: 3.28252
Iteration: 2, Log-Lik: -5476.342, Max-Change: 0.25885
Iteration: 3, Log-Lik: -5458.442, Max-Change: 0.33202
Iteration: 4, Log-Lik: -5435.990, Max-Change: 0.29978
Iteration: 5, Log-Lik: -5414.777, Max-Change: 0.26524
Iteration: 6, Log-Lik: -5400.050, Max-Change: 0.22590
Iteration: 7, Log-Lik: -5391.739, Max-Change: 0.20171
Iteration: 8, Log-Lik: -5387.367, Max-Change: 0.15274
Iteration: 9, Log-Lik: -5385.323, Max-Change: 0.11698
Iteration: 10, Log-Lik: -5384.367, Max-Change: 0.08408
Iteration: 11, Log-Lik: -5383.915, Max-Change: 0.05310
Iteration: 12, Log-Lik: -5383.711, Max-Change: 0.03829
Iteration: 13, Log-Lik: -5383.589, Max-Change: 0.02026
Iteration: 14, Log-Lik: -5383.565, Max-Change: 0.01736
Iteration: 15, Log-Lik: -5383.553, Max-Change: 0.01375
Iteration: 16, Log-Lik: -5383.542, Max-Change: 0.00651
Iteration: 17, Log-Lik: -5383.540, Max-Change: 0.00619
Iteration: 18, Log-Lik: -5383.540, Max-Change: 0.00497
Iteration: 19, Log-Lik: -5383.539, Max-Change: 0.00100
Iteration: 20, Log-Lik: -5383.538, Max-Change: 0.00085
Iteration: 21, Log-Lik: -5383.538, Max-Change: 0.00076
Iteration: 22, Log-Lik: -5383.538, Max-Change: 0.00045
Iteration: 23, Log-Lik: -5383.538, Max-Change: 0.00039
Iteration: 24, Log-Lik: -5383.538, Max-Change: 0.00035
Iteration: 25, Log-Lik: -5383.538, Max-Change: 0.00013
Iteration: 26, Log-Lik: -5383.538, Max-Change: 0.00056
Iteration: 27, Log-Lik: -5383.538, Max-Change: 0.00029
Iteration: 28, Log-Lik: -5383.538, Max-Change: 0.00006

## [1] "2PL ITERATION 14. STARTED WITH 45 ITEMS. 12 OUTLIERS REMOVED. 33 ITEMS RETAINED."
## 
Iteration: 1, Log-Lik: -5295.646, Max-Change: 6.66456
Iteration: 2, Log-Lik: -4051.365, Max-Change: 1.26485
Iteration: 3, Log-Lik: -4043.236, Max-Change: 0.26461
Iteration: 4, Log-Lik: -4032.323, Max-Change: 0.25731
Iteration: 5, Log-Lik: -4019.489, Max-Change: 0.23621
Iteration: 6, Log-Lik: -4007.469, Max-Change: 0.22204
Iteration: 7, Log-Lik: -3998.488, Max-Change: 0.19847
Iteration: 8, Log-Lik: -3992.816, Max-Change: 0.16280
Iteration: 9, Log-Lik: -3989.658, Max-Change: 0.13250
Iteration: 10, Log-Lik: -3987.876, Max-Change: 0.11149
Iteration: 11, Log-Lik: -3986.946, Max-Change: 0.10684
Iteration: 12, Log-Lik: -3986.474, Max-Change: 0.10070
Iteration: 13, Log-Lik: -3986.002, Max-Change: 0.29860
Iteration: 14, Log-Lik: -3985.991, Max-Change: 0.03843
Iteration: 15, Log-Lik: -3985.985, Max-Change: 0.00677
Iteration: 16, Log-Lik: -3985.984, Max-Change: 0.00664
Iteration: 17, Log-Lik: -3985.982, Max-Change: 0.00588
Iteration: 18, Log-Lik: -3985.981, Max-Change: 0.00509
Iteration: 19, Log-Lik: -3985.980, Max-Change: 0.00334
Iteration: 20, Log-Lik: -3985.980, Max-Change: 0.00051
Iteration: 21, Log-Lik: -3985.980, Max-Change: 0.00255
Iteration: 22, Log-Lik: -3985.980, Max-Change: 0.00051
Iteration: 23, Log-Lik: -3985.980, Max-Change: 0.01270
Iteration: 24, Log-Lik: -3985.980, Max-Change: 0.00328
Iteration: 25, Log-Lik: -3985.980, Max-Change: 0.00090
Iteration: 26, Log-Lik: -3985.980, Max-Change: 0.00050
Iteration: 27, Log-Lik: -3985.980, Max-Change: 0.00250
Iteration: 28, Log-Lik: -3985.980, Max-Change: 0.00050
Iteration: 29, Log-Lik: -3985.980, Max-Change: 0.00249
Iteration: 30, Log-Lik: -3985.980, Max-Change: 0.00249
Iteration: 31, Log-Lik: -3985.980, Max-Change: 0.00060
Iteration: 32, Log-Lik: -3985.980, Max-Change: 0.00248
Iteration: 33, Log-Lik: -3985.980, Max-Change: 0.00049
Iteration: 34, Log-Lik: -3985.980, Max-Change: 0.00049
Iteration: 35, Log-Lik: -3985.980, Max-Change: 0.00247
Iteration: 36, Log-Lik: -3985.980, Max-Change: 0.00246
Iteration: 37, Log-Lik: -3985.979, Max-Change: 0.00067
Iteration: 38, Log-Lik: -3985.979, Max-Change: 0.00246
Iteration: 39, Log-Lik: -3985.979, Max-Change: 0.00049
Iteration: 40, Log-Lik: -3985.979, Max-Change: 0.00049
Iteration: 41, Log-Lik: -3985.979, Max-Change: 0.00245
Iteration: 42, Log-Lik: -3985.979, Max-Change: 0.00049
Iteration: 43, Log-Lik: -3985.979, Max-Change: 0.00244
Iteration: 44, Log-Lik: -3985.979, Max-Change: 0.00059
Iteration: 45, Log-Lik: -3985.979, Max-Change: 0.00244
Iteration: 46, Log-Lik: -3985.979, Max-Change: 0.00068
Iteration: 47, Log-Lik: -3985.979, Max-Change: 0.00243
Iteration: 48, Log-Lik: -3985.979, Max-Change: 0.00050
Iteration: 49, Log-Lik: -3985.979, Max-Change: 0.00048
Iteration: 50, Log-Lik: -3985.979, Max-Change: 0.00242
Iteration: 51, Log-Lik: -3985.979, Max-Change: 0.00048
Iteration: 52, Log-Lik: -3985.979, Max-Change: 0.00241
Iteration: 53, Log-Lik: -3985.979, Max-Change: 0.00059
Iteration: 54, Log-Lik: -3985.979, Max-Change: 0.00241
Iteration: 55, Log-Lik: -3985.979, Max-Change: 0.00069
Iteration: 56, Log-Lik: -3985.979, Max-Change: 0.00240
Iteration: 57, Log-Lik: -3985.979, Max-Change: 0.00051
Iteration: 58, Log-Lik: -3985.979, Max-Change: 0.00048
Iteration: 59, Log-Lik: -3985.979, Max-Change: 0.00239
Iteration: 60, Log-Lik: -3985.979, Max-Change: 0.00048
Iteration: 61, Log-Lik: -3985.979, Max-Change: 0.00238
Iteration: 62, Log-Lik: -3985.979, Max-Change: 0.00061
Iteration: 63, Log-Lik: -3985.979, Max-Change: 0.00238
Iteration: 64, Log-Lik: -3985.979, Max-Change: 0.00073
Iteration: 65, Log-Lik: -3985.979, Max-Change: 0.00237
Iteration: 66, Log-Lik: -3985.979, Max-Change: 0.00054
Iteration: 67, Log-Lik: -3985.979, Max-Change: 0.00047
Iteration: 68, Log-Lik: -3985.979, Max-Change: 0.00236
Iteration: 69, Log-Lik: -3985.979, Max-Change: 0.00047
Iteration: 70, Log-Lik: -3985.979, Max-Change: 0.00047
Iteration: 71, Log-Lik: -3985.979, Max-Change: 0.00235
Iteration: 72, Log-Lik: -3985.979, Max-Change: 0.00235
Iteration: 73, Log-Lik: -3985.979, Max-Change: 0.00052
Iteration: 74, Log-Lik: -3985.979, Max-Change: 0.00234
Iteration: 75, Log-Lik: -3985.979, Max-Change: 0.00047
Iteration: 76, Log-Lik: -3985.979, Max-Change: 0.00047
Iteration: 77, Log-Lik: -3985.979, Max-Change: 0.00233
Iteration: 78, Log-Lik: -3985.979, Max-Change: 0.00047
Iteration: 79, Log-Lik: -3985.979, Max-Change: 0.00233
Iteration: 80, Log-Lik: -3985.979, Max-Change: 0.00047
Iteration: 81, Log-Lik: -3985.979, Max-Change: 0.00232
Iteration: 82, Log-Lik: -3985.979, Max-Change: 0.00059
Iteration: 83, Log-Lik: -3985.979, Max-Change: 0.00231
Iteration: 84, Log-Lik: -3985.979, Max-Change: 0.00047
Iteration: 85, Log-Lik: -3985.979, Max-Change: 0.00046
Iteration: 86, Log-Lik: -3985.979, Max-Change: 0.00231
Iteration: 87, Log-Lik: -3985.979, Max-Change: 0.00046
Iteration: 88, Log-Lik: -3985.979, Max-Change: 0.00230
Iteration: 89, Log-Lik: -3985.979, Max-Change: 0.00054
Iteration: 90, Log-Lik: -3985.979, Max-Change: 0.00229
Iteration: 91, Log-Lik: -3985.979, Max-Change: 0.00071
Iteration: 92, Log-Lik: -3985.979, Max-Change: 0.00228
Iteration: 93, Log-Lik: -3985.979, Max-Change: 0.00057
Iteration: 94, Log-Lik: -3985.979, Max-Change: 0.00046
Iteration: 95, Log-Lik: -3985.979, Max-Change: 0.00228
Iteration: 96, Log-Lik: -3985.979, Max-Change: 0.00045
Iteration: 97, Log-Lik: -3985.979, Max-Change: 0.00045
Iteration: 98, Log-Lik: -3985.979, Max-Change: 0.00227
Iteration: 99, Log-Lik: -3985.979, Max-Change: 0.00227
Iteration: 100, Log-Lik: -3985.979, Max-Change: 0.00054
Iteration: 101, Log-Lik: -3985.978, Max-Change: 0.00226
Iteration: 102, Log-Lik: -3985.978, Max-Change: 0.00045
Iteration: 103, Log-Lik: -3985.978, Max-Change: 0.00045
Iteration: 104, Log-Lik: -3985.978, Max-Change: 0.00225
Iteration: 105, Log-Lik: -3985.978, Max-Change: 0.00045
Iteration: 106, Log-Lik: -3985.978, Max-Change: 0.00225
Iteration: 107, Log-Lik: -3985.978, Max-Change: 0.00050
Iteration: 108, Log-Lik: -3985.978, Max-Change: 0.00224
Iteration: 109, Log-Lik: -3985.978, Max-Change: 0.00064
Iteration: 110, Log-Lik: -3985.978, Max-Change: 0.00223
Iteration: 111, Log-Lik: -3985.978, Max-Change: 0.00051
Iteration: 112, Log-Lik: -3985.978, Max-Change: 0.00045
Iteration: 113, Log-Lik: -3985.978, Max-Change: 0.00223
Iteration: 114, Log-Lik: -3985.978, Max-Change: 0.00044
Iteration: 115, Log-Lik: -3985.978, Max-Change: 0.00044
Iteration: 116, Log-Lik: -3985.978, Max-Change: 0.00222
Iteration: 117, Log-Lik: -3985.978, Max-Change: 0.00222
Iteration: 118, Log-Lik: -3985.978, Max-Change: 0.00054
Iteration: 119, Log-Lik: -3985.978, Max-Change: 0.00221
Iteration: 120, Log-Lik: -3985.978, Max-Change: 0.00044
Iteration: 121, Log-Lik: -3985.978, Max-Change: 0.00044
Iteration: 122, Log-Lik: -3985.978, Max-Change: 0.00220
Iteration: 123, Log-Lik: -3985.978, Max-Change: 0.00044
Iteration: 124, Log-Lik: -3985.978, Max-Change: 0.00220
Iteration: 125, Log-Lik: -3985.978, Max-Change: 0.00050
Iteration: 126, Log-Lik: -3985.978, Max-Change: 0.00219
Iteration: 127, Log-Lik: -3985.978, Max-Change: 0.00063
Iteration: 128, Log-Lik: -3985.978, Max-Change: 0.00218
Iteration: 129, Log-Lik: -3985.978, Max-Change: 0.00050
Iteration: 130, Log-Lik: -3985.978, Max-Change: 0.00044
Iteration: 131, Log-Lik: -3985.978, Max-Change: 0.00218
Iteration: 132, Log-Lik: -3985.978, Max-Change: 0.00043
Iteration: 133, Log-Lik: -3985.978, Max-Change: 0.00217
Iteration: 134, Log-Lik: -3985.978, Max-Change: 0.00058
Iteration: 135, Log-Lik: -3985.978, Max-Change: 0.00217
Iteration: 136, Log-Lik: -3985.978, Max-Change: 0.00077
Iteration: 137, Log-Lik: -3985.978, Max-Change: 0.00043
Iteration: 138, Log-Lik: -3985.978, Max-Change: 0.00216
Iteration: 139, Log-Lik: -3985.978, Max-Change: 0.00043
Iteration: 140, Log-Lik: -3985.978, Max-Change: 0.00215
Iteration: 141, Log-Lik: -3985.978, Max-Change: 0.00215
Iteration: 142, Log-Lik: -3985.978, Max-Change: 0.00056
Iteration: 143, Log-Lik: -3985.978, Max-Change: 0.00214
Iteration: 144, Log-Lik: -3985.978, Max-Change: 0.00044
Iteration: 145, Log-Lik: -3985.978, Max-Change: 0.00043
Iteration: 146, Log-Lik: -3985.978, Max-Change: 0.00214
Iteration: 147, Log-Lik: -3985.978, Max-Change: 0.00043
Iteration: 148, Log-Lik: -3985.978, Max-Change: 0.00213
Iteration: 149, Log-Lik: -3985.978, Max-Change: 0.00052
Iteration: 150, Log-Lik: -3985.978, Max-Change: 0.00213
Iteration: 151, Log-Lik: -3985.978, Max-Change: 0.00066
Iteration: 152, Log-Lik: -3985.978, Max-Change: 0.00212
Iteration: 153, Log-Lik: -3985.978, Max-Change: 0.00053
Iteration: 154, Log-Lik: -3985.978, Max-Change: 0.00043
Iteration: 155, Log-Lik: -3985.978, Max-Change: 0.00211
Iteration: 156, Log-Lik: -3985.978, Max-Change: 0.00042
Iteration: 157, Log-Lik: -3985.978, Max-Change: 0.00042
Iteration: 158, Log-Lik: -3985.978, Max-Change: 0.00211
Iteration: 159, Log-Lik: -3985.978, Max-Change: 0.00210
Iteration: 160, Log-Lik: -3985.978, Max-Change: 0.00054
Iteration: 161, Log-Lik: -3985.978, Max-Change: 0.00210
Iteration: 162, Log-Lik: -3985.978, Max-Change: 0.00043
Iteration: 163, Log-Lik: -3985.978, Max-Change: 0.00042
Iteration: 164, Log-Lik: -3985.978, Max-Change: 0.00209
Iteration: 165, Log-Lik: -3985.978, Max-Change: 0.00042
Iteration: 166, Log-Lik: -3985.978, Max-Change: 0.00209
Iteration: 167, Log-Lik: -3985.978, Max-Change: 0.00050
Iteration: 168, Log-Lik: -3985.978, Max-Change: 0.00208
Iteration: 169, Log-Lik: -3985.978, Max-Change: 0.00064
Iteration: 170, Log-Lik: -3985.978, Max-Change: 0.00207
Iteration: 171, Log-Lik: -3985.978, Max-Change: 0.00051
Iteration: 172, Log-Lik: -3985.978, Max-Change: 0.00041
Iteration: 173, Log-Lik: -3985.978, Max-Change: 0.00207
Iteration: 174, Log-Lik: -3985.978, Max-Change: 0.00041
Iteration: 175, Log-Lik: -3985.978, Max-Change: 0.00206
Iteration: 176, Log-Lik: -3985.978, Max-Change: 0.00059
Iteration: 177, Log-Lik: -3985.977, Max-Change: 0.00206
Iteration: 178, Log-Lik: -3985.977, Max-Change: 0.00078
Iteration: 179, Log-Lik: -3985.977, Max-Change: 0.00041
Iteration: 180, Log-Lik: -3985.977, Max-Change: 0.00205
Iteration: 181, Log-Lik: -3985.977, Max-Change: 0.00041
Iteration: 182, Log-Lik: -3985.977, Max-Change: 0.00205
Iteration: 183, Log-Lik: -3985.977, Max-Change: 0.00204
Iteration: 184, Log-Lik: -3985.977, Max-Change: 0.00056
Iteration: 185, Log-Lik: -3985.977, Max-Change: 0.00204
Iteration: 186, Log-Lik: -3985.977, Max-Change: 0.00044
Iteration: 187, Log-Lik: -3985.977, Max-Change: 0.00041
Iteration: 188, Log-Lik: -3985.977, Max-Change: 0.00203
Iteration: 189, Log-Lik: -3985.977, Max-Change: 0.00041
Iteration: 190, Log-Lik: -3985.977, Max-Change: 0.00203
Iteration: 191, Log-Lik: -3985.977, Max-Change: 0.00052
Iteration: 192, Log-Lik: -3985.977, Max-Change: 0.00202
Iteration: 193, Log-Lik: -3985.977, Max-Change: 0.00066
Iteration: 194, Log-Lik: -3985.977, Max-Change: 0.00201
Iteration: 195, Log-Lik: -3985.977, Max-Change: 0.00052
Iteration: 196, Log-Lik: -3985.977, Max-Change: 0.00043
Iteration: 197, Log-Lik: -3985.977, Max-Change: 0.00201
Iteration: 198, Log-Lik: -3985.977, Max-Change: 0.00040
Iteration: 199, Log-Lik: -3985.977, Max-Change: 0.00040
Iteration: 200, Log-Lik: -3985.977, Max-Change: 0.00200
Iteration: 201, Log-Lik: -3985.977, Max-Change: 0.00200
Iteration: 202, Log-Lik: -3985.977, Max-Change: 0.00054
Iteration: 203, Log-Lik: -3985.977, Max-Change: 0.00200
Iteration: 204, Log-Lik: -3985.977, Max-Change: 0.00043
Iteration: 205, Log-Lik: -3985.977, Max-Change: 0.00040
Iteration: 206, Log-Lik: -3985.977, Max-Change: 0.00199
Iteration: 207, Log-Lik: -3985.977, Max-Change: 0.00199
Iteration: 208, Log-Lik: -3985.977, Max-Change: 0.00056
Iteration: 209, Log-Lik: -3985.977, Max-Change: 0.00198
Iteration: 210, Log-Lik: -3985.977, Max-Change: 0.00044
Iteration: 211, Log-Lik: -3985.977, Max-Change: 0.00040
Iteration: 212, Log-Lik: -3985.977, Max-Change: 0.00198
Iteration: 213, Log-Lik: -3985.977, Max-Change: 0.00039
Iteration: 214, Log-Lik: -3985.977, Max-Change: 0.00197
Iteration: 215, Log-Lik: -3985.977, Max-Change: 0.00052
Iteration: 216, Log-Lik: -3985.977, Max-Change: 0.00197
Iteration: 217, Log-Lik: -3985.977, Max-Change: 0.00067
Iteration: 218, Log-Lik: -3985.977, Max-Change: 0.00196
Iteration: 219, Log-Lik: -3985.977, Max-Change: 0.00053
Iteration: 220, Log-Lik: -3985.977, Max-Change: 0.00043
Iteration: 221, Log-Lik: -3985.977, Max-Change: 0.00196
Iteration: 222, Log-Lik: -3985.977, Max-Change: 0.00039
Iteration: 223, Log-Lik: -3985.977, Max-Change: 0.00039
Iteration: 224, Log-Lik: -3985.977, Max-Change: 0.00195
Iteration: 225, Log-Lik: -3985.977, Max-Change: 0.00195
Iteration: 226, Log-Lik: -3985.977, Max-Change: 0.00053
Iteration: 227, Log-Lik: -3985.977, Max-Change: 0.00194
Iteration: 228, Log-Lik: -3985.977, Max-Change: 0.00042
Iteration: 229, Log-Lik: -3985.977, Max-Change: 0.00039
Iteration: 230, Log-Lik: -3985.977, Max-Change: 0.00194
Iteration: 231, Log-Lik: -3985.977, Max-Change: 0.00193
Iteration: 232, Log-Lik: -3985.977, Max-Change: 0.00055
Iteration: 233, Log-Lik: -3985.977, Max-Change: 0.00193
Iteration: 234, Log-Lik: -3985.977, Max-Change: 0.00044
Iteration: 235, Log-Lik: -3985.977, Max-Change: 0.00039
Iteration: 236, Log-Lik: -3985.977, Max-Change: 0.00193
Iteration: 237, Log-Lik: -3985.977, Max-Change: 0.00038
Iteration: 238, Log-Lik: -3985.977, Max-Change: 0.00192
Iteration: 239, Log-Lik: -3985.977, Max-Change: 0.00051
Iteration: 240, Log-Lik: -3985.977, Max-Change: 0.00192
Iteration: 241, Log-Lik: -3985.977, Max-Change: 0.00066
Iteration: 242, Log-Lik: -3985.977, Max-Change: 0.00191
Iteration: 243, Log-Lik: -3985.977, Max-Change: 0.00052
Iteration: 244, Log-Lik: -3985.977, Max-Change: 0.00043
Iteration: 245, Log-Lik: -3985.977, Max-Change: 0.00191
Iteration: 246, Log-Lik: -3985.977, Max-Change: 0.00038
Iteration: 247, Log-Lik: -3985.977, Max-Change: 0.00038
Iteration: 248, Log-Lik: -3985.977, Max-Change: 0.00190
Iteration: 249, Log-Lik: -3985.977, Max-Change: 0.00190
Iteration: 250, Log-Lik: -3985.977, Max-Change: 0.00052
Iteration: 251, Log-Lik: -3985.977, Max-Change: 0.00189
Iteration: 252, Log-Lik: -3985.977, Max-Change: 0.00041
Iteration: 253, Log-Lik: -3985.977, Max-Change: 0.00038
Iteration: 254, Log-Lik: -3985.977, Max-Change: 0.00189
Iteration: 255, Log-Lik: -3985.977, Max-Change: 0.00038
Iteration: 256, Log-Lik: -3985.977, Max-Change: 0.00189
Iteration: 257, Log-Lik: -3985.977, Max-Change: 0.00048
Iteration: 258, Log-Lik: -3985.977, Max-Change: 0.00188
Iteration: 259, Log-Lik: -3985.977, Max-Change: 0.00062
Iteration: 260, Log-Lik: -3985.977, Max-Change: 0.00188
Iteration: 261, Log-Lik: -3985.977, Max-Change: 0.00049
Iteration: 262, Log-Lik: -3985.977, Max-Change: 0.00040
Iteration: 263, Log-Lik: -3985.977, Max-Change: 0.00187
Iteration: 264, Log-Lik: -3985.977, Max-Change: 0.00037
Iteration: 265, Log-Lik: -3985.977, Max-Change: 0.00187
Iteration: 266, Log-Lik: -3985.977, Max-Change: 0.00057
Iteration: 267, Log-Lik: -3985.977, Max-Change: 0.00186
Iteration: 268, Log-Lik: -3985.977, Max-Change: 0.00076
Iteration: 269, Log-Lik: -3985.976, Max-Change: 0.00037
Iteration: 270, Log-Lik: -3985.976, Max-Change: 0.00186
Iteration: 271, Log-Lik: -3985.976, Max-Change: 0.00037
Iteration: 272, Log-Lik: -3985.976, Max-Change: 0.00185
Iteration: 273, Log-Lik: -3985.976, Max-Change: 0.00185
Iteration: 274, Log-Lik: -3985.976, Max-Change: 0.00053
Iteration: 275, Log-Lik: -3985.976, Max-Change: 0.00184
Iteration: 276, Log-Lik: -3985.976, Max-Change: 0.00042
Iteration: 277, Log-Lik: -3985.976, Max-Change: 0.00037
Iteration: 278, Log-Lik: -3985.976, Max-Change: 0.00184
Iteration: 279, Log-Lik: -3985.976, Max-Change: 0.00037
Iteration: 280, Log-Lik: -3985.976, Max-Change: 0.00184
Iteration: 281, Log-Lik: -3985.976, Max-Change: 0.00049
Iteration: 282, Log-Lik: -3985.976, Max-Change: 0.00183
Iteration: 283, Log-Lik: -3985.976, Max-Change: 0.00064
Iteration: 284, Log-Lik: -3985.976, Max-Change: 0.00183
Iteration: 285, Log-Lik: -3985.976, Max-Change: 0.00050
Iteration: 286, Log-Lik: -3985.976, Max-Change: 0.00041
Iteration: 287, Log-Lik: -3985.976, Max-Change: 0.00182
Iteration: 288, Log-Lik: -3985.976, Max-Change: 0.00036
Iteration: 289, Log-Lik: -3985.976, Max-Change: 0.00182
Iteration: 290, Log-Lik: -3985.976, Max-Change: 0.00058
Iteration: 291, Log-Lik: -3985.976, Max-Change: 0.00181
Iteration: 292, Log-Lik: -3985.976, Max-Change: 0.00079
Iteration: 293, Log-Lik: -3985.976, Max-Change: 0.00036
Iteration: 294, Log-Lik: -3985.976, Max-Change: 0.00181
Iteration: 295, Log-Lik: -3985.976, Max-Change: 0.00036
Iteration: 296, Log-Lik: -3985.976, Max-Change: 0.00180
Iteration: 297, Log-Lik: -3985.976, Max-Change: 0.00180
Iteration: 298, Log-Lik: -3985.976, Max-Change: 0.00054
Iteration: 299, Log-Lik: -3985.976, Max-Change: 0.00180
Iteration: 300, Log-Lik: -3985.976, Max-Change: 0.00043
Iteration: 301, Log-Lik: -3985.976, Max-Change: 0.00036
Iteration: 302, Log-Lik: -3985.976, Max-Change: 0.00179
Iteration: 303, Log-Lik: -3985.976, Max-Change: 0.00036
Iteration: 304, Log-Lik: -3985.976, Max-Change: 0.00179
Iteration: 305, Log-Lik: -3985.976, Max-Change: 0.00050
Iteration: 306, Log-Lik: -3985.976, Max-Change: 0.00179
Iteration: 307, Log-Lik: -3985.976, Max-Change: 0.00064
Iteration: 308, Log-Lik: -3985.976, Max-Change: 0.00178
Iteration: 309, Log-Lik: -3985.976, Max-Change: 0.00051
Iteration: 310, Log-Lik: -3985.976, Max-Change: 0.00041
Iteration: 311, Log-Lik: -3985.976, Max-Change: 0.00178
Iteration: 312, Log-Lik: -3985.976, Max-Change: 0.00035
Iteration: 313, Log-Lik: -3985.976, Max-Change: 0.00177
Iteration: 314, Log-Lik: -3985.976, Max-Change: 0.00059
Iteration: 315, Log-Lik: -3985.976, Max-Change: 0.00177
Iteration: 316, Log-Lik: -3985.976, Max-Change: 0.00080
Iteration: 317, Log-Lik: -3985.976, Max-Change: 0.00035
Iteration: 318, Log-Lik: -3985.976, Max-Change: 0.00176
Iteration: 319, Log-Lik: -3985.976, Max-Change: 0.00035
Iteration: 320, Log-Lik: -3985.976, Max-Change: 0.00176
Iteration: 321, Log-Lik: -3985.976, Max-Change: 0.00176
Iteration: 322, Log-Lik: -3985.976, Max-Change: 0.00054
Iteration: 323, Log-Lik: -3985.976, Max-Change: 0.00175
Iteration: 324, Log-Lik: -3985.976, Max-Change: 0.00043
Iteration: 325, Log-Lik: -3985.976, Max-Change: 0.00035
Iteration: 326, Log-Lik: -3985.976, Max-Change: 0.00175
Iteration: 327, Log-Lik: -3985.976, Max-Change: 0.00035
Iteration: 328, Log-Lik: -3985.976, Max-Change: 0.00174
Iteration: 329, Log-Lik: -3985.976, Max-Change: 0.00050
Iteration: 330, Log-Lik: -3985.976, Max-Change: 0.00174
Iteration: 331, Log-Lik: -3985.976, Max-Change: 0.00064
Iteration: 332, Log-Lik: -3985.976, Max-Change: 0.00174
Iteration: 333, Log-Lik: -3985.976, Max-Change: 0.00051
Iteration: 334, Log-Lik: -3985.976, Max-Change: 0.00041
Iteration: 335, Log-Lik: -3985.976, Max-Change: 0.00173
Iteration: 336, Log-Lik: -3985.976, Max-Change: 0.00035
Iteration: 337, Log-Lik: -3985.976, Max-Change: 0.00173
Iteration: 338, Log-Lik: -3985.976, Max-Change: 0.00059
Iteration: 339, Log-Lik: -3985.976, Max-Change: 0.00173
Iteration: 340, Log-Lik: -3985.976, Max-Change: 0.00080
Iteration: 341, Log-Lik: -3985.976, Max-Change: 0.00034
Iteration: 342, Log-Lik: -3985.976, Max-Change: 0.00172
Iteration: 343, Log-Lik: -3985.976, Max-Change: 0.00034
Iteration: 344, Log-Lik: -3985.976, Max-Change: 0.00172
Iteration: 345, Log-Lik: -3985.976, Max-Change: 0.00171
Iteration: 346, Log-Lik: -3985.976, Max-Change: 0.00053
Iteration: 347, Log-Lik: -3985.976, Max-Change: 0.00171
Iteration: 348, Log-Lik: -3985.976, Max-Change: 0.00042
Iteration: 349, Log-Lik: -3985.976, Max-Change: 0.00035
Iteration: 350, Log-Lik: -3985.976, Max-Change: 0.00171
Iteration: 351, Log-Lik: -3985.976, Max-Change: 0.00034
Iteration: 352, Log-Lik: -3985.976, Max-Change: 0.00170
Iteration: 353, Log-Lik: -3985.976, Max-Change: 0.00049
Iteration: 354, Log-Lik: -3985.976, Max-Change: 0.00170
Iteration: 355, Log-Lik: -3985.976, Max-Change: 0.00064
Iteration: 356, Log-Lik: -3985.976, Max-Change: 0.00169
Iteration: 357, Log-Lik: -3985.976, Max-Change: 0.00051
Iteration: 358, Log-Lik: -3985.976, Max-Change: 0.00041
Iteration: 359, Log-Lik: -3985.976, Max-Change: 0.00169
Iteration: 360, Log-Lik: -3985.976, Max-Change: 0.00034
Iteration: 361, Log-Lik: -3985.976, Max-Change: 0.00169
Iteration: 362, Log-Lik: -3985.976, Max-Change: 0.00059
Iteration: 363, Log-Lik: -3985.976, Max-Change: 0.00168
Iteration: 364, Log-Lik: -3985.976, Max-Change: 0.00079
Iteration: 365, Log-Lik: -3985.976, Max-Change: 0.00034
Iteration: 366, Log-Lik: -3985.976, Max-Change: 0.00168
Iteration: 367, Log-Lik: -3985.976, Max-Change: 0.00034
Iteration: 368, Log-Lik: -3985.976, Max-Change: 0.00167
Iteration: 369, Log-Lik: -3985.976, Max-Change: 0.00167
Iteration: 370, Log-Lik: -3985.976, Max-Change: 0.00053
Iteration: 371, Log-Lik: -3985.976, Max-Change: 0.00167
Iteration: 372, Log-Lik: -3985.976, Max-Change: 0.00042
Iteration: 373, Log-Lik: -3985.976, Max-Change: 0.00034
Iteration: 374, Log-Lik: -3985.976, Max-Change: 0.00166
Iteration: 375, Log-Lik: -3985.976, Max-Change: 0.00033
Iteration: 376, Log-Lik: -3985.976, Max-Change: 0.00166
Iteration: 377, Log-Lik: -3985.976, Max-Change: 0.00049
Iteration: 378, Log-Lik: -3985.976, Max-Change: 0.00166
Iteration: 379, Log-Lik: -3985.975, Max-Change: 0.00063
Iteration: 380, Log-Lik: -3985.975, Max-Change: 0.00165
Iteration: 381, Log-Lik: -3985.975, Max-Change: 0.00050
Iteration: 382, Log-Lik: -3985.975, Max-Change: 0.00041
Iteration: 383, Log-Lik: -3985.975, Max-Change: 0.00165
Iteration: 384, Log-Lik: -3985.975, Max-Change: 0.00033
Iteration: 385, Log-Lik: -3985.975, Max-Change: 0.00165
Iteration: 386, Log-Lik: -3985.975, Max-Change: 0.00058
Iteration: 387, Log-Lik: -3985.975, Max-Change: 0.00164
Iteration: 388, Log-Lik: -3985.975, Max-Change: 0.00079
Iteration: 389, Log-Lik: -3985.975, Max-Change: 0.00033
Iteration: 390, Log-Lik: -3985.975, Max-Change: 0.00164
Iteration: 391, Log-Lik: -3985.975, Max-Change: 0.00033
Iteration: 392, Log-Lik: -3985.975, Max-Change: 0.00164
Iteration: 393, Log-Lik: -3985.975, Max-Change: 0.00163
Iteration: 394, Log-Lik: -3985.975, Max-Change: 0.00053
Iteration: 395, Log-Lik: -3985.975, Max-Change: 0.00163
Iteration: 396, Log-Lik: -3985.975, Max-Change: 0.00042
Iteration: 397, Log-Lik: -3985.975, Max-Change: 0.00034
Iteration: 398, Log-Lik: -3985.975, Max-Change: 0.00163
Iteration: 399, Log-Lik: -3985.975, Max-Change: 0.00162
Iteration: 400, Log-Lik: -3985.975, Max-Change: 0.00053
Iteration: 401, Log-Lik: -3985.975, Max-Change: 0.00162
Iteration: 402, Log-Lik: -3985.975, Max-Change: 0.00042
Iteration: 403, Log-Lik: -3985.975, Max-Change: 0.00034
Iteration: 404, Log-Lik: -3985.975, Max-Change: 0.00162
Iteration: 405, Log-Lik: -3985.975, Max-Change: 0.00032
Iteration: 406, Log-Lik: -3985.975, Max-Change: 0.00161
Iteration: 407, Log-Lik: -3985.975, Max-Change: 0.00049
Iteration: 408, Log-Lik: -3985.975, Max-Change: 0.00161
Iteration: 409, Log-Lik: -3985.975, Max-Change: 0.00063
Iteration: 410, Log-Lik: -3985.975, Max-Change: 0.00161
Iteration: 411, Log-Lik: -3985.975, Max-Change: 0.00050
Iteration: 412, Log-Lik: -3985.975, Max-Change: 0.00041
Iteration: 413, Log-Lik: -3985.975, Max-Change: 0.00160
Iteration: 414, Log-Lik: -3985.975, Max-Change: 0.00032
Iteration: 415, Log-Lik: -3985.975, Max-Change: 0.00160
Iteration: 416, Log-Lik: -3985.975, Max-Change: 0.00058
Iteration: 417, Log-Lik: -3985.975, Max-Change: 0.00160
Iteration: 418, Log-Lik: -3985.975, Max-Change: 0.00079
Iteration: 419, Log-Lik: -3985.975, Max-Change: 0.00032
Iteration: 420, Log-Lik: -3985.975, Max-Change: 0.00159
Iteration: 421, Log-Lik: -3985.975, Max-Change: 0.00033
Iteration: 422, Log-Lik: -3985.975, Max-Change: 0.00159
Iteration: 423, Log-Lik: -3985.975, Max-Change: 0.00159
Iteration: 424, Log-Lik: -3985.975, Max-Change: 0.00052
Iteration: 425, Log-Lik: -3985.975, Max-Change: 0.00158
Iteration: 426, Log-Lik: -3985.975, Max-Change: 0.00041
Iteration: 427, Log-Lik: -3985.975, Max-Change: 0.00034
Iteration: 428, Log-Lik: -3985.975, Max-Change: 0.00158
Iteration: 429, Log-Lik: -3985.975, Max-Change: 0.00158
Iteration: 430, Log-Lik: -3985.975, Max-Change: 0.00052
Iteration: 431, Log-Lik: -3985.975, Max-Change: 0.00158
Iteration: 432, Log-Lik: -3985.975, Max-Change: 0.00041
Iteration: 433, Log-Lik: -3985.975, Max-Change: 0.00034
Iteration: 434, Log-Lik: -3985.975, Max-Change: 0.00157
Iteration: 435, Log-Lik: -3985.975, Max-Change: 0.00157
Iteration: 436, Log-Lik: -3985.975, Max-Change: 0.00052
Iteration: 437, Log-Lik: -3985.975, Max-Change: 0.00157
Iteration: 438, Log-Lik: -3985.975, Max-Change: 0.00041
Iteration: 439, Log-Lik: -3985.975, Max-Change: 0.00034
Iteration: 440, Log-Lik: -3985.975, Max-Change: 0.00156
Iteration: 441, Log-Lik: -3985.975, Max-Change: 0.00031
Iteration: 442, Log-Lik: -3985.975, Max-Change: 0.00156
Iteration: 443, Log-Lik: -3985.975, Max-Change: 0.00048
Iteration: 444, Log-Lik: -3985.975, Max-Change: 0.00156
Iteration: 445, Log-Lik: -3985.975, Max-Change: 0.00062
Iteration: 446, Log-Lik: -3985.975, Max-Change: 0.00155
Iteration: 447, Log-Lik: -3985.975, Max-Change: 0.00049
Iteration: 448, Log-Lik: -3985.975, Max-Change: 0.00040
Iteration: 449, Log-Lik: -3985.975, Max-Change: 0.00155
Iteration: 450, Log-Lik: -3985.975, Max-Change: 0.00032
Iteration: 451, Log-Lik: -3985.975, Max-Change: 0.00155
Iteration: 452, Log-Lik: -3985.975, Max-Change: 0.00057
Iteration: 453, Log-Lik: -3985.975, Max-Change: 0.00155
Iteration: 454, Log-Lik: -3985.975, Max-Change: 0.00077
Iteration: 455, Log-Lik: -3985.975, Max-Change: 0.00031
Iteration: 456, Log-Lik: -3985.975, Max-Change: 0.00154
Iteration: 457, Log-Lik: -3985.975, Max-Change: 0.00033
Iteration: 458, Log-Lik: -3985.975, Max-Change: 0.00154
Iteration: 459, Log-Lik: -3985.975, Max-Change: 0.00154
Iteration: 460, Log-Lik: -3985.975, Max-Change: 0.00051
Iteration: 461, Log-Lik: -3985.975, Max-Change: 0.00153
Iteration: 462, Log-Lik: -3985.975, Max-Change: 0.00040
Iteration: 463, Log-Lik: -3985.975, Max-Change: 0.00033
Iteration: 464, Log-Lik: -3985.975, Max-Change: 0.00153
Iteration: 465, Log-Lik: -3985.975, Max-Change: 0.00153
Iteration: 466, Log-Lik: -3985.975, Max-Change: 0.00051
Iteration: 467, Log-Lik: -3985.975, Max-Change: 0.00152
Iteration: 468, Log-Lik: -3985.975, Max-Change: 0.00040
Iteration: 469, Log-Lik: -3985.975, Max-Change: 0.00033
Iteration: 470, Log-Lik: -3985.975, Max-Change: 0.00152
Iteration: 471, Log-Lik: -3985.975, Max-Change: 0.00152
Iteration: 472, Log-Lik: -3985.975, Max-Change: 0.00051
Iteration: 473, Log-Lik: -3985.975, Max-Change: 0.00152
Iteration: 474, Log-Lik: -3985.975, Max-Change: 0.00040
Iteration: 475, Log-Lik: -3985.975, Max-Change: 0.00033
Iteration: 476, Log-Lik: -3985.975, Max-Change: 0.00151
Iteration: 477, Log-Lik: -3985.975, Max-Change: 0.00151
Iteration: 478, Log-Lik: -3985.975, Max-Change: 0.00050
Iteration: 479, Log-Lik: -3985.975, Max-Change: 0.00151
Iteration: 480, Log-Lik: -3985.975, Max-Change: 0.00040
Iteration: 481, Log-Lik: -3985.975, Max-Change: 0.00033
Iteration: 482, Log-Lik: -3985.975, Max-Change: 0.00151
Iteration: 483, Log-Lik: -3985.975, Max-Change: 0.00030
Iteration: 484, Log-Lik: -3985.975, Max-Change: 0.00150
Iteration: 485, Log-Lik: -3985.975, Max-Change: 0.00046
Iteration: 486, Log-Lik: -3985.975, Max-Change: 0.00150
Iteration: 487, Log-Lik: -3985.975, Max-Change: 0.00060
Iteration: 488, Log-Lik: -3985.975, Max-Change: 0.00150
Iteration: 489, Log-Lik: -3985.975, Max-Change: 0.00048
Iteration: 490, Log-Lik: -3985.975, Max-Change: 0.00039
Iteration: 491, Log-Lik: -3985.975, Max-Change: 0.00149
Iteration: 492, Log-Lik: -3985.975, Max-Change: 0.00031
Iteration: 493, Log-Lik: -3985.975, Max-Change: 0.00149
Iteration: 494, Log-Lik: -3985.975, Max-Change: 0.00055
Iteration: 495, Log-Lik: -3985.975, Max-Change: 0.00149
Iteration: 496, Log-Lik: -3985.975, Max-Change: 0.00075
Iteration: 497, Log-Lik: -3985.975, Max-Change: 0.00030
Iteration: 498, Log-Lik: -3985.975, Max-Change: 0.00149
Iteration: 499, Log-Lik: -3985.975, Max-Change: 0.00032
Iteration: 500, Log-Lik: -3985.975, Max-Change: 0.00148

## [1] "2PL ITERATION 15. STARTED WITH 33 ITEMS. 13 OUTLIERS REMOVED. 20 ITEMS RETAINED."
## 
Iteration: 1, Log-Lik: -3212.223, Max-Change: 3.18081
Iteration: 2, Log-Lik: -2531.816, Max-Change: 0.12606
Iteration: 3, Log-Lik: -2530.866, Max-Change: 0.10354
Iteration: 4, Log-Lik: -2527.900, Max-Change: 0.11667
Iteration: 5, Log-Lik: -2526.294, Max-Change: 0.12124
Iteration: 6, Log-Lik: -2524.577, Max-Change: 0.12794
Iteration: 7, Log-Lik: -2517.252, Max-Change: 0.08871
Iteration: 8, Log-Lik: -2516.631, Max-Change: 0.05610
Iteration: 9, Log-Lik: -2516.319, Max-Change: 0.04989
Iteration: 10, Log-Lik: -2515.764, Max-Change: 0.02830
Iteration: 11, Log-Lik: -2515.724, Max-Change: 0.02513
Iteration: 12, Log-Lik: -2515.697, Max-Change: 0.01854
Iteration: 13, Log-Lik: -2515.654, Max-Change: 0.01361
Iteration: 14, Log-Lik: -2515.649, Max-Change: 0.01152
Iteration: 15, Log-Lik: -2515.646, Max-Change: 0.00834
Iteration: 16, Log-Lik: -2515.640, Max-Change: 0.00660
Iteration: 17, Log-Lik: -2515.639, Max-Change: 0.00554
Iteration: 18, Log-Lik: -2515.638, Max-Change: 0.00463
Iteration: 19, Log-Lik: -2515.636, Max-Change: 0.00219
Iteration: 20, Log-Lik: -2515.636, Max-Change: 0.00183
Iteration: 21, Log-Lik: -2515.636, Max-Change: 0.00086
Iteration: 22, Log-Lik: -2515.636, Max-Change: 0.00079
Iteration: 23, Log-Lik: -2515.636, Max-Change: 0.00073
Iteration: 24, Log-Lik: -2515.636, Max-Change: 0.00067
Iteration: 25, Log-Lik: -2515.636, Max-Change: 0.00036
Iteration: 26, Log-Lik: -2515.636, Max-Change: 0.00033
Iteration: 27, Log-Lik: -2515.636, Max-Change: 0.00031
Iteration: 28, Log-Lik: -2515.636, Max-Change: 0.00019
Iteration: 29, Log-Lik: -2515.636, Max-Change: 0.00018
Iteration: 30, Log-Lik: -2515.636, Max-Change: 0.00017
Iteration: 31, Log-Lik: -2515.636, Max-Change: 0.00011
Iteration: 32, Log-Lik: -2515.636, Max-Change: 0.00011
Iteration: 33, Log-Lik: -2515.636, Max-Change: 0.00010
Iteration: 34, Log-Lik: -2515.636, Max-Change: 0.00007

## [1] "2PL ITERATION 16. STARTED WITH 20 ITEMS. 6 OUTLIERS REMOVED. 14 ITEMS RETAINED."
## 
Iteration: 1, Log-Lik: -2059.145, Max-Change: 3.16180
Iteration: 2, Log-Lik: -1539.881, Max-Change: 0.12486
Iteration: 3, Log-Lik: -1539.469, Max-Change: 0.07027
Iteration: 4, Log-Lik: -1538.657, Max-Change: 0.08132
Iteration: 5, Log-Lik: -1538.021, Max-Change: 0.08907
Iteration: 6, Log-Lik: -1537.268, Max-Change: 0.09693
Iteration: 7, Log-Lik: -1532.571, Max-Change: 0.13901
Iteration: 8, Log-Lik: -1531.761, Max-Change: 0.07736
Iteration: 9, Log-Lik: -1531.221, Max-Change: 0.07132
Iteration: 10, Log-Lik: -1529.641, Max-Change: 0.03567
Iteration: 11, Log-Lik: -1529.584, Max-Change: 0.02756
Iteration: 12, Log-Lik: -1529.555, Max-Change: 0.02395
Iteration: 13, Log-Lik: -1529.509, Max-Change: 0.01477
Iteration: 14, Log-Lik: -1529.503, Max-Change: 0.01287
Iteration: 15, Log-Lik: -1529.499, Max-Change: 0.01020
Iteration: 16, Log-Lik: -1529.491, Max-Change: 0.00448
Iteration: 17, Log-Lik: -1529.490, Max-Change: 0.00393
Iteration: 18, Log-Lik: -1529.490, Max-Change: 0.00375
Iteration: 19, Log-Lik: -1529.489, Max-Change: 0.00086
Iteration: 20, Log-Lik: -1529.489, Max-Change: 0.00062
Iteration: 21, Log-Lik: -1529.489, Max-Change: 0.00055
Iteration: 22, Log-Lik: -1529.489, Max-Change: 0.00042
Iteration: 23, Log-Lik: -1529.489, Max-Change: 0.00039
Iteration: 24, Log-Lik: -1529.489, Max-Change: 0.00035
Iteration: 25, Log-Lik: -1529.489, Max-Change: 0.00017
Iteration: 26, Log-Lik: -1529.489, Max-Change: 0.00016
Iteration: 27, Log-Lik: -1529.489, Max-Change: 0.00014
Iteration: 28, Log-Lik: -1529.489, Max-Change: 0.00007

## [1] "2PL ITERATION 17. STARTED WITH 14 ITEMS. 6 OUTLIERS REMOVED. 8 ITEMS RETAINED."
## 
Iteration: 1, Log-Lik: -1354.096, Max-Change: 3.16052
Iteration: 2, Log-Lik: -1075.055, Max-Change: 0.02446
Iteration: 3, Log-Lik: -1074.998, Max-Change: 0.02433
Iteration: 4, Log-Lik: -1074.776, Max-Change: 0.02659
Iteration: 5, Log-Lik: -1074.700, Max-Change: 0.02759
Iteration: 6, Log-Lik: -1074.619, Max-Change: 0.02859
Iteration: 7, Log-Lik: -1074.076, Max-Change: 0.03229
Iteration: 8, Log-Lik: -1073.965, Max-Change: 0.03264
Iteration: 9, Log-Lik: -1073.854, Max-Change: 0.03286
Iteration: 10, Log-Lik: -1073.222, Max-Change: 0.03148
Iteration: 11, Log-Lik: -1073.129, Max-Change: 0.03004
Iteration: 12, Log-Lik: -1073.044, Max-Change: 0.02926
Iteration: 13, Log-Lik: -1072.648, Max-Change: 0.02357
Iteration: 14, Log-Lik: -1072.610, Max-Change: 0.02276
Iteration: 15, Log-Lik: -1072.579, Max-Change: 0.02187
Iteration: 16, Log-Lik: -1072.451, Max-Change: 0.01298
Iteration: 17, Log-Lik: -1072.443, Max-Change: 0.01203
Iteration: 18, Log-Lik: -1072.437, Max-Change: 0.01106
Iteration: 19, Log-Lik: -1072.415, Max-Change: 0.00425
Iteration: 20, Log-Lik: -1072.414, Max-Change: 0.00464
Iteration: 21, Log-Lik: -1072.413, Max-Change: 0.00421
Iteration: 22, Log-Lik: -1072.410, Max-Change: 0.00178
Iteration: 23, Log-Lik: -1072.410, Max-Change: 0.00149
Iteration: 24, Log-Lik: -1072.410, Max-Change: 0.00146
Iteration: 25, Log-Lik: -1072.410, Max-Change: 0.00057
Iteration: 26, Log-Lik: -1072.410, Max-Change: 0.00041
Iteration: 27, Log-Lik: -1072.410, Max-Change: 0.00040
Iteration: 28, Log-Lik: -1072.410, Max-Change: 0.00021
Iteration: 29, Log-Lik: -1072.410, Max-Change: 0.00023
Iteration: 30, Log-Lik: -1072.410, Max-Change: 0.00021
Iteration: 31, Log-Lik: -1072.410, Max-Change: 0.00013
Iteration: 32, Log-Lik: -1072.410, Max-Change: 0.00012
Iteration: 33, Log-Lik: -1072.410, Max-Change: 0.00011
Iteration: 34, Log-Lik: -1072.410, Max-Change: 0.00007

## [1] "2PL ITERATION 18. STARTED WITH 8 ITEMS. 4 OUTLIERS REMOVED. 4 ITEMS RETAINED."
## 
Iteration: 1, Log-Lik: -1023.604, Max-Change: 2.78577
Iteration: 2, Log-Lik: -800.998, Max-Change: 0.02096
Iteration: 3, Log-Lik: -800.976, Max-Change: 0.02013
Iteration: 4, Log-Lik: -800.898, Max-Change: 0.01736
Iteration: 5, Log-Lik: -800.874, Max-Change: 0.01697
Iteration: 6, Log-Lik: -800.848, Max-Change: 0.01758
Iteration: 7, Log-Lik: -800.685, Max-Change: 0.02044
Iteration: 8, Log-Lik: -800.654, Max-Change: 0.02086
Iteration: 9, Log-Lik: -800.623, Max-Change: 0.02111
Iteration: 10, Log-Lik: -800.436, Max-Change: 0.02134
Iteration: 11, Log-Lik: -800.406, Max-Change: 0.02137
Iteration: 12, Log-Lik: -800.377, Max-Change: 0.02110
Iteration: 13, Log-Lik: -800.221, Max-Change: 0.01738
Iteration: 14, Log-Lik: -800.201, Max-Change: 0.01841
Iteration: 15, Log-Lik: -800.183, Max-Change: 0.01768
Iteration: 16, Log-Lik: -800.092, Max-Change: 0.01241
Iteration: 17, Log-Lik: -800.083, Max-Change: 0.01182
Iteration: 18, Log-Lik: -800.075, Max-Change: 0.01113
Iteration: 19, Log-Lik: -800.039, Max-Change: 0.00747
Iteration: 20, Log-Lik: -800.035, Max-Change: 0.00714
Iteration: 21, Log-Lik: -800.033, Max-Change: 0.00678
Iteration: 22, Log-Lik: -800.021, Max-Change: 0.00464
Iteration: 23, Log-Lik: -800.020, Max-Change: 0.00442
Iteration: 24, Log-Lik: -800.019, Max-Change: 0.00419
Iteration: 25, Log-Lik: -800.015, Max-Change: 0.00280
Iteration: 26, Log-Lik: -800.015, Max-Change: 0.00263
Iteration: 27, Log-Lik: -800.015, Max-Change: 0.00248
Iteration: 28, Log-Lik: -800.013, Max-Change: 0.00187
Iteration: 29, Log-Lik: -800.013, Max-Change: 0.00173
Iteration: 30, Log-Lik: -800.013, Max-Change: 0.00161
Iteration: 31, Log-Lik: -800.013, Max-Change: 0.00081
Iteration: 32, Log-Lik: -800.013, Max-Change: 0.00074
Iteration: 33, Log-Lik: -800.013, Max-Change: 0.00070
Iteration: 34, Log-Lik: -800.013, Max-Change: 0.00044
Iteration: 35, Log-Lik: -800.013, Max-Change: 0.00041
Iteration: 36, Log-Lik: -800.013, Max-Change: 0.00038
Iteration: 37, Log-Lik: -800.013, Max-Change: 0.00025
Iteration: 38, Log-Lik: -800.013, Max-Change: 0.00023
Iteration: 39, Log-Lik: -800.013, Max-Change: 0.00022
Iteration: 40, Log-Lik: -800.013, Max-Change: 0.00014
Iteration: 41, Log-Lik: -800.013, Max-Change: 0.00013
Iteration: 42, Log-Lik: -800.013, Max-Change: 0.00013
Iteration: 43, Log-Lik: -800.013, Max-Change: 0.00008

## [1] "2PL ITERATION 19. STARTED WITH 4 ITEMS. 1 OUTLIERS REMOVED. 3 ITEMS RETAINED."
## 
Iteration: 1, Log-Lik: -769.923, Max-Change: 3.15895
Iteration: 2, Log-Lik: -548.977, Max-Change: 0.01135
Iteration: 3, Log-Lik: -548.976, Max-Change: 0.00942
Iteration: 4, Log-Lik: -548.975, Max-Change: 0.00627
Iteration: 5, Log-Lik: -548.974, Max-Change: 0.00512
Iteration: 6, Log-Lik: -548.974, Max-Change: 0.00423
Iteration: 7, Log-Lik: -548.973, Max-Change: 0.00075
Iteration: 8, Log-Lik: -548.973, Max-Change: 0.00070
Iteration: 9, Log-Lik: -548.973, Max-Change: 0.00067
Iteration: 10, Log-Lik: -548.973, Max-Change: 0.00057
Iteration: 11, Log-Lik: -548.973, Max-Change: 0.00109
Iteration: 12, Log-Lik: -548.973, Max-Change: 0.00058
Iteration: 13, Log-Lik: -548.973, Max-Change: 0.00103
Iteration: 14, Log-Lik: -548.973, Max-Change: 0.00098
Iteration: 15, Log-Lik: -548.973, Max-Change: 0.00095
Iteration: 16, Log-Lik: -548.972, Max-Change: 0.00073
Iteration: 17, Log-Lik: -548.972, Max-Change: 0.00071
Iteration: 18, Log-Lik: -548.972, Max-Change: 0.00068
Iteration: 19, Log-Lik: -548.972, Max-Change: 0.00058
Iteration: 20, Log-Lik: -548.972, Max-Change: 0.00055
Iteration: 21, Log-Lik: -548.972, Max-Change: 0.00054
Iteration: 22, Log-Lik: -548.972, Max-Change: 0.00047
Iteration: 23, Log-Lik: -548.972, Max-Change: 0.00046
Iteration: 24, Log-Lik: -548.972, Max-Change: 0.00044
Iteration: 25, Log-Lik: -548.972, Max-Change: 0.00039
Iteration: 26, Log-Lik: -548.972, Max-Change: 0.00037
Iteration: 27, Log-Lik: -548.972, Max-Change: 0.00036
Iteration: 28, Log-Lik: -548.972, Max-Change: 0.00031
Iteration: 29, Log-Lik: -548.972, Max-Change: 0.00030
Iteration: 30, Log-Lik: -548.972, Max-Change: 0.00029
Iteration: 31, Log-Lik: -548.972, Max-Change: 0.00026
Iteration: 32, Log-Lik: -548.972, Max-Change: 0.00025
Iteration: 33, Log-Lik: -548.972, Max-Change: 0.00024
Iteration: 34, Log-Lik: -548.972, Max-Change: 0.00021
Iteration: 35, Log-Lik: -548.972, Max-Change: 0.00020
Iteration: 36, Log-Lik: -548.972, Max-Change: 0.00020
Iteration: 37, Log-Lik: -548.972, Max-Change: 0.00017
Iteration: 38, Log-Lik: -548.972, Max-Change: 0.00017
Iteration: 39, Log-Lik: -548.972, Max-Change: 0.00016
Iteration: 40, Log-Lik: -548.972, Max-Change: 0.00014
Iteration: 41, Log-Lik: -548.972, Max-Change: 0.00014
Iteration: 42, Log-Lik: -548.972, Max-Change: 0.00013
Iteration: 43, Log-Lik: -548.972, Max-Change: 0.00012
Iteration: 44, Log-Lik: -548.972, Max-Change: 0.00011
Iteration: 45, Log-Lik: -548.972, Max-Change: 0.00011
Iteration: 46, Log-Lik: -548.972, Max-Change: 0.00010

## [1] "2PL ITERATION 20. STARTED WITH 3 ITEMS. 4 OUTLIERS REMOVED. -1 ITEMS RETAINED."

Show items by difficulty

coefficients <- co %>%
  rename(slope = a, difficulty = b) %>%
  arrange(difficulty) 

coefficients %>%
  kable()

words	slope	difficulty	g	u
aloe_cactus	0.0000000	-Inf	0.5	1
acorn_key	0.0721736	-43.755986	0.5	1
acorn_coconut	0.1243297	-22.422637	0.5	1
aloe_bracket	0.1709534	1.194684	0.5	1

Refit for adults vs. kids with this set of items

kept_items <- coefficients$words 

For kids with subset of items from model fit

d_wide_kid_model_subset <- pilot_data_filtered %>%
  ungroup() %>%
  filter(kid_or_adult == 'Child') %>%
  filter(item_pair %in% kept_items) %>%
  select(sub_id, item_pair, correct) %>%
  arrange(item_pair) %>%
  ungroup() %>%
  pivot_wider(names_from=item_pair, values_from=correct, values_fn = ~mean(.x)) %>%
  ungroup()

d_mat_kid_model_subset  <- d_wide_kid_model_subset %>%
  select(-sub_id) %>%
  data.frame %>%
  data.matrix 

rownames(d_mat_kid_model_subset) <- d_wide_kid_model_subset$sub_id

assertthat::assert_that(dim(d_mat_kid_model_subset)[2]==length(kept_items))

## [1] TRUE

For adults with subset of items

d_wide_adult_model_subset<- pilot_data_filtered %>%
  ungroup() %>%
  filter(kid_or_adult == 'Adult') %>%
  filter(item_pair %in% kept_items) %>%
  select(sub_id, item_pair, correct) %>%
  arrange(item_pair) %>%
  ungroup() %>%
  pivot_wider(names_from=item_pair, values_from=correct, values_fn = ~mean(.x)) %>%
  ungroup()

d_mat_adult_model_subset <- d_wide_adult_model_subset %>%
  select(-sub_id) %>%
  data.frame %>%
  data.matrix 

rownames(d_mat_adult_model_subset) <- d_wide_adult_model_subset$sub_id

assertthat::assert_that(dim(d_mat_adult_model_subset)[2]==length(kept_items))

## [1] TRUE

Refit models to kids vs. adults

Fit a 2PL model with some basic priors on difficulty/slopes

start.dim = length(colnames(d_mat_adult_model_subset))-1
mm = (
  'F = 1-%d,
PRIOR = (1-%d, a1, norm, .2, 1),
PRIOR = (1-%d, d, norm, 0, 2)'
)
mm = mirt.model(sprintf(mm,start.dim,start.dim,start.dim))

mod_2pl_priors_kid_model_subset <- mirt::mirt(d_mat_kid_model_subset, mm, itemtype='2PL',guess=.5,  upper=1, verbose=TRUE)

## 
Iteration: 1, Log-Lik: -518.529, Max-Change: 12.74522
Iteration: 2, Log-Lik: -436.208, Max-Change: 1.01626
Iteration: 3, Log-Lik: -436.185, Max-Change: 9.58284
Iteration: 4, Log-Lik: -391.924, Max-Change: 1.13680
Iteration: 5, Log-Lik: -377.318, Max-Change: 0.01866
Iteration: 6, Log-Lik: -377.315, Max-Change: 0.01603
Iteration: 7, Log-Lik: -377.308, Max-Change: 0.00452
Iteration: 8, Log-Lik: -377.308, Max-Change: 0.00475
Iteration: 9, Log-Lik: -377.308, Max-Change: 0.00379
Iteration: 10, Log-Lik: -377.307, Max-Change: 0.00180
Iteration: 11, Log-Lik: -377.307, Max-Change: 0.00290
Iteration: 12, Log-Lik: -377.307, Max-Change: 0.00130
Iteration: 13, Log-Lik: -377.307, Max-Change: 0.00114
Iteration: 14, Log-Lik: -377.307, Max-Change: 0.00098
Iteration: 15, Log-Lik: -377.307, Max-Change: 0.00042
Iteration: 16, Log-Lik: -377.307, Max-Change: 0.00203
Iteration: 17, Log-Lik: -377.307, Max-Change: 0.00032
Iteration: 18, Log-Lik: -377.307, Max-Change: 0.00152
Iteration: 19, Log-Lik: -377.307, Max-Change: 0.00113
Iteration: 20, Log-Lik: -377.307, Max-Change: 0.00021
Iteration: 21, Log-Lik: -377.307, Max-Change: 0.00085
Iteration: 22, Log-Lik: -377.307, Max-Change: 0.00013
Iteration: 23, Log-Lik: -377.307, Max-Change: 0.00061
Iteration: 24, Log-Lik: -377.307, Max-Change: 0.00048
Iteration: 25, Log-Lik: -377.307, Max-Change: 0.00022
Iteration: 26, Log-Lik: -377.307, Max-Change: 0.00003

mod_2pl_priors_adult_model_subset <- mirt::mirt(d_mat_adult_model_subset, mm, itemtype='2PL',guess=.5,  upper=1, verbose=TRUE)

## 
Iteration: 1, Log-Lik: -252.045, Max-Change: 3.52957
Iteration: 2, Log-Lik: -141.779, Max-Change: 0.01096
Iteration: 3, Log-Lik: -141.778, Max-Change: 0.00527
Iteration: 4, Log-Lik: -141.777, Max-Change: 0.00400
Iteration: 5, Log-Lik: -141.777, Max-Change: 0.00309
Iteration: 6, Log-Lik: -141.777, Max-Change: 0.00070
Iteration: 7, Log-Lik: -141.777, Max-Change: 0.00060
Iteration: 8, Log-Lik: -141.777, Max-Change: 0.00029
Iteration: 9, Log-Lik: -141.777, Max-Change: 0.00027
Iteration: 10, Log-Lik: -141.777, Max-Change: 0.00019
Iteration: 11, Log-Lik: -141.777, Max-Change: 0.00019
Iteration: 12, Log-Lik: -141.777, Max-Change: 0.00018
Iteration: 13, Log-Lik: -141.777, Max-Change: 0.00013
Iteration: 14, Log-Lik: -141.777, Max-Change: 0.00013
Iteration: 15, Log-Lik: -141.777, Max-Change: 0.00012
Iteration: 16, Log-Lik: -141.777, Max-Change: 0.00009

Munge coefficeints for plotting

coefs_2pl_kid_model_subset <- as_data_frame(coef(mod_2pl_priors_kid_model_subset, simplify = TRUE)$items) %>%
  mutate(item_pair = rownames(coef(mod_2pl_priors_kid_model_subset, simplify = TRUE)$items))

coefs_2pl_adults_model_subset <- as_data_frame(coef(mod_2pl_priors_adult_model_subset, simplify = TRUE)$items) %>%
  mutate(item_pair = rownames(coef(mod_2pl_priors_adult_model_subset, simplify = TRUE)$items))

coefs_2pl_model_subset <- coefs_2pl_adults_model_subset  %>% 
  select(d, item_pair) %>%
  mutate(cohort = 'adults') %>%
  full_join(coefs_2pl_kid_model_subset %>% select(d, item_pair) %>% mutate(cohort = 'kid')) %>%
  pivot_wider(names_from = cohort, values_from = d) %>%
  mutate(differences = adults-kid)

coefs_a1_2pl <- coefs_2pl_adults_model_subset  %>% 
  select(a1, item_pair) %>%
  mutate(cohort = 'adults') %>%
  full_join(coefs_2pl_kid_model_subset %>% select(a1, item_pair) %>% mutate(cohort = 'kid')) %>%
  pivot_wider(names_from = cohort, values_from = a1) %>%
  mutate(differences = adults-kid)

Histograms of model parameters

h1 <- ggplot(co, aes(x=a)) + 
  geom_histogram(color="black", fill="white")
h2 <- ggplot(co, aes(x=b)) + 
  geom_histogram(color="black", fill="white")

# grid.arrange(h1, h2, nrow = 1)
# g <- arrangeGrob(h1,h2, nrow=1) 
# ggsave('2PL-ModelParamsHist.png',g)

Plot adult vs. kids 2pl model subset: difficulty

Doesn’t really improve the correlation thaaaat much .63 -> .69…

ggplot(data = coefs_2pl_model_subset, aes(x=adults, y=kid)) +
  geom_point(alpha=.8) +
  geom_smooth(method='lm', color='grey') +
  theme_few() +
  ggtitle('Adult vs. Kid - 2pl IRT model coefficients - difficulty - subset from iterative fit') +
  ggrepel::geom_label_repel(aes(label = item_pair), max.overlaps=20)

### Compute correlation for difficulty

cor.test(coefs_2pl_adults$d, coefs_2pl_kid$d)

## 
##  Pearson's product-moment correlation
## 
## data:  coefs_2pl_adults$d and coefs_2pl_kid$d
## t = 12.337, df = 234, p-value < 2.2e-16
## alternative hypothesis: true correlation is not equal to 0
## 95 percent confidence interval:
##  0.5436467 0.6993978
## sample estimates:
##       cor 
## 0.6277652

Get theta estimates and individual children and show against average pc / age

thetas <- fscores(m) %>%
  as_tibble() 

sub_data <- pilot_data_filtered %>%
  distinct(sub_id, item_pair, correct,age) %>%
  group_by(sub_id) %>%
  summarize(mean_pc = mean(correct), num_trials = length(correct)) %>%
  add_column(thetas = thetas$F) 

sub_data_with_demographics <- pilot_data_filtered %>%
  distinct(age, sub_id) %>%
  right_join(sub_data) %>%
  mutate(age = replace_na(age, '9')) %>%
  mutate(age = str_replace(age, '\\+', '')) %>%
  mutate(age = str_replace(age, 'Adult', '20')) %>%
  mutate(age = as.numeric(age))

hist(sub_data_with_demographics$thetas)

Theta vs. pc

How is it possible that some kids are having high pcs and low thetas? Combination of relatively few subjects per age bin x variable items across kids and adults?

ggplot(data = sub_data_with_demographics, aes(x=thetas, y=mean_pc, size=num_trials)) +
  geom_point(alpha=.15) +
  theme_few()  +
  xlab('Theta estimates') +
  ylab('Avg PC') +
  geom_smooth(aes(weight = num_trials)) +
  facet_wrap(~age)

theta_by_age <- sub_data_with_demographics %>%
  group_by(age) %>%
  filter(!is.na(thetas)) %>% # some participants who only responded to eliminated items
  multi_boot_standard(col = 'thetas')

theta_by_age_subs <- sub_data_with_demographics %>%
  group_by(age) %>%
  summarize(num_participants = length(unique(sub_id)))

Partially compensatory 2PL model – dead end for right now

Won’t run and get bizarre error – Error: Parameter ‘d’ does not exist for respective item

Lower performance on hard vs. easy trials across age?

Look at predicted AoA vs. actual accuracy (should correlate)

Adaptive item set based on IRT model only for kids? If goal is to make it only for rocketship kids, maybe fit to rocketship kids?