Multiple booklets 3PL items - tmatta/lsasim GitHub Wiki
We examined item parameter recovery under the following conditions: 1 (IRT model) x 2 (IRT R packages) x 3 (sample sizes) x 4 (test lengths) x 3 (test booklets)
- One IRT model was included: 3PL model
- Item parameters were randomly generated
- The bounds of the item difficulty parameter, b, are constrained to
b_bounds = (-2, 2)where -2 is the lowest generating value and 2 is the highest generating value - The bounds of the item discrimination parameter, a, are constrained to
a_bounds = (0.75, 1.25)where 0.75 is the lowest generating value and 1.25 is the highest generating value - The bounds of the item guessing parameter, c, are constrained to
c_bounds = (0, 0.25)where 0 is the lowest generating value and 0.25 is the highest generating value
- Two IRT R packages were evaluated:
TAM(version 2.4-9) andmirt(version 1.25) - Three sample sizes were used: 500, 1000, and 5000
- Simulated samples were based on one ability level from distribution N(0, 1)
- Four test lengths were used: 40, 60, 80, and 100
- Three test booklets were used: 5, 10, and 15 booklets
- One hundred replications were used for each condition for the calibration
- Item parameter recovery:
TAMandmirtdemonstrated a similar level of accuracy- Sample sizes of 5000 consistently produced more accurate results
- For b-parameter, recovery accuracy didn't perform well when small sample size, fewer items, and more number of booklets
- For a- and c-parameters, the estimates reproduced poorly.
# Load libraries
if(!require(lsasim)){
install.packages("lsasim")
library(lsasim) #version 1.0.1
}
if(!require(mirt)){
install.packages("mirt")
library(mirt) #version 1.25
}
if(!require(TAM)){
install.packages("TAM")
library(TAM) #version 2.4-9
}
# Set up conditions
N.cond <- c(500, 1000, 5000) #number of sample sizes
I.cond <- c(40, 60, 80, 100) #number of items
K.cond <- c(5, 10, 15) #number of booklets
# Set up number of replications
reps <- 100
# Create space for outputs
results <- NULL
#==============================================================================#
# START SIMULATION
#==============================================================================#
for (N in N.cond) { #sample size
for (I in I.cond) { #number of items
# generate item parameters for a 3PL model
set.seed(4365) # fix item parameters across replications
item_pool <- lsasim::item_gen(n_3pl = I,
thresholds = 1,
b_bounds = c(-2, 2),
a_bounds = c(0.75, 1.25),
c_bounds = c(0, .25))
for (K in K.cond) { #number of booklets
for (r in 1:reps) { #replication
set.seed(8088*(r+3))
# generate thetas
theta <- rnorm(N, mean=0, sd=1)
# assign items to blocks
blocks <- lsasim::block_design(n_blocks = K,
item_parameters = item_pool,
item_block_matrix = NULL)
#assign blocks to booklets
books <- lsasim::booklet_design(item_block_assignment =
blocks$block_assignment,
book_design = NULL)
#assign booklets to subjects
book_samp <- lsasim::booklet_sample(n_subj = N,
book_item_design = books,
book_prob = NULL)
# generate item responses
cog <- lsasim::response_gen(subject = book_samp$subject,
item = book_samp$item,
theta = theta,
b_par = item_pool$b,
a_par = item_pool$a,
c_par = item_pool$c)
# extract item responses (excluding "subject" column)
resp <- cog[, c(1:I)]
#------------------------------------------------------------------------------#
# Item calibration
#------------------------------------------------------------------------------#
# fit 3PL model using mirt package
mirt.mod <- NULL
mirt.mod <- mirt::mirt(resp, 1, itemtype = '3PL', verbose = F,
technical = list( NCYCLES = 500))
# fit 3PL model using tam package
tam.mod <- NULL
tam.mod <- TAM::tam.mml.3pl(resp, est.guess = c(1:I),
control = list(maxiter = 200))
#------------------------------------------------------------------------------#
# Item parameter extraction
#------------------------------------------------------------------------------#
# extract b, a, c in mirt package
mirt_b <- coef(mirt.mod, IRTpars = TRUE, simplify=TRUE)$items[,"b"]
mirt_a <- coef(mirt.mod, IRTpars = TRUE, simplify=TRUE)$items[,"a"]
mirt_c <- coef(mirt.mod, IRTpars = TRUE, simplify=TRUE)$items[,"g"]
# convert TAM output into 3PL parametrization
tam_b <- (tam.mod$item$AXsi_.Cat1) / (tam.mod$item$B.Cat1.Dim1)
tam_a <- (tam.mod$item$B.Cat1.Dim1)
tam_c <- tam.mod$item$guess
#------------------------------------------------------------------------------#
# Item parameter recovery
#------------------------------------------------------------------------------#
# summarize results
itempars <- data.frame(matrix(c(N, I, K, r), nrow=1))
colnames(itempars) <- c("N", "I", "K", "rep")
# calculate corr, bias, RMSE for item parameters in mirt pacakge
itempars$corr_mirt_b <- cor( item_pool$b, mirt_b)
itempars$bias_mirt_b <- mean( mirt_b - item_pool$b )
itempars$RMSE_mirt_b <- sqrt(mean( ( mirt_b - item_pool$b)^2 ))
itempars$corr_mirt_a <- cor( item_pool$a, mirt_a)
itempars$bias_mirt_a <- mean( mirt_a - item_pool$a )
itempars$RMSE_mirt_a <- sqrt(mean( ( mirt_a - item_pool$a)^2 ))
itempars$corr_mirt_c <- cor( item_pool$c, mirt_c)
itempars$bias_mirt_c <- mean( mirt_c - item_pool$c )
itempars$RMSE_mirt_c <- sqrt(mean( ( mirt_c - item_pool$c)^2 ))
# calculate corr, bias, RMSE for item parameters in TAM pacakge
itempars$corr_tam_b <- cor( item_pool$b, tam_b)
itempars$bias_tam_b <- mean( tam_b - item_pool$b )
itempars$RMSE_tam_b <- sqrt(mean( ( tam_b - item_pool$b)^2 ))
itempars$corr_tam_a <- cor( item_pool$a, tam_a)
itempars$bias_tam_a <- mean( tam_a - item_pool$a )
itempars$RMSE_tam_a <- sqrt(mean( ( tam_a - item_pool$a)^2 ))
itempars$corr_tam_c <- cor( item_pool$c, tam_c)
itempars$bias_tam_c <- mean( tam_c - item_pool$c )
itempars$RMSE_tam_c <- sqrt(mean( ( tam_c - item_pool$c)^2 ))
# combine results
results <- rbind(results, itempars)
}
}
}
}
\newpage
- Correlation, bias, and RMSE for item parameter recovery in
mirtpackage
mirt_recovery <- aggregate(cbind(corr_mirt_b, bias_mirt_b, RMSE_mirt_b,
corr_mirt_a, bias_mirt_a, RMSE_mirt_a,
corr_mirt_c, bias_mirt_c, RMSE_mirt_c) ~ N + I + K,
data=results, mean, na.rm=TRUE)
names(mirt_recovery) <- c("N", "I", "K",
"corr_b", "bias_b", "RMSE_b",
"corr_a", "bias_a", "RMSE_a",
"corr_c", "bias_c", "RMSE_c")
round(mirt_recovery, 3)
## N I K corr_b bias_b RMSE_b corr_a bias_a RMSE_a corr_c bias_c RMSE_c
## 1 500 40 5 0.638 0.559 3.305 0.014 1.090 3.367 0.108 0.005 0.188
## 2 1000 40 5 0.826 0.311 1.204 0.055 0.188 1.182 0.142 -0.018 0.156
## 3 5000 40 5 0.967 0.147 0.386 0.372 -0.186 0.344 0.286 -0.058 0.105
## 4 500 60 5 0.688 0.418 2.094 0.030 0.642 2.473 0.085 0.001 0.180
## 5 1000 60 5 0.826 0.214 1.157 0.114 0.043 0.753 0.116 -0.018 0.153
## 6 5000 60 5 0.961 0.163 0.376 0.390 -0.186 0.297 0.208 -0.052 0.110
## 7 500 80 5 0.645 0.317 2.824 0.007 0.634 2.563 0.105 -0.006 0.179
## 8 1000 80 5 0.846 0.266 0.914 0.108 0.020 0.744 0.121 -0.026 0.153
## 9 5000 80 5 0.968 0.176 0.381 0.321 -0.199 0.314 0.259 -0.061 0.112
## 10 500 100 5 0.575 0.837 8.561 0.008 0.568 2.418 0.062 -0.007 0.178
## 11 1000 100 5 0.851 0.251 0.822 0.104 0.008 0.721 0.110 -0.024 0.148
## 12 5000 100 5 0.971 0.174 0.394 0.299 -0.193 0.319 0.243 -0.057 0.108
## 13 500 40 10 0.298 26.174 279.504 0.018 4.851 9.613 0.092 0.045 0.221
## 14 1000 40 10 0.453 0.277 4.706 0.038 1.907 4.542 0.109 0.016 0.199
## 15 5000 40 10 0.893 0.238 0.652 0.153 -0.003 0.680 0.192 -0.031 0.141
## 16 500 60 10 0.299 0.280 8.514 0.017 4.111 9.161 0.064 0.035 0.216
## 17 1000 60 10 0.555 -1.613 17.239 -0.020 1.275 3.564 0.093 0.008 0.191
## 18 5000 60 10 0.908 0.204 0.552 0.194 -0.099 0.450 0.124 -0.037 0.137
## 19 500 80 10 0.234 0.742 8.811 0.009 3.817 9.146 0.068 0.021 0.216
## 20 1000 80 10 0.553 -1.172 17.970 -0.009 1.116 3.473 0.088 -0.003 0.192
## 21 5000 80 10 0.921 0.225 0.545 0.153 -0.126 0.455 0.161 -0.048 0.137
## 22 500 100 10 0.287 0.098 12.332 0.004 3.696 9.194 0.032 0.016 0.212
## 23 1000 100 10 0.514 -0.096 8.694 -0.019 0.942 3.229 0.064 -0.005 0.186
## 24 5000 100 10 0.930 0.199 0.544 0.182 -0.149 0.426 0.139 -0.050 0.132
## 25 500 40 15 0.238 0.446 5.853 0.036 6.446 13.155 0.063 0.050 0.229
## 26 1000 40 15 0.317 1.104 11.453 0.048 3.959 7.425 0.120 0.057 0.227
## 27 5000 40 15 0.768 0.224 1.676 0.040 0.533 1.685 0.172 -0.002 0.171
## 28 500 60 15 0.208 0.796 6.986 0.014 7.422 14.385 0.051 0.053 0.231
## 29 1000 60 15 0.316 -0.297 13.083 0.063 3.459 7.032 0.080 0.042 0.218
## 30 5000 60 15 0.832 0.282 0.731 0.063 0.147 1.017 0.114 -0.018 0.162
## 31 500 80 15 0.208 3.549 36.298 -0.005 8.194 16.032 0.051 0.049 0.235
## 32 1000 80 15 0.310 1.186 17.666 0.000 2.782 6.499 0.070 0.018 0.214
## 33 5000 80 15 0.838 0.256 1.126 0.082 0.036 0.834 0.138 -0.034 0.156
## 34 500 100 15 0.183 0.285 12.781 0.002 8.242 16.253 0.045 0.049 0.232
## 35 1000 100 15 0.351 0.473 5.032 0.017 2.322 5.764 0.034 0.013 0.210
## 36 5000 100 15 0.858 0.235 0.897 0.076 -0.031 0.689 0.121 -0.038 0.149
\newpage
- Correlation, bias, and RMSE for item parameter recovery in
TAMpackage
tam_recovery <- aggregate(cbind(corr_tam_b, bias_tam_b, RMSE_tam_b,
corr_tam_a, bias_tam_a, RMSE_tam_a,
corr_tam_c, bias_tam_c, RMSE_tam_c) ~ N + I + K,
data=results, mean, na.rm=TRUE)
names(tam_recovery) <- c("N", "I", "K",
"corr_b", "bias_b", "RMSE_b",
"corr_a", "bias_a", "RMSE_a",
"corr_c", "bias_c", "RMSE_c")
round(tam_recovery, 3)
## N I K corr_b bias_b RMSE_b corr_a bias_a RMSE_a corr_c bias_c RMSE_c
## 1 500 40 5 0.655 -0.088 3.739 0.133 0.384 1.405 0.039 -0.033 0.152
## 2 1000 40 5 0.831 0.191 1.103 0.153 0.129 0.756 0.065 -0.046 0.138
## 3 5000 40 5 0.975 0.078 0.277 0.395 -0.069 0.307 0.158 -0.079 0.116
## 4 500 60 5 0.698 0.395 3.156 0.110 0.399 1.279 0.039 -0.025 0.156
## 5 1000 60 5 0.866 0.145 0.698 0.198 0.152 0.653 0.073 -0.038 0.141
## 6 5000 60 5 0.975 0.082 0.259 0.416 -0.017 0.280 0.137 -0.069 0.117
## 7 500 80 5 0.661 0.212 1.747 0.084 0.484 1.356 0.061 -0.030 0.161
## 8 1000 80 5 0.869 0.144 0.784 0.169 0.205 0.707 0.078 -0.045 0.145
## 9 5000 80 5 0.975 0.094 0.281 0.343 0.018 0.307 0.196 -0.077 0.122
## 10 500 100 5 0.604 0.211 2.904 0.060 0.540 1.451 0.007 -0.031 0.159
## 11 1000 100 5 0.867 0.130 0.690 0.154 0.236 0.743 0.054 -0.042 0.142
## 12 5000 100 5 0.976 0.093 0.275 0.322 0.041 0.328 0.178 -0.071 0.118
## 13 500 40 10 0.244 0.484 35.223 0.095 0.823 2.619 0.010 -0.034 0.160
## 14 1000 40 10 0.507 0.294 5.543 0.138 0.367 1.542 -0.002 -0.041 0.153
## 15 5000 40 10 0.940 0.107 0.473 0.301 -0.095 0.434 0.028 -0.075 0.125
## 16 500 60 10 0.275 -0.572 14.039 0.083 0.874 2.549 0.036 -0.023 0.163
## 17 1000 60 10 0.579 0.617 7.065 0.074 0.403 1.407 0.033 -0.030 0.157
## 18 5000 60 10 0.947 0.093 0.372 0.282 -0.032 0.411 0.046 -0.063 0.129
## 19 500 80 10 0.237 -0.108 12.508 0.041 0.904 2.605 0.017 -0.028 0.175
## 20 1000 80 10 0.553 -0.011 6.316 0.073 0.366 1.400 0.032 -0.042 0.162
## 21 5000 80 10 0.949 0.119 0.374 0.222 -0.013 0.429 0.077 -0.072 0.136
## 22 500 100 10 0.264 -0.351 14.145 0.049 0.877 2.546 -0.013 -0.029 0.173
## 23 1000 100 10 0.516 0.110 4.218 0.047 0.409 1.443 -0.014 -0.039 0.162
## 24 5000 100 10 0.949 0.105 0.377 0.239 0.001 0.421 0.061 -0.070 0.133
## 25 500 40 15 0.140 -0.865 23.999 0.071 1.066 3.408 0.011 -0.030 0.161
## 26 1000 40 15 0.243 0.621 16.772 0.112 0.496 2.032 -0.008 -0.045 0.151
## 27 5000 40 15 0.810 0.125 1.700 0.233 -0.097 0.558 -0.013 -0.077 0.129
## 28 500 60 15 0.157 0.094 16.464 0.069 1.246 3.383 0.010 -0.022 0.168
## 29 1000 60 15 0.319 0.368 11.374 0.102 0.635 2.056 0.003 -0.032 0.160
## 30 5000 60 15 0.910 0.091 0.554 0.226 -0.009 0.567 0.007 -0.061 0.135
## 31 500 80 15 0.149 2.892 34.831 0.061 1.158 3.418 0.040 -0.030 0.174
## 32 1000 80 15 0.255 0.453 11.198 0.051 0.615 2.116 0.020 -0.042 0.167
## 33 5000 80 15 0.897 0.247 1.496 0.176 -0.005 0.579 0.047 -0.070 0.141
## 34 500 100 15 0.136 0.326 19.149 0.059 1.153 3.334 -0.005 -0.027 0.176
## 35 1000 100 15 0.310 4.103 55.752 0.052 0.637 2.103 -0.027 -0.037 0.170
## 36 5000 100 15 0.893 0.110 0.719 0.174 0.005 0.561 0.022 -0.068 0.141