Multiple booklets 2PL and GPC items - tmatta/lsasim GitHub Wiki
We examined item parameter recovery under the following conditions: 2 (IRT models) x 2 (IRT R packages) x 3 (sample sizes) x 4 (test lengths) x 3 (test booklets)
- Two types of IRT models were used: 2PL items and generalized partial credit (GPC) items.
- Item parameters were randomly generated
- The bounds of the item difficulty parameter, b, are constrained to
b_bounds = (-2, 2)where -2 is the lowest generating value and 2 is the highest generating value - The bounds of the item discrimination parameter, a, are constrained to
a_bounds = (0.75, 1.25)where 0.75 is the lowest generating value and 1.25 is the highest generating value.
- Two IRT R packages were evaluated:
TAM(version 2.4-9) andmirt(version 1.25) - Three sample sizes were used: 500, 1000, and 5000. Simulated samples were based on one ability level from distribution N(0, 1)
- Four test lengths were used: 40, 60, 80, and 100
- Three test booklets were used: 5, 10, and 15 booklets
- One hundred replications were used for each condition for the calibration.
- Summary of item parameter recovery:
TAMandmirtdemonstrated a similar level of accuracy on the values of correlation and RMSE, but not bias- In general:
- b-parameters recovered well. Sample sizes of 5000 consistently produced the most accurate results
- For a-parameter, when sample size increased, recovery accuracy inproved further
- For b- and a-parameters, four levels of test lengths performed very similarly
- For a-parameters, when number of booklets increased, recovery accuracy decreased
- Note: the following condition (N=100, I=40. K=15, r=7) was removed because of none/all subjects answered the items correctly:
- All of eleven subjects, who were assigned item 15 (i015), got the item uncorrectly
- All of fifteen subjects, who were assigned item 19 (i019), got the item correctly
# Load libraries
if(!require(lsasim)){
install.packages("lsasim")
library(lsasim) #version 1.0.1
}
if(!require(mirt)){
install.packages("mirt")
library(mirt) #version 1.25
}
if(!require(TAM)){
install.packages("TAM")
library(TAM) #version 2.4-9
}
# Set up conditions
N.cond <- c(500, 1000, 5000) #number of sample sizes
I.cond <- c(40, 60, 80, 100) #number of items
K.cond <- c(5, 10, 15) #number of booklets
# Set up number of replications
reps <- 100
# Create space for outputs
results <- NULL
#==============================================================================#
# START SIMULATION
#==============================================================================#
for (N in N.cond) { #sample size
for (I in I.cond) { #number of total items
# generate item parameters for 2PL and GPC models
set.seed(4374) # fix item parameters across replications
item_pool <- gen2PL_GPC <- lsasim::item_gen(n_2pl = c(I/2, I/2),
thresholds = c(1, 2),
b_bounds = c(-2, 2),
a_bounds = c(0.75, 1.25))
for (K in K.cond) { #number of booklets
for (r in 1:reps) { #number of replications
#------------------------------------------------------------------------------#
# Data simulation
#------------------------------------------------------------------------------#
set.seed(8088*(r+14))
# generate thetas
theta <- rnorm(N, mean=0, sd=1)
# assign items to blocks
blocks <- lsasim::block_design(n_blocks = K,
item_parameters = item_pool,
item_block_matrix = NULL)
#assign blocks to booklets
books <- lsasim::booklet_design(item_block_assignment = blocks$block_assignment,
book_design = NULL)
#assign booklets to subjects
book_samp <- lsasim::booklet_sample(n_subj = N,
book_item_design = books,
book_prob = NULL)
# generate item responses
cog <- lsasim::response_gen(subject = book_samp$subject,
item = book_samp$item,
theta = theta,
a_par = item_pool$a,
b_par = item_pool$b,
d_par = list(item_pool$d1,
item_pool$d2))
# extract item responses (excluding "subject" column)
resp <- cog[, c(1:I)]
#------------------------------------------------------------------------------#
# Item calibration
#------------------------------------------------------------------------------#
# fit 2PL and GPC models using mirt package
mirt.mod <- NULL
mirt.mod <- try(mirt::mirt(resp, 1,
temtype = c(rep('2PL', I/2), rep('gpcm', I/2)),
verbose = F))
# fit 2PL and GPC models using TAM package
tam.mod <- NULL
tam.mod <- try(TAM::tam.mml.2pl(resp, irtmodel = "GPCM",
control = list(maxiter = 200)))
#------------------------------------------------------------------------------#
# Item parameter extraction
#------------------------------------------------------------------------------#
if(!class(mirt.mod) == "try-error") {
if(!class(tam.mod) == "try-error") {
# extract item parameters in mirt package
mirt.mod.par <- coef(mirt.mod, IRTpars = TRUE, simplify=TRUE)$items
mirt.mod.par_2PL <- mirt.mod.par[c(1:I/2),]
mirt.mod.par_GPC <-mirt.mod.par[c((1/2*I+1):I),]
#--- 2PL items
mirt_2PL_b <- mirt.mod.par_2PL[,"b"]
mirt_2PL_a <- mirt.mod.par_2PL[,"a"]
#--- GPC items
mirt_GPC_b1 <- mirt.mod.par_GPC[,"b1"]
mirt_GPC_b2 <- mirt.mod.par_GPC[,"b2"]
mirt_GPC_a <- mirt.mod.par_GPC[,"a"]
# extract item parameters in TAM package
tam.mod.par_2PL <- tam.mod$item[c(1:I/2),]
tam.mod.par_GPC <- tam.mod$item[c((1/2*I+1):I),]
# convert TAM output into GPCM parametrization
#--- 2PL items
tam_2PL_b <- tam.mod.par_2PL$AXsi_.Cat1 / tam.mod.par_2PL$B.Cat1.Dim1
tam_2PL_a <- tam.mod.par_2PL$B.Cat1.Dim1
#--- GPC items
tam_GPC_b1 <- (tam.mod.par_GPC$AXsi_.Cat1 / tam.mod.par_GPC$B.Cat1.Dim1)
tam_GPC_b2 <- (tam.mod.par_GPC$AXsi_.Cat2 / tam.mod.par_GPC$B.Cat1.Dim1)
- tam_GPC_b1
tam_GPC_a <- tam.mod.par_GPC$B.Cat1.Dim1
#--------------------------------------------------------------------------#
# Item parameter recovery
#--------------------------------------------------------------------------#
# summarize results
itempars <- data.frame(matrix(c(N, I, K, r), nrow = 1))
colnames(itempars) <- c("N", "I", "K", "rep")
# retrieve generated item parameters
#--- 2PL items
gen2PL_b <- item_pool[c(1:I/2), "b"]
gen2PL_a <- item_pool[c(1:I/2), "a"]
#--- GPC items
genGPC_b1 <- item_pool[c((1/2*I+1):I), "b"] + item_pool[c((1/2*I+1):I), "d1"]
genGPC_b2 <- item_pool[c((1/2*I+1):I), "b"] + item_pool[c((1/2*I+1):I), "d2"]
genGPC_a <- item_pool[c((1/2*I+1):I), "a"]
# calculate corr, bias, RMSE for item parameters from mirt
#--- 2PL items
itempars$corr_mirt_2PL_b <- cor( gen2PL_b, mirt_2PL_b)
itempars$bias_mirt_2PL_b <- mean( mirt_2PL_b - gen2PL_b)
itempars$RMSE_mirt_2PL_b <- sqrt(mean( ( mirt_2PL_b - gen2PL_b)^2 ))
itempars$corr_mirt_2PL_a <- cor( gen2PL_a, mirt_2PL_a)
itempars$bias_mirt_2PL_a <- mean( mirt_2PL_a - gen2PL_a)
itempars$RMSE_mirt_2PL_a <- sqrt(mean( ( mirt_2PL_a - gen2PL_a)^2 ))
#--- GPC items
itempars$corr_mirt_GPC_b1 <- cor( genGPC_b1, mirt_GPC_b1)
itempars$bias_mirt_GPC_b1 <- mean( mirt_GPC_b1 - genGPC_b1)
itempars$RMSE_mirt_GPC_b1 <- sqrt(mean( ( mirt_GPC_b1 - genGPC_b1)^2 ))
itempars$corr_mirt_GPC_b2 <- cor( genGPC_b2, mirt_GPC_b2)
itempars$bias_mirt_GPC_b2 <- mean( mirt_GPC_b2 - genGPC_b2)
itempars$RMSE_mirt_GPC_b2 <- sqrt(mean( ( mirt_GPC_b2 - genGPC_b2)^2 ))
itempars$corr_mirt_GPC_a <- cor( genGPC_a, mirt_GPC_a)
itempars$bias_mirt_GPC_a <- mean( mirt_GPC_a - genGPC_a)
itempars$RMSE_mirt_GPC_a <- sqrt(mean( ( mirt_GPC_a - genGPC_a)^2 ))
# calculate corr, bias, RMSE for item parameters from TAM
#--- 2PL items
itempars$corr_tam_2PL_b <- cor( gen2PL_b, tam_2PL_b)
itempars$bias_tam_2PL_b <- mean( tam_2PL_b - gen2PL_b)
itempars$RMSE_tam_2PL_b <- sqrt(mean( ( tam_2PL_b - gen2PL_b)^2 ))
itempars$corr_tam_2PL_a <- cor( gen2PL_a, tam_2PL_a)
itempars$bias_tam_2PL_a <- mean( tam_2PL_a - gen2PL_a)
itempars$RMSE_tam_2PL_a <- sqrt(mean( ( tam_2PL_a - gen2PL_a)^2 ))
#--- GPC items
itempars$corr_tam_GPC_b1 <- cor( genGPC_b1, tam_GPC_b1)
itempars$bias_tam_GPC_b1 <- mean( tam_GPC_b1 - genGPC_b1)
itempars$RMSE_tam_GPC_b1 <- sqrt(mean( ( tam_GPC_b1 - genGPC_b1)^2 ))
itempars$corr_tam_GPC_b2 <- cor( genGPC_b2, tam_GPC_b2)
itempars$bias_tam_GPC_b2 <- mean( tam_GPC_b2 - genGPC_b2)
itempars$RMSE_tam_GPC_b2 <- sqrt(mean( ( tam_GPC_b2 - genGPC_b2)^2 ))
itempars$corr_tam_GPC_a <- cor( genGPC_a, tam_GPC_a)
itempars$bias_tam_GPC_a <- mean( tam_GPC_a - genGPC_a)
itempars$RMSE_tam_GPC_a <- sqrt(mean( ( tam_GPC_a - genGPC_a)^2 ))
#combine results
results <- rbind(results, itempars)
}
}
}
}
}
}
- Correlation, bias, and RMSE for item parameter recovery in
mirtpackage
#--- 2PL items
mirt_recovery_2PL <- aggregate(cbind(corr_mirt_2PL_b, bias_mirt_2PL_b, RMSE_mirt_2PL_b,
corr_mirt_2PL_a, bias_mirt_2PL_a, RMSE_mirt_2PL_a)
~ N + I + K,
data=results, mean, na.rm=TRUE)
names(mirt_recovery_2PL) <- c("N", "I", "K",
"corr_b", "bias_b", "RMSE_b",
"corr_a", "bias_a", "RMSE_a")
round(mirt_recovery_2PL, 3)
## N I K corr_b bias_b RMSE_b corr_a bias_a RMSE_a
## 1 500 40 5 0.967 0.021 0.345 0.559 0.037 0.281
## 2 1000 40 5 0.986 0.002 0.215 0.689 0.013 0.186
## 3 5000 40 5 0.997 0.001 0.094 0.906 0.002 0.081
## 4 500 60 5 0.969 0.005 0.288 0.578 0.032 0.256
## 5 1000 60 5 0.986 -0.006 0.189 0.727 0.014 0.171
## 6 5000 60 5 0.997 -0.004 0.081 0.922 0.002 0.076
## 7 500 80 5 0.966 0.011 0.376 0.588 0.030 0.256
## 8 1000 80 5 0.986 0.001 0.234 0.732 0.017 0.170
## 9 5000 80 5 0.998 -0.003 0.094 0.926 0.006 0.073
## 10 500 100 5 0.961 0.020 0.396 0.565 0.025 0.241
## 11 1000 100 5 0.986 -0.001 0.226 0.720 0.013 0.164
## 12 5000 100 5 0.997 -0.002 0.098 0.917 0.002 0.072
## 13 500 40 10 0.836 0.077 1.612 0.380 0.096 0.539
## 14 1000 40 10 0.956 0.037 0.412 0.500 0.055 0.350
## 15 5000 40 10 0.993 0.006 0.146 0.808 0.010 0.133
## 16 500 60 10 0.852 -0.105 2.515 0.411 0.065 0.452
## 17 1000 60 10 0.964 0.003 0.311 0.563 0.028 0.274
## 18 5000 60 10 0.994 -0.006 0.120 0.839 0.009 0.115
## 19 500 80 10 0.821 0.014 1.956 0.434 0.065 0.434
## 20 1000 80 10 0.958 0.018 0.432 0.573 0.023 0.270
## 21 5000 80 10 0.994 -0.001 0.143 0.845 0.005 0.113
## 22 500 100 10 0.820 0.095 1.603 0.410 0.070 0.436
## 23 1000 100 10 0.950 0.030 0.456 0.566 0.034 0.259
## 24 5000 100 10 0.993 0.001 0.153 0.831 0.004 0.111
## 25 500 40 15 0.657 -0.118 4.003 0.233 0.524 1.837
## 26 1000 40 15 0.849 0.071 1.338 0.352 0.112 0.598
## 27 5000 40 15 0.987 0.012 0.208 0.677 0.016 0.192
## 28 500 60 15 0.658 4.318 26.339 0.312 0.216 0.957
## 29 1000 60 15 0.898 -0.024 0.740 0.439 0.073 0.411
## 30 5000 60 15 0.989 -0.001 0.161 0.744 0.008 0.157
## 31 500 80 15 0.606 1.072 13.038 0.298 0.149 0.799
## 32 1000 80 15 0.904 0.001 0.729 0.435 0.055 0.402
## 33 5000 80 15 0.991 0.003 0.188 0.765 0.010 0.149
## 34 500 100 15 0.659 0.063 3.517 0.316 0.146 0.699
## 35 1000 100 15 0.892 -0.030 1.504 0.457 0.052 0.360
## 36 5000 100 15 0.989 0.006 0.200 0.752 0.009 0.145
#--- GPC items
mirt_recovery_GPC <- aggregate(cbind(corr_mirt_GPC_b1, bias_mirt_GPC_b1, RMSE_mirt_GPC_b1,
corr_mirt_GPC_b2, bias_mirt_GPC_b2, RMSE_mirt_GPC_b2,
corr_mirt_GPC_a, bias_mirt_GPC_a, RMSE_mirt_GPC_a)
~ N + I + K,
data=results, mean, na.rm=TRUE)
names(mirt_recovery_GPC) <- c("N", "I", "K",
"corr_b1", "bias_b1", "RMSE_b1",
"corr_b2", "bias_b2", "RMSE_b2",
"corr_a", "bias_a", "RMSE_a")
round(mirt_recovery_GPC, 3)
## N I K corr_b1 bias_b1 RMSE_b1 corr_b2 bias_b2 RMSE_b2 corr_a bias_a RMSE_a
## 1 500 40 5 0.954 -0.459 0.528 0.973 0.446 0.510 0.669 0.337 0.414
## 2 1000 40 5 0.972 -0.444 0.484 0.984 0.427 0.463 0.780 0.334 0.374
## 3 5000 40 5 0.987 -0.436 0.455 0.993 0.426 0.441 0.928 0.320 0.333
## 4 500 60 5 0.960 -0.379 0.464 0.964 0.392 0.484 0.588 0.303 0.377
## 5 1000 60 5 0.977 -0.378 0.426 0.979 0.364 0.418 0.716 0.293 0.331
## 6 5000 60 5 0.987 -0.369 0.397 0.989 0.366 0.393 0.894 0.282 0.294
## 7 500 80 5 0.951 -0.363 0.438 0.965 0.363 0.436 0.615 0.312 0.378
## 8 1000 80 5 0.970 -0.365 0.413 0.980 0.348 0.391 0.747 0.296 0.332
## 9 5000 80 5 0.985 -0.355 0.380 0.992 0.348 0.368 0.908 0.287 0.297
## 10 500 100 5 0.929 -0.383 0.461 0.958 0.370 0.433 0.647 0.313 0.376
## 11 1000 100 5 0.953 -0.387 0.438 0.973 0.360 0.401 0.761 0.300 0.334
## 12 5000 100 5 0.974 -0.372 0.399 0.986 0.359 0.382 0.923 0.291 0.301
## 13 500 40 10 0.871 -0.509 0.773 0.922 0.490 0.701 0.457 0.413 0.646
## 14 1000 40 10 0.942 -0.466 0.555 0.968 0.441 0.519 0.596 0.356 0.467
## 15 5000 40 10 0.981 -0.437 0.464 0.989 0.425 0.447 0.850 0.328 0.355
## 16 500 60 10 0.919 -0.404 0.574 0.919 0.424 0.626 0.407 0.345 0.520
## 17 1000 60 10 0.954 -0.387 0.482 0.959 0.384 0.488 0.543 0.315 0.411
## 18 5000 60 10 0.983 -0.365 0.400 0.986 0.370 0.406 0.806 0.290 0.314
## 19 500 80 10 0.892 -0.402 0.570 0.915 0.382 0.558 0.439 0.343 0.500
## 20 1000 80 10 0.945 -0.376 0.458 0.961 0.356 0.438 0.569 0.317 0.398
## 21 5000 80 10 0.980 -0.354 0.385 0.988 0.347 0.374 0.829 0.295 0.316
## 22 500 100 10 0.857 -0.420 0.592 0.913 0.386 0.518 0.484 0.340 0.479
## 23 1000 100 10 0.923 -0.394 0.479 0.953 0.369 0.439 0.622 0.312 0.384
## 24 5000 100 10 0.967 -0.371 0.407 0.982 0.357 0.386 0.858 0.296 0.315
## 25 500 40 15 0.736 -0.580 1.291 0.804 0.548 1.218 0.275 0.745 1.605
## 26 1000 40 15 0.902 -0.498 0.648 0.939 0.460 0.611 0.396 0.445 0.738
## 27 5000 40 15 0.977 -0.433 0.465 0.984 0.432 0.467 0.739 0.342 0.393
## 28 500 60 15 0.818 -0.148 2.633 0.795 2.192 10.424 0.301 0.469 0.938
## 29 1000 60 15 0.927 -0.409 0.557 0.933 0.402 0.565 0.408 0.347 0.516
## 30 5000 60 15 0.978 -0.369 0.415 0.982 0.369 0.415 0.705 0.298 0.340
## 31 500 80 15 0.821 -0.462 0.776 0.865 0.408 0.696 0.315 0.414 0.733
## 32 1000 80 15 0.910 -0.388 0.528 0.939 0.367 0.495 0.455 0.340 0.480
## 33 5000 80 15 0.974 -0.354 0.394 0.984 0.345 0.380 0.766 0.305 0.337
## 34 500 100 15 0.775 -0.431 0.887 0.854 0.422 0.645 0.361 0.394 0.654
## 35 1000 100 15 0.882 -0.402 0.538 0.927 0.373 0.481 0.533 0.340 0.458
## 36 5000 100 15 0.960 -0.373 0.416 0.978 0.358 0.392 0.795 0.304 0.333
- Correlation, bias, and RMSE for item parameter recovery in
TAMpackage
#--- 2PL items
tam_recovery_2PL <- aggregate(cbind(corr_tam_2PL_b, bias_tam_2PL_b, RMSE_tam_2PL_b,
corr_tam_2PL_a, bias_tam_2PL_a, RMSE_tam_2PL_a)
~ N + I + K,
data=results, mean, na.rm=TRUE)
names(tam_recovery_2PL) <- c("N", "I", "K",
"corr_b", "bias_b", "RMSE_b",
"corr_a", "bias_a", "RMSE_a")
round(tam_recovery_2PL, 3)
## N I K corr_b bias_b RMSE_b corr_a bias_a RMSE_a
## 1 500 40 5 0.967 0.022 0.343 0.561 0.038 0.280
## 2 1000 40 5 0.986 0.003 0.214 0.692 0.014 0.185
## 3 5000 40 5 0.997 0.003 0.094 0.907 0.003 0.081
## 4 500 60 5 0.969 0.006 0.288 0.576 0.033 0.257
## 5 1000 60 5 0.986 -0.004 0.189 0.728 0.015 0.170
## 6 5000 60 5 0.997 -0.002 0.081 0.922 0.003 0.076
## 7 500 80 5 0.967 0.013 0.375 0.588 0.031 0.256
## 8 1000 80 5 0.986 0.003 0.234 0.732 0.018 0.171
## 9 5000 80 5 0.998 -0.001 0.094 0.926 0.007 0.073
## 10 500 100 5 0.962 0.022 0.394 0.568 0.026 0.241
## 11 1000 100 5 0.986 0.004 0.228 0.722 0.013 0.164
## 12 5000 100 5 0.997 0.002 0.099 0.918 0.002 0.072
## 13 500 40 10 0.839 0.140 1.415 0.381 0.095 0.527
## 14 1000 40 10 0.956 0.037 0.410 0.503 0.055 0.346
## 15 5000 40 10 0.994 0.005 0.145 0.810 0.012 0.133
## 16 500 60 10 0.852 1.481 9.918 0.412 0.066 0.453
## 17 1000 60 10 0.964 0.003 0.310 0.563 0.030 0.274
## 18 5000 60 10 0.994 -0.006 0.120 0.840 0.010 0.115
## 19 500 80 10 0.820 -0.104 2.109 0.433 0.066 0.433
## 20 1000 80 10 0.959 0.019 0.431 0.573 0.024 0.271
## 21 5000 80 10 0.994 0.000 0.143 0.846 0.007 0.113
## 22 500 100 10 0.816 0.030 1.503 0.410 0.071 0.427
## 23 1000 100 10 0.951 0.031 0.456 0.568 0.036 0.259
## 24 5000 100 10 0.993 0.002 0.153 0.831 0.006 0.111
## 25 500 40 15 0.667 -0.035 3.185 0.247 0.465 1.662
## 26 1000 40 15 0.840 -0.678 4.222 0.351 0.114 0.599
## 27 5000 40 15 0.987 0.010 0.207 0.679 0.018 0.192
## 28 500 60 15 0.663 -0.201 3.066 0.316 0.205 0.907
## 29 1000 60 15 0.898 -0.024 0.737 0.441 0.075 0.410
## 30 5000 60 15 0.989 -0.001 0.160 0.745 0.010 0.157
## 31 500 80 15 0.608 0.403 6.499 0.298 0.149 0.787
## 32 1000 80 15 0.903 -0.014 0.796 0.438 0.056 0.398
## 33 5000 80 15 0.991 0.003 0.188 0.765 0.012 0.149
## 34 500 100 15 0.664 0.027 4.215 0.322 0.145 0.680
## 35 1000 100 15 0.887 15.328 109.465 0.459 0.055 0.362
## 36 5000 100 15 0.989 0.006 0.199 0.754 0.012 0.146
#--- GPC items
tam_recovery_GPC <- aggregate(cbind(corr_tam_GPC_b1, bias_tam_GPC_b1, RMSE_tam_GPC_b1,
corr_tam_GPC_b2, bias_tam_GPC_b2, RMSE_tam_GPC_b2,
corr_tam_GPC_a, bias_tam_GPC_a, RMSE_tam_GPC_a)
~ N + I + K,
data=results, mean, na.rm=TRUE)
names(tam_recovery_GPC) <- c("N", "I", "K",
"corr_b1", "bias_b1", "RMSE_b1",
"corr_b2", "bias_b2", "RMSE_b2",
"corr_a", "bias_a", "RMSE_a")
round(tam_recovery_GPC, 3)
## N I K corr_b1 bias_b1 RMSE_b1 corr_b2 bias_b2 RMSE_b2 corr_a bias_a RMSE_a
## 1 500 40 5 0.955 -0.011 0.255 0.969 0.002 0.264 0.649 0.023 0.198
## 2 1000 40 5 0.976 -0.008 0.178 0.984 -0.005 0.179 0.774 0.018 0.135
## 3 5000 40 5 0.995 -0.003 0.079 0.997 -0.004 0.081 0.939 0.004 0.061
## 4 500 60 5 0.967 -0.003 0.253 0.969 0.011 0.261 0.638 0.025 0.192
## 5 1000 60 5 0.984 -0.007 0.172 0.984 -0.008 0.178 0.769 0.013 0.128
## 6 5000 60 5 0.997 -0.003 0.078 0.997 -0.001 0.080 0.939 0.003 0.057
## 7 500 80 5 0.950 -0.001 0.236 0.966 0.004 0.250 0.629 0.027 0.184
## 8 1000 80 5 0.975 -0.006 0.161 0.984 -0.008 0.172 0.764 0.011 0.125
## 9 5000 80 5 0.995 -0.002 0.072 0.997 -0.001 0.074 0.935 0.002 0.055
## 10 500 100 5 0.937 -0.003 0.235 0.958 0.001 0.239 0.667 0.024 0.178
## 11 1000 100 5 0.966 -0.014 0.170 0.979 0.000 0.165 0.785 0.011 0.122
## 12 5000 100 5 0.993 0.000 0.073 0.996 -0.001 0.072 0.948 0.001 0.052
## 13 500 40 10 0.879 -0.162 1.141 0.900 -0.107 1.022 0.427 0.094 0.435
## 14 1000 40 10 0.949 -0.012 0.271 0.964 -0.009 0.288 0.575 0.037 0.251
## 15 5000 40 10 0.989 -0.003 0.115 0.993 -0.007 0.118 0.842 0.006 0.104
## 16 500 60 10 0.922 -0.005 0.410 0.926 0.016 0.431 0.438 0.065 0.348
## 17 1000 60 10 0.963 -0.007 0.265 0.965 0.000 0.277 0.583 0.033 0.229
## 18 5000 60 10 0.993 0.000 0.111 0.993 0.001 0.116 0.857 0.007 0.094
## 19 500 80 10 0.889 -0.021 0.374 0.919 0.005 0.416 0.443 0.058 0.320
## 20 1000 80 10 0.944 -0.017 0.245 0.965 -0.001 0.254 0.579 0.030 0.205
## 21 5000 80 10 0.989 -0.002 0.103 0.993 -0.002 0.106 0.855 0.006 0.087
## 22 500 100 10 0.867 -0.022 0.376 0.910 0.003 0.378 0.497 0.051 0.293
## 23 1000 100 10 0.930 -0.014 0.250 0.957 -0.001 0.244 0.638 0.022 0.189
## 24 5000 100 10 0.986 0.000 0.104 0.991 -0.005 0.105 0.883 0.005 0.082
## 25 500 40 15 0.758 -0.156 1.475 0.771 -0.010 1.559 0.262 0.352 1.147
## 26 1000 40 15 0.920 -0.030 0.367 0.931 -0.003 0.417 0.376 0.122 0.515
## 27 5000 40 15 0.984 0.001 0.142 0.988 -0.001 0.158 0.725 0.017 0.155
## 28 500 60 15 0.807 -0.369 2.680 0.802 -0.759 5.301 0.317 0.179 0.715
## 29 1000 60 15 0.940 -0.017 0.350 0.940 0.005 0.372 0.438 0.065 0.341
## 30 5000 60 15 0.988 -0.004 0.144 0.989 -0.001 0.147 0.762 0.013 0.132
## 31 500 80 15 0.830 -0.046 0.525 0.867 -0.002 0.579 0.311 0.132 0.558
## 32 1000 80 15 0.914 -0.015 0.319 0.942 -0.003 0.342 0.464 0.051 0.296
## 33 5000 80 15 0.982 -0.003 0.132 0.989 -0.004 0.134 0.786 0.013 0.116
## 34 500 100 15 0.788 -0.063 0.766 0.848 -0.002 0.638 0.368 0.106 0.489
## 35 1000 100 15 0.893 -0.012 0.321 0.930 -0.008 0.322 0.544 0.046 0.266
## 36 5000 100 15 0.978 -0.003 0.130 0.986 -0.002 0.132 0.814 0.009 0.110