--- title: "sim-validation" output: rmarkdown::html_vignette vignette: > %\VignetteIndexEntry{sim-validation} %\VignetteEngine{knitr::rmarkdown} %\VignetteEncoding{UTF-8} --- ```{r, include = FALSE} knitr::opts_chunk$set( collapse = TRUE, comment = "#>" ) knitr::opts_chunk$set(echo = TRUE) knitr::opts_chunk$set(message = FALSE) knitr::opts_chunk$set(warning = FALSE) knitr::opts_chunk$set(results = "hide") knitr::opts_chunk$set(fig.dim = c(6,4)) ``` ```{r setup} library("splithalf") library("tidyr") ``` # Validaion Sim for splithalf This script runs _splithalf_ with many possible combinations of the options. Its main use is to ensure that the _splithalf_ function # Splithalf() ```{r generate_data} # Thanks to Nathaniel Haines for the suggestion and the starting code (https://github.com/sdparsons/splithalf/issues/9). set.seed(01042021) n_participants <- 60 ## sample size n_trials <- 80 n_blocks <- 2 sim_data <- data.frame(participant_number = rep(1:n_participants, each = n_blocks * n_trials), trial_number = rep(1:n_trials, times = n_blocks * n_participants), block_name = rep(c("A","B"), each = n_trials, length.out = n_participants * n_trials * n_blocks), trial_type = rep(c("congruent","congruent", "incongruent","incongruent"), length.out = n_participants * n_trials * n_blocks / 2), RT = rnorm(n_participants * n_trials * n_blocks, 500, 200), ACC = 1) sim_data_acc <- data.frame(participant_number = rep(1:n_participants, each = n_blocks * n_trials), trial_number = rep(1:n_trials, times = n_blocks * n_participants), block_name = rep(c("A","B"), each = n_trials, length.out = n_participants * n_trials * n_blocks), trial_type = rep(c("congruent","congruent", "incongruent","incongruent"), length.out = n_participants * n_trials * n_blocks / 2), RT = rnorm(n_participants * n_trials * n_blocks, 500, 200), ACC = c(rbinom(n_participants * n_trials * n_blocks / 6, 1, .5), rbinom(n_participants * n_trials * n_blocks / 6, 1, .7), rbinom(n_participants * n_trials * n_blocks / 6, 1, .9), rbinom(n_participants * n_trials * n_blocks / 6, 1, .5), rbinom(n_participants * n_trials * n_blocks / 6, 1, .7), rbinom(n_participants * n_trials * n_blocks / 6, 1, .9))) ``` ```{r test_setup} params <- list(score = c("average", "difference", "difference_of_difference"), halftype = c("oddeven", "halfs", "random"), averae = c("mean", "median") ) expand.grid(params) %>% dplyr::arrange(., score, halftype) ``` ## Response times ```{r rt_test1} splithalf(data = sim_data, outcome = "RT", score = "average", conditionlist = c("A", "B"), halftype = "oddeven", permutations = 500, var.RT = "RT", var.condition = "block_name", var.participant = "participant_number", var.compare = "trial_type", compare1 = "congruent", compare2 = "incongruent", average = "mean") ``` ```{r rt_test2} splithalf(data = sim_data, outcome = "RT", score = "average", conditionlist = c("A", "B"), halftype = "oddeven", permutations = 500, var.RT = "RT", var.condition = "block_name", var.participant = "participant_number", var.compare = "trial_type", compare1 = "congruent", compare2 = "incongruent", average = "median") ``` ```{r rt_test3} splithalf(data = sim_data, outcome = "RT", score = "average", conditionlist = c("A", "B"), halftype = "halfs", permutations = 500, var.RT = "RT", var.condition = "block_name", var.participant = "participant_number", var.compare = "trial_type", compare1 = "congruent", compare2 = "incongruent", average = "mean") ``` ```{r rt_test4} splithalf(data = sim_data, outcome = "RT", score = "average", conditionlist = c("A", "B"), halftype = "halfs", permutations = 500, var.RT = "RT", var.condition = "block_name", var.participant = "participant_number", var.compare = "trial_type", compare1 = "congruent", compare2 = "incongruent", average = "median") ``` ```{r rt_test5} splithalf(data = sim_data, outcome = "RT", score = "average", conditionlist = c("A", "B"), halftype = "random", permutations = 500, var.RT = "RT", var.condition = "block_name", var.participant = "participant_number", var.compare = "trial_type", compare1 = "congruent", compare2 = "incongruent", average = "mean", plot = TRUE) ``` ```{r rt_test6} splithalf(data = sim_data, outcome = "RT", score = "average", conditionlist = c("A", "B"), halftype = "random", permutations = 500, var.RT = "RT", var.condition = "block_name", var.participant = "participant_number", var.compare = "trial_type", compare1 = "congruent", compare2 = "incongruent", average = "median", plot = TRUE) ``` ```{r rt_test7} splithalf(data = sim_data, outcome = "RT", score = "difference", conditionlist = c("A", "B"), halftype = "oddeven", permutations = 500, var.RT = "RT", var.condition = "block_name", var.participant = "participant_number", var.compare = "trial_type", compare1 = "congruent", compare2 = "incongruent", average = "mean") ``` ```{r rt_test8} splithalf(data = sim_data, outcome = "RT", score = "difference", conditionlist = c("A", "B"), halftype = "oddeven", permutations = 500, var.RT = "RT", var.condition = "block_name", var.participant = "participant_number", var.compare = "trial_type", compare1 = "congruent", compare2 = "incongruent", average = "median") ``` ```{r rt_test9} splithalf(data = sim_data, outcome = "RT", score = "difference", conditionlist = c("A", "B"), halftype = "halfs", permutations = 500, var.RT = "RT", var.condition = "block_name", var.participant = "participant_number", var.compare = "trial_type", compare1 = "congruent", compare2 = "incongruent", average = "mean") ``` ```{r rt_test10} splithalf(data = sim_data, outcome = "RT", score = "difference", conditionlist = c("A", "B"), halftype = "halfs", permutations = 500, var.RT = "RT", var.condition = "block_name", var.participant = "participant_number", var.compare = "trial_type", compare1 = "congruent", compare2 = "incongruent", average = "median") ``` ```{r rt_test11} splithalf(data = sim_data, outcome = "RT", score = "difference", conditionlist = c("A", "B"), halftype = "random", permutations = 500, var.RT = "RT", var.condition = "block_name", var.participant = "participant_number", var.compare = "trial_type", compare1 = "congruent", compare2 = "incongruent", average = "mean", plot = TRUE) ``` ```{r rt_test12} splithalf(data = sim_data, outcome = "RT", score = "difference", conditionlist = c("A", "B"), halftype = "random", permutations = 500, var.RT = "RT", var.condition = "block_name", var.participant = "participant_number", var.compare = "trial_type", compare1 = "congruent", compare2 = "incongruent", average = "median", plot = TRUE) ``` ```{r rt_test13} splithalf(data = sim_data, outcome = "RT", score = "difference_of_difference", conditionlist = c("A", "B"), halftype = "oddeven", permutations = 500, var.RT = "RT", var.condition = "block_name", var.participant = "participant_number", var.compare = "trial_type", compare1 = "congruent", compare2 = "incongruent", average = "mean") ``` ```{r rt_test14} splithalf(data = sim_data, outcome = "RT", score = "difference_of_difference", conditionlist = c("A", "B"), halftype = "oddeven", permutations = 500, var.RT = "RT", var.condition = "block_name", var.participant = "participant_number", var.compare = "trial_type", compare1 = "congruent", compare2 = "incongruent", average = "median") ``` ```{r rt_test15} splithalf(data = sim_data, outcome = "RT", score = "difference_of_difference", conditionlist = c("A", "B"), halftype = "halfs", permutations = 500, var.RT = "RT", var.condition = "block_name", var.participant = "participant_number", var.compare = "trial_type", compare1 = "congruent", compare2 = "incongruent", average = "mean") ``` ```{r rt_test16} splithalf(data = sim_data, outcome = "RT", score = "difference_of_difference", conditionlist = c("A", "B"), halftype = "halfs", permutations = 500, var.RT = "RT", var.condition = "block_name", var.participant = "participant_number", var.compare = "trial_type", compare1 = "congruent", compare2 = "incongruent", average = "median") ``` ```{r rt_test17} splithalf(data = sim_data, outcome = "RT", score = "difference_of_difference", conditionlist = c("A", "B"), halftype = "random", permutations = 500, var.RT = "RT", var.condition = "block_name", var.participant = "participant_number", var.compare = "trial_type", compare1 = "congruent", compare2 = "incongruent", average = "mean", plot = TRUE) ``` ```{r rt_test18} splithalf(data = sim_data, outcome = "RT", score = "difference_of_difference", conditionlist = c("A", "B"), halftype = "random", permutations = 500, var.RT = "RT", var.condition = "block_name", var.participant = "participant_number", var.compare = "trial_type", compare1 = "congruent", compare2 = "incongruent", average = "median", plot = TRUE) ``` ## Average accuracy rates ```{r acc_test1} splithalf(data = sim_data_acc, outcome = "accuracy", score = "average", conditionlist = c("A", "B"), halftype = "oddeven", permutations = 500, var.RT = "RT", var.condition = "block_name", var.participant = "participant_number", var.compare = "trial_type", var.ACC = "ACC", compare1 = "congruent", compare2 = "incongruent", average = "mean") ``` ```{r acc_test2} splithalf(data = sim_data_acc, outcome = "accuracy", score = "average", conditionlist = c("A", "B"), halftype = "oddeven", permutations = 500, var.RT = "RT", var.condition = "block_name", var.participant = "participant_number", var.compare = "trial_type", var.ACC = "ACC", compare1 = "congruent", compare2 = "incongruent", average = "median") ``` ```{r acc_test3} splithalf(data = sim_data_acc, outcome = "accuracy", score = "average", conditionlist = c("A", "B"), halftype = "halfs", permutations = 500, var.RT = "RT", var.condition = "block_name", var.participant = "participant_number", var.compare = "trial_type", var.ACC = "ACC", compare1 = "congruent", compare2 = "incongruent", average = "mean") ``` ```{r acc_test4} splithalf(data = sim_data_acc, outcome = "accuracy", score = "average", conditionlist = c("A", "B"), halftype = "halfs", permutations = 500, var.RT = "RT", var.condition = "block_name", var.participant = "participant_number", var.compare = "trial_type", var.ACC = "ACC", compare1 = "congruent", compare2 = "incongruent", average = "median") ``` ```{r acc_test5} splithalf(data = sim_data_acc, outcome = "accuracy", score = "average", conditionlist = c("A", "B"), halftype = "random", permutations = 500, var.RT = "RT", var.condition = "block_name", var.participant = "participant_number", var.compare = "trial_type", var.ACC = "ACC", compare1 = "congruent", compare2 = "incongruent", average = "mean", plot = TRUE) ``` ```{r acc_test6} splithalf(data = sim_data_acc, outcome = "accuracy", score = "average", conditionlist = c("A", "B"), halftype = "random", permutations = 500, var.RT = "RT", var.condition = "block_name", var.participant = "participant_number", var.compare = "trial_type", var.ACC = "ACC", compare1 = "congruent", compare2 = "incongruent", average = "median", plot = TRUE) ``` ```{r acc_test7} splithalf(data = sim_data_acc, outcome = "accuracy", score = "difference", conditionlist = c("A", "B"), halftype = "oddeven", permutations = 500, var.RT = "RT", var.condition = "block_name", var.participant = "participant_number", var.compare = "trial_type", var.ACC = "ACC", compare1 = "congruent", compare2 = "incongruent", average = "mean") ``` ```{r acc_test8} splithalf(data = sim_data_acc, outcome = "accuracy", score = "difference", conditionlist = c("A", "B"), halftype = "oddeven", permutations = 500, var.RT = "RT", var.condition = "block_name", var.participant = "participant_number", var.compare = "trial_type", var.ACC = "ACC", compare1 = "congruent", compare2 = "incongruent", average = "median") ``` ```{r acc_test9} splithalf(data = sim_data_acc, outcome = "accuracy", score = "difference", conditionlist = c("A", "B"), halftype = "halfs", permutations = 500, var.RT = "RT", var.condition = "block_name", var.participant = "participant_number", var.compare = "trial_type", var.ACC = "ACC", compare1 = "congruent", compare2 = "incongruent", average = "mean") ``` ```{r acc_test10} splithalf(data = sim_data_acc, outcome = "accuracy", score = "difference", conditionlist = c("A", "B"), halftype = "halfs", permutations = 500, var.RT = "RT", var.condition = "block_name", var.participant = "participant_number", var.compare = "trial_type", var.ACC = "ACC", compare1 = "congruent", compare2 = "incongruent", average = "median") ``` ```{r acc_test11} splithalf(data = sim_data_acc, outcome = "accuracy", score = "difference", conditionlist = c("A", "B"), halftype = "random", permutations = 500, var.RT = "RT", var.condition = "block_name", var.participant = "participant_number", var.compare = "trial_type", var.ACC = "ACC", compare1 = "congruent", compare2 = "incongruent", average = "mean", plot = TRUE) ``` ```{r acc_test12} splithalf(data = sim_data_acc, outcome = "accuracy", score = "difference", conditionlist = c("A", "B"), halftype = "random", permutations = 500, var.RT = "RT", var.condition = "block_name", var.participant = "participant_number", var.compare = "trial_type", var.ACC = "ACC", compare1 = "congruent", compare2 = "incongruent", average = "median", plot = TRUE) ``` ```{r acc_test13} splithalf(data = sim_data_acc, outcome = "accuracy", score = "difference_of_difference", conditionlist = c("A", "B"), halftype = "oddeven", permutations = 500, var.RT = "RT", var.condition = "block_name", var.participant = "participant_number", var.compare = "trial_type", var.ACC = "ACC", compare1 = "congruent", compare2 = "incongruent", average = "mean") ``` ```{r acc_test14} splithalf(data = sim_data_acc, outcome = "accuracy", score = "difference_of_difference", conditionlist = c("A", "B"), halftype = "oddeven", permutations = 500, var.RT = "RT", var.condition = "block_name", var.participant = "participant_number", var.compare = "trial_type", var.ACC = "ACC", compare1 = "congruent", compare2 = "incongruent", average = "median") ``` ```{r acc_test15} splithalf(data = sim_data_acc, outcome = "accuracy", score = "difference_of_difference", conditionlist = c("A", "B"), halftype = "halfs", permutations = 500, var.RT = "RT", var.condition = "block_name", var.participant = "participant_number", var.compare = "trial_type", var.ACC = "ACC", compare1 = "congruent", compare2 = "incongruent", average = "mean") ``` ```{r acc_test16} splithalf(data = sim_data_acc, outcome = "accuracy", score = "difference_of_difference", conditionlist = c("A", "B"), halftype = "halfs", permutations = 500, var.RT = "RT", var.condition = "block_name", var.participant = "participant_number", var.compare = "trial_type", var.ACC = "ACC", compare1 = "congruent", compare2 = "incongruent", average = "median") ``` ```{r acc_test17} splithalf(data = sim_data_acc, outcome = "accuracy", score = "difference_of_difference", conditionlist = c("A", "B"), halftype = "random", permutations = 500, var.RT = "RT", var.condition = "block_name", var.participant = "participant_number", var.compare = "trial_type", var.ACC = "ACC", compare1 = "congruent", compare2 = "incongruent", average = "mean", plot = TRUE) ``` ```{r acc_test18} splithalf(data = sim_data_acc, outcome = "accuracy", score = "difference_of_difference", conditionlist = c("A", "B"), halftype = "random", permutations = 500, var.RT = "RT", var.condition = "block_name", var.participant = "participant_number", var.compare = "trial_type", var.ACC = "ACC", compare1 = "congruent", compare2 = "incongruent", average = "median", plot = TRUE) ``` # multiverse Runs several multiverse ```{r generate_mv_data} set.seed(01042021) n_participants <- 80 ## sample size n_trials <- 120 n_blocks <- 2 sim_data_mv <- data.frame(participant_number = rep(1:n_participants, each = n_blocks * n_trials), trial_number = rep(1:n_trials, times = n_blocks * n_participants), block_name = rep(c(1,2), each = n_trials, length.out = n_participants * n_trials * n_blocks), trial_type = rep(c("congruent","congruent", "incongruent","incongruent"), length.out = n_participants * n_trials * n_blocks / 2), RT = rnorm(n_participants * n_trials * n_blocks, 500, 200), ACC = c(rbinom(n_participants * n_trials * n_blocks / 6, 1, .5), rbinom(n_participants * n_trials * n_blocks / 6, 1, .7), rbinom(n_participants * n_trials * n_blocks / 6, 1, .9), rbinom(n_participants * n_trials * n_blocks / 6, 1, .5), rbinom(n_participants * n_trials * n_blocks / 6, 1, .7), rbinom(n_participants * n_trials * n_blocks / 6, 1, .9))) ``` ```{r create_specs} specifications <- list( ACC_cutoff = c(0, 0.5), RT_min = c(0, 200), RT_max = c(2000, 3000), RT_sd_cutoff = c(0, 2), split_by = c("subject", "trial"), averaging_method = c("mean") ) ``` ## internal consistency ```{r splithalf_for_mv, comment=FALSE, results='hide'} mv_splithalf <- splithalf(data = sim_data, outcome = "RT", score = "difference", conditionlist = c("A"), halftype = "random", permutations = 50, var.RT = "RT", var.condition = "block_name", var.participant = "participant_number", var.compare = "trial_type", var.ACC = "ACC", compare1 = "congruent", compare2 = "incongruent", average = "mean") multiverse <- splithalf.multiverse(input = mv_splithalf, specifications = specifications) ``` ```{r splithalf_for_mv_fig} multiverse.plot(multiverse = multiverse, title = "sim_multiverse") ``` ```{r} ``` ## test-retest ```{r retest_mv1, comment=FALSE, results='hide'} icc2 <- testretest.multiverse(data = sim_data_acc, specifications, test = "ICC2", score = "difference", var.participant = "participant_number", var.ACC = "ACC", var.RT = "RT", var.time = "block_name", var.compare = "trial_type", compare1 = "congruent", compare2 = "incongruent") multiverse.plot(icc2) ``` ```{r retest_mv2, comment=FALSE, results='hide'} icc3 <- testretest.multiverse(data = sim_data_acc, specifications, test = "ICC3", score = "difference", var.participant = "participant_number", var.ACC = "ACC", var.RT = "RT", var.time = "block_name", var.compare = "trial_type", compare1 = "congruent", compare2 = "incongruent") multiverse.plot(icc3) ``` ```{r retest_mv3, comment=FALSE, results='hide'} retest_cor <- testretest.multiverse(data = sim_data_acc, specifications, test = "cor", score = "difference", var.participant = "participant_number", var.ACC = "ACC", var.RT = "RT", var.time = "block_name", var.compare = "trial_type", compare1 = "congruent", compare2 = "incongruent") multiverse.plot(retest_cor) ```