Model Evaluation

Holdout Split

Train and Test Rows

Model evaluation starts by separating rows used for fitting from rows saved for checking predictions.

Program

Play the script to move the holdout boundary and watch the train/test counts change.

test_start
holdout_split.R
Replay: real traced execution (multi-file project)
ids <- 1:6
test_start <- 5
train_ids <- ids[ids < test_start]
test_ids <- ids[ids >= test_start]
label <- paste(length(train_ids), length(test_ids), sep = ":")
cat(label, "\n", sep = "")

ids <- 1:6
test_start <- 4
train_ids <- ids[ids < test_start]
test_ids <- ids[ids >= test_start]
label <- paste(length(train_ids), length(test_ids), sep = ":")
cat(label, "\n", sep = "")

ids <- 1:6
test_start <- 6
train_ids <- ids[ids < test_start]
test_ids <- ids[ids >= test_start]
label <- paste(length(train_ids), length(test_ids), sep = ":")
cat(label, "\n", sep = "")

  1. ids ← 1, 2, 3, 4, 5, 6

    1ids <- 1:62test_start <- 5
    values this step1, 2, 3, 4, 5, 6ids

  2. test_start ← 5

    1ids <- 1:62test_start <- 53train_ids <- ids[ids < test_start]
    values this step5test_start

  3. train_ids ← 1, 2, 3, 4

    2test_start <- 53train_ids <- ids[ids < test_start]4test_ids <- ids[ids >= test_start]
    values this step1, 2, 3, 4train_ids1:6ids5test_start

  4. test_ids ← 5, 6

    3train_ids <- ids[ids < test_start]4test_ids <- ids[ids >= test_start]5label <- paste(length(train_ids), length(test_ids), sep = ":")
    values this step5, 6test_ids1:6ids5test_start

  5. label ← 4:2

    4test_ids <- ids[ids >= test_start]5label <- paste(length(train_ids), length(test_ids), sep = ":")6cat(label, "\n", sep = "")
    values this step4:2label4 rowstrain_ids2 rowstest_ids

  6. cat(label, " ", sep = "")

    5label <- paste(length(train_ids), length(test_ids), sep = ":")6cat(label, "\n", sep = "")
    output4:2
    values this step4:2label

  1. ids ← 1, 2, 3, 4, 5, 6

    1ids <- 1:62test_start <- 4
    values this step1, 2, 3, 4, 5, 6ids

  2. test_start ← 4

    1ids <- 1:62test_start <- 43train_ids <- ids[ids < test_start]
    values this step4test_start

  3. train_ids ← 1, 2, 3

    2test_start <- 43train_ids <- ids[ids < test_start]4test_ids <- ids[ids >= test_start]
    values this step1, 2, 3train_ids1:6ids4test_start

  4. test_ids ← 4, 5, 6

    3train_ids <- ids[ids < test_start]4test_ids <- ids[ids >= test_start]5label <- paste(length(train_ids), length(test_ids), sep = ":")
    values this step4, 5, 6test_ids1:6ids4test_start

  5. label ← 3:3

    4test_ids <- ids[ids >= test_start]5label <- paste(length(train_ids), length(test_ids), sep = ":")6cat(label, "\n", sep = "")
    values this step3:3label3 rowstrain_ids3 rowstest_ids

  6. cat(label, " ", sep = "")

    5label <- paste(length(train_ids), length(test_ids), sep = ":")6cat(label, "\n", sep = "")
    output3:3
    values this step3:3label

  1. ids ← 1, 2, 3, 4, 5, 6

    1ids <- 1:62test_start <- 6
    values this step1, 2, 3, 4, 5, 6ids

  2. test_start ← 6

    1ids <- 1:62test_start <- 63train_ids <- ids[ids < test_start]
    values this step6test_start

  3. train_ids ← 1, 2, 3, 4, 5

    2test_start <- 63train_ids <- ids[ids < test_start]4test_ids <- ids[ids >= test_start]
    values this step1, 2, 3, 4, 5train_ids1:6ids6test_start

  4. test_ids ← 6

    3train_ids <- ids[ids < test_start]4test_ids <- ids[ids >= test_start]5label <- paste(length(train_ids), length(test_ids), sep = ":")
    values this step6test_ids1:6ids6test_start

  5. label ← 5:1

    4test_ids <- ids[ids >= test_start]5label <- paste(length(train_ids), length(test_ids), sep = ":")6cat(label, "\n", sep = "")
    values this step5:1label5 rowstrain_ids1 rowtest_ids

  6. cat(label, " ", sep = "")

    5label <- paste(length(train_ids), length(test_ids), sep = ":")6cat(label, "\n", sep = "")
    output5:1
    values this step5:1label
holdout A holdout set is kept separate from rows used to fit a model.
logical index `ids < test_start` selects rows before the test boundary.
length `length(train_ids)` counts selected row identifiers.