Data Import Checks

Type Scan

Count Numeric Problems

Imported columns often arrive as text, so a type scan should find values that do not match the intended shape.

Program

Play the script to change how many bad numeric values are tolerated.

allowed_bad
type_scan.R
Replay: real traced execution (multi-file project)
score_text <- c("91", "84", "oops", "76")
allowed_bad <- 1
is_number <- grepl("^[0-9]+$", score_text)
bad <- sum(!is_number)
status <- if (bad <= allowed_bad) "ok" else "fix"
label <- paste(status, bad, sep = ":")
cat(label, "\n", sep = "")

score_text <- c("91", "84", "oops", "76")
allowed_bad <- 0
is_number <- grepl("^[0-9]+$", score_text)
bad <- sum(!is_number)
status <- if (bad <= allowed_bad) "ok" else "fix"
label <- paste(status, bad, sep = ":")
cat(label, "\n", sep = "")

score_text <- c("91", "84", "oops", "76")
allowed_bad <- 2
is_number <- grepl("^[0-9]+$", score_text)
bad <- sum(!is_number)
status <- if (bad <= allowed_bad) "ok" else "fix"
label <- paste(status, bad, sep = ":")
cat(label, "\n", sep = "")

  1. score_text ← 91, 84, oops, 76

    1score_text <- c("91", "84", "oops", "76")2allowed_bad <- 1
    values this step91, 84, oops, 76score_text

  2. allowed_bad ← 1

    1score_text <- c("91", "84", "oops", "76")2allowed_bad <- 13is_number <- grepl("^[0-9]+$", score_text)
    values this step1allowed_bad

  3. is_number ← TRUE, TRUE, FALSE, TRUE

    2allowed_bad <- 13is_number <- grepl("^[0-9]+$", score_text)4bad <- sum(!is_number)
    values this stepTRUE, TRUE, FALSE, TRUEis_number91, 84, oops, 76score_text

  4. bad ← 1

    3is_number <- grepl("^[0-9]+$", score_text)4bad <- sum(!is_number)5status <- if (bad <= allowed_bad) "ok" else "fix"
    values this step1badTRUE, TRUE, FALSE, TRUEis_number

  5. status ← ok

    4bad <- sum(!is_number)5status <- if (bad <= allowed_bad) "ok" else "fix"6label <- paste(status, bad, sep = ":")
    values this stepokstatus1bad1allowed_bad

  6. label ← ok:1

    5status <- if (bad <= allowed_bad) "ok" else "fix"6label <- paste(status, bad, sep = ":")7cat(label, "\n", sep = "")
    values this stepok:1labelokstatus1bad

  7. cat(label, " ", sep = "")

    6label <- paste(status, bad, sep = ":")7cat(label, "\n", sep = "")
    outputok:1
    values this stepok:1label

  1. score_text ← 91, 84, oops, 76

    1score_text <- c("91", "84", "oops", "76")2allowed_bad <- 0
    values this step91, 84, oops, 76score_text

  2. allowed_bad ← 0

    1score_text <- c("91", "84", "oops", "76")2allowed_bad <- 03is_number <- grepl("^[0-9]+$", score_text)
    values this step0allowed_bad

  3. is_number ← TRUE, TRUE, FALSE, TRUE

    2allowed_bad <- 03is_number <- grepl("^[0-9]+$", score_text)4bad <- sum(!is_number)
    values this stepTRUE, TRUE, FALSE, TRUEis_number91, 84, oops, 76score_text

  4. bad ← 1

    3is_number <- grepl("^[0-9]+$", score_text)4bad <- sum(!is_number)5status <- if (bad <= allowed_bad) "ok" else "fix"
    values this step1badTRUE, TRUE, FALSE, TRUEis_number

  5. status ← fix

    4bad <- sum(!is_number)5status <- if (bad <= allowed_bad) "ok" else "fix"6label <- paste(status, bad, sep = ":")
    values this stepfixstatus1bad0allowed_bad

  6. label ← fix:1

    5status <- if (bad <= allowed_bad) "ok" else "fix"6label <- paste(status, bad, sep = ":")7cat(label, "\n", sep = "")
    values this stepfix:1labelfixstatus1bad

  7. cat(label, " ", sep = "")

    6label <- paste(status, bad, sep = ":")7cat(label, "\n", sep = "")
    outputfix:1
    values this stepfix:1label

  1. score_text ← 91, 84, oops, 76

    1score_text <- c("91", "84", "oops", "76")2allowed_bad <- 2
    values this step91, 84, oops, 76score_text

  2. allowed_bad ← 2

    1score_text <- c("91", "84", "oops", "76")2allowed_bad <- 23is_number <- grepl("^[0-9]+$", score_text)
    values this step2allowed_bad

  3. is_number ← TRUE, TRUE, FALSE, TRUE

    2allowed_bad <- 23is_number <- grepl("^[0-9]+$", score_text)4bad <- sum(!is_number)
    values this stepTRUE, TRUE, FALSE, TRUEis_number91, 84, oops, 76score_text

  4. bad ← 1

    3is_number <- grepl("^[0-9]+$", score_text)4bad <- sum(!is_number)5status <- if (bad <= allowed_bad) "ok" else "fix"
    values this step1badTRUE, TRUE, FALSE, TRUEis_number

  5. status ← ok

    4bad <- sum(!is_number)5status <- if (bad <= allowed_bad) "ok" else "fix"6label <- paste(status, bad, sep = ":")
    values this stepokstatus1bad2allowed_bad

  6. label ← ok:1

    5status <- if (bad <= allowed_bad) "ok" else "fix"6label <- paste(status, bad, sep = ":")7cat(label, "\n", sep = "")
    values this stepok:1labelokstatus1bad

  7. cat(label, " ", sep = "")

    6label <- paste(status, bad, sep = ":")7cat(label, "\n", sep = "")
    outputok:1
    values this stepok:1label
text input Imported values can start as strings even when the column should be numeric.
shape test `grepl("^[0-9]+$", score_text)` marks values that look numeric.
tolerance `allowed_bad` decides whether the import can continue or needs repair.