Diagnostics and Quality Checks

Duplicate Check

Find Repeated IDs

Repeated identifiers can merge records incorrectly unless the policy is explicit.

Program

Play the script to change the duplicate policy and see the resulting action.

policy_index
duplicate_check.R
Replay: real traced execution (multi-file project)
policy_index <- 2
id <- c("A1", "B2", "A1", "C3")
policy <- c("ignore", "flag", "stop")[policy_index]
duplicate_id <- unique(id[duplicated(id)])
duplicate_count <- length(duplicate_id)
action <- if (duplicate_count == 0) "ok" else policy
label <- paste(action, duplicate_count, sep = ":")
cat(label, "\n", sep = "")

policy_index <- 1
id <- c("A1", "B2", "A1", "C3")
policy <- c("ignore", "flag", "stop")[policy_index]
duplicate_id <- unique(id[duplicated(id)])
duplicate_count <- length(duplicate_id)
action <- if (duplicate_count == 0) "ok" else policy
label <- paste(action, duplicate_count, sep = ":")
cat(label, "\n", sep = "")

policy_index <- 3
id <- c("A1", "B2", "A1", "C3")
policy <- c("ignore", "flag", "stop")[policy_index]
duplicate_id <- unique(id[duplicated(id)])
duplicate_count <- length(duplicate_id)
action <- if (duplicate_count == 0) "ok" else policy
label <- paste(action, duplicate_count, sep = ":")
cat(label, "\n", sep = "")

  1. policy_index ← 2

    1policy_index <- 22id <- c("A1", "B2", "A1", "C3")
    values this step2policy_index

  2. id ← A1, B2, A1, C3

    1policy_index <- 22id <- c("A1", "B2", "A1", "C3")3policy <- c("ignore", "flag", "stop")[policy_index]
    values this stepA1, B2, A1, C3id

  3. policy ← flag

    2id <- c("A1", "B2", "A1", "C3")3policy <- c("ignore", "flag", "stop")[policy_index]4duplicate_id <- unique(id[duplicated(id)])
    values this stepflagpolicy2policy_index

  4. duplicate_id ← A1

    3policy <- c("ignore", "flag", "stop")[policy_index]4duplicate_id <- unique(id[duplicated(id)])5duplicate_count <- length(duplicate_id)
    values this stepA1duplicate_idA1 repeatsid

  5. duplicate_count ← 1

    4duplicate_id <- unique(id[duplicated(id)])5duplicate_count <- length(duplicate_id)6action <- if (duplicate_count == 0) "ok" else policy
    values this step1duplicate_countA1duplicate_id

  6. action ← flag

    5duplicate_count <- length(duplicate_id)6action <- if (duplicate_count == 0) "ok" else policy7label <- paste(action, duplicate_count, sep = ":")
    values this stepflagaction1duplicate_countflagpolicy

  7. label ← flag:1

    6action <- if (duplicate_count == 0) "ok" else policy7label <- paste(action, duplicate_count, sep = ":")8cat(label, "\n", sep = "")
    values this stepflag:1labelflagaction1duplicate_count

  8. cat(label, " ", sep = "")

    7label <- paste(action, duplicate_count, sep = ":")8cat(label, "\n", sep = "")
    outputflag:1
    values this stepflag:1label

  1. policy_index ← 1

    1policy_index <- 12id <- c("A1", "B2", "A1", "C3")
    values this step1policy_index

  2. id ← A1, B2, A1, C3

    1policy_index <- 12id <- c("A1", "B2", "A1", "C3")3policy <- c("ignore", "flag", "stop")[policy_index]
    values this stepA1, B2, A1, C3id

  3. policy ← ignore

    2id <- c("A1", "B2", "A1", "C3")3policy <- c("ignore", "flag", "stop")[policy_index]4duplicate_id <- unique(id[duplicated(id)])
    values this stepignorepolicy1policy_index

  4. duplicate_id ← A1

    3policy <- c("ignore", "flag", "stop")[policy_index]4duplicate_id <- unique(id[duplicated(id)])5duplicate_count <- length(duplicate_id)
    values this stepA1duplicate_idA1 repeatsid

  5. duplicate_count ← 1

    4duplicate_id <- unique(id[duplicated(id)])5duplicate_count <- length(duplicate_id)6action <- if (duplicate_count == 0) "ok" else policy
    values this step1duplicate_countA1duplicate_id

  6. action ← ignore

    5duplicate_count <- length(duplicate_id)6action <- if (duplicate_count == 0) "ok" else policy7label <- paste(action, duplicate_count, sep = ":")
    values this stepignoreaction1duplicate_countignorepolicy

  7. label ← ignore:1

    6action <- if (duplicate_count == 0) "ok" else policy7label <- paste(action, duplicate_count, sep = ":")8cat(label, "\n", sep = "")
    values this stepignore:1labelignoreaction1duplicate_count

  8. cat(label, " ", sep = "")

    7label <- paste(action, duplicate_count, sep = ":")8cat(label, "\n", sep = "")
    outputignore:1
    values this stepignore:1label

  1. policy_index ← 3

    1policy_index <- 32id <- c("A1", "B2", "A1", "C3")
    values this step3policy_index

  2. id ← A1, B2, A1, C3

    1policy_index <- 32id <- c("A1", "B2", "A1", "C3")3policy <- c("ignore", "flag", "stop")[policy_index]
    values this stepA1, B2, A1, C3id

  3. policy ← stop

    2id <- c("A1", "B2", "A1", "C3")3policy <- c("ignore", "flag", "stop")[policy_index]4duplicate_id <- unique(id[duplicated(id)])
    values this stepstoppolicy3policy_index

  4. duplicate_id ← A1

    3policy <- c("ignore", "flag", "stop")[policy_index]4duplicate_id <- unique(id[duplicated(id)])5duplicate_count <- length(duplicate_id)
    values this stepA1duplicate_idA1 repeatsid

  5. duplicate_count ← 1

    4duplicate_id <- unique(id[duplicated(id)])5duplicate_count <- length(duplicate_id)6action <- if (duplicate_count == 0) "ok" else policy
    values this step1duplicate_countA1duplicate_id

  6. action ← stop

    5duplicate_count <- length(duplicate_id)6action <- if (duplicate_count == 0) "ok" else policy7label <- paste(action, duplicate_count, sep = ":")
    values this stepstopaction1duplicate_countstoppolicy

  7. label ← stop:1

    6action <- if (duplicate_count == 0) "ok" else policy7label <- paste(action, duplicate_count, sep = ":")8cat(label, "\n", sep = "")
    values this stepstop:1labelstopaction1duplicate_count

  8. cat(label, " ", sep = "")

    7label <- paste(action, duplicate_count, sep = ":")8cat(label, "\n", sep = "")
    outputstop:1
    values this stepstop:1label
identifier An ID column should usually identify one record.
duplicate scan `duplicated(id)` marks repeated IDs after their first appearance.
policy The duplicate policy decides whether to ignore, flag, or stop.