Text Mining Basics

Term Frequency

Count Tokens

table counts repeated values. Text mining uses this idea to turn word tokens into term frequencies.

Program

Play the script to choose a term and watch its frequency come out of the token table.

target_index
term_frequency.R
Replay: real traced execution (multi-file project)
tokens <- c("r", "data", "r", "model", "data", "data")
terms <- c("r", "data", "model")
target_index <- 2
target <- terms[target_index]
counts <- table(tokens)
frequency <- as.integer(counts[[target]])
label <- paste(target, frequency, sep = "=")
cat(label, "\n", sep = "")

tokens <- c("r", "data", "r", "model", "data", "data")
terms <- c("r", "data", "model")
target_index <- 1
target <- terms[target_index]
counts <- table(tokens)
frequency <- as.integer(counts[[target]])
label <- paste(target, frequency, sep = "=")
cat(label, "\n", sep = "")

tokens <- c("r", "data", "r", "model", "data", "data")
terms <- c("r", "data", "model")
target_index <- 3
target <- terms[target_index]
counts <- table(tokens)
frequency <- as.integer(counts[[target]])
label <- paste(target, frequency, sep = "=")
cat(label, "\n", sep = "")

  1. tokens ← r, data, r, model, data, data

    1tokens <- c("r", "data", "r", "model", "data", "data")2terms <- c("r", "data", "model")
    values this stepr, data, r, model, data, datatokens

  2. terms ← r, data, model

    1tokens <- c("r", "data", "r", "model", "data", "data")2terms <- c("r", "data", "model")3target_index <- 2
    values this stepr, data, modelterms

  3. target_index ← 2

    2terms <- c("r", "data", "model")3target_index <- 24target <- terms[target_index]
    values this step2target_index

  4. target ← data

    3target_index <- 24target <- terms[target_index]5counts <- table(tokens)
    values this stepdatatargetr, data, modelterms2target_index

  5. counts ← data=3, model=1, r=2

    4target <- terms[target_index]5counts <- table(tokens)6frequency <- as.integer(counts[[target]])
    values this stepdata=3, model=1, r=2counts6 tokenstokens

  6. frequency ← 3

    5counts <- table(tokens)6frequency <- as.integer(counts[[target]])7label <- paste(target, frequency, sep = "=")
    values this step3frequency3counts[[target]]datatarget

  7. label ← data=3

    6frequency <- as.integer(counts[[target]])7label <- paste(target, frequency, sep = "=")8cat(label, "\n", sep = "")
    values this stepdata=3labeldatatarget3frequency

  8. cat(label, " ", sep = "")

    7label <- paste(target, frequency, sep = "=")8cat(label, "\n", sep = "")
    outputdata=3
    values this stepdata=3label

  1. tokens ← r, data, r, model, data, data

    1tokens <- c("r", "data", "r", "model", "data", "data")2terms <- c("r", "data", "model")
    values this stepr, data, r, model, data, datatokens

  2. terms ← r, data, model

    1tokens <- c("r", "data", "r", "model", "data", "data")2terms <- c("r", "data", "model")3target_index <- 1
    values this stepr, data, modelterms

  3. target_index ← 1

    2terms <- c("r", "data", "model")3target_index <- 14target <- terms[target_index]
    values this step1target_index

  4. target ← r

    3target_index <- 14target <- terms[target_index]5counts <- table(tokens)
    values this steprtargetr, data, modelterms1target_index

  5. counts ← data=3, model=1, r=2

    4target <- terms[target_index]5counts <- table(tokens)6frequency <- as.integer(counts[[target]])
    values this stepdata=3, model=1, r=2counts6 tokenstokens

  6. frequency ← 2

    5counts <- table(tokens)6frequency <- as.integer(counts[[target]])7label <- paste(target, frequency, sep = "=")
    values this step2frequency2counts[[target]]rtarget

  7. label ← r=2

    6frequency <- as.integer(counts[[target]])7label <- paste(target, frequency, sep = "=")8cat(label, "\n", sep = "")
    values this stepr=2labelrtarget2frequency

  8. cat(label, " ", sep = "")

    7label <- paste(target, frequency, sep = "=")8cat(label, "\n", sep = "")
    outputr=2
    values this stepr=2label

  1. tokens ← r, data, r, model, data, data

    1tokens <- c("r", "data", "r", "model", "data", "data")2terms <- c("r", "data", "model")
    values this stepr, data, r, model, data, datatokens

  2. terms ← r, data, model

    1tokens <- c("r", "data", "r", "model", "data", "data")2terms <- c("r", "data", "model")3target_index <- 3
    values this stepr, data, modelterms

  3. target_index ← 3

    2terms <- c("r", "data", "model")3target_index <- 34target <- terms[target_index]
    values this step3target_index

  4. target ← model

    3target_index <- 34target <- terms[target_index]5counts <- table(tokens)
    values this stepmodeltargetr, data, modelterms3target_index

  5. counts ← data=3, model=1, r=2

    4target <- terms[target_index]5counts <- table(tokens)6frequency <- as.integer(counts[[target]])
    values this stepdata=3, model=1, r=2counts6 tokenstokens

  6. frequency ← 1

    5counts <- table(tokens)6frequency <- as.integer(counts[[target]])7label <- paste(target, frequency, sep = "=")
    values this step1frequency1counts[[target]]modeltarget

  7. label ← model=1

    6frequency <- as.integer(counts[[target]])7label <- paste(target, frequency, sep = "=")8cat(label, "\n", sep = "")
    values this stepmodel=1labelmodeltarget1frequency

  8. cat(label, " ", sep = "")

    7label <- paste(target, frequency, sep = "=")8cat(label, "\n", sep = "")
    outputmodel=1
    values this stepmodel=1label
table `table(tokens)` counts how often each token appears.
term `target <- terms[target_index]` chooses which count to inspect.
frequency `as.integer(counts[[target]])` extracts the chosen count as a number.