Text Mining Basics

Token Cleanup

Text to Words

A text-mining workflow often starts by normalizing punctuation and case, then splitting text into word tokens.

Program

Play the script to change the minimum token length and see which words remain.

min_chars
token_cleanup.R
Replay: real traced execution (multi-file project)
text <- "Data science uses data, code, and questions"
words <- strsplit(tolower(gsub("[^A-Za-z ]", "", text)), "\\s+")[[1]]
min_chars <- 4
kept <- words[nchar(words) >= min_chars]
label <- paste(kept, collapse = ",")
cat(label, "\n", sep = "")

text <- "Data science uses data, code, and questions"
words <- strsplit(tolower(gsub("[^A-Za-z ]", "", text)), "\\s+")[[1]]
min_chars <- 3
kept <- words[nchar(words) >= min_chars]
label <- paste(kept, collapse = ",")
cat(label, "\n", sep = "")

text <- "Data science uses data, code, and questions"
words <- strsplit(tolower(gsub("[^A-Za-z ]", "", text)), "\\s+")[[1]]
min_chars <- 5
kept <- words[nchar(words) >= min_chars]
label <- paste(kept, collapse = ",")
cat(label, "\n", sep = "")

  1. text ← Data science uses data, code, and questions

    1text <- "Data science uses data, code, and questions"2words <- strsplit(tolower(gsub("[^A-Za-z ]", "", text)), "\\s+")[[1]]
    values this stepData science uses data, code, and questionstext

  2. words ← data, science, uses, data, code, and, questions

    1text <- "Data science uses data, code, and questions"2words <- strsplit(tolower(gsub("[^A-Za-z ]", "", text)), "\\s+")[[1]]3min_chars <- 4
    values this stepdata, science, uses, data, code, and, questionswordsnormalizedtext

  3. min_chars ← 4

    2words <- strsplit(tolower(gsub("[^A-Za-z ]", "", text)), "\\s+")[[1]]3min_chars <- 44kept <- words[nchar(words) >= min_chars]
    values this step4min_chars

  4. kept ← data, science, uses, data, code, questions

    3min_chars <- 44kept <- words[nchar(words) >= min_chars]5label <- paste(kept, collapse = ",")
    values this stepdata, science, uses, data, code, questionskept7 tokenswords4min_chars

  5. label ← data,science,uses,data,code,questions

    4kept <- words[nchar(words) >= min_chars]5label <- paste(kept, collapse = ",")6cat(label, "\n", sep = "")
    values this stepdata,science,uses,data,code,questionslabel6 tokenskept

  6. cat(label, " ", sep = "")

    5label <- paste(kept, collapse = ",")6cat(label, "\n", sep = "")
    outputdata,science,uses,data,code,questions
    values this stepdata,science,uses,data,code,questionslabel

  1. text ← Data science uses data, code, and questions

    1text <- "Data science uses data, code, and questions"2words <- strsplit(tolower(gsub("[^A-Za-z ]", "", text)), "\\s+")[[1]]
    values this stepData science uses data, code, and questionstext

  2. words ← data, science, uses, data, code, and, questions

    1text <- "Data science uses data, code, and questions"2words <- strsplit(tolower(gsub("[^A-Za-z ]", "", text)), "\\s+")[[1]]3min_chars <- 3
    values this stepdata, science, uses, data, code, and, questionswordsnormalizedtext

  3. min_chars ← 3

    2words <- strsplit(tolower(gsub("[^A-Za-z ]", "", text)), "\\s+")[[1]]3min_chars <- 34kept <- words[nchar(words) >= min_chars]
    values this step3min_chars

  4. kept ← data, science, uses, data, code, and, questions

    3min_chars <- 34kept <- words[nchar(words) >= min_chars]5label <- paste(kept, collapse = ",")
    values this stepdata, science, uses, data, code, and, questionskept7 tokenswords3min_chars

  5. label ← data,science,uses,data,code,and,questions

    4kept <- words[nchar(words) >= min_chars]5label <- paste(kept, collapse = ",")6cat(label, "\n", sep = "")
    values this stepdata,science,uses,data,code,and,questionslabel7 tokenskept

  6. cat(label, " ", sep = "")

    5label <- paste(kept, collapse = ",")6cat(label, "\n", sep = "")
    outputdata,science,uses,data,code,and,questions
    values this stepdata,science,uses,data,code,and,questionslabel

  1. text ← Data science uses data, code, and questions

    1text <- "Data science uses data, code, and questions"2words <- strsplit(tolower(gsub("[^A-Za-z ]", "", text)), "\\s+")[[1]]
    values this stepData science uses data, code, and questionstext

  2. words ← data, science, uses, data, code, and, questions

    1text <- "Data science uses data, code, and questions"2words <- strsplit(tolower(gsub("[^A-Za-z ]", "", text)), "\\s+")[[1]]3min_chars <- 5
    values this stepdata, science, uses, data, code, and, questionswordsnormalizedtext

  3. min_chars ← 5

    2words <- strsplit(tolower(gsub("[^A-Za-z ]", "", text)), "\\s+")[[1]]3min_chars <- 54kept <- words[nchar(words) >= min_chars]
    values this step5min_chars

  4. kept ← science, questions

    3min_chars <- 54kept <- words[nchar(words) >= min_chars]5label <- paste(kept, collapse = ",")
    values this stepscience, questionskept7 tokenswords5min_chars

  5. label ← science,questions

    4kept <- words[nchar(words) >= min_chars]5label <- paste(kept, collapse = ",")6cat(label, "\n", sep = "")
    values this stepscience,questionslabel2 tokenskept

  6. cat(label, " ", sep = "")

    5label <- paste(kept, collapse = ",")6cat(label, "\n", sep = "")
    outputscience,questions
    values this stepscience,questionslabel
gsub `gsub("[^A-Za-z ]", "", text)` removes punctuation for this small example.
tolower `tolower` normalizes words before counting or filtering.
nchar `nchar(words) >= min_chars` filters tokens by length.