Text Mining Basics

Stopword Filter

Keep Content Words

Stopword filtering removes common words so the remaining tokens carry more topic signal.

Program

Play the script to choose a stopword set and compare the remaining content tokens.

stop_choice
stopword_filter.R
Replay: real traced execution (multi-file project)
tokens <- c("data", "is", "useful", "when", "data", "answers", "questions")
stop_choice <- 1
stop_sets <- list(c("is", "when"), c("is", "when", "data"), c("is", "when", "questions"))
stopwords <- stop_sets[[stop_choice]]
content <- tokens[!(tokens %in% stopwords)]
summary <- paste(content, collapse = ",")
cat(summary, "\n", sep = "")

tokens <- c("data", "is", "useful", "when", "data", "answers", "questions")
stop_choice <- 2
stop_sets <- list(c("is", "when"), c("is", "when", "data"), c("is", "when", "questions"))
stopwords <- stop_sets[[stop_choice]]
content <- tokens[!(tokens %in% stopwords)]
summary <- paste(content, collapse = ",")
cat(summary, "\n", sep = "")

tokens <- c("data", "is", "useful", "when", "data", "answers", "questions")
stop_choice <- 3
stop_sets <- list(c("is", "when"), c("is", "when", "data"), c("is", "when", "questions"))
stopwords <- stop_sets[[stop_choice]]
content <- tokens[!(tokens %in% stopwords)]
summary <- paste(content, collapse = ",")
cat(summary, "\n", sep = "")

  1. tokens ← data, is, useful, when, data, answers, questions

    1tokens <- c("data", "is", "useful", "when", "data", "answers", "questions")2stop_choice <- 1
    values this stepdata, is, useful, when, data, answers, questionstokens

  2. stop_choice ← 1

    1tokens <- c("data", "is", "useful", "when", "data", "answers", "questions")2stop_choice <- 13stop_sets <- list(c("is", "when"), c("is", "when", "data"), c("is", "when", "questions"))
    values this step1stop_choice

  3. stop_sets ← 3 choices

    2stop_choice <- 13stop_sets <- list(c("is", "when"), c("is", "when", "data"), c("is", "when", "questions"))4stopwords <- stop_sets[[stop_choice]]
    values this step3 choicesstop_sets

  4. stopwords ← is, when

    3stop_sets <- list(c("is", "when"), c("is", "when", "data"), c("is", "when", "questions"))4stopwords <- stop_sets[[stop_choice]]5content <- tokens[!(tokens %in% stopwords)]
    values this stepis, whenstopwords1stop_choice

  5. content ← data, useful, data, answers, questions

    4stopwords <- stop_sets[[stop_choice]]5content <- tokens[!(tokens %in% stopwords)]6summary <- paste(content, collapse = ",")
    values this stepdata, useful, data, answers, questionscontent7 tokenstokensis, whenstopwords

  6. summary ← data,useful,data,answers,questions

    5content <- tokens[!(tokens %in% stopwords)]6summary <- paste(content, collapse = ",")7cat(summary, "\n", sep = "")
    values this stepdata,useful,data,answers,questionssummary5 tokenscontent

  7. cat(summary, " ", sep = "")

    6summary <- paste(content, collapse = ",")7cat(summary, "\n", sep = "")
    outputdata,useful,data,answers,questions
    values this stepdata,useful,data,answers,questionssummary

  1. tokens ← data, is, useful, when, data, answers, questions

    1tokens <- c("data", "is", "useful", "when", "data", "answers", "questions")2stop_choice <- 2
    values this stepdata, is, useful, when, data, answers, questionstokens

  2. stop_choice ← 2

    1tokens <- c("data", "is", "useful", "when", "data", "answers", "questions")2stop_choice <- 23stop_sets <- list(c("is", "when"), c("is", "when", "data"), c("is", "when", "questions"))
    values this step2stop_choice

  3. stop_sets ← 3 choices

    2stop_choice <- 23stop_sets <- list(c("is", "when"), c("is", "when", "data"), c("is", "when", "questions"))4stopwords <- stop_sets[[stop_choice]]
    values this step3 choicesstop_sets

  4. stopwords ← is, when, data

    3stop_sets <- list(c("is", "when"), c("is", "when", "data"), c("is", "when", "questions"))4stopwords <- stop_sets[[stop_choice]]5content <- tokens[!(tokens %in% stopwords)]
    values this stepis, when, datastopwords2stop_choice

  5. content ← useful, answers, questions

    4stopwords <- stop_sets[[stop_choice]]5content <- tokens[!(tokens %in% stopwords)]6summary <- paste(content, collapse = ",")
    values this stepuseful, answers, questionscontent7 tokenstokensis, when, datastopwords

  6. summary ← useful,answers,questions

    5content <- tokens[!(tokens %in% stopwords)]6summary <- paste(content, collapse = ",")7cat(summary, "\n", sep = "")
    values this stepuseful,answers,questionssummary3 tokenscontent

  7. cat(summary, " ", sep = "")

    6summary <- paste(content, collapse = ",")7cat(summary, "\n", sep = "")
    outputuseful,answers,questions
    values this stepuseful,answers,questionssummary

  1. tokens ← data, is, useful, when, data, answers, questions

    1tokens <- c("data", "is", "useful", "when", "data", "answers", "questions")2stop_choice <- 3
    values this stepdata, is, useful, when, data, answers, questionstokens

  2. stop_choice ← 3

    1tokens <- c("data", "is", "useful", "when", "data", "answers", "questions")2stop_choice <- 33stop_sets <- list(c("is", "when"), c("is", "when", "data"), c("is", "when", "questions"))
    values this step3stop_choice

  3. stop_sets ← 3 choices

    2stop_choice <- 33stop_sets <- list(c("is", "when"), c("is", "when", "data"), c("is", "when", "questions"))4stopwords <- stop_sets[[stop_choice]]
    values this step3 choicesstop_sets

  4. stopwords ← is, when, questions

    3stop_sets <- list(c("is", "when"), c("is", "when", "data"), c("is", "when", "questions"))4stopwords <- stop_sets[[stop_choice]]5content <- tokens[!(tokens %in% stopwords)]
    values this stepis, when, questionsstopwords3stop_choice

  5. content ← data, useful, data, answers

    4stopwords <- stop_sets[[stop_choice]]5content <- tokens[!(tokens %in% stopwords)]6summary <- paste(content, collapse = ",")
    values this stepdata, useful, data, answerscontent7 tokenstokensis, when, questionsstopwords

  6. summary ← data,useful,data,answers

    5content <- tokens[!(tokens %in% stopwords)]6summary <- paste(content, collapse = ",")7cat(summary, "\n", sep = "")
    values this stepdata,useful,data,answerssummary4 tokenscontent

  7. cat(summary, " ", sep = "")

    6summary <- paste(content, collapse = ",")7cat(summary, "\n", sep = "")
    outputdata,useful,data,answers
    values this stepdata,useful,data,answerssummary
stopwords Stopwords are tokens the analysis chooses to ignore.
list `stop_sets[[stop_choice]]` picks one candidate vector from a list.
%in% `tokens %in% stopwords` marks which tokens should be removed.