Data Pipeline Patterns

Filter Records

Filtering keeps only the records that match the rule for the next step.

filtering records A filter stage copies matching records into a new list while preserving their order.

Filter Records

min_score
filter_records.py
Replay: real traced execution (multi-file project)
records = [
    {"name": "Ada", "score": 92},
    {"name": "Linus", "score": 78},
    {"name": "Grace", "score": 86},
]

min_score = 80

passed = []
for record in records:
    if record["score"] >= min_score:
        passed.append(record["name"])

print("passed=" + ",".join(passed))

records = [
    {"name": "Ada", "score": 92},
    {"name": "Linus", "score": 78},
    {"name": "Grace", "score": 86},
]

min_score = 90

passed = []
for record in records:
    if record["score"] >= min_score:
        passed.append(record["name"])

print("passed=" + ",".join(passed))

  1. records ← [{'name': 'Ada', 'score': 92}, {'name': 'Linus', 'score': 78}, {'name': 'Grace', 'score': 86}]

    1records→ [{'name': 'Ada', 'score': 92}, {'name': 'Linus', 'score': 78}, {'name': 'Grace', 'score': 86}] = [2    {"name": "Ada", "score": 92},3    {"name": "Linus", "score": 78},4    {"name": "Grace", "score": 86},5]67min_score→ 80 = 80  #@min_score=9089passed→ [] = []10for record in records:

  2. for record in records:

    pass 1 of 3
    9passed = []10for record{'name': 'Ada', 'score': 92} in records[{'name': 'Ada', 'score': 92}, {'name': 'Linus', 'score': 78}, {'name': 'Grace', 'score': 86}]:11    if record["score"] >= min_score:12        passed.append(record["name"])
    All 3 passes — pass 1 is the card above
    passrecordrecord[”score”]min_scorerecord[”name”]passed
    1{'name': 'Ada', 'score': 92}9280Ada[] → ['Ada']
    2{'name': 'Linus', 'score': 78}————
    3{'name': 'Grace', 'score': 86}8680Grace['Ada'] → ['Ada', 'Grace']

  3. passed ← ['Ada']

    pass 1 of 2
    10for record in records:11    if record["score"]92 >= min_score80:12        passed→ ['Ada'].append(record["name"]Ada)

  4. passed ← ['Ada', 'Grace']

    pass 2 of 2
    10for record in records:11    if record["score"]86 >= min_score80:12        passed→ ['Ada', 'Grace'].append(record["name"]Grace)

  5. print("passed=" + ",".join(passed))

    14print("passed=" + ",".join(passed['Ada', 'Grace']))
    outputpassed=Ada,Grace

  1. records ← [{'name': 'Ada', 'score': 92}, {'name': 'Linus', 'score': 78}, {'name': 'Grace', 'score': 86}]

    1records→ [{'name': 'Ada', 'score': 92}, {'name': 'Linus', 'score': 78}, {'name': 'Grace', 'score': 86}] = [2    {"name": "Ada", "score": 92},3    {"name": "Linus", "score": 78},4    {"name": "Grace", "score": 86},5]67min_score→ 90 = 9089passed→ [] = []10for record in records:

  2. for record in records:

    pass 1 of 3
    9passed = []10for record{'name': 'Ada', 'score': 92} in records[{'name': 'Ada', 'score': 92}, {'name': 'Linus', 'score': 78}, {'name': 'Grace', 'score': 86}]:11    if record["score"] >= min_score:12        passed.append(record["name"])
    All 3 passes — pass 1 is the card above
    passrecordrecord[”score”]min_scorerecord[”name”]passed
    1{'name': 'Ada', 'score': 92}9290Ada[] → ['Ada']
    2{'name': 'Linus', 'score': 78}————
    3{'name': 'Grace', 'score': 86}————

  3. passed ← ['Ada']

    10for record in records:11    if record["score"]92 >= min_score90:12        passed→ ['Ada'].append(record["name"]Ada)

  4. print("passed=" + ",".join(passed))

    14print("passed=" + ",".join(passed['Ada']))
    outputpassed=Ada