Iterators & Generators

Itertools Introduction

Need to generate all possible password combinations, batch process millions of records, or create a sliding window over streaming data? The itertools module provides memory-efficient building blocks for these iterator patterns, implemented in C for speed.

Practical Batch Processing

infinite_iterators.py 
# Infinite iterators

from itertools import count, cycle, repeat

# count: infinite counter
print("count(10, 2) - first 5:")
counter = count(10, 2)  # Start at 10, step by 2
for _ in range(5):
    print(next(counter), end=" ")
print()

# cycle: infinite cycle through iterable
print("\ncycle(['A', 'B', 'C']) - first 7:")
cycler = cycle(['A', 'B', 'C'])
for _ in range(7):
    print(next(cycler), end=" ")
print()

# repeat: repeat value n times
print("\nrepeat('X', 5):")
for item in repeat('X', 5):
    print(item, end=" ")
print()

# Practical: enumerate with start offset
print("\nUsing count for custom enumerate:")
for i, letter in zip(count(1), ['a', 'b', 'c', 'd']):
    print(f"{i}. {letter}")
combinatorics.py 
# Combinations and permutations

from itertools import combinations, permutations, combinations_with_replacement

items = 

# Combinations: order doesn't matter, no repetition
print("Combinations of 2:")
for combo in combinations(items, 2):
    print(combo)

# Permutations: order matters, no repetition
print("\nPermutations of 2:")
for perm in permutations(items, 2):
    print(perm)

# Combinations with replacement
print("\nCombinations with replacement:")
for combo in combinations_with_replacement(items, 2):
    print(combo)

# Practical: all 3-digit PIN codes
from itertools import product
print("\nSample PIN codes (first 10):")
pins = product(range(10), repeat=3)
for i, pin in enumerate(pins):
    if i >= 10:
        break
    print(''.join(map(str, pin)))

# Combinations and permutations

from itertools import combinations, permutations, combinations_with_replacement

items = 

# Combinations: order doesn't matter, no repetition
print("Combinations of 2:")
for combo in combinations(items, 2):
    print(combo)

# Permutations: order matters, no repetition
print("\nPermutations of 2:")
for perm in permutations(items, 2):
    print(perm)

# Combinations with replacement
print("\nCombinations with replacement:")
for combo in combinations_with_replacement(items, 2):
    print(combo)

# Practical: all 3-digit PIN codes
from itertools import product
print("\nSample PIN codes (first 10):")
pins = product(range(10), repeat=3)
for i, pin in enumerate(pins):
    if i >= 10:
        break
    print(''.join(map(str, pin)))

# Combinations and permutations

from itertools import combinations, permutations, combinations_with_replacement

items = 

# Combinations: order doesn't matter, no repetition
print("Combinations of 2:")
for combo in combinations(items, 2):
    print(combo)

# Permutations: order matters, no repetition
print("\nPermutations of 2:")
for perm in permutations(items, 2):
    print(perm)

# Combinations with replacement
print("\nCombinations with replacement:")
for combo in combinations_with_replacement(items, 2):
    print(combo)

# Practical: all 3-digit PIN codes
from itertools import product
print("\nSample PIN codes (first 10):")
pins = product(range(10), repeat=3)
for i, pin in enumerate(pins):
    if i >= 10:
        break
    print(''.join(map(str, pin)))
filtering_iterators.py 
# Filtering iterators

from itertools import filterfalse, takewhile, dropwhile, islice

numbers = range(10)

# filterfalse: opposite of filter
print("filterfalse (odd numbers):")
evens = filterfalse(lambda x: x % 2, numbers)
print(list(evens))

# takewhile: take while condition is true
print("\ntakewhile (x < 5):")
result = takewhile(lambda x: x < 5, numbers)
print(list(result))

# dropwhile: drop while condition is true, then take rest
print("\ndropwhile (x < 5):")
result = dropwhile(lambda x: x < 5, numbers)
print(list(result))

# islice: slice iterator
print("\nislice (skip 2, take 4):")
result = islice(numbers, 2, 6)  # Start at 2, stop before 6
print(list(result))

print("\nislice (every other item):")
result = islice(range(20), 0, None, 2)  # Start, stop, step
print(list(result))
groupby_examples.py 
# Grouping with groupby

from itertools import groupby

# Group consecutive identical items
data = [1, 1, 1, 2, 2, 3, 3, 3, 3, 1, 1]
print("Grouping consecutive numbers:")
for key, group in groupby(data):
    print(f"{key}: {list(group)}")

# Group by custom key
people = [
    ('Alice', 25),
    ('Bob', 30),
    ('Charlie', 25),
    ('David', 30),
    ('Eve', 25)
]

# Must sort first for groupby to work correctly
people_sorted = sorted(people, key=lambda x: x[1])

print("\nGrouping by age:")
for age, group in groupby(people_sorted, key=lambda x: x[1]):
    names = [person[0] for person in group]
    print(f"Age {age}: {names}")

# Count consecutive runs
data = ['A', 'A', 'A', 'B', 'B', 'C', 'A', 'A']
print("\nRun-length encoding:")
for key, group in groupby(data):
    count = len(list(group))
    print(f"{key}: {count}")
chain_accumulate.py 
# Chain and accumulate

from itertools import chain, accumulate
import operator

# chain: concatenate iterables
list1 = [1, 2, 3]
list2 = [4, 5, 6]
list3 = [7, 8, 9]

print("chain:")
result = chain(list1, list2, list3)
print(list(result))

# chain.from_iterable: flatten nested lists
nested = [[1, 2], [3, 4], [5, 6]]
print("\nchain.from_iterable:")
result = chain.from_iterable(nested)
print(list(result))

# accumulate: running total
numbers = [1, 2, 3, 4, 5]
print("\naccumulate (sum):")
result = accumulate(numbers)
print(list(result))

print("\naccumulate (product):")
result = accumulate(numbers, operator.mul)
print(list(result))

print("\naccumulate (max):")
values = [3, 4, 6, 2, 1, 9, 0, 7, 5]
result = accumulate(values, max)
print(list(result))
practical.py 
# Practical example: batch processing with itertools

from itertools import islice, chain, groupby

def chunked(iterable, size):
    """Split iterable into chunks of given size"""
    iterator = iter(iterable)
    while True:
        chunk = list(islice(iterator, size))
        if not chunk:
            break
        yield chunk

# Process data in batches
data = range(1, 26)
print("Processing in batches of 5:")
for batch_num, batch in enumerate(chunked(data, 5), 1):
    print(f"Batch {batch_num}: {batch}")

# Flatten and group
nested_data = [
    ['apple', 'apricot', 'avocado'],
    ['banana', 'blueberry'],
    ['cherry', 'cranberry']
]

print("\nFlattened fruits:")
all_fruits = list(chain.from_iterable(nested_data))
print(all_fruits)

print("\nGrouped by first letter:")
all_fruits.sort()
for letter, fruits in groupby(all_fruits, key=lambda x: x[0]):
    print(f"{letter}: {list(fruits)}")

# Pairwise iteration
def pairwise(iterable):
    """s -> (s0,s1), (s1,s2), (s2,s3), ..."""
    a, b = iter(iterable), iter(iterable)
    next(b, None)
    return zip(a, b)

numbers = [1, 2, 3, 4, 5]
print("\nPairwise iteration:")
for pair in pairwise(numbers):
    print(pair)
infinite iterator - an iterator that never ends, like count(), cycle(), and repeat()
combinatorics - generating all combinations, permutations, or products of elements
filtering iterators - tools like takewhile, dropwhile, and filterfalse that select elements
groupby - groups consecutive elements by a key function (requires sorted input for full grouping)
chain - concatenates multiple iterables into one seamless iterator

Exercise: practical.py

Build a log file analyzer using itertools for batching and grouping