Generator Functions with yield - Python Programming

Processing a 10GB log file shouldn't require 10GB of memory. Generator functions with yield produce values one at a time on demand, letting you work with massive datasets or infinite sequences using constant memory.

Infinite Generators

basic.py

# Basic generator


def count_up(n):
    """Generate numbers from 0 to n-1"""
    i = 0
    while i < n:
        yield i
        i += 1


# Use generator in for loop
limit = 
for num in count_up(limit):
    print(num, end=" ")

print("\n")

# Manual iteration
gen = count_up(3)
print(next(gen))  # 0
print(next(gen))  # 1
print(next(gen))  # 2

# Basic generator


def count_up(n):
    """Generate numbers from 0 to n-1"""
    i = 0
    while i < n:
        yield i
        i += 1


# Use generator in for loop
limit = 
for num in count_up(limit):
    print(num, end=" ")

print("\n")

# Manual iteration
gen = count_up(3)
print(next(gen))  # 0
print(next(gen))  # 1
print(next(gen))  # 2

# Basic generator


def count_up(n):
    """Generate numbers from 0 to n-1"""
    i = 0
    while i < n:
        yield i
        i += 1


# Use generator in for loop
limit = 
for num in count_up(limit):
    print(num, end=" ")

print("\n")

# Manual iteration
gen = count_up(3)
print(next(gen))  # 0
print(next(gen))  # 1
print(next(gen))  # 2

state.py

# State preservation


def fibonacci(n):
    """Generate first n Fibonacci numbers"""
    a, b = 0, 1
    for _ in range(n):
        yield a
        a, b = b, a + b  # state preserved across yields


# State is preserved
print("Fibonacci sequence:")
for num in fibonacci(10):
    print(num, end=" ")

lazy_vs_eager.py

# Generator vs list


def eager_range(n):
    """Returns a list (all values in memory)"""
    result = []
    for i in range(n):
        result.append(i)
    return result


def lazy_range(n):
    """Returns a generator (values on demand)"""
    for i in range(n):
        yield i


# Eager: all values created upfront
print("Eager (list):")
nums = eager_range(5)
print(type(nums), nums)

# Lazy: values created on demand
print("\nLazy (generator):")
gen = lazy_range(5)
print(type(gen), gen)
print("Values:", list(gen))

expression.py

# Generator expressions


# Generator expression (lazy)
gen = (x * x for x in range(5))
print("Type:", type(gen))
print("Values:", list(gen))

# Compare with list comprehension (eager)
lst = [x * x for x in range(5)]
print("\nList:", type(lst), lst)

# Pipeline with generator expressions
numbers = range(10)
squares = (x * x for x in numbers)
evens = (x for x in squares if x % 2 == 0)
print("\nEven squares:", list(evens))

infinite.py

# Infinite generators


def infinite_count(start=0):
    """Infinite counter"""
    n = start
    while True:
        yield n
        n += 1


# Use with break or takewhile
counter = infinite_count(1)
for num in counter:
    print(num, end=" ")
    if num >= 10:
        break

print("\n")

# With itertools.islice
from itertools import islice

counter = infinite_count(100)
print("First 5:", list(islice(counter, 5)))

yield vs return

return ends the function and returns a value
yield pauses the function and produces a value, resuming later

Key Benefits

Lazy evaluation: values produced on demand, not all at once
State preservation: local variables preserved between yields
Memory efficient: only one value in memory at a time
Automatic iterator: no need to write __iter__ and __next__

yield - a keyword that pauses the function and produces a value, resuming execution on the next iteration

lazy evaluation - computing values only when needed, rather than all at once upfront

generator expression - a compact syntax `(expr for x in iterable)` that creates a generator inline

Exercise: practical.py

Build a file reader that yields lines matching a pattern