Data Pipeline Patterns
Group Counts
Grouping turns individual records into counts keyed by a shared field.
Group Counts
group_counts.py
orders = [
{"region": "east", "status": "open"},
{"region": "west", "status": "open"},
{"region": "west", "status": "closed"},
]
status_filter =
counts = {}
for order in orders:
if status_filter != "all" and order["status"] != status_filter:
continue
region = order["region"]
counts[region] = counts.get(region, 0) + 1
parts = []
for region in sorted(counts):
parts.append(region + "=" + str(counts[region]))
print("counts=" + ";".join(parts))
orders = [
{"region": "east", "status": "open"},
{"region": "west", "status": "open"},
{"region": "west", "status": "closed"},
]
status_filter =
counts = {}
for order in orders:
if status_filter != "all" and order["status"] != status_filter:
continue
region = order["region"]
counts[region] = counts.get(region, 0) + 1
parts = []
for region in sorted(counts):
parts.append(region + "=" + str(counts[region]))
print("counts=" + ";".join(parts))
grouping records
A grouping stage updates one dictionary entry for each record key.