Data Pipeline Patterns

Validation Summary

Validation can produce a compact summary instead of stopping at the first failure.

validation threshold The threshold decides which records count as accepted.
summary record A summary record returns both counts together without exposing mutable state.

Validation Summary

minimumScore
ValidationSummary.java
Replay: real traced execution (multi-file project)
public class ValidationSummary {
    record Submission(String name, int score) {}
    record Summary(int accepted, int rejected) {}

    static Summary summarize(Submission[] submissions, int minimumScore) {
        int accepted = 0;
        int rejected = 0;

        for (Submission submission : submissions) {
            if (submission.score() >= minimumScore) {
                accepted++;
            } else {
                rejected++;
            }
        }

        return new Summary(accepted, rejected);
    }

    public static void main(String[] args) {
        int minimumScore = 70;
        Submission[] submissions = {
            new Submission("alpha", 95),
            new Submission("beta", 68),
            new Submission("gamma", 82)
        };
        Summary summary = summarize(submissions, minimumScore);

        System.out.println("accepted=" + summary.accepted());
        System.out.println("rejected=" + summary.rejected());
    }
}

public class ValidationSummary {
    record Submission(String name, int score) {}
    record Summary(int accepted, int rejected) {}

    static Summary summarize(Submission[] submissions, int minimumScore) {
        int accepted = 0;
        int rejected = 0;

        for (Submission submission : submissions) {
            if (submission.score() >= minimumScore) {
                accepted++;
            } else {
                rejected++;
            }
        }

        return new Summary(accepted, rejected);
    }

    public static void main(String[] args) {
        int minimumScore = 80;
        Submission[] submissions = {
            new Submission("alpha", 95),
            new Submission("beta", 68),
            new Submission("gamma", 82)
        };
        Summary summary = summarize(submissions, minimumScore);

        System.out.println("accepted=" + summary.accepted());
        System.out.println("rejected=" + summary.rejected());
    }
}

public class ValidationSummary {
    record Submission(String name, int score) {}
    record Summary(int accepted, int rejected) {}

    static Summary summarize(Submission[] submissions, int minimumScore) {
        int accepted = 0;
        int rejected = 0;

        for (Submission submission : submissions) {
            if (submission.score() >= minimumScore) {
                accepted++;
            } else {
                rejected++;
            }
        }

        return new Summary(accepted, rejected);
    }

    public static void main(String[] args) {
        int minimumScore = 90;
        Submission[] submissions = {
            new Submission("alpha", 95),
            new Submission("beta", 68),
            new Submission("gamma", 82)
        };
        Summary summary = summarize(submissions, minimumScore);

        System.out.println("accepted=" + summary.accepted());
        System.out.println("rejected=" + summary.rejected());
    }
}

  1. minimumScore ← 70

    20public static void main(String[] args) {21    int minimumScore→ 70 = 70;  //@minimumScore=80, 9022    Submission[] submissions = {23        new Submission("alpha", 95),24        new Submission("beta", 68),25        new Submission("gamma", 82)26    };27    Summary summary = summarize(submissions, minimumScore70);

  2. accepted ← 0, rejected ← 0

    5static Summary summarize(Submission[] submissions, int minimumScore70) {6    int accepted→ 0 = 0;7    int rejected→ 0 = 0;

  3. for (Submission submission : submissions)

    pass 1 of 3
    9for (Submission submissionSubmission[name=alpha, score=95] : submissions) {10    if (submission.score() >= minimumScore) {
    All 3 passes — pass 1 is the card above
    passsubmissionminimumScoreacceptedrejected
    1Submission[name=alpha, score=95]700 → 1—
    2Submission[name=beta, score=68]——0 → 1
    3Submission[name=gamma, score=82]701 → 2—

  4. accepted ← 1

    pass 1 of 2
    9for (Submission submission : submissions) {10    if (submission.score() >= minimumScore70) {11        accepted→ 1++;12    } else {

  5. rejected ← 1

    11    accepted++;12} else {13    rejected→ 1++;14}

  6. accepted ← 2

    pass 2 of 2
    9for (Submission submission : submissions) {10    if (submission.score() >= minimumScore70) {11        accepted→ 2++;12    } else {

  7. return new Summary(accepted, rejected);

    17    return new Summary(accepted, rejected);18}

  8. summary ← Summary[accepted=2, rejected=1]

    26    };27    Summary summary→ Summary[accepted=2, rejected=1] = summarize(submissions, minimumScore70);2829    System.out.println("accepted=" + summary.accepted());30    System.out.println("rejected=" + summary.rejected());31}
    outputaccepted=2
rejected=1

  1. minimumScore ← 80

    20public static void main(String[] args) {21    int minimumScore→ 80 = 80;22    Submission[] submissions = {23        new Submission("alpha", 95),24        new Submission("beta", 68),25        new Submission("gamma", 82)26    };27    Summary summary = summarize(submissions, minimumScore80);

  2. accepted ← 0, rejected ← 0

    5static Summary summarize(Submission[] submissions, int minimumScore80) {6    int accepted→ 0 = 0;7    int rejected→ 0 = 0;

  3. for (Submission submission : submissions)

    pass 1 of 3
    9for (Submission submissionSubmission[name=alpha, score=95] : submissions) {10    if (submission.score() >= minimumScore) {
    All 3 passes — pass 1 is the card above
    passsubmissionminimumScoreacceptedrejected
    1Submission[name=alpha, score=95]800 → 1—
    2Submission[name=beta, score=68]——0 → 1
    3Submission[name=gamma, score=82]801 → 2—

  4. accepted ← 1

    pass 1 of 2
    9for (Submission submission : submissions) {10    if (submission.score() >= minimumScore80) {11        accepted→ 1++;12    } else {

  5. rejected ← 1

    11    accepted++;12} else {13    rejected→ 1++;14}

  6. accepted ← 2

    pass 2 of 2
    9for (Submission submission : submissions) {10    if (submission.score() >= minimumScore80) {11        accepted→ 2++;12    } else {

  7. return new Summary(accepted, rejected);

    17    return new Summary(accepted, rejected);18}

  8. summary ← Summary[accepted=2, rejected=1]

    26    };27    Summary summary→ Summary[accepted=2, rejected=1] = summarize(submissions, minimumScore80);2829    System.out.println("accepted=" + summary.accepted());30    System.out.println("rejected=" + summary.rejected());31}
    outputaccepted=2
rejected=1

  1. minimumScore ← 90

    20public static void main(String[] args) {21    int minimumScore→ 90 = 90;22    Submission[] submissions = {23        new Submission("alpha", 95),24        new Submission("beta", 68),25        new Submission("gamma", 82)26    };27    Summary summary = summarize(submissions, minimumScore90);

  2. accepted ← 0, rejected ← 0

    5static Summary summarize(Submission[] submissions, int minimumScore90) {6    int accepted→ 0 = 0;7    int rejected→ 0 = 0;

  3. for (Submission submission : submissions)

    pass 1 of 3
    9for (Submission submissionSubmission[name=alpha, score=95] : submissions) {10    if (submission.score() >= minimumScore) {
    All 3 passes — pass 1 is the card above
    passsubmissionminimumScoreacceptedrejected
    1Submission[name=alpha, score=95]900 → 1—
    2Submission[name=beta, score=68]——0 → 1
    3Submission[name=gamma, score=82]——1 → 2

  4. accepted ← 1

    9for (Submission submission : submissions) {10    if (submission.score() >= minimumScore90) {11        accepted→ 1++;12    } else {

  5. rejected ← 1

    pass 1 of 2
    11    accepted++;12} else {13    rejected→ 1++;14}

  6. rejected ← 2

    pass 2 of 2
    11    accepted++;12} else {13    rejected→ 2++;14}

  7. return new Summary(accepted, rejected);

    17    return new Summary(accepted, rejected);18}

  8. summary ← Summary[accepted=1, rejected=2]

    26    };27    Summary summary→ Summary[accepted=1, rejected=2] = summarize(submissions, minimumScore90);2829    System.out.println("accepted=" + summary.accepted());30    System.out.println("rejected=" + summary.rejected());31}
    outputaccepted=1
rejected=2