Data Pipeline Patterns

Config Merge

A merge step can combine default settings with a specific override.

default settings Defaults give the pipeline a complete configuration even when no override is present.
override rule The merge rule chooses the override only when it is meaningful.

Config Merge

overrideTimeout
ConfigMerge.java
Replay: real traced execution (multi-file project)
public class ConfigMerge {
    record Settings(int timeoutSeconds, boolean audit) {}

    static Settings merge(Settings defaults, int overrideTimeout) {
        int timeout = overrideTimeout > 0 ? overrideTimeout : defaults.timeoutSeconds();
        return new Settings(timeout, defaults.audit());
    }

    public static void main(String[] args) {
        int overrideTimeout = 0;
        Settings defaults = new Settings(30, true);
        Settings settings = merge(defaults, overrideTimeout);

        System.out.println("timeout=" + settings.timeoutSeconds());
        System.out.println("audit=" + settings.audit());
    }
}

public class ConfigMerge {
    record Settings(int timeoutSeconds, boolean audit) {}

    static Settings merge(Settings defaults, int overrideTimeout) {
        int timeout = overrideTimeout > 0 ? overrideTimeout : defaults.timeoutSeconds();
        return new Settings(timeout, defaults.audit());
    }

    public static void main(String[] args) {
        int overrideTimeout = 15;
        Settings defaults = new Settings(30, true);
        Settings settings = merge(defaults, overrideTimeout);

        System.out.println("timeout=" + settings.timeoutSeconds());
        System.out.println("audit=" + settings.audit());
    }
}

public class ConfigMerge {
    record Settings(int timeoutSeconds, boolean audit) {}

    static Settings merge(Settings defaults, int overrideTimeout) {
        int timeout = overrideTimeout > 0 ? overrideTimeout : defaults.timeoutSeconds();
        return new Settings(timeout, defaults.audit());
    }

    public static void main(String[] args) {
        int overrideTimeout = 45;
        Settings defaults = new Settings(30, true);
        Settings settings = merge(defaults, overrideTimeout);

        System.out.println("timeout=" + settings.timeoutSeconds());
        System.out.println("audit=" + settings.audit());
    }
}

  1. overrideTimeout ← 0, defaults ← Settings[timeoutSeconds=30, audit=true]

    9public static void main(String[] args) {10    int overrideTimeout→ 0 = 0;  //@overrideTimeout=15, 4511    Settings defaults→ Settings[timeoutSeconds=30, audit=true] = new Settings(30, true);12    Settings settings = merge(defaultsSettings[timeoutSeconds=30, audit=true], overrideTimeout0);

  2. timeout ← 30

    4static Settings merge(Settings defaultsSettings[timeoutSeconds=30, audit=true], int overrideTimeout0) {5    int timeout→ 30 = overrideTimeout0 > 0 ? overrideTimeout : defaults.timeoutSeconds();6    return new Settings(timeout, defaults.audit());7}

  3. settings ← Settings[timeoutSeconds=30, audit=true]

    11    Settings defaults = new Settings(30, true);12    Settings settings→ Settings[timeoutSeconds=30, audit=true] = merge(defaultsSettings[timeoutSeconds=30, audit=true], overrideTimeout0);1314    System.out.println("timeout=" + settings.timeoutSeconds());15    System.out.println("audit=" + settings.audit());16}
    outputtimeout=30
audit=true

  1. overrideTimeout ← 15, defaults ← Settings[timeoutSeconds=30, audit=true]

    9public static void main(String[] args) {10    int overrideTimeout→ 15 = 15;11    Settings defaults→ Settings[timeoutSeconds=30, audit=true] = new Settings(30, true);12    Settings settings = merge(defaultsSettings[timeoutSeconds=30, audit=true], overrideTimeout15);

  2. timeout ← 15

    4static Settings merge(Settings defaultsSettings[timeoutSeconds=30, audit=true], int overrideTimeout15) {5    int timeout→ 15 = overrideTimeout15 > 0 ? overrideTimeout : defaults.timeoutSeconds();6    return new Settings(timeout, defaults.audit());7}

  3. settings ← Settings[timeoutSeconds=15, audit=true]

    11    Settings defaults = new Settings(30, true);12    Settings settings→ Settings[timeoutSeconds=15, audit=true] = merge(defaultsSettings[timeoutSeconds=30, audit=true], overrideTimeout15);1314    System.out.println("timeout=" + settings.timeoutSeconds());15    System.out.println("audit=" + settings.audit());16}
    outputtimeout=15
audit=true

  1. overrideTimeout ← 45, defaults ← Settings[timeoutSeconds=30, audit=true]

    9public static void main(String[] args) {10    int overrideTimeout→ 45 = 45;11    Settings defaults→ Settings[timeoutSeconds=30, audit=true] = new Settings(30, true);12    Settings settings = merge(defaultsSettings[timeoutSeconds=30, audit=true], overrideTimeout45);

  2. timeout ← 45

    4static Settings merge(Settings defaultsSettings[timeoutSeconds=30, audit=true], int overrideTimeout45) {5    int timeout→ 45 = overrideTimeout45 > 0 ? overrideTimeout : defaults.timeoutSeconds();6    return new Settings(timeout, defaults.audit());7}

  3. settings ← Settings[timeoutSeconds=45, audit=true]

    11    Settings defaults = new Settings(30, true);12    Settings settings→ Settings[timeoutSeconds=45, audit=true] = merge(defaultsSettings[timeoutSeconds=30, audit=true], overrideTimeout45);1314    System.out.println("timeout=" + settings.timeoutSeconds());15    System.out.println("audit=" + settings.audit());16}
    outputtimeout=45
audit=true