Flattening a 3 level nested JSON string in java

wandermonk :

The requirement is to create a generic flattening utility for an input JSON object to a flattened JSON object.

The sample JSON looks like the below

{
  "Source": "source-1",
  "Rows": [
    {
      "Keys": {
        "device-id": "BC04-EBH-N3K-01",
        "interface-name": "TenGigE0/0/0/39",
        "node-name": "0/0/CPU0"
      },
      "Timestamp": 1567621527656,
      "inner": {
        "donm": {
          "id": "0062",
          "mol": {
            "rem": 30,
            "len": 11,
            "org": {
              "ldp": [
                {
                  "t": 486,
                  "o": 322
                },
                {
                  "t": 487,
                  "o": 32,
                  "twss": 1,
                  "tlv": "00:01"
                }
              ]
            },
            "chlen": 14,
            "poe": 5,
            "combs": 10,
            "chaype": 4,
            "rek": 0,
            "rem-um": 67
          },
          "detail": {
            "enas": "B,R",
            "systes": "B,R",
            "timng": 91,
            "syn": "C",
            "met-type": 0,
            "neses": {
              "lldEDIT": [
                {
                  "ium": 830,
                  "m": 1,
                  "ass": {
                    "ape": "ipv4",
                    "ipvs": "94"
                  }
                }
              ]
            },
            "pess": "0008",
            "por]d": 0,
            "pon": "BCtive",
            "sysme": "BC1"
          },
          "reme": "Bu1",
          "hean": 0,
          "porl": "Et1"
        }
      }
    }

  ],
  "Tey": {
    "epath": "Cgetail",
    "sustr": "MX",
    "coime": 1567621527653,
    "msp": 1567621527653,
    "come": 1567621527660,
    "nor": "BC5",
    "cid": 14789654
  }
}

I have been trying to flatten it to 3 levels and came up with the below utility. But, The things are getting complicated when I have to deal with Arrays and values of type String, long, Timestamp, etc. Also, I am unable to understand how the nested keys can be maintained for uniqueness.

public static Map<String,Object> flattenJson(JsonNode input){
        Map<String,Object> finalMap = new HashMap<>();

        ObjectMapper datamapper = new ObjectMapper();
        Map<String,Object> topLevelJsonMap = datamapper.convertValue(input,Map.class);
        Set<String> topLevelKeys = topLevelJsonMap.keySet();

        for(String topLevelKey : topLevelKeys){

            System.out.println("Key :::: "+topLevelKey);

            Object topLevelData = topLevelJsonMap.get(topLevelKey);
            System.out.println("value :::: "+topLevelData.toString());

            if(topLevelData instanceof ArrayNode){
                ArrayNode arrayOfData = (ArrayNode) topLevelData;
                for(JsonNode dataNode : arrayOfData){
                    flattenJson(input);
                }
            } else if(topLevelData instanceof JsonNode){
                Map<String,Object> innerLevelJsonMap = datamapper.convertValue(topLevelData,Map.class);
                Set<String> innerLevelKeys = innerLevelJsonMap.keySet();
                for(String innerLevelKey : innerLevelKeys){
                    System.out.println("inner key :::: "+innerLevelKey);
                    flattenJson((JsonNode) innerLevelJsonMap.get(innerLevelKey));
                }
            }else {
               finalMap.put(topLevelKey,topLevelData);
            }
        }
        return finalMap;
    }

Any help is greatly appreciated.

Michał Ziober :

To avoid conflicts with key names you can use JSON Pointer specification to create them. It is also supported by Jackson library, so you can use them later to traverse the JsonNode node.

Simple implementation could look like below:

import com.fasterxml.jackson.databind.JsonNode;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.fasterxml.jackson.databind.node.ArrayNode;
import com.fasterxml.jackson.databind.node.ObjectNode;
import java.io.File;
import java.util.LinkedHashMap;
import java.util.Map;
import java.util.Objects;
import java.util.concurrent.atomic.AtomicInteger;

public class JsonApp {

  public static void main(String[] args) throws Exception {
    File jsonFile = new File("./test.json");

    ObjectMapper mapper = new ObjectMapper();
    JsonNode root = mapper.readTree(jsonFile);
    Map<String, JsonNode> map = new JsonFlattener(root).flatten();

    System.out.println("Use key-value pairs:");
    map.forEach(
        (k, v) -> {
          System.out.println(k + " => " + v);
        });

    System.out.println();
    System.out.println("Use pointers:");
    map.forEach(
        (k, v) -> {
          System.out.println(k + " => " + root.at(k));
        });
  }
}

class JsonFlattener {

  private final Map<String, JsonNode> json = new LinkedHashMap<>(64);
  private final JsonNode root;

  JsonFlattener(JsonNode node) {
    this.root = Objects.requireNonNull(node);
  }

  public Map<String, JsonNode> flatten() {
    process(root, "");
    return json;
  }

  private void process(JsonNode node, String prefix) {
    if (node.isObject()) {
      ObjectNode object = (ObjectNode) node;
      object
          .fields()
          .forEachRemaining(
              entry -> {
                  process(entry.getValue(), prefix + "/" + entry.getKey());
              });
    } else if (node.isArray()) {
      ArrayNode array = (ArrayNode) node;
      AtomicInteger counter = new AtomicInteger();
      array
          .elements()
          .forEachRemaining(
              item -> {
                process(item, prefix + "/" + counter.getAndIncrement());
              });
    } else {
      json.put(prefix, node);
    }
  }
}

Above code prints:

Use key-value pairs:
/Source => "source-1"
/Rows/0/Keys/device-id => "BC04-EBH-N3K-01"
/Rows/0/Keys/interface-name => "TenGigE0/0/0/39"
/Rows/0/Keys/node-name => "0/0/CPU0"
/Rows/0/Timestamp => 1567621527656
/Rows/0/inner/donm/id => "0062"
/Rows/0/inner/donm/mol/rem => 30
/Rows/0/inner/donm/mol/len => 11
/Rows/0/inner/donm/mol/org/ldp/0/t => 486
/Rows/0/inner/donm/mol/org/ldp/0/o => 322
/Rows/0/inner/donm/mol/org/ldp/1/t => 487
/Rows/0/inner/donm/mol/org/ldp/1/o => 32
/Rows/0/inner/donm/mol/org/ldp/1/twss => 1
/Rows/0/inner/donm/mol/org/ldp/1/tlv => "00:01"
/Rows/0/inner/donm/mol/chlen => 14
/Rows/0/inner/donm/mol/poe => 5
/Rows/0/inner/donm/mol/combs => 10
/Rows/0/inner/donm/mol/chaype => 4
/Rows/0/inner/donm/mol/rek => 0
/Rows/0/inner/donm/mol/rem-um => 67
/Rows/0/inner/donm/detail/enas => "B,R"
/Rows/0/inner/donm/detail/systes => "B,R"
/Rows/0/inner/donm/detail/timng => 91
/Rows/0/inner/donm/detail/syn => "C"
/Rows/0/inner/donm/detail/met-type => 0
/Rows/0/inner/donm/detail/neses/lldEDIT/0/ium => 830
/Rows/0/inner/donm/detail/neses/lldEDIT/0/m => 1
/Rows/0/inner/donm/detail/neses/lldEDIT/0/ass/ape => "ipv4"
/Rows/0/inner/donm/detail/neses/lldEDIT/0/ass/ipvs => "94"
/Rows/0/inner/donm/detail/pess => "0008"
/Rows/0/inner/donm/detail/por]d => 0
/Rows/0/inner/donm/detail/pon => "BCtive"
/Rows/0/inner/donm/detail/sysme => "BC1"
/Rows/0/inner/donm/reme => "Bu1"
/Rows/0/inner/donm/hean => 0
/Rows/0/inner/donm/porl => "Et1"
/Tey/epath => "Cgetail"
/Tey/sustr => "MX"
/Tey/coime => 1567621527653
/Tey/msp => 1567621527653
/Tey/come => 1567621527660
/Tey/nor => "BC5"
/Tey/cid => 14789654

Use pointers:
/Source => "source-1"
/Rows/0/Keys/device-id => "BC04-EBH-N3K-01"
/Rows/0/Keys/interface-name => "TenGigE0/0/0/39"
/Rows/0/Keys/node-name => "0/0/CPU0"
/Rows/0/Timestamp => 1567621527656
/Rows/0/inner/donm/id => "0062"
/Rows/0/inner/donm/mol/rem => 30
/Rows/0/inner/donm/mol/len => 11
/Rows/0/inner/donm/mol/org/ldp/0/t => 486
/Rows/0/inner/donm/mol/org/ldp/0/o => 322
/Rows/0/inner/donm/mol/org/ldp/1/t => 487
/Rows/0/inner/donm/mol/org/ldp/1/o => 32
/Rows/0/inner/donm/mol/org/ldp/1/twss => 1
/Rows/0/inner/donm/mol/org/ldp/1/tlv => "00:01"
/Rows/0/inner/donm/mol/chlen => 14
/Rows/0/inner/donm/mol/poe => 5
/Rows/0/inner/donm/mol/combs => 10
/Rows/0/inner/donm/mol/chaype => 4
/Rows/0/inner/donm/mol/rek => 0
/Rows/0/inner/donm/mol/rem-um => 67
/Rows/0/inner/donm/detail/enas => "B,R"
/Rows/0/inner/donm/detail/systes => "B,R"
/Rows/0/inner/donm/detail/timng => 91
/Rows/0/inner/donm/detail/syn => "C"
/Rows/0/inner/donm/detail/met-type => 0
/Rows/0/inner/donm/detail/neses/lldEDIT/0/ium => 830
/Rows/0/inner/donm/detail/neses/lldEDIT/0/m => 1
/Rows/0/inner/donm/detail/neses/lldEDIT/0/ass/ape => "ipv4"
/Rows/0/inner/donm/detail/neses/lldEDIT/0/ass/ipvs => "94"
/Rows/0/inner/donm/detail/pess => "0008"
/Rows/0/inner/donm/detail/por]d => 0
/Rows/0/inner/donm/detail/pon => "BCtive"
/Rows/0/inner/donm/detail/sysme => "BC1"
/Rows/0/inner/donm/reme => "Bu1"
/Rows/0/inner/donm/hean => 0
/Rows/0/inner/donm/porl => "Et1"
/Tey/epath => "Cgetail"
/Tey/sustr => "MX"
/Tey/coime => 1567621527653
/Tey/msp => 1567621527653
/Tey/come => 1567621527660
/Tey/nor => "BC5"
/Tey/cid => 14789654