The traditional parsing of JSON is to completely construct a complete JSON text content and then parse and deserialize it through the JSON framework, but when the JSON content is very long, for example, a JSON list stores N tuples, and it is very long. Thousands. So how to achieve streaming when transmitting JSON text content? Of course, there are methods. There is such a piece of code in Solr's Xport code. The class name is: JSONTupleStream.
import java.io.IOException; import java.io.Reader; import java.util.List; import java.util.Map; import org.apache.solr.client.solrj.io.stream.SolrStream; import org.noggit.JSONParser; import org.noggit.ObjectBuilder; public class JSONTupleStream { private List<String> path; // future... for more general stream handling private Reader reader; private JSONParser parser; private boolean atDocs; public JSONTupleStream(Reader reader) { this.reader = reader; this.parser = new JSONParser(reader); } /** returns the next Tuple or null */ public Map<String,Object> next() throws IOException { if (!atDocs) { boolean found = advanceToDocs(); atDocs = true; if (!found) return null; } // advance past ARRAY_START (in the case that we just advanced to docs, or OBJECT_END left over from the last call. int event = parser.nextEvent(); if (event == JSONParser.ARRAY_END) return null; Object o = ObjectBuilder.getVal(parser); // right now, getVal will leave the last event read as OBJECT_END return (Map<String,Object>)o; } public void close() throws IOException { reader.close(); } private void expect(int parserEventType) throws IOException { int event = parser.nextEvent(); if (event != parserEventType) { throw new IOException("JSONTupleStream: expected " + JSONParser.getEventString(parserEventType) + " but got " + JSONParser.getEventString(event) ); } } private void expect(String mapKey) { } private boolean advanceToMapKey(String key, boolean deepSearch) throws IOException { for (;;) { int event = parser.nextEvent(); switch (event) { case JSONParser.STRING: if (key != null) { String val = parser.getString(); if (key.equals(val)) { return true; } else if("error".equals(val)) { handleError(); } } break; case JSONParser.OBJECT_END: return false; case JSONParser.OBJECT_START: if (deepSearch) { boolean found = advanceToMapKey(key, true); if (found) { return true; } } else { advanceToMapKey(null, false); } break; case JSONParser.ARRAY_START: skipArray(key, deepSearch); break; } } } private void handleError() throws IOException { for (;;) { int event = parser.nextEvent(); if(event == JSONParser.STRING) { String val = parser.getString(); if("msg".equals(val)) { event = parser.nextEvent(); if(event == JSONParser.STRING) { String msg = parser.getString(); if(msg != null) { throw new SolrStream.HandledException(msg); } } } } else if (event == JSONParser.OBJECT_END) { throw new IOException(""); } } } private void skipArray(String key, boolean deepSearch) throws IOException { for (;;) { int event = parser.nextEvent(); switch (event) { case JSONParser.OBJECT_START: advanceToMapKey(key, deepSearch); break; case JSONParser.ARRAY_START: skipArray(key, deepSearch); break; case JSONParser.ARRAY_END: return; } } } private boolean advanceToDocs() throws IOException { expect(JSONParser.OBJECT_START); advanceToMapKey("numFound", true); expect(JSONParser.LONG); // int event = parser.nextEvent(); //if (event == JSONParser.ARRAY_END) return null; System.out.println( ObjectBuilder.getVal(parser)); boolean found = advanceToMapKey("docs", true); expect(JSONParser.ARRAY_START); return found; } }
Write another test script:
import java.io.StringReader; import java.util.Map; import com.dfire.tis.solrextend.core.JSONTupleStream; import junit.framework.TestCase; public class TestTupleStreamParser extends TestCase { public void test() throws Exception { StringReader reader = new StringReader("{\"responseHeader\": {\"status\": 0}, \"response\":{\"numFound\":" + 100 + ", \"docs\":[{name:'baisui'},{name:'xxxx'}]}"); JSONTupleStream tupleStream = new JSONTupleStream(reader); while (true) { Map<String, Object> tuple = tupleStream.next(); if (tuple != null) { System.out.println(tuple.get("name")); } else { break; } } } }
In this way, the sender can keep sending, and the client can consume continuously. The advantage of this is that when the server client transmits a large amount of structured data, it can avoid the need to open up a Large memory space, which can effectively prevent memory overflow, and at the same time, can also effectively improve the system throughput.