In the native solr, there is a request handler implementation of /get (org.apache.solr.handler.component.RealTimeGetComponent). The principle of implementation is to first go to the memory of tlog to find records, and then go to indexSearch to find them if they are not found. In this way, after the user submits to the solr server using the solrj client, the newly submitted record information can be immediately obtained from the server without waiting for the server softcommit.
This can avoid the problem of dirty data writing. However, the collection index structure uses Nested Document. After the client submits a Nested Document record, and then uses solrj to call the "getById" method, only the parent document can be returned . For this reason, in the production environment, the time interval between two business operations on the same record is shorter than the time period of soft commit, which will cause the problem of dirty data writing.
To solve this problem, it is necessary to extend Solr's SearchComponent, so that all the child documents (child docuemnt) of the document can be loaded through the ID.
import java.io.IOException;
import java.util.ArrayList;
import java.util.List;
import org.apache.lucene.document.Document;
import org.apache.lucene.index.IndexableField;
import org.apache.lucene.index.LeafReaderContext;
import org.apache.lucene.util.BytesRefBuilder;
import org.apache.solr.common.SolrDocument;
import org.apache.solr.common.SolrDocumentList;
import org.apache.solr.common.SolrInputDocument;
import org.apache.solr.common.params.SolrParams;
import org.apache.solr.core.SolrCore;
import org.apache.solr.handler.component.RealTimeGetComponent;
import org.apache.solr.handler.component.ResponseBuilder;
import org.apache.solr.handler.component.SearchComponent;
import org.apache.solr.request.SolrQueryRequest;
import org.apache.solr.response.BasicResultContext;
import org.apache.solr.response.ResultContext;
import org.apache.solr.response.SolrQueryResponse;
import org.apache.solr.response.transform.DocTransformer;
import org.apache.solr.schema.FieldType;
import org.apache.solr.schema.IndexSchema;
import org.apache.solr.schema.SchemaField;
import org.apache.solr.search.ReturnFields;
import org.apache.solr.search.SolrIndexSearcher;
import org.apache.solr.search.SolrReturnFields;
import org.apache.solr.util.RefCounted;
public class NestRealtimeGetComponet extends SearchComponent {
public static final String COMPONENT_NAME = "nestget";
@Override
public void prepare(ResponseBuilder rb) throws IOException {
SolrQueryRequest req = rb.req;
SolrQueryResponse rsp = rb.rsp;
SolrParams params = req.getParams();
if (!params.getBool(COMPONENT_NAME, false)) {
return;
}
// Set field flags
ReturnFields returnFields = new SolrReturnFields(rb.req);
rb.rsp.setReturnFields(returnFields);
}
@Override
public void process(ResponseBuilder rb) throws IOException {
SolrQueryRequest req = rb.req;
SolrQueryResponse rsp = rb.rsp;
SolrParams params = req.getParams();
if (!params.getBool(COMPONENT_NAME, false)) {
return;
}
String id = params.get("id");
SchemaField idField = req.getSchema().getUniqueKeyField();
FieldType fieldType = idField.getType();
BytesRefBuilder idBytes = new BytesRefBuilder ();
fieldType.readableToIndexed(id, idBytes);
SolrCore core = req.getCore();
SolrInputDocument doc = RealTimeGetComponent.getInputDocumentFromTlog(
core, idBytes.get());
SolrDocumentList docList = new SolrDocumentList();
if (doc != null) {
docList.add(convertDocument(doc));
docList.setNumFound (1);
} else {
RefCounted<SolrIndexSearcher> searchHolder = req.getCore()
.getSearcher();
SolrIndexSearcher searcher = searchHolder.get();
// get transfer
DocTransformer transformer = rsp.getReturnFields().getTransformer();
if (transformer != null) {
ResultContext context = new BasicResultContext(null,
rsp.getReturnFields(), null, null, req);
transformer.setContext(context);
}
try {
int docid = -1;
long segAndId = searcher.lookupId(idBytes.get());
if (segAndId >= 0) {
int segid = (int) segAndId;
LeafReaderContext ctx = searcher.getTopReaderContext()
.leaves().get((int) (segAndId >> 32));
docid = segid + ctx.docBase;
}
if (docid >= 0) {
Document luceneDocument = searcher.doc(docid, rsp
.getReturnFields().getLuceneFieldNames());
SolrDocument d = toSolrDoc(luceneDocument,
core.getLatestSchema());
searcher.decorateDocValueFields(d, docid,
searcher.getNonStoredDVs(true));
if (transformer != null) {
transformer.transform(d, docid, 0);
}
docList.add(d);
docList.setNumFound (1);
}
} finally {
searchHolder.decref();
}
}
rb.rsp.addResponse(docList);
}
private static SolrDocument toSolrDoc(Document doc, IndexSchema schema) {
SolrDocument out = new SolrDocument();
for (IndexableField f : doc.getFields()) {
// Make sure multivalued fields are represented as lists
Object existing = out.get(f.name());
if (existing == null) {
SchemaField sf = schema.getFieldOrNull(f.name());
// don't return copyField targets
if (sf != null && schema.isCopyFieldTarget(sf))
continue;
if (sf != null && sf.multiValued()) {
List<Object> vals = new ArrayList<>();
vals.add(f);
out.setField(f.name(), vals);
} else {
out.setField(f.name(), f);
}
} else {
out.addField(f.name(), f);
}
}
return out;
}
protected SolrDocument convertDocument(SolrInputDocument doc) {
SolrDocument sdoc = new SolrDocument();
for (String k : doc.getFieldNames()) {
sdoc.setField(k, doc.getFieldValue(k));
}
if (doc.hasChildDocuments()) {
for (SolrInputDocument s : doc.getChildDocuments()) {
sdoc.addChildDocument(convertDocument(s));
}
}
return sdoc;
}
}
Configuration in solrconfig.xml:
<searchComponent name="nestget"
class="com.dfire.tis.solrextend.handler.component.NestRealtimeGetComponet" />
requestHandler name="/select" class="solr.SearchHandler">
<lst name="defaults">
<str name="echoParams">explicit</str>
<int name="rows">10</int>
<str name="df">text</str>
</lst>
<arr name="last-components">
<str>nestget</str>
</arr>
</requestHandler>
Client query example:
SolrQuery query = new SolrQuery();
query.setParam("nestget", true);
query.set("id", pid);
query.setQuery("id:0");
query.setFields("*", "[child parentFilter=type:p childFilter=\"{!terms f=id}" + cid + "\" limit=100]");
QueryResponse r = this.client.query(collection, pid, query);
SolrDocumentList doclist = r.getResults();
for (SolrDocument d : doclist) {
System.out.println(d.get("id"));
System.out.println();
if (d.getChildDocumentCount() > 0) {
for (SolrDocument c : d.getChildDocuments()) {
StringBuffer f = new StringBuffer();
for (String key : c.getFieldNames()) {
f.append(key).append(":").append(c.getFirstValue(key));
}
System.out.println(f.toString());
}
}
}