kafka consumers pull message

This article only tracking process pull message consumers. For java client, kafka producers and consumers reuse the same network NetworkClient io class.

KafkaConsumer # pollOnce the inlet, extracting main steps:

// configuration FetchRequest request, the request unsent objects in the collection, transmission wait 
fetcher.sendFetches (); 

// remove the request unsent, # Send call NetworkClient, NetworkClinet # poll 
client.poll (pollTimeout, nowMs, new new PollCondition () { 
    @Override 
    public  Boolean shouldBlock () {
         // Operating since FETCH A by Might BE Completed The background Thread, the this poll WE need for condition Condition
         // to that Ensure® WE do Not Block unnecessarily in poll () 
        return ! fetcher.hasCompletedFetches (); 
    } 
}); 

// returns the data to the user 
return fetcher.fetchedRecords ();

Fetcher#sendFetches

public  the synchronized  int sendFetches () {
     // configured pull message requests. From which node, which partition, pull message what position 
    the Map <the Node, FetchSessionHandler.FetchRequestData> fetchRequestMap = prepareFetchRequests ();
     for (of Map.Entry <the Node, FetchSessionHandler.FetchRequestData> entry: fetchRequestMap.entrySet ()) {
         Final the Node fetchTarget = entry.getKey ();
         Final FetchSessionHandler.FetchRequestData Data = entry.getValue ();
         // 1. FetchRequest objects configured by Builder 
        Final FetchRequest.Builder Request = FetchRequest.Builder 
                .forConsumer ( the this.maxWaitMs, this.minBytes, data.toSend())
                .isolationLevel(isolationLevel)
                .setMaxBytes(this.maxBytes)
                .metadata(data.metadata())
                .toForget(data.toForget());
        if (log.isDebugEnabled()) {
            log.debug("Sending {} {} to broker {}", isolationLevel, data.toString(), fetchTarget);
        }

        
        client.send(fetchTarget, request)
                //4. 给 RequestFutureCompletionHandler.future 添加 RequestFutureListener
                .addListener(new RequestFutureListener<ClientResponse>() {
                    @Override
                    public void onSuccess(ClientResponse resp) {
                        synchronized (Fetcher.this) {
                            FetchResponse response = (FetchResponse) resp.responseBody();
                            FetchSessionHandler handler = sessionHandler(fetchTarget.id());
                            if (handler == null) {
                                log.error("Unable to find FetchSessionHandler for node {}. Ignoring fetch response.",
                                    fetchTarget.id());
                                return;
                            }
                            if (!handler.handleResponse(response)) {
                                return;
                            }

                            Set<TopicPartition> partitions = new HashSet<>(response.responseData().keySet());
                            FetchResponseMetricAggregator metricAggregator = new FetchResponseMetricAggregator(sensors, partitions);

                            for (Map.Entry<TopicPartition, FetchResponse.PartitionData> entry : response.responseData().entrySet()) {
                                TopicPartition partition = entry.getKey();
                                long fetchOffset = data.sessionPartitions().get(partition).fetchOffset;
                                FetchResponse.PartitionData fetchData = entry.getValue();

                                log.debug("Fetch {} at offset {} for partition {} returned fetch data {}",
                                        isolationLevel, fetchOffset, partition, fetchData);
                                // 10. 把数据放入 completedFetches，最终返回给用户
                                completedFetches.add(new CompletedFetch(partition, fetchOffset, fetchData, metricAggregator,
                                        resp.requestHeader().apiVersion()));
                            }

                            sensors.fetchLatency.record(resp.requestLatencyMs());
                        }
                    }

                    @Override
                    public void onFailure(RuntimeException e) {
                        synchronized (Fetcher.this) {
                            FetchSessionHandler handler = sessionHandler(fetchTarget.id());
                            if (handler != null) {
                                handler.handleError(e);
                            }
                        }
                    }
                });
    }
    return fetchRequestMap.size();
}

ConsumerNetworkClient#send

public RequestFuture<ClientResponse> send(Node node, AbstractRequest.Builder<?> requestBuilder) {
    long now = time.milliseconds();
    //2. 使用 RequestFutureCompletionHandler 作为回调函数
    RequestFutureCompletionHandler completionHandler = new RequestFutureCompletionHandler();
    ClientRequest clientRequest = client.newClientRequest(node.idString(), requestBuilder, now, true,
            completionHandler);
    //3. 请求放入 unsent 集合        
    unsent.put(node, clientRequest);

    // wakeup the client in case it is blocking in poll so that we can send the queued request
    client.wakeup();
    return completionHandler.future;
}

ConsumerNetworkClient#poll

// The transmission request is unsent, and no network IO 
trySend (now); 

// real writing and reading data network
 // 6. transmission request
 @ 7 receives a response
 // 8. RequestFutureCompletionHandler trigger callback 
client. poll (0 , now); 

// 9. the triggering in the callback RequestFutureListener 
firePendingCompletedRequests ();

NetworkClient#handleCompletedReceives

private void handleCompletedReceives(List<ClientResponse> responses, long now) {
    for (NetworkReceive receive : this.selector.completedReceives()) {
        String source = receive.source();
        InFlightRequest req = inFlightRequests.completeNext(source);
        Struct responseStruct = parseStructMaybeUpdateThrottleTimeMetrics(receive.payload(), req.header,
            throttleTimeSensor, now);
        if (log.isTraceEnabled()) {
            log.trace("Completed receive from node {} for {} with correlation id {}, received {}", req.destination,
                req.header.apiKey(), req.header.correlationId(), responseStruct);
        }
        AbstractResponse body = AbstractResponse.parseResponse(req.header.apiKey(), responseStruct);
        if (req.isInternalRequest && body instanceof MetadataResponse)
            metadataUpdater.handleCompletedMetadataResponse(req.header, now, (MetadataResponse) body);
        else if (req.isInternalRequest && body instanceof ApiVersionsResponse)
            handleApiVersionsResponse(responses, req, now, (ApiVersionsResponse) body);
        else
            // 此处给 responses 添加元素
            //new new ClientResponse return (header, callback, Where do you want, createdTimeMs, timeMs, to false, null, Response);
             // directly assigned to the response callback request 
             // Manufacturer transmitted callback message, the user is passed via the parameter
             // consumption by pulling the callback message, ConsumerNetworkClient # send is designated, is RequestFutureCompletionHandler 
            responses.add (req.completed (body, now)); 
    } 
}

NetworkClient#completeResponses

private void completeResponses(List<ClientResponse> responses) {
    for (ClientResponse response : responses) {
        try {
            // callback.onComplete(this);
            response.onComplete();
        } catch (Exception e) {
            log.error("Uncaught error in request completion:", e);
        }
    }
}

RequestFutureCompletionHandler#onComplete

public void onComplete(ClientResponse response) {
    this.response = response;
    pendingCompletion.add(this);
}

ConsumerNetworkClient#firePendingCompletedRequests

private void firePendingCompletedRequests() {
    boolean completedRequestsFired = false;
    for (;;) {
        RequestFutureCompletionHandler completionHandler = pendingCompletion.poll();
        if (completionHandler == null)
            break;

        completionHandler.fireCompletion();
        completedRequestsFired = true;
    }

    // wakeup the client in case it is blocking in poll for this future's completion
    if (completedRequestsFired)
        client.wakeup();
}

ConsumerNetworkClient.RequestFutureCompletionHandler#fireCompletion

public void fireCompletion() {
    if (e != null) {
        future.raise(e);
    } else if (response.wasDisconnected()) {
        RequestHeader requestHeader = response.requestHeader();
        int correlation = requestHeader.correlationId();
        log.debug("Cancelled {} request {} with correlation id {} due to node {} being disconnected",
                requestHeader.apiKey(), requestHeader, correlation, response.destination());
        future.raise(DisconnectException.INSTANCE);
    } else if (response.versionMismatch() != null) {
        future.raise(response.versionMismatch());
    } else {
        future.complete(response);
    }
}

RequestFuture#complete

public void complete(T value) {
    try {
        if (value instanceof RuntimeException)
            throw new IllegalArgumentException("The argument to complete can not be an instance of RuntimeException");

        if (!result.compareAndSet(INCOMPLETE_SENTINEL, value))
            throw new IllegalStateException("Invalid attempt to complete a request future which is already complete");
        fireSuccess();
    } finally {
        completedLatch.countDown();
    }
}

private void fireSuccess() {
    T value = value();
    while (true) {
        RequestFutureListener<T> listener = listeners.poll();
        if (listener == null)
            break;
        // 终于调到 RequestFutureListener
        listener.onSuccess(value);
    }
}

 

If you do not consider heartbeat thread, consumer first poll is no data, in which case the request was sent out, the response has not come back, to be in the second poll, in order to process network events to read and write at the same time.

After finished with, personally I feel the call chain is very long. Feel a thing, the entire process only one thread, but each time taking the branch are not the same, giving inspiration is that they do not wait for the single-threaded, fast.