There is a new system needs to read the file data rotate through the flume of TaildirSource, the so-called rotate file refers to the data file reaches the specified size of our time, the system will automatically rename the backup history file to a new file, for example, specify the size of the log file 1M, the system continuously generates log information in the log file, if the file reaches 1M, then the system will generate log.1, log.2 this history file; flume by the combined use of TaildirSource to monitor log file when found, can not generate log.1, log.2 this historical document, that is, if there is TaildirSource to monitor this file, then the file after reaching the specified size can not be renamed because file occupied the process, can not modify its name, the equivalent of us in the windows system is open to rename a file, it is not.
In fact, we can test a simple java test class RandomAccessFile and FileChannel two classes read difference file time, by the time RandomAccessFile read the file, it is not possible to modify the file name, and I want java in other obstructive IO streams too, and use FileChannel is possible, but can not get FileChannel use the following way:
After the study TaildirSource source code found, TailFile class operation when reading a file, it uses RandomAccessFile, it is an obstructive java file operations like read, we need to change it to a non-blocking file operations categories: FileChannel, and acquires the object's constructor method is carried out by its own methods:
FileChannel inChannel = FileChannel.open(Paths.get(file.getAbsolutePath()), StandardOpenOption.READ);In fact, we can test a simple java test class RandomAccessFile and FileChannel two classes read difference file time, by the time RandomAccessFile read the file, it is not possible to modify the file name, and I want java in other obstructive IO streams too, and use FileChannel is possible, but can not get FileChannel use the following way:
RandomAccessFile aFile = new RandomAccessFile("data/nio-data.txt", "rw");
FileChannel inChannel = aFile.getChannel();After testing, the corresponding file will TailFile RandomAccessFile TaildirSource way into the source code of the corresponding FileChannel method can solve this problem. Class source code is modified as follows:
package com.centrify.platform.analytics.agent.mysource;
import static org.apache.flume.source.taildir.TaildirSourceConfigurationConstants.BYTE_OFFSET_HEADER_KEY;
import java.io.File;
import java.io.IOException;
import java.nio.ByteBuffer;
import java.nio.channels.FileChannel;
import java.nio.file.Paths;
import java.nio.file.StandardOpenOption;
import java.util.List;
import java.util.Map;
import org.apache.flume.Event;
import org.apache.flume.event.EventBuilder;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
import com.google.common.collect.Lists;
public class TailFile {
private static final Logger logger = LoggerFactory.getLogger(TailFile.class);
private static final byte BYTE_NL = (byte) 10;
private static final byte BYTE_CR = (byte) 13;
private static final int BUFFER_SIZE = 8192;
private static final int NEED_READING = -1;
//private RandomAccessFile raf;
private FileChannel inChannel;
private final String path;
private final long inode;
private long pos;
private long lastUpdated;
private boolean needTail;
private final Map<String, String> headers;
private byte[] buffer;
private byte[] oldBuffer;
private int bufferPos;
private long lineReadPos;
public TailFile(File file, Map<String, String> headers, long inode, long pos)
throws IOException {
//this.raf = new RandomAccessFile(file, "r");
this.inChannel = FileChannel.open(Paths.get(file.getAbsolutePath()), StandardOpenOption.READ);
if (pos > 0) {
//raf.seek(pos);
inChannel.position(pos);
lineReadPos = pos;
}
this.path = file.getAbsolutePath();
this.inode = inode;
this.pos = pos;
this.lastUpdated = 0L;
this.needTail = true;
this.headers = headers;
this.oldBuffer = new byte[0];
this.bufferPos = NEED_READING;
}
//public RandomAccessFile getRaf() {
// return raf;
// }
public FileChannel getFileChannel(){
return inChannel;
}
public String getPath() {
return path;
}
public long getInode() {
return inode;
}
public long getPos() {
return pos;
}
public long getLastUpdated() {
return lastUpdated;
}
public boolean needTail() {
return needTail;
}
public Map<String, String> getHeaders() {
return headers;
}
public long getLineReadPos() {
return lineReadPos;
}
public void setPos(long pos) {
this.pos = pos;
}
public void setLastUpdated(long lastUpdated) {
this.lastUpdated = lastUpdated;
}
public void setNeedTail(boolean needTail) {
this.needTail = needTail;
}
public void setLineReadPos(long lineReadPos) {
this.lineReadPos = lineReadPos;
}
public boolean updatePos(String path, long inode, long pos) throws IOException {
if (this.inode == inode && this.path.equals(path)) {
setPos(pos);
updateFilePos(pos);
logger.info("Updated position, file: " + path + ", inode: " + inode + ", pos: " + pos);
return true;
}
return false;
}
public void updateFilePos(long pos) throws IOException {
//raf.seek(pos);
inChannel.position(pos);
lineReadPos = pos;
bufferPos = NEED_READING;
oldBuffer = new byte[0];
}
public List<Event> readEvents(int numEvents, boolean backoffWithoutNL,
boolean addByteOffset) throws IOException {
List<Event> events = Lists.newLinkedList();
for (int i = 0; i < numEvents; i++) {
Event event = readEvent(backoffWithoutNL, addByteOffset);
if (event == null) {
break;
}
events.add(event);
}
return events;
}
private Event readEvent(boolean backoffWithoutNL, boolean addByteOffset) throws IOException {
Long posTmp = getLineReadPos();
LineResult line = readLine();
if (line == null) {
return null;
}
if (backoffWithoutNL && !line.lineSepInclude) {
// logger.info("Backing off in file without newline: "
// + path + ", inode: " + inode + ", pos: " + raf.getFilePointer());
logger.info("Backing off in file without newline: "
+ path + ", inode: " + inode + ", pos: " + inChannel.position());
updateFilePos(posTmp);
return null;
}
Event event = EventBuilder.withBody(line.line);
if (addByteOffset == true) {
event.getHeaders().put(BYTE_OFFSET_HEADER_KEY, posTmp.toString());
}
return event;
}
private void readFile() throws IOException {
/*if ((raf.length() - raf.getFilePointer()) < BUFFER_SIZE) {
buffer = new byte[(int) (raf.length() - raf.getFilePointer())];
} else {
buffer = new byte[BUFFER_SIZE];
}
raf.read(buffer, 0, buffer.length);*/
if ((inChannel.size() - inChannel.position()) < BUFFER_SIZE) {
buffer = new byte[(int) (inChannel.size() - inChannel.position())];
} else {
buffer = new byte[BUFFER_SIZE];
}
ByteBuffer byteBuffer = ByteBuffer.wrap(buffer);
inChannel.read(byteBuffer);
bufferPos = 0;
}
private byte[] concatByteArrays(byte[] a, int startIdxA, int lenA,
byte[] b, int startIdxB, int lenB) {
byte[] c = new byte[lenA + lenB];
System.arraycopy(a, startIdxA, c, 0, lenA);
System.arraycopy(b, startIdxB, c, lenA, lenB);
return c;
}
public LineResult readLine() throws IOException {
LineResult lineResult = null;
while (true) {
if (bufferPos == NEED_READING) {
//if (raf.getFilePointer() < raf.length()) {
if (inChannel.position() < inChannel.size()) {
readFile();
} else {
if (oldBuffer.length > 0) {
lineResult = new LineResult(false, oldBuffer);
oldBuffer = new byte[0];
setLineReadPos(lineReadPos + lineResult.line.length);
}
break;
}
}
for (int i = bufferPos; i < buffer.length; i++) {
if (buffer[i] == BYTE_NL) {
int oldLen = oldBuffer.length;
// Don't copy last byte(NEW_LINE)
int lineLen = i - bufferPos;
// For windows, check for CR
if (i > 0 && buffer[i - 1] == BYTE_CR) {
lineLen -= 1;
} else if (oldBuffer.length > 0 && oldBuffer[oldBuffer.length - 1] == BYTE_CR) {
oldLen -= 1;
}
lineResult = new LineResult(true,
concatByteArrays(oldBuffer, 0, oldLen, buffer, bufferPos, lineLen));
setLineReadPos(lineReadPos + (oldBuffer.length + (i - bufferPos + 1)));
oldBuffer = new byte[0];
if (i + 1 < buffer.length) {
bufferPos = i + 1;
} else {
bufferPos = NEED_READING;
}
break;
}
}
if (lineResult != null) {
break;
}
// NEW_LINE not showed up at the end of the buffer
oldBuffer = concatByteArrays(oldBuffer, 0, oldBuffer.length,
buffer, bufferPos, buffer.length - bufferPos);
bufferPos = NEED_READING;
}
return lineResult;
}
public void close() {
try {
//raf.close();
//raf = null;
inChannel.close();
inChannel = null;
long now = System.currentTimeMillis();
setLastUpdated(now);
} catch (IOException e) {
logger.error("Failed closing file: " + path + ", inode: " + inode, e);
}
}
private class LineResult {
final boolean lineSepInclude;
final byte[] line;
public LineResult(boolean lineSepInclude, byte[] line) {
super();
this.lineSepInclude = lineSepInclude;
this.line = line;
}
}
}