腾讯mars xlog日志解压缩工具go语言版 暂时只实现未加密日志的解压缩

python 工具依赖太多

mars 地址

https://github.com/Tencent/mars/wiki/Xlog-%E5%8A%A0%E5%AF%86%E4%BD%BF%E7%94%A8%E6%8C%87%E5%BC%95

解密工具地址

https://github.com/wuhaowen/xlog-go-parser/blob/master/Parser.go

package main

import (
    "io/ioutil"
    "encoding/binary"
    "bytes"
    "errors"
    "fmt"
    "compress/zlib"
    "io"
)

const MAGIC_NO_COMPRESS_START = 0x03
const MAGIC_NO_COMPRESS_START1 = 0x06
const MAGIC_NO_COMPRESS_NO_CRYPT_START = 0x08
const MAGIC_COMPRESS_START = 0x04
const MAGIC_COMPRESS_START1 = 0x05
const MAGIC_COMPRESS_START2 = 0x07
const MAGIC_COMPRESS_NO_CRYPT_START = 0x09

const MAGIC_END = 0x00

var lastseq = 0

func BytesToInt(b []byte) int {
    bytesBuffer := bytes.NewBuffer(b)
    var tmp uint32
    binary.Read(bytesBuffer, binary.LittleEndian, &tmp)
    return int(tmp)
}

func BytesToShort(b []byte) int {
    bytesBuffer := bytes.NewBuffer(b)
    var tmp uint16
    binary.Read(bytesBuffer, binary.LittleEndian, &tmp)
    return int(tmp)
}

func IsGoodLogBuffer(_buffer []byte, _offset int, count int) (bool, error) {
    if _offset == len(_buffer) {
        return true, nil
    }
    magic_start := _buffer[_offset]
    crypt_key_len := 0
    if MAGIC_NO_COMPRESS_START == magic_start || MAGIC_COMPRESS_START == magic_start || MAGIC_COMPRESS_START1 == magic_start {
        crypt_key_len = 4
    } else if MAGIC_COMPRESS_START2 == magic_start || MAGIC_NO_COMPRESS_START1 == magic_start || MAGIC_NO_COMPRESS_NO_CRYPT_START == magic_start || MAGIC_COMPRESS_NO_CRYPT_START == magic_start {
        crypt_key_len = 64
    } else {
        return false, errors.New("_buffer[" + string(_offset) + "]:" + string(_buffer[_offset]) + " != MAGIC_NUM_START")
    }
    headerLen := 1 + 2 + 1 + 1 + 4 + crypt_key_len

    if _offset+headerLen+1+1 > len(_buffer) {
        return false, errors.New(fmt.Sprintf("'offset:%d > len(buffer):%d", _offset, len(_buffer)))
    }
    s := _buffer[ _offset+headerLen-4-crypt_key_len:  _offset+headerLen-crypt_key_len]
    length := BytesToInt(s)

    if _offset+headerLen+length+1 > len(_buffer) {
        return false, errors.New(fmt.Sprintf("log length:%d, end pos %d > len(buffer):%d", length, _offset+headerLen+length+1, len(_buffer)))

    }
    if MAGIC_END != _buffer[_offset+headerLen+length] {
        return false, errors.New(fmt.Sprintf("log length:%d, buffer[%d]:%d != MAGIC_END", length, _offset+headerLen+length, _buffer[_offset+headerLen+length]))
    }
    if 1 >= count {
        return true, nil
    }
    return IsGoodLogBuffer(_buffer, _offset+headerLen+length+1, count-1)
}

func GetLogStartPos(_buffer []byte, _count int) int {
    offset := 0
    for ; ; {
        if offset >= len(_buffer) {
            break
        }
        if MAGIC_NO_COMPRESS_START == _buffer[offset] || MAGIC_NO_COMPRESS_START1 == _buffer[offset] || MAGIC_COMPRESS_START == _buffer[offset] || MAGIC_COMPRESS_START1 == _buffer[offset] || MAGIC_COMPRESS_START2 == _buffer[offset] || MAGIC_COMPRESS_NO_CRYPT_START == _buffer[offset] || MAGIC_NO_COMPRESS_NO_CRYPT_START == _buffer[offset] {
            if isGood, _ := IsGoodLogBuffer(_buffer, offset, _count); isGood {
                return offset
            }
        }
        offset += 1
    }
    return -1
}

func DecodeBuffer(_buffer []byte, _offset int, _outbuffer *[]byte) int {

    if _offset >= len(_buffer) {
        return -1
    }
    ret, err := IsGoodLogBuffer(_buffer, _offset, 1)
    fixpos := 0
    if !ret {
        fixpos = GetLogStartPos(_buffer[_offset:], 1)
        if fixpos == -1 {
            return -1
        } else {
            //append(_outbuffer, )
            *_outbuffer = append(*_outbuffer, []byte(fmt.Sprintf("decode error len=%d, result:%s\n", fixpos, err.Error()))...)

            _offset += fixpos

        }
    }
    magic_start := _buffer[_offset]
    crypt_key_len := 0
    if MAGIC_NO_COMPRESS_START == magic_start || MAGIC_COMPRESS_START == magic_start || MAGIC_COMPRESS_START1 == magic_start {
        crypt_key_len = 4
    } else if MAGIC_COMPRESS_START2 == magic_start || MAGIC_NO_COMPRESS_START1 == magic_start || MAGIC_NO_COMPRESS_NO_CRYPT_START == magic_start || MAGIC_COMPRESS_NO_CRYPT_START == magic_start {
        crypt_key_len = 64
    } else {
        *_outbuffer = append(*_outbuffer, []byte(fmt.Sprintf("in DecodeBuffer _buffer[%d]:%d != MAGIC_NUM_START\n", fixpos, magic_start))...)

        return -1
    }
    headerLen := 1 + 2 + 1 + 1 + 4 + crypt_key_len
    length_buffer := _buffer[_offset+headerLen-4-crypt_key_len:_offset+headerLen-crypt_key_len]
    length := BytesToInt(length_buffer)
    seq_buffer := _buffer[ _offset+headerLen-4-crypt_key_len-2-2:   _offset+headerLen-4-crypt_key_len-2]
    seq := BytesToShort(seq_buffer)
    if seq != 0 && seq != 1 && lastseq != 0 && seq != lastseq+1 {
        *_outbuffer = append(*_outbuffer, []byte(fmt.Sprintf("log seq:%d-%d is missing\n", lastseq+1, seq-1))...)

    }

    if seq != 0 {
        lastseq = seq
    }
    tmpbuffer := _buffer[_offset+headerLen:_offset+headerLen+length]
    if MAGIC_NO_COMPRESS_START1 == _buffer[_offset] || MAGIC_COMPRESS_START2 == _buffer[_offset] {
        fmt.Println("use wrong decode script")
    } else if MAGIC_COMPRESS_START == _buffer[_offset] || MAGIC_COMPRESS_NO_CRYPT_START == _buffer[_offset] {
        tmpbuffer, err = DoZlibUnCompress(tmpbuffer)
    } else if MAGIC_COMPRESS_START1 == _buffer[_offset] {
        var decompress_data []byte
        for len(tmpbuffer) > 0 {
            single_log_len_buffer := tmpbuffer[0: 2]
            single_log_len := BytesToShort(single_log_len_buffer)
            decompress_data = append(decompress_data, tmpbuffer[2:single_log_len+2]...)
            tmpbuffer = tmpbuffer[single_log_len+2: ]
        }
        tmpbuffer, err = DoZlibUnCompress(decompress_data)
    }
    if err != nil {
        *_outbuffer = append(*_outbuffer, []byte("decompress err, "+err.Error()+"\n")...)
        return _offset + headerLen + length + 1
    }
    *_outbuffer = append(*_outbuffer, tmpbuffer...)
    return _offset + headerLen + length + 1

}

func DoZlibUnCompress(compressSrc []byte) ([]byte, error) {
    var i int16 = 0x78da
    b := make([]byte, 2)
    binary.BigEndian.PutUint16(b, uint16(i))
    b = append(b, compressSrc...)
    source := bytes.NewReader(b)
    var out bytes.Buffer
    r, err := zlib.NewReader(source)
    if err != nil {
        return nil, err
    }
    io.Copy(&out, r)
    return out.Bytes(), nil
}

func ParseFile(_file string, _outFile string) {
    _buffer, err := ioutil.ReadFile(_file)
    if err != nil {
        panic(err)
    }
    startpos := GetLogStartPos(_buffer, 2)
    if -1 == startpos {
        return
    }
    var outbuffer []byte
    for startpos != -1 {
        startpos = DecodeBuffer(_buffer, startpos, &outbuffer)
    }
    if len(outbuffer) == 0 {
        return
    }

    ioutil.WriteFile(_outFile, outbuffer, 0644)

}

func main() {
    ParseFile("./1.xlog", "./2.log")
}

猜你喜欢

转载自blog.csdn.net/csdn9990/article/details/111084228