python之NLP词性标注

1、知识点

包括中文和英文的词性标注
主要使用的库是nltk和jiaba

2、代码

# coding = utf-8

import nltk
from nltk.corpus import stopwords
from nltk.corpus import brown
import numpy as np
"""
标注步骤:
    1、清洗，分词
    2、标注
    
FAQ:
    1、 Resource punkt not found.
        请安装punkt模块 
    2、安装average_perceptron tagger
    3、Resource sinica_treebank not found
        请安装sinica_treebank模块
"""
def english_label():
    """
    英文词性标注
    :return:
    """
    # 分词
    text = "Sentiment analysis is a challenging subject in machine learning.\
     People express their emotions in language that is often obscured by sarcasm,\
      ambiguity, and plays on words, all of which could be very misleading for \
      both humans and computers.".lower()
    text_list = nltk.word_tokenize(text)
    # 去掉标点符号
    english_punctuations = [',', '.', ':', ';', '?', '(', ')', '[', ']', '&', '!', '*', '@', '#', '$', '%']
    text_list = [word for word in text_list if word not in english_punctuations]
    # 去掉停用词
    stops = set(stopwords.words("english"))
    text_list = [word for word in text_list if word not in stops]

    list = nltk.pos_tag(text_list) #打标签
    print(list)


def chineses_label():
    import jieba.posseg as pseg
    import re
    """
    fool也可以针对中文词性标注
    HanLP词性标注集
    案例使用jieba进行词性标注
    :return:
    """
    str = "我爱你，是粉色，舒服 ，舒服，士大夫"
    posseg_list = re.sub(r'[，]', " ", str)
    posseg_list =pseg.cut(posseg_list)
    print(posseg_list)
    print(' '.join('%s/%s' % (word, tag) for (word, tag) in posseg_list))