Spark Mlib(四)用spark计算tf-idf值

tf-idf算法是用统计的手法衡量一个元素在一个集合中的重要程度。在自然语言处理中，该算法可以衡量一个词在语料中的重要程度。其本思想很简单，字词的重要性随着它在文件中出现的次数成正比增加，但同时会随着它在语料库中出现的频率成反比下降。下面是spark官网（http://spark.apache.org/docs/latest/ml-features.html#tf-idf）给出的例子

package alg
import org.apache.spark.ml.feature.{HashingTF, IDF, Tokenizer}
import org.apache.spark.sql.SparkSession

object tfidf {


  def main(args:Array[String]):Unit={


    val spark: SparkSession = SparkSession.builder
      .appName("My")
      .master("local[*]")
      .getOrCreate()

    val sentenceData = spark.createDataFrame(Seq(
      (0.0, "Hi I heard about Spark"),
      (0.0, "I wish Java could use case classes"),
      (1.0, "Logistic regression models are neat")
    )).toDF("label", "sentence")

    val tokenizer = new Tokenizer().setInputCol("sentence").setOutputCol("words")
    val wordsData = tokenizer.transform(sentenceData)

    val hashingTF = new HashingTF()
      .setInputCol("words").setOutputCol("rawFeatures").setNumFeatures(20)

    val featurizedData = hashingTF.transform(wordsData)
    // alternatively, CountVectorizer can also be used to get term frequency vectors

    val idf = new IDF().setInputCol("rawFeatures").setOutputCol("features")
    val idfModel = idf.fit(featurizedData)

    val rescaledData = idfModel.transform(featurizedData)

    rescaledData.collect().foreach(print(_))
    //rescaledData.select("label", "features").show()
  }

}