Spark实践 | 数据挖掘实验_推荐

其他 2018-05-08 12:28:58 阅读次数: 3

数据来源：

http://files.grouplens.org/datasets/movielens/ml-100k.zip

代码：

import org.apache.spark.SparkConf
import org.apache.spark.SparkContext
import org.apache.spark.SparkContext._
import org.apache.spark.mllib.recommendation.ALS
import org.apache.spark.mllib.recommendation.Rating

/**
 * @author root
 */
object ExtFeatureRC {
   def main(args: Array[String]) {
   // if (args.length < 1) {
   //   System.err.println("Usage: <file>")
   //   System.exit(1)
   // }
    val conf = new SparkConf()
    val sc = new SparkContext(conf)
    //Extracting features from the MovieLens 100k dataset
    val rawData = sc.textFile("/user/ml-100k/u.data")
    //rawData.first()
    //rawData.foreach(println);
    //Extract the first three fields
    val rawRatings = rawData.map(_.split("\t").take(3))

   // rawRatings.foreach(println);
    //create our rating dataset using the map method and transforming the array of IDs and ratings into a Rating object
  val ratings = rawRatings.map{ case Array(user,movie,rating) =>
            Rating(user.toInt,movie.toInt,rating.toDouble)}
   //ratings.foreach(println);
    /*Training a model on the MovieLens 100k dataset
    rank: This refers to the number of factors in our ALS model, 
    that is, the number of hidden features in our low-rank approximation matrices
    iterations: This refers to the number of iterations to run
    lambda: This parameter controls the regularization of our model. Thus, lambda
controls over fitting. The higher the value of lambda, the more is the regularization applied
    */
   val model = ALS.train(ratings,5,5,0.01)
   // println("model.userFeatures " + model.userFeatures + "\n")
    println("model.userFeatures.count = " + model.userFeatures.count + "\n")
    //compute a predicted score for a given user and item combination
     val predictedRating = model.predict(789,123)
     println("predictedRating = " + predictedRating + "\n")

     //generate the top 10 recommended items for user 789
    val userId = 789
   val K = 10
    val topKRecs = model.recommendProducts(userId, K)
    println(topKRecs.mkString("\n"))

    //give these recommendations a sense check by taking a quick look at the titles of the movies a user has rated and the recommended movies
    //collect this data as a Map[Int, String] method mapping the movie ID to the title
    val movies = sc.textFile("/user/ml-100k/u.item")
    val titles = movies.map(line => line.split("\\|").take(2)).map(array => 
      (array(0).toInt, array(1))).collectAsMap()

    titles(123)

    /*For our user 789, we can find out what movies they have rated, 
     * take the 10 movies with the highest rating,
     *  and then check the titles.
     *   We will do this now by first using the keyBy Spark function
     *    to create an RDD of key-value pairs from our ratings RDD, 
     *   where the key will be the user ID.
     *    We will then use the lookup function to return 
     *    just the ratings for this key (that is, that particular user ID) to the driver
     * */

    val moviesForUser = ratings.keyBy(_.user).lookup(789)
    println(moviesForUser.size)

    /*
     * Next, we will take the 10 movies with the highest ratings 
     *by sorting the moviesForUser collection using the rating field of the Rating object. 
     * We will then extract the movie title for the relevant product ID 
     * attached to the Rating class from our mapping of movie titles 
     * and print out the top 10 titles with their ratings
     */
    moviesForUser.sortBy(-_.rating).take(10).map(rating =>
      (titles(rating.product),rating.rating)).foreach(println)

      /*Now, let’s take a look at the top 10 recommendations for this user and see what the titles are using the same approach as the one we used earlier 
       * (note that the recommendations are already sorted)
       * 
       */
    topKRecs.map(rating => (titles(rating.product), rating.rating)).foreach(println) 


    //Evaluating the performance of recommendation models
    //The Mean Squared Error (MSE) is a direct measure of the reconstruction error of the user-item rating matrix

    val actualRating = moviesForUser.take(1)(0)
    val predictedRating2 = model.predict(789, actualRating.product)
    //compute the squared error between the actual rating and the predicted rating
    val squaredError = math.pow(predictedRating2 - actualRating.rating, 2.0)

    /*
     * First, we will extract the user and product IDs from the ratings RDD
     *  and make predictions for each user-item pair  using model.predict. 
     * We will use the user-item pair as the key and the predicted rating as the value
     */

    val usersProducts = ratings.map{ case Rating(user, product, rating) => (user, product)}
    val predictions = model.predict(usersProducts).map{
          case Rating(user, product, rating) => ((user, product), rating)
         }


    val ratingsAndPredictions = ratings.map{
           case Rating(user, product, rating) => ((user, product), rating)
            }.join(predictions)

    // compute the MSE by summing up the squared errors using reduce and dividing by the count method of the number of records
    val MSE = ratingsAndPredictions.map{case ((user, product),
        (actual, predicted)) =>  math.pow((actual - predicted), 2)
         }.reduce(_ + _) / ratingsAndPredictions.count 
     println("Mean Squared Error = " + MSE)



    sc.stop()
   }
}

部署：

spark-submit  --name ExtFeatureRC --class test.ExtFeatureRC --executor-memory 256M /home/jackherrick/Documents/ExtFeatureRC.jar

日志：

18/05/06 18:49:43 INFO spark.SparkContext: Running Spark version 2.2.0
18/05/06 18:49:43 WARN util.NativeCodeLoader: Unable to load native-hadoop library for your platform... using builtin-java classes where applicable
18/05/06 18:49:44 INFO spark.SparkContext: Submitted application: ExtFeatureRC
18/05/06 18:49:44 INFO spark.SecurityManager: Changing view acls to: jackherrick
18/05/06 18:49:44 INFO spark.SecurityManager: Changing modify acls to: jackherrick
18/05/06 18:49:44 INFO spark.SecurityManager: Changing view acls groups to: 
18/05/06 18:49:44 INFO spark.SecurityManager: Changing modify acls groups to: 
18/05/06 18:49:44 INFO spark.SecurityManager: SecurityManager: authentication disabled; ui acls disabled; users  with view permissions: Set(jackherrick); groups with view permissions: Set(); users  with modify permissions: Set(jackherrick); groups with modify permissions: Set()
18/05/06 18:49:44 INFO util.Utils: Successfully started service 'sparkDriver' on port 37537.
18/05/06 18:49:45 INFO spark.SparkEnv: Registering MapOutputTracker
18/05/06 18:49:45 INFO spark.SparkEnv: Registering BlockManagerMaster
18/05/06 18:49:45 INFO storage.BlockManagerMasterEndpoint: Using org.apache.spark.storage.DefaultTopologyMapper for getting topology information
18/05/06 18:49:45 INFO storage.BlockManagerMasterEndpoint: BlockManagerMasterEndpoint up
18/05/06 18:49:45 INFO storage.DiskBlockManager: Created local directory at /tmp/blockmgr-cd48e204-9dea-4d38-af03-f46b136d6440
18/05/06 18:49:45 INFO memory.MemoryStore: MemoryStore started with capacity 366.3 MB
18/05/06 18:49:45 INFO spark.SparkEnv: Registering OutputCommitCoordinator
18/05/06 18:49:45 INFO util.log: Logging initialized @4220ms
18/05/06 18:49:45 INFO server.Server: jetty-9.3.z-SNAPSHOT
18/05/06 18:49:45 INFO server.Server: Started @4509ms
18/05/06 18:49:45 INFO server.AbstractConnector: Started ServerConnector@23c388c2{HTTP/1.1,[http/1.1]}{0.0.0.0:4040}
18/05/06 18:49:45 INFO util.Utils: Successfully started service 'SparkUI' on port 4040.
18/05/06 18:49:46 INFO handler.ContextHandler: Started o.s.j.s.ServletContextHandler@30b34287{/jobs,null,AVAILABLE,@Spark}
18/05/06 18:49:46 INFO handler.ContextHandler: Started o.s.j.s.ServletContextHandler@6bc28a83{/jobs/json,null,AVAILABLE,@Spark}
18/05/06 18:49:46 INFO handler.ContextHandler: Started o.s.j.s.ServletContextHandler@13579834{/jobs/job,null,AVAILABLE,@Spark}
18/05/06 18:49:46 INFO handler.ContextHandler: Started o.s.j.s.ServletContextHandler@aec50a1{/jobs/job/json,null,AVAILABLE,@Spark}
18/05/06 18:49:46 INFO handler.ContextHandler: Started o.s.j.s.ServletContextHandler@70d2e40b{/stages,null,AVAILABLE,@Spark}
18/05/06 18:49:46 INFO handler.ContextHandler: Started o.s.j.s.ServletContextHandler@7a0e1b5e{/stages/json,null,AVAILABLE,@Spark}
18/05/06 18:49:46 INFO handler.ContextHandler: Started o.s.j.s.ServletContextHandler@173b9122{/stages/stage,null,AVAILABLE,@Spark}
18/05/06 18:49:46 INFO handler.ContextHandler: Started o.s.j.s.ServletContextHandler@3b1bb3ab{/stages/stage/json,null,AVAILABLE,@Spark}
18/05/06 18:49:46 INFO handler.ContextHandler: Started o.s.j.s.ServletContextHandler@40bffbca{/stages/pool,null,AVAILABLE,@Spark}
18/05/06 18:49:46 INFO handler.ContextHandler: Started o.s.j.s.ServletContextHandler@42a9a63e{/stages/pool/json,null,AVAILABLE,@Spark}
18/05/06 18:49:46 INFO handler.ContextHandler: Started o.s.j.s.ServletContextHandler@5d8445d7{/storage,null,AVAILABLE,@Spark}
18/05/06 18:49:46 INFO handler.ContextHandler: Started o.s.j.s.ServletContextHandler@384fc774{/storage/json,null,AVAILABLE,@Spark}
18/05/06 18:49:46 INFO handler.ContextHandler: Started o.s.j.s.ServletContextHandler@71e9a896{/storage/rdd,null,AVAILABLE,@Spark}
18/05/06 18:49:46 INFO handler.ContextHandler: Started o.s.j.s.ServletContextHandler@408b35bf{/storage/rdd/json,null,AVAILABLE,@Spark}
18/05/06 18:49:46 INFO handler.ContextHandler: Started o.s.j.s.ServletContextHandler@15bcf458{/environment,null,AVAILABLE,@Spark}
18/05/06 18:49:46 INFO handler.ContextHandler: Started o.s.j.s.ServletContextHandler@43c67247{/environment/json,null,AVAILABLE,@Spark}
18/05/06 18:49:46 INFO handler.ContextHandler: Started o.s.j.s.ServletContextHandler@726386ed{/executors,null,AVAILABLE,@Spark}
18/05/06 18:49:46 INFO handler.ContextHandler: Started o.s.j.s.ServletContextHandler@14bb2297{/executors/json,null,AVAILABLE,@Spark}
18/05/06 18:49:46 INFO handler.ContextHandler: Started o.s.j.s.ServletContextHandler@797501a{/executors/threadDump,null,AVAILABLE,@Spark}
18/05/06 18:49:46 INFO handler.ContextHandler: Started o.s.j.s.ServletContextHandler@57f791c6{/executors/threadDump/json,null,AVAILABLE,@Spark}
18/05/06 18:49:46 INFO handler.ContextHandler: Started o.s.j.s.ServletContextHandler@6c4f9535{/static,null,AVAILABLE,@Spark}
18/05/06 18:49:46 INFO handler.ContextHandler: Started o.s.j.s.ServletContextHandler@200606de{/,null,AVAILABLE,@Spark}
18/05/06 18:49:46 INFO handler.ContextHandler: Started o.s.j.s.ServletContextHandler@f8908f6{/api,null,AVAILABLE,@Spark}
18/05/06 18:49:46 INFO handler.ContextHandler: Started o.s.j.s.ServletContextHandler@7544a1e4{/jobs/job/kill,null,AVAILABLE,@Spark}
18/05/06 18:49:46 INFO handler.ContextHandler: Started o.s.j.s.ServletContextHandler@7957dc72{/stages/stage/kill,null,AVAILABLE,@Spark}
18/05/06 18:49:46 INFO ui.SparkUI: Bound SparkUI to 0.0.0.0, and started at http://192.168.201.128:4040
18/05/06 18:49:46 INFO spark.SparkContext: Added JAR file:/home/jackherrick/Documents/ExtFeatureRC.jar at spark://192.168.201.128:37537/jars/ExtFeatureRC.jar with timestamp 1525657786364
18/05/06 18:49:46 INFO executor.Executor: Starting executor ID driver on host localhost
18/05/06 18:49:46 INFO util.Utils: Successfully started service 'org.apache.spark.network.netty.NettyBlockTransferService' on port 39601.
18/05/06 18:49:46 INFO netty.NettyBlockTransferService: Server created on 192.168.201.128:39601
18/05/06 18:49:46 INFO storage.BlockManager: Using org.apache.spark.storage.RandomBlockReplicationPolicy for block replication policy
18/05/06 18:49:46 INFO storage.BlockManagerMaster: Registering BlockManager BlockManagerId(driver, 192.168.201.128, 39601, None)
18/05/06 18:49:46 INFO storage.BlockManagerMasterEndpoint: Registering block manager 192.168.201.128:39601 with 366.3 MB RAM, BlockManagerId(driver, 192.168.201.128, 39601, None)
18/05/06 18:49:46 INFO storage.BlockManagerMaster: Registered BlockManager BlockManagerId(driver, 192.168.201.128, 39601, None)
18/05/06 18:49:46 INFO storage.BlockManager: Initialized BlockManager: BlockManagerId(driver, 192.168.201.128, 39601, None)
18/05/06 18:49:47 INFO handler.ContextHandler: Started o.s.j.s.ServletContextHandler@41477a6d{/metrics/json,null,AVAILABLE,@Spark}
18/05/06 18:49:48 INFO memory.MemoryStore: Block broadcast_0 stored as values in memory (estimated size 240.0 KB, free 366.1 MB)
18/05/06 18:49:49 INFO memory.MemoryStore: Block broadcast_0_piece0 stored as bytes in memory (estimated size 23.3 KB, free 366.0 MB)
18/05/06 18:49:49 INFO storage.BlockManagerInfo: Added broadcast_0_piece0 in memory on 192.168.201.128:39601 (size: 23.3 KB, free: 366.3 MB)
18/05/06 18:49:49 INFO spark.SparkContext: Created broadcast 0 from textFile at ExtFeatureRC.scala:21
18/05/06 18:49:50 INFO mapred.FileInputFormat: Total input paths to process : 1
18/05/06 18:49:50 INFO spark.SparkContext: Starting job: isEmpty at ALS.scala:240
18/05/06 18:49:50 INFO scheduler.DAGScheduler: Got job 0 (isEmpty at ALS.scala:240) with 1 output partitions
18/05/06 18:49:50 INFO scheduler.DAGScheduler: Final stage: ResultStage 0 (isEmpty at ALS.scala:240)
18/05/06 18:49:51 INFO scheduler.DAGScheduler: Parents of final stage: List()
18/05/06 18:49:51 INFO scheduler.DAGScheduler: Missing parents: List()
18/05/06 18:49:51 INFO scheduler.DAGScheduler: Submitting ResultStage 0 (MapPartitionsRDD[3] at map at ExtFeatureRC.scala:29), which has no missing parents
18/05/06 18:49:51 INFO memory.MemoryStore: Block broadcast_1 stored as values in memory (estimated size 3.6 KB, free 366.0 MB)
18/05/06 18:49:51 INFO memory.MemoryStore: Block broadcast_1_piece0 stored as bytes in memory (estimated size 2.1 KB, free 366.0 MB)
18/05/06 18:49:51 INFO storage.BlockManagerInfo: Added broadcast_1_piece0 in memory on 192.168.201.128:39601 (size: 2.1 KB, free: 366.3 MB)
18/05/06 18:49:51 INFO spark.SparkContext: Created broadcast 1 from broadcast at DAGScheduler.scala:1006
18/05/06 18:49:51 INFO scheduler.DAGScheduler: Submitting 1 missing tasks from ResultStage 0 (MapPartitionsRDD[3] at map at ExtFeatureRC.scala:29) (first 15 tasks are for partitions Vector(0))
18/05/06 18:49:51 INFO scheduler.TaskSchedulerImpl: Adding task set 0.0 with 1 tasks
18/05/06 18:49:51 INFO scheduler.TaskSetManager: Starting task 0.0 in stage 0.0 (TID 0, localhost, executor driver, partition 0, ANY, 4857 bytes)
18/05/06 18:49:51 INFO executor.Executor: Running task 0.0 in stage 0.0 (TID 0)
18/05/06 18:49:51 INFO executor.Executor: Fetching spark://192.168.201.128:37537/jars/ExtFeatureRC.jar with timestamp 1525657786364
18/05/06 18:49:51 INFO client.TransportClientFactory: Successfully created connection to /192.168.201.128:37537 after 48 ms (0 ms spent in bootstraps)
18/05/06 18:49:51 INFO util.Utils: Fetching spark://192.168.201.128:37537/jars/ExtFeatureRC.jar to /tmp/spark-3e2c979f-9940-4333-9c41-313105080560/userFiles-75a66de3-fd8d-45a3-b152-8a361ef592c5/fetchFileTemp7523857094345728563.tmp
18/05/06 18:49:51 INFO executor.Executor: Adding file:/tmp/spark-3e2c979f-9940-4333-9c41-313105080560/userFiles-75a66de3-fd8d-45a3-b152-8a361ef592c5/ExtFeatureRC.jar to class loader
18/05/06 18:49:52 INFO rdd.HadoopRDD: Input split: hdfs://master:9000/user/ml-100k/u.data:0+989586
18/05/06 18:49:52 INFO executor.Executor: Finished task 0.0 in stage 0.0 (TID 0). 968 bytes result sent to driver
18/05/06 18:49:52 INFO scheduler.TaskSetManager: Finished task 0.0 in stage 0.0 (TID 0) in 1426 ms on localhost (executor driver) (1/1)
18/05/06 18:49:52 INFO scheduler.TaskSchedulerImpl: Removed TaskSet 0.0, whose tasks have all completed, from pool 
18/05/06 18:49:52 INFO scheduler.DAGScheduler: ResultStage 0 (isEmpty at ALS.scala:240) finished in 1.516 s
18/05/06 18:49:52 INFO scheduler.DAGScheduler: Job 0 finished: isEmpty at ALS.scala:240, took 2.001484 s
18/05/06 18:49:53 INFO spark.SparkContext: Starting job: isEmpty at ALS.scala:843
18/05/06 18:49:53 INFO scheduler.DAGScheduler: Got job 1 (isEmpty at ALS.scala:843) with 1 output partitions
18/05/06 18:49:53 INFO scheduler.DAGScheduler: Final stage: ResultStage 1 (isEmpty at ALS.scala:843)
18/05/06 18:49:53 INFO scheduler.DAGScheduler: Parents of final stage: List()
18/05/06 18:49:53 INFO scheduler.DAGScheduler: Missing parents: List()
18/05/06 18:49:53 INFO scheduler.DAGScheduler: Submitting ResultStage 1 (MapPartitionsRDD[4] at map at ALS.scala:256), which has no missing parents
18/05/06 18:49:53 INFO memory.MemoryStore: Block broadcast_2 stored as values in memory (estimated size 3.8 KB, free 366.0 MB)
18/05/06 18:49:53 INFO memory.MemoryStore: Block broadcast_2_piece0 stored as bytes in memory (estimated size 2.2 KB, free 366.0 MB)
18/05/06 18:49:53 INFO storage.BlockManagerInfo: Added broadcast_2_piece0 in memory on 192.168.201.128:39601 (size: 2.2 KB, free: 366.3 MB)
18/05/06 18:49:53 INFO spark.SparkContext: Created broadcast 2 from broadcast at DAGScheduler.scala:1006
18/05/06 18:49:53 INFO scheduler.DAGScheduler: Submitting 1 missing tasks from ResultStage 1 (MapPartitionsRDD[4] at map at ALS.scala:256) (first 15 tasks are for partitions Vector(0))
18/05/06 18:49:53 INFO scheduler.TaskSchedulerImpl: Adding task set 1.0 with 1 tasks
18/05/06 18:49:53 INFO scheduler.TaskSetManager: Starting task 0.0 in stage 1.0 (TID 1, localhost, executor driver, partition 0, ANY, 4857 bytes)
18/05/06 18:49:53 INFO executor.Executor: Running task 0.0 in stage 1.0 (TID 1)
18/05/06 18:49:53 INFO rdd.HadoopRDD: Input split: hdfs://master:9000/user/ml-100k/u.data:0+989586
18/05/06 18:49:53 INFO executor.Executor: Finished task 0.0 in stage 1.0 (TID 1). 1089 bytes result sent to driver
18/05/06 18:49:53 INFO scheduler.TaskSetManager: Finished task 0.0 in stage 1.0 (TID 1) in 106 ms on localhost (executor driver) (1/1)
18/05/06 18:49:53 INFO scheduler.DAGScheduler: ResultStage 1 (isEmpty at ALS.scala:843) finished in 0.102 s
18/05/06 18:49:53 INFO scheduler.DAGScheduler: Job 1 finished: isEmpty at ALS.scala:843, took 0.197033 s
18/05/06 18:49:53 INFO scheduler.TaskSchedulerImpl: Removed TaskSet 1.0, whose tasks have all completed, from pool 
18/05/06 18:49:53 INFO spark.SparkContext: Starting job: count at ALS.scala:857
18/05/06 18:49:53 INFO scheduler.DAGScheduler: Registering RDD 5 (mapPartitions at ALS.scala:1101)
18/05/06 18:49:53 INFO scheduler.DAGScheduler: Registering RDD 8 (map at ALS.scala:1344)
18/05/06 18:49:53 INFO scheduler.DAGScheduler: Got job 2 (count at ALS.scala:857) with 2 output partitions
18/05/06 18:49:53 INFO scheduler.DAGScheduler: Final stage: ResultStage 4 (count at ALS.scala:857)
18/05/06 18:49:53 INFO scheduler.DAGScheduler: Parents of final stage: List(ShuffleMapStage 3)
18/05/06 18:49:53 INFO scheduler.DAGScheduler: Missing parents: List(ShuffleMapStage 3)
18/05/06 18:49:53 INFO scheduler.DAGScheduler: Submitting ShuffleMapStage 2 (MapPartitionsRDD[5] at mapPartitions at ALS.scala:1101), which has no missing parents
18/05/06 18:49:53 INFO memory.MemoryStore: Block broadcast_3 stored as values in memory (estimated size 6.0 KB, free 366.0 MB)
18/05/06 18:49:53 INFO memory.MemoryStore: Block broadcast_3_piece0 stored as bytes in memory (estimated size 3.2 KB, free 366.0 MB)
18/05/06 18:49:53 INFO storage.BlockManagerInfo: Added broadcast_3_piece0 in memory on 192.168.201.128:39601 (size: 3.2 KB, free: 366.3 MB)
18/05/06 18:49:53 INFO spark.SparkContext: Created broadcast 3 from broadcast at DAGScheduler.scala:1006
18/05/06 18:49:53 INFO scheduler.DAGScheduler: Submitting 2 missing tasks from ShuffleMapStage 2 (MapPartitionsRDD[5] at mapPartitions at ALS.scala:1101) (first 15 tasks are for partitions Vector(0, 1))
18/05/06 18:49:53 INFO scheduler.TaskSchedulerImpl: Adding task set 2.0 with 2 tasks
18/05/06 18:49:53 INFO scheduler.TaskSetManager: Starting task 0.0 in stage 2.0 (TID 2, localhost, executor driver, partition 0, ANY, 4846 bytes)
18/05/06 18:49:53 INFO scheduler.TaskSetManager: Starting task 1.0 in stage 2.0 (TID 3, localhost, executor driver, partition 1, ANY, 4846 bytes)
18/05/06 18:49:53 INFO executor.Executor: Running task 0.0 in stage 2.0 (TID 2)
18/05/06 18:49:53 INFO executor.Executor: Running task 1.0 in stage 2.0 (TID 3)
18/05/06 18:49:53 INFO rdd.HadoopRDD: Input split: hdfs://master:9000/user/ml-100k/u.data:0+989586
18/05/06 18:49:53 INFO rdd.HadoopRDD: Input split: hdfs://master:9000/user/ml-100k/u.data:989586+989587
18/05/06 18:49:54 INFO storage.BlockManagerInfo: Removed broadcast_1_piece0 on 192.168.201.128:39601 in memory (size: 2.1 KB, free: 366.3 MB)
18/05/06 18:49:55 INFO storage.BlockManagerInfo: Removed broadcast_2_piece0 on 192.168.201.128:39601 in memory (size: 2.2 KB, free: 366.3 MB)
18/05/06 18:49:55 INFO executor.Executor: Finished task 0.0 in stage 2.0 (TID 2). 1071 bytes result sent to driver
18/05/06 18:49:55 INFO executor.Executor: Finished task 1.0 in stage 2.0 (TID 3). 1028 bytes result sent to driver
18/05/06 18:49:55 INFO scheduler.TaskSetManager: Finished task 0.0 in stage 2.0 (TID 2) in 2082 ms on localhost (executor driver) (1/2)
18/05/06 18:49:55 INFO scheduler.TaskSetManager: Finished task 1.0 in stage 2.0 (TID 3) in 2095 ms on localhost (executor driver) (2/2)
18/05/06 18:49:55 INFO scheduler.DAGScheduler: ShuffleMapStage 2 (mapPartitions at ALS.scala:1101) finished in 2.106 s
18/05/06 18:49:55 INFO scheduler.DAGScheduler: looking for newly runnable stages
18/05/06 18:49:55 INFO scheduler.DAGScheduler: running: Set()
18/05/06 18:49:55 INFO scheduler.DAGScheduler: waiting: Set(ShuffleMapStage 3, ResultStage 4)
18/05/06 18:49:55 INFO scheduler.DAGScheduler: failed: Set()
18/05/06 18:49:55 INFO scheduler.TaskSchedulerImpl: Removed TaskSet 2.0, whose tasks have all completed, from pool 
18/05/06 18:49:55 INFO scheduler.DAGScheduler: Submitting ShuffleMapStage 3 (MapPartitionsRDD[8] at map at ALS.scala:1344), which has no missing parents
18/05/06 18:49:55 INFO memory.MemoryStore: Block broadcast_4 stored as values in memory (estimated size 7.3 KB, free 366.0 MB)
18/05/06 18:49:55 INFO memory.MemoryStore: Block broadcast_4_piece0 stored as bytes in memory (estimated size 3.7 KB, free 366.0 MB)
18/05/06 18:49:55 INFO storage.BlockManagerInfo: Added broadcast_4_piece0 in memory on 192.168.201.128:39601 (size: 3.7 KB, free: 366.3 MB)
18/05/06 18:49:55 INFO spark.SparkContext: Created broadcast 4 from broadcast at DAGScheduler.scala:1006
18/05/06 18:49:55 INFO scheduler.DAGScheduler: Submitting 2 missing tasks from ShuffleMapStage 3 (MapPartitionsRDD[8] at map at ALS.scala:1344) (first 15 tasks are for partitions Vector(0, 1))
18/05/06 18:49:55 INFO scheduler.TaskSchedulerImpl: Adding task set 3.0 with 2 tasks
18/05/06 18:49:55 INFO scheduler.TaskSetManager: Starting task 0.0 in stage 3.0 (TID 4, localhost, executor driver, partition 0, ANY, 4610 bytes)
18/05/06 18:49:55 INFO scheduler.TaskSetManager: Starting task 1.0 in stage 3.0 (TID 5, localhost, executor driver, partition 1, ANY, 4610 bytes)
18/05/06 18:49:55 INFO executor.Executor: Running task 0.0 in stage 3.0 (TID 4)
18/05/06 18:49:55 INFO executor.Executor: Running task 1.0 in stage 3.0 (TID 5)
18/05/06 18:49:55 INFO storage.ShuffleBlockFetcherIterator: Getting 2 non-empty blocks out of 2 blocks
18/05/06 18:49:55 INFO storage.ShuffleBlockFetcherIterator: Started 0 remote fetches in 16 ms
18/05/06 18:49:55 INFO storage.ShuffleBlockFetcherIterator: Getting 2 non-empty blocks out of 2 blocks
18/05/06 18:49:55 INFO storage.ShuffleBlockFetcherIterator: Started 0 remote fetches in 16 ms
18/05/06 18:49:56 INFO memory.MemoryStore: Block rdd_7_1 stored as values in memory (estimated size 586.0 KB, free 365.5 MB)
18/05/06 18:49:56 INFO storage.BlockManagerInfo: Added rdd_7_1 in memory on 192.168.201.128:39601 (size: 586.0 KB, free: 365.7 MB)
18/05/06 18:49:56 INFO memory.MemoryStore: Block rdd_7_0 stored as values in memory (estimated size 586.4 KB, free 364.9 MB)
18/05/06 18:49:56 INFO storage.BlockManagerInfo: Added rdd_7_0 in memory on 192.168.201.128:39601 (size: 586.4 KB, free: 365.1 MB)
18/05/06 18:49:56 INFO executor.Executor: Finished task 0.0 in stage 3.0 (TID 4). 2026 bytes result sent to driver
18/05/06 18:49:56 INFO executor.Executor: Finished task 1.0 in stage 3.0 (TID 5). 2026 bytes result sent to driver
18/05/06 18:49:56 INFO scheduler.TaskSetManager: Finished task 1.0 in stage 3.0 (TID 5) in 504 ms on localhost (executor driver) (1/2)
18/05/06 18:49:56 INFO scheduler.TaskSetManager: Finished task 0.0 in stage 3.0 (TID 4) in 509 ms on localhost (executor driver) (2/2)
18/05/06 18:49:56 INFO scheduler.TaskSchedulerImpl: Removed TaskSet 3.0, whose tasks have all completed, from pool 
18/05/06 18:49:56 INFO scheduler.DAGScheduler: ShuffleMapStage 3 (map at ALS.scala:1344) finished in 0.512 s
18/05/06 18:49:56 INFO scheduler.DAGScheduler: looking for newly runnable stages
18/05/06 18:49:56 INFO scheduler.DAGScheduler: running: Set()
18/05/06 18:49:56 INFO scheduler.DAGScheduler: waiting: Set(ResultStage 4)
18/05/06 18:49:56 INFO scheduler.DAGScheduler: failed: Set()
18/05/06 18:49:56 INFO scheduler.DAGScheduler: Submitting ResultStage 4 (userOutBlocks MapPartitionsRDD[11] at mapValues at ALS.scala:1381), which has no missing parents
18/05/06 18:49:56 INFO memory.MemoryStore: Block broadcast_5 stored as values in memory (estimated size 7.9 KB, free 364.9 MB)
18/05/06 18:49:56 INFO memory.MemoryStore: Block broadcast_5_piece0 stored as bytes in memory (estimated size 3.9 KB, free 364.9 MB)
18/05/06 18:49:56 INFO storage.BlockManagerInfo: Added broadcast_5_piece0 in memory on 192.168.201.128:39601 (size: 3.9 KB, free: 365.1 MB)
18/05/06 18:49:56 INFO spark.SparkContext: Created broadcast 5 from broadcast at DAGScheduler.scala:1006
18/05/06 18:49:56 INFO scheduler.DAGScheduler: Submitting 2 missing tasks from ResultStage 4 (userOutBlocks MapPartitionsRDD[11] at mapValues at ALS.scala:1381) (first 15 tasks are for partitions Vector(0, 1))
18/05/06 18:49:56 INFO scheduler.TaskSchedulerImpl: Adding task set 4.0 with 2 tasks
18/05/06 18:49:56 INFO scheduler.TaskSetManager: Starting task 0.0 in stage 4.0 (TID 6, localhost, executor driver, partition 0, ANY, 4621 bytes)
18/05/06 18:49:56 INFO scheduler.TaskSetManager: Starting task 1.0 in stage 4.0 (TID 7, localhost, executor driver, partition 1, ANY, 4621 bytes)
18/05/06 18:49:56 INFO executor.Executor: Running task 0.0 in stage 4.0 (TID 6)
18/05/06 18:49:56 INFO executor.Executor: Running task 1.0 in stage 4.0 (TID 7)
18/05/06 18:49:56 INFO storage.ShuffleBlockFetcherIterator: Getting 2 non-empty blocks out of 2 blocks
18/05/06 18:49:56 INFO storage.ShuffleBlockFetcherIterator: Started 0 remote fetches in 1 ms
18/05/06 18:49:56 INFO storage.ShuffleBlockFetcherIterator: Getting 2 non-empty blocks out of 2 blocks
18/05/06 18:49:56 INFO storage.ShuffleBlockFetcherIterator: Started 0 remote fetches in 1 ms
18/05/06 18:49:56 INFO memory.MemoryStore: Block rdd_10_1 stored as values in memory (estimated size 394.7 KB, free 364.5 MB)
18/05/06 18:49:56 INFO memory.MemoryStore: Block rdd_10_0 stored as values in memory (estimated size 394.3 KB, free 364.1 MB)
18/05/06 18:49:56 INFO storage.BlockManagerInfo: Added rdd_10_1 in memory on 192.168.201.128:39601 (size: 394.7 KB, free: 364.7 MB)
18/05/06 18:49:56 INFO memory.MemoryStore: Block rdd_11_1 stored as values in memory (estimated size 3.8 KB, free 364.1 MB)
18/05/06 18:49:56 INFO storage.BlockManagerInfo: Added rdd_10_0 in memory on 192.168.201.128:39601 (size: 394.3 KB, free: 364.4 MB)
18/05/06 18:49:56 INFO storage.BlockManagerInfo: Added rdd_11_1 in memory on 192.168.201.128:39601 (size: 3.8 KB, free: 364.3 MB)
18/05/06 18:49:56 INFO executor.Executor: Finished task 1.0 in stage 4.0 (TID 7). 1920 bytes result sent to driver
18/05/06 18:49:56 INFO memory.MemoryStore: Block rdd_11_0 stored as values in memory (estimated size 3.8 KB, free 364.1 MB)
18/05/06 18:49:56 INFO storage.BlockManagerInfo: Added rdd_11_0 in memory on 192.168.201.128:39601 (size: 3.8 KB, free: 364.3 MB)
18/05/06 18:49:56 INFO executor.Executor: Finished task 0.0 in stage 4.0 (TID 6). 1877 bytes result sent to driver
18/05/06 18:49:56 INFO scheduler.TaskSetManager: Finished task 1.0 in stage 4.0 (TID 7) in 536 ms on localhost (executor driver) (1/2)
18/05/06 18:49:56 INFO scheduler.TaskSetManager: Finished task 0.0 in stage 4.0 (TID 6) in 539 ms on localhost (executor driver) (2/2)
18/05/06 18:49:56 INFO scheduler.TaskSchedulerImpl: Removed TaskSet 4.0, whose tasks have all completed, from pool 
18/05/06 18:49:56 INFO scheduler.DAGScheduler: ResultStage 4 (count at ALS.scala:857) finished in 0.509 s
18/05/06 18:49:57 INFO scheduler.DAGScheduler: Job 2 finished: count at ALS.scala:857, took 3.411162 s
18/05/06 18:49:57 INFO spark.SparkContext: Starting job: count at ALS.scala:865
18/05/06 18:49:57 INFO spark.MapOutputTrackerMaster: Size of output statuses for shuffle 1 is 160 bytes
18/05/06 18:49:57 INFO scheduler.DAGScheduler: Registering RDD 13 (map at ALS.scala:1344)
18/05/06 18:49:57 INFO scheduler.DAGScheduler: Got job 3 (count at ALS.scala:865) with 2 output partitions
18/05/06 18:49:57 INFO scheduler.DAGScheduler: Final stage: ResultStage 7 (count at ALS.scala:865)
18/05/06 18:49:57 INFO scheduler.DAGScheduler: Parents of final stage: List(ShuffleMapStage 6)
18/05/06 18:49:57 INFO scheduler.DAGScheduler: Missing parents: List(ShuffleMapStage 6)
18/05/06 18:49:57 INFO scheduler.DAGScheduler: Submitting ShuffleMapStage 6 (MapPartitionsRDD[13] at map at ALS.scala:1344), which has no missing parents
18/05/06 18:49:57 INFO memory.MemoryStore: Block broadcast_6 stored as values in memory (estimated size 7.5 KB, free 364.1 MB)
18/05/06 18:49:57 INFO memory.MemoryStore: Block broadcast_6_piece0 stored as bytes in memory (estimated size 3.8 KB, free 364.1 MB)
18/05/06 18:49:57 INFO storage.BlockManagerInfo: Added broadcast_6_piece0 in memory on 192.168.201.128:39601 (size: 3.8 KB, free: 364.3 MB)
18/05/06 18:49:57 INFO spark.SparkContext: Created broadcast 6 from broadcast at DAGScheduler.scala:1006
18/05/06 18:49:57 INFO scheduler.DAGScheduler: Submitting 2 missing tasks from ShuffleMapStage 6 (MapPartitionsRDD[13] at map at ALS.scala:1344) (first 15 tasks are for partitions Vector(0, 1))
18/05/06 18:49:57 INFO scheduler.TaskSchedulerImpl: Adding task set 6.0 with 2 tasks
18/05/06 18:49:57 INFO scheduler.TaskSetManager: Starting task 0.0 in stage 6.0 (TID 8, localhost, executor driver, partition 0, PROCESS_LOCAL, 4610 bytes)
18/05/06 18:49:57 INFO scheduler.TaskSetManager: Starting task 1.0 in stage 6.0 (TID 9, localhost, executor driver, partition 1, PROCESS_LOCAL, 4610 bytes)
18/05/06 18:49:57 INFO executor.Executor: Running task 0.0 in stage 6.0 (TID 8)
18/05/06 18:49:57 INFO executor.Executor: Running task 1.0 in stage 6.0 (TID 9)
18/05/06 18:49:57 INFO storage.BlockManager: Found block rdd_7_1 locally
18/05/06 18:49:57 INFO storage.BlockManager: Found block rdd_7_0 locally
18/05/06 18:49:57 INFO executor.Executor: Finished task 1.0 in stage 6.0 (TID 9). 985 bytes result sent to driver
18/05/06 18:49:57 INFO scheduler.TaskSetManager: Finished task 1.0 in stage 6.0 (TID 9) in 122 ms on localhost (executor driver) (1/2)
18/05/06 18:49:57 INFO storage.BlockManagerInfo: Removed broadcast_4_piece0 on 192.168.201.128:39601 in memory (size: 3.7 KB, free: 364.3 MB)
18/05/06 18:49:57 INFO storage.BlockManagerInfo: Removed broadcast_5_piece0 on 192.168.201.128:39601 in memory (size: 3.9 KB, free: 364.3 MB)
18/05/06 18:49:57 INFO executor.Executor: Finished task 0.0 in stage 6.0 (TID 8). 1028 bytes result sent to driver
18/05/06 18:49:57 INFO scheduler.TaskSetManager: Finished task 0.0 in stage 6.0 (TID 8) in 223 ms on localhost (executor driver) (2/2)
18/05/06 18:49:57 INFO scheduler.TaskSchedulerImpl: Removed TaskSet 6.0, whose tasks have all completed, from pool 
18/05/06 18:49:57 INFO scheduler.DAGScheduler: ShuffleMapStage 6 (map at ALS.scala:1344) finished in 0.223 s
18/05/06 18:49:57 INFO scheduler.DAGScheduler: looking for newly runnable stages
18/05/06 18:49:57 INFO scheduler.DAGScheduler: running: Set()
18/05/06 18:49:57 INFO scheduler.DAGScheduler: waiting: Set(ResultStage 7)
18/05/06 18:49:57 INFO scheduler.DAGScheduler: failed: Set()
18/05/06 18:49:57 INFO scheduler.DAGScheduler: Submitting ResultStage 7 (itemOutBlocks MapPartitionsRDD[16] at mapValues at ALS.scala:1381), which has no missing parents
18/05/06 18:49:57 INFO memory.MemoryStore: Block broadcast_7 stored as values in memory (estimated size 8.1 KB, free 364.1 MB)
18/05/06 18:49:57 INFO memory.MemoryStore: Block broadcast_7_piece0 stored as bytes in memory (estimated size 3.9 KB, free 364.1 MB)
18/05/06 18:49:57 INFO storage.BlockManagerInfo: Added broadcast_7_piece0 in memory on 192.168.201.128:39601 (size: 3.9 KB, free: 364.3 MB)
18/05/06 18:49:57 INFO spark.SparkContext: Created broadcast 7 from broadcast at DAGScheduler.scala:1006
18/05/06 18:49:57 INFO scheduler.DAGScheduler: Submitting 2 missing tasks from ResultStage 7 (itemOutBlocks MapPartitionsRDD[16] at mapValues at ALS.scala:1381) (first 15 tasks are for partitions Vector(0, 1))
18/05/06 18:49:57 INFO scheduler.TaskSchedulerImpl: Adding task set 7.0 with 2 tasks
18/05/06 18:49:57 INFO scheduler.TaskSetManager: Starting task 0.0 in stage 7.0 (TID 10, localhost, executor driver, partition 0, ANY, 4621 bytes)
18/05/06 18:49:57 INFO scheduler.TaskSetManager: Starting task 1.0 in stage 7.0 (TID 11, localhost, executor driver, partition 1, ANY, 4621 bytes)
18/05/06 18:49:57 INFO executor.Executor: Running task 0.0 in stage 7.0 (TID 10)
18/05/06 18:49:57 INFO executor.Executor: Running task 1.0 in stage 7.0 (TID 11)
18/05/06 18:49:57 INFO storage.ShuffleBlockFetcherIterator: Getting 2 non-empty blocks out of 2 blocks
18/05/06 18:49:57 INFO storage.ShuffleBlockFetcherIterator: Started 0 remote fetches in 1 ms
18/05/06 18:49:57 INFO storage.ShuffleBlockFetcherIterator: Getting 2 non-empty blocks out of 2 blocks
18/05/06 18:49:57 INFO storage.ShuffleBlockFetcherIterator: Started 0 remote fetches in 6 ms
18/05/06 18:49:57 INFO memory.MemoryStore: Block rdd_15_1 stored as values in memory (estimated size 398.8 KB, free 363.7 MB)
18/05/06 18:49:57 INFO storage.BlockManagerInfo: Added rdd_15_1 in memory on 192.168.201.128:39601 (size: 398.8 KB, free: 364.0 MB)
18/05/06 18:49:57 INFO memory.MemoryStore: Block rdd_16_1 stored as values in memory (estimated size 6.3 KB, free 363.7 MB)
18/05/06 18:49:57 INFO storage.BlockManagerInfo: Added rdd_16_1 in memory on 192.168.201.128:39601 (size: 6.3 KB, free: 363.9 MB)
18/05/06 18:49:57 INFO executor.Executor: Finished task 1.0 in stage 7.0 (TID 11). 1877 bytes result sent to driver
18/05/06 18:49:57 INFO scheduler.TaskSetManager: Finished task 1.0 in stage 7.0 (TID 11) in 198 ms on localhost (executor driver) (1/2)
18/05/06 18:49:57 INFO memory.MemoryStore: Block rdd_15_0 stored as values in memory (estimated size 395.9 KB, free 363.3 MB)
18/05/06 18:49:57 INFO storage.BlockManagerInfo: Added rdd_15_0 in memory on 192.168.201.128:39601 (size: 395.9 KB, free: 363.6 MB)
18/05/06 18:49:57 INFO memory.MemoryStore: Block rdd_16_0 stored as values in memory (estimated size 6.3 KB, free 363.3 MB)
18/05/06 18:49:57 INFO storage.BlockManagerInfo: Added rdd_16_0 in memory on 192.168.201.128:39601 (size: 6.3 KB, free: 363.6 MB)
18/05/06 18:49:57 INFO executor.Executor: Finished task 0.0 in stage 7.0 (TID 10). 1877 bytes result sent to driver
18/05/06 18:49:57 INFO scheduler.TaskSetManager: Finished task 0.0 in stage 7.0 (TID 10) in 321 ms on localhost (executor driver) (2/2)
18/05/06 18:49:57 INFO scheduler.DAGScheduler: ResultStage 7 (count at ALS.scala:865) finished in 0.325 s
18/05/06 18:49:57 INFO scheduler.DAGScheduler: Job 3 finished: count at ALS.scala:865, took 0.666716 s
18/05/06 18:49:57 INFO scheduler.TaskSchedulerImpl: Removed TaskSet 7.0, whose tasks have all completed, from pool 
18/05/06 18:49:58 INFO spark.SparkContext: Starting job: count at ALS.scala:279
18/05/06 18:49:58 INFO spark.MapOutputTrackerMaster: Size of output statuses for shuffle 1 is 160 bytes
18/05/06 18:49:58 INFO spark.MapOutputTrackerMaster: Size of output statuses for shuffle 0 is 160 bytes
18/05/06 18:49:58 INFO scheduler.DAGScheduler: Registering RDD 17 (map at ALS.scala:1017)
18/05/06 18:49:58 INFO scheduler.DAGScheduler: Registering RDD 22 (flatMap at ALS.scala:1433)
18/05/06 18:49:58 INFO spark.MapOutputTrackerMaster: Size of output statuses for shuffle 2 is 160 bytes
18/05/06 18:49:58 INFO scheduler.DAGScheduler: Registering RDD 31 (flatMap at ALS.scala:1433)
18/05/06 18:49:58 INFO scheduler.DAGScheduler: Registering RDD 40 (flatMap at ALS.scala:1433)
18/05/06 18:49:58 INFO scheduler.DAGScheduler: Registering RDD 49 (flatMap at ALS.scala:1433)
18/05/06 18:49:58 INFO scheduler.DAGScheduler: Registering RDD 58 (flatMap at ALS.scala:1433)
18/05/06 18:49:58 INFO scheduler.DAGScheduler: Registering RDD 67 (flatMap at ALS.scala:1433)
18/05/06 18:49:58 INFO scheduler.DAGScheduler: Registering RDD 76 (flatMap at ALS.scala:1433)
18/05/06 18:49:58 INFO scheduler.DAGScheduler: Registering RDD 85 (flatMap at ALS.scala:1433)
18/05/06 18:49:58 INFO scheduler.DAGScheduler: Registering RDD 94 (flatMap at ALS.scala:1433)
18/05/06 18:49:58 INFO scheduler.DAGScheduler: Registering RDD 103 (flatMap at ALS.scala:1433)
18/05/06 18:49:58 INFO scheduler.DAGScheduler: Got job 4 (count at ALS.scala:279) with 2 output partitions
18/05/06 18:49:58 INFO scheduler.DAGScheduler: Final stage: ResultStage 22 (count at ALS.scala:279)
18/05/06 18:49:58 INFO scheduler.DAGScheduler: Parents of final stage: List(ShuffleMapStage 9, ShuffleMapStage 21)
18/05/06 18:49:58 INFO scheduler.DAGScheduler: Missing parents: List(ShuffleMapStage 21)
18/05/06 18:49:58 INFO scheduler.DAGScheduler: Submitting ShuffleMapStage 10 (MapPartitionsRDD[17] at map at ALS.scala:1017), which has no missing parents
18/05/06 18:49:58 INFO memory.MemoryStore: Block broadcast_8 stored as values in memory (estimated size 7.9 KB, free 363.3 MB)
18/05/06 18:49:58 INFO memory.MemoryStore: Block broadcast_8_piece0 stored as bytes in memory (estimated size 3.9 KB, free 363.3 MB)
18/05/06 18:49:58 INFO storage.BlockManagerInfo: Added broadcast_8_piece0 in memory on 192.168.201.128:39601 (size: 3.9 KB, free: 363.6 MB)
18/05/06 18:49:58 INFO spark.SparkContext: Created broadcast 8 from broadcast at DAGScheduler.scala:1006
18/05/06 18:49:58 INFO scheduler.DAGScheduler: Submitting 2 missing tasks from ShuffleMapStage 10 (MapPartitionsRDD[17] at map at ALS.scala:1017) (first 15 tasks are for partitions Vector(0, 1))
18/05/06 18:49:58 INFO scheduler.TaskSchedulerImpl: Adding task set 10.0 with 2 tasks
18/05/06 18:49:58 INFO scheduler.TaskSetManager: Starting task 0.0 in stage 10.0 (TID 12, localhost, executor driver, partition 0, PROCESS_LOCAL, 4610 bytes)
18/05/06 18:49:58 INFO scheduler.TaskSetManager: Starting task 1.0 in stage 10.0 (TID 13, localhost, executor driver, partition 1, PROCESS_LOCAL, 4610 bytes)
18/05/06 18:49:58 INFO executor.Executor: Running task 0.0 in stage 10.0 (TID 12)
18/05/06 18:49:58 INFO executor.Executor: Running task 1.0 in stage 10.0 (TID 13)
18/05/06 18:49:58 INFO storage.BlockManager: Found block rdd_10_1 locally
18/05/06 18:49:58 INFO storage.BlockManager: Found block rdd_10_0 locally
18/05/06 18:49:58 WARN netlib.BLAS: Failed to load implementation from: com.github.fommil.netlib.NativeSystemBLAS
18/05/06 18:49:58 WARN netlib.BLAS: Failed to load implementation from: com.github.fommil.netlib.NativeRefBLAS
18/05/06 18:49:58 INFO executor.Executor: Finished task 0.0 in stage 10.0 (TID 12). 985 bytes result sent to driver
18/05/06 18:49:58 INFO scheduler.TaskSetManager: Finished task 0.0 in stage 10.0 (TID 12) in 88 ms on localhost (executor driver) (1/2)
18/05/06 18:49:58 INFO executor.Executor: Finished task 1.0 in stage 10.0 (TID 13). 985 bytes result sent to driver
18/05/06 18:49:58 INFO scheduler.TaskSetManager: Finished task 1.0 in stage 10.0 (TID 13) in 108 ms on localhost (executor driver) (2/2)
18/05/06 18:49:58 INFO scheduler.TaskSchedulerImpl: Removed TaskSet 10.0, whose tasks have all completed, from pool 
18/05/06 18:49:58 INFO scheduler.DAGScheduler: ShuffleMapStage 10 (map at ALS.scala:1017) finished in 0.086 s
18/05/06 18:49:58 INFO scheduler.DAGScheduler: looking for newly runnable stages
18/05/06 18:49:58 INFO scheduler.DAGScheduler: running: Set()
18/05/06 18:49:58 INFO scheduler.DAGScheduler: waiting: Set(ShuffleMapStage 15, ShuffleMapStage 19, ShuffleMapStage 16, ShuffleMapStage 13, ShuffleMapStage 20, ShuffleMapStage 17, ShuffleMapStage 21, ShuffleMapStage 18, ResultStage 22, ShuffleMapStage 14, ShuffleMapStage 11)
18/05/06 18:49:58 INFO scheduler.DAGScheduler: failed: Set()
18/05/06 18:49:58 INFO scheduler.DAGScheduler: Submitting ShuffleMapStage 11 (MapPartitionsRDD[22] at flatMap at ALS.scala:1433), which has no missing parents
18/05/06 18:49:58 INFO memory.MemoryStore: Block broadcast_9 stored as values in memory (estimated size 9.0 KB, free 363.3 MB)
18/05/06 18:49:58 INFO memory.MemoryStore: Block broadcast_9_piece0 stored as bytes in memory (estimated size 4.2 KB, free 363.3 MB)
18/05/06 18:49:58 INFO storage.BlockManagerInfo: Added broadcast_9_piece0 in memory on 192.168.201.128:39601 (size: 4.2 KB, free: 363.5 MB)
18/05/06 18:49:58 INFO spark.SparkContext: Created broadcast 9 from broadcast at DAGScheduler.scala:1006
18/05/06 18:49:58 INFO scheduler.DAGScheduler: Submitting 2 missing tasks from ShuffleMapStage 11 (MapPartitionsRDD[22] at flatMap at ALS.scala:1433) (first 15 tasks are for partitions Vector(0, 1))
18/05/06 18:49:58 INFO scheduler.TaskSchedulerImpl: Adding task set 11.0 with 2 tasks
18/05/06 18:49:58 INFO scheduler.TaskSetManager: Starting task 0.0 in stage 11.0 (TID 14, localhost, executor driver, partition 0, PROCESS_LOCAL, 4827 bytes)
18/05/06 18:49:58 INFO scheduler.TaskSetManager: Starting task 1.0 in stage 11.0 (TID 15, localhost, executor driver, partition 1, PROCESS_LOCAL, 4827 bytes)
18/05/06 18:49:58 INFO executor.Executor: Running task 0.0 in stage 11.0 (TID 14)
18/05/06 18:49:58 INFO executor.Executor: Running task 1.0 in stage 11.0 (TID 15)
18/05/06 18:49:58 INFO storage.BlockManager: Found block rdd_11_0 locally
18/05/06 18:49:58 INFO storage.ShuffleBlockFetcherIterator: Getting 1 non-empty blocks out of 2 blocks
18/05/06 18:49:58 INFO storage.ShuffleBlockFetcherIterator: Started 0 remote fetches in 0 ms
18/05/06 18:49:58 INFO storage.BlockManager: Found block rdd_11_1 locally
18/05/06 18:49:58 INFO storage.ShuffleBlockFetcherIterator: Getting 1 non-empty blocks out of 2 blocks
18/05/06 18:49:58 INFO storage.ShuffleBlockFetcherIterator: Started 0 remote fetches in 1 ms
18/05/06 18:49:58 INFO executor.Executor: Finished task 0.0 in stage 11.0 (TID 14). 1372 bytes result sent to driver
18/05/06 18:49:58 INFO scheduler.TaskSetManager: Finished task 0.0 in stage 11.0 (TID 14) in 129 ms on localhost (executor driver) (1/2)
18/05/06 18:49:58 INFO executor.Executor: Finished task 1.0 in stage 11.0 (TID 15). 1415 bytes result sent to driver
18/05/06 18:49:58 INFO scheduler.TaskSetManager: Finished task 1.0 in stage 11.0 (TID 15) in 157 ms on localhost (executor driver) (2/2)
18/05/06 18:49:58 INFO scheduler.TaskSchedulerImpl: Removed TaskSet 11.0, whose tasks have all completed, from pool 
18/05/06 18:49:58 INFO scheduler.DAGScheduler: ShuffleMapStage 11 (flatMap at ALS.scala:1433) finished in 0.160 s
18/05/06 18:49:58 INFO scheduler.DAGScheduler: looking for newly runnable stages
18/05/06 18:49:58 INFO scheduler.DAGScheduler: running: Set()
18/05/06 18:49:58 INFO scheduler.DAGScheduler: waiting: Set(ShuffleMapStage 15, ShuffleMapStage 19, ShuffleMapStage 16, ShuffleMapStage 13, ShuffleMapStage 20, ShuffleMapStage 17, ShuffleMapStage 21, ShuffleMapStage 18, ResultStage 22, ShuffleMapStage 14)
18/05/06 18:49:58 INFO scheduler.DAGScheduler: failed: Set()
18/05/06 18:49:58 INFO scheduler.DAGScheduler: Submitting ShuffleMapStage 13 (MapPartitionsRDD[31] at flatMap at ALS.scala:1433), which has no missing parents
18/05/06 18:49:58 INFO memory.MemoryStore: Block broadcast_10 stored as values in memory (estimated size 11.2 KB, free 363.3 MB)
18/05/06 18:49:58 INFO storage.BlockManagerInfo: Removed broadcast_8_piece0 on 192.168.201.128:39601 in memory (size: 3.9 KB, free: 363.6 MB)
18/05/06 18:49:58 INFO storage.BlockManagerInfo: Removed broadcast_6_piece0 on 192.168.201.128:39601 in memory (size: 3.8 KB, free: 363.6 MB)
18/05/06 18:49:58 INFO storage.BlockManagerInfo: Removed broadcast_7_piece0 on 192.168.201.128:39601 in memory (size: 3.9 KB, free: 363.6 MB)
18/05/06 18:49:58 INFO memory.MemoryStore: Block broadcast_10_piece0 stored as bytes in memory (estimated size 5.1 KB, free 363.3 MB)
18/05/06 18:49:58 INFO storage.BlockManagerInfo: Added broadcast_10_piece0 in memory on 192.168.201.128:39601 (size: 5.1 KB, free: 363.6 MB)
18/05/06 18:49:58 INFO spark.SparkContext: Created broadcast 10 from broadcast at DAGScheduler.scala:1006
18/05/06 18:49:58 INFO scheduler.DAGScheduler: Submitting 2 missing tasks from ShuffleMapStage 13 (MapPartitionsRDD[31] at flatMap at ALS.scala:1433) (first 15 tasks are for partitions Vector(0, 1))
18/05/06 18:49:59 INFO scheduler.TaskSchedulerImpl: Adding task set 13.0 with 2 tasks
18/05/06 18:49:59 INFO scheduler.TaskSetManager: Starting task 0.0 in stage 13.0 (TID 16, localhost, executor driver, partition 0, PROCESS_LOCAL, 4900 bytes)
18/05/06 18:49:59 INFO scheduler.TaskSetManager: Starting task 1.0 in stage 13.0 (TID 17, localhost, executor driver, partition 1, PROCESS_LOCAL, 4900 bytes)
18/05/06 18:49:59 INFO executor.Executor: Running task 0.0 in stage 13.0 (TID 16)
18/05/06 18:49:59 INFO executor.Executor: Running task 1.0 in stage 13.0 (TID 17)
18/05/06 18:49:59 INFO storage.BlockManager: Found block rdd_16_0 locally
18/05/06 18:49:59 INFO storage.BlockManager: Found block rdd_15_0 locally
18/05/06 18:49:59 INFO storage.ShuffleBlockFetcherIterator: Getting 2 non-empty blocks out of 2 blocks
18/05/06 18:49:59 INFO storage.ShuffleBlockFetcherIterator: Started 0 remote fetches in 0 ms
18/05/06 18:49:59 INFO storage.BlockManager: Found block rdd_16_1 locally
18/05/06 18:49:59 INFO storage.BlockManager: Found block rdd_15_1 locally
18/05/06 18:49:59 INFO storage.ShuffleBlockFetcherIterator: Getting 2 non-empty blocks out of 2 blocks
18/05/06 18:49:59 INFO storage.ShuffleBlockFetcherIterator: Started 0 remote fetches in 9 ms
18/05/06 18:49:59 WARN netlib.LAPACK: Failed to load implementation from: com.github.fommil.netlib.NativeSystemLAPACK
18/05/06 18:49:59 WARN netlib.LAPACK: Failed to load implementation from: com.github.fommil.netlib.NativeRefLAPACK
18/05/06 18:49:59 INFO executor.Executor: Finished task 0.0 in stage 13.0 (TID 16). 1372 bytes result sent to driver
18/05/06 18:49:59 INFO scheduler.TaskSetManager: Finished task 0.0 in stage 13.0 (TID 16) in 408 ms on localhost (executor driver) (1/2)
18/05/06 18:49:59 INFO executor.Executor: Finished task 1.0 in stage 13.0 (TID 17). 1372 bytes result sent to driver
18/05/06 18:49:59 INFO scheduler.TaskSetManager: Finished task 1.0 in stage 13.0 (TID 17) in 434 ms on localhost (executor driver) (2/2)
18/05/06 18:49:59 INFO scheduler.TaskSchedulerImpl: Removed TaskSet 13.0, whose tasks have all completed, from pool 
18/05/06 18:49:59 INFO scheduler.DAGScheduler: ShuffleMapStage 13 (flatMap at ALS.scala:1433) finished in 0.434 s
18/05/06 18:49:59 INFO scheduler.DAGScheduler: looking for newly runnable stages
18/05/06 18:49:59 INFO scheduler.DAGScheduler: running: Set()
18/05/06 18:49:59 INFO scheduler.DAGScheduler: waiting: Set(ShuffleMapStage 15, ShuffleMapStage 19, ShuffleMapStage 16, ShuffleMapStage 20, ShuffleMapStage 17, ShuffleMapStage 21, ShuffleMapStage 18, ResultStage 22, ShuffleMapStage 14)
18/05/06 18:49:59 INFO scheduler.DAGScheduler: failed: Set()
18/05/06 18:49:59 INFO scheduler.DAGScheduler: Submitting ShuffleMapStage 14 (MapPartitionsRDD[40] at flatMap at ALS.scala:1433), which has no missing parents
18/05/06 18:49:59 INFO memory.MemoryStore: Block broadcast_11 stored as values in memory (estimated size 12.2 KB, free 363.3 MB)
18/05/06 18:49:59 INFO memory.MemoryStore: Block broadcast_11_piece0 stored as bytes in memory (estimated size 5.4 KB, free 363.3 MB)
18/05/06 18:49:59 INFO storage.BlockManagerInfo: Added broadcast_11_piece0 in memory on 192.168.201.128:39601 (size: 5.4 KB, free: 363.5 MB)
18/05/06 18:49:59 INFO spark.SparkContext: Created broadcast 11 from broadcast at DAGScheduler.scala:1006
18/05/06 18:49:59 INFO scheduler.DAGScheduler: Submitting 2 missing tasks from ShuffleMapStage 14 (MapPartitionsRDD[40] at flatMap at ALS.scala:1433) (first 15 tasks are for partitions Vector(0, 1))
18/05/06 18:49:59 INFO scheduler.TaskSchedulerImpl: Adding task set 14.0 with 2 tasks
18/05/06 18:49:59 INFO scheduler.TaskSetManager: Starting task 0.0 in stage 14.0 (TID 18, localhost, executor driver, partition 0, PROCESS_LOCAL, 4900 bytes)
18/05/06 18:49:59 INFO scheduler.TaskSetManager: Starting task 1.0 in stage 14.0 (TID 19, localhost, executor driver, partition 1, PROCESS_LOCAL, 4900 bytes)
18/05/06 18:49:59 INFO executor.Executor: Running task 0.0 in stage 14.0 (TID 18)
18/05/06 18:49:59 INFO executor.Executor: Running task 1.0 in stage 14.0 (TID 19)
18/05/06 18:49:59 INFO storage.BlockManager: Found block rdd_11_0 locally
18/05/06 18:49:59 INFO storage.BlockManager: Found block rdd_10_0 locally
18/05/06 18:49:59 INFO storage.BlockManager: Found block rdd_11_1 locally
18/05/06 18:49:59 INFO storage.ShuffleBlockFetcherIterator: Getting 2 non-empty blocks out of 2 blocks
18/05/06 18:49:59 INFO storage.ShuffleBlockFetcherIterator: Started 0 remote fetches in 0 ms
18/05/06 18:49:59 INFO storage.BlockManager: Found block rdd_10_1 locally
18/05/06 18:49:59 INFO storage.ShuffleBlockFetcherIterator: Getting 2 non-empty blocks out of 2 blocks
18/05/06 18:49:59 INFO storage.ShuffleBlockFetcherIterator: Started 0 remote fetches in 3 ms
18/05/06 18:49:59 INFO executor.Executor: Finished task 1.0 in stage 14.0 (TID 19). 1372 bytes result sent to driver
18/05/06 18:49:59 INFO scheduler.TaskSetManager: Finished task 1.0 in stage 14.0 (TID 19) in 159 ms on localhost (executor driver) (1/2)
18/05/06 18:49:59 INFO executor.Executor: Finished task 0.0 in stage 14.0 (TID 18). 1372 bytes result sent to driver
18/05/06 18:49:59 INFO scheduler.TaskSetManager: Finished task 0.0 in stage 14.0 (TID 18) in 228 ms on localhost (executor driver) (2/2)
18/05/06 18:49:59 INFO scheduler.TaskSchedulerImpl: Removed TaskSet 14.0, whose tasks have all completed, from pool 
18/05/06 18:49:59 INFO scheduler.DAGScheduler: ShuffleMapStage 14 (flatMap at ALS.scala:1433) finished in 0.210 s
18/05/06 18:49:59 INFO scheduler.DAGScheduler: looking for newly runnable stages
18/05/06 18:49:59 INFO scheduler.DAGScheduler: running: Set()
18/05/06 18:49:59 INFO scheduler.DAGScheduler: waiting: Set(ShuffleMapStage 15, ShuffleMapStage 19, ShuffleMapStage 16, ShuffleMapStage 20, ShuffleMapStage 17, ShuffleMapStage 21, ShuffleMapStage 18, ResultStage 22)
18/05/06 18:49:59 INFO scheduler.DAGScheduler: failed: Set()
18/05/06 18:49:59 INFO scheduler.DAGScheduler: Submitting ShuffleMapStage 15 (MapPartitionsRDD[49] at flatMap at ALS.scala:1433), which has no missing parents
18/05/06 18:49:59 INFO memory.MemoryStore: Block broadcast_12 stored as values in memory (estimated size 13.1 KB, free 363.3 MB)
18/05/06 18:49:59 INFO memory.MemoryStore: Block broadcast_12_piece0 stored as bytes in memory (estimated size 5.6 KB, free 363.3 MB)
18/05/06 18:49:59 INFO storage.BlockManagerInfo: Added broadcast_12_piece0 in memory on 192.168.201.128:39601 (size: 5.6 KB, free: 363.5 MB)
18/05/06 18:49:59 INFO spark.SparkContext: Created broadcast 12 from broadcast at DAGScheduler.scala:1006
18/05/06 18:49:59 INFO scheduler.DAGScheduler: Submitting 2 missing tasks from ShuffleMapStage 15 (MapPartitionsRDD[49] at flatMap at ALS.scala:1433) (first 15 tasks are for partitions Vector(0, 1))
18/05/06 18:49:59 INFO scheduler.TaskSchedulerImpl: Adding task set 15.0 with 2 tasks
18/05/06 18:49:59 INFO scheduler.TaskSetManager: Starting task 0.0 in stage 15.0 (TID 20, localhost, executor driver, partition 0, PROCESS_LOCAL, 4900 bytes)
18/05/06 18:49:59 INFO scheduler.TaskSetManager: Starting task 1.0 in stage 15.0 (TID 21, localhost, executor driver, partition 1, PROCESS_LOCAL, 4900 bytes)
18/05/06 18:49:59 INFO executor.Executor: Running task 0.0 in stage 15.0 (TID 20)
18/05/06 18:49:59 INFO executor.Executor: Running task 1.0 in stage 15.0 (TID 21)
18/05/06 18:49:59 INFO storage.BlockManager: Found block rdd_16_1 locally
18/05/06 18:49:59 INFO storage.BlockManager: Found block rdd_15_1 locally
18/05/06 18:49:59 INFO storage.ShuffleBlockFetcherIterator: Getting 2 non-empty blocks out of 2 blocks
18/05/06 18:49:59 INFO storage.ShuffleBlockFetcherIterator: Started 0 remote fetches in 0 ms
18/05/06 18:49:59 INFO storage.BlockManager: Found block rdd_16_0 locally
18/05/06 18:49:59 INFO storage.BlockManager: Found block rdd_15_0 locally
18/05/06 18:49:59 INFO storage.ShuffleBlockFetcherIterator: Getting 2 non-empty blocks out of 2 blocks
18/05/06 18:49:59 INFO storage.ShuffleBlockFetcherIterator: Started 0 remote fetches in 1 ms
18/05/06 18:49:59 INFO executor.Executor: Finished task 0.0 in stage 15.0 (TID 20). 1415 bytes result sent to driver
18/05/06 18:49:59 INFO scheduler.TaskSetManager: Finished task 0.0 in stage 15.0 (TID 20) in 203 ms on localhost (executor driver) (1/2)
18/05/06 18:50:00 INFO executor.Executor: Finished task 1.0 in stage 15.0 (TID 21). 1372 bytes result sent to driver
18/05/06 18:50:00 INFO scheduler.TaskSetManager: Finished task 1.0 in stage 15.0 (TID 21) in 241 ms on localhost (executor driver) (2/2)
18/05/06 18:50:00 INFO scheduler.TaskSchedulerImpl: Removed TaskSet 15.0, whose tasks have all completed, from pool 
18/05/06 18:50:00 INFO scheduler.DAGScheduler: ShuffleMapStage 15 (flatMap at ALS.scala:1433) finished in 0.246 s
18/05/06 18:50:00 INFO scheduler.DAGScheduler: looking for newly runnable stages
18/05/06 18:50:00 INFO scheduler.DAGScheduler: running: Set()
18/05/06 18:50:00 INFO scheduler.DAGScheduler: waiting: Set(ShuffleMapStage 19, ShuffleMapStage 16, ShuffleMapStage 20, ShuffleMapStage 17, ShuffleMapStage 21, ShuffleMapStage 18, ResultStage 22)
18/05/06 18:50:00 INFO scheduler.DAGScheduler: failed: Set()
18/05/06 18:50:00 INFO scheduler.DAGScheduler: Submitting ShuffleMapStage 16 (MapPartitionsRDD[58] at flatMap at ALS.scala:1433), which has no missing parents
18/05/06 18:50:00 INFO memory.MemoryStore: Block broadcast_13 stored as values in memory (estimated size 14.0 KB, free 363.2 MB)
18/05/06 18:50:00 INFO memory.MemoryStore: Block broadcast_13_piece0 stored as bytes in memory (estimated size 5.9 KB, free 363.2 MB)
18/05/06 18:50:00 INFO storage.BlockManagerInfo: Added broadcast_13_piece0 in memory on 192.168.201.128:39601 (size: 5.9 KB, free: 363.5 MB)
18/05/06 18:50:00 INFO spark.SparkContext: Created broadcast 13 from broadcast at DAGScheduler.scala:1006
18/05/06 18:50:00 INFO scheduler.DAGScheduler: Submitting 2 missing tasks from ShuffleMapStage 16 (MapPartitionsRDD[58] at flatMap at ALS.scala:1433) (first 15 tasks are for partitions Vector(0, 1))
18/05/06 18:50:00 INFO scheduler.TaskSchedulerImpl: Adding task set 16.0 with 2 tasks
18/05/06 18:50:00 INFO scheduler.TaskSetManager: Starting task 0.0 in stage 16.0 (TID 22, localhost, executor driver, partition 0, PROCESS_LOCAL, 4900 bytes)
18/05/06 18:50:00 INFO scheduler.TaskSetManager: Starting task 1.0 in stage 16.0 (TID 23, localhost, executor driver, partition 1, PROCESS_LOCAL, 4900 bytes)
18/05/06 18:50:00 INFO executor.Executor: Running task 0.0 in stage 16.0 (TID 22)
18/05/06 18:50:00 INFO executor.Executor: Running task 1.0 in stage 16.0 (TID 23)
18/05/06 18:50:00 INFO storage.BlockManager: Found block rdd_11_1 locally
18/05/06 18:50:00 INFO storage.BlockManager: Found block rdd_10_1 locally
18/05/06 18:50:00 INFO storage.ShuffleBlockFetcherIterator: Getting 2 non-empty blocks out of 2 blocks
18/05/06 18:50:00 INFO storage.ShuffleBlockFetcherIterator: Started 0 remote fetches in 1 ms
18/05/06 18:50:00 INFO storage.BlockManager: Found block rdd_11_0 locally
18/05/06 18:50:00 INFO storage.BlockManager: Found block rdd_10_0 locally
18/05/06 18:50:00 INFO storage.ShuffleBlockFetcherIterator: Getting 2 non-empty blocks out of 2 blocks
18/05/06 18:50:00 INFO storage.ShuffleBlockFetcherIterator: Started 0 remote fetches in 17 ms
18/05/06 18:50:00 INFO executor.Executor: Finished task 1.0 in stage 16.0 (TID 23). 1372 bytes result sent to driver
18/05/06 18:50:00 INFO scheduler.TaskSetManager: Finished task 1.0 in stage 16.0 (TID 23) in 205 ms on localhost (executor driver) (1/2)
18/05/06 18:50:00 INFO executor.Executor: Finished task 0.0 in stage 16.0 (TID 22). 1372 bytes result sent to driver
18/05/06 18:50:00 INFO scheduler.TaskSetManager: Finished task 0.0 in stage 16.0 (TID 22) in 277 ms on localhost (executor driver) (2/2)
18/05/06 18:50:00 INFO scheduler.TaskSchedulerImpl: Removed TaskSet 16.0, whose tasks have all completed, from pool 
18/05/06 18:50:00 INFO scheduler.DAGScheduler: ShuffleMapStage 16 (flatMap at ALS.scala:1433) finished in 0.250 s
18/05/06 18:50:00 INFO scheduler.DAGScheduler: looking for newly runnable stages
18/05/06 18:50:00 INFO scheduler.DAGScheduler: running: Set()
18/05/06 18:50:00 INFO scheduler.DAGScheduler: waiting: Set(ShuffleMapStage 19, ShuffleMapStage 20, ShuffleMapStage 17, ShuffleMapStage 21, ShuffleMapStage 18, ResultStage 22)
18/05/06 18:50:00 INFO scheduler.DAGScheduler: failed: Set()
18/05/06 18:50:00 INFO scheduler.DAGScheduler: Submitting ShuffleMapStage 17 (MapPartitionsRDD[67] at flatMap at ALS.scala:1433), which has no missing parents
18/05/06 18:50:00 INFO memory.MemoryStore: Block broadcast_14 stored as values in memory (estimated size 14.9 KB, free 363.2 MB)
18/05/06 18:50:00 INFO memory.MemoryStore: Block broadcast_14_piece0 stored as bytes in memory (estimated size 6.0 KB, free 363.2 MB)
18/05/06 18:50:00 INFO storage.BlockManagerInfo: Added broadcast_14_piece0 in memory on 192.168.201.128:39601 (size: 6.0 KB, free: 363.5 MB)
18/05/06 18:50:00 INFO spark.SparkContext: Created broadcast 14 from broadcast at DAGScheduler.scala:1006
18/05/06 18:50:00 INFO scheduler.DAGScheduler: Submitting 2 missing tasks from ShuffleMapStage 17 (MapPartitionsRDD[67] at flatMap at ALS.scala:1433) (first 15 tasks are for partitions Vector(0, 1))
18/05/06 18:50:00 INFO scheduler.TaskSchedulerImpl: Adding task set 17.0 with 2 tasks
18/05/06 18:50:00 INFO scheduler.TaskSetManager: Starting task 0.0 in stage 17.0 (TID 24, localhost, executor driver, partition 0, PROCESS_LOCAL, 4900 bytes)
18/05/06 18:50:00 INFO scheduler.TaskSetManager: Starting task 1.0 in stage 17.0 (TID 25, localhost, executor driver, partition 1, PROCESS_LOCAL, 4900 bytes)
18/05/06 18:50:00 INFO executor.Executor: Running task 0.0 in stage 17.0 (TID 24)
18/05/06 18:50:00 INFO executor.Executor: Running task 1.0 in stage 17.0 (TID 25)
18/05/06 18:50:00 INFO storage.BlockManager: Found block rdd_16_0 locally
18/05/06 18:50:00 INFO storage.BlockManager: Found block rdd_15_0 locally
18/05/06 18:50:00 INFO storage.ShuffleBlockFetcherIterator: Getting 2 non-empty blocks out of 2 blocks
18/05/06 18:50:00 INFO storage.ShuffleBlockFetcherIterator: Started 0 remote fetches in 0 ms
18/05/06 18:50:00 INFO storage.BlockManager: Found block rdd_16_1 locally
18/05/06 18:50:00 INFO storage.BlockManager: Found block rdd_15_1 locally
18/05/06 18:50:00 INFO storage.ShuffleBlockFetcherIterator: Getting 2 non-empty blocks out of 2 blocks
18/05/06 18:50:00 INFO storage.ShuffleBlockFetcherIterator: Started 0 remote fetches in 3 ms
18/05/06 18:50:00 INFO executor.Executor: Finished task 1.0 in stage 17.0 (TID 25). 1415 bytes result sent to driver
18/05/06 18:50:00 INFO scheduler.TaskSetManager: Finished task 1.0 in stage 17.0 (TID 25) in 173 ms on localhost (executor driver) (1/2)
18/05/06 18:50:00 INFO executor.Executor: Finished task 0.0 in stage 17.0 (TID 24). 1372 bytes result sent to driver
18/05/06 18:50:00 INFO scheduler.TaskSetManager: Finished task 0.0 in stage 17.0 (TID 24) in 257 ms on localhost (executor driver) (2/2)
18/05/06 18:50:00 INFO scheduler.TaskSchedulerImpl: Removed TaskSet 17.0, whose tasks have all completed, from pool 
18/05/06 18:50:00 INFO scheduler.DAGScheduler: ShuffleMapStage 17 (flatMap at ALS.scala:1433) finished in 0.240 s
18/05/06 18:50:00 INFO scheduler.DAGScheduler: looking for newly runnable stages
18/05/06 18:50:00 INFO scheduler.DAGScheduler: running: Set()
18/05/06 18:50:00 INFO scheduler.DAGScheduler: waiting: Set(ShuffleMapStage 19, ShuffleMapStage 20, ShuffleMapStage 21, ShuffleMapStage 18, ResultStage 22)
18/05/06 18:50:00 INFO scheduler.DAGScheduler: failed: Set()
18/05/06 18:50:00 INFO scheduler.DAGScheduler: Submitting ShuffleMapStage 18 (MapPartitionsRDD[76] at flatMap at ALS.scala:1433), which has no missing parents
18/05/06 18:50:00 INFO memory.MemoryStore: Block broadcast_15 stored as values in memory (estimated size 15.8 KB, free 363.2 MB)
18/05/06 18:50:00 INFO memory.MemoryStore: Block broadcast_15_piece0 stored as bytes in memory (estimated size 6.3 KB, free 363.2 MB)
18/05/06 18:50:00 INFO storage.BlockManagerInfo: Added broadcast_15_piece0 in memory on 192.168.201.128:39601 (size: 6.3 KB, free: 363.5 MB)
18/05/06 18:50:00 INFO spark.SparkContext: Created broadcast 15 from broadcast at DAGScheduler.scala:1006
18/05/06 18:50:00 INFO scheduler.DAGScheduler: Submitting 2 missing tasks from ShuffleMapStage 18 (MapPartitionsRDD[76] at flatMap at ALS.scala:1433) (first 15 tasks are for partitions Vector(0, 1))
18/05/06 18:50:00 INFO scheduler.TaskSchedulerImpl: Adding task set 18.0 with 2 tasks
18/05/06 18:50:00 INFO scheduler.TaskSetManager: Starting task 0.0 in stage 18.0 (TID 26, localhost, executor driver, partition 0, PROCESS_LOCAL, 4900 bytes)
18/05/06 18:50:00 INFO scheduler.TaskSetManager: Starting task 1.0 in stage 18.0 (TID 27, localhost, executor driver, partition 1, PROCESS_LOCAL, 4900 bytes)
18/05/06 18:50:00 INFO executor.Executor: Running task 0.0 in stage 18.0 (TID 26)
18/05/06 18:50:00 INFO executor.Executor: Running task 1.0 in stage 18.0 (TID 27)
18/05/06 18:50:00 INFO storage.BlockManager: Found block rdd_11_0 locally
18/05/06 18:50:00 INFO storage.BlockManager: Found block rdd_10_0 locally
18/05/06 18:50:00 INFO storage.ShuffleBlockFetcherIterator: Getting 2 non-empty blocks out of 2 blocks
18/05/06 18:50:00 INFO storage.ShuffleBlockFetcherIterator: Started 0 remote fetches in 3 ms
18/05/06 18:50:00 INFO storage.BlockManager: Found block rdd_11_1 locally
18/05/06 18:50:00 INFO storage.BlockManager: Found block rdd_10_1 locally
18/05/06 18:50:00 INFO storage.ShuffleBlockFetcherIterator: Getting 2 non-empty blocks out of 2 blocks
18/05/06 18:50:00 INFO storage.ShuffleBlockFetcherIterator: Started 0 remote fetches in 11 ms
18/05/06 18:50:00 INFO executor.Executor: Finished task 0.0 in stage 18.0 (TID 26). 1372 bytes result sent to driver
18/05/06 18:50:00 INFO scheduler.TaskSetManager: Finished task 0.0 in stage 18.0 (TID 26) in 225 ms on localhost (executor driver) (1/2)
18/05/06 18:50:01 INFO executor.Executor: Finished task 1.0 in stage 18.0 (TID 27). 1372 bytes result sent to driver
18/05/06 18:50:01 INFO scheduler.TaskSetManager: Finished task 1.0 in stage 18.0 (TID 27) in 347 ms on localhost (executor driver) (2/2)
18/05/06 18:50:01 INFO scheduler.TaskSchedulerImpl: Removed TaskSet 18.0, whose tasks have all completed, from pool 
18/05/06 18:50:01 INFO scheduler.DAGScheduler: ShuffleMapStage 18 (flatMap at ALS.scala:1433) finished in 0.332 s
18/05/06 18:50:01 INFO scheduler.DAGScheduler: looking for newly runnable stages
18/05/06 18:50:01 INFO scheduler.DAGScheduler: running: Set()
18/05/06 18:50:01 INFO scheduler.DAGScheduler: waiting: Set(ShuffleMapStage 19, ShuffleMapStage 20, ShuffleMapStage 21, ResultStage 22)
18/05/06 18:50:01 INFO scheduler.DAGScheduler: failed: Set()
18/05/06 18:50:01 INFO scheduler.DAGScheduler: Submitting ShuffleMapStage 19 (MapPartitionsRDD[85] at flatMap at ALS.scala:1433), which has no missing parents
18/05/06 18:50:01 INFO memory.MemoryStore: Block broadcast_16 stored as values in memory (estimated size 16.7 KB, free 363.2 MB)
18/05/06 18:50:01 INFO memory.MemoryStore: Block broadcast_16_piece0 stored as bytes in memory (estimated size 6.4 KB, free 363.2 MB)
18/05/06 18:50:01 INFO storage.BlockManagerInfo: Added broadcast_16_piece0 in memory on 192.168.201.128:39601 (size: 6.4 KB, free: 363.5 MB)
18/05/06 18:50:01 INFO spark.SparkContext: Created broadcast 16 from broadcast at DAGScheduler.scala:1006
18/05/06 18:50:01 INFO scheduler.DAGScheduler: Submitting 2 missing tasks from ShuffleMapStage 19 (MapPartitionsRDD[85] at flatMap at ALS.scala:1433) (first 15 tasks are for partitions Vector(0, 1))
18/05/06 18:50:01 INFO scheduler.TaskSchedulerImpl: Adding task set 19.0 with 2 tasks
18/05/06 18:50:01 INFO scheduler.TaskSetManager: Starting task 0.0 in stage 19.0 (TID 28, localhost, executor driver, partition 0, PROCESS_LOCAL, 4900 bytes)
18/05/06 18:50:01 INFO scheduler.TaskSetManager: Starting task 1.0 in stage 19.0 (TID 29, localhost, executor driver, partition 1, PROCESS_LOCAL, 4900 bytes)
18/05/06 18:50:01 INFO executor.Executor: Running task 0.0 in stage 19.0 (TID 28)
18/05/06 18:50:01 INFO executor.Executor: Running task 1.0 in stage 19.0 (TID 29)
18/05/06 18:50:01 INFO storage.BlockManager: Found block rdd_16_1 locally
18/05/06 18:50:01 INFO storage.BlockManager: Found block rdd_15_1 locally
18/05/06 18:50:01 INFO storage.ShuffleBlockFetcherIterator: Getting 2 non-empty blocks out of 2 blocks
18/05/06 18:50:01 INFO storage.ShuffleBlockFetcherIterator: Started 0 remote fetches in 0 ms
18/05/06 18:50:01 INFO storage.BlockManager: Found block rdd_16_0 locally
18/05/06 18:50:01 INFO storage.BlockManager: Found block rdd_15_0 locally
18/05/06 18:50:01 INFO storage.ShuffleBlockFetcherIterator: Getting 2 non-empty blocks out of 2 blocks
18/05/06 18:50:01 INFO storage.ShuffleBlockFetcherIterator: Started 0 remote fetches in 6 ms
18/05/06 18:50:01 INFO executor.Executor: Finished task 0.0 in stage 19.0 (TID 28). 1372 bytes result sent to driver
18/05/06 18:50:01 INFO scheduler.TaskSetManager: Finished task 0.0 in stage 19.0 (TID 28) in 229 ms on localhost (executor driver) (1/2)
18/05/06 18:50:01 INFO executor.Executor: Finished task 1.0 in stage 19.0 (TID 29). 1415 bytes result sent to driver
18/05/06 18:50:01 INFO scheduler.TaskSetManager: Finished task 1.0 in stage 19.0 (TID 29) in 290 ms on localhost (executor driver) (2/2)
18/05/06 18:50:01 INFO scheduler.TaskSchedulerImpl: Removed TaskSet 19.0, whose tasks have all completed, from pool 
18/05/06 18:50:01 INFO scheduler.DAGScheduler: ShuffleMapStage 19 (flatMap at ALS.scala:1433) finished in 0.274 s
18/05/06 18:50:01 INFO scheduler.DAGScheduler: looking for newly runnable stages
18/05/06 18:50:01 INFO scheduler.DAGScheduler: running: Set()
18/05/06 18:50:01 INFO scheduler.DAGScheduler: waiting: Set(ShuffleMapStage 20, ShuffleMapStage 21, ResultStage 22)
18/05/06 18:50:01 INFO scheduler.DAGScheduler: failed: Set()
18/05/06 18:50:01 INFO scheduler.DAGScheduler: Submitting ShuffleMapStage 20 (MapPartitionsRDD[94] at flatMap at ALS.scala:1433), which has no missing parents
18/05/06 18:50:01 INFO memory.MemoryStore: Block broadcast_17 stored as values in memory (estimated size 17.7 KB, free 363.2 MB)
18/05/06 18:50:01 INFO memory.MemoryStore: Block broadcast_17_piece0 stored as bytes in memory (estimated size 6.7 KB, free 363.2 MB)
18/05/06 18:50:01 INFO storage.BlockManagerInfo: Added broadcast_17_piece0 in memory on 192.168.201.128:39601 (size: 6.7 KB, free: 363.5 MB)
18/05/06 18:50:01 INFO spark.SparkContext: Created broadcast 17 from broadcast at DAGScheduler.scala:1006
18/05/06 18:50:01 INFO scheduler.DAGScheduler: Submitting 2 missing tasks from ShuffleMapStage 20 (MapPartitionsRDD[94] at flatMap at ALS.scala:1433) (first 15 tasks are for partitions Vector(0, 1))
18/05/06 18:50:01 INFO scheduler.TaskSchedulerImpl: Adding task set 20.0 with 2 tasks
18/05/06 18:50:01 INFO scheduler.TaskSetManager: Starting task 0.0 in stage 20.0 (TID 30, localhost, executor driver, partition 0, PROCESS_LOCAL, 4900 bytes)
18/05/06 18:50:01 INFO scheduler.TaskSetManager: Starting task 1.0 in stage 20.0 (TID 31, localhost, executor driver, partition 1, PROCESS_LOCAL, 4900 bytes)
18/05/06 18:50:01 INFO executor.Executor: Running task 1.0 in stage 20.0 (TID 31)
18/05/06 18:50:01 INFO executor.Executor: Running task 0.0 in stage 20.0 (TID 30)
18/05/06 18:50:01 INFO storage.BlockManager: Found block rdd_11_1 locally
18/05/06 18:50:01 INFO storage.BlockManager: Found block rdd_10_1 locally
18/05/06 18:50:01 INFO storage.ShuffleBlockFetcherIterator: Getting 2 non-empty blocks out of 2 blocks
18/05/06 18:50:01 INFO storage.ShuffleBlockFetcherIterator: Started 0 remote fetches in 0 ms
18/05/06 18:50:01 INFO storage.BlockManager: Found block rdd_11_0 locally
18/05/06 18:50:01 INFO storage.BlockManager: Found block rdd_10_0 locally
18/05/06 18:50:01 INFO storage.ShuffleBlockFetcherIterator: Getting 2 non-empty blocks out of 2 blocks
18/05/06 18:50:01 INFO storage.ShuffleBlockFetcherIterator: Started 0 remote fetches in 12 ms
18/05/06 18:50:01 INFO executor.Executor: Finished task 1.0 in stage 20.0 (TID 31). 1372 bytes result sent to driver
18/05/06 18:50:01 INFO scheduler.TaskSetManager: Finished task 1.0 in stage 20.0 (TID 31) in 247 ms on localhost (executor driver) (1/2)
18/05/06 18:50:01 INFO executor.Executor: Finished task 0.0 in stage 20.0 (TID 30). 1372 bytes result sent to driver
18/05/06 18:50:01 INFO scheduler.TaskSetManager: Finished task 0.0 in stage 20.0 (TID 30) in 263 ms on localhost (executor driver) (2/2)
18/05/06 18:50:01 INFO scheduler.TaskSchedulerImpl: Removed TaskSet 20.0, whose tasks have all completed, from pool 
18/05/06 18:50:01 INFO scheduler.DAGScheduler: ShuffleMapStage 20 (flatMap at ALS.scala:1433) finished in 0.223 s
18/05/06 18:50:01 INFO scheduler.DAGScheduler: looking for newly runnable stages
18/05/06 18:50:01 INFO scheduler.DAGScheduler: running: Set()
18/05/06 18:50:01 INFO scheduler.DAGScheduler: waiting: Set(ShuffleMapStage 21, ResultStage 22)
18/05/06 18:50:01 INFO scheduler.DAGScheduler: failed: Set()
18/05/06 18:50:01 INFO scheduler.DAGScheduler: Submitting ShuffleMapStage 21 (MapPartitionsRDD[103] at flatMap at ALS.scala:1433), which has no missing parents
18/05/06 18:50:01 INFO memory.MemoryStore: Block broadcast_18 stored as values in memory (estimated size 18.6 KB, free 363.1 MB)
18/05/06 18:50:01 INFO memory.MemoryStore: Block broadcast_18_piece0 stored as bytes in memory (estimated size 6.9 KB, free 363.1 MB)
18/05/06 18:50:01 INFO storage.BlockManagerInfo: Added broadcast_18_piece0 in memory on 192.168.201.128:39601 (size: 6.9 KB, free: 363.5 MB)
18/05/06 18:50:01 INFO spark.SparkContext: Created broadcast 18 from broadcast at DAGScheduler.scala:1006
18/05/06 18:50:01 INFO scheduler.DAGScheduler: Submitting 2 missing tasks from ShuffleMapStage 21 (MapPartitionsRDD[103] at flatMap at ALS.scala:1433) (first 15 tasks are for partitions Vector(0, 1))
18/05/06 18:50:01 INFO scheduler.TaskSchedulerImpl: Adding task set 21.0 with 2 tasks
18/05/06 18:50:01 INFO scheduler.TaskSetManager: Starting task 0.0 in stage 21.0 (TID 32, localhost, executor driver, partition 0, PROCESS_LOCAL, 4900 bytes)
18/05/06 18:50:01 INFO scheduler.TaskSetManager: Starting task 1.0 in stage 21.0 (TID 33, localhost, executor driver, partition 1, PROCESS_LOCAL, 4900 bytes)
18/05/06 18:50:01 INFO executor.Executor: Running task 1.0 in stage 21.0 (TID 33)
18/05/06 18:50:01 INFO executor.Executor: Running task 0.0 in stage 21.0 (TID 32)
18/05/06 18:50:01 INFO storage.BlockManager: Found block rdd_16_0 locally
18/05/06 18:50:01 INFO storage.BlockManager: Found block rdd_15_0 locally
18/05/06 18:50:01 INFO storage.ShuffleBlockFetcherIterator: Getting 2 non-empty blocks out of 2 blocks
18/05/06 18:50:01 INFO storage.ShuffleBlockFetcherIterator: Started 0 remote fetches in 1 ms
18/05/06 18:50:01 INFO storage.BlockManager: Found block rdd_16_1 locally
18/05/06 18:50:01 INFO storage.BlockManager: Found block rdd_15_1 locally
18/05/06 18:50:01 INFO storage.ShuffleBlockFetcherIterator: Getting 2 non-empty blocks out of 2 blocks
18/05/06 18:50:01 INFO storage.ShuffleBlockFetcherIterator: Started 0 remote fetches in 1 ms
18/05/06 18:50:01 INFO executor.Executor: Finished task 0.0 in stage 21.0 (TID 32). 1415 bytes result sent to driver
18/05/06 18:50:01 INFO executor.Executor: Finished task 1.0 in stage 21.0 (TID 33). 1415 bytes result sent to driver
18/05/06 18:50:01 INFO scheduler.TaskSetManager: Finished task 0.0 in stage 21.0 (TID 32) in 182 ms on localhost (executor driver) (1/2)
18/05/06 18:50:01 INFO scheduler.TaskSetManager: Finished task 1.0 in stage 21.0 (TID 33) in 185 ms on localhost (executor driver) (2/2)
18/05/06 18:50:01 INFO scheduler.TaskSchedulerImpl: Removed TaskSet 21.0, whose tasks have all completed, from pool 
18/05/06 18:50:01 INFO scheduler.DAGScheduler: ShuffleMapStage 21 (flatMap at ALS.scala:1433) finished in 0.145 s
18/05/06 18:50:01 INFO scheduler.DAGScheduler: looking for newly runnable stages
18/05/06 18:50:01 INFO scheduler.DAGScheduler: running: Set()
18/05/06 18:50:01 INFO scheduler.DAGScheduler: waiting: Set(ResultStage 22)
18/05/06 18:50:01 INFO scheduler.DAGScheduler: failed: Set()
18/05/06 18:50:01 INFO scheduler.DAGScheduler: Submitting ResultStage 22 (users MapPartitionsRDD[119] at mapValues at ALS.scala:271), which has no missing parents
18/05/06 18:50:02 INFO memory.MemoryStore: Block broadcast_19 stored as values in memory (estimated size 19.8 KB, free 363.1 MB)
18/05/06 18:50:02 INFO memory.MemoryStore: Block broadcast_19_piece0 stored as bytes in memory (estimated size 7.2 KB, free 363.1 MB)
18/05/06 18:50:02 INFO storage.BlockManagerInfo: Added broadcast_19_piece0 in memory on 192.168.201.128:39601 (size: 7.2 KB, free: 363.5 MB)
18/05/06 18:50:02 INFO spark.SparkContext: Created broadcast 19 from broadcast at DAGScheduler.scala:1006
18/05/06 18:50:02 INFO scheduler.DAGScheduler: Submitting 2 missing tasks from ResultStage 22 (users MapPartitionsRDD[119] at mapValues at ALS.scala:271) (first 15 tasks are for partitions Vector(0, 1))
18/05/06 18:50:02 INFO scheduler.TaskSchedulerImpl: Adding task set 22.0 with 2 tasks
18/05/06 18:50:02 INFO scheduler.TaskSetManager: Starting task 0.0 in stage 22.0 (TID 34, localhost, executor driver, partition 0, PROCESS_LOCAL, 4911 bytes)
18/05/06 18:50:02 INFO scheduler.TaskSetManager: Starting task 1.0 in stage 22.0 (TID 35, localhost, executor driver, partition 1, PROCESS_LOCAL, 4911 bytes)
18/05/06 18:50:02 INFO executor.Executor: Running task 0.0 in stage 22.0 (TID 34)
18/05/06 18:50:02 INFO executor.Executor: Running task 1.0 in stage 22.0 (TID 35)
18/05/06 18:50:02 INFO storage.BlockManager: Found block rdd_10_1 locally
18/05/06 18:50:02 INFO storage.BlockManager: Found block rdd_10_0 locally
18/05/06 18:50:02 INFO storage.BlockManager: Found block rdd_10_0 locally
18/05/06 18:50:02 INFO storage.ShuffleBlockFetcherIterator: Getting 2 non-empty blocks out of 2 blocks
18/05/06 18:50:02 INFO storage.ShuffleBlockFetcherIterator: Started 0 remote fetches in 1 ms
18/05/06 18:50:02 INFO storage.BlockManager: Found block rdd_10_1 locally
18/05/06 18:50:02 INFO storage.ShuffleBlockFetcherIterator: Getting 2 non-empty blocks out of 2 blocks
18/05/06 18:50:02 INFO storage.ShuffleBlockFetcherIterator: Started 0 remote fetches in 3 ms
18/05/06 18:50:02 INFO memory.MemoryStore: Block rdd_119_1 stored as values in memory (estimated size 39.4 KB, free 363.1 MB)
18/05/06 18:50:02 INFO storage.BlockManagerInfo: Added rdd_119_1 in memory on 192.168.201.128:39601 (size: 39.4 KB, free: 363.5 MB)
18/05/06 18:50:02 INFO executor.Executor: Finished task 1.0 in stage 22.0 (TID 35). 1916 bytes result sent to driver
18/05/06 18:50:02 INFO memory.MemoryStore: Block rdd_119_0 stored as values in memory (estimated size 39.4 KB, free 363.0 MB)
18/05/06 18:50:02 INFO scheduler.TaskSetManager: Finished task 1.0 in stage 22.0 (TID 35) in 321 ms on localhost (executor driver) (1/2)
18/05/06 18:50:02 INFO storage.BlockManagerInfo: Added rdd_119_0 in memory on 192.168.201.128:39601 (size: 39.4 KB, free: 363.4 MB)
18/05/06 18:50:02 INFO executor.Executor: Finished task 0.0 in stage 22.0 (TID 34). 1959 bytes result sent to driver
18/05/06 18:50:02 INFO scheduler.TaskSetManager: Finished task 0.0 in stage 22.0 (TID 34) in 338 ms on localhost (executor driver) (2/2)
18/05/06 18:50:02 INFO scheduler.TaskSchedulerImpl: Removed TaskSet 22.0, whose tasks have all completed, from pool 
18/05/06 18:50:02 INFO scheduler.DAGScheduler: ResultStage 22 (count at ALS.scala:279) finished in 0.349 s
18/05/06 18:50:02 INFO scheduler.DAGScheduler: Job 4 finished: count at ALS.scala:279, took 3.877865 s
18/05/06 18:50:02 INFO spark.SparkContext: Starting job: count at ALS.scala:280
18/05/06 18:50:02 INFO spark.MapOutputTrackerMaster: Size of output statuses for shuffle 1 is 160 bytes
18/05/06 18:50:02 INFO spark.MapOutputTrackerMaster: Size of output statuses for shuffle 2 is 160 bytes
18/05/06 18:50:02 INFO spark.MapOutputTrackerMaster: Size of output statuses for shuffle 0 is 160 bytes
18/05/06 18:50:02 INFO spark.MapOutputTrackerMaster: Size of output statuses for shuffle 13 is 163 bytes
18/05/06 18:50:02 INFO spark.MapOutputTrackerMaster: Size of output statuses for shuffle 12 is 160 bytes
18/05/06 18:50:02 INFO spark.MapOutputTrackerMaster: Size of output statuses for shuffle 11 is 160 bytes
18/05/06 18:50:02 INFO spark.MapOutputTrackerMaster: Size of output statuses for shuffle 10 is 160 bytes
18/05/06 18:50:02 INFO spark.MapOutputTrackerMaster: Size of output statuses for shuffle 9 is 160 bytes
18/05/06 18:50:02 INFO spark.MapOutputTrackerMaster: Size of output statuses for shuffle 8 is 160 bytes
18/05/06 18:50:02 INFO spark.MapOutputTrackerMaster: Size of output statuses for shuffle 7 is 160 bytes
18/05/06 18:50:02 INFO spark.MapOutputTrackerMaster: Size of output statuses for shuffle 6 is 160 bytes
18/05/06 18:50:02 INFO spark.MapOutputTrackerMaster: Size of output statuses for shuffle 5 is 160 bytes
18/05/06 18:50:02 INFO spark.MapOutputTrackerMaster: Size of output statuses for shuffle 4 is 160 bytes
18/05/06 18:50:02 INFO scheduler.DAGScheduler: Got job 5 (count at ALS.scala:280) with 2 output partitions
18/05/06 18:50:02 INFO scheduler.DAGScheduler: Final stage: ResultStage 36 (count at ALS.scala:280)
18/05/06 18:50:02 INFO scheduler.DAGScheduler: Parents of final stage: List(ShuffleMapStage 35, ShuffleMapStage 24)
18/05/06 18:50:02 INFO scheduler.DAGScheduler: Missing parents: List()
18/05/06 18:50:02 INFO scheduler.DAGScheduler: Submitting ResultStage 36 (products MapPartitionsRDD[120] at mapValues at ALS.scala:275), which has no missing parents
18/05/06 18:50:02 INFO memory.MemoryStore: Block broadcast_20 stored as values in memory (estimated size 18.9 KB, free 363.0 MB)
18/05/06 18:50:02 INFO memory.MemoryStore: Block broadcast_20_piece0 stored as bytes in memory (estimated size 7.0 KB, free 363.0 MB)
18/05/06 18:50:02 INFO storage.BlockManagerInfo: Added broadcast_20_piece0 in memory on 192.168.201.128:39601 (size: 7.0 KB, free: 363.4 MB)
18/05/06 18:50:02 INFO spark.SparkContext: Created broadcast 20 from broadcast at DAGScheduler.scala:1006
18/05/06 18:50:02 INFO scheduler.DAGScheduler: Submitting 2 missing tasks from ResultStage 36 (products MapPartitionsRDD[120] at mapValues at ALS.scala:275) (first 15 tasks are for partitions Vector(0, 1))
18/05/06 18:50:02 INFO scheduler.TaskSchedulerImpl: Adding task set 36.0 with 2 tasks
18/05/06 18:50:02 INFO scheduler.TaskSetManager: Starting task 0.0 in stage 36.0 (TID 36, localhost, executor driver, partition 0, PROCESS_LOCAL, 4911 bytes)
18/05/06 18:50:02 INFO scheduler.TaskSetManager: Starting task 1.0 in stage 36.0 (TID 37, localhost, executor driver, partition 1, PROCESS_LOCAL, 4911 bytes)
18/05/06 18:50:02 INFO executor.Executor: Running task 0.0 in stage 36.0 (TID 36)
18/05/06 18:50:02 INFO executor.Executor: Running task 1.0 in stage 36.0 (TID 37)
18/05/06 18:50:02 INFO storage.BlockManager: Found block rdd_15_1 locally
18/05/06 18:50:02 INFO storage.BlockManager: Found block rdd_15_1 locally
18/05/06 18:50:02 INFO storage.ShuffleBlockFetcherIterator: Getting 2 non-empty blocks out of 2 blocks
18/05/06 18:50:02 INFO storage.ShuffleBlockFetcherIterator: Started 0 remote fetches in 0 ms
18/05/06 18:50:02 INFO storage.BlockManager: Found block rdd_15_0 locally
18/05/06 18:50:02 INFO storage.BlockManager: Found block rdd_15_0 locally
18/05/06 18:50:02 INFO storage.ShuffleBlockFetcherIterator: Getting 2 non-empty blocks out of 2 blocks
18/05/06 18:50:02 INFO storage.ShuffleBlockFetcherIterator: Started 0 remote fetches in 35 ms
18/05/06 18:50:02 INFO memory.MemoryStore: Block rdd_120_1 stored as values in memory (estimated size 78.3 KB, free 362.9 MB)
18/05/06 18:50:02 INFO storage.BlockManagerInfo: Added rdd_120_1 in memory on 192.168.201.128:39601 (size: 78.3 KB, free: 363.3 MB)
18/05/06 18:50:02 INFO executor.Executor: Finished task 1.0 in stage 36.0 (TID 37). 1916 bytes result sent to driver
18/05/06 18:50:02 INFO scheduler.TaskSetManager: Finished task 1.0 in stage 36.0 (TID 37) in 338 ms on localhost (executor driver) (1/2)
18/05/06 18:50:02 INFO memory.MemoryStore: Block rdd_120_0 stored as values in memory (estimated size 78.3 KB, free 362.8 MB)
18/05/06 18:50:02 INFO storage.BlockManagerInfo: Added rdd_120_0 in memory on 192.168.201.128:39601 (size: 78.3 KB, free: 363.3 MB)
18/05/06 18:50:02 INFO executor.Executor: Finished task 0.0 in stage 36.0 (TID 36). 1916 bytes result sent to driver
18/05/06 18:50:02 INFO scheduler.TaskSetManager: Finished task 0.0 in stage 36.0 (TID 36) in 405 ms on localhost (executor driver) (2/2)
18/05/06 18:50:02 INFO scheduler.DAGScheduler: ResultStage 36 (count at ALS.scala:280) finished in 0.410 s
18/05/06 18:50:02 INFO scheduler.DAGScheduler: Job 5 finished: count at ALS.scala:280, took 0.579904 s
18/05/06 18:50:02 INFO scheduler.TaskSchedulerImpl: Removed TaskSet 36.0, whose tasks have all completed, from pool 
18/05/06 18:50:03 INFO spark.SparkContext: Starting job: first at MatrixFactorizationModel.scala:67
18/05/06 18:50:03 INFO spark.MapOutputTrackerMaster: Size of output statuses for shuffle 3 is 160 bytes
18/05/06 18:50:03 INFO scheduler.DAGScheduler: Got job 6 (first at MatrixFactorizationModel.scala:67) with 1 output partitions
18/05/06 18:50:03 INFO scheduler.DAGScheduler: Final stage: ResultStage 51 (first at MatrixFactorizationModel.scala:67)
18/05/06 18:50:03 INFO scheduler.DAGScheduler: Parents of final stage: List(ShuffleMapStage 38, ShuffleMapStage 50)
18/05/06 18:50:03 INFO scheduler.DAGScheduler: Missing parents: List()
18/05/06 18:50:03 INFO scheduler.DAGScheduler: Submitting ResultStage 51 (users MapPartitionsRDD[119] at mapValues at ALS.scala:271), which has no missing parents
18/05/06 18:50:03 INFO memory.MemoryStore: Block broadcast_21 stored as values in memory (estimated size 20.0 KB, free 362.8 MB)
18/05/06 18:50:03 INFO memory.MemoryStore: Block broadcast_21_piece0 stored as bytes in memory (estimated size 7.3 KB, free 362.8 MB)
18/05/06 18:50:03 INFO storage.BlockManagerInfo: Added broadcast_21_piece0 in memory on 192.168.201.128:39601 (size: 7.3 KB, free: 363.3 MB)
18/05/06 18:50:03 INFO spark.SparkContext: Created broadcast 21 from broadcast at DAGScheduler.scala:1006
18/05/06 18:50:03 INFO scheduler.DAGScheduler: Submitting 1 missing tasks from ResultStage 51 (users MapPartitionsRDD[119] at mapValues at ALS.scala:271) (first 15 tasks are for partitions Vector(0))
18/05/06 18:50:03 INFO scheduler.TaskSchedulerImpl: Adding task set 51.0 with 1 tasks
18/05/06 18:50:03 INFO scheduler.TaskSetManager: Starting task 0.0 in stage 51.0 (TID 38, localhost, executor driver, partition 0, PROCESS_LOCAL, 4911 bytes)
18/05/06 18:50:03 INFO executor.Executor: Running task 0.0 in stage 51.0 (TID 38)
18/05/06 18:50:03 INFO storage.BlockManager: Found block rdd_119_0 locally
18/05/06 18:50:03 INFO executor.Executor: 1 block locks were not released by TID = 38:
[rdd_119_0]
18/05/06 18:50:03 INFO executor.Executor: Finished task 0.0 in stage 51.0 (TID 38). 995 bytes result sent to driver
18/05/06 18:50:03 INFO scheduler.TaskSetManager: Finished task 0.0 in stage 51.0 (TID 38) in 32 ms on localhost (executor driver) (1/1)
18/05/06 18:50:03 INFO scheduler.TaskSchedulerImpl: Removed TaskSet 51.0, whose tasks have all completed, from pool 
18/05/06 18:50:03 INFO scheduler.DAGScheduler: ResultStage 51 (first at MatrixFactorizationModel.scala:67) finished in 0.035 s
18/05/06 18:50:03 INFO scheduler.DAGScheduler: Job 6 finished: first at MatrixFactorizationModel.scala:67, took 0.130999 s
18/05/06 18:50:03 INFO spark.SparkContext: Starting job: first at MatrixFactorizationModel.scala:67
18/05/06 18:50:03 INFO scheduler.DAGScheduler: Got job 7 (first at MatrixFactorizationModel.scala:67) with 1 output partitions
18/05/06 18:50:03 INFO scheduler.DAGScheduler: Final stage: ResultStage 65 (first at MatrixFactorizationModel.scala:67)
18/05/06 18:50:03 INFO scheduler.DAGScheduler: Parents of final stage: List(ShuffleMapStage 64, ShuffleMapStage 53)
18/05/06 18:50:03 INFO scheduler.DAGScheduler: Missing parents: List()
18/05/06 18:50:03 INFO scheduler.DAGScheduler: Submitting ResultStage 65 (products MapPartitionsRDD[120] at mapValues at ALS.scala:275), which has no missing parents
18/05/06 18:50:03 INFO memory.MemoryStore: Block broadcast_22 stored as values in memory (estimated size 19.1 KB, free 362.8 MB)
18/05/06 18:50:03 INFO memory.MemoryStore: Block broadcast_22_piece0 stored as bytes in memory (estimated size 7.1 KB, free 362.8 MB)
18/05/06 18:50:03 INFO storage.BlockManagerInfo: Added broadcast_22_piece0 in memory on 192.168.201.128:39601 (size: 7.1 KB, free: 363.2 MB)
18/05/06 18:50:03 INFO spark.SparkContext: Created broadcast 22 from broadcast at DAGScheduler.scala:1006
18/05/06 18:50:03 INFO scheduler.DAGScheduler: Submitting 1 missing tasks from ResultStage 65 (products MapPartitionsRDD[120] at mapValues at ALS.scala:275) (first 15 tasks are for partitions Vector(0))
18/05/06 18:50:03 INFO scheduler.TaskSchedulerImpl: Adding task set 65.0 with 1 tasks
18/05/06 18:50:03 INFO scheduler.TaskSetManager: Starting task 0.0 in stage 65.0 (TID 39, localhost, executor driver, partition 0, PROCESS_LOCAL, 4911 bytes)
18/05/06 18:50:03 INFO executor.Executor: Running task 0.0 in stage 65.0 (TID 39)
18/05/06 18:50:03 INFO storage.BlockManager: Found block rdd_120_0 locally
18/05/06 18:50:03 INFO executor.Executor: 1 block locks were not released by TID = 39:
[rdd_120_0]
18/05/06 18:50:03 INFO executor.Executor: Finished task 0.0 in stage 65.0 (TID 39). 995 bytes result sent to driver
18/05/06 18:50:03 INFO scheduler.TaskSetManager: Finished task 0.0 in stage 65.0 (TID 39) in 20 ms on localhost (executor driver) (1/1)
18/05/06 18:50:03 INFO scheduler.TaskSchedulerImpl: Removed TaskSet 65.0, whose tasks have all completed, from pool 
18/05/06 18:50:03 INFO scheduler.DAGScheduler: ResultStage 65 (first at MatrixFactorizationModel.scala:67) finished in 0.020 s
18/05/06 18:50:03 INFO scheduler.DAGScheduler: Job 7 finished: first at MatrixFactorizationModel.scala:67, took 0.079481 s
18/05/06 18:50:03 INFO spark.SparkContext: Starting job: count at ExtFeatureRC.scala:41
18/05/06 18:50:03 INFO scheduler.DAGScheduler: Got job 8 (count at ExtFeatureRC.scala:41) with 2 output partitions
18/05/06 18:50:03 INFO scheduler.DAGScheduler: Final stage: ResultStage 80 (count at ExtFeatureRC.scala:41)
18/05/06 18:50:03 INFO scheduler.DAGScheduler: Parents of final stage: List(ShuffleMapStage 67, ShuffleMapStage 79)
18/05/06 18:50:03 INFO scheduler.DAGScheduler: Missing parents: List()
18/05/06 18:50:03 INFO scheduler.DAGScheduler: Submitting ResultStage 80 (users MapPartitionsRDD[119] at mapValues at ALS.scala:271), which has no missing parents
18/05/06 18:50:03 INFO memory.MemoryStore: Block broadcast_23 stored as values in memory (estimated size 19.8 KB, free 362.8 MB)
18/05/06 18:50:03 INFO memory.MemoryStore: Block broadcast_23_piece0 stored as bytes in memory (estimated size 7.2 KB, free 362.8 MB)
18/05/06 18:50:03 INFO storage.BlockManagerInfo: Added broadcast_23_piece0 in memory on 192.168.201.128:39601 (size: 7.2 KB, free: 363.2 MB)
18/05/06 18:50:03 INFO spark.SparkContext: Created broadcast 23 from broadcast at DAGScheduler.scala:1006
18/05/06 18:50:03 INFO scheduler.DAGScheduler: Submitting 2 missing tasks from ResultStage 80 (users MapPartitionsRDD[119] at mapValues at ALS.scala:271) (first 15 tasks are for partitions Vector(0, 1))
18/05/06 18:50:03 INFO scheduler.TaskSchedulerImpl: Adding task set 80.0 with 2 tasks
18/05/06 18:50:03 INFO scheduler.TaskSetManager: Starting task 0.0 in stage 80.0 (TID 40, localhost, executor driver, partition 0, PROCESS_LOCAL, 4911 bytes)
18/05/06 18:50:03 INFO scheduler.TaskSetManager: Starting task 1.0 in stage 80.0 (TID 41, localhost, executor driver, partition 1, PROCESS_LOCAL, 4911 bytes)
18/05/06 18:50:03 INFO executor.Executor: Running task 0.0 in stage 80.0 (TID 40)
18/05/06 18:50:03 INFO executor.Executor: Running task 1.0 in stage 80.0 (TID 41)
18/05/06 18:50:03 INFO storage.BlockManager: Found block rdd_119_1 locally
18/05/06 18:50:03 INFO executor.Executor: Finished task 1.0 in stage 80.0 (TID 41). 832 bytes result sent to driver
18/05/06 18:50:03 INFO storage.BlockManager: Found block rdd_119_0 locally
18/05/06 18:50:03 INFO executor.Executor: Finished task 0.0 in stage 80.0 (TID 40). 875 bytes result sent to driver
18/05/06 18:50:03 INFO scheduler.TaskSetManager: Finished task 1.0 in stage 80.0 (TID 41) in 60 ms on localhost (executor driver) (1/2)
18/05/06 18:50:03 INFO scheduler.TaskSetManager: Finished task 0.0 in stage 80.0 (TID 40) in 65 ms on localhost (executor driver) (2/2)
18/05/06 18:50:03 INFO scheduler.TaskSchedulerImpl: Removed TaskSet 80.0, whose tasks have all completed, from pool 
18/05/06 18:50:03 INFO scheduler.DAGScheduler: ResultStage 80 (count at ExtFeatureRC.scala:41) finished in 0.065 s
18/05/06 18:50:03 INFO scheduler.DAGScheduler: Job 8 finished: count at ExtFeatureRC.scala:41, took 0.156099 s
model.userFeatures.count = 943

18/05/06 18:50:03 INFO spark.SparkContext: Starting job: lookup at MatrixFactorizationModel.scala:81
18/05/06 18:50:03 INFO scheduler.DAGScheduler: Got job 9 (lookup at MatrixFactorizationModel.scala:81) with 1 output partitions
18/05/06 18:50:03 INFO scheduler.DAGScheduler: Final stage: ResultStage 95 (lookup at MatrixFactorizationModel.scala:81)
18/05/06 18:50:03 INFO scheduler.DAGScheduler: Parents of final stage: List(ShuffleMapStage 82, ShuffleMapStage 94)
18/05/06 18:50:03 INFO scheduler.DAGScheduler: Missing parents: List()
18/05/06 18:50:03 INFO scheduler.DAGScheduler: Submitting ResultStage 95 (users MapPartitionsRDD[119] at mapValues at ALS.scala:271), which has no missing parents
18/05/06 18:50:03 INFO memory.MemoryStore: Block broadcast_24 stored as values in memory (estimated size 20.1 KB, free 362.7 MB)
18/05/06 18:50:03 INFO memory.MemoryStore: Block broadcast_24_piece0 stored as bytes in memory (estimated size 7.4 KB, free 362.7 MB)
18/05/06 18:50:03 INFO storage.BlockManagerInfo: Added broadcast_24_piece0 in memory on 192.168.201.128:39601 (size: 7.4 KB, free: 363.2 MB)
18/05/06 18:50:03 INFO spark.SparkContext: Created broadcast 24 from broadcast at DAGScheduler.scala:1006
18/05/06 18:50:03 INFO scheduler.DAGScheduler: Submitting 1 missing tasks from ResultStage 95 (users MapPartitionsRDD[119] at mapValues at ALS.scala:271) (first 15 tasks are for partitions Vector(1))
18/05/06 18:50:03 INFO scheduler.TaskSchedulerImpl: Adding task set 95.0 with 1 tasks
18/05/06 18:50:03 INFO scheduler.TaskSetManager: Starting task 0.0 in stage 95.0 (TID 42, localhost, executor driver, partition 1, PROCESS_LOCAL, 4911 bytes)
18/05/06 18:50:03 INFO executor.Executor: Running task 0.0 in stage 95.0 (TID 42)
18/05/06 18:50:03 INFO storage.BlockManager: Found block rdd_119_1 locally
18/05/06 18:50:03 INFO executor.Executor: Finished task 0.0 in stage 95.0 (TID 42). 1028 bytes result sent to driver
18/05/06 18:50:03 INFO scheduler.TaskSetManager: Finished task 0.0 in stage 95.0 (TID 42) in 62 ms on localhost (executor driver) (1/1)
18/05/06 18:50:03 INFO scheduler.TaskSchedulerImpl: Removed TaskSet 95.0, whose tasks have all completed, from pool 
18/05/06 18:50:03 INFO scheduler.DAGScheduler: ResultStage 95 (lookup at MatrixFactorizationModel.scala:81) finished in 0.063 s
18/05/06 18:50:03 INFO scheduler.DAGScheduler: Job 9 finished: lookup at MatrixFactorizationModel.scala:81, took 0.110734 s
18/05/06 18:50:03 INFO spark.SparkContext: Starting job: lookup at MatrixFactorizationModel.scala:82
18/05/06 18:50:03 INFO scheduler.DAGScheduler: Got job 10 (lookup at MatrixFactorizationModel.scala:82) with 1 output partitions
18/05/06 18:50:03 INFO scheduler.DAGScheduler: Final stage: ResultStage 109 (lookup at MatrixFactorizationModel.scala:82)
18/05/06 18:50:03 INFO scheduler.DAGScheduler: Parents of final stage: List(ShuffleMapStage 108, ShuffleMapStage 97)
18/05/06 18:50:03 INFO scheduler.DAGScheduler: Missing parents: List()
18/05/06 18:50:03 INFO scheduler.DAGScheduler: Submitting ResultStage 109 (products MapPartitionsRDD[120] at mapValues at ALS.scala:275), which has no missing parents
18/05/06 18:50:03 INFO memory.MemoryStore: Block broadcast_25 stored as values in memory (estimated size 19.2 KB, free 362.7 MB)
18/05/06 18:50:03 INFO memory.MemoryStore: Block broadcast_25_piece0 stored as bytes in memory (estimated size 7.1 KB, free 362.7 MB)
18/05/06 18:50:03 INFO storage.BlockManagerInfo: Added broadcast_25_piece0 in memory on 192.168.201.128:39601 (size: 7.1 KB, free: 363.2 MB)
18/05/06 18:50:03 INFO spark.SparkContext: Created broadcast 25 from broadcast at DAGScheduler.scala:1006
18/05/06 18:50:03 INFO scheduler.DAGScheduler: Submitting 1 missing tasks from ResultStage 109 (products MapPartitionsRDD[120] at mapValues at ALS.scala:275) (first 15 tasks are for partitions Vector(1))
18/05/06 18:50:03 INFO scheduler.TaskSchedulerImpl: Adding task set 109.0 with 1 tasks
18/05/06 18:50:03 INFO scheduler.TaskSetManager: Starting task 0.0 in stage 109.0 (TID 43, localhost, executor driver, partition 1, PROCESS_LOCAL, 4911 bytes)
18/05/06 18:50:03 INFO executor.Executor: Running task 0.0 in stage 109.0 (TID 43)
18/05/06 18:50:03 INFO storage.BlockManager: Found block rdd_120_1 locally
18/05/06 18:50:03 INFO executor.Executor: Finished task 0.0 in stage 109.0 (TID 43). 1028 bytes result sent to driver
18/05/06 18:50:03 INFO scheduler.TaskSetManager: Finished task 0.0 in stage 109.0 (TID 43) in 12 ms on localhost (executor driver) (1/1)
18/05/06 18:50:03 INFO scheduler.TaskSchedulerImpl: Removed TaskSet 109.0, whose tasks have all completed, from pool 
18/05/06 18:50:03 INFO scheduler.DAGScheduler: ResultStage 109 (lookup at MatrixFactorizationModel.scala:82) finished in 0.015 s
18/05/06 18:50:03 INFO scheduler.DAGScheduler: Job 10 finished: lookup at MatrixFactorizationModel.scala:82, took 0.067510 s
predictedRating = 3.1529348917041577

18/05/06 18:50:03 INFO spark.SparkContext: Starting job: lookup at MatrixFactorizationModel.scala:168
18/05/06 18:50:03 INFO scheduler.DAGScheduler: Got job 11 (lookup at MatrixFactorizationModel.scala:168) with 1 output partitions
18/05/06 18:50:03 INFO scheduler.DAGScheduler: Final stage: ResultStage 124 (lookup at MatrixFactorizationModel.scala:168)
18/05/06 18:50:03 INFO scheduler.DAGScheduler: Parents of final stage: List(ShuffleMapStage 111, ShuffleMapStage 123)
18/05/06 18:50:03 INFO scheduler.DAGScheduler: Missing parents: List()
18/05/06 18:50:03 INFO scheduler.DAGScheduler: Submitting ResultStage 124 (users MapPartitionsRDD[119] at mapValues at ALS.scala:271), which has no missing parents
18/05/06 18:50:03 INFO memory.MemoryStore: Block broadcast_26 stored as values in memory (estimated size 20.1 KB, free 362.7 MB)
18/05/06 18:50:03 INFO memory.MemoryStore: Block broadcast_26_piece0 stored as bytes in memory (estimated size 7.4 KB, free 362.7 MB)
18/05/06 18:50:03 INFO storage.BlockManagerInfo: Added broadcast_26_piece0 in memory on 192.168.201.128:39601 (size: 7.4 KB, free: 363.2 MB)
18/05/06 18:50:03 INFO spark.SparkContext: Created broadcast 26 from broadcast at DAGScheduler.scala:1006
18/05/06 18:50:03 INFO scheduler.DAGScheduler: Submitting 1 missing tasks from ResultStage 124 (users MapPartitionsRDD[119] at mapValues at ALS.scala:271) (first 15 tasks are for partitions Vector(1))
18/05/06 18:50:03 INFO scheduler.TaskSchedulerImpl: Adding task set 124.0 with 1 tasks
18/05/06 18:50:03 INFO scheduler.TaskSetManager: Starting task 0.0 in stage 124.0 (TID 44, localhost, executor driver, partition 1, PROCESS_LOCAL, 4911 bytes)
18/05/06 18:50:03 INFO executor.Executor: Running task 0.0 in stage 124.0 (TID 44)
18/05/06 18:50:03 INFO storage.BlockManager: Found block rdd_119_1 locally
18/05/06 18:50:03 INFO executor.Executor: Finished task 0.0 in stage 124.0 (TID 44). 985 bytes result sent to driver
18/05/06 18:50:03 INFO scheduler.TaskSetManager: Finished task 0.0 in stage 124.0 (TID 44) in 16 ms on localhost (executor driver) (1/1)
18/05/06 18:50:03 INFO scheduler.TaskSchedulerImpl: Removed TaskSet 124.0, whose tasks have all completed, from pool 
18/05/06 18:50:03 INFO scheduler.DAGScheduler: ResultStage 124 (lookup at MatrixFactorizationModel.scala:168) finished in 0.020 s
18/05/06 18:50:03 INFO scheduler.DAGScheduler: Job 11 finished: lookup at MatrixFactorizationModel.scala:168, took 0.078790 s
18/05/06 18:50:03 INFO spark.SparkContext: Starting job: top at MatrixFactorizationModel.scala:259
18/05/06 18:50:03 INFO scheduler.DAGScheduler: Got job 12 (top at MatrixFactorizationModel.scala:259) with 2 output partitions
18/05/06 18:50:03 INFO scheduler.DAGScheduler: Final stage: ResultStage 138 (top at MatrixFactorizationModel.scala:259)
18/05/06 18:50:03 INFO scheduler.DAGScheduler: Parents of final stage: List(ShuffleMapStage 137, ShuffleMapStage 126)
18/05/06 18:50:03 INFO scheduler.DAGScheduler: Missing parents: List()
18/05/06 18:50:03 INFO scheduler.DAGScheduler: Submitting ResultStage 138 (MapPartitionsRDD[122] at top at MatrixFactorizationModel.scala:259), which has no missing parents
18/05/06 18:50:03 INFO memory.MemoryStore: Block broadcast_27 stored as values in memory (estimated size 20.1 KB, free 362.7 MB)
18/05/06 18:50:03 INFO memory.MemoryStore: Block broadcast_27_piece0 stored as bytes in memory (estimated size 7.5 KB, free 362.7 MB)
18/05/06 18:50:03 INFO storage.BlockManagerInfo: Added broadcast_27_piece0 in memory on 192.168.201.128:39601 (size: 7.5 KB, free: 363.2 MB)
18/05/06 18:50:03 INFO spark.SparkContext: Created broadcast 27 from broadcast at DAGScheduler.scala:1006
18/05/06 18:50:03 INFO scheduler.DAGScheduler: Submitting 2 missing tasks from ResultStage 138 (MapPartitionsRDD[122] at top at MatrixFactorizationModel.scala:259) (first 15 tasks are for partitions Vector(0, 1))
18/05/06 18:50:03 INFO scheduler.TaskSchedulerImpl: Adding task set 138.0 with 2 tasks
18/05/06 18:50:03 INFO scheduler.TaskSetManager: Starting task 0.0 in stage 138.0 (TID 45, localhost, executor driver, partition 0, PROCESS_LOCAL, 4911 bytes)
18/05/06 18:50:03 INFO scheduler.TaskSetManager: Starting task 1.0 in stage 138.0 (TID 46, localhost, executor driver, partition 1, PROCESS_LOCAL, 4911 bytes)
18/05/06 18:50:03 INFO executor.Executor: Running task 1.0 in stage 138.0 (TID 46)
18/05/06 18:50:03 INFO executor.Executor: Running task 0.0 in stage 138.0 (TID 45)
18/05/06 18:50:03 INFO storage.BlockManager: Found block rdd_120_0 locally
18/05/06 18:50:03 INFO storage.BlockManager: Found block rdd_120_1 locally
18/05/06 18:50:04 INFO executor.Executor: Finished task 0.0 in stage 138.0 (TID 45). 1795 bytes result sent to driver
18/05/06 18:50:04 INFO executor.Executor: Finished task 1.0 in stage 138.0 (TID 46). 1795 bytes result sent to driver
18/05/06 18:50:04 INFO scheduler.TaskSetManager: Finished task 0.0 in stage 138.0 (TID 45) in 59 ms on localhost (executor driver) (1/2)
18/05/06 18:50:04 INFO scheduler.TaskSetManager: Finished task 1.0 in stage 138.0 (TID 46) in 60 ms on localhost (executor driver) (2/2)
18/05/06 18:50:04 INFO scheduler.TaskSchedulerImpl: Removed TaskSet 138.0, whose tasks have all completed, from pool 
18/05/06 18:50:04 INFO scheduler.DAGScheduler: ResultStage 138 (top at MatrixFactorizationModel.scala:259) finished in 0.061 s
18/05/06 18:50:04 INFO scheduler.DAGScheduler: Job 12 finished: top at MatrixFactorizationModel.scala:259, took 0.169724 s
Rating(789,838,7.988870096266704)
Rating(789,1288,7.139100057786628)
Rating(789,74,7.089206512169639)
Rating(789,1607,7.070558913016638)
Rating(789,1664,7.025680340760473)
Rating(789,1643,7.021651958848989)
Rating(789,1324,6.688334395551978)
Rating(789,868,6.651602999961195)
Rating(789,75,6.642213102693804)
Rating(789,1203,6.622753623010645)
18/05/06 18:50:04 INFO memory.MemoryStore: Block broadcast_28 stored as values in memory (estimated size 240.0 KB, free 362.4 MB)
18/05/06 18:50:04 INFO memory.MemoryStore: Block broadcast_28_piece0 stored as bytes in memory (estimated size 23.3 KB, free 362.4 MB)
18/05/06 18:50:04 INFO storage.BlockManagerInfo: Added broadcast_28_piece0 in memory on 192.168.201.128:39601 (size: 23.3 KB, free: 363.2 MB)
18/05/06 18:50:04 INFO spark.SparkContext: Created broadcast 28 from textFile at ExtFeatureRC.scala:54
18/05/06 18:50:04 INFO mapred.FileInputFormat: Total input paths to process : 1
18/05/06 18:50:04 INFO spark.SparkContext: Starting job: collectAsMap at ExtFeatureRC.scala:56
18/05/06 18:50:04 INFO scheduler.DAGScheduler: Got job 13 (collectAsMap at ExtFeatureRC.scala:56) with 2 output partitions
18/05/06 18:50:04 INFO scheduler.DAGScheduler: Final stage: ResultStage 139 (collectAsMap at ExtFeatureRC.scala:56)
18/05/06 18:50:04 INFO scheduler.DAGScheduler: Parents of final stage: List()
18/05/06 18:50:04 INFO scheduler.DAGScheduler: Missing parents: List()
18/05/06 18:50:04 INFO scheduler.DAGScheduler: Submitting ResultStage 139 (MapPartitionsRDD[126] at map at ExtFeatureRC.scala:55), which has no missing parents
18/05/06 18:50:04 INFO memory.MemoryStore: Block broadcast_29 stored as values in memory (estimated size 3.5 KB, free 362.4 MB)
18/05/06 18:50:04 INFO memory.MemoryStore: Block broadcast_29_piece0 stored as bytes in memory (estimated size 2043.0 B, free 362.4 MB)
18/05/06 18:50:04 INFO storage.BlockManagerInfo: Added broadcast_29_piece0 in memory on 192.168.201.128:39601 (size: 2043.0 B, free: 363.2 MB)
18/05/06 18:50:04 INFO spark.SparkContext: Created broadcast 29 from broadcast at DAGScheduler.scala:1006
18/05/06 18:50:04 INFO scheduler.DAGScheduler: Submitting 2 missing tasks from ResultStage 139 (MapPartitionsRDD[126] at map at ExtFeatureRC.scala:55) (first 15 tasks are for partitions Vector(0, 1))
18/05/06 18:50:04 INFO scheduler.TaskSchedulerImpl: Adding task set 139.0 with 2 tasks
18/05/06 18:50:04 INFO scheduler.TaskSetManager: Starting task 0.0 in stage 139.0 (TID 47, localhost, executor driver, partition 0, ANY, 4857 bytes)
18/05/06 18:50:04 INFO scheduler.TaskSetManager: Starting task 1.0 in stage 139.0 (TID 48, localhost, executor driver, partition 1, ANY, 4857 bytes)
18/05/06 18:50:04 INFO executor.Executor: Running task 0.0 in stage 139.0 (TID 47)
18/05/06 18:50:04 INFO executor.Executor: Running task 1.0 in stage 139.0 (TID 48)
18/05/06 18:50:04 INFO rdd.HadoopRDD: Input split: hdfs://master:9000/user/ml-100k/u.item:0+118172
18/05/06 18:50:04 INFO rdd.HadoopRDD: Input split: hdfs://master:9000/user/ml-100k/u.item:118172+118172
18/05/06 18:50:04 INFO executor.Executor: Finished task 0.0 in stage 139.0 (TID 47). 36684 bytes result sent to driver
18/05/06 18:50:04 INFO executor.Executor: Finished task 1.0 in stage 139.0 (TID 48). 37023 bytes result sent to driver
18/05/06 18:50:04 INFO scheduler.TaskSetManager: Finished task 0.0 in stage 139.0 (TID 47) in 184 ms on localhost (executor driver) (1/2)
18/05/06 18:50:04 INFO scheduler.TaskSetManager: Finished task 1.0 in stage 139.0 (TID 48) in 186 ms on localhost (executor driver) (2/2)
18/05/06 18:50:04 INFO scheduler.TaskSchedulerImpl: Removed TaskSet 139.0, whose tasks have all completed, from pool 
18/05/06 18:50:04 INFO scheduler.DAGScheduler: ResultStage 139 (collectAsMap at ExtFeatureRC.scala:56) finished in 0.164 s
18/05/06 18:50:04 INFO scheduler.DAGScheduler: Job 13 finished: collectAsMap at ExtFeatureRC.scala:56, took 0.201497 s
18/05/06 18:50:04 INFO storage.BlockManagerInfo: Removed broadcast_21_piece0 on 192.168.201.128:39601 in memory (size: 7.3 KB, free: 363.2 MB)
18/05/06 18:50:04 INFO storage.BlockManagerInfo: Removed broadcast_25_piece0 on 192.168.201.128:39601 in memory (size: 7.1 KB, free: 363.2 MB)
18/05/06 18:50:04 INFO storage.BlockManagerInfo: Removed broadcast_12_piece0 on 192.168.201.128:39601 in memory (size: 5.6 KB, free: 363.2 MB)
18/05/06 18:50:04 INFO storage.BlockManagerInfo: Removed broadcast_26_piece0 on 192.168.201.128:39601 in memory (size: 7.4 KB, free: 363.2 MB)
18/05/06 18:50:04 INFO storage.BlockManagerInfo: Removed broadcast_24_piece0 on 192.168.201.128:39601 in memory (size: 7.4 KB, free: 363.2 MB)
18/05/06 18:50:04 INFO storage.BlockManagerInfo: Removed broadcast_13_piece0 on 192.168.201.128:39601 in memory (size: 5.9 KB, free: 363.2 MB)
18/05/06 18:50:04 INFO storage.BlockManagerInfo: Removed broadcast_11_piece0 on 192.168.201.128:39601 in memory (size: 5.4 KB, free: 363.2 MB)
18/05/06 18:50:04 INFO storage.BlockManagerInfo: Removed broadcast_15_piece0 on 192.168.201.128:39601 in memory (size: 6.3 KB, free: 363.2 MB)
18/05/06 18:50:04 INFO storage.BlockManagerInfo: Removed broadcast_19_piece0 on 192.168.201.128:39601 in memory (size: 7.2 KB, free: 363.2 MB)
18/05/06 18:50:04 INFO storage.BlockManagerInfo: Removed broadcast_23_piece0 on 192.168.201.128:39601 in memory (size: 7.2 KB, free: 363.3 MB)
18/05/06 18:50:04 INFO storage.BlockManagerInfo: Removed broadcast_18_piece0 on 192.168.201.128:39601 in memory (size: 6.9 KB, free: 363.3 MB)
18/05/06 18:50:04 INFO storage.BlockManagerInfo: Removed broadcast_27_piece0 on 192.168.201.128:39601 in memory (size: 7.5 KB, free: 363.3 MB)
18/05/06 18:50:04 INFO storage.BlockManagerInfo: Removed broadcast_16_piece0 on 192.168.201.128:39601 in memory (size: 6.4 KB, free: 363.3 MB)
18/05/06 18:50:04 INFO storage.BlockManagerInfo: Removed broadcast_22_piece0 on 192.168.201.128:39601 in memory (size: 7.1 KB, free: 363.3 MB)
18/05/06 18:50:04 INFO storage.BlockManagerInfo: Removed broadcast_9_piece0 on 192.168.201.128:39601 in memory (size: 4.2 KB, free: 363.3 MB)
18/05/06 18:50:04 INFO storage.BlockManagerInfo: Removed broadcast_10_piece0 on 192.168.201.128:39601 in memory (size: 5.1 KB, free: 363.3 MB)
18/05/06 18:50:04 INFO storage.BlockManagerInfo: Removed broadcast_14_piece0 on 192.168.201.128:39601 in memory (size: 6.0 KB, free: 363.3 MB)
18/05/06 18:50:04 INFO storage.BlockManagerInfo: Removed broadcast_20_piece0 on 192.168.201.128:39601 in memory (size: 7.0 KB, free: 363.3 MB)
18/05/06 18:50:04 INFO storage.BlockManagerInfo: Removed broadcast_17_piece0 on 192.168.201.128:39601 in memory (size: 6.7 KB, free: 363.3 MB)
18/05/06 18:50:04 INFO spark.SparkContext: Starting job: lookup at ExtFeatureRC.scala:70
18/05/06 18:50:04 INFO scheduler.DAGScheduler: Got job 14 (lookup at ExtFeatureRC.scala:70) with 2 output partitions
18/05/06 18:50:04 INFO scheduler.DAGScheduler: Final stage: ResultStage 140 (lookup at ExtFeatureRC.scala:70)
18/05/06 18:50:04 INFO scheduler.DAGScheduler: Parents of final stage: List()
18/05/06 18:50:04 INFO scheduler.DAGScheduler: Missing parents: List()
18/05/06 18:50:04 INFO scheduler.DAGScheduler: Submitting ResultStage 140 (MapPartitionsRDD[129] at lookup at ExtFeatureRC.scala:70), which has no missing parents
18/05/06 18:50:04 INFO memory.MemoryStore: Block broadcast_30 stored as values in memory (estimated size 4.4 KB, free 362.8 MB)
18/05/06 18:50:04 INFO memory.MemoryStore: Block broadcast_30_piece0 stored as bytes in memory (estimated size 2.4 KB, free 362.8 MB)
18/05/06 18:50:04 INFO storage.BlockManagerInfo: Added broadcast_30_piece0 in memory on 192.168.201.128:39601 (size: 2.4 KB, free: 363.3 MB)
18/05/06 18:50:04 INFO spark.SparkContext: Created broadcast 30 from broadcast at DAGScheduler.scala:1006
18/05/06 18:50:04 INFO scheduler.DAGScheduler: Submitting 2 missing tasks from ResultStage 140 (MapPartitionsRDD[129] at lookup at ExtFeatureRC.scala:70) (first 15 tasks are for partitions Vector(0, 1))
18/05/06 18:50:04 INFO scheduler.TaskSchedulerImpl: Adding task set 140.0 with 2 tasks
18/05/06 18:50:04 INFO scheduler.TaskSetManager: Starting task 0.0 in stage 140.0 (TID 49, localhost, executor driver, partition 0, ANY, 4857 bytes)
18/05/06 18:50:04 INFO scheduler.TaskSetManager: Starting task 1.0 in stage 140.0 (TID 50, localhost, executor driver, partition 1, ANY, 4857 bytes)
18/05/06 18:50:04 INFO executor.Executor: Running task 0.0 in stage 140.0 (TID 49)
18/05/06 18:50:04 INFO executor.Executor: Running task 1.0 in stage 140.0 (TID 50)
18/05/06 18:50:04 INFO rdd.HadoopRDD: Input split: hdfs://master:9000/user/ml-100k/u.data:0+989586
18/05/06 18:50:04 INFO rdd.HadoopRDD: Input split: hdfs://master:9000/user/ml-100k/u.data:989586+989587
18/05/06 18:50:04 INFO executor.Executor: Finished task 1.0 in stage 140.0 (TID 50). 1675 bytes result sent to driver
18/05/06 18:50:04 INFO scheduler.TaskSetManager: Finished task 1.0 in stage 140.0 (TID 50) in 370 ms on localhost (executor driver) (1/2)
18/05/06 18:50:05 INFO executor.Executor: Finished task 0.0 in stage 140.0 (TID 49). 822 bytes result sent to driver
18/05/06 18:50:05 INFO scheduler.TaskSetManager: Finished task 0.0 in stage 140.0 (TID 49) in 453 ms on localhost (executor driver) (2/2)
18/05/06 18:50:05 INFO scheduler.TaskSchedulerImpl: Removed TaskSet 140.0, whose tasks have all completed, from pool 
18/05/06 18:50:05 INFO scheduler.DAGScheduler: ResultStage 140 (lookup at ExtFeatureRC.scala:70) finished in 0.451 s
18/05/06 18:50:05 INFO scheduler.DAGScheduler: Job 14 finished: lookup at ExtFeatureRC.scala:70, took 0.466367 s
33
(Godfather, The (1972),5.0)
(Trainspotting (1996),5.0)
(Dead Man Walking (1995),5.0)
(Star Wars (1977),5.0)
(Swingers (1996),5.0)
(Leaving Las Vegas (1995),5.0)
(Bound (1996),5.0)
(Fargo (1996),5.0)
(Last Supper, The (1995),5.0)
(Private Parts (1997),4.0)
(In the Line of Duty 2 (1987),7.988870096266704)
(Denise Calls Up (1995),7.139100057786628)
(Faster Pussycat! Kill! Kill! (1965),7.089206512169639)
(Hurricane Streets (1998),7.070558913016638)
(8 Heads in a Duffel Bag (1997),7.025680340760473)
(Angel Baby (1995),7.021651958848989)
(Loaded (1994),6.688334395551978)
(Hearts and Minds (1996),6.651602999961195)
(Brother Minister: The Assassination of Malcolm X (1994),6.642213102693804)
(Top Hat (1935),6.622753623010645)
18/05/06 18:50:05 INFO spark.SparkContext: Starting job: lookup at MatrixFactorizationModel.scala:81
18/05/06 18:50:05 INFO scheduler.DAGScheduler: Got job 15 (lookup at MatrixFactorizationModel.scala:81) with 1 output partitions
18/05/06 18:50:05 INFO scheduler.DAGScheduler: Final stage: ResultStage 155 (lookup at MatrixFactorizationModel.scala:81)
18/05/06 18:50:05 INFO scheduler.DAGScheduler: Parents of final stage: List(ShuffleMapStage 154, ShuffleMapStage 142)
18/05/06 18:50:05 INFO scheduler.DAGScheduler: Missing parents: List()
18/05/06 18:50:05 INFO scheduler.DAGScheduler: Submitting ResultStage 155 (users MapPartitionsRDD[119] at mapValues at ALS.scala:271), which has no missing parents
18/05/06 18:50:05 INFO memory.MemoryStore: Block broadcast_31 stored as values in memory (estimated size 20.1 KB, free 362.8 MB)
18/05/06 18:50:05 INFO memory.MemoryStore: Block broadcast_31_piece0 stored as bytes in memory (estimated size 7.4 KB, free 362.8 MB)
18/05/06 18:50:05 INFO storage.BlockManagerInfo: Added broadcast_31_piece0 in memory on 192.168.201.128:39601 (size: 7.4 KB, free: 363.3 MB)
18/05/06 18:50:05 INFO spark.SparkContext: Created broadcast 31 from broadcast at DAGScheduler.scala:1006
18/05/06 18:50:05 INFO scheduler.DAGScheduler: Submitting 1 missing tasks from ResultStage 155 (users MapPartitionsRDD[119] at mapValues at ALS.scala:271) (first 15 tasks are for partitions Vector(1))
18/05/06 18:50:05 INFO scheduler.TaskSchedulerImpl: Adding task set 155.0 with 1 tasks
18/05/06 18:50:05 INFO scheduler.TaskSetManager: Starting task 0.0 in stage 155.0 (TID 51, localhost, executor driver, partition 1, PROCESS_LOCAL, 4911 bytes)
18/05/06 18:50:05 INFO executor.Executor: Running task 0.0 in stage 155.0 (TID 51)
18/05/06 18:50:05 INFO storage.BlockManager: Found block rdd_119_1 locally
18/05/06 18:50:05 INFO executor.Executor: Finished task 0.0 in stage 155.0 (TID 51). 985 bytes result sent to driver
18/05/06 18:50:05 INFO scheduler.TaskSetManager: Finished task 0.0 in stage 155.0 (TID 51) in 14 ms on localhost (executor driver) (1/1)
18/05/06 18:50:05 INFO scheduler.TaskSchedulerImpl: Removed TaskSet 155.0, whose tasks have all completed, from pool 
18/05/06 18:50:05 INFO scheduler.DAGScheduler: ResultStage 155 (lookup at MatrixFactorizationModel.scala:81) finished in 0.001 s
18/05/06 18:50:05 INFO scheduler.DAGScheduler: Job 15 finished: lookup at MatrixFactorizationModel.scala:81, took 0.054156 s
18/05/06 18:50:05 INFO spark.SparkContext: Starting job: lookup at MatrixFactorizationModel.scala:82
18/05/06 18:50:05 INFO scheduler.DAGScheduler: Got job 16 (lookup at MatrixFactorizationModel.scala:82) with 1 output partitions
18/05/06 18:50:05 INFO scheduler.DAGScheduler: Final stage: ResultStage 169 (lookup at MatrixFactorizationModel.scala:82)
18/05/06 18:50:05 INFO scheduler.DAGScheduler: Parents of final stage: List(ShuffleMapStage 168, ShuffleMapStage 157)
18/05/06 18:50:05 INFO scheduler.DAGScheduler: Missing parents: List()
18/05/06 18:50:05 INFO scheduler.DAGScheduler: Submitting ResultStage 169 (products MapPartitionsRDD[120] at mapValues at ALS.scala:275), which has no missing parents
18/05/06 18:50:05 INFO memory.MemoryStore: Block broadcast_32 stored as values in memory (estimated size 19.2 KB, free 362.8 MB)
18/05/06 18:50:05 INFO memory.MemoryStore: Block broadcast_32_piece0 stored as bytes in memory (estimated size 7.1 KB, free 362.8 MB)
18/05/06 18:50:05 INFO storage.BlockManagerInfo: Added broadcast_32_piece0 in memory on 192.168.201.128:39601 (size: 7.1 KB, free: 363.3 MB)
18/05/06 18:50:05 INFO spark.SparkContext: Created broadcast 32 from broadcast at DAGScheduler.scala:1006
18/05/06 18:50:05 INFO scheduler.DAGScheduler: Submitting 1 missing tasks from ResultStage 169 (products MapPartitionsRDD[120] at mapValues at ALS.scala:275) (first 15 tasks are for partitions Vector(0))
18/05/06 18:50:05 INFO scheduler.TaskSchedulerImpl: Adding task set 169.0 with 1 tasks
18/05/06 18:50:05 INFO scheduler.TaskSetManager: Starting task 0.0 in stage 169.0 (TID 52, localhost, executor driver, partition 0, PROCESS_LOCAL, 4911 bytes)
18/05/06 18:50:05 INFO executor.Executor: Running task 0.0 in stage 169.0 (TID 52)
18/05/06 18:50:05 INFO storage.BlockManager: Found block rdd_120_0 locally
18/05/06 18:50:05 INFO executor.Executor: Finished task 0.0 in stage 169.0 (TID 52). 985 bytes result sent to driver
18/05/06 18:50:05 INFO scheduler.TaskSetManager: Finished task 0.0 in stage 169.0 (TID 52) in 14 ms on localhost (executor driver) (1/1)
18/05/06 18:50:05 INFO scheduler.TaskSchedulerImpl: Removed TaskSet 169.0, whose tasks have all completed, from pool 
18/05/06 18:50:05 INFO scheduler.DAGScheduler: ResultStage 169 (lookup at MatrixFactorizationModel.scala:82) finished in 0.007 s
18/05/06 18:50:05 INFO scheduler.DAGScheduler: Job 16 finished: lookup at MatrixFactorizationModel.scala:82, took 0.029319 s
18/05/06 18:50:05 INFO spark.SparkContext: Starting job: aggregate at MatrixFactorizationModel.scala:96
18/05/06 18:50:05 INFO scheduler.DAGScheduler: Got job 17 (aggregate at MatrixFactorizationModel.scala:96) with 2 output partitions
18/05/06 18:50:05 INFO scheduler.DAGScheduler: Final stage: ResultStage 170 (aggregate at MatrixFactorizationModel.scala:96)
18/05/06 18:50:05 INFO scheduler.DAGScheduler: Parents of final stage: List()
18/05/06 18:50:05 INFO scheduler.DAGScheduler: Missing parents: List()
18/05/06 18:50:05 INFO scheduler.DAGScheduler: Submitting ResultStage 170 (MapPartitionsRDD[130] at map at ExtFeatureRC.scala:104), which has no missing parents
18/05/06 18:50:05 INFO memory.MemoryStore: Block broadcast_33 stored as values in memory (estimated size 4.1 KB, free 362.8 MB)
18/05/06 18:50:05 INFO memory.MemoryStore: Block broadcast_33_piece0 stored as bytes in memory (estimated size 2.4 KB, free 362.8 MB)
18/05/06 18:50:05 INFO storage.BlockManagerInfo: Added broadcast_33_piece0 in memory on 192.168.201.128:39601 (size: 2.4 KB, free: 363.3 MB)
18/05/06 18:50:05 INFO spark.SparkContext: Created broadcast 33 from broadcast at DAGScheduler.scala:1006
18/05/06 18:50:05 INFO scheduler.DAGScheduler: Submitting 2 missing tasks from ResultStage 170 (MapPartitionsRDD[130] at map at ExtFeatureRC.scala:104) (first 15 tasks are for partitions Vector(0, 1))
18/05/06 18:50:05 INFO scheduler.TaskSchedulerImpl: Adding task set 170.0 with 2 tasks
18/05/06 18:50:05 INFO scheduler.TaskSetManager: Starting task 0.0 in stage 170.0 (TID 53, localhost, executor driver, partition 0, ANY, 4857 bytes)
18/05/06 18:50:05 INFO scheduler.TaskSetManager: Starting task 1.0 in stage 170.0 (TID 54, localhost, executor driver, partition 1, ANY, 4857 bytes)
18/05/06 18:50:05 INFO executor.Executor: Running task 0.0 in stage 170.0 (TID 53)
18/05/06 18:50:05 INFO executor.Executor: Running task 1.0 in stage 170.0 (TID 54)
18/05/06 18:50:05 INFO rdd.HadoopRDD: Input split: hdfs://master:9000/user/ml-100k/u.data:0+989586
18/05/06 18:50:05 INFO rdd.HadoopRDD: Input split: hdfs://master:9000/user/ml-100k/u.data:989586+989587
18/05/06 18:50:05 INFO storage.BlockManagerInfo: Removed broadcast_31_piece0 on 192.168.201.128:39601 in memory (size: 7.4 KB, free: 363.3 MB)
18/05/06 18:50:05 INFO storage.BlockManagerInfo: Removed broadcast_29_piece0 on 192.168.201.128:39601 in memory (size: 2043.0 B, free: 363.3 MB)
18/05/06 18:50:05 INFO storage.BlockManagerInfo: Removed broadcast_30_piece0 on 192.168.201.128:39601 in memory (size: 2.4 KB, free: 363.3 MB)
18/05/06 18:50:05 INFO storage.BlockManagerInfo: Removed broadcast_32_piece0 on 192.168.201.128:39601 in memory (size: 7.1 KB, free: 363.3 MB)
18/05/06 18:50:05 INFO executor.Executor: Finished task 1.0 in stage 170.0 (TID 54). 1011 bytes result sent to driver
18/05/06 18:50:05 INFO scheduler.TaskSetManager: Finished task 1.0 in stage 170.0 (TID 54) in 412 ms on localhost (executor driver) (1/2)
18/05/06 18:50:05 INFO executor.Executor: Finished task 0.0 in stage 170.0 (TID 53). 1011 bytes result sent to driver
18/05/06 18:50:05 INFO scheduler.TaskSetManager: Finished task 0.0 in stage 170.0 (TID 53) in 502 ms on localhost (executor driver) (2/2)
18/05/06 18:50:05 INFO scheduler.TaskSchedulerImpl: Removed TaskSet 170.0, whose tasks have all completed, from pool 
18/05/06 18:50:05 INFO scheduler.DAGScheduler: ResultStage 170 (aggregate at MatrixFactorizationModel.scala:96) finished in 0.485 s
18/05/06 18:50:05 INFO scheduler.DAGScheduler: Job 17 finished: aggregate at MatrixFactorizationModel.scala:96, took 0.516152 s
18/05/06 18:50:05 INFO spark.SparkContext: Starting job: reduce at ExtFeatureRC.scala:117
18/05/06 18:50:05 INFO scheduler.DAGScheduler: Registering RDD 140 (map at ExtFeatureRC.scala:110)
18/05/06 18:50:05 INFO scheduler.DAGScheduler: Registering RDD 130 (map at ExtFeatureRC.scala:104)
18/05/06 18:50:05 INFO scheduler.DAGScheduler: Registering RDD 134 (map at MatrixFactorizationModel.scala:129)
18/05/06 18:50:05 INFO scheduler.DAGScheduler: Registering RDD 139 (map at ExtFeatureRC.scala:105)
18/05/06 18:50:05 INFO scheduler.DAGScheduler: Got job 18 (reduce at ExtFeatureRC.scala:117) with 2 output partitions
18/05/06 18:50:05 INFO scheduler.DAGScheduler: Final stage: ResultStage 189 (reduce at ExtFeatureRC.scala:117)
18/05/06 18:50:05 INFO scheduler.DAGScheduler: Parents of final stage: List(ShuffleMapStage 171, ShuffleMapStage 188)
18/05/06 18:50:05 INFO scheduler.DAGScheduler: Missing parents: List(ShuffleMapStage 171, ShuffleMapStage 188)
18/05/06 18:50:05 INFO scheduler.DAGScheduler: Submitting ShuffleMapStage 171 (MapPartitionsRDD[140] at map at ExtFeatureRC.scala:110), which has no missing parents
18/05/06 18:50:05 INFO memory.MemoryStore: Block broadcast_34 stored as values in memory (estimated size 4.3 KB, free 362.8 MB)
18/05/06 18:50:05 INFO memory.MemoryStore: Block broadcast_34_piece0 stored as bytes in memory (estimated size 2.5 KB, free 362.8 MB)
18/05/06 18:50:05 INFO storage.BlockManagerInfo: Added broadcast_34_piece0 in memory on 192.168.201.128:39601 (size: 2.5 KB, free: 363.3 MB)
18/05/06 18:50:05 INFO spark.SparkContext: Created broadcast 34 from broadcast at DAGScheduler.scala:1006
18/05/06 18:50:05 INFO scheduler.DAGScheduler: Submitting 2 missing tasks from ShuffleMapStage 171 (MapPartitionsRDD[140] at map at ExtFeatureRC.scala:110) (first 15 tasks are for partitions Vector(0, 1))
18/05/06 18:50:05 INFO scheduler.TaskSchedulerImpl: Adding task set 171.0 with 2 tasks
18/05/06 18:50:05 INFO scheduler.TaskSetManager: Starting task 0.0 in stage 171.0 (TID 55, localhost, executor driver, partition 0, ANY, 4846 bytes)
18/05/06 18:50:05 INFO scheduler.TaskSetManager: Starting task 1.0 in stage 171.0 (TID 56, localhost, executor driver, partition 1, ANY, 4846 bytes)
18/05/06 18:50:05 INFO executor.Executor: Running task 0.0 in stage 171.0 (TID 55)
18/05/06 18:50:05 INFO executor.Executor: Running task 1.0 in stage 171.0 (TID 56)
18/05/06 18:50:05 INFO rdd.HadoopRDD: Input split: hdfs://master:9000/user/ml-100k/u.data:0+989586
18/05/06 18:50:05 INFO rdd.HadoopRDD: Input split: hdfs://master:9000/user/ml-100k/u.data:989586+989587
18/05/06 18:50:05 INFO scheduler.DAGScheduler: Submitting ShuffleMapStage 185 (MapPartitionsRDD[130] at map at ExtFeatureRC.scala:104), which has no missing parents
18/05/06 18:50:05 INFO memory.MemoryStore: Block broadcast_35 stored as values in memory (estimated size 4.3 KB, free 362.8 MB)
18/05/06 18:50:05 INFO memory.MemoryStore: Block broadcast_35_piece0 stored as bytes in memory (estimated size 2.5 KB, free 362.8 MB)
18/05/06 18:50:05 INFO storage.BlockManagerInfo: Added broadcast_35_piece0 in memory on 192.168.201.128:39601 (size: 2.5 KB, free: 363.3 MB)
18/05/06 18:50:05 INFO spark.SparkContext: Created broadcast 35 from broadcast at DAGScheduler.scala:1006
18/05/06 18:50:05 INFO scheduler.DAGScheduler: Submitting 2 missing tasks from ShuffleMapStage 185 (MapPartitionsRDD[130] at map at ExtFeatureRC.scala:104) (first 15 tasks are for partitions Vector(0, 1))
18/05/06 18:50:05 INFO scheduler.TaskSchedulerImpl: Adding task set 185.0 with 2 tasks
18/05/06 18:50:06 INFO storage.BlockManagerInfo: Removed broadcast_33_piece0 on 192.168.201.128:39601 in memory (size: 2.4 KB, free: 363.3 MB)
18/05/06 18:50:06 INFO executor.Executor: Finished task 1.0 in stage 171.0 (TID 56). 1071 bytes result sent to driver
18/05/06 18:50:06 INFO scheduler.TaskSetManager: Starting task 0.0 in stage 185.0 (TID 57, localhost, executor driver, partition 0, ANY, 4846 bytes)
18/05/06 18:50:06 INFO scheduler.TaskSetManager: Finished task 1.0 in stage 171.0 (TID 56) in 728 ms on localhost (executor driver) (1/2)
18/05/06 18:50:06 INFO executor.Executor: Running task 0.0 in stage 185.0 (TID 57)
18/05/06 18:50:06 INFO rdd.HadoopRDD: Input split: hdfs://master:9000/user/ml-100k/u.data:0+989586
18/05/06 18:50:06 INFO executor.Executor: Finished task 0.0 in stage 171.0 (TID 55). 1028 bytes result sent to driver
18/05/06 18:50:06 INFO scheduler.TaskSetManager: Starting task 1.0 in stage 185.0 (TID 58, localhost, executor driver, partition 1, ANY, 4846 bytes)
18/05/06 18:50:06 INFO scheduler.TaskSetManager: Finished task 0.0 in stage 171.0 (TID 55) in 911 ms on localhost (executor driver) (2/2)
18/05/06 18:50:06 INFO scheduler.TaskSchedulerImpl: Removed TaskSet 171.0, whose tasks have all completed, from pool 
18/05/06 18:50:06 INFO executor.Executor: Running task 1.0 in stage 185.0 (TID 58)
18/05/06 18:50:06 INFO scheduler.DAGScheduler: ShuffleMapStage 171 (map at ExtFeatureRC.scala:110) finished in 0.880 s
18/05/06 18:50:06 INFO scheduler.DAGScheduler: looking for newly runnable stages
18/05/06 18:50:06 INFO scheduler.DAGScheduler: running: Set(ShuffleMapStage 185)
18/05/06 18:50:06 INFO scheduler.DAGScheduler: waiting: Set(ShuffleMapStage 187, ShuffleMapStage 188, ResultStage 189)
18/05/06 18:50:06 INFO scheduler.DAGScheduler: failed: Set()
18/05/06 18:50:06 INFO rdd.HadoopRDD: Input split: hdfs://master:9000/user/ml-100k/u.data:989586+989587
18/05/06 18:50:07 INFO executor.Executor: Finished task 0.0 in stage 185.0 (TID 57). 985 bytes result sent to driver
18/05/06 18:50:07 INFO scheduler.TaskSetManager: Finished task 0.0 in stage 185.0 (TID 57) in 851 ms on localhost (executor driver) (1/2)
18/05/06 18:50:07 INFO executor.Executor: Finished task 1.0 in stage 185.0 (TID 58). 985 bytes result sent to driver
18/05/06 18:50:07 INFO scheduler.TaskSetManager: Finished task 1.0 in stage 185.0 (TID 58) in 804 ms on localhost (executor driver) (2/2)
18/05/06 18:50:07 INFO scheduler.TaskSchedulerImpl: Removed TaskSet 185.0, whose tasks have all completed, from pool 
18/05/06 18:50:07 INFO scheduler.DAGScheduler: ShuffleMapStage 185 (map at ExtFeatureRC.scala:104) finished in 1.601 s
18/05/06 18:50:07 INFO scheduler.DAGScheduler: looking for newly runnable stages
18/05/06 18:50:07 INFO scheduler.DAGScheduler: running: Set()
18/05/06 18:50:07 INFO scheduler.DAGScheduler: waiting: Set(ShuffleMapStage 187, ShuffleMapStage 188, ResultStage 189)
18/05/06 18:50:07 INFO scheduler.DAGScheduler: failed: Set()
18/05/06 18:50:07 INFO scheduler.DAGScheduler: Submitting ShuffleMapStage 187 (MapPartitionsRDD[134] at map at MatrixFactorizationModel.scala:129), which has no missing parents
18/05/06 18:50:07 INFO memory.MemoryStore: Block broadcast_36 stored as values in memory (estimated size 20.2 KB, free 362.8 MB)
18/05/06 18:50:07 INFO memory.MemoryStore: Block broadcast_36_piece0 stored as bytes in memory (estimated size 7.3 KB, free 362.8 MB)
18/05/06 18:50:07 INFO storage.BlockManagerInfo: Added broadcast_36_piece0 in memory on 192.168.201.128:39601 (size: 7.3 KB, free: 363.3 MB)
18/05/06 18:50:07 INFO spark.SparkContext: Created broadcast 36 from broadcast at DAGScheduler.scala:1006
18/05/06 18:50:07 INFO scheduler.DAGScheduler: Submitting 2 missing tasks from ShuffleMapStage 187 (MapPartitionsRDD[134] at map at MatrixFactorizationModel.scala:129) (first 15 tasks are for partitions Vector(0, 1))
18/05/06 18:50:07 INFO scheduler.TaskSchedulerImpl: Adding task set 187.0 with 2 tasks
18/05/06 18:50:07 INFO scheduler.TaskSetManager: Starting task 0.0 in stage 187.0 (TID 59, localhost, executor driver, partition 0, PROCESS_LOCAL, 4938 bytes)
18/05/06 18:50:07 INFO scheduler.TaskSetManager: Starting task 1.0 in stage 187.0 (TID 60, localhost, executor driver, partition 1, PROCESS_LOCAL, 4938 bytes)
18/05/06 18:50:07 INFO executor.Executor: Running task 1.0 in stage 187.0 (TID 60)
18/05/06 18:50:07 INFO executor.Executor: Running task 0.0 in stage 187.0 (TID 59)
18/05/06 18:50:07 INFO storage.BlockManager: Found block rdd_119_1 locally
18/05/06 18:50:07 INFO storage.ShuffleBlockFetcherIterator: Getting 2 non-empty blocks out of 2 blocks
18/05/06 18:50:07 INFO storage.ShuffleBlockFetcherIterator: Started 0 remote fetches in 1 ms
18/05/06 18:50:07 INFO storage.BlockManager: Found block rdd_119_0 locally
18/05/06 18:50:07 INFO storage.ShuffleBlockFetcherIterator: Getting 2 non-empty blocks out of 2 blocks
18/05/06 18:50:07 INFO storage.ShuffleBlockFetcherIterator: Started 0 remote fetches in 1 ms
18/05/06 18:50:08 INFO storage.BlockManagerInfo: Removed broadcast_34_piece0 on 192.168.201.128:39601 in memory (size: 2.5 KB, free: 363.3 MB)
18/05/06 18:50:08 INFO storage.BlockManagerInfo: Removed broadcast_35_piece0 on 192.168.201.128:39601 in memory (size: 2.5 KB, free: 363.3 MB)
18/05/06 18:50:08 INFO executor.Executor: Finished task 1.0 in stage 187.0 (TID 60). 1415 bytes result sent to driver
18/05/06 18:50:08 INFO scheduler.TaskSetManager: Finished task 1.0 in stage 187.0 (TID 60) in 1376 ms on localhost (executor driver) (1/2)
18/05/06 18:50:09 INFO executor.Executor: Finished task 0.0 in stage 187.0 (TID 59). 1415 bytes result sent to driver
18/05/06 18:50:09 INFO scheduler.TaskSetManager: Finished task 0.0 in stage 187.0 (TID 59) in 1479 ms on localhost (executor driver) (2/2)
18/05/06 18:50:09 INFO scheduler.TaskSchedulerImpl: Removed TaskSet 187.0, whose tasks have all completed, from pool 
18/05/06 18:50:09 INFO scheduler.DAGScheduler: ShuffleMapStage 187 (map at MatrixFactorizationModel.scala:129) finished in 1.453 s
18/05/06 18:50:09 INFO scheduler.DAGScheduler: looking for newly runnable stages
18/05/06 18:50:09 INFO scheduler.DAGScheduler: running: Set()
18/05/06 18:50:09 INFO scheduler.DAGScheduler: waiting: Set(ShuffleMapStage 188, ResultStage 189)
18/05/06 18:50:09 INFO scheduler.DAGScheduler: failed: Set()
18/05/06 18:50:09 INFO scheduler.DAGScheduler: Submitting ShuffleMapStage 188 (MapPartitionsRDD[139] at map at ExtFeatureRC.scala:105), which has no missing parents
18/05/06 18:50:09 INFO memory.MemoryStore: Block broadcast_37 stored as values in memory (estimated size 19.4 KB, free 362.8 MB)
18/05/06 18:50:09 INFO memory.MemoryStore: Block broadcast_37_piece0 stored as bytes in memory (estimated size 7.1 KB, free 362.8 MB)
18/05/06 18:50:09 INFO storage.BlockManagerInfo: Added broadcast_37_piece0 in memory on 192.168.201.128:39601 (size: 7.1 KB, free: 363.3 MB)
18/05/06 18:50:09 INFO spark.SparkContext: Created broadcast 37 from broadcast at DAGScheduler.scala:1006
18/05/06 18:50:09 INFO scheduler.DAGScheduler: Submitting 2 missing tasks from ShuffleMapStage 188 (MapPartitionsRDD[139] at map at ExtFeatureRC.scala:105) (first 15 tasks are for partitions Vector(0, 1))
18/05/06 18:50:09 INFO scheduler.TaskSchedulerImpl: Adding task set 188.0 with 2 tasks
18/05/06 18:50:09 INFO scheduler.TaskSetManager: Starting task 0.0 in stage 188.0 (TID 61, localhost, executor driver, partition 0, PROCESS_LOCAL, 4938 bytes)
18/05/06 18:50:09 INFO scheduler.TaskSetManager: Starting task 1.0 in stage 188.0 (TID 62, localhost, executor driver, partition 1, PROCESS_LOCAL, 4938 bytes)
18/05/06 18:50:09 INFO executor.Executor: Running task 1.0 in stage 188.0 (TID 62)
18/05/06 18:50:09 INFO executor.Executor: Running task 0.0 in stage 188.0 (TID 61)
18/05/06 18:50:09 INFO storage.ShuffleBlockFetcherIterator: Getting 2 non-empty blocks out of 2 blocks
18/05/06 18:50:09 INFO storage.ShuffleBlockFetcherIterator: Started 0 remote fetches in 10 ms
18/05/06 18:50:09 INFO storage.BlockManager: Found block rdd_120_1 locally
18/05/06 18:50:09 INFO storage.ShuffleBlockFetcherIterator: Getting 2 non-empty blocks out of 2 blocks
18/05/06 18:50:09 INFO storage.ShuffleBlockFetcherIterator: Started 0 remote fetches in 4 ms
18/05/06 18:50:09 INFO storage.BlockManager: Found block rdd_120_0 locally
18/05/06 18:50:10 INFO executor.Executor: Finished task 1.0 in stage 188.0 (TID 62). 1372 bytes result sent to driver
18/05/06 18:50:10 INFO scheduler.TaskSetManager: Finished task 1.0 in stage 188.0 (TID 62) in 944 ms on localhost (executor driver) (1/2)
18/05/06 18:50:10 INFO executor.Executor: Finished task 0.0 in stage 188.0 (TID 61). 1372 bytes result sent to driver
18/05/06 18:50:10 INFO scheduler.TaskSetManager: Finished task 0.0 in stage 188.0 (TID 61) in 1059 ms on localhost (executor driver) (2/2)
18/05/06 18:50:10 INFO scheduler.TaskSchedulerImpl: Removed TaskSet 188.0, whose tasks have all completed, from pool 
18/05/06 18:50:10 INFO scheduler.DAGScheduler: ShuffleMapStage 188 (map at ExtFeatureRC.scala:105) finished in 1.060 s
18/05/06 18:50:10 INFO scheduler.DAGScheduler: looking for newly runnable stages
18/05/06 18:50:10 INFO scheduler.DAGScheduler: running: Set()
18/05/06 18:50:10 INFO scheduler.DAGScheduler: waiting: Set(ResultStage 189)
18/05/06 18:50:10 INFO scheduler.DAGScheduler: failed: Set()
18/05/06 18:50:10 INFO scheduler.DAGScheduler: Submitting ResultStage 189 (MapPartitionsRDD[144] at map at ExtFeatureRC.scala:115), which has no missing parents
18/05/06 18:50:10 INFO memory.MemoryStore: Block broadcast_38 stored as values in memory (estimated size 3.4 KB, free 362.8 MB)
18/05/06 18:50:10 INFO memory.MemoryStore: Block broadcast_38_piece0 stored as bytes in memory (estimated size 1998.0 B, free 362.8 MB)
18/05/06 18:50:10 INFO storage.BlockManagerInfo: Added broadcast_38_piece0 in memory on 192.168.201.128:39601 (size: 1998.0 B, free: 363.3 MB)
18/05/06 18:50:10 INFO spark.SparkContext: Created broadcast 38 from broadcast at DAGScheduler.scala:1006
18/05/06 18:50:10 INFO scheduler.DAGScheduler: Submitting 2 missing tasks from ResultStage 189 (MapPartitionsRDD[144] at map at ExtFeatureRC.scala:115) (first 15 tasks are for partitions Vector(0, 1))
18/05/06 18:50:10 INFO scheduler.TaskSchedulerImpl: Adding task set 189.0 with 2 tasks
18/05/06 18:50:10 INFO scheduler.TaskSetManager: Starting task 0.0 in stage 189.0 (TID 63, localhost, executor driver, partition 0, PROCESS_LOCAL, 4684 bytes)
18/05/06 18:50:10 INFO scheduler.TaskSetManager: Starting task 1.0 in stage 189.0 (TID 64, localhost, executor driver, partition 1, PROCESS_LOCAL, 4684 bytes)
18/05/06 18:50:10 INFO executor.Executor: Running task 1.0 in stage 189.0 (TID 64)
18/05/06 18:50:10 INFO executor.Executor: Running task 0.0 in stage 189.0 (TID 63)
18/05/06 18:50:10 INFO storage.ShuffleBlockFetcherIterator: Getting 2 non-empty blocks out of 2 blocks
18/05/06 18:50:10 INFO storage.ShuffleBlockFetcherIterator: Started 0 remote fetches in 5 ms
18/05/06 18:50:10 INFO storage.ShuffleBlockFetcherIterator: Getting 2 non-empty blocks out of 2 blocks
18/05/06 18:50:10 INFO storage.ShuffleBlockFetcherIterator: Started 0 remote fetches in 4 ms
18/05/06 18:50:10 INFO storage.ShuffleBlockFetcherIterator: Getting 2 non-empty blocks out of 2 blocks
18/05/06 18:50:10 INFO storage.ShuffleBlockFetcherIterator: Started 0 remote fetches in 8 ms
18/05/06 18:50:10 INFO storage.ShuffleBlockFetcherIterator: Getting 2 non-empty blocks out of 2 blocks
18/05/06 18:50:10 INFO storage.ShuffleBlockFetcherIterator: Started 0 remote fetches in 0 ms
18/05/06 18:50:10 INFO storage.BlockManagerInfo: Removed broadcast_36_piece0 on 192.168.201.128:39601 in memory (size: 7.3 KB, free: 363.3 MB)
18/05/06 18:50:10 INFO storage.BlockManagerInfo: Removed broadcast_37_piece0 on 192.168.201.128:39601 in memory (size: 7.1 KB, free: 363.3 MB)
18/05/06 18:50:11 INFO executor.Executor: Finished task 1.0 in stage 189.0 (TID 64). 1257 bytes result sent to driver
18/05/06 18:50:11 INFO scheduler.TaskSetManager: Finished task 1.0 in stage 189.0 (TID 64) in 1139 ms on localhost (executor driver) (1/2)
18/05/06 18:50:11 INFO executor.Executor: Finished task 0.0 in stage 189.0 (TID 63). 1257 bytes result sent to driver
18/05/06 18:50:11 INFO scheduler.TaskSetManager: Finished task 0.0 in stage 189.0 (TID 63) in 1375 ms on localhost (executor driver) (2/2)
18/05/06 18:50:11 INFO scheduler.TaskSchedulerImpl: Removed TaskSet 189.0, whose tasks have all completed, from pool 
18/05/06 18:50:11 INFO scheduler.DAGScheduler: ResultStage 189 (reduce at ExtFeatureRC.scala:117) finished in 1.377 s
18/05/06 18:50:11 INFO scheduler.DAGScheduler: Job 18 finished: reduce at ExtFeatureRC.scala:117, took 5.763970 s
18/05/06 18:50:11 INFO spark.SparkContext: Starting job: count at ExtFeatureRC.scala:117
18/05/06 18:50:11 INFO spark.MapOutputTrackerMaster: Size of output statuses for shuffle 16 is 160 bytes
18/05/06 18:50:11 INFO spark.MapOutputTrackerMaster: Size of output statuses for shuffle 1 is 160 bytes
18/05/06 18:50:11 INFO spark.MapOutputTrackerMaster: Size of output statuses for shuffle 0 is 160 bytes
18/05/06 18:50:11 INFO spark.MapOutputTrackerMaster: Size of output statuses for shuffle 13 is 163 bytes
18/05/06 18:50:11 INFO spark.MapOutputTrackerMaster: Size of output statuses for shuffle 12 is 160 bytes
18/05/06 18:50:11 INFO spark.MapOutputTrackerMaster: Size of output statuses for shuffle 2 is 160 bytes
18/05/06 18:50:11 INFO spark.MapOutputTrackerMaster: Size of output statuses for shuffle 11 is 160 bytes
18/05/06 18:50:11 INFO spark.MapOutputTrackerMaster: Size of output statuses for shuffle 10 is 160 bytes
18/05/06 18:50:11 INFO spark.MapOutputTrackerMaster: Size of output statuses for shuffle 9 is 160 bytes
18/05/06 18:50:11 INFO spark.MapOutputTrackerMaster: Size of output statuses for shuffle 8 is 160 bytes
18/05/06 18:50:11 INFO spark.MapOutputTrackerMaster: Size of output statuses for shuffle 7 is 160 bytes
18/05/06 18:50:11 INFO spark.MapOutputTrackerMaster: Size of output statuses for shuffle 6 is 160 bytes
18/05/06 18:50:11 INFO spark.MapOutputTrackerMaster: Size of output statuses for shuffle 5 is 160 bytes
18/05/06 18:50:11 INFO spark.MapOutputTrackerMaster: Size of output statuses for shuffle 4 is 160 bytes
18/05/06 18:50:11 INFO spark.MapOutputTrackerMaster: Size of output statuses for shuffle 15 is 160 bytes
18/05/06 18:50:11 INFO spark.MapOutputTrackerMaster: Size of output statuses for shuffle 3 is 160 bytes
18/05/06 18:50:11 INFO spark.MapOutputTrackerMaster: Size of output statuses for shuffle 14 is 160 bytes
18/05/06 18:50:11 INFO spark.MapOutputTrackerMaster: Size of output statuses for shuffle 17 is 160 bytes
18/05/06 18:50:11 INFO scheduler.DAGScheduler: Got job 19 (count at ExtFeatureRC.scala:117) with 2 output partitions
18/05/06 18:50:11 INFO scheduler.DAGScheduler: Final stage: ResultStage 208 (count at ExtFeatureRC.scala:117)
18/05/06 18:50:11 INFO scheduler.DAGScheduler: Parents of final stage: List(ShuffleMapStage 190, ShuffleMapStage 207)
18/05/06 18:50:11 INFO scheduler.DAGScheduler: Missing parents: List()
18/05/06 18:50:11 INFO scheduler.DAGScheduler: Submitting ResultStage 208 (MapPartitionsRDD[143] at join at ExtFeatureRC.scala:112), which has no missing parents
18/05/06 18:50:11 INFO memory.MemoryStore: Block broadcast_39 stored as values in memory (estimated size 3.0 KB, free 362.8 MB)
18/05/06 18:50:11 INFO memory.MemoryStore: Block broadcast_39_piece0 stored as bytes in memory (estimated size 1804.0 B, free 362.8 MB)
18/05/06 18:50:11 INFO storage.BlockManagerInfo: Added broadcast_39_piece0 in memory on 192.168.201.128:39601 (size: 1804.0 B, free: 363.3 MB)
18/05/06 18:50:11 INFO spark.SparkContext: Created broadcast 39 from broadcast at DAGScheduler.scala:1006
18/05/06 18:50:11 INFO scheduler.DAGScheduler: Submitting 2 missing tasks from ResultStage 208 (MapPartitionsRDD[143] at join at ExtFeatureRC.scala:112) (first 15 tasks are for partitions Vector(0, 1))
18/05/06 18:50:11 INFO scheduler.TaskSchedulerImpl: Adding task set 208.0 with 2 tasks
18/05/06 18:50:11 INFO scheduler.TaskSetManager: Starting task 0.0 in stage 208.0 (TID 65, localhost, executor driver, partition 0, PROCESS_LOCAL, 4684 bytes)
18/05/06 18:50:11 INFO scheduler.TaskSetManager: Starting task 1.0 in stage 208.0 (TID 66, localhost, executor driver, partition 1, PROCESS_LOCAL, 4684 bytes)
18/05/06 18:50:11 INFO executor.Executor: Running task 0.0 in stage 208.0 (TID 65)
18/05/06 18:50:11 INFO storage.ShuffleBlockFetcherIterator: Getting 2 non-empty blocks out of 2 blocks
18/05/06 18:50:11 INFO executor.Executor: Running task 1.0 in stage 208.0 (TID 66)
18/05/06 18:50:11 INFO storage.ShuffleBlockFetcherIterator: Getting 2 non-empty blocks out of 2 blocks
18/05/06 18:50:11 INFO storage.ShuffleBlockFetcherIterator: Started 0 remote fetches in 1 ms
18/05/06 18:50:11 INFO storage.ShuffleBlockFetcherIterator: Getting 2 non-empty blocks out of 2 blocks
18/05/06 18:50:11 INFO storage.ShuffleBlockFetcherIterator: Started 0 remote fetches in 0 ms
18/05/06 18:50:11 INFO storage.ShuffleBlockFetcherIterator: Started 0 remote fetches in 4 ms
18/05/06 18:50:11 INFO storage.ShuffleBlockFetcherIterator: Getting 2 non-empty blocks out of 2 blocks
18/05/06 18:50:11 INFO storage.ShuffleBlockFetcherIterator: Started 0 remote fetches in 0 ms
18/05/06 18:50:12 INFO executor.Executor: Finished task 0.0 in stage 208.0 (TID 65). 1176 bytes result sent to driver
18/05/06 18:50:12 INFO scheduler.TaskSetManager: Finished task 0.0 in stage 208.0 (TID 65) in 1187 ms on localhost (executor driver) (1/2)
18/05/06 18:50:12 INFO executor.Executor: Finished task 1.0 in stage 208.0 (TID 66). 1176 bytes result sent to driver
18/05/06 18:50:12 INFO scheduler.TaskSetManager: Finished task 1.0 in stage 208.0 (TID 66) in 1216 ms on localhost (executor driver) (2/2)
18/05/06 18:50:12 INFO scheduler.TaskSchedulerImpl: Removed TaskSet 208.0, whose tasks have all completed, from pool 
18/05/06 18:50:12 INFO scheduler.DAGScheduler: ResultStage 208 (count at ExtFeatureRC.scala:117) finished in 1.224 s
18/05/06 18:50:12 INFO scheduler.DAGScheduler: Job 19 finished: count at ExtFeatureRC.scala:117, took 1.329562 s
Mean Squared Error = 0.6465301944614723
18/05/06 18:50:12 INFO server.AbstractConnector: Stopped Spark@23c388c2{HTTP/1.1,[http/1.1]}{0.0.0.0:4040}
18/05/06 18:50:12 INFO ui.SparkUI: Stopped Spark web UI at http://192.168.201.128:4040
18/05/06 18:50:12 INFO spark.MapOutputTrackerMasterEndpoint: MapOutputTrackerMasterEndpoint stopped!
18/05/06 18:50:13 INFO memory.MemoryStore: MemoryStore cleared
18/05/06 18:50:13 INFO storage.BlockManager: BlockManager stopped
18/05/06 18:50:13 INFO storage.BlockManagerMaster: BlockManagerMaster stopped
18/05/06 18:50:13 INFO scheduler.OutputCommitCoordinator$OutputCommitCoordinatorEndpoint: OutputCommitCoordinator stopped!
18/05/06 18:50:13 INFO spark.SparkContext: Successfully stopped SparkContext
18/05/06 18:50:13 INFO util.ShutdownHookManager: Shutdown hook called
18/05/06 18:50:13 INFO util.ShutdownHookManager: Deleting directory /tmp/spark-3e2c979f-9940-4333-9c41-313105080560

猜你喜欢

转载自blog.csdn.net/jh_zhai/article/details/80221758
今日推荐
《美国对全球网络空间安全与发展的威胁和破坏》报告发布
火速冲上 GitHub 热榜 —— 开源编程语言、框架哪有这么可爱？
北京人形机器人创新中心发布全球首个纯电驱拟人奔跑的全尺寸人形机器人“天工”
LFOSSA 源来如此公开课 | 掌握云原生未来：CNCF 认证全面攻略与备考秘籍
周排行
让自己的头脑极度开放
CentOS 6.5(x64) 和Redhat6.5操作系误删libc
高可用注册中心
【日记】12.28/【题解】AtCoder AGC041
XML（5）_XML 约束_DTD
Java集合Map（四）
树梅派安装桌面环境教程
pipenv 的 使用和安装
小程序白屏问题和内存研究
C语言简单选择排序
每日归档
更多
2024-05-02(0)
2024-05-01(4)
2024-04-30(1)
2024-04-29(40)
2024-04-28(0)
2024-04-27(56)
2024-04-26(39)
2024-04-25(22)
2024-04-24(36)
2024-04-23(26)

代码天地 © 2018 codetd.com

境外服务器   最新电视剧2022