Scala利用akka实现spark通信

master

import akka.actor.{Actor, ActorSystem, Props}
import com.typesafe.config.ConfigFactory

import scala.collection.mutable
import scala.collection.mutable.ArrayBuffer
import scala.concurrent.duration._

class spark_master extends Actor {
  //todo  定义一个map，用于保存注册信息，key是workerID，value是spark_workerInfo
  private val workerInfoMap = new mutable.HashMap[String, spark_workerInfo]
  //todo  定义一个ArrayBuffer,用于保存worker信息
  private val workerInfoArray = new ArrayBuffer[spark_workerInfo]()

  println("master启动了")

  //todo  初始化方法，在构造方法之后执行一次
  override def preStart(): Unit = {
    //todo  在初始化方法中完成心跳超时检测，后续间隔指定的时间检测
    import context.dispatcher
    context.system.scheduler.schedule(0 millis, 15000 millis, self, checkTimeOut)
  }

  //todo  receive方法用于消息的接受处,
  override def receive = {
    //todo  用于接收注册信息
    case RegisterMessage(workerID, memory, cores) => {
      //todo  判断该worker是否被注册
      if (!workerInfoMap.contains(workerID)) {
        val workerInfo = new spark_workerInfo(workerID, memory, cores)
        //todo   将workerID,memory,cores信息封装到spark_workerInfo中
        workerInfoMap.put(workerID, workerInfo)
        workerInfoArray += workerInfo
        println(s"注册成功$workerInfo")

        //todo  注册成功给客户端返回注册成功的信息
        sender ! RegistedMessage(s"congratulations you guy $workerID")
      } else {
        sender ! failRegisterMessage("you has been registed")
      }
    }

    case heartBeatMessage(workerID) => {
      //todo  判断是否注册过，如果注册过更新心跳时间
      if (workerInfoMap.contains(workerID)) {
        //todo  获取当前时间
        val nowTime = System.currentTimeMillis()
        //todo 过去worker对象
        val info = workerInfoMap(workerID)
        //todo  将当前心跳时间更新为上次心跳时间
        info.lastHeartTime = nowTime

      }
    }

    case checkTimeOut => {
      //todo  找到超时的worker
      val outOfTime = workerInfoArray.filter(x => System.currentTimeMillis() - x.lastHeartTime > 15000)

      for (w <- outOfTime) {
        val workerId = w.workerID
        workerInfoMap -= workerId
        workerInfoArray -= w
        println(s"小老弟超时了哦...@$workerId ")

      }
      println("活着的worker总数：" + workerInfoArray.size)


    }

  }
}

object spark_master {
  def main(args: Array[String]): Unit = {

    //        akka.actor.provider = "akka.remote.RemoteActorRefProvider"
    //        akka.remote.netty.tcp.hostname = "$host"
    //        akka.remote.netty.tcp.port = "$port"
    //todo  利用参数获取配置，增加代码灵活性
    val host = "192.168.1.4"
    val port = 8888
    val configStr =
      s"""
         |akka.actor.provider = "akka.remote.RemoteActorRefProvider"
         |        akka.remote.netty.tcp.hostname = "$host"
         |        akka.remote.netty.tcp.port = "$port"
      """.stripMargin
    //todo  准备配置文件
    val config = ConfigFactory.parseString(configStr)

    //todo  创建ActorSystem，用于管理众多的actor
    val masterActorSystem = ActorSystem.create("masterActorSystem", config)
    //todo  通过masterActorSystem创建masterActor
    val master = masterActorSystem.actorOf(Props(new spark_master), "masterActor")
  }
}

worker

import akka.actor.{Actor, ActorSelection, ActorSystem, Props}
import com.typesafe.config.ConfigFactory
import scala.concurrent.duration._

//todo  通过构造吧memory,cores传递出来
class spark_worker(val memory: Int, val cores: Int) extends Actor {
  var master: ActorSelection = _
  val workerID = UUID.randomUUID().toString
  override def preStart(): Unit = {
    //todo  在初始化中拿到master引用(协议、host、port、actorSystem、masterActor、actor层级)
    master = context.actorSelection("akka.tcp://[email protected]:8888/user/masterActor")
    //todo  发送注册信息

    master ! RegisterMessage(workerID, memory, cores)

  }

  override def receive = {
    //todo  接收master返回的消息
    case RegistedMessage(msg) => {
      println("伞兵一号卢本伟闪亮登场")

      //todo  注册成功，立即开始首次心跳，以后间隔指定时间进行心跳发射
      /** todo
        * initialDelay: FiniteDuration, todo  首次心跳
        * interval: FiniteDuration,  todo  心跳间隔
        * receiver: ActorRef,  todo  心跳发送目标
        * message: Any  todo   心跳发送内容
        */
      //todo  定时的信息只能发送给自己，心跳定时发送给自己，体系自己开始真正的心跳
      import context.dispatcher
      context.system.scheduler.schedule(0 millis, 10000 millis, self, heartBeat)
    }
    case failRegisterMessage(msg) => println("是吗？没睡好，有点健忘了。嘻嘻00.")
    //todo  接收心跳提醒，发送心跳
    case heartBeat => {
      master ! heartBeatMessage(workerID)

    }
    case RegistedMessage(msg) => println(msg)

  }
}

object spark_worker {

  def main(args: Array[String]): Unit = {
    val host = args(0)
    val port = args(1)
    val memory = 8
    val cores = 2

    //todo  解析字符串配置
    val config = ConfigFactory.parseString(
      s"""akka.actor.provider = "akka.remote.RemoteActorRefProvider"
         |akka.remote.netty.tcp.hostname = "$host"
         |akka.remote.netty.tcp.port = "$port"""".stripMargin)
    //todo  获取workerActorSystem
    val workerActorSystem = ActorSystem.create("workerActorSystem", config)
    //todo  创建workerActor
    val worker = workerActorSystem.actorOf(Props(new spark_worker(memory, cores)), "workerActor")
  }
}

park_message 定义样例类来封装类型参数

 trait spark_message extends Serializable {

}
//todo  定义样例类用于注册成功信息
case class RegistedMessage(msg:String) extends spark_message
//todo  定义样例类用于给客户端返回成功消息worker给master发送
case class RegisterMessage(workerID:String,memory:Int,cores:Int) extends spark_message
//todo  定义样例类用于worker给master发送注册失败信息
case class failRegisterMessage(msg:String) extends spark_message
//todo  定义样例对象，用于worker自己给自己发送心跳提醒
case object heartBeat
//todo  定义样例类，用于worker给master发送心跳信息
case class heartBeatMessage(workerID:String)extends spark_message
//todo  定义样例对象，用于worker自己给自己发送心跳超时检测提醒
case object checkTimeOut

workerInfo 封装worker信息

class spark_workerInfo(var workerID: String, var memory: Int, var ores: Int) {
  //todo  上次心跳时间
  var lastHeartTime :Long = 0

  override def toString = s"spark_workerInfo($workerID)"
}