上一节介绍了master如何处理driver提交的应用程序，找到满足条件的worker之后给worker发消息，进而启动executor，这一节我们来继续看一下worker在收到master发过来的LaunchExecutor消息之后如何调度分配资源。
Worker源码
package org.apache.spark.deploy.worker
import java.io.{File, IOException}
import java.text.SimpleDateFormat
import java.util.{Date, Locale, UUID}
import java.util.concurrent._
import java.util.concurrent.{Future => JFuture, ScheduledFuture => JScheduledFuture}
import java.util.function.Supplier
import scala.collection.mutable.{HashMap, HashSet, LinkedHashMap}
import scala.concurrent.ExecutionContext
import scala.util.Random
import scala.util.control.NonFatal
import org.apache.spark.{SecurityManager, SparkConf}
import org.apache.spark.deploy.{Command, ExecutorDescription, ExecutorState}
import org.apache.spark.deploy.DeployMessages._
import org.apache.spark.deploy.ExternalShuffleService
import org.apache.spark.deploy.StandaloneResourceUtils._
import org.apache.spark.deploy.master.{DriverState, Master}
import org.apache.spark.deploy.worker.ui.WorkerWebUI
import org.apache.spark.internal.{config, Logging}
import org.apache.spark.internal.config.Tests.IS_TESTING
import org.apache.spark.internal.config.UI._
import org.apache.spark.internal.config.Worker._
import org.apache.spark.metrics.{MetricsSystem, MetricsSystemInstances}
import org.apache.spark.resource.ResourceInformation
import org.apache.spark.resource.ResourceUtils._
import org.apache.spark.rpc._
import org.apache.spark.util.{SignalUtils, SparkUncaughtExceptionHandler, ThreadUtils, Utils}
private[deploy] class Worker(
override val rpcEnv: RpcEnv,
webUiPort: Int,
cores: Int,
memory: Int,
masterRpcAddresses: Array[RpcAddress],
endpointName: String,
workDirPath: String = null,
val conf: SparkConf,
val securityMgr: SecurityManager,
resourceFileOpt: Option[String] = None,
externalShuffleServiceSupplier: Supplier[ExternalShuffleService] = null)
extends ThreadSafeRpcEndpoint with Logging {
//获取rpcenv的host
private val host = rpcEnv.address.host
//获取rpcenv的端口
private val port = rpcEnv.address.port
//检查主机名和端口格式是否正确
Utils.checkHost(host)
assert (port > 0)
// If worker decommissioning is enabled register a handler on PWR to shutdown.
//如果启用了worker退役，则在PWR上注册一个处理程序以关闭。
//如果启用了worker退役
if (conf.get(config.DECOMMISSION_ENABLED)) {
//提示正在注册SIGPWR处理程序以触发停用。
logInfo("Registering SIGPWR handler to trigger decommissioning.")
SignalUtils.register("PWR", "Failed to register SIGPWR handler - " +
"disabling worker decommission feature.") {
//发送注册SIGPWR处理程序
self.send(WorkerSigPWRReceived)
true
}
} else {
//提示没有开启worker通过应用程序退役 SIGPWR将会退出
logInfo("Worker decommissioning not enabled, SIGPWR will result in exiting.")
}
// A scheduled executor used to send messages at the specified time.
//用于在指定时间发送消息的计划执行器。
private val forwardMessageScheduler =
ThreadUtils.newDaemonSingleThreadScheduledExecutor("worker-forward-message-scheduler")
// A separated thread to clean up the workDir and the directories of finished applications.
// Used to provide the implicit parameter of `Future` methods.
//一个单独的线程，用于清理workDir和已完成应用程序的目录。用于提供“Future”方法的隐式参数。
private val cleanupThreadExecutor = ExecutionContext.fromExecutorService(
ThreadUtils.newDaemonSingleThreadExecutor("worker-cleanup-thread"))
// For worker and executor IDs
//用于worker和executor创建ID
private def createDateFormat = new SimpleDateFormat("yyyyMMddHHmmss", Locale.US)
// Send a heartbeat every (heartbeat timeout) / 4 milliseconds
//每隔（检测信号超时）/4毫秒发送一次心跳
private val HEARTBEAT_MILLIS = conf.get(WORKER_TIMEOUT) * 1000 / 4
// Model retries to connect to the master, after Hadoop's model.
// The first six attempts to reconnect are in shorter intervals (between 5 and 15 seconds)
// Afterwards, the next 10 attempts are between 30 and 90 seconds.
// A bit of randomness is introduced so that not all of the workers attempt to reconnect at
// the same time.
//模型在Hadoop的模型之后重试连接到Master。
//前六次重新连接的尝试间隔较短（在5到15秒之间）。之后，接下来的10次尝试间隔在30到90秒之间。
//引入了一点随机性，因此并非所有worker都试图在同一时间重新连接。
//开始的注册重试6次
private val INITIAL_REGISTRATION_RETRIES = 6
//总共重试次数
private val TOTAL_REGISTRATION_RETRIES = INITIAL_REGISTRATION_RETRIES + 10
//模糊乘子区间下界
private val FUZZ_MULTIPLIER_INTERVAL_LOWER_BOUND = 0.500
//注册重试模糊乘法器
private val REGISTRATION_RETRY_FUZZ_MULTIPLIER = {
val randomNumberGenerator = new Random(UUID.randomUUID.getMostSignificantBits)
randomNumberGenerator.nextDouble + FUZZ_MULTIPLIER_INTERVAL_LOWER_BOUND
}
//初始注册重试间隔秒
private val INITIAL_REGISTRATION_RETRY_INTERVAL_SECONDS = (math.round(10 *
REGISTRATION_RETRY_FUZZ_MULTIPLIER))
//延长的注册重试间隔秒
private val PROLONGED_REGISTRATION_RETRY_INTERVAL_SECONDS = (math.round(60
* REGISTRATION_RETRY_FUZZ_MULTIPLIER))
//能否清理应用程序的目录
private val CLEANUP_ENABLED = conf.get(WORKER_CLEANUP_ENABLED)
// How often worker will clean up old app folders
//worker清理旧应用文件夹的频率
private val CLEANUP_INTERVAL_MILLIS = conf.get(WORKER_CLEANUP_INTERVAL) * 1000
// TTL for app folders/data; after TTL expires it will be cleaned up
//应用程序文件夹/数据的TTL；TTL到期后，它将被清除
private val APP_DATA_RETENTION_SECONDS = conf.get(APP_DATA_RETENTION)
// Whether or not cleanup the non-shuffle service served files on executor exits.
//是否在executor退出时清除非混洗服务提供的文件。
private val CLEANUP_FILES_AFTER_EXECUTOR_EXIT =
conf.get(config.STORAGE_CLEANUP_FILES_AFTER_EXECUTOR_EXIT)
//master引用
private var master: Option[RpcEndpointRef] = None
/**
* Whether to use the master address in `masterRpcAddresses` if possible. If it's disabled, Worker
* will just use the address received from Master.
*/
//如果可能，是否使用“masterRpcAddresses”中的主地址。如果它被禁用，Worker将只使用从Master收到的地址。
private val preferConfiguredMasterAddress = conf.get(PREFER_CONFIGURED_MASTER_ADDRESS)
/**
* The master address to connect in case of failure. When the connection is broken, worker will
* use this address to connect. This is usually just one of `masterRpcAddresses`. However, when
* a master is restarted or takes over leadership, it will be an address sent from master, which
* may not be in `masterRpcAddresses`.
*/
//发生故障时要连接的主地址。当连接断开时，worker将使用此地址进行连接。这通常只是“masterRpcAddresses”中的一个。
//但是，当master重新启动或接管领导时，它将是master发送的地址，该地址可能不在“masterRpcAddresses”中。
private var masterAddressToConnect: Option[RpcAddress] = None
//活动的master的url
private var activeMasterUrl: String = ""
//活动的master的web url
private[worker] var activeMasterWebUiUrl : String = ""
//worker的web url
private var workerWebUiUrl: String = ""
//worker的uri地址
private val workerUri = RpcEndpointAddress(rpcEnv.address, endpointName).toString
//是否在master中注册
private var registered = false
//是否连接上了master
private var connected = false
//是否退役了
private var decommissioned = false
// expose for test
//暴露测试
//生成worker id
private[spark] val workerId = generateWorkerId()
//创建测试文件
private val sparkHome =
if (sys.props.contains(IS_TESTING.key)) {
//断言 如果已经有则抛出异常
assert(sys.props.contains("spark.test.home"), "spark.test.home is not set!")
//在系统中创建目录文件
new File(sys.props("spark.test.home"))
} else {
//直接在SPARK_HOME下创建
new File(sys.env.getOrElse("SPARK_HOME", "."))
}
//工作文件
var workDir: File = null
//已经结束的exector
val finishedExecutors = new LinkedHashMap[String, ExecutorRunner]
//driver列表
val drivers = new HashMap[String, DriverRunner]
//executor列表
val executors = new HashMap[String, ExecutorRunner]
//已经结束的driver列表
val finishedDrivers = new LinkedHashMap[String, DriverRunner]
//应用程序目录
val appDirectories = new HashMap[String, Seq[String]]
//已经结束的应用程序
val finishedApps = new HashSet[String]
//保留的executor
val retainedExecutors = conf.get(WORKER_UI_RETAINED_EXECUTORS)
//保留的driver
val retainedDrivers = conf.get(WORKER_UI_RETAINED_DRIVERS)
// The shuffle service is not actually started unless configured.
//除非进行了配置，否则shuffle服务实际上不会启动。
private val shuffleService = if (externalShuffleServiceSupplier != null) {
//如果shuffle服务不为空则获取
externalShuffleServiceSupplier.get()
} else {
//创建新的shuffle服务
new ExternalShuffleService(conf, securityMgr)
}
//公开地址
private val publicAddress = {
//获取配置的dns地址
val envVar = conf.getenv("SPARK_PUBLIC_DNS")
//如果dns地址不为空 则取dns地址 否则以host地址为对外的地址
if (envVar != null) envVar else host
}
//web ui地址
private var webUi: WorkerWebUI = null
//尝试连接次数
private var connectionAttemptCount = 0
//metrics系统
private val metricsSystem =
MetricsSystem.createMetricsSystem(MetricsSystemInstances.WORKER, conf, securityMgr)
//worker来源
private val workerSource = new WorkerSource(this)
//反向代理
val reverseProxy = conf.get(UI_REVERSE_PROXY)
//注册master结果
private var registerMasterFutures: Array[JFuture[_]] = null
//注册master重试次数
private var registrationRetryTimer: Option[JScheduledFuture[_]] = None
// A thread pool for registering with masters. Because registering with a master is a blocking
// action, this thread pool must be able to create "masterRpcAddresses.size" threads at the same
// time so that we can register with all masters.
//用于向master注册的线程池。因为向master注册是一个阻塞操作，
//所以这个线程池必须能够同时创建“masterRpcAddresses.size”线程，这样我们才能向所有master注册。
private val registerMasterThreadPool = ThreadUtils.newDaemonCachedThreadPool(
"worker-register-master-threadpool",
masterRpcAddresses.length // Make sure we can register with all masters at the same time
) //确保能在同一时间在所有的Master中注册
// visible for tests
//测试可见
//资源信息
private[deploy] var resources: Map[String, ResourceInformation] = Map.empty
//已经使用的core
var coresUsed = 0
//已经使用的内存
var memoryUsed = 0
//已经使用的资源
val resourcesUsed = new HashMap[String, MutableResourceInfo]()
//空闲的core
def coresFree: Int = cores - coresUsed
//空闲的内存
def memoryFree: Int = memory - memoryUsed
//创建工作目录
private def createWorkDir(): Unit = {
workDir = Option(workDirPath).map(new File(_)).getOrElse(new File(sparkHome, "work"))
if (!Utils.createDirectory(workDir)) {
System.exit(1)
}
}
//启动worker
override def onStart(): Unit = {
//断言 如果没有注册则抛出异常
assert(!registered)
//提示启动worker
logInfo("Starting Spark worker %s:%d with %d cores, %s RAM".format(
host, port, cores, Utils.megabytesToString(memory)))
//提示运行的Spark版本
logInfo(s"Running Spark version ${org.apache.spark.SPARK_VERSION}")
logInfo("Spark home: " + sparkHome)
//创建工作目录
createWorkDir()
//启动外部的shuffle服务
startExternalShuffleService()
//配置worker的资源
setupWorkerResources()
//webUi初始化
webUi = new WorkerWebUI(this, workDir, webUiPort)
//绑定到端口
webUi.bind()
//url赋值
workerWebUiUrl = s"${webUi.scheme}$publicAddress:${webUi.boundPort}"
//注册到master
registerWithMaster()
//在metrics系统中加入worker来源 用于采集worker的信息 并发送显示在UI上
metricsSystem.registerSource(workerSource)
//启动metrics系统
metricsSystem.start()
// Attach the worker metrics servlet handler to the web ui after the metrics system is started.
//在度量系统启动后，将worker度量servlet处理程序附加到web ui。
metricsSystem.getServletHandlers.foreach(webUi.attachHandler)
}
//设置worker的资源
private def setupWorkerResources(): Unit = {
try {
//从输入资源文件和应用程序级Spark配置中获取输入组件的所有已分配资源信息。
//它首先查看资源文件中是否明确指定了资源（这将包括指定的地址分配，并且仅在某些集群管理器中指定），
//然后查看Spark配置以获取文件中未指定的任何其他配置。文件中未明确设置的资源需要运行发现脚本才能获取资源的地址。
//它还验证资源分配是否满足每个资源所需的数量。
resources = getOrDiscoverAllResources(conf, SPARK_WORKER_PREFIX, resourceFileOpt)
//打印已经分配的资源信息
logResourceInfo(SPARK_WORKER_PREFIX, resources)
} catch {
case e: Exception =>
//提示未能设置worker资源
logError("Failed to setup worker resources: ", e)
if (!Utils.isTesting) {
System.exit(1)
}
}
resources.keys.foreach { rName =>
//已经使用的资源 MutableResourceInfo--一种可变的资源信息，可对地址进行更有效的修改。
resourcesUsed(rName) = MutableResourceInfo(rName, new HashSet[String])
}
}
//添加已使用的资源
private def addResourcesUsed(deltaInfo: Map[String, ResourceInformation]): Unit = {
deltaInfo.foreach { case (rName, rInfo) =>
//更新已经使用的资源
resourcesUsed(rName) = resourcesUsed(rName) + rInfo
}
}
//移除已经使用的资源
private def removeResourcesUsed(deltaInfo: Map[String, ResourceInformation]): Unit = {
deltaInfo.foreach { case (rName, rInfo) =>
//更新已经使用的资源 减去使用的
resourcesUsed(rName) = resourcesUsed(rName) - rInfo
}
}
/**
* Change to use the new master.
*
* @param masterRef the new master ref
* @param uiUrl the new master Web UI address
* @param masterAddress the new master address which the worker should use to connect in case of
* failure
*/
//改变连接到新的master
//masterRef表示新master的引用
//uiUrl表示新master的web ui 地址
//masterAddress表示新master的地址 master在出现故障时应用于连接的新主地址
private def changeMaster(masterRef: RpcEndpointRef, uiUrl: String,
masterAddress: RpcAddress): Unit = {
// activeMasterUrl it's a valid Spark url since we receive it from master.
//activeMasterUrl这是一个有效的Spark url，因为我们从master收到它。
activeMasterUrl = masterRef.address.toSparkURL
//有效的master web ui url
activeMasterWebUiUrl = uiUrl
//master用于连接的地址初始化
masterAddressToConnect = Some(masterAddress)
//master初始化
master = Some(masterRef)
//已经连接上
connected = true
//如果设置了反向代理
if (reverseProxy) {
//提示WorkerWebUI的的可用地址
logInfo("WorkerWebUI is available at %s/proxy/%s".format(
activeMasterWebUiUrl.stripSuffix("/"), workerId))
// if reverseProxyUrl is not set, then we continue to generate relative URLs
// starting with "/" throughout the UI and do not use activeMasterWebUiUrl
//如果未设置reverseProxyUrl，则我们将继续在整个UI中生成以“/”开头的相对URL，并且不使用activeMasterWebUiUrl
val proxyUrl = conf.get(UI_REVERSE_PROXY_URL.key, "").stripSuffix("/")
// In the method `UIUtils.makeHref`, the URL segment "/proxy/$worker_id" will be appended
// after `proxyUrl`, so no need to set the worker ID in the `spark.ui.proxyBase` here.
//在方法`UIUtils.makeHref`中，URL段“/proxy/$worker_id”将附加在`proxyUrl`之后，因此无需在此处的`spark.ui.proxyBase`中设置工作者id。
System.setProperty("spark.ui.proxyBase", proxyUrl)
}
// Cancel any outstanding re-registration attempts because we found a new master
//取消任何未完成的重新注册尝试，因为我们找到了新的master
cancelLastRegistrationRetry()
}
//向所有的master注册
private def tryRegisterAllMasters(): Array[JFuture[_]] = {
//遍历master地址
masterRpcAddresses.map { masterAddress =>
registerMasterThreadPool.submit(new Runnable {
override def run(): Unit = {
try {
//提示连接到master
logInfo("Connecting to master " + masterAddress + "...")
//初始化master引用
val masterEndpoint = rpcEnv.setupEndpointRef(masterAddress, Master.ENDPOINT_NAME)
//发送注册消息到该master
sendRegisterMessageToMaster(masterEndpoint)
} catch {
//异常取消注册
case ie: InterruptedException => // Cancelled
//提示无法连接到master
case NonFatal(e) => logWarning(s"Failed to connect to master $masterAddress", e)
}
}
})
}
}
/**
* Re-register with the master because a network failure or a master failure has occurred.
* If the re-registration attempt threshold is exceeded, the worker exits with error.
* Note that for thread-safety this should only be called from the rpcEndpoint.
*/
//由于发生网络故障或主机故障，请重新向主机注册。如果超过了重新注册尝试阈值，则工作程序将错误退出。
//请注意，为了线程安全，应该仅从rpcEndpoint调用此函数。
private def reregisterWithMaster(): Unit = {
Utils.tryOrExit {
//注册连接次数加一
connectionAttemptCount += 1
//如果已经注册了
if (registered) {
//取消上一次注册重试
cancelLastRegistrationRetry()
}
//如果注册次数小于注册阈值
else if (connectionAttemptCount <= TOTAL_REGISTRATION_RETRIES) {
//提示正在尝试连接到master
logInfo(s"Retrying connection to master (attempt # $connectionAttemptCount)")
/**
* Re-register with the active master this worker has been communicating with. If there
* is none, then it means this worker is still bootstrapping and hasn't established a
* connection with a master yet, in which case we should re-register with all masters.
*
* It is important to re-register only with the active master during failures. Otherwise,
* if the worker unconditionally attempts to re-register with all masters, the following
* race condition may arise and cause a "duplicate worker" error detailed in SPARK-4592:
*
* (1) Master A fails and Worker attempts to reconnect to all masters
* (2) Master B takes over and notifies Worker
* (3) Worker responds by registering with Master B
* (4) Meanwhile, Worker's previous reconnection attempt reaches Master B,
* causing the same Worker to register with Master B twice
*
* Instead, if we only register with the known active master, we can assume that the
* old master must have died because another master has taken over. Note that this is
* still not safe if the old master recovers within this interval, but this is a much
* less likely scenario.
*/
//重新注册到该worker一直在与之通信的活动master。
//如果没有，则意味着该工作进程仍在自举，尚未与master建立连接，在这种情况下，我们应该向所有master重新注册。
//在发生故障时，仅向活动master重新注册非常重要。
//否则，如果worker无条件地尝试向所有master重新注册，则可能会出现以下竞争条件，并导致SPARK-4592中详细说明的“重复工作程序”错误：
//（1） master A失败，Worker尝试重新连接到所有master
//（2） master B接管并通知worker
//（3） Worker通过向Master B注册进行响应
//（4） 同时，Worker的上一次重新连接尝试到达Master B，导致同一Worker向Master B注册两次
//相反，如果我们只向已知的活动master注册，我们可以假设旧的master一定是因为另一位master接管了而去世的。
//请注意，如果旧master在此间隔内恢复，这仍然是不安全的，但这种情况的可能性要小得多。
master match {
case Some(masterRef) =>
// registered == false && master != None means we lost the connection to master, so
// masterRef cannot be used and we need to recreate it again. Note: we must not set
// master to None due to the above comments.
//registered==false&&master！= None 这并不意味着我们失去了与master的连接，
//因此不能使用masterRef，我们需要重新创建它。注意：由于上述评论，我们不能将master设置为None。
//如果注册结果不为空
if (registerMasterFutures != null) {
//取消所有注册
registerMasterFutures.foreach(_.cancel(true))
}
//如果配置了master地址 则使用配置地址 否则使用当前master引用地址
val masterAddress =
if (preferConfiguredMasterAddress) masterAddressToConnect.get else masterRef.address
//
registerMasterFutures = Array(registerMasterThreadPool.submit(new Runnable {
override def run(): Unit = {
try {
//提示正在连接到master
logInfo("Connecting to master " + masterAddress + "...")
//获取master引用
val masterEndpoint = rpcEnv.setupEndpointRef(masterAddress, Master.ENDPOINT_NAME)
//向master发请求注册消息
sendRegisterMessageToMaster(masterEndpoint)
} catch {
//发生异常 取消注册
case ie: InterruptedException => // Cancelled
//提示连接到master失败
case NonFatal(e) => logWarning(s"Failed to connect to master $masterAddress", e)
}
}
}))
case None =>
//如果注册结果不为空 取消所有的注册
if (registerMasterFutures != null) {
registerMasterFutures.foreach(_.cancel(true))
}
// We are retrying the initial registration
//我们正在重试初始注册
registerMasterFutures = tryRegisterAllMasters()
}
// We have exceeded the initial registration retry threshold
// All retries from now on should use a higher interval
//我们已经超过了初始注册重试阈值从现在起所有重试都应该使用更高的间隔
if (connectionAttemptCount == INITIAL_REGISTRATION_RETRIES) {
//已经超过初始注册阈值 直接取消所有的重试注册行为
registrationRetryTimer.foreach(_.cancel(true))
//使用延长时间间隔 重新尝试注册 在超时时间之内
registrationRetryTimer = Some(
forwardMessageScheduler.scheduleAtFixedRate(
() => Utils.tryLogNonFatalError { self.send(ReregisterWithMaster) },
PROLONGED_REGISTRATION_RETRY_INTERVAL_SECONDS,
PROLONGED_REGISTRATION_RETRY_INTERVAL_SECONDS,
TimeUnit.SECONDS))
}
} else {
//提示 所有的master都没有回应 放弃注册
logError("All masters are unresponsive! Giving up.")
System.exit(1)
}
}
}
/**
* Cancel last registration retry, or do nothing if no retry
*/
//取消上次注册重试，如果没有重试则不执行任何操作
private def cancelLastRegistrationRetry(): Unit = {
//如果注册结果不为空
if (registerMasterFutures != null) {
//所有的注册都取消
registerMasterFutures.foreach(_.cancel(true))
//注册结果都设置为空
registerMasterFutures = null
}
//重试注册都取消
registrationRetryTimer.foreach(_.cancel(true))
registrationRetryTimer = None
}
//注册到master
private def registerWithMaster(): Unit = {
// onDisconnected may be triggered multiple times, so don't attempt registration
// if there are outstanding registration attempts scheduled.
//onDisconnected可能会被触发多次，因此如果计划了未完成的注册尝试，请不要尝试注册。
registrationRetryTimer match {
//
case None =>
//标记没有注册
registered = false
//向所有的master注册
registerMasterFutures = tryRegisterAllMasters()
//连接尝试次数
connectionAttemptCount = 0
//通过线程注册 间隔一定时间之后再重试 在超时时间之内
registrationRetryTimer = Some(forwardMessageScheduler.scheduleAtFixedRate(
() => Utils.tryLogNonFatalError { Option(self).foreach(_.send(ReregisterWithMaster)) },
INITIAL_REGISTRATION_RETRY_INTERVAL_SECONDS,
INITIAL_REGISTRATION_RETRY_INTERVAL_SECONDS,
TimeUnit.SECONDS))
case Some(_) =>
//由于已经安排了一次尝试，因此不会再次尝试向主机注册。
logInfo("Not spawning another attempt to register with the master, since there is an" +
" attempt scheduled already.")
}
}
//启动外部的shuffle服务
private def startExternalShuffleService(): Unit = {
try {
//启动服务
shuffleService.startIfEnabled()
} catch {
case e: Exception =>
//提示 无法启动外部混洗服务
logError("Failed to start external shuffle service", e)
//退出
System.exit(1)
}
}
//向master发请求注册消息
private def sendRegisterMessageToMaster(masterEndpoint: RpcEndpointRef): Unit = {
//引用向master发消息worker注册
masterEndpoint.send(RegisterWorker(
workerId,
host,
port,
self,
cores,
memory,
workerWebUiUrl,
masterEndpoint.address,
resources))
}
//处理worker注册的回应
private def handleRegisterResponse(msg: RegisterWorkerResponse): Unit = synchronized {
msg match {
//已经注册的worker
case RegisteredWorker(masterRef, masterWebUiUrl, masterAddress, duplicate) =>
//如果配置了master地址 则使用配置地址 否则使用master返回的地址
val preferredMasterAddress = if (preferConfiguredMasterAddress) {
masterAddress.toSparkURL
} else {
masterRef.address.toSparkURL
}
// there're corner cases which we could hardly avoid duplicate worker registration,
// e.g. Master disconnect(maybe due to network drop) and recover immediately, see
// SPARK-23191 for more details.
//有些情况下，我们很难避免重复的worker注册，例如主master断开连接（可能是由于网络中断）并立即恢复，请参阅SPARK-23191了解更多详细信息。
//如果是重复注册
if (duplicate) {
//提示重复注册了
logWarning(s"Duplicate registration at master $preferredMasterAddress")
}
//提示已经注册成功
logInfo(s"Successfully registered with master $preferredMasterAddress")
//标记已经注册成功
registered = true
//改变连接到新的master
changeMaster(masterRef, masterWebUiUrl, masterAddress)
//发送心跳
forwardMessageScheduler.scheduleAtFixedRate(
() => Utils.tryLogNonFatalError { self.send(SendHeartbeat) },
0, HEARTBEAT_MILLIS, TimeUnit.MILLISECONDS)
//如果可以清理应用程序的目录
if (CLEANUP_ENABLED) {
//提示worker清理功能启动 旧应用目录将会被删除
logInfo(
s"Worker cleanup enabled; old application directories will be deleted in: $workDir")
//向worker发消息 清理旧应用目录
forwardMessageScheduler.scheduleAtFixedRate(
() => Utils.tryLogNonFatalError { self.send(WorkDirCleanup) },
CLEANUP_INTERVAL_MILLIS, CLEANUP_INTERVAL_MILLIS, TimeUnit.MILLISECONDS)
}
//获取所有的executor
val execs = executors.values.map { e =>
new ExecutorDescription(e.appId, e.execId, e.cores, e.state)
}
//向master发消息worker的信息 包括workerid executor列表 提交应用的driver列表
masterRef.send(WorkerLatestState(workerId, execs.toList, drivers.keys.toSeq))
//如果worker注册失败
case RegisterWorkerFailed(message) =>
//如果没有注册成功
if (!registered) {
//提示注册失败
logError("Worker registration failed: " + message)
System.exit(1)
}
//如果是注册到备用master 则不处理
case MasterInStandby =>
// Ignore. Master not yet ready.
}
}
//处理接收到的消息 先获取锁
override def receive: PartialFunction[Any, Unit] = synchronized {
//worker注册回应
case msg: RegisterWorkerResponse =>
handleRegisterResponse(msg)
//发送心跳
case SendHeartbeat =>
//如果连接成功 则向master发送心跳
if (connected) { sendToMaster(Heartbeat(workerId, self)) }
//清理应用目录
case WorkDirCleanup =>
// Spin up a separate thread (in a future) to do the dir cleanup; don't tie up worker
// rpcEndpoint.
// Copy ids so that it can be used in the cleanup thread.
//（将来）起一个单独的线程来进行目录清理；不要绑定工作进程rpcEndpoint。
//复制id，以便可以在清理线程中使用它。
//获取executor的应用id和driver的应用id
val appIds = (executors.values.map(_.appId) ++ drivers.values.map(_.driverId)).toSet
try {
val cleanupFuture: concurrent.Future[Unit] = concurrent.Future {
//获取工作目录下的文件列表
val appDirs = workDir.listFiles()
//如果文件列表为空
if (appDirs == null) {
//抛出异常
throw new IOException("ERROR: Failed to list files in " + appDirs)
}
appDirs.filter { dir =>
// the directory is used by an application - check that the application is not running
// when cleaning up
//目录由应用程序使用-在清理时检查应用程序是否未运行
//获取应用程序目录名
val appIdFromDir = dir.getName
//判断应用程序是否还在执行
val isAppStillRunning = appIds.contains(appIdFromDir)
//不是目录且应用不在运行且目录下不包含新文件
dir.isDirectory && !isAppStillRunning &&
!Utils.doesDirectoryContainAnyNewFiles(dir, APP_DATA_RETENTION_SECONDS)
}.foreach { dir =>
//满足上面的条件 才能清理
//提示正在移除目录
logInfo(s"Removing directory: ${dir.getPath}")
//删除目录下的文件
Utils.deleteRecursively(dir)
// Remove some registeredExecutors information of DB in external shuffle service when
// #spark.shuffle.service.db.enabled=true, the one which comes to mind is, what happens
// if an application is stopped while the external shuffle service is down?
// So then it'll leave an entry in the DB and the entry should be removed.
//当#spark.shuffle.service.DB.enabled=true时，删除外部shuffle服务中DB的一些registeredExecutors信息，
//脑海中浮现的问题是，如果在外部shuffler服务关闭时停止应用程序，会发生什么？
//因此，它将在数据库中留下一个条目，该条目应该被删除。
if (conf.get(config.SHUFFLE_SERVICE_DB_ENABLED) &&
conf.get(config.SHUFFLE_SERVICE_ENABLED)) {
//shuffle服务停止时应该将其留下的文件删除
shuffleService.applicationRemoved(dir.getName)
}
}
}(cleanupThreadExecutor)//由清理线程负责执行
//清理失败结果遍历
cleanupFuture.failed.foreach(e =>
//提示应用程序目录清理失败
logError("App dir cleanup failed: " + e.getMessage, e)
)(cleanupThreadExecutor)
} catch {
//拒绝执行异常
case _: RejectedExecutionException if cleanupThreadExecutor.isShutdown =>
//提示清理失败因为执行线程池已经关闭
logWarning("Failed to cleanup work dir as executor pool was shutdown")
}
//master已经改变
case MasterChanged(masterRef, masterWebUiUrl) =>
//提示 master已经切换 新的master是.
logInfo("Master has changed, new master is at " + masterRef.address.toSparkURL)
//改变连接到新的master
changeMaster(masterRef, masterWebUiUrl, masterRef.address)
//executor回应
val executorResponses = executors.values.map { e =>
WorkerExecutorStateResponse(new ExecutorDescription(
e.appId, e.execId, e.cores, e.state), e.resources)
}
//driver回应
val driverResponses = drivers.keys.map { id =>
WorkerDriverStateResponse(id, drivers(id).resources)}
//向master发消息 更新worker资源状态
masterRef.send(WorkerSchedulerStateResponse(
workerId, executorResponses.toList, driverResponses.toSeq))
//worker重新连接
case ReconnectWorker(masterUrl) =>
//url为$masterUrl的Master请求此worker重新连接。
logInfo(s"Master with url $masterUrl requested this worker to reconnect.")
//向master注册
registerWithMaster()
//启动executor
case LaunchExecutor(masterUrl, appId, execId, appDesc, cores_, memory_, resources_) =>
//如果传入的master url不是有效的master url
if (masterUrl != activeMasterUrl) {
//提示 无效的master尝试启动executor
logWarning("Invalid Master (" + masterUrl + ") attempted to launch executor.")
} else if (decommissioned) {//如果worker已经退役
//worker已经退役却请求启动executor
logWarning("Asked to launch an executor while decommissioned. Not launching executor.")
} else {
try {
//提示 请求启动executor
logInfo("Asked to launch executor %s/%d for %s".format(appId, execId, appDesc.name))
// Create the executor's working directory
//创建executor工作目录
val executorDir = new File(workDir, appId + "/" + execId)
if (!executorDir.mkdirs()) {
//如果创建目录失败则抛出异常
throw new IOException("Failed to create directory " + executorDir)
}
// Create local dirs for the executor. These are passed to the executor via the
// SPARK_EXECUTOR_DIRS environment variable, and deleted by the Worker when the
// application finishes.
//为executor创建本地目录。这些信息通过SPARK_EXECUTOR_DIRS环境变量，并在应用程序完成时由Worker删除。
//创建应用本地目录
val appLocalDirs = appDirectories.getOrElse(appId, {
//根据配置创建本地根目录
val localRootDirs = Utils.getOrCreateLocalRootDirs(conf)
//根目录遍历
val dirs = localRootDirs.flatMap { dir =>
try {
//创建应用目录
val appDir = Utils.createDirectory(dir, namePrefix = "executor")
//设置目录权限
Utils.chmod700(appDir)
//返回应用目录地址
Some(appDir.getAbsolutePath())
} catch {
//创建失败抛出IO异常
case e: IOException =>
//提示忽视这个目录
logWarning(s"${e.getMessage}. Ignoring this directory.")
None
}
}.toSeq
//如果目录为空
if (dirs.isEmpty) {
//抛出异常 子目录无法被创建
throw new IOException("No subfolder can be created in " +
s"${localRootDirs.mkString(",")}.")
}
dirs
})
//应用程序目录初始化
appDirectories(appId) = appLocalDirs
//ExecutorRunner--管理一个执行程序进程的执行。这目前仅在独立模式下使用。
val manager = new ExecutorRunner(
appId,
execId,
appDesc.copy(command = Worker.maybeUpdateSSLSettings(appDesc.command, conf)),
cores_,
memory_,
self,
workerId,
webUi.scheme,
host,
webUi.boundPort,
publicAddress,
sparkHome,
executorDir,
workerUri,
conf,
appLocalDirs,
ExecutorState.LAUNCHING,
resources_)
//添加executor管理者
executors(appId + "/" + execId) = manager
//管理者启动
manager.start()
//已经使用的core更新为加上启动该executor所用的core
coresUsed += cores_
//已经使用的内存更新为加上启动该executor所用的内存
memoryUsed += memory_
//更新已经使用的资源
addResourcesUsed(resources_)
} catch {
case e: Exception =>
//启动executor失败
logError(s"Failed to launch executor $appId/$execId for ${appDesc.name}.", e)
//如果executors列表中已经包含要注册的executor
if (executors.contains(appId + "/" + execId)) {
//杀死旧executor
executors(appId + "/" + execId).kill()
//在executors列表中移除旧executor
executors -= appId + "/" + execId
}
//给master发消息executor状态已经改变 即失败了
sendToMaster(ExecutorStateChanged(appId, execId, ExecutorState.FAILED,
Some(e.toString), None))
}
}
//executor状态改变
case executorStateChanged: ExecutorStateChanged =>
handleExecutorStateChanged(executorStateChanged)
//杀死executor
case KillExecutor(masterUrl, appId, execId) =>
//如果master地址不是有效的地址
if (masterUrl != activeMasterUrl) {
//提示 无效的master尝试杀死executor
logWarning("Invalid Master (" + masterUrl + ") attempted to kill executor " + execId)
} else {
//获取id全称
val fullId = appId + "/" + execId
//遍历executors列表
executors.get(fullId) match {
//如果匹配到executor
case Some(executor) =>
//提示 请求杀死executor
logInfo("Asked to kill executor " + fullId)
//杀死executor
executor.kill()
case None =>
//如果没匹配到executor id则提示请求杀死未知的executor
logInfo("Asked to kill unknown executor " + fullId)
}
}
//启动driver
case LaunchDriver(driverId, driverDesc, resources_) =>
//请求启动driver
logInfo(s"Asked to launch driver $driverId")
//DriverRunner--管理一个driver的执行，包括在出现故障时自动重新启动驱动程序。这目前仅用于独立集群部署模式。
val driver = new DriverRunner(
conf,
driverId,
workDir,
sparkHome,
driverDesc.copy(command = Worker.maybeUpdateSSLSettings(driverDesc.command, conf)),
self,
workerUri,
workerWebUiUrl,
securityMgr,
resources_)
//在drivers列表中添加该driver
drivers(driverId) = driver
//启动该driver
driver.start()
//已经使用的core更新为加上启动该driver所用的core
coresUsed += driverDesc.cores
//已经使用的内存更新为加上启动该driver所用的内存
memoryUsed += driverDesc.mem
//更新已经使用的资源
addResourcesUsed(resources_)
//杀死driver
case KillDriver(driverId) =>
//请求杀死driver
logInfo(s"Asked to kill driver $driverId")
//在drivers列表中找到该driver并匹配
drivers.get(driverId) match {
case Some(runner) =>
//如果找到则杀死该driver
runner.kill()
case None =>
//没找到则提示 请求杀死一个未知的driver
logError(s"Asked to kill unknown driver $driverId")
}
//driver状态改变 @ 此处作用 应该是模式匹配 后面补充
case driverStateChanged @ DriverStateChanged(driverId, state, exception) =>
handleDriverStateChanged(driverStateChanged)
//注册到master
case ReregisterWithMaster =>
reregisterWithMaster()
//应用程序已经结束
case ApplicationFinished(id) =>
//在结束应用列表中加入该应用
finishedApps += id
//清理应用
maybeCleanupApplication(id)
//使worker退役
case DecommissionWorker =>
decommissionSelf()
//使用PWR处理程序使worker退役
case WorkerSigPWRReceived =>
//worker退役
decommissionSelf()
// Tell the Master that we are starting decommissioning
// so it stops trying to launch executor/driver on us
//告诉master 我正在开始退役 不要在我这儿启动executor或者driver
sendToMaster(WorkerDecommissioning(workerId, self))
}
//收到消息并回复
override def receiveAndReply(context: RpcCallContext): PartialFunction[Any, Unit] = {
//请求worker状态信息
case RequestWorkerState =>
//返回worker上所有的资源和应用的信息
context.reply(WorkerStateResponse(host, port, workerId, executors.values.toList,
finishedExecutors.values.toList, drivers.values.toList,
finishedDrivers.values.toList, activeMasterUrl, cores, memory,
coresUsed, memoryUsed, activeMasterWebUiUrl, resources,
resourcesUsed.toMap.map { case (k, v) => (k, v.toResourceInformation)}))
}
//连接断开
override def onDisconnected(remoteAddress: RpcAddress): Unit = {
//如果master中包含该地址或者master中用于连接的地址中包含该地址
if (master.exists(_.address == remoteAddress) ||
masterAddressToConnect.contains(remoteAddress)) {
//提示 该地址已经断开连接
logInfo(s"$remoteAddress Disassociated !")
//已经与master断开连接
masterDisconnected()
}
}
//与master断开连接
private def masterDisconnected(): Unit = {
//提示 连接到master失败 等待master重新连接
logError("Connection to master failed! Waiting for master to reconnect...")
//标记连接断开
connected = false
//重新注册到master
registerWithMaster()
}
//也许可以清理应用程序
private def maybeCleanupApplication(id: String): Unit = {
//判断是否能清理应用程序 如果已经结束应用列表中包含该应用id 并且executors列表中不包含该应用id
val shouldCleanup = finishedApps.contains(id) && !executors.values.exists(_.appId == id)
//如果可以清理
if (shouldCleanup) {
//在已经结束的应用列表中移除该应用
finishedApps -= id
try {
//移除该应用的目录
appDirectories.remove(id).foreach { dirList =>
concurrent.Future {
//提示 清楚该应用的本地目录
logInfo(s"Cleaning up local directories for application $id")
//目录遍历
dirList.foreach { dir =>
//删除目录
Utils.deleteRecursively(new File(dir))
}
}(cleanupThreadExecutor).failed.foreach(e =>
//提示 清理应用目录失败
logError(s"Clean up app dir $dirList failed: ${e.getMessage}", e)
)(cleanupThreadExecutor)
}
} catch {
//清理操作被拒绝异常
case _: RejectedExecutionException if cleanupThreadExecutor.isShutdown =>
//清理应用失败原因是执行线程池被关闭
logWarning("Failed to cleanup application as executor pool was shutdown")
}
shuffleService.applicationRemoved(id)
}
}
/**
* Send a message to the current master. If we have not yet registered successfully with any
* master, the message will be dropped.
*/
//向当前master发送消息。如果我们还没有成功注册到任何主机，消息将被删除。
private def sendToMaster(message: Any): Unit = {
master match {
//如果匹配到master 则向该master发消息
case Some(masterRef) => masterRef.send(message)
//如果没有匹配到
case None =>
//提示 丢弃该消息 因为连接到master还没有建立
logWarning(
s"Dropping $message because the connection to master has not yet been established")
}
}
//生成worker id
private def generateWorkerId(): String = {
"worker-%s-%s-%d".format(createDateFormat.format(new Date), host, port)
}
//worker停止
override def onStop(): Unit = {
//清理线程关闭
cleanupThreadExecutor.shutdownNow()
//metrics系统发送信息
metricsSystem.report()
//取消注册重试
cancelLastRegistrationRetry()
//发送消息线程关闭
forwardMessageScheduler.shutdownNow()
//注册线程池关闭
registerMasterThreadPool.shutdownNow()
//杀死每一个executor
executors.values.foreach(_.kill())
//杀死每一个driver
drivers.values.foreach(_.kill())
//shuffle服务停止
shuffleService.stop()
//页面ui停止
webUi.stop()
///metrics系统退出
metricsSystem.stop()
}
//整理已完成的executor（如有必要）
private def trimFinishedExecutorsIfNecessary(): Unit = {
// do not need to protect with locks since both WorkerPage and Restful server get data through
// thread-safe RpcEndPoint
//不需要使用锁进行保护，因为WorkerPage和Restful服务器都通过线程安全的RpcEndPoint获取数据
//如果已经结束的executor数量比保留的executor数量多
if (finishedExecutors.size > retainedExecutors) {
//在已经结束executor列表中移除一些
finishedExecutors.take(math.max(finishedExecutors.size / 10, 1)).foreach {
case (executorId, _) => finishedExecutors.remove(executorId)
}
}
}
//整理已经完成的driver列表
private def trimFinishedDriversIfNecessary(): Unit = {
// do not need to protect with locks since both WorkerPage and Restful server get data through
// thread-safe RpcEndPoint
//不需要使用锁进行保护，因为WorkerPage和Restful服务器都通过线程安全的RpcEndPoint获取数据
//如果已经结束的driver数量比保留的driver数量多
if (finishedDrivers.size > retainedDrivers) {
//在已经结束的driver列表中移除部分driver
finishedDrivers.take(math.max(finishedDrivers.size / 10, 1)).foreach {
case (driverId, _) => finishedDrivers.remove(driverId)
}
}
}
//worker使自己退役
private[deploy] def decommissionSelf(): Unit = {
//如果配置了可以退役且worker当前没有退役
if (conf.get(config.DECOMMISSION_ENABLED) && !decommissioned) {
//标记已经退役
decommissioned = true
//提示 使worker退役
logInfo(s"Decommission worker $workerId.")
} else if (decommissioned) {//如果已经退役了
//提示 worker正在退役
logWarning(s"Worker $workerId already started decommissioning.")
} else {
//提示 收到退役请求 但是退役操作无法执行
logWarning(s"Receive decommission request, but decommission feature is disabled.")
}
}
//处理driver状态变化
private[worker] def handleDriverStateChanged(driverStateChanged: DriverStateChanged): Unit = {
//获取状态改变的driver id
val driverId = driverStateChanged.driverId
//获取状态改变的driver的异常信息
val exception = driverStateChanged.exception
//获取状态改变的driver的状态
val state = driverStateChanged.state
state match {
//如果是driver发生错误
case DriverState.ERROR =>
//提示 driver失败不可用
logWarning(s"Driver $driverId failed with unrecoverable exception: ${exception.get}")
//driver失败
case DriverState.FAILED =>
//提示 driver失败退出了
logWarning(s"Driver $driverId exited with failure")
//driver正常结束
case DriverState.FINISHED =>
//提示 driver成功退出
logInfo(s"Driver $driverId exited successfully")
//driver被杀死
case DriverState.KILLED =>
//提示 driver被用户杀死
logInfo(s"Driver $driverId was killed by user")
//其他情况
case _ =>
//提示 driver的状态变到了当前状态
logDebug(s"Driver $driverId changed state to $state")
}
//给master发消息driver状态已经改变
sendToMaster(driverStateChanged)
//获取该driver
val driver = drivers.remove(driverId).get
//在已经结束的driver列表中加入该driver
finishedDrivers(driverId) = driver
//整理已经完成的driver列表
trimFinishedDriversIfNecessary()
//更新已经使用的内存 已经使用的内存减去该driver使用的内存
memoryUsed -= driver.driverDesc.mem
//更新已经使用的core 已经使用的core减去该driver使用的core
coresUsed -= driver.driverDesc.cores
//更新已经使用的资源 在已经使用的资源中减去该driver使用的资源
removeResourcesUsed(driver.resources)
}
//处理executor状态变化
private[worker] def handleExecutorStateChanged(executorStateChanged: ExecutorStateChanged):
Unit = {
//向master发消息executor状态变化
sendToMaster(executorStateChanged)
//获取executor状态
val state = executorStateChanged.state
//如果executor已经结束了
if (ExecutorState.isFinished(state)) {
//获取executor执行的应用id
val appId = executorStateChanged.appId
//获取id全称
val fullId = appId + "/" + executorStateChanged.execId
//获取变化信息
val message = executorStateChanged.message
//获取executor退出状态
val exitStatus = executorStateChanged.exitStatus
//遍历executors列表 进行匹配
executors.get(fullId) match {
//匹配到某个executor
case Some(executor) =>
//提示 executor以xx状态结束
logInfo("Executor " + fullId + " finished with state " + state +
message.map(" message " + _).getOrElse("") +
exitStatus.map(" exitStatus " + _).getOrElse(""))
//在executors列表中移除该executor
executors -= fullId
//在结束executors列表中添加该executor
finishedExecutors(fullId) = executor
//整理已完成的执行程序 移除一些已经完成的executor
trimFinishedExecutorsIfNecessary()
//已经使用的core中减去该executor使用的core
coresUsed -= executor.cores
//已经使用的内存中减去该executor使用的内存
memoryUsed -= executor.memory
//移除已经使用的资源
removeResourcesUsed(executor.resources)
if (CLEANUP_FILES_AFTER_EXECUTOR_EXIT) {
shuffleService.executorRemoved(executorStateChanged.execId.toString, appId)
}
case None =>
logInfo("Unknown Executor " + fullId + " finished with state " + state +
message.map(" message " + _).getOrElse("") +
exitStatus.map(" exitStatus " + _).getOrElse(""))
}
maybeCleanupApplication(appId)
}
}
}
//worker对象
private[deploy] object Worker extends Logging {
val SYSTEM_NAME = "sparkWorker"
val ENDPOINT_NAME = "Worker"
private val SSL_NODE_LOCAL_CONFIG_PATTERN = """\-Dspark\.ssl\.useNodeLocalConf\=(.+)""".r
def main(argStrings: Array[String]): Unit = {
Thread.setDefaultUncaughtExceptionHandler(new SparkUncaughtExceptionHandler(
exitOnUncaughtException = false))
Utils.initDaemon(log)
val conf = new SparkConf
val args = new WorkerArguments(argStrings, conf)
val rpcEnv = startRpcEnvAndEndpoint(args.host, args.port, args.webUiPort, args.cores,
args.memory, args.masters, args.workDir, conf = conf,
resourceFileOpt = conf.get(SPARK_WORKER_RESOURCE_FILE))
// With external shuffle service enabled, if we request to launch multiple workers on one host,
// we can only successfully launch the first worker and the rest fails, because with the port
// bound, we may launch no more than one external shuffle service on each host.
// When this happens, we should give explicit reason of failure instead of fail silently. For
// more detail see SPARK-20989.
//启用外部shuffle服务后，如果我们请求在一台主机上启动多个工作程序，
//我们只能成功地启动第一个工作进程，其余的都失败了，因为有了端口
//绑定后，我们可以在每个主机上启动不超过一个外部shuffle服务。
//当这种情况发生时，我们应该给出失败的明确原因，而不是默默地失败。对于
//更多详细信息，请参阅SPARK-20989。
val externalShuffleServiceEnabled = conf.get(config.SHUFFLE_SERVICE_ENABLED)
val sparkWorkerInstances = scala.sys.env.getOrElse("SPARK_WORKER_INSTANCES", "1").toInt
require(externalShuffleServiceEnabled == false || sparkWorkerInstances <= 1,
"Starting multiple workers on one host is failed because we may launch no more than one " +
"external shuffle service on each host, please set spark.shuffle.service.enabled to " +
"false or set SPARK_WORKER_INSTANCES to 1 to resolve the conflict.")
rpcEnv.awaitTermination()
}
def startRpcEnvAndEndpoint(
host: String,
port: Int,
webUiPort: Int,
cores: Int,
memory: Int,
masterUrls: Array[String],
workDir: String,
workerNumber: Option[Int] = None,
conf: SparkConf = new SparkConf,
resourceFileOpt: Option[String] = None): RpcEnv = {
// The LocalSparkCluster runs multiple local sparkWorkerX RPC Environments
val systemName = SYSTEM_NAME + workerNumber.map(_.toString).getOrElse("")
val securityMgr = new SecurityManager(conf)
val rpcEnv = RpcEnv.create(systemName, host, port, conf, securityMgr)
val masterAddresses = masterUrls.map(RpcAddress.fromSparkURL)
rpcEnv.setupEndpoint(ENDPOINT_NAME, new Worker(rpcEnv, webUiPort, cores, memory,
masterAddresses, ENDPOINT_NAME, workDir, conf, securityMgr, resourceFileOpt))
rpcEnv
}
def isUseLocalNodeSSLConfig(cmd: Command): Boolean = {
val result = cmd.javaOpts.collectFirst {
case SSL_NODE_LOCAL_CONFIG_PATTERN(_result) => _result.toBoolean
}
result.getOrElse(false)
}
def maybeUpdateSSLSettings(cmd: Command, conf: SparkConf): Command = {
val prefix = "spark.ssl."
val useNLC = "spark.ssl.useNodeLocalConf"
if (isUseLocalNodeSSLConfig(cmd)) {
val newJavaOpts = cmd.javaOpts
.filter(opt => !opt.startsWith(s"-D$prefix")) ++
conf.getAll.collect { case (key, value) if key.startsWith(prefix) => s"-D$key=$value" } :+
s"-D$useNLC=true"
cmd.copy(javaOpts = newJavaOpts)
} else {
cmd
}
}
}
worker在收到master发过来的LaunchExecutor消息之后，开始启动执行器：
//启动executor
case LaunchExecutor(masterUrl, appId, execId, appDesc, cores_, memory_, resources_) =>
//如果传入的master url不是有效的master url
if (masterUrl != activeMasterUrl) {
//提示 无效的master尝试启动executor
logWarning("Invalid Master (" + masterUrl + ") attempted to launch executor.")
} else if (decommissioned) {//如果worker已经退役
//worker已经退役却请求启动executor
logWarning("Asked to launch an executor while decommissioned. Not launching executor.")
} else {
try {
//提示 请求启动executor
logInfo("Asked to launch executor %s/%d for %s".format(appId, execId, appDesc.name))
// Create the executor's working directory
//创建executor工作目录
val executorDir = new File(workDir, appId + "/" + execId)
if (!executorDir.mkdirs()) {
//如果创建目录失败则抛出异常
throw new IOException("Failed to create directory " + executorDir)
}
// Create local dirs for the executor. These are passed to the executor via the
// SPARK_EXECUTOR_DIRS environment variable, and deleted by the Worker when the
// application finishes.
//为executor创建本地目录。这些信息通过SPARK_EXECUTOR_DIRS环境变量，并在应用程序完成时由Worker删除。
//创建应用本地目录
val appLocalDirs = appDirectories.getOrElse(appId, {
//根据配置创建本地根目录
val localRootDirs = Utils.getOrCreateLocalRootDirs(conf)
//根目录遍历
val dirs = localRootDirs.flatMap { dir =>
try {
//创建应用目录
val appDir = Utils.createDirectory(dir, namePrefix = "executor")
//设置目录权限
Utils.chmod700(appDir)
//返回应用目录地址
Some(appDir.getAbsolutePath())
} catch {
//创建失败抛出IO异常
case e: IOException =>
//提示忽视这个目录
logWarning(s"${e.getMessage}. Ignoring this directory.")
None
}
}.toSeq
//如果目录为空
if (dirs.isEmpty) {
//抛出异常 子目录无法被创建
throw new IOException("No subfolder can be created in " +
s"${localRootDirs.mkString(",")}.")
}
dirs
})
//应用程序目录初始化
appDirectories(appId) = appLocalDirs
//ExecutorRunner--管理一个执行程序进程的执行。这目前仅在独立模式下使用。
val manager = new ExecutorRunner(
appId,
execId,
appDesc.copy(command = Worker.maybeUpdateSSLSettings(appDesc.command, conf)),
cores_,
memory_,
self,
workerId,
webUi.scheme,
host,
webUi.boundPort,
publicAddress,
sparkHome,
executorDir,
workerUri,
conf,
appLocalDirs,
ExecutorState.LAUNCHING,
resources_)
//添加executor管理者
executors(appId + "/" + execId) = manager
//管理者启动
manager.start()
//已经使用的core更新为加上启动该executor所用的core
coresUsed += cores_
//已经使用的内存更新为加上启动该executor所用的内存
memoryUsed += memory_
//更新已经使用的资源
addResourcesUsed(resources_)
} catch {
case e: Exception =>
//启动executor失败
logError(s"Failed to launch executor $appId/$execId for ${appDesc.name}.", e)
//如果executors列表中已经包含要注册的executor
if (executors.contains(appId + "/" + execId)) {
//杀死旧executor
executors(appId + "/" + execId).kill()
//在executors列表中移除旧executor
executors -= appId + "/" + execId
}
//给master发消息executor状态已经改变 即失败了
sendToMaster(ExecutorStateChanged(appId, execId, ExecutorState.FAILED,
Some(e.toString), None))
}
}
首先判断发消息的master是不是有效的master，如果不是则忽略，接着判断worker是不是退役了，如果退役也忽略，最后正式开始启动：先创建executor执行目录，然后创建应用目录，创建应用子目录，进行目录初始化，设置目录权限，创建ExecutorRunner（只在Standalone模式下才会启动）管理executor进程的执行，最后更新已经使用的资源。ExecutorRunner在启动的时候会创建executor进程，下一节介绍该部分源码。
