@InterfaceAudience.LimitedPrivate(value="yarn") @InterfaceStability.Unstable public class FairScheduler extends AbstractYarnScheduler<FSAppAttempt,FSSchedulerNode>
Modifier and Type | Field and Description |
---|---|
protected boolean |
assignMultiple |
static org.apache.hadoop.yarn.api.records.Resource |
CONTAINER_RESERVED |
protected boolean |
continuousSchedulingEnabled
Deprecated.
|
protected int |
continuousSchedulingSleepMs
Deprecated.
|
protected int |
maxAssign |
protected long |
nodeLocalityDelayMs
Deprecated.
|
protected double |
nodeLocalityThreshold |
protected long |
rackLocalityDelayMs
Deprecated.
|
protected double |
rackLocalityThreshold |
protected boolean |
sizeBasedWeight |
activitiesManager, applications, EMPTY_ALLOCATION, lastNodeUpdateTime, minimumAllocation, nmExpireInterval, nmHeartbeatInterval, nodeTracker, readLock, rmContext, schedulerHealth, schedulingMonitorManager, THREAD_JOIN_TIMEOUT_MS, updateInterval, writeLock
Constructor and Description |
---|
FairScheduler() |
Modifier and Type | Method and Description |
---|---|
protected void |
addApplication(org.apache.hadoop.yarn.api.records.ApplicationId applicationId,
String queueName,
String user,
boolean isAppRecovering,
ApplicationPlacementContext placementContext)
Add a new application to the scheduler, with a given id, queue name, and
user.
|
protected void |
addApplicationAttempt(org.apache.hadoop.yarn.api.records.ApplicationAttemptId applicationAttemptId,
boolean transferStateFromPreviousAttempt,
boolean isAttemptRecovering)
Add a new application attempt to the scheduler.
|
Allocation |
allocate(org.apache.hadoop.yarn.api.records.ApplicationAttemptId appAttemptId,
List<org.apache.hadoop.yarn.api.records.ResourceRequest> ask,
List<org.apache.hadoop.yarn.api.records.SchedulingRequest> schedulingRequests,
List<org.apache.hadoop.yarn.api.records.ContainerId> release,
List<String> blacklistAdditions,
List<String> blacklistRemovals,
ContainerUpdates updateRequests)
The main API between the ApplicationMaster and the Scheduler.
|
boolean |
checkAccess(org.apache.hadoop.security.UserGroupInformation callerUGI,
org.apache.hadoop.yarn.api.records.QueueACL acl,
String queueName)
Check if the user has permission to perform the operation.
|
long |
checkAndGetApplicationLifetime(String queueName,
long lifetime)
Verify whether a submitted application lifetime is valid as per configured
Queue lifetime.
|
protected void |
completedContainerInternal(RMContainer rmContainer,
org.apache.hadoop.yarn.api.records.ContainerStatus containerStatus,
RMContainerEventType event)
Clean up a completed container.
|
protected void |
createPreemptionThread() |
AllocationConfiguration |
getAllocationConfiguration() |
List<org.apache.hadoop.yarn.api.records.ApplicationAttemptId> |
getAppsInQueue(String queueName)
Gets the apps under a given queue
|
FairSchedulerConfiguration |
getConf() |
RMContainerTokenSecretManager |
getContainerTokenSecretManager() |
FSContext |
getContext() |
int |
getContinuousSchedulingSleepMs()
Deprecated.
linked to
isContinuousSchedulingEnabled() deprecation |
org.apache.hadoop.yarn.api.records.Resource |
getIncrementResourceCapability() |
org.apache.hadoop.yarn.api.records.Resource |
getMaximumResourceCapability(String queueName)
Get maximum allocatable
Resource for the queue specified. |
long |
getNodeLocalityDelayMs()
Deprecated.
linked to
isContinuousSchedulingEnabled() deprecation |
double |
getNodeLocalityThreshold() |
org.apache.hadoop.yarn.api.records.Resource |
getNormalizedResource(org.apache.hadoop.yarn.api.records.Resource requestedResource,
org.apache.hadoop.yarn.api.records.Resource maxResourceCapability)
Normalize a resource request using scheduler level maximum resource or
queue based maximum resource.
|
int |
getNumClusterNodes()
Get the number of nodes available in the cluster.
|
int |
getNumNodesInRack(String rackName) |
Set<String> |
getPlanQueues()
Gets the list of names for queues managed by the Reservation System.
|
org.apache.hadoop.yarn.api.records.QueueInfo |
getQueueInfo(String queueName,
boolean includeChildQueues,
boolean recursive)
Get queue information.
|
QueueManager |
getQueueManager() |
List<org.apache.hadoop.yarn.api.records.QueueUserACLInfo> |
getQueueUserAclInfo()
Get acls for queues for current user.
|
long |
getRackLocalityDelayMs()
Deprecated.
linked to
isContinuousSchedulingEnabled() deprecation |
double |
getRackLocalityThreshold() |
float |
getReservableNodesRatio() |
org.apache.hadoop.yarn.util.resource.ResourceCalculator |
getResourceCalculator() |
RMContext |
getRMContext() |
QueueMetrics |
getRootQueueMetrics()
Get the root queue for the scheduler.
|
FSAppAttempt |
getSchedulerApp(org.apache.hadoop.yarn.api.records.ApplicationAttemptId appAttemptId) |
EnumSet<org.apache.hadoop.yarn.proto.YarnServiceProtos.SchedulerResourceTypes> |
getSchedulingResourceTypes()
Return a collection of the resource types that are considered when
scheduling
|
void |
handle(SchedulerEvent event) |
boolean |
isAtLeastReservationThreshold(org.apache.hadoop.yarn.util.resource.ResourceCalculator resourceCalculator,
org.apache.hadoop.yarn.api.records.Resource resource) |
boolean |
isContinuousSchedulingEnabled()
Deprecated.
Continuous scheduling should not be turned ON. It is
deprecated because it can cause scheduler slowness due to locking issues.
Schedulers should use assignmultiple as a replacement.
|
boolean |
isNoTerminalRuleCheck() |
boolean |
isSizeBasedWeight() |
void |
killContainer(RMContainer container)
Kill a RMContainer.
|
String |
moveApplication(org.apache.hadoop.yarn.api.records.ApplicationId appId,
String queueName)
Moves the given application to the given queue.
|
protected void |
nodeUpdate(RMNode nm)
Process a heartbeat update from a node.
|
void |
preValidateMoveApplication(org.apache.hadoop.yarn.api.records.ApplicationId appId,
String newQueue)
Perform pre-checks while moving the application.
|
void |
recover(RMStateStore.RMState state) |
void |
reinitialize(org.apache.hadoop.conf.Configuration conf,
RMContext rmContext)
Re-initialize the
ResourceScheduler . |
void |
removeQueue(String queueName)
Only supports removing empty leaf queues
|
void |
serviceInit(org.apache.hadoop.conf.Configuration conf) |
void |
serviceStart() |
void |
serviceStop() |
void |
setEntitlement(String queueName,
QueueEntitlement entitlement)
This method increase the entitlement for current queue (must respect
invariants, e.g., no overcommit of parents, non negative, etc.).
|
void |
setRMContext(RMContext rmContext)
Set RMContext for
ResourceScheduler . |
void |
update()
Recompute the internal variables used by the scheduler - per-job weights,
fair shares, deficits, minimum slot allocations, and amount of used and
required resources per job.
|
org.apache.hadoop.yarn.api.records.Priority |
updateApplicationPriority(org.apache.hadoop.yarn.api.records.Priority newPriority,
org.apache.hadoop.yarn.api.records.ApplicationId applicationId,
org.apache.hadoop.thirdparty.com.google.common.util.concurrent.SettableFuture<Object> future,
org.apache.hadoop.security.UserGroupInformation user)
Change application priority of a submitted application at runtime
|
void |
updateNodeResource(RMNode nm,
org.apache.hadoop.yarn.api.records.ResourceOption resourceOption)
Process resource update on a node and update Queue.
|
addQueue, asyncContainerRelease, attemptAllocationOnNode, checkAndGetApplicationPriority, clearPendingContainerCache, completedContainer, containerIncreasedOnNode, containerLaunchedOnNode, createReleaseCache, createSchedContainerChangeRequests, getActivitiesManager, getApplicationAttempt, getAppResourceUsageReport, getBlacklistedNodes, getClock, getClusterResource, getCurrentAttemptForContainer, getLastNodeUpdateTime, getMaxClusterLevelAppPriority, getMaximumAllocation, getMaximumApplicationLifetime, getMaximumResourceCapability, getMinimumAllocation, getMinimumResourceCapability, getNode, getNodeIds, getNodeReport, getNodes, getNodeTracker, getPendingResourceRequestsForAttempt, getPendingSchedulingRequestsForAttempt, getRMContainer, getSchedulerAppInfo, getSchedulerApplications, getSchedulerHealth, getSchedulerNode, getSchedulingMonitorManager, getSkipNodeInterval, getTransferredContainers, handleContainerUpdates, initMaximumResourceCapability, killAllAppsInQueue, moveAllApps, normalizeResourceRequests, normalizeResourceRequests, placementConstraintEnabled, recoverContainersOnNode, refreshMaximumAllocation, releaseContainers, resetSchedulerMetrics, rollbackContainerUpdate, setClock, setClusterMaxPriority, setLastNodeUpdateTime, shouldContainersBeAutoUpdated, triggerUpdate, updateNodeResourceUtilization, updateSchedulerHealthInformation
close, getBlockers, getConfig, getFailureCause, getFailureState, getLifecycleHistory, getName, getServiceState, getStartTime, init, isInState, noteFailure, putBlocker, registerGlobalListener, registerServiceListener, removeBlocker, setConfig, start, stop, toString, unregisterGlobalListener, unregisterServiceListener, waitForServiceToStop
public static final org.apache.hadoop.yarn.api.records.Resource CONTAINER_RESERVED
protected boolean sizeBasedWeight
@Deprecated protected boolean continuousSchedulingEnabled
@Deprecated protected volatile int continuousSchedulingSleepMs
protected double nodeLocalityThreshold
protected double rackLocalityThreshold
@Deprecated protected long nodeLocalityDelayMs
@Deprecated protected long rackLocalityDelayMs
protected boolean assignMultiple
protected int maxAssign
public FSContext getContext()
public RMContext getRMContext()
public boolean isAtLeastReservationThreshold(org.apache.hadoop.yarn.util.resource.ResourceCalculator resourceCalculator, org.apache.hadoop.yarn.api.records.Resource resource)
public FairSchedulerConfiguration getConf()
public int getNumNodesInRack(String rackName)
public QueueManager getQueueManager()
@VisibleForTesting public void update()
update
in class AbstractYarnScheduler<FSAppAttempt,FSSchedulerNode>
public RMContainerTokenSecretManager getContainerTokenSecretManager()
public boolean isSizeBasedWeight()
public org.apache.hadoop.yarn.api.records.Resource getIncrementResourceCapability()
public double getNodeLocalityThreshold()
public double getRackLocalityThreshold()
@Deprecated public long getNodeLocalityDelayMs()
isContinuousSchedulingEnabled()
deprecation@Deprecated public long getRackLocalityDelayMs()
isContinuousSchedulingEnabled()
deprecation@Deprecated public boolean isContinuousSchedulingEnabled()
@Deprecated public int getContinuousSchedulingSleepMs()
isContinuousSchedulingEnabled()
deprecationprotected void addApplication(org.apache.hadoop.yarn.api.records.ApplicationId applicationId, String queueName, String user, boolean isAppRecovering, ApplicationPlacementContext placementContext)
applicationId
- applicationId.queueName
- queue name.user
- submit application user.isAppRecovering
- true, app recover; false, app not recover.placementContext
- application placement context.protected void addApplicationAttempt(org.apache.hadoop.yarn.api.records.ApplicationAttemptId applicationAttemptId, boolean transferStateFromPreviousAttempt, boolean isAttemptRecovering)
applicationAttemptId
- application AttemptId.transferStateFromPreviousAttempt
- transferStateFromPreviousAttempt.isAttemptRecovering
- true, attempt recovering;false, attempt not recovering.protected void completedContainerInternal(RMContainer rmContainer, org.apache.hadoop.yarn.api.records.ContainerStatus containerStatus, RMContainerEventType event)
completedContainerInternal
in class AbstractYarnScheduler<FSAppAttempt,FSSchedulerNode>
public org.apache.hadoop.yarn.api.records.Resource getNormalizedResource(org.apache.hadoop.yarn.api.records.Resource requestedResource, org.apache.hadoop.yarn.api.records.Resource maxResourceCapability)
YarnScheduler
getNormalizedResource
in interface YarnScheduler
getNormalizedResource
in class AbstractYarnScheduler<FSAppAttempt,FSSchedulerNode>
requestedResource
- the resource to be normalizedmaxResourceCapability
- Maximum container allocation value, if null or
empty scheduler level maximum container allocation value will be
usedpublic org.apache.hadoop.yarn.api.records.Resource getMaximumResourceCapability(String queueName)
YarnScheduler
Resource
for the queue specified.getMaximumResourceCapability
in interface YarnScheduler
getMaximumResourceCapability
in class AbstractYarnScheduler<FSAppAttempt,FSSchedulerNode>
queueName
- queue name@VisibleForTesting public void killContainer(RMContainer container)
AbstractYarnScheduler
killContainer
in class AbstractYarnScheduler<FSAppAttempt,FSSchedulerNode>
container
- the container to killpublic Allocation allocate(org.apache.hadoop.yarn.api.records.ApplicationAttemptId appAttemptId, List<org.apache.hadoop.yarn.api.records.ResourceRequest> ask, List<org.apache.hadoop.yarn.api.records.SchedulingRequest> schedulingRequests, List<org.apache.hadoop.yarn.api.records.ContainerId> release, List<String> blacklistAdditions, List<String> blacklistRemovals, ContainerUpdates updateRequests)
YarnScheduler
appAttemptId
- the id of the application attempt.ask
- the request made by an application to obtain various allocations
like host/rack, resource, number of containers, relaxLocality etc.,
see ResourceRequest
.schedulingRequests
- similar to ask, but with added ability to specify
allocation tags etc., see SchedulingRequest
.release
- the list of containers to be released.blacklistAdditions
- places (node/rack) to be added to the blacklist.blacklistRemovals
- places (node/rack) to be removed from the
blacklist.updateRequests
- container promotion/demotion updates.Allocation
for the application.protected void nodeUpdate(RMNode nm)
AbstractYarnScheduler
nodeUpdate
in class AbstractYarnScheduler<FSAppAttempt,FSSchedulerNode>
nm
- The RMNode corresponding to the NodeManagerpublic FSAppAttempt getSchedulerApp(org.apache.hadoop.yarn.api.records.ApplicationAttemptId appAttemptId)
public org.apache.hadoop.yarn.util.resource.ResourceCalculator getResourceCalculator()
public QueueMetrics getRootQueueMetrics()
YarnScheduler
public void handle(SchedulerEvent event)
public void recover(RMStateStore.RMState state) throws Exception
Exception
public void setRMContext(RMContext rmContext)
ResourceScheduler
ResourceScheduler
.
This method should be called immediately after instantiating
a scheduler once.rmContext
- created by ResourceManager@VisibleForTesting protected void createPreemptionThread()
public void serviceInit(org.apache.hadoop.conf.Configuration conf) throws Exception
serviceInit
in class AbstractYarnScheduler<FSAppAttempt,FSSchedulerNode>
Exception
public void serviceStart() throws Exception
serviceStart
in class AbstractYarnScheduler<FSAppAttempt,FSSchedulerNode>
Exception
public void serviceStop() throws Exception
serviceStop
in class AbstractYarnScheduler<FSAppAttempt,FSSchedulerNode>
Exception
public void reinitialize(org.apache.hadoop.conf.Configuration conf, RMContext rmContext) throws IOException
ResourceScheduler
ResourceScheduler
.reinitialize
in interface ResourceScheduler
reinitialize
in class AbstractYarnScheduler<FSAppAttempt,FSSchedulerNode>
conf
- configurationrmContext
- RMContext.IOException
- an I/O exception has occurred.public org.apache.hadoop.yarn.api.records.QueueInfo getQueueInfo(String queueName, boolean includeChildQueues, boolean recursive) throws IOException
YarnScheduler
queueName
- queue nameincludeChildQueues
- include child queues?recursive
- get children queues?IOException
- an I/O exception has occurred.public List<org.apache.hadoop.yarn.api.records.QueueUserACLInfo> getQueueUserAclInfo()
YarnScheduler
public int getNumClusterNodes()
YarnScheduler
public boolean checkAccess(org.apache.hadoop.security.UserGroupInformation callerUGI, org.apache.hadoop.yarn.api.records.QueueACL acl, String queueName)
YarnScheduler
QueueACL.ADMINISTER_QUEUE
permission,
this user can view/modify the applications in this queue.callerUGI
- caller UserGroupInformation.acl
- queue ACL.queueName
- queue Name.true
if the user has the permission,
false
otherwisepublic AllocationConfiguration getAllocationConfiguration()
public List<org.apache.hadoop.yarn.api.records.ApplicationAttemptId> getAppsInQueue(String queueName)
YarnScheduler
queueName
- the name of the queue.public String moveApplication(org.apache.hadoop.yarn.api.records.ApplicationId appId, String queueName) throws org.apache.hadoop.yarn.exceptions.YarnException
YarnScheduler
moveApplication
in interface YarnScheduler
moveApplication
in class AbstractYarnScheduler<FSAppAttempt,FSSchedulerNode>
appId
- application IdqueueName
- the given queue.org.apache.hadoop.yarn.exceptions.YarnException
- if the move cannot be carried outpublic void preValidateMoveApplication(org.apache.hadoop.yarn.api.records.ApplicationId appId, String newQueue) throws org.apache.hadoop.yarn.exceptions.YarnException
preValidateMoveApplication
in interface YarnScheduler
preValidateMoveApplication
in class AbstractYarnScheduler<FSAppAttempt,FSSchedulerNode>
appId
- The ID of the app to be movednewQueue
- The name of the queue the app should move toorg.apache.hadoop.yarn.exceptions.YarnException
- if the validate failspublic void updateNodeResource(RMNode nm, org.apache.hadoop.yarn.api.records.ResourceOption resourceOption)
updateNodeResource
in class AbstractYarnScheduler<FSAppAttempt,FSSchedulerNode>
nm
- RMNode.resourceOption
- resourceOption.public EnumSet<org.apache.hadoop.yarn.proto.YarnServiceProtos.SchedulerResourceTypes> getSchedulingResourceTypes()
getSchedulingResourceTypes
in interface YarnScheduler
getSchedulingResourceTypes
in class AbstractYarnScheduler<FSAppAttempt,FSSchedulerNode>
public Set<String> getPlanQueues() throws org.apache.hadoop.yarn.exceptions.YarnException
YarnScheduler
getPlanQueues
in interface YarnScheduler
getPlanQueues
in class AbstractYarnScheduler<FSAppAttempt,FSSchedulerNode>
org.apache.hadoop.yarn.exceptions.YarnException
- when yarn exception occur.public void setEntitlement(String queueName, QueueEntitlement entitlement) throws org.apache.hadoop.yarn.exceptions.YarnException
YarnScheduler
setEntitlement
in interface YarnScheduler
setEntitlement
in class AbstractYarnScheduler<FSAppAttempt,FSSchedulerNode>
queueName
- the queue for which we change entitlemententitlement
- the new entitlement for the queue (capacity,
maxCapacity, etc..)org.apache.hadoop.yarn.exceptions.YarnException
- when yarn exception occur.public void removeQueue(String queueName) throws org.apache.hadoop.yarn.exceptions.YarnException
removeQueue
in interface YarnScheduler
removeQueue
in class AbstractYarnScheduler<FSAppAttempt,FSSchedulerNode>
queueName
- name of queue to removeorg.apache.hadoop.yarn.exceptions.YarnException
- if queue to remove is either not a leaf or if its
not emptypublic float getReservableNodesRatio()
public long checkAndGetApplicationLifetime(String queueName, long lifetime)
YarnScheduler
checkAndGetApplicationLifetime
in interface YarnScheduler
checkAndGetApplicationLifetime
in class AbstractYarnScheduler<FSAppAttempt,FSSchedulerNode>
queueName
- Name of the Queuelifetime
- configured application lifetimepublic org.apache.hadoop.yarn.api.records.Priority updateApplicationPriority(org.apache.hadoop.yarn.api.records.Priority newPriority, org.apache.hadoop.yarn.api.records.ApplicationId applicationId, org.apache.hadoop.thirdparty.com.google.common.util.concurrent.SettableFuture<Object> future, org.apache.hadoop.security.UserGroupInformation user) throws org.apache.hadoop.yarn.exceptions.YarnException
YarnScheduler
updateApplicationPriority
in interface YarnScheduler
updateApplicationPriority
in class AbstractYarnScheduler<FSAppAttempt,FSSchedulerNode>
newPriority
- Submitted Application priority.applicationId
- Application IDfuture
- Sets any type of exception happened from StateStoreuser
- who submitted the applicationorg.apache.hadoop.yarn.exceptions.YarnException
- when yarn exception occur.public boolean isNoTerminalRuleCheck()
Copyright © 2008–2024 Apache Software Foundation. All rights reserved.