@InterfaceAudience.Private @InterfaceStability.Unstable public class JobHistoryUtils extends Object
Modifier and Type | Field and Description |
---|---|
static String |
CONF_FILE_NAME_SUFFIX
Suffix for configuration files.
|
static org.apache.hadoop.fs.permission.FsPermission |
HISTORY_DONE_DIR_PERMISSION
Permissions for the history done dir and derivatives.
|
static org.apache.hadoop.fs.permission.FsPermission |
HISTORY_DONE_DIR_UMASK
Umask for the done dir and derivatives.
|
static org.apache.hadoop.fs.permission.FsPermission |
HISTORY_DONE_FILE_PERMISSION |
static org.apache.hadoop.fs.permission.FsPermission |
HISTORY_INTERMEDIATE_DONE_DIR_PERMISSIONS
Permissions for the intermediate done directory.
|
static org.apache.hadoop.fs.permission.FsPermission |
HISTORY_STAGING_DIR_PERMISSIONS
Permissions for the history staging dir while JobInProgress.
|
static org.apache.hadoop.fs.permission.FsPermission |
HISTORY_STAGING_USER_DIR_PERMISSIONS
Permissions for the user directory under the staging directory.
|
static String |
JOB_HISTORY_FILE_EXTENSION
Job History File extension.
|
static int |
SERIAL_NUMBER_DIRECTORY_DIGITS |
static String |
SUMMARY_FILE_NAME_SUFFIX
Suffix for summary files.
|
static Pattern |
TIMESTAMP_DIR_PATTERN |
static String |
TIMESTAMP_DIR_REGEX |
static int |
VERSION |
Constructor and Description |
---|
JobHistoryUtils() |
Modifier and Type | Method and Description |
---|---|
static String |
doneSubdirsBeforeSerialTail() |
static org.apache.hadoop.fs.PathFilter |
getConfFileFilter()
Gets a PathFilter which would match configuration files.
|
static String |
getConfiguredHistoryIntermediateDoneDirPrefix(org.apache.hadoop.conf.Configuration conf)
Gets the configured directory prefix for intermediate done history files.
|
static org.apache.hadoop.fs.permission.FsPermission |
getConfiguredHistoryIntermediateUserDoneDirPermissions(org.apache.hadoop.conf.Configuration conf)
Gets the configured directory permissions for the user directories in the
Gets the configured permissions for the user directories and files in the
both need full permissions, this is enforced by this method.
|
static String |
getConfiguredHistoryServerDoneDirPrefix(org.apache.hadoop.conf.Configuration conf)
Gets the configured directory prefix for Done history files.
|
static String |
getConfiguredHistoryStagingDirPrefix(org.apache.hadoop.conf.Configuration conf,
String jobId)
Gets the configured directory prefix for In Progress history files.
|
static List<org.apache.hadoop.fs.FileStatus> |
getHistoryDirsForCleaning(org.apache.hadoop.fs.FileContext fc,
org.apache.hadoop.fs.Path root,
long cutoff)
Looks for the dirs to clean.
|
static org.apache.hadoop.fs.PathFilter |
getHistoryFileFilter()
Gets a PathFilter which would match job history file names.
|
static String |
getHistoryIntermediateDoneDirForUser(org.apache.hadoop.conf.Configuration conf)
Gets the user directory for intermediate done history files.
|
static String |
getIntermediateConfFileName(JobId jobId)
Get the done configuration file name for a job.
|
static String |
getIntermediateSummaryFileName(JobId jobId)
Get the done summary file name for a job.
|
static org.apache.hadoop.mapreduce.JobID |
getJobIDFromHistoryFilePath(String pathString)
Returns the jobId from a job history file name.
|
static org.apache.hadoop.fs.Path |
getPreviousJobHistoryPath(org.apache.hadoop.conf.Configuration conf,
org.apache.hadoop.yarn.api.records.ApplicationAttemptId applicationAttemptId) |
static org.apache.hadoop.fs.Path |
getStagingConfFile(org.apache.hadoop.fs.Path logDir,
JobId jobId,
int attempt)
Gets the conf file path for jobs in progress.
|
static org.apache.hadoop.fs.Path |
getStagingJobHistoryFile(org.apache.hadoop.fs.Path dir,
JobId jobId,
int attempt)
Get the job history file path for non Done history files.
|
static org.apache.hadoop.fs.Path |
getStagingJobHistoryFile(org.apache.hadoop.fs.Path dir,
String jobId,
int attempt)
Get the job history file path for non Done history files.
|
static String |
getTimestampPartFromPath(String path)
Extracts the timstamp component from the path.
|
static String |
historyLogSubdirectory(JobId id,
String timestampComponent,
String serialNumberFormat)
Gets the history subdirectory based on the jobId, timestamp and serial number format.
|
static boolean |
isValidJobHistoryFileName(String pathString)
Checks whether the provided path string is a valid job history file.
|
static int |
jobSerialNumber(JobId id)
Computes a serial number used as part of directory naming for the given jobId.
|
static List<org.apache.hadoop.fs.FileStatus> |
localGlobber(org.apache.hadoop.fs.FileContext fc,
org.apache.hadoop.fs.Path root,
String tail) |
static List<org.apache.hadoop.fs.FileStatus> |
localGlobber(org.apache.hadoop.fs.FileContext fc,
org.apache.hadoop.fs.Path root,
String tail,
org.apache.hadoop.fs.PathFilter filter) |
static List<org.apache.hadoop.fs.FileStatus> |
localGlobber(org.apache.hadoop.fs.FileContext fc,
org.apache.hadoop.fs.Path root,
String tail,
org.apache.hadoop.fs.PathFilter filter,
AtomicBoolean hasFlatFiles) |
static String |
serialNumberDirectoryComponent(JobId id,
String serialNumberFormat)
Gets the serial number part of the path based on the jobId and serialNumber format.
|
static boolean |
shouldCreateNonUserDirectory(org.apache.hadoop.conf.Configuration conf) |
static String |
timestampDirectoryComponent(long millisecondTime)
Gets the timestamp component based on millisecond time.
|
public static final org.apache.hadoop.fs.permission.FsPermission HISTORY_STAGING_DIR_PERMISSIONS
public static final org.apache.hadoop.fs.permission.FsPermission HISTORY_STAGING_USER_DIR_PERMISSIONS
public static final org.apache.hadoop.fs.permission.FsPermission HISTORY_DONE_DIR_PERMISSION
public static final org.apache.hadoop.fs.permission.FsPermission HISTORY_DONE_FILE_PERMISSION
public static final org.apache.hadoop.fs.permission.FsPermission HISTORY_DONE_DIR_UMASK
public static final org.apache.hadoop.fs.permission.FsPermission HISTORY_INTERMEDIATE_DONE_DIR_PERMISSIONS
public static final String CONF_FILE_NAME_SUFFIX
public static final String SUMMARY_FILE_NAME_SUFFIX
public static final String JOB_HISTORY_FILE_EXTENSION
public static final int VERSION
public static final int SERIAL_NUMBER_DIRECTORY_DIGITS
public static final String TIMESTAMP_DIR_REGEX
public static final Pattern TIMESTAMP_DIR_PATTERN
public static boolean isValidJobHistoryFileName(String pathString)
pathString
- the path to be checked.public static org.apache.hadoop.mapreduce.JobID getJobIDFromHistoryFilePath(String pathString) throws IOException
pathString
- the path string.IOException
- if the filename format is invalid.public static org.apache.hadoop.fs.PathFilter getConfFileFilter()
PathFilter
for matching conf files.public static org.apache.hadoop.fs.PathFilter getHistoryFileFilter()
PathFilter
matching job history files.public static String getConfiguredHistoryStagingDirPrefix(org.apache.hadoop.conf.Configuration conf, String jobId) throws IOException
conf
- the configuration for hte jobjobId
- the id of the job the history file is for.IOException
public static String getConfiguredHistoryIntermediateDoneDirPrefix(org.apache.hadoop.conf.Configuration conf)
conf
- public static org.apache.hadoop.fs.permission.FsPermission getConfiguredHistoryIntermediateUserDoneDirPermissions(org.apache.hadoop.conf.Configuration conf)
conf
- The configuration objectpublic static String getConfiguredHistoryServerDoneDirPrefix(org.apache.hadoop.conf.Configuration conf)
conf
- the configuration objectpublic static String getHistoryIntermediateDoneDirForUser(org.apache.hadoop.conf.Configuration conf) throws IOException
conf
- the configuration objectIOException
public static boolean shouldCreateNonUserDirectory(org.apache.hadoop.conf.Configuration conf)
public static org.apache.hadoop.fs.Path getStagingJobHistoryFile(org.apache.hadoop.fs.Path dir, JobId jobId, int attempt)
public static org.apache.hadoop.fs.Path getStagingJobHistoryFile(org.apache.hadoop.fs.Path dir, String jobId, int attempt)
public static String getIntermediateConfFileName(JobId jobId)
jobId
- the jobId.public static String getIntermediateSummaryFileName(JobId jobId)
jobId
- the jobId.public static org.apache.hadoop.fs.Path getStagingConfFile(org.apache.hadoop.fs.Path logDir, JobId jobId, int attempt)
logDir
- the log directory prefix.jobId
- the jobId.attempt
- attempt number for this job.public static String serialNumberDirectoryComponent(JobId id, String serialNumberFormat)
id
- serialNumberFormat
- public static String getTimestampPartFromPath(String path)
path
- public static String historyLogSubdirectory(JobId id, String timestampComponent, String serialNumberFormat)
id
- timestampComponent
- serialNumberFormat
- public static String timestampDirectoryComponent(long millisecondTime)
millisecondTime
- public static String doneSubdirsBeforeSerialTail()
public static int jobSerialNumber(JobId id)
id
- the jobId.public static List<org.apache.hadoop.fs.FileStatus> localGlobber(org.apache.hadoop.fs.FileContext fc, org.apache.hadoop.fs.Path root, String tail) throws IOException
IOException
public static List<org.apache.hadoop.fs.FileStatus> localGlobber(org.apache.hadoop.fs.FileContext fc, org.apache.hadoop.fs.Path root, String tail, org.apache.hadoop.fs.PathFilter filter) throws IOException
IOException
public static List<org.apache.hadoop.fs.FileStatus> localGlobber(org.apache.hadoop.fs.FileContext fc, org.apache.hadoop.fs.Path root, String tail, org.apache.hadoop.fs.PathFilter filter, AtomicBoolean hasFlatFiles) throws IOException
IOException
public static org.apache.hadoop.fs.Path getPreviousJobHistoryPath(org.apache.hadoop.conf.Configuration conf, org.apache.hadoop.yarn.api.records.ApplicationAttemptId applicationAttemptId) throws IOException
IOException
public static List<org.apache.hadoop.fs.FileStatus> getHistoryDirsForCleaning(org.apache.hadoop.fs.FileContext fc, org.apache.hadoop.fs.Path root, long cutoff) throws IOException
fc
- done dir FileContextroot
- folder for completed jobscutoff
- The cutoff for the max history ageIOException
Copyright © 2008–2024 Apache Software Foundation. All rights reserved.