Secure your code as it's written. Use Snyk Code to scan source code in minutes - no build needed - and fix issues immediately.
def initHeapUsageAndGCPause(self, gclogFile, gcpauseFile):
fileLines = FileReader.readLines(gclogFile)
for line in fileLines:
if (line.strip != ""):
heapUsage = self.parseUsage(line)
if(line.startswith("[Young]")):
self.youngGen.append(heapUsage)
elif(line.startswith("[Old]")):
self.oldGen.append(heapUsage)
elif(line.startswith("[Metaspace]")):
self.metaGen.append(heapUsage)
fileLines2 = FileReader.readLines(gcpauseFile)
self.gcTotal["GCPause"] = []
self.gcTimeLine["GCPause"] = []
for line in fileLines2:
if not line[0].isdigit():
continue
def initHeapUsageAndGCPause(self, gclogFile, gcpauseFile):
fileLines = FileReader.readLines(gclogFile)
for line in fileLines:
if (line.strip != ""):
heapUsage = self.parseUsage(line)
if(line.startswith("[Young]")):
self.youngGen.append(heapUsage)
elif(line.startswith("[Old]")):
self.oldGen.append(heapUsage)
elif(line.startswith("[Metaspace]")):
self.metaGen.append(heapUsage)
fileLines2 = FileReader.readLines(gcpauseFile)
for line in fileLines2:
if not line[0].isdigit():
continue
items = re.split(",", line)
gc_type = items[-1]
def initHeapUsage(self, gclogFile, timeOffset):
fileLines = FileReader.readLines(gclogFile)
for line in fileLines:
if (line.strip != ""):
heapUsage = self.parseUsage(line, timeOffset)
if(line.startswith("[Young]")):
self.youngGen.append(heapUsage)
elif(line.startswith("[Old]")):
self.oldGen.append(heapUsage)
def plotResourceUsage(topMetricsFile, slowestTasksDir, appName):
fileLines = FileReader.readLines(topMetricsFile)
isExecutorMetric = False
isSlaveMetric = False
executorTime = []
executorCPU = []
executorMemory = []
slaveTime = []
slaveCPU = []
slaveMemory = []
first_time = -1
for line in fileLines:
if (line.startswith("[Top Metrics][Executor")):
def fillStatistics(self, metrics, statisticsDir, statisticsFiles, metricsMap, withMax):
metricsTupleDict = {} # ["app.duration", ("app.duration", "Time (s)", 1000)]
for tuple in metrics:
metricsTupleDict[tuple[0]] = tuple
for file in statisticsFiles:
if file.endswith("stat.txt"): # [RDDJoin-CMS-1-7G-stat.txt, RDDJoin-CMS-2-14G-stat.txt, ...]
# [app.duration] mean = 224339.20, stdVar = 8311.91, median = 225233.00, min = 211999.00, quantile25 = 216682.50, quantile75 = 231549.00, max = 233837.00
# -------------------------------------------------------------------[Stage 0]-------------------------------------------------------------------
# [stage0.duration] mean = 42360.60, stdVar = 4069.63, median = 41404.00, min = 37094.00, quantile25 = 38942.50, quantile75 = 46257.00, max = 47801.00
# [stage0.inputBytes] mean = 8588671743.00, stdVar = 0.00, median = 8588671743.00, min = 8588671743.00, quantile25 = 8588671743.00, quantile75 = 8588671743.00, max = 8588671743.00
# [stage0.inputRecords] mean = 66000000.00, stdVar = 0.00, median = 66000000.00, min = 66000000.00, quantile25 = 66000000.00, quantile75 = 66000000.00, max = 66000000.00
for line in FileReader.readLines(os.path.join(statisticsDir, file)):
metricName = line[line.find('[') + 1: line.find(']')]
if metricsTupleDict.has_key(metricName):
if metricsMap.has_key(metricName):
metricsMap[metricName].addStatistics(line, file, withMax)
else:
statistics = st.BoxPlotStatistics(metricsTupleDict[metricName])
statistics.addStatistics(line, file, withMax)
metricsMap[metricName] = statistics
# Fill the NaA values
for metricName, statistics in metricsMap.items():
statistics.checkAndFillNulls()
def initHeapUsage(self, gclogFile):
fileLines = FileReader.readLines(gclogFile)
for line in fileLines:
if (line.strip != ""):
heapUsage = self.parseUsage(line)
if(line.startswith("[Young]")):
self.youngGen.append(heapUsage)
elif(line.startswith("[Old]")):
self.oldGen.append(heapUsage)
def initHeapUsage(self, gclogFile, timeOffset):
fileLines = FileReader.readLines(gclogFile)
for line in fileLines:
if (line.strip != ""):
heapUsage = self.parseUsage(line, timeOffset)
if(line.startswith("[Young]")):
self.youngGen.append(heapUsage)
elif(line.startswith("[Old]")):
self.oldGen.append(heapUsage)
def initHeapUsage(self, gclogFile, timeOffset):
fileLines = FileReader.readLines(gclogFile)
for line in fileLines:
if (line.strip != ""):
heapUsage = self.parseUsage(line, timeOffset)
if(line.startswith("[Young]")):
self.youngGen.append(heapUsage)
elif(line.startswith("[Old]")):
self.oldGen.append(heapUsage)
executor = ""
if (file.find("CMS") != -1):
GC = "CMS"
elif (file.find("G1") != -1):
GC = "G1"
elif (file.find("Parallel") != -1):
GC = "Parallel"
if (file.find("1-7G") != -1):
executor = "E1"
elif (file.find("2-14G") != -1):
executor = "E2"
elif (file.find("4-28G") != -1):
executor = "E4"
for line in FileReader.readLines(os.path.join(self.taskInfoDir, file)):
if line.startswith('[appName'):
appName = line[line.find('=') + 2: line.find(']')]
elif line.startswith('[appId'):
appId = line[line.find('=') + 2: line.find(']')]
elif line.startswith('[stageId'):
stageId = int(line[line.find('=') + 2: line.find(']')])
elif line.startswith('['):
metricName = line[line.find('task') + 5: line.find(']')]
metricValue = float(line[line.find(']') + 2:])
if (metricName == 'index'):
taskAttempt = ta.TaskAttempt(appId, appName, stageId, int(metricValue), GC, executor)
taskAttempt.set(metricName, metricValue)
self.taskList.append(taskAttempt)
def plotResourceUsage(topMetricsFile, slowestTasksDir, appName):
fileLines = FileReader.readLines(topMetricsFile)
isExecutorMetric = False
isSlaveMetric = False
executorTime = []
executorCPU = []
executorMemory = []
slaveTime = []
slaveCPU = []
slaveMemory = []
for line in fileLines:
if (line.startswith("[Top Metrics][Executor")):
isExecutorMetric = True
elif (line.startswith("[Top Metrics][aliSlave")):