Source code for PyFoam.Applications.ListProfilingInfo

"""
Application-class that implements pyFoamListProfilingInfo.py
"""
from optparse import OptionGroup

from .PyFoamApplication import PyFoamApplication
from .CommonSelectTimesteps import CommonSelectTimesteps

from PyFoam.ThirdParty.six import print_

from PyFoam.RunDictionary.ParsedParameterFile import ParsedParameterFile
from PyFoam.RunDictionary.SolutionDirectory import SolutionDirectory

from os import path
from glob import glob

[docs]class ListProfilingInfo(PyFoamApplication, CommonSelectTimesteps): def __init__(self, args=None, **kwargs): description="""\ List the profiling information in time directories (either created by a patched OpenFOAM-version, Foam-extend or by a OpenFOAM-version with the profiling-patch applied) and prints them in a human-readable form. Either gets files or a case directory from which it tries to get the proper files. If more than one file is specified it is assumed that this is a parallel run and the data is accumulated Results are printed as a table. The first column is the name of the profiling entry. Children entries (entries that are called by that entry) are indented. The first numeric column is the percentage that this entry used of the total time (including time spent in children). The next entry is the percentages of the 'self' time (the time spent in this entry minus the time spent in the children) as the percentage of the total execution time. After that the percentages of the parent time (total and 'self'). After that the number of times this entry was called is printed. Then the total time spent in this entry and the time without the child entries are printed. If the data from multiple processors is used then the totalTime and the calls are the average of all processors. Also are there three more columns: the range of the totalTime (maximum-minimum). How big that range is compared to the average totalTime and the range of the calls """ examples="""\ %prog aCase/100/uniform/profilingInfo Print the profiling info of a case named aCase at time 100 %prog aCase --time=100 Also print the profiling info of a case named aCase at time 100 %prog aCase --latest-time Print the profiling info from the latest timestep in case aCase %prog aCase --latest-time --parallel Print the profiling information from the latest timestep in the case aCase but use the data from all processors and accumulate them %prog aCase --latest-time --parallel --sort-by=totalTime --depth=2 Sort the profiling data by the total time that was used and only print the first two levels %prog aCase --latest-time --threshold-low=1 --graphviz-dot | dot -Tsvg -o aCase.svg Remove all nodes that need less than 1% of the computation time and generate an SVG-file with a graph. This requires the dot-utility from GraphViz to be present %prog aCase --latest-time --threshold-low=1 --graphviz-dot --dot-append-title="$(date)" | dot -Tsvg -o aCase.svg Plots the same with the current time added to the title (whether "$(date)" works depends on the used shell) """ PyFoamApplication.__init__(self, args=args, description=description, examples=examples, usage="%prog [<caseDirectory>|<profiling file>]", interspersed=True, changeVersion=True, nr=1, exactNr=False, **kwargs)
[docs] def addOptions(self): CommonSelectTimesteps.addOptions(self,False,singleTime=True) output=OptionGroup(self.parser, "Output", "How data should be output") self.parser.add_option_group(output) sortModes=["id","totalTime","selfTime","description","calls"] output.add_option("--sort-by", type="choice", dest="sortBy", default="id", choices=sortModes, help="How the entries should be sorted in their 'subtree'. Possible values are "+", ".join(sortModes)+". Default is 'id' which is more or less the order in which these entries were registered") output.add_option("--depth", type="int", dest="depth", default=0, help="How many levels of the tree should be printed. If 0 or smaller then everything is printed") output.add_option("--threshold-low", type="float", dest="threshold_low", default=None, help="A percent value. If a node has less than this value it (and its children) will not be displayed") output.add_option("--graphviz-dot", default=False, dest="graphviz_dot", action="store_true", help="Instead of the list print out an output that can be piped into the utility called 'dot' of the GraphViz suite to produce a graphical tree representation of the profiling data") dot = OptionGroup(self.parser, "Dot", "Options that are used for the graphviz-dot output") self.parser.add_option_group(dot) dot.add_option("--dot-theme", dest="dot_theme", default=themes.keys()[0], help="Theme to be used. possible values are: " + ", ".join(themes.keys()) + ". Default: %default") dot.add_option("--dot-override-title", dest="dot_override_title", default=None, help="For dot plots use this title instead of the automatically generated title of either case-name and time or filename. Can have '\\n' for multiple lines") dot.add_option("--dot-append-title", dest="dot_append_title", default=None, help="Append this to the title on a new line. Can have '\\n' for multiple lines")
[docs] def readProfilingInfo(self,fName): """Read the info from a file and return a tuple with (date,children,root)""" pf=ParsedParameterFile(fName, treatBinaryAsASCII=True) try: # Foam-extend and Patch pi=pf["profilingInfo"] newFormat=False except KeyError: try: pi=pf["profiling"] newFormat=True except KeyError: self.error("No profiling info found in",fName) data={} children={} root=None for p in pi: if newFormat: p=pi[p] if p["id"] in data: print_("Duplicate definition of",p["id"]) sys.exit(-1) if p["description"][0]=='"': p["description"]=p["description"][1:] if p["description"][-1]=='"': p["description"]=p["description"][:-1] data[p["id"]]=p if "parentId" in p: if p["parentId"] in children: children[p["parentId"]].append(p["id"]) else: children[p["parentId"]]=[p["id"]] else: if root!=None: print_("Two root elements") sys-exit(-1) else: root=p["id"] p["selfTime"]=p["totalTime"]-p["childTime"] return data,children,root
[docs] def printProfilingInfo(self,data,children,root,parallel=False): """Prints the profiling info in a pseudo-graphical form""" def depth(i): if i in children: return max([depth(j) for j in children[i]])+1 else: return 0 maxdepth=depth(root) depths={} def nameLen(i,d=0): depths[i]=d maxi=len(data[i]["description"]) if i in children: maxi=max(maxi,max([nameLen(j,d+1) for j in children[i]])) if self.opts.depth>0 and depths[i]>self.opts.depth: return 0 else: return maxi+3 maxLen=nameLen(root) format=" %5.1f%% (%5.1f%%) - %5.1f%% (%5.1f%%) | %8d %9.4gs %9.4gs" if parallel: parallelFormat=" | %9.4gs %5.1f%% %9.4g" totalTime=data[root]["totalTime"] header=" "*(maxLen)+" | total ( self ) - parent ( self ) | calls total self " if parallel: header+="| range(total) / % range(calls) " print_(header) print_("-"*len(header)) def printItem(i): result="" if self.opts.depth>0 and depths[i]>self.opts.depth: return "" if depths[i]>1: result+=" "*(depths[i]-1) if depths[i]>0: result+="|- " result+=data[i]["description"] result+=" "*(maxLen-len(result)+1)+"| " parentTime=data[i]["totalTime"] if "parentId" in data[i]: parentTime=data[data[i]["parentId"]]["totalTime"] tt=data[i]["totalTime"] ct=data[i]["childTime"] st=data[i]["selfTime"] result+=format % (100*tt/totalTime, 100*st/totalTime, 100*tt/parentTime, 100*st/tt, data[i]["calls"], tt, st) if parallel: timeRange=data[i]["totalTimeMax"]-data[i]["totalTimeMin"] result+=parallelFormat % (timeRange, 100*timeRange/tt, data[i]["callsMax"]-data[i]["callsMin"]) print_(result) if i in children: def getKey(k): def keyF(i): return data[i][k] return keyF #make sure that children are printed in the correct order if self.opts.sortBy=="id": children[i].sort() else: children[i].sort( key=getKey(self.opts.sortBy), reverse=self.opts.sortBy in ["totalTime","selfTime","calls"]) for c in children[i]: printItem(c) printItem(root)
AddFields = ["totalTime", "totalTimeMin", "totalTimeMax", "childTime", "selfTime", "nr_removed", "calls", "callsMin", "callsMax"]
[docs] def printDotGraph(self, data, children, root, theme, title=None): totalTime = data[root]["totalTime"] terminal_nodes = [] non_terminal = [] def get_nodes(node): if node in children: non_terminal.append(node) for c in children[node]: get_nodes(c) else: terminal_nodes.append(node) get_nodes(root) dot_nodes = {} translation = {} for i, n in enumerate(non_terminal): translation[n] = i dot_nodes[i] = data[n] descr = {} for n in terminal_nodes: d = data[n] nm = d["description"] if nm not in descr: new_id = max(dot_nodes.keys()) + 1 descr[nm] = new_id dot_nodes[new_id] = d.copy() else: dst = dot_nodes[descr[nm]] for k in self.AddFields: if k in dst: dst[k] += d[k] translation[n] = descr[nm] def color(rgb): r, g, b = rgb def float2int(f): if f <= 0.0: return 0 if f >= 1.0: return 255 return int(255.0*f + 0.5) return "#" + "".join(["%02x" % float2int(c) for c in (r, g, b)]) print_("""digraph {{ graph [fontname={fontname}, nodesep=0.125, ranksep=0.25]; node [fontcolor={fontcolor}, fontsize=10, fontname={fontname}, height=0, shape=box, style={nodestyle}, width=0]; edge [fontname={fontname}, fontsize=7];\n""".format(fontname=theme.graph_fontname(), fontcolor=theme.graph_fontcolor(), nodestyle=theme.node_style())) if title is not None: print_('label = "{}";'.format(title.replace("\n","\\n"))) print_('labelloc = "top";') print_('labeljust = "left";') def node_name(n): return "node{}".format(translation[n]) def make_edges(node): if node not in children: return p = data[node] for c in children[node]: d = data[c] label_text = "{:6g}s = {:.2f}%".format(d["totalTime"], 100*d["totalTime"] / totalTime) label_text += "\\n{:6g} x".format(d["calls"]) frac = d["totalTime"] / p["totalTime"] label_text += "\\n{:.2f}% of parent".format(100 * frac) total_frac = d["totalTime"] / totalTime ratio = 0.5 + 10 * total_frac try: label_text += "\\nReplaces {} paths".format(d["nr_removed"]) except KeyError: pass print_(' {src} -> {dst} [color="{color}", fontcolor="{fontcolor}", fontsize="{fontsize}", label="{label}", penwidth="{width}", arrowsize="{arrow}", labeldistance="{width}"];'.format( src=node_name(node), dst=node_name(c), label=label_text, color=color(theme.edge_color(frac)), fontcolor=color(theme.edge_color(frac)), fontsize=theme.edge_fontsize(total_frac), width=ratio, arrow=1/ratio)) make_edges(c) make_edges(root) for t, n in translation.items(): d = dot_nodes[n] label_text = d["description"] label_text += "\\n{:6g}s = {:.2f}%".format(d["totalTime"], 100*d["totalTime"]/totalTime) if t not in terminal_nodes: label_text += "\\nSelf: {:.2f}%".format(100*(d["selfTime"]/d["totalTime"])) label_text += "\\n{:6g} x".format(d["calls"]) try: label_text += "\\nReplaces {} nodes".format(d["nr_removed"]) except KeyError: pass weight = (d["selfTime"] if t in children else d["totalTime"]) / totalTime print(' {node} [ color="{bgcolor}", fontcolor="{fontcolor}", fontsize="{fontsize}", label="{label}"];'.format( node="node{}".format(n), bgcolor=color(theme.node_bgcolor(weight)), fontcolor=color(theme.node_fgcolor(weight)), fontsize=theme.node_fontsize(weight), label=label_text)) print_("}")
[docs] def clip_small(self, threshold, data, children, root): thres = data[root]["totalTime"] * threshold / 100. def remove_small_children(node): if node not in children: return remove = [] for c in children[node]: if data[c]["totalTime"] < thres: remove.append(c) for c in remove: children[node].remove(c) for c in children[node]: remove_small_children(c) if len(remove) > 0: rest = data[remove[0]].copy() rest["description"] = "< less than {}% >".format(threshold) new_id = max(data.keys())+1 rest["id"] = new_id rest["nr_removed"] = 1 for c in remove[1:]: for k in self.AddFields: if k in data[c]: rest[k] += data[c][k] rest["nr_removed"] += 1 data[new_id] = rest children[node].append(new_id) remove_small_children(root) return data, children
[docs] def run(self): files=self.parser.getArgs()[0:] usedTime = None usedCase = None if self.opts.graphviz_dot and self.opts.depth > 0: self.error("The depth-option can't be used for the graphviz plots") if len(files)==1 and path.isdir(files[0]): usedCase = path.abspath(self.parser.getArgs()[0]) sol=SolutionDirectory( usedCase, archive=None, parallel=self.opts.parallelTimes, paraviewLink=False) self.processTimestepOptions(sol) if len(self.opts.time)<1: self.error("No time specified") globStr=self.parser.getArgs()[0] if self.opts.parallelTimes: globStr=path.join(globStr,"processor*") usedTime=sol.timeName(self.opts.time[0]) globStr=path.join(globStr, usedTime, "uniform","profiling*") files=glob(globStr) if not self.opts.graphviz_dot: print_("Profiling info from time", usedTime) used_data = None parallel = False if len(files)<1: self.error("No profiling data found") elif len(files)>1: lst=[] for f in files: lst.append(self.readProfilingInfo(f)) dataAll,children0,root0=lst[0] for i in dataAll: d=dataAll[i] d["totalTimeMin"]=d["totalTime"] d["totalTimeMax"]=d["totalTime"] d["callsMin"]=d["calls"] d["callsMax"]=d["calls"] for data,children,root in lst[1:]: if root0!=root or children!=children0 or data.keys()!=dataAll.keys(): self.error("Inconsistent profiling data. Probably not from same run/timestep") for i in data: d=data[i] s=dataAll[i] s["totalTime"]+=d["totalTime"] s["totalTimeMin"]=min(s["totalTimeMin"],d["totalTime"]) s["totalTimeMax"]=max(s["totalTimeMax"],d["totalTime"]) s["calls"]+=d["calls"] s["callsMin"]=min(s["callsMin"],d["calls"]) s["callsMax"]=max(s["callsMax"],d["calls"]) s["childTime"]+=d["childTime"] for i in dataAll: d=dataAll[i] d["totalTime"]=d["totalTime"]/len(lst) d["childTime"]=d["childTime"]/len(lst) d["calls"]=d["calls"]/len(lst) used_data = dataAll parallel = True else: used_data, children, root = self.readProfilingInfo(files[0]) if self.opts.threshold_low is not None: used_data, children = self.clip_small(max(min(self.opts.threshold_low, 99.99), 0), used_data, children, root) if self.opts.graphviz_dot: if self.opts.dot_theme not in themes.keys(): self.error("Unknown theme name '{}'. Possible values are: {}".format( self.opts.dot_theme, ", ".join(themes.keys()))) title = None if usedTime: title = "Case: {}\nt={}".format(path.basename(usedCase), usedTime) else: title = "Files: " + ", ".join(files) if self.opts.dot_override_title is not None: title = self.opts.dot_override_title if self.opts.dot_append_title is not None: title += "\n" + self.opts.dot_append_title self.printDotGraph(used_data, children, root, themes[self.opts.dot_theme], title=title) else: self.printProfilingInfo(used_data, children, root, parallel)
# Color handling. Lifted from # https://github.com/jrfonseca/gprof2dot/blob/master/gprof2dot.py # Will be moved to basic should th graphviz-dot stuff ever be needed elsewhere
[docs]class Theme: def __init__(self, bgcolor = (0.0, 0.0, 1.0), mincolor = (0.0, 0.0, 0.0), maxcolor = (0.0, 0.0, 1.0), fontname = "Arial", fontcolor = "white", nodestyle = "filled", minfontsize = 10.0, maxfontsize = 10.0, minpenwidth = 0.5, maxpenwidth = 4.0, gamma = 2.2, skew = 1.0): self.bgcolor = bgcolor self.mincolor = mincolor self.maxcolor = maxcolor self.fontname = fontname self.fontcolor = fontcolor self.nodestyle = nodestyle self.minfontsize = minfontsize self.maxfontsize = maxfontsize self.minpenwidth = minpenwidth self.maxpenwidth = maxpenwidth self.gamma = gamma self.skew = skew
[docs] def graph_bgcolor(self): return self.hsl_to_rgb(*self.bgcolor)
[docs] def graph_fontname(self): return self.fontname
[docs] def graph_fontcolor(self): return self.fontcolor
[docs] def graph_fontsize(self): return self.minfontsize
[docs] def node_bgcolor(self, weight): return self.color(weight)
[docs] def node_fgcolor(self, weight): if self.nodestyle == "filled": return self.graph_bgcolor() else: return self.color(weight)
[docs] def node_fontsize(self, weight): return self.fontsize(weight)
[docs] def node_style(self): return self.nodestyle
[docs] def edge_color(self, weight): return self.color(weight)
[docs] def edge_fontsize(self, weight): return self.fontsize(weight)
[docs] def edge_penwidth(self, weight): return max(weight*self.maxpenwidth, self.minpenwidth)
[docs] def edge_arrowsize(self, weight): return 0.5 * math.sqrt(self.edge_penwidth(weight))
[docs] def fontsize(self, weight): return max(weight**2 * self.maxfontsize, self.minfontsize)
[docs] def color(self, weight): weight = min(max(weight, 0.0), 1.0) hmin, smin, lmin = self.mincolor hmax, smax, lmax = self.maxcolor if self.skew < 0: raise ValueError("Skew must be greater than 0") elif self.skew == 1.0: h = hmin + weight*(hmax - hmin) s = smin + weight*(smax - smin) l = lmin + weight*(lmax - lmin) else: base = self.skew h = hmin + ((hmax-hmin)*(-1.0 + (base ** weight)) / (base - 1.0)) s = smin + ((smax-smin)*(-1.0 + (base ** weight)) / (base - 1.0)) l = lmin + ((lmax-lmin)*(-1.0 + (base ** weight)) / (base - 1.0)) return self.hsl_to_rgb(h, s, l)
[docs] def hsl_to_rgb(self, h, s, l): """Convert a color from HSL color-model to RGB. See also: - http://www.w3.org/TR/css3-color/#hsl-color """ h = h % 1.0 s = min(max(s, 0.0), 1.0) l = min(max(l, 0.0), 1.0) if l <= 0.5: m2 = l*(s + 1.0) else: m2 = l + s - l*s m1 = l*2.0 - m2 r = self._hue_to_rgb(m1, m2, h + 1.0/3.0) g = self._hue_to_rgb(m1, m2, h) b = self._hue_to_rgb(m1, m2, h - 1.0/3.0) # Apply gamma correction r **= self.gamma g **= self.gamma b **= self.gamma return (r, g, b)
def _hue_to_rgb(self, m1, m2, h): if h < 0.0: h += 1.0 elif h > 1.0: h -= 1.0 if h*6 < 1.0: return m1 + (m2 - m1)*h*6.0 elif h*2 < 1.0: return m2 elif h*3 < 2.0: return m1 + (m2 - m1)*(2.0/3.0 - h)*6.0 else: return m1
TEMPERATURE_COLORMAP = Theme( mincolor = (2.0/3.0, 0.80, 0.25), # dark blue maxcolor = (0.0, 1.0, 0.5), # satured red gamma = 1.0 ) PINK_COLORMAP = Theme( mincolor = (0.0, 1.0, 0.90), # pink maxcolor = (0.0, 1.0, 0.5), # satured red ) GRAY_COLORMAP = Theme( mincolor = (0.0, 0.0, 0.85), # light gray maxcolor = (0.0, 0.0, 0.0), # black ) BW_COLORMAP = Theme( minfontsize = 8.0, maxfontsize = 24.0, mincolor = (0.0, 0.0, 0.0), # black maxcolor = (0.0, 0.0, 0.0), # black minpenwidth = 0.1, maxpenwidth = 8.0, ) PRINT_COLORMAP = Theme( minfontsize = 18.0, maxfontsize = 30.0, fontcolor = "black", nodestyle = "solid", mincolor = (0.0, 0.0, 0.0), # black maxcolor = (0.0, 0.0, 0.0), # black minpenwidth = 0.1, maxpenwidth = 8.0, ) themes = { "color": TEMPERATURE_COLORMAP, "pink": PINK_COLORMAP, "gray": GRAY_COLORMAP, "bw": BW_COLORMAP, "print": PRINT_COLORMAP, }