Source code for PyFoam.Execution.AnalyzedCommon

#  ICE Revision: $Id: AnalyzedCommon.py,v 3f8df529776e 2020-02-28 20:07:20Z bgschaid $
"""Common stuff for classes that use analyzers"""

from os import path,mkdir
from shutil import move,rmtree

from PyFoam.Basics.PlotTimelinesFactory import createPlotTimelines,createPlotTimelinesDirect
from PyFoam.Basics.TimeLineCollection import signedMax
from PyFoam.LogAnalysis.RegExpLineAnalyzer import RegExpLineAnalyzer
from PyFoam.LogAnalysis.PhaseChangerLineAnalyzer import PhaseChangerLineAnalyzer
from PyFoam.LogAnalysis.CountLineAnalyzer import CountLineAnalyzer
from PyFoam.LogAnalysis.TriggerLineAnalyzer import TriggerLineAnalyzer
from PyFoam.LogAnalysis.ExecNameLineAnalyzer import ExecNameLineAnalyzer
from PyFoam.LogAnalysis.ReplayDataFileAnalyzer import ReplayDataFileAnalyzer

from PyFoam.Error import error,warning

# import pickle
from PyFoam.ThirdParty.six.moves import cPickle as pickle
from PyFoam.ThirdParty.six import print_

from PyFoam.Basics.GeneralPlotTimelines import allPlots
from PyFoam.Basics.TimeLineCollection import allLines

from threading import Lock

[docs]class AnalyzedCommon(object): """This class collects information and methods that are needed for handling analyzers""" def __init__(self, filenames, analyzer, splitThres=2048, split_fraction_unchanged=0.2, doPickling=True): """:param filename: name of the file that is being analyzed :param analyzer: the analyzer itself :param doPickling: write the pickled plot data""" if type(filenames) is list: filename=filenames[0] else: filename=filenames self.analyzer=analyzer self.analyzer.addResetFileTrigger(self.resetFile) if 'dir' in dir(self): self.logDir=path.join(self.dir,filename+".analyzed") else: self.logDir=filename+".analyzed" if path.exists(self.logDir): # Clean away rmtree(self.logDir,ignore_errors=True) mkdir(self.logDir) self.doPickling=doPickling if self.doPickling: self.pickleLock=Lock() self.reset() if hasattr(self,"data"): pickleFile=path.join(self.logDir,"pickledStartData") pick=pickle.Pickler(open(pickleFile,"wb")) pick.dump(self.data) self.persist=None self.start_=None self.end=None self.raiseit=False self.writeFiles=False self.splitThres=splitThres self.split_fraction_unchanged=split_fraction_unchanged self.plottingImplementation="dummy" self.gnuplotTerminal=None eName=ExecNameLineAnalyzer() eName.addListener(self.execNameFound) self.addAnalyzer("ExecName",eName) self.automaticCustom=None self.tickers=[] self.plots={}
[docs] def addTicker(self,ticker): """Add a callable that will be called at every timestep""" if ticker is not None: self.tickers.append(ticker)
[docs] def addPlots(self,plots): "Add plots. To be overriden" pass
[docs] def execNameFound(self,execName): if hasattr(self,"oldExecName"): if execName==self.oldExecName: return self.automaticCustom=[] self.oldExecName=execName from PyFoam import configuration as conf from PyFoam.Basics.CustomPlotInfo import CustomPlotInfo solvers=set([execName]) for name,lst in conf().items("SolverBase"): if execName.lower().startswith(name): solvers|=set(eval(lst)) import re autoplots=[a.lower() for a in conf().getList("Plotting","autoplots")] for name,dataString in conf().items("Autoplots"): found=False try: data=eval(dataString) solverPatterns=data["solvers"] info=data["plotinfo"] for s in solverPatterns: try: for eName in solvers: if re.compile(s).fullmatch(eName): found=True break if found: break except AttributeError: # python 2 has no fullmatch for eName in solvers: if re.compile("(?:" + s + r")\Z").match(eName): found=True break if found: break except KeyError: import sys e = sys.exc_info()[1] warning("Miss-configured automatic expression",name, "Missing key",e.args[0]) except re.error: import sys warning("Problem with regular expression",s,"of",name, ":",sys.exc_info()[1]) except SyntaxError: warning("Syntax error in",dataString) if found or name.lower() in autoplots: self.automaticCustom.append(CustomPlotInfo(raw=info, name=name))
[docs] def tearDown(self): self.analyzer.tearDown() if hasattr(self,"data"): pickleFile=path.join(self.logDir,"pickledData") pick=pickle.Pickler(open(pickleFile,"wb")) pick.dump(self.data)
[docs] def listAnalyzers(self): """:returns: A list with the names of the analyzers""" return self.analyzer.listAnalyzers()
[docs] def getAnalyzer(self,name): """:param name: name of the LineAnalyzer to get""" return self.analyzer.getAnalyzer(name)
[docs] def hasAnalyzer(self,name): """:param name: name of the LineAnalyzer we ask for""" return self.analyzer.hasAnalyzer(name)
[docs] def addAnalyzer(self,name,analyzer): """:param name: name of the LineAnalyzer to add :param analyzer: the analyzer to add""" analyzer.setDirectory(self.logDir) return self.analyzer.addAnalyzer(name,analyzer)
[docs] def lineHandle(self,line): """Not to be called: calls the analyzer for the current line""" self.analyzer.analyzeLine(line) if self.automaticCustom: # if one of the readers added custom plots add them AFTER # processing the whole line if len(self.automaticCustom)>0: print_("Adding automatic plots:",", ".join( [e.name for e in self.automaticCustom])) automaticPlots=self.addCustomExpressions(self.automaticCustom, persist=self.persist, quiet=self.quiet, start=self.start_, end=self.end, raiseit=self.raiseit, writeFiles=self.writeFiles, splitThres=self.splitThres, split_fraction_unchanged=self.split_fraction_unchanged, gnuplotTerminal=self.gnuplotTerminal, plottingImplementation=self.plottingImplementation) self.reset() self.addPlots(automaticPlots) self.automaticCustom=None
[docs] def reset(self): """reset the analyzer""" self.analyzer.setDirectory(self.logDir)
[docs] def getDirname(self): """Get the name of the directory where the data is written to""" return self.logDir
[docs] def getTime(self): """Get the execution time""" return self.analyzer.getTime()
[docs] def addTrigger(self,time,func,once=True,until=None): """Adds a timed trigger to the Analyzer :param time: the time at which the function should be triggered :param func: the trigger function :param once: Should this function be called once or at every time-step :param until: The time until which the trigger should be called""" self.analyzer.addTrigger(time,func,once=once,until=until)
[docs] def createPlots(self, persist=None, quiet=False, raiseit=False, splitThres=2048, split_fraction_unchanged=0.2, plotLinear=True, plotCont=True, plotBound=True, plotIterations=True, plotCourant=True, plotExecution=True, plotDeltaT=True, start=None, end=None, writeFiles=False, customRegexp=None, gnuplotTerminal=None, plottingImplementation="dummy"): plots={} self.persist=persist self.quiet=quiet self.start_=start self.end=end self.raiseit=raiseit self.writeFiles=writeFiles self.splitThres=splitThres self.plottingImplementation=plottingImplementation self.gnuplotTerminal=gnuplotTerminal if plotLinear and self.hasAnalyzer("Linear"): plots["linear"]=createPlotTimelinesDirect("linear", self.getAnalyzer("Linear").lines, persist=persist, quiet=quiet, raiseit=raiseit, forbidden=["final","iterations"], start=start, end=end, logscale=True, gnuplotTerminal=gnuplotTerminal, implementation=plottingImplementation) self.getAnalyzer("Linear").lines.setSplitting(splitThres=splitThres, split_fraction_unchanged=split_fraction_unchanged, splitFun=max, advancedSplit=True) plots["linear"].setTitle("Residuals") plots["linear"].setYLabel("Initial residual") if plotCont and self.hasAnalyzer("Continuity"): plots["cont"]=createPlotTimelinesDirect("continuity", self.getAnalyzer("Continuity").lines, persist=persist, quiet=quiet, alternateAxis=["Global"], raiseit=raiseit, start=start, end=end, gnuplotTerminal=gnuplotTerminal, implementation=plottingImplementation) plots["cont"].setYLabel("Cumulative") plots["cont"].setYLabel2("Global") self.getAnalyzer("Continuity").lines.setSplitting(splitThres=splitThres, split_fraction_unchanged=split_fraction_unchanged, advancedSplit=True) plots["cont"].setTitle("Continuity") if plotBound and self.hasAnalyzer("Bounding"): plots["bound"]=createPlotTimelinesDirect("bounding", self.getAnalyzer("Bounding").lines, persist=persist, quiet=quiet, raiseit=raiseit, start=start, end=end, gnuplotTerminal=gnuplotTerminal, implementation=plottingImplementation) self.getAnalyzer("Bounding").lines.setSplitting(splitThres=splitThres, split_fraction_unchanged=split_fraction_unchanged, splitFun=signedMax, advancedSplit=True) plots["bound"].setTitle("Bounded variables") if plotIterations and self.hasAnalyzer("Iterations"): plots["iter"]=createPlotTimelinesDirect("iterations", self.getAnalyzer("Iterations").lines, persist=persist, quiet=quiet, with_="steps", raiseit=raiseit, start=start, end=end, gnuplotTerminal=gnuplotTerminal, implementation=plottingImplementation) self.getAnalyzer("Iterations").lines.setSplitting(splitThres=splitThres, split_fraction_unchanged=split_fraction_unchanged, advancedSplit=True) plots["iter"].setTitle("Iterations") plots["iter"].setYLabel("Sum of iterations") if plotCourant and self.hasAnalyzer("Courant"): plots["courant"]=createPlotTimelinesDirect("courant", self.getAnalyzer("Courant").lines, persist=persist, quiet=quiet, raiseit=raiseit, start=start, end=end, gnuplotTerminal=gnuplotTerminal, implementation=plottingImplementation) self.getAnalyzer("Courant").lines.setSplitting(splitThres=splitThres, split_fraction_unchanged=split_fraction_unchanged, advancedSplit=True) plots["courant"].setTitle("Courant") plots["courant"].setYLabel("Courant Number [1]") if plotDeltaT and self.hasAnalyzer("DeltaT"): plots["deltaT"]=createPlotTimelinesDirect("timestep", self.getAnalyzer("DeltaT").lines, persist=persist, quiet=quiet, raiseit=raiseit, start=start, end=end, logscale=True, gnuplotTerminal=gnuplotTerminal, implementation=plottingImplementation) self.getAnalyzer("DeltaT").lines.setSplitting(splitThres=splitThres, split_fraction_unchanged=split_fraction_unchanged, advancedSplit=True) plots["deltaT"].setTitle("DeltaT") plots["deltaT"].setYLabel("dt [s]") if plotExecution and self.hasAnalyzer("Execution"): plots["execution"]=createPlotTimelinesDirect("execution", self.getAnalyzer("Execution").lines, persist=persist, quiet=quiet, with_="steps", raiseit=raiseit, start=start, end=end, gnuplotTerminal=gnuplotTerminal, implementation=plottingImplementation) self.getAnalyzer("Execution").lines.setSplitting(splitThres=splitThres, split_fraction_unchanged=split_fraction_unchanged, advancedSplit=True) plots["execution"].setTitle("Execution Time") plots["execution"].setYLabel("Time [s]") self.plots.update(plots) if customRegexp: customPlots=self.addCustomExpressions(customRegexp, persist=persist, quiet=quiet, start=start, end=end, raiseit=raiseit, writeFiles=writeFiles, splitThres=splitThres, split_fraction_unchanged=split_fraction_unchanged, gnuplotTerminal=gnuplotTerminal, plottingImplementation=plottingImplementation) plots.update(customPlots) self.reset() self.addPlots(plots)
[docs] def addCustomExpressions(self, customRegexp, persist=None, quiet=False, start=None, end=None, raiseit=False, writeFiles=False, splitThres=2048, split_fraction_unchanged=0.2, gnuplotTerminal=None, plottingImplementation="dummy"): plots={} publishers={} collectors=[] canonical={} marks=[] plotTypes= [ ("regular" , "Matches regular expression and plots"), ("collector" , "Plot data on a different plot (the 'publisher')"), ("slave" , "Old (deprecated) terminology for 'collector'"), ("dynamic" , "Dynamically creates lines depending on the match found at 'idNr'"), ("dynamiccollector" , "Combination of 'dynamic' and 'collector'"), ("dynamicslave" , "Old (deprecated) terminology for 'dynamiccollector'"), ("data" , "Reads data from a file and plots it"), ("datacollector" , "Combination of 'data' and 'collector'"), ("dataslave" , "Old (deprecated) terminology for 'datacollector'"), ("count" , "Counts how often an expression occured (no plotting but used for 'alternateTime')"), ("mark" , "If the expression matches then a vertical marker is drawn on the 'targets'"), ("phase" , "Changes the phase prefix (for multi-region cases)") ] for i,custom in enumerate(customRegexp): if not custom.enabled: continue if persist!=None: custom.persist=persist if start!=None: custom.start=start if end!=None: custom.end=end custom.raiseit=raiseit if custom.type not in [p[0] for p in plotTypes]: error("type '{}' of custom plot '{}' not in known types:\n {}".format( custom.type,custom.id, "\n ".join(["{:15} : {}".format(n,d) for n,d in plotTypes]))) createPlot=True if custom.type=="phase": self.addAnalyzer(custom.name, PhaseChangerLineAnalyzer(custom.expr, idNr=custom.idNr)) createPlot=False elif custom.type=="count": self.addAnalyzer(custom.name, CountLineAnalyzer(custom.expr)) createPlot=False elif custom.type=="mark": a=TriggerLineAnalyzer(custom.expr) self.addAnalyzer(custom.name, a) marks.append((custom,a)) createPlot=False elif custom.type in ["dynamic", "dynamicslave", "dynamiccollector"]: self.addAnalyzer(custom.name, RegExpLineAnalyzer(custom.name.lower(), custom.expr, titles=custom.titles, doTimelines=True, doFiles=writeFiles or custom.writeFiles, accumulation=custom.accumulation, dataTransformations=custom.dataTransformations, progressTemplate=custom.progress, singleFile=True, idNr=custom.idNr, stringValues=custom.stringValues, startTime=custom.start, endTime=custom.end)) elif custom.type in ["regular", "slave", "collector"]: self.addAnalyzer(custom.name, RegExpLineAnalyzer(custom.name.lower(), custom.expr, titles=custom.titles, doTimelines=True, doFiles=writeFiles or custom.writeFiles, accumulation=custom.accumulation, dataTransformations=custom.dataTransformations, progressTemplate=custom.progress, stringValues=custom.stringValues, singleFile=True, startTime=custom.start, endTime=custom.end)) elif custom.type in ["data", "dataslave", "datacollector"]: self.addAnalyzer(custom.name, ReplayDataFileAnalyzer(timeName=custom.timeName, validData=custom.validData, validMatchRegexp=custom.validMatchRegexp, csvName=custom.csvName, txtName=custom.txtName, namePrefix=custom.namePrefix, excelName=custom.excelName, skip_header=custom.skip_header, stripCharacters=custom.stripCharacters, progressTemplate=custom.progress, replaceFirstLine=custom.replaceFirstLine, startTime=custom.start, endTime=custom.end)) else: error("Unknown type",custom.type,"in custom expression",custom.name) canonical[custom.id]=custom.name if createPlot: if custom.publisher==None: if custom.type in ["slave", "dynamicslave", "dataslave", "collector", "dynamiccollector", "datacollector"]: error("Custom expression", custom.name, "is supposed to be a 'collector' but no publisher is defined") publishers[custom.id] = custom plotCustom = createPlotTimelines(self.getAnalyzer(custom.name).lines, quiet=quiet, custom=custom, gnuplotTerminal=gnuplotTerminal, implementation=plottingImplementation) self.getAnalyzer(custom.name).lines.setSplitting(splitThres=splitThres, split_fraction_unchanged=split_fraction_unchanged, advancedSplit=True) plotCustom.setTitle(custom.theTitle) plots["custom%04d" % i]=plotCustom self.plots[custom.id]=plotCustom else: if custom.type not in ["slave", "dynamicslave", "dataslave", "collector", "dynamiccollector", "datacollector"]: error("'publisher' only makes sense if type is 'collector' or 'dynamiccollector' for",custom.name) if getattr(custom,"alternateAxis",None): error("Specify alternate values in 'alternateAxis' of publisher", custom.publisher, "for", custom.name) collectors.append(custom) for custom in customRegexp: if custom.alternateTime: self.getAnalyzer(custom.name).setParent( self.getAnalyzer(canonical[custom.alternateTime]) ) self.getAnalyzer(canonical[custom.alternateTime]).addTimeListener(self.getAnalyzer(custom.name)) for s in collectors: if s.publisher not in publishers: error("The custom plot", s.id, "wants the publisher plot", s.publisher, "but it is not found in the list of publishers", list(publishers.keys())) else: collector=self.getAnalyzer(s.name) m = publishers[s.publisher] if m.xvalue is not None and s.xvalue is None: error("Publisher plot", m.id, "is parameteric with xvalue", m.xvalue, "so", s.id, "has to be parameteric as well") if m.xvalue is None and s.xvalue is not None: error("Collector plot", s.id, "is parameteric with xvalue", s.xvalue, "so", m.id, "has to be parameteric as well") publisher=self.getAnalyzer(m.name) collector.setPublisher(publisher) for custom,analyzer in marks: for pName in custom.targets: try: p=self.plots[pName] except KeyError as e: print_("Available plots:",list(self.plots.keys())) raise e def mark(p): def makeMark(): p.addVerticalMarker() return makeMark analyzer.addFunction(mark(p)) return plots
[docs] def picklePlots(self,wait=False): """Writes the necessary information for the plots permanently to disc, so that it doesn't have to be generated again :param wait: wait for the lock to be allowed to pickle""" # print "Putting some pickles in the jar" lines=allLines() plots=allPlots() if lines and plots: # print "Getting lock" gotIt=self.pickleLock.acquire(wait) # print "Got lock" if not gotIt: return pickleFile=path.join(self.logDir,"pickledPlots") pick=pickle.Pickler(open(pickleFile+".tmp","wb")) pick.dump(lines.prepareForTransfer()) pick.dump(plots.prepareForTransfer()) move(pickleFile+".tmp",pickleFile) if hasattr(self,"data"): pickleFile=path.join(self.logDir,"pickledUnfinishedData") pick=pickle.Pickler(open(pickleFile+".tmp","wb")) pick.dump(self.data) del pick move(pickleFile+".tmp",pickleFile) # print "Releasing lock" self.pickleLock.release()
# print "Released lock"
[docs] def setDataSet(self,data): if hasattr(self,"data"): self.data["analyzed"]=data
[docs] def resetFile(self): """The input file changed and we add a marker to all plots""" for n in self.plots: p = self.plots[n] p.addVerticalMarker(colorRGB=(1.,0,0),label="Restart")
# Should work with Python3 and Python2