# ICE Revision: $Id$
"""Run a OpenFOAM command"""
import sys
import string
import gzip
from os import path
from platform import uname
from threading import Timer
from time import time,asctime
import uuid
from PyFoam.FoamInformation import oldAppConvention as oldApp
from PyFoam.ThirdParty.six import print_
import PyFoam.Basics.FoamFileGenerator
from PyFoam.Basics.DataStructures import makePrimitiveString
from PyFoam.Basics.Utilities import rmtree
from PyFoam.Basics.RunDatabase import RunDatabase
if not 'curdir' in dir(path) or not 'sep' in dir(path):
print_("Warning: Inserting symbols into os.path (Python-Version<2.3)")
path.curdir='.'
path.sep ='/'
from PyFoam.Execution.FoamThread import FoamThread
from PyFoam.Infrastructure.FoamServer import FoamServer
from PyFoam.Infrastructure.Logging import foamLogger
from PyFoam.RunDictionary.SolutionDirectory import SolutionDirectory
from PyFoam.RunDictionary.ParameterFile import ParameterFile
from PyFoam.Error import warning,error,debug
from PyFoam import configuration as config
[docs]def restoreControlDict(ctrl,runner):
"""Timed function to avoid time-stamp-problems"""
warning("Restoring the controlDict")
ctrl.restore()
runner.controlDict=None
[docs]def calcLogname(logname,argv):
if logname==None:
return "PyFoam."+path.basename(argv[0])
else:
return logname
[docs]def findRestartFiles(logfile,sol):
isRestart=False
restartnr=None
restartName=None
lastlog=None
if path.exists(logfile) or path.exists(logfile+".gz"):
ctrlDict=ParameterFile(path.join(sol.controlDict()),backup=False)
if ctrlDict.readParameter("startFrom")=="latestTime":
if len(sol)>1:
isRestart=True
restartnr=0
restartName=logfile+".restart00"
lastlog=logfile
while path.exists(restartName) or path.exists(restartName+".gz"):
restartnr+=1
lastlog=restartName
restartName=logfile+".restart%02d" % restartnr
return isRestart,restartnr,restartName,lastlog
[docs]class BasicRunner(object):
"""Base class for the running of commands
When the command is run the output is copied to a LogFile and
(optionally) standard-out
The argument list assumes for the first three elements the
OpenFOAM-convention:
<cmd> <dir> <case>
The directory name for outputs is therefor created from <dir> and
<case>
Provides some handle-methods that are to be overloaded for
additional functionality"""
def __init__(self,
argv=None,
silent=False,
logname=None,
compressLog=False,
lam=None,
server=False,
restart=False,
noLog=False,
logTail=None,
remark=None,
jobId=None,
parameters=None,
writeState=True,
echoCommandLine=None):
""":param argv: list with the tokens that are the command line
if not set the standard command line is used
:param silent: if True no output is sent to stdout
:param logname: name of the logfile
:param compressLog: Compress the logfile into a gzip
:param lam: Information about a parallel run
:param server: Whether or not to start the network-server
:type lam: PyFoam.Execution.ParallelExecution.LAMMachine
:param noLog: Don't output a log file
:param logTail: only the last lines of the log should be written
:param remark: User defined remark about the job
:param parameters: User defined dictionary with parameters for
documentation purposes
:param jobId: Job ID of the controlling system (Queueing system)
:param writeState: Write the state to some files in the case
:param echoCommandLine: Prefix that is printed with the command line. If unset nothing is printed
"""
if sys.version_info < (2,3):
# Python 2.2 does not have the capabilities for the Server-Thread
if server:
warning("Can not start server-process because Python-Version is too old")
server=False
if argv==None:
self.argv=sys.argv[1:]
else:
self.argv=argv
if oldApp():
self.dir=path.join(self.argv[1],self.argv[2])
if self.argv[2][-1]==path.sep:
self.argv[2]=self.argv[2][:-1]
else:
self.dir=path.curdir
if "-case" in self.argv:
self.dir=self.argv[self.argv.index("-case")+1]
logname=calcLogname(logname,argv)
try:
sol=self.getSolutionDirectory()
except OSError:
e = sys.exc_info()[1] # compatible with 2.x and 3.x
error("Solution directory",self.dir,"does not exist. No use running. Problem:",e)
self.echoCommandLine=echoCommandLine
self.silent=silent
self.lam=lam
self.origArgv=self.argv
self.writeState=writeState
self.__lastLastSeenWrite=0
self.__lastNowTimeWrite=0
if self.lam!=None:
self.argv=lam.buildMPIrun(self.argv)
if config().getdebug("ParallelExecution"):
debug("Command line:"," ".join(self.argv))
self.cmd=" ".join(self.argv)
foamLogger().info("Starting: "+self.cmd+" in "+path.abspath(path.curdir))
self.logFile=path.join(self.dir,logname+".logfile")
isRestart,restartnr,restartName,lastlog=findRestartFiles(self.logFile,sol)
if restartName:
self.logFile=restartName
if not isRestart:
from os import unlink
from glob import glob
for g in glob(self.logFile+".restart*"):
if path.isdir(g):
rmtree(g)
else:
unlink(g)
self.noLog=noLog
self.logTail=logTail
if self.logTail:
if self.noLog:
warning("Log tail",self.logTail,"and no-log specified. Using logTail")
self.noLog=True
self.lastLines=[]
self.compressLog=compressLog
if self.compressLog:
self.logFile+=".gz"
self.fatalError=False
self.fatalFPE=False
self.fatalStackdump=False
self.endSeen=False
self.warnings=0
self.started=False
self.isRestarted=False
if restart:
self.controlDict=ParameterFile(path.join(self.dir,"system","controlDict"),backup=True)
self.controlDict.replaceParameter("startFrom","latestTime")
self.isRestarted=True
else:
self.controlDict=None
self.run=FoamThread(self.cmd,self)
self.server=None
if server:
self.server=FoamServer(run=self.run,master=self)
self.server.setDaemon(True)
self.server.start()
try:
IP,PID,Port=self.server.info()
f=open(path.join(self.dir,"PyFoamServer.info"),"w")
print_(IP,PID,Port,file=f)
f.close()
except AttributeError:
warning("There seems to be a problem with starting the server:",self.server,"with attributes",dir(self.server))
self.server=None
self.createTime=None
self.nowTime=None
self.startTimestamp=time()
self.stopMe=False
self.writeRequested=False
self.endTriggers=[]
self.lastLogLineSeen=None
self.lastTimeStepSeen=None
self.remark=remark
self.jobId=jobId
self.data={"lines":0} # self.data={"lines":0L}
self.data[RunDatabase.unique_id]=str(uuid.uuid1())
self.data["logfile"]=self.logFile
self.data["casefullname"]=path.abspath(self.dir)
self.data["casename"]=path.basename(path.abspath(self.dir))
self.data["solver"]=path.basename(self.argv[0])
self.data["solverFull"]=self.argv[0]
self.data["commandLine"]=self.cmd
self.data["hostname"]=uname()[1]
if remark:
self.data["remark"]=remark
else:
self.data["remark"]="No remark given"
if jobId:
self.data["jobId"]=jobId
parameterFile=sol.getParametersFromFile()
if len(parameterFile):
self.data["parameters"]={}
for k,v in parameterFile.items():
self.data["parameters"][k]=makePrimitiveString(v)
if parameters:
if "parameters" not in self.data:
self.data["parameters"]={}
self.data["parameters"].update(parameters)
self.data["starttime"]=asctime()
[docs] def appendTailLine(self,line):
"""Append lines to the tail of the log"""
if len(self.lastLines)>10*self.logTail:
# truncate the lines, but not too often
self.lastLines=self.lastLines[-self.logTail:]
self.writeTailLog()
self.lastLines.append(line+"\n")
[docs] def writeTailLog(self):
"""Write the last lines to the log"""
fh=open(self.logFile,"w")
if len(self.lastLines)<=self.logTail:
fh.writelines(self.lastLines)
else:
fh.writelines(self.lastLines[-self.logTail:])
fh.close()
[docs] def start(self):
"""starts the command and stays with it till the end"""
self.started=True
if not self.noLog:
if self.compressLog:
fh=gzip.open(self.logFile,"w")
else:
fh=open(self.logFile,"w")
self.startHandle()
self.writeStartTime()
self.writeTheState("Running")
check=BasicRunnerCheck()
if self.echoCommandLine:
print_(self.echoCommandLine+" "+" ".join(self.argv))
self.run.start()
interrupted=False
totalWarningLines=0
addLinesToWarning=0
collectWarnings=True
while self.run.check():
try:
self.run.read()
if not self.run.check():
break
line=self.run.getLine()
if "errorText" in self.data:
self.data["errorText"]+=line+"\n"
if addLinesToWarning>0:
self.data["warningText"]+=line+"\n"
addLinesToWarning-=1
totalWarningLines+=1
if totalWarningLines>500:
collectWarnings=False
addLinesToWarning=0
self.data["warningText"]+="No more warnings added because limit of 500 lines exceeded"
self.data["lines"]+=1
self.lastLogLineSeen=time()
self.writeLastSeen()
tmp=check.getTime(line)
if check.controlDictRead(line):
if self.writeRequested:
duration=config().getfloat("Execution","controlDictRestoreWait",default=30.)
warning("Preparing to reset controlDict to old glory in",duration,"seconds")
Timer(duration,
restoreControlDict,
args=[self.controlDict,self]).start()
self.writeRequested=False
if tmp!=None:
self.data["time"]=tmp
self.nowTime=tmp
self.writeTheState("Running",always=False)
self.writeNowTime()
self.lastTimeStepSeen=time()
if self.createTime==None:
# necessary because interFoam reports no creation time
self.createTime=tmp
try:
self.data["stepNr"]+=1
except KeyError:
self.data["stepNr"]=1 # =1L
self.data["lasttimesteptime"]=asctime()
tmp=check.getCreateTime(line)
self.endSeen=check.endSeen
if tmp!=None:
self.createTime=tmp
if not self.silent:
try:
print_(line)
except IOError:
e = sys.exc_info()[1] # compatible with 2.x and 3.x
if e.errno!=32:
raise e
else:
# Pipe was broken
self.run.interrupt()
if line.find("FOAM FATAL ERROR")>=0 or line.find("FOAM FATAL IO ERROR")>=0:
self.fatalError=True
self.data["errorText"]="PyFoam found a Fatal Error "
if "time" in self.data:
self.data["errorText"]+="at time "+str(self.data["time"])+"\n"
else:
self.data["errorText"]+="before time started\n"
self.data["errorText"]+="\n"+line+"\n"
if line.find("Foam::sigFpe::sigFpeHandler")>=0:
self.fatalFPE=True
if line.find("Foam::error::printStack")>=0:
self.fatalStackdump=True
if self.fatalError and line!="":
foamLogger().error(line)
if line.find("FOAM Warning")>=0:
self.warnings+=1
try:
self.data["warnings"]+=1
except KeyError:
self.data["warnings"]=1
if collectWarnings:
addLinesToWarning=20
if not "warningText" in self.data:
self.data["warningText"]=""
else:
self.data["warningText"]+=("-"*40)+"\n"
self.data["warningText"]+="Warning found by PyFoam on line "
self.data["warningText"]+=str(self.data["lines"])+" "
if "time" in self.data:
self.data["warningText"]+="at time "+str(self.data["time"])+"\n"
else:
self.data["warningText"]+="before time started\n"
self.data["warningText"]+="\n"+line+"\n"
if self.server!=None:
self.server._insertLine(line)
self.lineHandle(line)
if not self.noLog:
fh.write(line+"\n")
fh.flush()
elif self.logTail:
self.appendTailLine(line)
except KeyboardInterrupt:
e = sys.exc_info()[1] # compatible with 2.x and 3.x
foamLogger().warning("Keyboard Interrupt")
self.run.interrupt()
self.writeTheState("Interrupted")
self.data["keyboardInterrupt"]=True
interrupted=True
if not "keyboardInterrupt" in self.data:
self.data["keyboardInterrupt"]=self.run.keyboardInterupted
self.data["interrupted"]=interrupted
self.data["OK"]=self.runOK()
self.data["endSeen"]=self.endSeen
self.data["cpuTime"]=self.run.cpuTime()
self.data["cpuUserTime"]=self.run.cpuUserTime()
self.data["cpuSystemTime"]=self.run.cpuSystemTime()
self.data["wallTime"]=self.run.wallTime()
self.data["usedMemory"]=self.run.usedMemory()
self.data["endtime"]=asctime()
self.data["fatalError"]=self.fatalError
self.data["fatalFPE"]=self.fatalFPE
self.data["fatalStackdump"]=self.fatalStackdump
self.writeNowTime(force=True)
self.stopHandle()
if not interrupted:
if self.endSeen:
self.writeTheState("Finished - Ended")
else:
self.writeTheState("Finished")
for t in self.endTriggers:
t()
if not self.noLog:
fh.close()
elif self.logTail:
self.writeTailLog()
if self.server!=None:
self.server.deregister()
self.server.kill()
foamLogger().info("Finished")
return self.data
[docs] def writeToStateFile(self,fName,message):
"""Write a message to a state file"""
if self.writeState:
open(path.join(self.dir,"PyFoamState."+fName),"w").write(message+"\n")
[docs] def writeStartTime(self):
"""Write the real time the run was started at"""
self.writeToStateFile("StartedAt",asctime())
[docs] def writeTheState(self,state,always=True):
"""Write the current state the run is in"""
if always or (time()-self.__lastLastSeenWrite)>9:
self.writeToStateFile("TheState",state)
[docs] def writeLastSeen(self):
if (time()-self.__lastLastSeenWrite)>10:
self.writeToStateFile("LastOutputSeen",asctime())
self.__lastLastSeenWrite=time()
[docs] def writeNowTime(self,force=False):
if (time()-self.__lastNowTimeWrite)>10 or force:
self.writeToStateFile("CurrentTime",str(self.nowTime))
self.__lastNowTimeWrite=time()
[docs] def runOK(self):
"""checks whether the run was successful"""
if self.started:
return not self.fatalError and not self.fatalFPE and not self.fatalStackdump # and self.run.getReturnCode()==0
else:
return False
[docs] def startHandle(self):
"""to be called before the program is started"""
pass
def _writeStopAt(self,value,message):
"""Write stopAt to stop the run gracefully"""
if not self.stopMe:
self.stopMe=True
if not self.isRestarted:
if self.controlDict:
warning("The controlDict has already been modified. Restoring will be problementic")
self.controlDict=ParameterFile(path.join(self.dir,"system","controlDict"),backup=True)
self.controlDict.replaceParameter("stopAt",value)
warning(message)
[docs] def stopGracefully(self):
"""Tells the runner to stop at the next convenient time"""
self._writeStopAt("writeNow","Stopping run and writting")
[docs] def stopAtNextWrite(self):
"""Tells the runner to stop at the next write"""
self._writeStopAt("nextWrite","Stopping run at next write")
[docs] def stopWithoutWrite(self):
"""Tells the runner to stop without writing"""
self._writeStopAt("noWriteNow","Stopping run without writing")
[docs] def writeResults(self):
"""Writes the next possible time-step"""
# warning("writeResult is not yet implemented")
if not self.writeRequested:
if not self.isRestarted:
if self.controlDict:
warning("The controlDict has already been modified. Restoring will be problementic")
self.controlDict=ParameterFile(path.join(self.dir,"system","controlDict"),backup=True)
self.controlDict.replaceParameter("writeControl","timeStep")
self.controlDict.replaceParameter("writeInterval","1")
self.writeRequested=True
[docs] def stopHandle(self):
"""called after the program has stopped"""
if self.stopMe or self.isRestarted:
self.controlDict.restore()
[docs] def lineHandle(self,line):
"""called every time a new line is read"""
pass
[docs] def logName(self):
"""Get the name of the logfiles"""
return self.logFile
[docs] def getSolutionDirectory(self,archive=None):
""":return: The directory of the case
:rtype: PyFoam.RunDictionary.SolutionDirectory
:param archive: Name of the directory for archiving results"""
return SolutionDirectory(self.dir,archive=archive,parallel=True)
[docs] def addEndTrigger(self,f):
""":param f: A function that is to be executed at the end of the simulation"""
self.endTriggers.append(f)
import re
[docs]class BasicRunnerCheck(object):
"""A small class that does primitve checking for BasicRunner
Duplicates other efforts, but ...."""
floatRegExp="[-+]?[0-9]*\.?[0-9]+(?:[eE][-+]?[0-9]+)?"
def __init__(self):
# self.timeExpr=re.compile("^Time = (%f%)$".replace("%f%",self.floatRegExp))
self.timeExpr=config().getRegexp("SolverOutput","timeregexp")
self.createExpr=re.compile("^Create mesh for time = (%f%)$".replace("%f%",self.floatRegExp))
self.endSeen=False
[docs] def getTime(self,line):
"""Does this line contain time information?"""
m=self.timeExpr.match(line)
if m:
self.endSeen=False
try:
return float(m.group(2))
except ValueError:
warning("Problem while converting",m.group(2),"to float")
return None
else:
if line.strip()=="End":
self.endSeen=True
return None
[docs] def getCreateTime(self,line):
"""Does this line contain mesh time information?"""
m=self.createExpr.match(line)
if m:
return float(m.group(1))
else:
return None
[docs] def controlDictRead(self,line):
"""Was the controlDict reread?"""
phrases=["Reading object controlDict from file",
"Re-reading object controlDict from file"]
for p in phrases:
if line.find(p)>=0:
return True
return False
# Should work with Python3 and Python2