Source code for PyFoam.Basics.TimeLineCollection

#  ICE Revision: $Id$
"""Collection of array of timelines"""

from PyFoam.Error import error
from math import ceil
from copy import deepcopy
from threading import Lock
import sys

from PyFoam.ThirdParty.six import print_,iteritems

transmissionLock=Lock()

[docs]def mean(a,b): """Mean value of a and b""" return 0.5*(a+b)
[docs]def signedMax(a,b): """Absolute Maximum of a and b with the sign preserved""" if a<0. or b<0.: return min(a,b) else: return max(a,b)
[docs]class TimeLinesRegistry(object): """Collects references to TimeLineCollection objects""" nr=1 def __init__(self): self.lines={}
[docs] def clear(self): self.lines={} TimeLinesRegistry.nr=1
[docs] def add(self,line,nr=None): if nr: if nr in self.lines: error("Number",nr,"already existing") TimeLinesRegistry.nr=max(nr+1,TimeLinesRegistry.nr) else: nr=TimeLinesRegistry.nr TimeLinesRegistry.nr+=1 self.lines[nr]=line return nr
[docs] def get(self,nr): try: return self.lines[nr] except KeyError: error(nr,"not a known data set:",list(self.lines.keys()))
[docs] def prepareForTransfer(self): """Makes sure that the data about the timelines is to be transfered via XMLRPC""" # print("Get transmission") transmissionLock.acquire() # print("Got transmission") lst={} for i,p in iteritems(self.lines): collectors=[] for s in p.collectors: collectors.append(s.lineNr) lst[str(i)]={ "nr" : i, "times" : deepcopy(p.times), "values": deepcopy(p.values), "lastValid" : deepcopy(p.lastValid), "collectors": collectors } transmissionLock.release() return lst
[docs] def resolveSlaves(self): self.resolveCollectors()
[docs] def resolveCollectors(self): """Looks through all the registered lines and replaces integers with the actual registered line""" for i,p in iteritems(self.lines): if len(p.collectors)>0: collectors=[] for s in p.collectors: if type(s)==int: try: collectors.append(self.lines[s]) except KeyError: error(s,"not a known data set:",list(self.lines.keys())) else: collectors.append(s) p.collectors=collectors
_allLines=TimeLinesRegistry()
[docs]def allLines(): return _allLines
[docs]class TimeLineCollection(object): possibleAccumulations=["first", "last", "min", "max", "average", "sum","count"] def __init__(self, deflt=float("NaN"), extendCopy=False, splitThres=None, split_fraction_unchanged=0.2, splitFun=None, noEmptyTime=True, advancedSplit=False, preloadData=None, accumulation="first", registry=None): """:param deflt: default value for timelines if none has been defined before :param extendCopy: Extends the timeline by cpying the last element :param splitThres: Threshold after which the number of points is halved :param splitFun: Function that is used for halving. If none is specified the mean function is used :param noEmptyTime: if there is no valid entry no data is stored for this time :param advancedSplit: Use another split algorithm than one that condenses two values into one :param preloadData: a dictionary with a dictionary to initialize the values :param accumulation: if more than one value is given at any time-step, how to accumulate them (possible values: "first", "last", "min", "max", "average", "sum","count") """ self.cTime=None self.addTimeOnDemand=False self.times=[] self.values={} self.lastValid={} self.setDefault(deflt) self.setExtend(extendCopy) self.thres=None self.fun=None self.is_parametric = False if not (accumulation in TimeLineCollection.possibleAccumulations): error("Value",accumulation,"not in list of possible values:",TimeLineCollection.possibleAccumulations) self.accumulation=accumulation self.accumulations={} self.occured={} self.collectors=[] self.setSplitting(splitThres=splitThres, split_fraction_unchanged=split_fraction_unchanged, splitFun=splitFun, advancedSplit=advancedSplit, noEmptyTime=noEmptyTime) self.lineNr=None if preloadData: self.times=preloadData["times"] self.values=preloadData["values"] self.collectors=preloadData["collectors"] self.lineNr=int(preloadData["nr"]) if "lastValid" in preloadData: self.lastValid=preloadData["lastValid"] else: self.resetValid(val=True) if registry==None: registry=allLines() self.lineNr=registry.add(self,self.lineNr)
[docs] def resetValid(self,val=False): """Helper function that resets the information whether the last entry is valid""" self.lastValid={} for n in self.values: self.lastValid[n]=val for s in self.collectors: s.resetValid(val=val)
[docs] def nrValid(self): """Helper function that gets the number of valid values""" nr=list(self.lastValid.values()).count(True) for s in self.collectors: nr+=s.nrValid() return nr
[docs] def addSlave(self,slave): self.addCollector(slave)
[docs] def addCollector(self,collector): """Adds a collector time-line-collection""" self.collectors.append(collector) collector.setSplitting(splitThres=self.thres, split_fraction_unchanged=self.split_fraction_unchanged, splitFun=self.fun, advancedSplit=self.advancedSplit, noEmptyTime=self.noEmptyTime)
[docs] def setAccumulator(self,name,accu): """Sets a special accumulator fopr a timeline :param name: Name of the timeline :param accu: Name of the accumulator""" if not (accu in TimeLineCollection.possibleAccumulations): error("Value",accu,"not in list of possible values:",TimeLineCollection.possibleAccumulations,"When setting for",name) self.accumulations[name]=accu
[docs] def setSplitting(self, splitThres=None, split_fraction_unchanged=0.2, splitFun=None, advancedSplit=False, noEmptyTime=True): """Sets the parameters for splitting""" self.advancedSplit = advancedSplit if self.advancedSplit: self.splitLevels = [] if splitThres: self.thres=splitThres if (self.thres % 2)==1: self.thres+=1 if splitFun: self.fun=splitFun elif not self.fun: self.fun=mean for s in self.collectors: s.setSplitting(splitThres=splitThres, split_fraction_unchanged=split_fraction_unchanged, splitFun=splitFun, advancedSplit=advancedSplit, noEmptyTime=noEmptyTime) self.noEmptyTime=noEmptyTime self.split_fraction_unchanged = split_fraction_unchanged
[docs] def setDefault(self,deflt): """:param deflt: default value to be used""" self.defaultValue=float(deflt)
[docs] def setExtend(self,mode): """:param mode: whether or not to extend the timeline by copying or setting the default value""" self.extendCopy=mode
[docs] def nr(self): """Number of elements in timelines""" return len(self.times)
[docs] def setTime(self,time,noLock=False,forceAppend=False): """Sets the time. If time is new all the timelines are extended :param time: the new current time :param noLock: do not acquire the lock that ensures consistent data transmission""" if not noLock: transmissionLock.acquire() dTime=float(time) append=False self.addTimeOnDemand=False if dTime!=self.cTime: self.cTime=dTime append=True if self.noEmptyTime and not forceAppend: if self.nrValid()==0: # no valid data yet. Extend the timeline when the first data set is added append=False self.addTimeOnDemand=True if append: self.times.append(self.cTime) for v in list(self.values.values()): if len(v)>0 and self.extendCopy: val=v[-1] else: val=self.defaultValue v.append(val) else: if len(self.times)>0: self.times[-1]=self.cTime self.resetValid() if self.thres and append and not self.is_parametric: try: if len(self.times)>=self.thres: if self.advancedSplit: # self._advanced_split() self._time_resolution_split() else: self.times = self.split(self.times, min) for k in list(self.values.keys()): self.values[k] = self.split(self.values[k], self.fun) except Exception: e = sys.exc_info()[1] # Needed because python 2.5 does not support 'as e' err, detail, tb = sys.exc_info() print_(e) error("Problem splitting",e) self.occured={} if not self.is_parametric: for s in self.collectors: s.setTime(time,noLock=True,forceAppend=append) if not noLock: transmissionLock.release()
def _time_resolution_split(self): # This algorithm tries to have an evenly spacing of the values # and preserve maxima and minima and their order def concatenate(lists): result = lists[0] for l in lists[1:]: result.extend(l) return result def time_ranges(orig, delta): lngth = len(orig) ranges = [] t, ind = orig[0], 0 tend = orig[-1] last = orig[0] while ind < lngth: limit = orig[ind] + delta start = ind while ind < lngth and orig[ind] < limit: ind += 1 if ind == lngth: limit = orig[-1] ranges.append((start, ind, last, limit)) last = limit return ranges def base_times(orig, ranges): bases = [] for start, stop, base, end in ranges: nr = stop-start assert nr > 0 if nr == 1: bases.append(([orig[start]], (start, stop))) elif nr==2: bases.append(([orig[start], orig[start+1]], (start, stop))) else: delta = end - base bases.append(([base + 0.25*delta, base + 0.75*delta], (start, stop))) return bases def reduce_value(vals, bases): result = [] for _, ind in bases: start, stop = ind if (stop-start) <= 2: result.append(list(vals[start:stop])) else: v_min, i_min = vals[start], 0 v_max, i_max = vals[start], 0 for i,v in enumerate(vals[(start+1):stop]): if v < v_min: v_min = v i_min = i + 1 if v > v_max: v_max = v i_max = i + 1 if i_max < i_min: result.append([v_max, v_min]) else: result.append([v_min, v_max]) return concatenate(result) nr_points = min(max(int(self.thres/2), 20), len(self.times)) nr_back = max(int(nr_points*self.split_fraction_unchanged), 5) nr_rest = nr_points - nr_back times_back = self.times[-nr_back:] times = self.times[:-nr_back] vals_back = {k: self.values[k][-nr_back:] for k in self.values} vals = {k: self.values[k][:-nr_back] for k in self.values} delta = (times[-1]-times[0])/nr_rest bases = base_times(times, time_ranges(times, delta)) self.times = concatenate([b[0] for b in bases]) + times_back for k in self.values: self.values[k] = reduce_value(vals[k], bases) + vals_back[k] def _advanced_split(self): # Clumsy algorithm where the maximum and the minimum of a # data-window are preserved in that order if len(self.splitLevels)<len(self.times): self.splitLevels+=[0]*(len(self.times)-len(self.splitLevels)) splitTill=int(len(self.times)*0.75) if self.splitLevels[splitTill]!=0: # Shouldn't happen. But just in case splitTill=self.splitLevels.index(0) splitFrom=0 maxLevel=self.splitLevels[0] for l in range(maxLevel): try: li=self.splitLevels.index(l) if li>=0 and li<splitTill/2: splitFrom=li break except ValueError: pass window=4 if ((splitTill-splitFrom)/window)!=0: splitTill=splitFrom+window*int(ceil((splitTill-splitFrom)/float(window))) # prepare data that will not be split times=self.times[:splitFrom] levels=self.splitLevels[:splitFrom] values={} for k in self.values: values[k]=self.values[k][:splitFrom] for start in range(splitFrom,splitTill,window): end=start+window-1 sTime=self.times[start] eTime=self.times[end] times+=[sTime,(eTime-sTime)*(2./3)+sTime] levels+=[self.splitLevels[start]+1,self.splitLevels[end]+1] for k in self.values: minV=self.values[k][start] minI=0 maxV=self.values[k][start] maxI=0 for j in range(1,window): val=self.values[k][start+j] if val>maxV: maxV=val maxI=j if val<minV: minV=val minI=j if minI<maxI: values[k]+=[minV,maxV] else: values[k]+=[maxV,minV] firstUnsplit=int(splitTill/window)*window self.times=times+self.times[firstUnsplit:] self.splitLevels=levels+self.splitLevels[firstUnsplit:] for k in self.values: self.values[k]=values[k]+self.values[k][firstUnsplit:] assert len(self.times)==len(self.values[k])
[docs] def split(self,array,func): """Makes the array smaller by joining every two points :param array: the field to split :param func: The function to use for joining two points""" newLen=len(array)/2 newArray=[0.]*newLen for i in range(newLen): newArray[i]=func(array[2*i],array[2*i+1]) return newArray
[docs] def getTimes(self,name=None): """:return: A list of the time values""" tm=None if name in self.values or name is None: tm=self.times else: for s in self.collectors: if name in s.values: tm=s.times break return tm
[docs] def getValueNames(self): """:return: A list with the names of the safed values""" names=list(self.values.keys()) for i,s in enumerate(self.collectors): for n in s.getValueNames(): names.append("%s-collector%02d" % (n,i)) return names
[docs] def getValues(self,name): """Gets a timeline :param name: Name of the timeline :return: List with the values""" if name not in self.values: if len(self.collectors)>0: if name.rfind("-collector")>0: nr=int(name[-2:]) nm=name[:name.rfind("-collector")] return self.collectors[nr].getValues(nm) self.values[name]=self.nr()*[self.defaultValue] return self.values[name]
[docs] def setValue(self,name,value): """Sets the value of the last element in a timeline :param name: name of the timeline :param value: the last element""" val=float(value) transmissionLock.acquire() if self.addTimeOnDemand: self.times.append(self.cTime) for v in list(self.values.values()): if len(v)>0 and self.extendCopy: val=v[-1] else: val=self.defaultValue v.append(val) self.addTimeOnDemand=False data=self.getValues(name) if len(data)>0: accu=self.accumulation if name not in self.occured: if accu=="count": newValue=1 # =1L else: newValue=val self.occured[name]=1 else: oldValue=data[-1] n=self.occured[name] self.occured[name]+=1 if name in self.accumulations: accu=self.accumulations[name] if accu=="first": newValue=oldValue elif accu=="last": newValue=val elif accu=="max": newValue=max(val,oldValue) elif accu=="min": newValue=min(val,oldValue) elif accu=="sum": newValue=val+oldValue elif accu=="average": newValue=(n*oldValue+val)/(n+1) elif accu=="count": newValue=n+1 else: error("Unimplemented accumulator",accu,"for",name) data[-1]=newValue self.lastValid[name]=True transmissionLock.release()
[docs] def getData(self): """Return the whole current data as a SpreadsheetData-object""" from .SpreadsheetData import SpreadsheetData try: import numpy except ImportError: # assume this is pypy and retry import numpypy import numpy names=["time"]+list(self.values.keys()) data=[] data.append(self.times) for k in list(self.values.keys()): data.append(self.values[k]) return SpreadsheetData(names=names,data=numpy.asarray(data).transpose())
[docs] def getLatestData(self,structured=False): """Return a dictionary with the latest values from all data sets""" result={} for n,d in iteritems(self.values): if len(d)>0: if self.lastValid[n] or len(d)<2: result[n]=d[-1] else: result[n]=d[-2] return result
# Should work with Python3 and Python2