Source code for altimetry.data.hydro

# -*- coding: utf-8 -*-
import os
import glob
import inspect
import fnmatch
import datetime

from warnings import warn

import numpy as np
import matplotlib.pyplot as plt
from netCDF4 import Dataset as ncfile
import copy
import tempfile

from altimetry.tools.nctools import load_ncVar, load_ncVar_v2, nc as ncobj, dimStr,attrStr,\
    ncStr, varStr
try:
    import seawater.csiro as csw
except ImportError:
    warn("[WARNING:%s] Module seawater doesn't exists. not loading it" % __name__)
    pass # module doesn't exist, deal with it.

#import alti_tools as atools
try : from scipy import interpolate
except ImportError :
    warn("[WARNING:%s] module scipy not found" % __name__)

from altimetry.tools import recale_limits, in_limits, cumulative_distance, calcul_distance, \
    where_list, \
    cnes_convert, \
    plot_map, \
    get_caller
from collections import OrderedDict


[docs]class hydro_data(object): ''' A base object dedicated to handle oceanographic data (in-situ or remote sensing) with upper level processing methods. .. note:: This object SHOULD NOT be called directly but through a subclass heritating of it (eg. :class:`altimetry.data.alti_data`) '''
[docs] def __init__(self,file_pattern,limit=None,verbose=1,round=True,zero_2pi=True,output_is_dict=True,flatten=False,**kwargs): ''' Returns the object filled with the data loaded from a single file or a concatenated set of files :parameter file_pattern: a pattern of files to be globbed (:func:`glob.glob`) or a list of file names. :keyword limit: the limits of the domain to handle ([latmin,lonmin,latmax,lonmax]). :keyword verbose: verbosity level on a scale of 0 (silent) to 4 (max verobsity) :keyword round: round limits (cf. :func:`altimetry.tools.in_limits`) :keyword zero_2pi: limits goes from 0 to 360 degrees (not -180/180). :keyword output_is_dict: data structures are dictionnaries (eg. my_hydro_data.variable['data']). If false uses an object with attributes (eg. my_hydro_data.variable.data). .. note:: This methodes init all the attributes, then loads the data from files (:meth:`altimetry.data.hydro_data.read`) and appends it to the object (:meth:`altimetry.data.hydro_data.update_dataset`) before checking its content (:meth:`altimetry.data.hydro_data.check_variables`). .. note:: The method :meth:`altimetry.data.hydro_data.read` MUST be defined (typically by overloading it). This method must return a data structure. ''' #Init system variables # if limit is None : limit=[-90.,0.,90.,360.] self.zero_2pi=zero_2pi self.limit_set=False if limit is None : limit=[-90.,0.,90.,360.] else : self.limit_set = True self.limit = np.array(recale_limits(limit, zero_2pi=self.zero_2pi)) ''' limits of the domain : [latmin,lonmin,latmax,lonmax] (default = [-90.,0.,90.,360.])/ .. note:: limits are automatically reset using :func:`altimetry.tools.recale_limits` ''' self.verbose = verbose ''' verbosity level on a scale of 0 (silent) to 4 (max verbosity) ''' self.fileid = np.array([]) ''' array of file IDs ''' self.count=0 ''' number of files loaded ''' self.size=0 ''' length of the dataset ''' #Setup file list if isinstance(file_pattern, str) : ls=glob.glob(file_pattern) else : ls = file_pattern.tolist() if not isinstance(file_pattern,list) else file_pattern file_pattern=file_pattern[0] if len(file_pattern) > 0 else [] if len(ls) == 0 : if isinstance(file_pattern, str) : self.Error('File pattern not matched '+file_pattern) if isinstance(file_pattern, list) : self.warning(2, 'Empty file list - will return empty object') self._filename=None ''' Name of the file currently used ''' self._ncfile=None ''' Name of the file currently used ''' self.__tempfile=[] ''' Temporary buffer where to unzip data file if asked for ''' self.filelist=[os.path.basename(j) for i,j in enumerate(ls)] #~ self.filelist=[j for i,j in enumerate(ls)] ''' list of files being loaded ''' self.filelist_count = [0]*len(self.filelist) ''' number of counted values by files ''' enum = list(enumerate(ls)) enum = zip(*enum) self.fid_list=np.array(enum[0]) if len(enum) > 0 else np.array([]) self.dirname=os.getcwd() ''' Directory name of the file pattern being globbed (:func:`glob.glob`). Defaulted to current directory ''' if isinstance(file_pattern,str) : self.dirname=os.path.dirname(os.path.abspath(file_pattern)) elif len(file_pattern) > 0 : self.dirname=os.path.dirname(os.path.abspath(file_pattern[0])) self.par_list=np.array([]) ''' array of parameters ''' self.dim_list=np.array([]) ''' array containing the dimensions of each parameter ''' self._dimensions=dimStr() ''' dimensional strucutre ''' #Loop over data files ##################### for i in np.arange(len(self.fid_list)) : #Read data file ############### filename = enum[1][i] self.message(1,"Loading "+os.path.basename(filename)) #Dezip if required filename=self.dezip(filename) if filename != enum[1][i]: self.filelist[i]=os.path.basename(filename) res=self.read(filename,output_is_dict=output_is_dict,**kwargs) #read() function is specific of each class self.update_dataset(res,flatten=flatten) #update class with loaded data self.check_variables() if not self.limit_set : self.limit=self.extension(round=round) if self.count == 0 : self.warning(1,'Empty object!')
[docs] def update_dataset(self,dataStr,flatten=False): ''' update class with a data structure. :keyword flatten: use this to automatically flatten variables (squeeze dimensions) ''' #Load keys and dimensions ######################### dataDim = dataStr.pop('_dimensions',{}) attrStr = dataStr.pop('_attributes',{}) ndims = dataDim.pop('_ndims',0) dimensions = [dataDim.keys(),dataDim.values()] keys = dataStr.keys() if len(keys) == 0: self.warning(2, 'No data loaded') return self.message(2, 'Loaded variables : '+str(keys)) #Check what is the current variable type isStructure = True if isinstance(dataStr[keys[0]],dict) else False # datalen = [np.size(dataStr[key]) for key in keys] datalen = [list(np.shape(dataStr[key]['data'])[::-1]) for key in keys] if isStructure else [list(np.shape(dataStr[key])[::-1]) for key in keys] #Shape is inverted wrt to order of dimensions to be consistent with check_variable if isStructure : varDim = [list(dataStr[key]['_dimensions'])[1:] for key in keys] ind = [where_list(vDim,dimensions[0]) for vDim in varDim] #Dimensions indices from actual variables' dimensions #Check dimension lengths # dimOk = np.array([enum[1][0] == dimensions[1][ind[enum[0]][0]] for enum in enumerate(datalen)]) dimOk = [any([enum[1][ii] == dimensions[1][jj] for ii,jj in enumerate(ind[enum[0]])]) for enum in enumerate(datalen)] if any([not d for d in dimOk]) : notOk = np.where(~np.array(dimOk))[0] print datalen self.Error('Problem with {0} variables : {1}'.format(len(notOk),','.join(np.array(dataStr.keys())[notOk]))) else : ind = [where_list(dlen,dimensions[1]) for dlen in datalen] #Dimensions indices from variable length if (np.array(ind).sum() == -1)!= 0 : self.Error('At least one variable have not been properly defined') dimname = [np.array(dimensions[0])[i].tolist() for i in ind] #Get correspondance between data structure dimensions and variables curDim, nself=self.get_currentDim() createDim=np.array([np.array([w == -1 for w in where_list(j, curDim[0])]) for i,j in enumerate(dimname) ]) createDim=np.squeeze(createDim) # curInd = atools.where_list(dimname_reduced,curDim[0]) #Get correspondance between data structure dimensions and object dimensions # createDim = (np.array(curInd) == -1) #Get dimensions to be created toCreate = np.array([not self.__dict__.has_key(key) for key in keys]) updateDim=[] self.message(2, 'Updating object with '+str(['{0}({1}:{2})'.format(i[0],i[1],i[2]) for i in zip(*(keys,dimname,datalen))])) #Update variables available in files for enum in enumerate(keys) : ind=enum[0] key=enum[1] #Load variable ############## # var=dataStr.get(key) dum=dataStr.get(key).pop('data') if isStructure else copy.deepcopy(dataStr.get(key)) if flatten : if isinstance(dum,dict) :dum['data']=dum['data'].flatten() else : dum=dum.flatten() if not isStructure : dum={'_dimensions':dum._dimensions if hasattr(dum,'_dimensions') else {}, '_attributes':dum._attributes if hasattr(dum,'_attributes') else {}, 'data':dum} else : dumStr=dataStr.get(key) dumStr.update({'data':dum}) dum=dumStr dumDim=dimStr(dimname[ind],datalen[ind]) # if dataStr[key].has_key('_attributes'): # dum.update(dataStr[key]['_attributes']) # if isinstance(dum,np.ma.masked_array) : # #Get associated dimensions # ################################## # datalen = datalen[ind]#[len(dataStr[key]) for key in keys] # ind = atools.where_list([datalen],dimensions[1])[0] # if (ind == -1) : self.Error('Dimensions of current variable ('+key+') have not been properly defined') # dimname = dimensions[ #Initialize variable if required # if toCreate : # updateDim.append(self.create_Variable(key, dum, dimensions={dimname[ind]:datalen[ind]},toCreate=toCreate[ind],createDim=createDim[ind])) updateDim.append(self.create_Variable(key, dum, dimensions=dumDim,toCreate=toCreate[ind],createDim=createDim[ind])) #Extend missing variables # missing__keys = list(set(self.par_list).difference(keys)) # for enum in enumerate(missing__keys) : # ind=enum[0] # key=enum[1] # updateDim.append(self.create_Variable(key, np.ma.repeat(self.dist_to_coast_leuliette.fill_value), dimensions=dumDim,toCreate=False,createDim=False)) #Final sequence zipped_upd=zip(*(np.hstack(dimname)[~np.hstack(createDim)],np.hstack(datalen)[~np.hstack(createDim)])) updateDim_List = np.array(list(set(tuple(i) for i in np.array(zipped_upd,dtype='|S16').tolist()))) #2D unique # updateDim_List = np.unique(np.array(zipped_upd,dtype='|S16')) #[str(i) for i in datalen] # if updateDim_List.size > 0 : updateDim_List.resize((2,updateDim_List.size/2)) # updateDim_List = np.unique(zip(*(np.array(dimname)[~createDim],np.array(datalen)[~createDim]))) #[str(i) for i in datalen] zipped_dims=zip(*(np.hstack(dimname)[np.hstack(createDim)],np.hstack(datalen)[np.hstack(createDim)])) createDim_list = np.array(list(set(tuple(i) for i in np.array(zipped_dims,dtype='|S16').tolist()))) #2D unique # clist, inv = np.unique(np.array(zipped_dims,dtype='|S16'),return_inverse=True) #RQ : THIS WILL FAIL IF NUMBERS HAVE MORE THAN 16 DIGITS #[str(i) for i in datalen] # if createDim_list.size > 0 : createDim_list.resize((2,createDim_list.size/2)) # createDim_list = np.unique(zip(*(np.array(dimname)[createDim],np.array(datalen)[createDim]))) #[str(i) for i in datalen] for dname,dim in createDim_list : self.create_Dim(dname, np.int(dim)) for dname,dim in updateDim_List: self.update_Dim(dname, np.int(dim))
[docs] def check_variables(self): """ Forces variables to respect dimensions """ self.count = len(self.fileid) self.size = np.size([np.size(self.__dict__.get(par)) for par in self.par_list]) infos = zip(*(self.par_list.tolist(),self.dim_list.tolist())) # curDim, nself = self.get_currentDim() for enum in enumerate(infos): varSize = np.size(self.__dict__.get(enum[1][0])) varShape = np.shape(self.__dict__.get(enum[1][0]))[::-1] #Data and Netcdf dimensions are inverted (not allways?) dimSize = None if hasattr(self.__dict__.get(enum[1][0]),'_dimensions') : dumDim = self.__dict__.get(enum[1][0])._dimensions dumDim.pop('_ndims') dimSize = tuple(dumDim.values()) elif isinstance(self.__dict__.get(enum[1][0]),dict): if self.__dict__.get(enum[1][0]).has_key('_dimensions'): dumDim = self.__dict__.get(enum[1][0])['_dimensions'] dumDim.pop('_ndims') dimSize = tuple(dumDim.values()) if dimSize is None : if isinstance(self._dimensions,dimStr) : dimSize = (self._dimensions).get(enum[1][1]) else : dimSize = tuple([(self._dimensions).get([d]) for d in enum[1][1]]) masked = isinstance(self.__dict__.get(enum[1][0]), np.ma.masked_array) #Check mask consistency (mask length should be the same as data) if masked : if (self.__dict__.get(enum[1][0]).mask.size != self.__dict__.get(enum[1][0]).data.size): raise np.ma.core.MaskError("Mask length is not consistent with data") #Check dimensions self.message(4, 'checking variables -> {0}(N={1}) - {2}:{3}'.format(enum[1][0],varSize,enum[1][1],dimSize)) for n,sh in enumerate(varShape) : if (sh > dimSize[n]) : self.Error('Object variable {0} greater than corresponding dimension ({1})'.format(enum[1][0],enum[1][1])) elif (sh < dimSize[n]): self.message(3, 'Variable {0}(N={1}) being extended to match dimension {2}:{3}'.format(enum[1][0],varSize,enum[1][1],dimSize)) # self.__dict__[enum[1][0]] = np.ma.concatenate((self.__dict__[enum[1][0]], np.ma.masked_array(np.repeat(np.nan,dimSize - varSize),mask=np.zeros(dimSize - varSize,dtype='bool')))) #Save additonnal attributes attrStr={} for a in set(self.__dict__[enum[1][0]].__dict__.keys()).difference(np.ma.empty(0,dtype=bool).__dict__.keys()): attrStr.update({a:self.__dict__[enum[1][0]].__dict__[a]}) self.__dict__[enum[1][0]] = np.ma.masked_array( np.append(self.__dict__[enum[1][0]].data,np.repeat(self.__dict__[enum[1][0]].fill_value if hasattr(self.__dict__[enum[1][0]],'fill_value') else np.NaN,dimSize[n] - varSize)), mask=np.append(self.__dict__[enum[1][0]].mask,np.ones(dimSize[n] - varSize,dtype='bool')) ) self.__dict__[enum[1][0]].__dict__.update(attrStr)
[docs] def create_Dim(self, name,value): ''' Adds a dimension to class. :parameter name: dimension name :parameter value: dimension value ''' if not self._dimensions.has_key(name) : self.message(3, 'Create dimension {0}:{1}'.format(name,value)) self._dimensions[name]=value self._dimensions['_ndims']=len(self._dimensions) - 1 else : self.message(3, 'Dimension {0} already exists'.format(name))
[docs] def update_Dim(self,name,value): ''' update a dimension by appending the number of added elements to the dimensions :: <upddated dimension> = <old dimension> + <number of added elements along this dimension> ''' oldVal=self._dimensions[name] self._dimensions[name] += value self.message(2, 'Updating dimension {0} (from {1} to {2})'.format(name,oldVal,self._dimensions[name])) #Update dimensions within all variables for p in self.par_list: if self.__dict__[p].__dict__.has_key('_dimensions'): for d in self.__dict__[p]._dimensions.keys(): self.__dict__[p]._dimensions.update({d:self._dimensions[d]})
[docs] def update_fid_list(self,filename,N): ''' update file indices attribute `altimetry.data.hydro_data.fileid` ''' self.filelist_count[self.filelist.index(filename)] = N fid=self.fid_list.compress([enum[1][0] == os.path.basename(filename) for enum in enumerate(zip(*(self.filelist,self.fid_list)))]) self.__dict__.update({'fileid' :np.append(self.fileid,np.repeat(fid,N))})
[docs] def delete_Variable(self,name): ''' pops a variable from class and delete it from parameter list :parameter name: name of the parameter to delete ''' self.message(1,'Deleting variable {0}'.format(name)) self.par_list=self.par_list[self.par_list != name] return self.__dict__.pop(name)
[docs] def create_Variable(self,name,value,dimensions,toCreate=None,createDim=None,extend=True): """ create_Variable : This function adds data to :class:`altimetry.data.hydro_data` :parameter name: name of the parameter to create :parameter value: values associated to the variable. Must be a numpy masked_array or a data structure. :parameter dimensions: dimensional structure (cf. notes). .. _structures: .. note:: altimetry tools package handles the NetCDF data using specific structures. NetCDF data is structured this way: .. code-block:: python :emphasize-lines: 1,3 NetCDF_data = {'_dimensions':dimension_structure, #File dimensions (COMPULSORY) '_attributes':attribute_structure, #Global attributes 'dimension_1':data_structure, #Data associated to the dimensions. (COMPULSORY) ..., 'variable_1':data_structure, #Variables ... } In standard NetCDF files, dimensions are always associated to a variable. If it is not the case, an array of indices the length of the dimension is generated and a warning is issued. Moreover, dimensions MUST be defined to be accepted by :class:`altimetry.tools.nctools.nc` (empty NetCDF files would fail). * a dimensional structure should be of the form : .. code-block:: python dimension_structure = {'_ndims':N, #Attribute setting the number of dimensions. 'dims':{'dim_A':A, #Structure containing the name 'dim_B':B, #of the dimensions and their size. ..., 'dim_N':N } } * an attribute structure is a very simple structure containing the attribute names and values: .. code-block:: python data_structure = {'attribute_1':attribute_1, ..., 'attribute_N':attribute_N} * a data structure should be of the form : .. code-block:: python :emphasize-lines: 1-2 data_structure = {'_dimensions':dimension_structure, #dimensions of hte variable (COMPULSORY) 'data':data, #data associated to the variable (COMPULSORY) 'long_name':long_name, #Variable attributes 'units':units, ... } DATA and _DIMENSIONS fields are compulsory. Other fields are optional and will be treated as attributes. Furthermore, code will have a special look at **scale**, **scale_factor** and **add_offset** while reading and writing data and to **_FillValue** and missing_value while reading (_FillValue being automatically filled by :class:`NetCDF4.Dataset` when writing) """ #Check variable name #################### #This allows to remove impossible variable names #!!!! This is not a good solution name=name.replace('.','_') #Check if data is structured or not isStructure = True if isinstance(value,dict) else False #Get dimensions dimName = np.array(dimensions.keys()) dimVal = np.array(dimensions.values()) keys=np.array(self._dimensions.keys()) # if createDim is None : createDim = self._dimensions.has_key(dimName[0]) createDim = np.array([not self._dimensions.has_key(dim) for dim in dimName]) if createDim is None else np.array(createDim) if toCreate is None : toCreate = np.sum(self.par_list == name) == 0 self.message(3,'Loading {0} ({1}:{2}) from {3}'.format(name,dimName,dimVal,os.path.basename(self._filename))) #Cast variable into masked array first ###################################### if (not isinstance(value['data'],np.ma.core.MaskedArray) if isStructure else not isinstance(value,np.ma.core.MaskedArray)) : value['data'] = np.ma.masked_array(value['data'],mask=np.zeros(tuple(dimVal),dtype='bool')) if isStructure else np.ma.masked_array(value,mask=np.zeros(tuple(dimVal),dtype='bool')) self.message(4,'Casting variable to np.ma.MaskedArray') #Restructure dataset if structure if isStructure : dumvalue=value.pop('data') if value.has_key('_attributes'): for a in value['_attributes'].keys(): self.message(4, "copying attribute %s" % a) dumvalue.__setattr__(a,value['_attributes'][a]) value=copy.deepcopy(dumvalue) curDim, nself=self.get_currentDim() curInd=np.array(where_list(dimName,curDim[0])) curDimVal=np.array(where_list(dimVal,curDim[1])) existDims= (curInd != -1) createDim = (curInd == -1) createInd = np.where(createDim)[0] appendDim=existDims & (curDimVal == -1) appendInd=curInd[appendDim] # curInd = set(atools.where_list(dimVal,curDim[1])).intersection(set(atools.where_list(dimName,curDim[0]))) #Get dims to be created ####################### #Choose case between all different solutions : ############################################## # 1: create a new variable with at least 1 new dimension # 2: extend -> create a new variable using existing dimensions # 3: append exisiting variable with data # 4: impossible case ? #1) Create variable if createDim.any() & toCreate : #Create Variable self.message(4,'Create variable %s '+name) # self.__setattr__(name,value) # cmd='self.'+name+'=value' #Append variable infos to object self.par_list=np.append(self.par_list,name) dimlist_cp=self.dim_list.tolist() dimlist_cp.append(dimName.tolist()) self.dim_list=np.array(dimlist_cp) #np.append(self.dim_list,dimName.tolist()) updateDim=False #2) Extend elif (not createDim.any()) & toCreate : #extend variable if extend : dumvalue = np.ma.masked_array(np.append(np.zeros(curDim[1][curInd]),value.data),mask=np.append(np.ones(curDim[1][curInd],dtype='bool'),value.mask)) for a in set(value.__dict__.keys()).difference(dumvalue.__dict__.keys()) : dumvalue.__setattr__(a,value.__dict__[a] if hasattr(value, a) else self.__getattribute__(name).__getattribute__(a)) value=copy.deepcopy(dumvalue) self.message(4,'Extend variable '+name) # self.__setattr__(name,value) # cmd='self.'+name+'=value' # self.message(4,'exec : '+cmd) #Append variable infos to object self.par_list=np.append(self.par_list,name) dimlist_cp=self.dim_list.tolist() dimlist_cp.append(dimName.tolist()) self.dim_list=np.array(dimlist_cp) # self.dim_list=np.append(self.dim_list,dimName) updateDim=True #3) Append elif (not createDim.any()) & (not toCreate) : #append variable self.message(4,'Append data to variable '+name) dumvalue = np.ma.masked_array(np.append(self.__getattribute__(name).data,value.data),mask=np.append(self.__getattribute__(name).mask,value.mask)) #We gather a list of attributes : # - already in data structure, # - in current data file # - and not in output structure attributes=set(self.__getattribute__(name).__dict__.keys()) attributes=attributes.union(value.__dict__.keys()) # attributes=attributes.difference(self.__getattribute__(name).__dict__.keys()) attributes=attributes.difference(dumvalue.__dict__.keys()) #Then : # - we add attributes of current file not in data structure # - we keep attributes of current data structure if they exist for a in attributes : dumvalue.__setattr__(a,value.__dict__[a] if hasattr(value, a) else self.__getattribute__(name).__getattribute__(a)) value=copy.deepcopy(dumvalue) updateDim=True elif createDim.any() & (not toCreate) : #Impossible case ? self.Error('Impossible case : create dimensions and variable {0} already existing'.format(name)) #Append dimensions to variable if not dimensions.has_key('_ndims') : dumDim=dimStr(dimensions) dimensions=dumDim.copy() #Update variable dimensions if updateDim : for k in dimensions.keys(): dimensions.update({k:self._dimensions[k]}) value.__setattr__('_dimensions',dimensions) try : self.__setattr__(name,value) except np.ma.core.MaskError : raise 'mask error' # # try : exec(cmd) # except np.ma.core.MaskError : # raise 'mask error' # exec(cmd) return updateDim
[docs] def get_currentDim(self): ''' returns the current dimensions of the object ''' selfDim = self._dimensions.copy() if not isinstance(selfDim,dimStr): if selfDim.has_key('_ndims') : nself = selfDim.pop('_ndims') else : self.warning(1, 'self._dimensions does not have the _ndims key') nself = len(selfDim) else : nself = selfDim['_ndims'] curDim = [[key for key in selfDim.keys()],[selfDim[key] for key in selfDim.keys()]] return curDim, nself
[docs] def update(self,*args,**kwargs): ''' Wrapper to :func:`altimetry.data.hydro_data.update_with_slice`. ''' self.update_with_slice(*args,**kwargs)
[docs] def message(self,MSG_LEVEL,str): """ print function wrapper. Print a message depending on the verbose level :parameter {in}{required}{type=int} MSG_LEVEL: level of the message to be compared with self.verbose :example: To write a message .. code-block:: python self.message(0,'This message will be shown for any verbose level') """ caller=get_caller() if MSG_LEVEL <= self.verbose : print('[{0}.{1}()] {2}'.format(__name__,caller.co_name,str))
[docs] def warning(self,MSG_LEVEL,str): """ Wrapper to :func:`warning.warn`. Returns a warning when verbose level is not 0. :param MSG_LEVEL: level of the message to be compared with self.verbose :example: To issued a warning .. code-block:: python self.warning(1,'Warning being issued) """ if self.verbose >= 1 : warn(str)
[docs] def Error(self,ErrorMsg): ''' raises an exception ''' raise Exception(ErrorMsg)
[docs] def updated_copy(self,flag,deep=True): ''' Returns a sliced (updated) copy of current data object :summary: This has the same effect as `obj.copy();obj.update(flag)` but is much less memory consumming. .. note:: TypeError could arise if some object attributes are setted outside the :func:`__init__` function (eg. for data objects derived from :class:`hydro_data`). If this is the case, initialise these attributes within their respective :func:`__init__`. ''' emptyObj=self.__class__([]) if deep : func_copy = copy.deepcopy else : func_copy = copy.copy #Get object attributes from self for a in set(self.__dict__.keys()).intersection(emptyObj.__dict__.keys()): self.message(4, 'updating attribute %s to returned object' % a) emptyObj.__dict__[a] = func_copy(self.__dict__[a]) #Get slices for p in set(self.__dict__.keys()).difference(emptyObj.__dict__.keys()): self.message(4, 'updating parameter %s to returned object' % p) try : #Save additionnal attributes attrlist=list(set(dir(self.__dict__[p])).difference(dir(np.ma.array(0)))) attrvalues=[self.__dict__[p].__dict__[l] for l in attrlist] attr=attrStr(attrlist,attrvalues) emptyObj.__dict__[p] = func_copy(self.__dict__[p][flag]) for a in attr.keys(): setattr(emptyObj.__dict__[p], a, attr[a]) except TypeError: self.warning(1,'Could not slice %s - check if this attribute is in self.__init__') #update dimensions N=flag.sum() emptyObj._dimensions['time']=N emptyObj.count=N # self.message(4, "Checking variables consistency") # emptyObj.check_variables() return emptyObj
[docs] def copy(self,*args,**kwargs): ''' Returns a copy of the current data object :param flag: if an argument is provided, this returns an updated copy of current object (ie. equivalent to obj.copy();obj.update(flag)), optimising the memory ( :keyword True deep: deep copies the object (object data will be copied as well). ''' deep=kwargs.get('deep',True) if len(args) > 0: return self.updated_copy(*args) else : return copy.deepcopy(self) if deep else copy.copy(self)
[docs] def slice(self,param,range,surf=False): ''' get a flag for indexing based on values (ange of fixed values). :parameter param: variable name :parameter range: numpy array defining the range of the values. If size(range) == 2 : * flag is computed between min and max values of range * flag is computed based on equality to range value. ''' if np.size(range) == 2 : if not surf : fg = (self.__dict__[param] >= np.min(range)) & (self.__dict__[param] < np.max(range)) else : fg = (self.__dict__[param+'_surf'] >= np.min(range)) & (self.__dict__[param+'_surf'] < np.max(range)) elif np.size(range) == 1 : if not surf : fg = (self.__dict__[param] == range) else : fg = (self.__dict__[param+'_surf'] == range) else : self.Error('Range array must have 1 or 2 elements max') return fg
[docs] def time_slice(self,timerange,surf=False): ''' slice object given a time range :parameter timerange: rime range to be used. ''' if isinstance(timerange[0],str): trange = cnes_convert(timerange)[0] else: trange=timerange return self.slice('date',trange,surf=surf)
[docs] def update_with_slice(self,flag): ''' update object with a given time slice flag :parameter (boolean array) flag: a flag for indexing data along the ''time'' dimension ''' N=flag.sum() #Get object attributes to update varSize = np.array([np.size(self.__dict__[k]) for k in self.__dict__.keys()]) par_list=np.array(self.__dict__.keys())[varSize == self._dimensions['time']].tolist() self._dimensions['time']=N self.count=N for par in par_list : # self.__setattr__(par,self.__dict__[par][flag]) # inVar=copy.deepcopy(self.__dict__[par]) if isinstance(self.__dict__[par],np.ma.masked_array) : tmpVar=self.__dict__.pop(par) dumVar=copy.deepcopy(tmpVar) tmpVar=tmpVar.compress(flag) for a in set(dumVar.__dict__.keys()).difference(tmpVar.__dict__.keys()) : tmpVar.__dict__[a]=dumVar.__dict__[a] self.__dict__[par] = tmpVar elif isinstance(self.__dict__[par],np.ndarray): self.__dict__[par] = self.__dict__[par].compress(flag) elif isinstance(self.__dict__[par],list): self.__dict__[par] = (np.array(self.__dict__[par])[flag]).tolist() if (hasattr(self.__dict__[par], '_dimensions')) : self.__dict__[par]._dimensions['time']=self._dimensions['time']
[docs] def get_file(self,pattern): ''' returns a flag array of the data loaded from a given file pattern :parameter pattern: pattern to match in the file list. ''' flag=[fnmatch.fnmatch(l,pattern) for l in self.filelist] id=self.fid_list.compress(flag) flag = self.fileid == id return flag
def dezip(self,filename): fname,extzip = os.path.splitext(filename) __tempfile = [k for k in self.__dict__.keys() if k.endswith("__tempfile")][0] if [".gz"].count(extzip) == 0: return filename _,extension =os.path.splitext(fname) self.__tempfile += [tempfile.mktemp(extension,__tempfile,'/tmp')] self.message(2,"Unzipping %s to %s" % (os.path.basename(filename),os.path.basename(self.__tempfile[-1]))) if extzip == ".gz": self.message(2,"gunzip -c %s > %s" % (filename,self.__tempfile[-1])) os.system("gunzip -c %s > %s" % (filename,self.__tempfile[-1])) return self.__tempfile[-1]
[docs] def time_range(self,flag=None): ''' time range of the current dataset :keyword flag: use a flag array to know the time range of an indexed slice of the object ''' if self.count==0: return [[None,None],[None,None]] if flag is None : return cnes_convert([self.date.min(),self.date.max()]) else : return cnes_convert([self.date.compress(flag).min(),self.date.compress(flag).max()])
[docs] def extension(self,flag=None,round=True): ''' returns the limits of the dataset. :keyword flag: an indexation flag array :keyword round: round the limits to the south-west and north-east. ''' if flag is None : limit = [self.lat.min(),self.lon.min(),self.lat.max(),self.lon.max()] \ if (self.__dict__.has_key('lat') and self.__dict__.has_key('lon')) \ else [-90,0,90,360] else : limit = [self.lat.compress(flag).min(),self.lon.compress(flag).min(),self.lat.compress(flag).max(),self.lon.compress(flag).max()] \ if (self.__dict__.has_key('lat') and self.__dict__.has_key('lon')) \ else [-90,0,90,360] if round : limit[0]=np.floor(limit[0]) limit[2]=np.floor(limit[2]) limit[1]=np.ceil(limit[1]) limit[3]=np.ceil(limit[3]) return limit
def get_id_list(self,flag=None): if flag is None : return np.unique(self.id) else : return np.unique(self.id.compress(flag))
[docs] def get(self,name): ''' retunrs a variable ''' return self.__dict__[name]
[docs] def pop(self,*args,**kwargs): ''' This is a wrapper to :meth:`altimetry.data.hydro_data.delete_Variable` ''' return self.delete_Variable(*args,**kwargs)
[docs] def get_timestats(self,flag,par_list=None,full=True,bins=None): ''' get temporal statistics about a part of the dataset :keyword par_list: List of parameters :keyword True full: cf. :return: section :return: If not FULL returs par_stats only (dict of dicts), else returns (par_list, valid, per_valid, fname, trange, extent, N, avail_par, avail_par_per, par_stats) :example: par_list, valid, per_valid, fname, trange, extent, N, avail_par, avail_par_per, par_stats = self.stats(flag) ''' if par_list is None : par_list=self.par_list.compress([(not par.endswith('_surf')) & ( par != 'id') for par in self.par_list]) else : if not isinstance(par_list,np.ma.masked_array) : par_list=np.ma.array(par_list,mask=np.zeros(len(par_list),dtype=bool)) valid= np.array([len(self.__dict__[par].compress(flag).compressed()) for par in par_list]) per_valid = (100.0* valid) /float(np.sum(flag)) fname = [self.filelist[i] for i in np.unique(self.fileid.compress(flag)).astype('i')] trange = self.time_range(flag) extent = self.extension(flag) N = np.sum(flag) avail_par = par_list.compress(per_valid > 0) avail_par_per = per_valid.compress(per_valid > 0) if bins is None and trange is not None: bins=(max(trange[1])-min(trange[1])).days + 1 else: bins=() par_stats = OrderedDict() for p in avail_par : var = self.__dict__[p].compress(flag).compressed() N,t_edges=np.histogram(self.date,bins=bins) H,_=np.histogram(self.date,bins=bins,weights=var) t_edges=t_edges[:-1]+(t_edges[1]-t_edges[0])/2. par_stats.update({p:{'nb':N, 'mean':H/N, 't_axis':t_edges } }) if full : out = par_list, valid, per_valid, fname, trange, extent, N, avail_par, avail_par_per, par_stats else : out = par_stats return out
[docs] def get_stats(self,flag,par_list=None,full=True): ''' get some statistics about a part of the dataset :keyword par_list: List of parameters :keyword True full: cf. :return: section :return: If not FULL returs par_stats only (dict of dicts), else returns (par_list, valid, per_valid, fname, trange, extent, N, avail_par, avail_par_per, par_stats) :example: par_list, valid, per_valid, fname, trange, extent, N, avail_par, avail_par_per, par_stats = self.stats(flag) ''' if par_list is None : par_list=self.par_list.compress([(not par.endswith('_surf')) & ( par != 'id') for par in self.par_list]) else : if not isinstance(par_list,np.ma.masked_array) : par_list=np.ma.array(par_list,mask=np.zeros(len(par_list),dtype=bool)) valid= np.array([len(self.__dict__[par].compress(flag).compressed()) for par in par_list]) per_valid = (100.0* valid) /float(np.sum(flag)) fname = [self.filelist[i] for i in np.unique(self.fileid.compress(flag)).astype('i')] trange = self.time_range(flag) extent = self.extension(flag) N = np.sum(flag) avail_par = par_list.compress(per_valid > 0) avail_par_per = per_valid.compress(per_valid > 0) par_stats = OrderedDict() for p in avail_par : var = self.__dict__[p].compress(flag).compressed() par_stats.update({p:{'nb':N, 'mean':np.mean(var), 'median':np.median(var), 'std':np.std(var), 'min':np.min(var), 'max':np.max(var), # 'ptp':np.ptp(var)} #peak-to-peak }) if full : out = par_list, valid, per_valid, fname, trange, extent, N, avail_par, avail_par_per, par_stats else : out = par_stats return out
[docs] def get_platform_stats(self,id,functype='overall'): ''' get statistics based on `altimetry.data.hydro_data.id` :param functype: set to 'overall' to get variables moments or 'time'|'temporal' to get temporal variability ''' if functype == 'overall': get_stats = self.get_stats elif functype == 'time' or functype == 'temporall': get_stats = self.get_timestats else: get_stats = self.get_stats par_list, valid, per_valid, fname, trange, extent, N, avail_par, avail_par_per, par_stats = get_stats(self.id == id) return (fname,trange,extent,N,avail_par,avail_par_per, par_stats)
[docs] def get_object_stats(self,functype='overall',**kwargs): ''' get some statistics about the whole dataset. :param functype: set to 'overall' to get variables moments or 'time' to get temporal variability :return: par_list, valid, per_valid, fname, trange, extent, N, avail_par, avail_par_per, par_stats :example: par_list, valid, per_valid, fname, trange, extent, N, avail_par, avail_par_per, par_stats = self.get_object_stats() ''' if functype == 'overall': get_stats = self.get_stats elif functype == 'time': get_stats = self.get_timestats return get_stats(np.ones(self.count,dtype='bool'),**kwargs)
[docs] def platform_summary(self,id,col='.k',functype='overall'): ''' outputs a summary of the statistics for a given platform ''' stats = self.get_platform_stats(id,functype=functype) self.message(0, '\tPlatform {0} : '.format(id)) self.message(0, '\t-> file : {0}'.format(stats[0])) self.message(0, '\t-> from : '+' - '.join(map(str,stats[1][0]))) self.message(0, '\t-> extent : ['+', '.join(['{0:.1f}'.format(x) for x in stats[2]])+']') self.message(0, '\t-> size : {0} pts'.format(stats[3])) self.message(0, '\t-> variables : [%s]' % ', '.join(['{0}({1:.0f} %)'.format(i[0],i[1]) for i in zip(*(stats[4],stats[5]))])+']') if functype == 'time' or functype == 'temporal': nsub=len(stats[6].keys()) nx,ny = ((nsub-1)/2) + 1, 2 plt.suptitle('Platform : %s' % id) for i,item in enumerate(stats[6].iteritems()): s,v=item[0],copy.deepcopy(item[1]) t=v.pop('t_axis') ax1=plt.subplot(nx,ny,i+1) ax1.plot(t,v['nb'],'-k') plt.title(s) if i == nsub: ax1.set_xlabel('time') if i%2 == 0: ax1.set_ylabel('#') ax2 = ax1.twinx() ax2.set_ylabel('mean') if i%2 == 1: ax1.set_ylabel('#') ax2.plot(t,v['mean'],'-r') plt.show() else: for p in stats[6].keys() : self.message(0,'\t\to %s : {%s}' % (p,','.join(["%s:%f" % (s,stats[6][p][s]) for s in stats[6][p].keys()])))
[docs] def map(self, flag=None, fname=None, zoom=False, pmap=None, show=True, bathy=False, **kwargs): ''' display (or not) a map based on a :class:`altimetry.tools.plot_map` object. :keyword show: set to False not to show (and neither apply :meth:`altimetry.tools.plot_map.setup_map`) .. note:: This function creates a :class:`altimetry.tools.plot_map` instance, plot a partion of the dataset using :meth:`altimetry.data.hydro_data.plot_track` and displays it if asked to. ''' if zoom : limit = self.extension(flag) else : limit = self.limit if pmap is None : pmap=plot_map(0,0,0,limit=limit,bathy=bathy) p,=self.plot_track(pmap, flag,**kwargs) if show : pmap.setup_map(bathy=bathy) pmap.show() return p,pmap
[docs] def summary(self,all=False,fig=None,col='.k',legend=None,bathy=False,functype='overall',**kwargs): """ outputs a summary of the whole current dataset :param functype: set to 'overall' to get variables moments or 'time' to get temporal variability """ par_list, valid, per_valid, fname, trange, extent, N, avail_par, avail_par_per, par_stats = self.get_object_stats() #Print parameter list self.message(0, '\n\t DATA SUMMARY') self.message(0, '\tObject type <class \'{0}\'> ({1})'.format(self.__class__.__name__,','.join(map(str,self.__class__.__bases__)))) self.message(0, '\tLoaded from : '+self.dirname)#', '.join(map(str,[os.path.basename(file) for file in self.filelist]))) self.message(0, '\tLimits = '+', '.join(map(str,self.limit))) self.message(0, '\tDate range : '+' - '.join(map(str,trange[0]))) self.message(0, '\tSpatial extension: ['+', '.join(['{0:.1f}'.format(x) for x in extent])+']') self.message(0, '\tRecords : [{0}]'.format(N)) self.message(0, '\tAvailable parameters : '+', '.join(map(str,self.par_list))) self.message(0, '\tParam summary : [%s]' % ', '.join(['{0}({1:.0f} %)'.format(i[0],i[1]) for i in zip(*(avail_par,avail_par_per))])) for p in avail_par : self.message(0,'\t\to %s : {%s}' % (p,','.join(["%s:%s" % (s,par_stats[p][s]) for s in par_stats[p].keys()]))) if isinstance(fig,str) : ffig = fig+'dataset.png' show=False else : ffig = fig show=True if fig is not None : if self.__dict__.has_key('id_surf') : n=len(self.id_surf) else : n=N # p,pmap=self.map(np.ones(n,dtype='bool'),ffig,col=col,show=show,**kwargs) p,pmap=self.map(col=col,show=show,bathy=bathy,**kwargs) pmap.setup_map() if legend is not None : plt.legend([p],[legend]) if not show : pmap.savefig(ffig) plt.clf() if all is not True : return self.message(0, '\t##### Details by float ID') #Get all platform ID's id_list=self.get_id_list().astype('a') for id in id_list : self.platform_summary(id,functype=functype) if isinstance(fig,str) : ffig = fig+'{0}.png'.format(str(id)) else : ffig = None if fig is not None : if self.__dict__.has_key('id_surf') : flag=self.id_surf == id n=len(self.id_surf.compress(flag)) else : flag=self.id == id n=len(self.id.compress(flag)) p,pmap=self.map(flag=flag,col=col,show=show,**kwargs) pmap.setup_map() if legend is not None : plt.legend([p],['#{0}'.format(id)]) if show is not True : plt.savefig(ffig) plt.clf()
[docs] def in_limits(self,limit=None): ''' wrapper to :func:`altimetry.tools.in_limits` based on dataset limits. ''' if limit is None : limit = self.limit flag=in_limits(self.lon, self.lat, limit) return flag
[docs] def plot_track(self,pmap,flag=None,col='.k',endpoint='*r',endpoint_size=None,title=None,fontsize=8,textcolor='b',ms=5,linewidth=1,**kwargs): ''' plot trajectories based on platform IDs :parameter pmap: a :class:`altimetry.tools.plot_map` instance :parameter col: color to be used along the trajectory. If this is an array of values, calls :func:`altimetry.tools.plot_map.scatter` instead of :func:`altimetry.tools.plot_map.plot` .. note:: This method loops on data IDs. Then it calls :func:`altimetry.tools.plot_map.plot` or :func:`altimetry.tools.plot_map.scatter` to plot the trajectory and then labels the trajectory using :func:`altimetry.tools.plot_map.text` ''' if self.__dict__.has_key('lon_surf') : if flag is None : flag=np.ones(self.lon_surf.size,dtype='bool') lon=self.lon_surf.compress(flag) lat=self.lat_surf.compress(flag) dat=self.date_surf.compress(flag) id=self.id_surf.compress(flag) else : if flag is None : flag=np.ones(self.lon.size,dtype='bool') lon=self.lon.compress(flag) lat=self.lat.compress(flag) dat=self.date.compress(flag) id=self.id.compress(flag) if endpoint_size is None : endpoint_size=2*ms id_list=np.unique(id) cnt=np.size(id_list) # #Go to next iteration when no data # if cnt == 0 : # continue for j in id_list : dumflag = (id == j) dumdat=dat.compress(dumflag) sort = dumdat.argsort() dumlon=lon.compress(dumflag)[sort] dumlat=lat.compress(dumflag)[sort] dumid=id.compress(dumflag)[sort] dumcnt=np.size(dumid) dumvalid=(~dumlon.mask & ~dumlat.mask & ~dumid.mask).sum() if dumvalid > 0: if isinstance(col,str) : p=pmap.plot(dumlon,dumlat,col,ms=ms,linewidth=linewidth,**kwargs) else : p=pmap.scatter(dumlon,dumlat,col.compress(dumflag)[sort],s=ms,linewidth=linewidth,**kwargs) pmap.plot(dumlon[dumcnt-1],dumlat[dumcnt-1],endpoint,ms=endpoint_size) pmap.text(dumlon[dumcnt-1],dumlat[dumcnt-1],'{0}'.format(j),fontsize=fontsize,color=textcolor) if title is not None : pmap.title(title) # print j # pmap.show() try : return p finally : return pmap.plot(-100,-370,'') #this is a hack to plot data outside the globe...
[docs] def plot_track_old(self,*args,**kwargs): ''' plot a surface map of sampling track .. warning:: DEPRECATED method! ''' #Get map object (plot_map or Basemap) flag_list=[inspect.ismodule(type(i)) | inspect.ismodule(type(i)) for i in args] if (np.array(flag_list).max()) : map_obj = args[0] obj_plot=map_obj args.pop(0) else : obj_plot=plt #Retrieve other params lon=args[0] lat=args[1] #override if passed directly through 3rd argument if len(args) == 3 : var = args[2] scatter=True else : if 'c' in kwargs : c = kwargs['c'] del kwargs['c'] else : c = '.k' scatter=False if scatter :obj_plot.scatter(lon,lat,var,**kwargs) else : obj_plot.plot(lon,lat,**kwargs)
[docs] def plot_transect(self, x, z, var, xrange=None, zrange=None, vmin=None,vmax=None, #colormap limits xstep=1,zstep=10, #Interpolation step s=10, edgecolor='none', **kwargs): ''' shows a 2d space-depth section plotting point (using :func:`altimetry.tools.plot_map.scatter`) :example: plot a temperature section along a glider transect ''' ax=plt.gca() if isinstance(var,np.ma.masked_array) : flag=np.reshape(var.mask ,var.size) values=var.compress(~flag) x=x.compress(~flag) z=z.compress(~flag) else : values = var return ax.scatter(x,z,c=values,s=s,edgecolor=edgecolor,vmin=vmin,vmax=vmax) # plt.show() pass
[docs] def contour_transect(self, x, z, var, xrange=None,zrange=None, xstep=1,zstep=10,vmin=None,vmax=None,marker='.k', **kwargs): ''' shows a 2d space-depth section by interpolating the data along the section. .. note:: This method interpolates using :func:`scipy.interpolate.griddata` and plots using :func:`matplotlib.pyplot.meshcolorgrid` ''' if xrange is None : xrange=(x.min(),x.max()) if zrange is None : zrange=(z.min(),z.max()) gx=np.arange(xrange[0],xrange[1],xstep) gz=np.arange(zrange[0],zrange[1],zstep) grid_x, grid_y = np.meshgrid(gx,gz) #Remove masks flag=np.reshape(var.mask ,var.size) values=var.compress(~flag) x=x.compress(~flag) z=z.compress(~flag) npts=values.size # plt.tricontourf(x,z,values) #VERY long!! points = zip(*(np.reshape(x,npts),np.reshape(z,npts))) Z = interpolate.griddata(points, values, (grid_x, grid_y), method='cubic') # plt.imshow(gdist,gdepth,Z,vmin=13.1,vmax=13.6) plt.pcolormesh(gx,gz,Z,vmin=vmin,vmax=vmax) if marker is not None : plt.plot(x,z,marker,ms=1) # plt.scatter(dist2d,depth,s=10,c=values,edgecolor='none',vmin=13.1,vmax=13.6) # plt.show() return Z, gx, gz
[docs] def read_ArgoNC(self,filename,params=None,force=False,dephrange=None,timerange=None,**kwargs): """ An Argo network NetCDF reader :return outStr: Output data stricture (dict) containing all recorded parameters as specificied by NetCDF file PARAMETER list. :author: Renaud Dussurget """ #Open file self._filename = filename self._ncfile = ncfile(self._filename, "r") #Get list of recorded parameters: dum=self._ncfile.variables['STATION_PARAMETERS'][0,:] nparam=np.shape(dum)[0] par_list=np.array([''.join(self._ncfile.variables['STATION_PARAMETERS'][0,i,:].compressed()) for i in np.arange(nparam)]) #remove empty items and update nparam par_list=par_list.compress([len(par) != 0 for par in par_list]) nparam=par_list.size if params is not None : if force : par_list=[i.upper() for i in params] else :par_list=list(set(params).intersection(par_list)) else : par_list=par_list.tolist() self.message(1,'Recorded parameters : '+str(nparam)+' -> '+str(par_list)) lon = self.load_ncVar('LONGITUDE',**kwargs) lat = self.load_ncVar('LATITUDE',**kwargs) date = self.load_ncVar('JULD',**kwargs) # lon = self._ncfile.variables['LONGITUDE'][:] # lat = self._ncfile.variables['LATITUDE'][:] #Extract within limits ind, flag = in_limits(lon['data'],lat['data'],limit=self.limit) if timerange is not None : dflag = (date['data'] >= np.min(timerange)) & (date['data'] < np.max(timerange)) flag = flag & dflag ind = np.where(flag)[0] dim_lon = lon['_dimensions'] dim_date = date['_dimensions'] lat['data'] = lat['data'].compress(flag) lon['data'] = lon['data'].compress(flag) date['data'] = date['data'].compress(flag) #Update dimensions lat['_dimensions']['N_PROF']=flag.sum() lon['_dimensions']['N_PROF']=flag.sum() date['_dimensions']['N_PROF']=flag.sum() # dist=cumulative_distance(lat['data'], lon['data']) sz=np.shape(lon) ndims=np.size(sz) # if dephrange is not None : # pres = self.load_ncVar('PRES',N_PROF=ind,**kwargs) # deph=gsw.z_from_p(pres['data'],lat[0]) # depind=(np.abs(tutu-(-np.max(dephrange)))).argmin(1) # if timerange is not None :deph=gsw.z_from_p(pres['data'],lat[0]) date = self.load_ncVar('JULD',N_PROF=ind,**kwargs) dimStr = date['_dimensions'] # date=date['data'] # #Create dimension structure # curDim = [str(dimname) for dimname in dimStr.keys()[1:]] #[str(dimname) for dimname in self._ncfile.variables['LONGITUDE'].dimensions] # curDimval = [dimStr[dim] for dim in curDim] #[len(self._ncfile.dimensions[dimname]) for dimname in curDim] # # outStr={'_dimensions':{'_ndims':ndims,'nbpoints':sz[0]},'lon':lon,'lat':lat,'date':date} outStr={'_dimensions':dimStr,'lon':lon,'lat':lat,'date':date} for param in par_list : dumVar = self.load_ncVar(param,N_PROF=ind,**kwargs) #Load variables dimStr=dumVar.get('_dimensions') # dimStr=dumVar.pop('_dimensions') # dumVar['data'].__setattr__('_dimensions',dimStr) #update current object dimensions with missing variable dimensions curDim = [str(dimname) for dimname in dimStr.keys()[1:]] #[str(dimname) for dimname in self._ncfile.variables['LONGITUDE'].dimensions] curDimval = [dimStr[dim] for dim in curDim] #[len(self._ncfile.dimensions[dimname]) for dimname in curDim] # curDim = [str(dimname) for dimname in self._ncfile.variables[param].dimensions] # curDimval = [len(self._ncfile.dimensions[dimname]) for dimname in curDim] flag = [(np.array(dimname) == outStr['_dimensions'].keys()).sum() == 0 for dimname in curDim] #find dimensions to update dimUpdate = np.array(curDim).compress(flag) for enum in enumerate(dimUpdate) : self.message(2, 'Appending dimensions {0}:{1} to dataStructure'.format(enum[1],np.array(curDimval).compress(flag)[enum[0]])) outStr['_dimensions'].update({enum[1]:np.array(curDimval).compress(flag)[enum[0]]}) #Append new dimension outStr['_dimensions']['_ndims']+=1 #update dimension counts self.message(4, 'Loading {0} ({1})'.format(param.lower(),','.join(str(dimStr[key]) for key in dimStr.keys()[1:]))) dumStr=dumVar.copy() # locals()[param.lower()] = {param.lower():dumVar['data']} # cmd = 'dumStr = {\''+param.lower()+'\':dumVar[\'data\']}' # self.message(4, 'exec : '+cmd) # exec(cmd) outStr.update({param.lower():dumStr}) # id=np.array([''.join([num for num in j.compressed()]) for j in self._ncfile.variables['PLATFORM_NUMBER'][ind]]) #get platform id into array pid=self.load_ncVar('PLATFORM_NUMBER',N_PROF=ind,**kwargs) pid_var=np.ma.MaskedArray([''.join([num for num in j.compressed()]) for j in pid['data']],mask=lon['data'].mask) pid['data']=pid_var.copy() for key in pid['_dimensions'].keys() : if key.startswith('STRING') : pid['_dimensions'].pop(key) pid['_dimensions']['_ndims']-=1 outStr.update({'id':pid}) self._ncfile.close() return outStr
def load_ncVar(self,varName,nc=None,**kwargs): return load_ncVar_v2(varName, nc=self._ncfile, **kwargs)
[docs] def ncstruct(self,**kwargs): ''' returns a data structure (dict) of the dataset. :keyword params: Add this keyword to provide a list of variables to export. Default : all variables contained is self.par_list ''' par_list = kwargs.get('params',self.par_list.tolist()) dimStr=self._dimensions dimlist = dimStr.keys() #outStr = OrderedDict({'_dimensions':dimStr}) outStr=ncStr() outStr['_dimensions'].update(dimStr) if (np.array(par_list) == 'sat').sum() : par_list.pop(par_list.index('sat')) #Remove satellite info varlist=np.append(np.array(dimlist[1:]),np.array(par_list)) for d in varlist : self.message(2, 'Updating output structure with {0}'.format(d)) curDim=getattr(self.__getattribute__(d),'_dimensions',None) attributes=[a for a in self.__getattribute__(d).__dict__.keys() if not a.startswith('_')] attr=attrStr(keys=attributes, values=[self.__getattribute__(d).__getattribute__(a) for a in attributes]) data=self.__getattribute__(d) outStr[d]=varStr(dimensions=curDim if curDim is not None else self._dimensions, attributes=attr, data=data) return outStr
[docs] def write_nc(self,filename,clobber=False,**kwargs): ''' write a NetCDF file from current dataset :keyword kwargs: additional arguments are passed to :meth:`altimetry.tools.nctools.nc.write` ''' obj=ncobj(verbose=kwargs.pop('verbose',self.verbose),limit=self.limit,use_local_dims=True) ncalti=self.ncstruct() #Get an netcdf structure from data return obj.write(ncalti,filename,clobber=clobber,**kwargs) #Save processed datase
[docs] def push_nc(self,*args,**kwargs): ''' append a data structure to an exisiting netcdf file ''' obj=ncobj(verbose=self.verbose,limit=self.limit,use_local_dims=True) res=obj.push(*args,**kwargs)
def __len__(self): return self.count def __del__(self): for f in self.__tempfile : os.unlink(f)
class buoy_data(hydro_data): def __init__(self,file_pattern,**kwargs): hydro_data.__init__(self,file_pattern,**kwargs) # self.count=np.size(self.lon) def read(self,filename,**kwargs): fname,extension = os.path.splitext(filename) # if extension == '.nc' : outStr=self.read_ArgoNC(filename,N_LEVELS=0,**kwargs) if extension == '.nc' : outStr=self.read_ArgoNC(filename,**kwargs) elif extension == '.dat' : outStr = self.read_txt(filename) elif extension == '.asc' : kwread=kwargs.get('lon_name',{}) kwread=kwargs.get('lat_name',{}) outStr = self.read_asc(filename,**kwread) #CLS dumps else : self.Error('Unknown formatting') #Rework dimensions ################## # Dimensions may be in 2D (more not compliant with current code) # Dataobject only accepts flattened (1D) variables dimStr=outStr.pop('_dimensions') # if dimStr['_ndims'] != 2 : self.Error('Variables with {0} dimensions can not be used as glider profiles (2 dimensions)'.format(dimStr['_ndims'])) if not dimStr.has_key('N_LEVELS') or not dimStr.has_key('N_PROF') : self.Error('Data structure must contain dimensions N_LEVELS and N_PROF (only contains {0})'.format(dimStr.keys()[1:])) datalen = [np.size(outStr[key]) for key in outStr.keys()] #[np.shape(outStr[key]) for key in outStr.keys()] nblevels = dimStr['N_LEVELS'] nbprofiles = dimStr['N_PROF'] nbpoints = nblevels*nbprofiles #Change dimension names towards new ones for key in outStr.keys() : if outStr[key].has_key('_dimensions'): outStr[key]['_dimensions'].pop('N_LEVELS',None) outStr[key]['_dimensions'].pop('N_PROF',None) # if outStr[key]['_dimensions'].has_key('N_PROF') : outStr[key]['_dimensions'].update({'nbprofiles':outStr[key]['_dimensions'].pop('N_PROF')}) outStr[key]['_dimensions'].update({'nbpoints':nbpoints}) #Reform 2D variables from 1D ############################ twoD_flag = [] for key in outStr.keys() : twoD_flag.append(len(np.shape(outStr[key]['data'])) == 2) if isinstance(outStr[key],dict) else twoD_flag.append(len(np.shape(outStr[key])) == 2) twoD_flag = np.array(twoD_flag) oneD_vars = np.array(outStr.keys()).compress(~twoD_flag) twoD_vars = np.array(outStr.keys()).compress(twoD_flag) #Transform 2D variables into 1D # -> For each record (N_PROF), we only keep the valid depth bin (N_PROF) for key in twoD_vars: if isinstance(outStr[key],dict) : outStr[key]['data']=outStr[key]['data'][:,outStr[key]['data'].mask.argmin(1)].diagonal() else : outStr[key]=outStr[key][:,outStr[key].mask.argmin(1)].diagonal() #Flatten variables (force flattening in case...) for key in outStr.keys(): if isinstance(outStr[key],dict) : outStr[key]['data']=outStr[key]['data'].flatten() else : outStr[key]=outStr[key].flatten() #Additonnal variables dst=outStr['lat'].copy() if isinstance(dst,dict) : dst['data']=calcul_distance(outStr['lat']['data'],outStr['lon']['data']) outStr.update({'dist':dst}) else : outStr.update({'dist':calcul_distance(outStr['lat'],outStr['lon'])}) #Update dimensions newDim = {'_dimensions' : {'_ndims':1,'nbprofiles':nbprofiles}} outStr.update(newDim) self.update_fid_list(os.path.basename(filename),outStr['_dimensions']['nbprofiles']) return outStr def read_txt(self,filename): #Open file self._filename = filename data=np.genfromtxt(filename,skip_header=1) #Convert to numpy arrays lon=np.ma.masked_array(data[:,3]) lat=np.ma.masked_array(data[:,2]) import datetime date=np.ma.masked_array(data[:,1])-datetime.date(1951,1,2).toordinal() id=np.ma.masked_array(data[:,0]) sz=np.shape(lon) ndims=np.size(sz) #Get SST, U & V data i available basename=os.path.basename(filename) dirname=os.path.dirname(filename) sstuv_name=dirname+'/tempuv_'+basename[basename.find('_')+1:len(basename)] data=np.genfromtxt(sstuv_name,skip_header=1) temp=np.ma.masked_array(data[:,0]) temp=np.ma.masked_equal(temp, 9999.) u=np.ma.masked_array(data[:,1]) u=np.ma.masked_equal(u, 9999.) v=np.ma.masked_array(data[:,2]) v=np.ma.masked_equal(v, 9999.) return {'_dimensions':{'_ndims':ndims,'N_PROF':sz[0],'N_LEVELS':1},'lon':lon,'lat':lat,'date':date,'id':id,'temp':temp,'u':u,'v':v} def read_asc(self,filename,lon_name='LON_FILTRE',lat_name='LAT_FILTRE'): #Open file self._filename = filename asc=open(self._filename) #get header properties l=0 par_list=[] par_dv=[] for line in asc.readlines(): if not line.startswith('//') : break else : if line.startswith('//\tClip') : par_list.append(line.split(' ')[-1].replace('\n','')) if line.startswith('//\tDefaut') : par_dv.append(line.split('=')[-1].replace('\n','')) l+=1 nPar=len(par_list) col_id=np.arange(nPar)+2 #Get lon & lat par_list[par_list.index(lon_name)]='lon' par_list[par_list.index(lat_name)]='lat' data=np.genfromtxt(filename,skip_header=l) sz=data.shape[0] mask=np.zeros(sz,dtype=bool) #Construct output structure dimStr={'_dimensions':{'_ndims':2,'N_PROF':sz,'N_LEVELS':1}} outStr = OrderedDict() outStr.update(dimStr) outStr.update({'N_PROF':{'_dimensions':{'_ndims':1,'N_PROF':sz},'data':np.ma.array(np.arange(sz),mask=mask.copy())}}) outStr.update({'N_LEVELS':{'_dimensions':{'_ndims':1,'N_PROF':sz},'data':np.ma.array(np.repeat(25.,sz),mask=mask.copy())}}) #1st row is assigned to float ID, 2nd to date (julian seconds) outStr.update({'id':{'_dimensions':{'_ndims':1,'N_PROF':sz},'data':np.ma.masked_array(data[:,0],mask=mask.copy())}}) outStr.update({'date':{'_dimensions':{'_ndims':1,'N_PROF':sz},'data':np.ma.masked_array(data[:,1]/86400.,mask=mask.copy())}}) for i,p in enumerate(par_list): dumVar=data[:,col_id[i]] mask=dumVar==eval('np.{0}'.format(dumVar.dtype))(par_dv[i]) outStr.update({p:{'_dimensions':{'_ndims':1,'N_PROF':sz},'data':np.ma.masked_array(dumVar,mask=mask.copy())}}) #recompute mask based on lon,lat,date validity mask=outStr['lon']['data'].mask.copy() \ | outStr['lat']['data'].mask.copy() \ | outStr['date']['data'].mask.copy() #Reapply mask (we may check dimensions?) outStr['date']['data'].mask=mask.copy() for k in par_list : if outStr[k].has_key('data') : outStr[k]['data'].mask=mask.copy() return outStr # def plot_track(self,pmap,date): # # bflag=abs(self.date - date) <= 0.5 # blon=self.lon.compress(bflag) # blat=self.lat.compress(bflag) # bdat=self.date.compress(bflag) # bid=self.id.compress(bflag) # # bid_list=np.unique(bid) # cnt=np.size(bid_list) # ## #Go to next iteration when no data ## if cnt == 0 : ## continue # # for j in bid_list : # dumflag = (bid == j) # dumdat=bdat.compress(dumflag) # id = dumdat.argsort() # dumlon=blon.compress(dumflag)[id] # dumlat=blat.compress(dumflag)[id] # dumid=bid.compress(dumflag)[id] # dumcnt=np.size(dumid) # pmap.plot(dumlon,dumlat,'.k',ms=5) # pmap.plot(dumlon[dumcnt-1],dumlat[dumcnt-1],'*r',ms=10) # pmap.text(dumlon[dumcnt-1],dumlat[dumcnt-1],str(int(j)),fontsize=8,color='b') ## pmap.title('IMEDIA day '+str(i)) ## # plt.text(, y, s, fontsize=12) ## pmap.savefig(FIG_DIR+"/IMEDIA_alti_buoys_"+str(i)+".png") class argo_trajectory(hydro_data): def __init__(self,file_pattern,**kwargs): hydro_data.__init__(self,file_pattern,**kwargs) # self.count=np.size(self.lon) def read(self,filename,**kwargs): fname,extension = os.path.splitext(filename) # if extension == '.nc' : outStr=self.read_ArgoNC(filename,N_LEVELS=0,**kwargs) if extension == '.nc' : outStr=self.read_ArgoNC(filename,**kwargs) elif extension == '.dat' : outStr = self.read_txt(filename) else : self.Error('Unknown formatting') #Rework dimensions ################## # Dimensions may be in 2D (more not compliant with current code) # Dataobject only accepts flattened (1D) variables dimStr=outStr.pop('_dimensions') # if dimStr['_ndims'] != 2 : self.Error('Variables with {0} dimensions can not be used as glider profiles (2 dimensions)'.format(dimStr['_ndims'])) if not dimStr.has_key('N_PROF') : self.Error('Data structure must contain dimensions N_LEVELS and N_PROF (only contains {0})'.format(dimStr.keys()[1:])) datalen = [np.size(outStr[key]) for key in outStr.keys()] #[np.shape(outStr[key]) for key in outStr.keys()] nbprofiles = dimStr['N_PROF'] nbpoints = nbprofiles #Reform 2D variables from 1D ############################ twoD_flag = np.array([len(np.shape(outStr[key])) for key in outStr.keys()]) == 2 oneD_vars = np.array(outStr.keys()).compress(~twoD_flag) twoD_vars = np.array(outStr.keys()).compress(twoD_flag) #Transform 2D variables into 1D # -> For each record (N_PROF), we only keep the valid depth bin (N_PROF) for key in twoD_vars: outStr.update({key:outStr[key][:,outStr[key].mask.argmin(1)].diagonal()}) #Flatten variables (force flattening in case...) for key in outStr.keys(): outStr.update({key:outStr[key].flatten()}) #Additonnal variables outStr.update({'dist':calcul_distance(outStr['lat'],outStr['lon'])}) #Update dimensions newDim = {'_dimensions' : {'_ndims':1,'nbpoints':nbprofiles}} outStr.update(newDim) self.update_fid_list(os.path.basename(filename),outStr['_dimensions']['nbpoints']) return outStr class glider_data(hydro_data): def __init__(self,file_pattern,**kwargs): #Init hydro_data class (calling gilder_data.read() function) hydro_data.__init__(self,file_pattern,**kwargs) def read(self,filename,**kwargs): fname,extension = os.path.splitext(filename) outStr=self.read_ArgoNC(filename,**kwargs) #Rework dimensions ################## # Dimensions may be in 2D (more not compliant with current code) # Dataobject only accepts flattened (1D) variables dimStr=outStr.pop('_dimensions') if dimStr['_ndims'] != 2 : self.Error('Variables with {0} dimensions can not be used as glider profiles (2 dimensions)'.format(dimStr['_ndims'])) if not dimStr.has_key('N_LEVELS') or not dimStr.has_key('N_PROF') : self.Error('Data structure must contain dimensions N_LEVELS and N_PROF (only contains {0})'.format(dimStr.keys()[1:])) # datalen = [np.size(outStr[key]) for key in outStr.keys()] #[np.shape(outStr[key]) for key in outStr.keys()] nblevels = dimStr['N_LEVELS'] # nblevels = nblevels.astype(nblevels.dtype) nbprofiles = dimStr['N_PROF'] nbpoints = nblevels*nbprofiles #Change dimension names towards new ones for key in outStr.keys() : if outStr[key].has_key('_dimensions'): outStr[key]['_dimensions'].pop('N_LEVELS',None) outStr[key]['_dimensions'].pop('N_PROF',None) # if outStr[key]['_dimensions'].has_key('N_PROF') : outStr[key]['_dimensions'].update({'nbprofiles':outStr[key]['_dimensions'].pop('N_PROF')}) outStr[key]['_dimensions'].update({'nbpoints':nbpoints}) #Reform 2D variables from 1D ############################ twoD_flag = [] for key in outStr.keys() : twoD_flag.append(len(np.shape(outStr[key]['data'])) == 2) if isinstance(outStr[key],dict) else twoD_flag.append(len(np.shape(outStr[key])) == 2) twoD_flag = np.array(twoD_flag) oneD_vars = np.array(outStr.keys()).compress(~twoD_flag) twoD_vars = np.array(outStr.keys()).compress(twoD_flag) #Transform 1D variables into 2D for key in oneD_vars: if isinstance(outStr[key],dict) : outStr[key]['data'] = np.reshape(np.repeat(outStr[key]['data'],nblevels),[nbprofiles,nblevels]) else : outStr[key]=np.reshape(np.repeat(outStr[key],nblevels),[nbprofiles,nblevels]) # [{key:np.shape(outStr[key])} for key in outStr.keys()] #Check variables #Load surface data surfStr=self.read_ArgoNC(filename,N_LEVELS=0) #read surface data surfStr.pop('_dimensions') #Change dimension names towards new ones for key in surfStr.keys() : if surfStr[key].has_key('_dimensions'): surfStr[key]['_dimensions'].pop('N_LEVELS',None) if surfStr[key]['_dimensions'].has_key('N_PROF') : surfStr[key]['_dimensions'].update({'nbprofiles':surfStr[key]['_dimensions'].pop('N_PROF')}) #Flatten surface variables for key in surfStr.keys(): if isinstance(surfStr[key],dict) : surfStr[key]['data'] = surfStr[key]['data'].flatten() else : surfStr[key] = surfStr[key].flatten() outStr.update({key+'_surf':surfStr[key]}) #Flatten 2D variables ( for key in outStr.keys(): if isinstance(outStr[key],dict) : outStr[key]['data']=outStr[key]['data'].flatten() else : outStr[key]=outStr[key].flatten() #Additonnal variables dst_surf=outStr['lat_surf'].copy() dst=outStr['lat'].copy() if isinstance(dst_surf,dict) : dst_surf['data']=calcul_distance(outStr['lat_surf']['data'],outStr['lon_surf']['data']) dst['data']=np.repeat(dst_surf['data'],nblevels) outStr.update({'dist_surf':dst_surf}) outStr.update({'dist':dst}) else : outStr.update({'dist_surf':calcul_distance(outStr['lat_surf'],outStr['lon_surf'])}) outStr.update({'dist':np.repeat(outStr['dist_surf'],nblevels)}) ###TODO : Could be simplified? (if has_key(var) --> must have key var_surf if outStr.has_key('pres') : deph = outStr['pres'].copy() deph_surf = outStr['pres_surf'].copy() try: deph['data']=gsw.z_from_p(outStr['pres']['data'],outStr['lat']['data']) deph_surf['data']=gsw.z_from_p(outStr['pres_surf']['data'],outStr['lat_surf']['data']) except : deph['data'][:]=deph['data'].fill_value deph_surf['data'][:]=deph['data'].fill_value outStr.update({'deph' : deph}) outStr.update({'deph_surf' : deph_surf}) if outStr.has_key('psal') and outStr.has_key('temp') and outStr.has_key('pres') : rho=outStr['psal'].copy() rho_surf=outStr['psal_surf'].copy() try: rho['data']=gsw.rho(outStr['psal']['data'],outStr['temp']['data'],outStr['pres']['data']) rho_surf['data']=gsw.rho(outStr['psal_surf']['data'],outStr['temp_surf']['data'],outStr['pres_surf']['data']) except : rho['data'][:]=deph['data'].fill_value rho_surf['data'][:]=deph['data'].fill_value outStr.update({'rho' : rho}) outStr.update({'rho_surf' : rho_surf}) #Update dimensions newDim = {'_dimensions' : {'_ndims':2,'nbpoints':nbpoints,'nbprofiles':nbprofiles}} outStr.update(newDim) #Update fid self.update_fid_list(os.path.basename(filename),outStr['_dimensions']['nbpoints']) return outStr class TSG_data(hydro_data) : def __init__(self,file_pattern,**kwargs): hydro_data.__init__(self,file_pattern,**kwargs) def read(self,filename): fname,extension = os.path.splitext(filename) if extension == '.nc' : outStr=self.read_ArgoNC(filename,params=['TEM2','PSAL']) outStr.update({'temp':outStr.pop('tem2')}) #Use TEM2 as temperature field # outStr.update({'depth':outStr.pop('deph')}) #Use DEPH as depth field return outStr elif extension == '.dat' : return self.read_txt(filename) else : self.Error('Unknown formatting') def read_txt(self,filename): #Open file self._filename = filename data=np.genfromtxt(filename,skip_header=1) #Convert to numpy arrays lon=np.ma.masked_array(data[:,2]) lat=np.ma.masked_array(data[:,1]) import datetime date=np.ma.masked_array(data[:,0])+datetime.date(2012,1,1).toordinal()-datetime.date(1950,1,2).toordinal() temp=np.ma.masked_array(data[:,3]) temp=np.ma.masked_greater(temp, 99.) psal=np.ma.masked_array(data[:,4]) psal=np.ma.masked_less(psal, 35.) fluo=np.ma.masked_array(data[:,5]) fluo=np.ma.masked_where((fluo == 0.0) | (fluo >= 99.),fluo) id=np.repeat('IMEDIA_TSG',len(psal)) sz=np.shape(lon) ndims=np.size(sz) return {'_dimensions':{'_ndims':ndims,'nbpoints':sz[0]},'lon':lon,'lat':lat,'date':date,'id':id,'temp':temp,'psal':psal,'fluo':fluo} class CTD_data(hydro_data): def __init__(self,file_pattern,**kwargs): #Init hydro_data class (calling gilder_data.read() function) hydro_data.__init__(self,file_pattern,**kwargs) def read(self,filename,**kwargs): fname,extension = os.path.splitext(filename) if extension == '.nc' : outStr=self.read_ArgoNC(filename,params=['TEM2','PSAL']) outStr.update({'temp':outStr.pop('tem2')}) #Use TEM2 as temperature field # outStr.update({'depth':outStr.pop('deph')}) #Use DEPH as depth field return outStr elif extension == '.dat' : return self.read_txt(filename) #Seabird CTD data elif extension == '.cnv' : return self.read_cnv(filename,**kwargs) elif extension == '.asc' : return self.read_asc(filename,**kwargs) else : self.Error('Unknown formatting') def read_asc(self,filename,**kwargs): self._filename = filename #Check file length nlines=0 for line in open(filename): nlines+=1 if nlines > 1 : #Open file data=np.genfromtxt(filename,skip_header=1) #Convert to numpy arrays lon=kwargs['lon'] lat=kwargs['lat'] import datetime date=np.ma.masked_array(data[:,0])+datetime.date(2012,1,1).toordinal()-datetime.date(1950,1,2).toordinal() pres=np.ma.masked_array(data[:,1]) temp=np.ma.masked_array(data[:,2]) cond=np.ma.masked_array(data[:,3]) obs1=np.ma.masked_array(data[:,4]) obs2=np.ma.masked_array(data[:,5]) descend_rate=np.ma.masked_array(data[:,6]) scan=np.ma.masked_array(data[:,7]) fluoro=np.ma.masked_array(data[:,8]) depth=np.ma.masked_array(data[:,9]) potemp=np.ma.masked_array(data[:,10]) psal=np.ma.masked_array(data[:,11]) dens=np.ma.masked_array(data[:,12]) svCM=np.ma.masked_array(data[:,13]) flag=np.ma.masked_array(data[:,14]) reclen=len(pres) date.mask=np.zeros(reclen,dtype='bool') pres.mask=np.zeros(reclen,dtype='bool') temp.mask=np.zeros(reclen,dtype='bool') cond.mask=np.zeros(reclen,dtype='bool') obs1.mask=np.zeros(reclen,dtype='bool') obs2.mask=np.zeros(reclen,dtype='bool') descend_rate.mask=np.zeros(reclen,dtype='bool') scan.mask=np.zeros(reclen,dtype='bool') fluoro.mask=np.zeros(reclen,dtype='bool') depth.mask=np.zeros(reclen,dtype='bool') potemp.mask=np.zeros(reclen,dtype='bool') psal.mask=np.zeros(reclen,dtype='bool') dens.mask=np.zeros(reclen,dtype='bool') svCM.mask=np.zeros(reclen,dtype='bool') flag.mask=np.zeros(reclen,dtype='bool') id=np.repeat('{0}'.format(kwargs['stationid']),reclen) lon=np.ma.masked_array(np.repeat(lon,reclen),mask=np.zeros(reclen,dtype='bool')) lat=np.ma.masked_array(np.repeat(lat,reclen),mask=np.zeros(reclen,dtype='bool')) # psal=csw.salt(cond/gsw.cte.C3515, temp, pres) # depth=gsw.z_from_p(pres,lat) # dens= gsw.rho(psal,temp,pres) sz=np.shape(lon) ndims=np.size(sz) else : ndims = 1. sz=(1,1) lon=np.array(np.NaN) lat=np.array(np.NaN) date=np.array(np.NaN) id=np.array(np.NaN) depth=np.array(np.NaN) pres=np.array(np.NaN) temp=np.array(np.NaN) psal=np.array(np.NaN) fluoro=np.array(np.NaN) dens=np.array(np.NaN) potemp=np.array(np.NaN) cond=np.array(np.NaN) obs1=np.array(np.NaN) obs2=np.array(np.NaN) svCM=np.array(np.NaN) descend_rate=np.array(np.NaN) flag=np.array(np.NaN) return {'_dimensions':{'_ndims':ndims,'nbpoints':sz[0]},'lon':lon,'lat':lat,'date':date,'id':id,'depth':depth,'pres':pres,'temp':temp, \ 'psal':psal,'fluoro':fluoro,'dens':dens,'potemp':potemp,'cond':cond,'obs1':obs1,'obs2':obs2,'svCM':svCM,'descend_rate':descend_rate,'flag':flag} def read_cnv(self,filename,**kwargs): #Open file data=np.genfromtxt(filename,skip_header=328) #Convert to numpy arrays lon=kwargs['lon'] lat=kwargs['lat'] import datetime date=np.ma.masked_array(data[:,0])+datetime.date(2012,1,1).toordinal()-datetime.date(1950,1,2).toordinal() pres=np.ma.masked_array(data[:,1]) temp=np.ma.masked_array(data[:,2]) cond=np.ma.masked_array(data[:,3]) obs1=np.ma.masked_array(data[:,6]) obs2=np.ma.masked_array(data[:,7]) fluoro=np.ma.masked_array(data[:,]) pres=np.ma.masked_greater(pres, 99999.) temp=np.ma.masked_greater(temp, 99.) cond=np.ma.masked_greater(cond, 99.) #fluo=np.ma.masked_where((fluo == 0.0) | (fluo >= 99.),fluo) reclen=len(pres) id=np.repeat('{0}'.format(kwargs['stationid']),reclen) lon=np.ma.masked_array(np.repeat(lon,reclen)) lat=np.ma.masked_array(np.repeat(lat,reclen)) try : psal=csw.salt(cond/gsw.cte.C3515, temp, pres) depth=gsw.z_from_p(pres,lat) dens= gsw.rho(psal,temp,pres) except : psal=np.ma.masked_array(np.repeat(lon.fill_value,reclen),mask=True) depth=np.ma.masked_array(np.repeat(lon.fill_value,reclen),mask=True) dens=np.ma.masked_array(np.repeat(lon.fill_value,reclen),mask=True) sz=np.shape(lon) ndims=np.size(sz) return {'_dimensions':{'_ndims':ndims,'nbpoints':sz[0]},'lon':lon,'lat':lat,'date':date,'id':id,'depth':depth,'pres':pres,'temp':temp,'psal':psal}