r'''
The pipelines module contains classes that represent data procesing
pipelines. At this moment, only NDPPP is implemented here, but later
on, this module will also support calibrator/target pipelines and
imaging settings.
'''
from lofarobsxml.observationspecificationbase import ObservationSpecificationBase
from lofarobsxml.utilities import AutoReprBaseClass, typecheck, lower_case, unique, indent
from lofarobsxml.momformats import mom_duration, mom_timestamp
import ephem
[docs]class NDPPP(AutoReprBaseClass):
r'''
The demixing- and averaging parameters for NDPPP.
**Parameters**
avg_freq_step : int
How many channels to average.
avg_time_step : int
How many time steps to average.
demix_freq_step : int
Width (in channels) of the window over which demixing is
done. This must be a multiple of ``avg_freq_step``.
demix_time_step : int
Length (in time slots) of the demixing window. This must be a
multiple of ``avg_time_step``.
demix_always : None or list of strings
Sources to always demix. Valid source names are: ['CasA',
'CygA', 'TauA', 'HydraA', 'VirA', 'HerA'].
ignore_target : None or bool
See imagoing cookbook documentation. None implies observatory
default.
**Examples**
>>> dmx = NDPPP(16, 2, demix_freq_step=64, demix_time_step=10,
... demix_always=['CygA', 'CasA'])
>>> dmx
NDPPP(avg_freq_step = 16,
avg_time_step = 2,
demix_always = ['CygA', 'CasA'],
demix_freq_step = 64,
demix_if_needed = None,
demix_time_step = 10,
ignore_target = None)
>>> print(dmx.xml())
<BLANKLINE>
<demixingParameters>
<averagingFreqStep>16</averagingFreqStep>
<averagingTimeStep>2</averagingTimeStep>
<demixFreqStep>64</demixFreqStep>
<demixTimeStep>10</demixTimeStep>
<demixAlways>[CygA,CasA]</demixAlways>
<demixIfNeeded></demixIfNeeded>
<ignoreTarget></ignoreTarget>
</demixingParameters>
Of course, several consistency checks are made:
>>> NDPPP(16, 2, demix_freq_step=64, demix_time_step=5,
... demix_always=['CygA', 'CasA'])
Traceback (most recent call last):
...
ValueError: NDPPP.demix_time_step(5) is not a multiple of NDPPP.avg_time_step(2)
>>> NDPPP(16, 2.5, demix_freq_step=64, demix_time_step=5,
... demix_always=['CygA', 'CasA'])
Traceback (most recent call last):
...
TypeError: type(NDPPP.avg_time_step)(2.5) not in ['int']
'''
def __init__(self,
avg_freq_step = 64, avg_time_step = 1,
demix_freq_step = 64, demix_time_step = 10,
demix_always = None, demix_if_needed = None,
ignore_target = None):
self.avg_freq_step = avg_freq_step
self.avg_time_step = avg_time_step
self.demix_freq_step = demix_freq_step
self.demix_time_step = demix_time_step
self.demix_always = demix_always
if type(self.demix_always) is str:
self.demix_always = [self.demix_always]
self.demix_if_needed = demix_if_needed
if type(self.demix_if_needed) is str:
self.demix_if_needed = [self.demix_if_needed]
self.ignore_target = ignore_target
self.validate()
[docs] def validate(self):
r'''
Raise a ValueError or TypeError if problems with the
specification are detected.
'''
typecheck(self.avg_freq_step, int, 'NDPPP.avg_freq_step')
typecheck(self.avg_time_step, int, 'NDPPP.avg_time_step')
typecheck(self.demix_freq_step, int, 'NDPPP.demix_freq_step')
typecheck(self.demix_time_step, int, 'NDPPP.demix_time_step')
typecheck(self.demix_always, [list, type(None)],
'NDPPP.demix_always')
typecheck(self.demix_if_needed, [list, type(None)],
'NDPPP.demix_if_needed')
if self.demix_freq_step % self.avg_freq_step != 0:
raise ValueError('NDPPP.demix_freq_step(%r) is not a multiple of NDPPP.avg_freq_step(%r)' %
(self.demix_freq_step,
self.avg_freq_step))
if self.demix_time_step % self.avg_time_step != 0:
raise ValueError('NDPPP.demix_time_step(%r) is not a multiple of NDPPP.avg_time_step(%r)' %
(self.demix_time_step,
self.avg_time_step))
typecheck(self.ignore_target, [type(None), type(True)],
'NDPPP.ignore_target')
[docs] def xml(self):
r'''
Produce an xml representation of demixing settings.
'''
template = '''
<demixingParameters>
<averagingFreqStep>%(avg_freq_step)d</averagingFreqStep>
<averagingTimeStep>%(avg_time_step)d</averagingTimeStep>
<demixFreqStep>%(demix_freq_step)d</demixFreqStep>
<demixTimeStep>%(demix_time_step)d</demixTimeStep>
<demixAlways>%(demix_always)s</demixAlways>
<demixIfNeeded>%(demix_if_needed)s</demixIfNeeded>
<ignoreTarget>%(ignore_target)s</ignoreTarget>
</demixingParameters>'''
args = {'avg_freq_step' : self.avg_freq_step,
'avg_time_step' : self.avg_time_step,
'demix_freq_step' : self.demix_freq_step,
'demix_time_step' : self.demix_time_step,
'demix_always' : '',
'demix_if_needed' : '',
'ignore_target' : ''}
if self.demix_always is not None:
args['demix_always'] = '['+','.join(self.demix_always)+']'
if self.demix_if_needed is not None:
args['demix_if_needed'] = '['+','.join(self.demix_if_needed)+']'
if self.ignore_target is not None:
args['ignore_target'] = lower_case(self.ignore_target)
return template % args
[docs]class AveragingPipeline(ObservationSpecificationBase):
r'''
**Parameters**
flagging_strategy: 'LBAdefault', 'HBAdefault', or None
NDPPP flagging strategy.
initial_status : string
Either 'opened' or 'approved'
**Examples**
>>> from lofarobsxml import TargetSource, Angle
>>> from lofarobsxml.backend import BackendProcessing
>>> from lofarobsxml.observation import Observation
>>> from lofarobsxml.beam import Beam
>>> target = TargetSource(name = 'Cyg A',
... ra_angle = Angle(hms = (19, 59, 28.3566)),
... dec_angle = Angle(sdms = ('+', 40, 44, 2.097)))
>>> bm = Beam(target, '77..324')
>>> obs = Observation('HBA_DUAL_INNER', 'LBA_LOW', (2013, 10, 20, 18, 5, 0),
... duration_seconds = 600, name = 'Main observation',
... stations = ['CS001', 'RS106', 'DE601'],
... clock_mhz = 200, beam_list = [bm],
... backend = BackendProcessing())
>>> avg = AveragingPipeline(name = 'Avg Pipeline', ndppp = NDPPP())
>>> avg.add_input_data_product(obs.children[0])
>>> obs.append_child(avg)
>>> avg
AveragingPipeline(parent = Observation('Main observation'),
children = None,
default_template = 'Preprocessing Pipeline',
duration_s = None,
flagging_strategy = None,
initial_status = 'opened',
input_data = [Beam(parent = Observation('Main observation'),
children = None,
duration_s = None,
initial_status = 'opened',
measurement_type = 'Target',
name = 'Cyg A',
subband_spec = '77..324',
target_source = TargetSource(name = 'Cyg A',
ra_angle = Angle(shms = ('+', 19, 59, 28.3566)),
dec_angle = Angle(sdms = ('+', 40, 44, 2.097))),
tied_array_beams = None)],
name = 'Avg Pipeline',
ndppp = NDPPP(avg_freq_step = 64,
avg_time_step = 1,
demix_always = None,
demix_freq_step = 64,
demix_if_needed = None,
demix_time_step = 10,
ignore_target = None),
predecessor_label = None,
start_date = None)
>>> print(avg.xml('Project name'))
<lofar:pipeline xsi:type="lofar:AveragingPipelineType">
<topology>Main_observation.1.Avg_Pipeline</topology>
<predecessor_topology>Main_observation</predecessor_topology>
<name>Avg Pipeline</name>
<description>Avg Pipeline: "Preprocessing Pipeline"</description>
<currentStatus>
<mom2:openedStatus/>
</currentStatus>
<averagingPipelineAttributes>
<defaultTemplate>Preprocessing Pipeline</defaultTemplate>
<duration></duration>
<startTime></startTime>
<endTime></endTime>
<demixingParameters>
<averagingFreqStep>64</averagingFreqStep>
<averagingTimeStep>1</averagingTimeStep>
<demixFreqStep>64</demixFreqStep>
<demixTimeStep>10</demixTimeStep>
<demixAlways></demixAlways>
<demixIfNeeded></demixIfNeeded>
<ignoreTarget></ignoreTarget>
</demixingParameters>
<flaggingStrategy>HBAdefault</flaggingStrategy>
</averagingPipelineAttributes>
<usedDataProducts>
<item>
<lofar:uvDataProduct topology="Main_observation.0.Cyg_A.dps">
<name>Main_observation.0.Cyg_A.dps</name>
</lofar:uvDataProduct>
</item>
</usedDataProducts>
<resultDataProducts>
<item>
<lofar:uvDataProduct>
<name>Main_observation.1.Avg_Pipeline.dps</name>
<topology>Main_observation.1.Avg_Pipeline.dps</topology>
<status>no_data</status>
</lofar:uvDataProduct>
</item>
</resultDataProducts>
</lofar:pipeline>
'''
def __init__(self, name, ndppp, input_data = None,
duration_s = None, start_date = None,
flagging_strategy = None,
parent = None, children = None,
predecessor_label = None,
initial_status='opened'):
super(AveragingPipeline, self).__init__(name=name,
parent=parent,
children=children,
initial_status=initial_status)
self.ndppp = ndppp
self.input_data = None
self.duration_s = duration_s
self.start_date = start_date
self.flagging_strategy = flagging_strategy
self.default_template = 'Preprocessing Pipeline'
self.predecessor_label = predecessor_label
if input_data is not None:
for item in input_data:
self.add_input_data_product(item)
self.validate()
[docs] def validate(self):
r'''
'''
typecheck(self.ndppp, NDPPP,
'AveragingPipeline.NDPPP')
typecheck(self.predecessor_label, [type(None), str],
'AveragingPipeline.predecessor_label')
[docs] def predecessor(self):
r'''
'''
if self.predecessor_label is not None:
return self.predecessor_label
predecessor_observations = unique(
[data_set.parent.label()
for data_set in self.input_data
if data_set.parent is not None])
if len(predecessor_observations) != 1:
raise ValueError('AveragingPipeline: more than one predecessor (%r)' %
predecessor_observations)
return predecessor_observations[0]
def xml_prefix(self, project_name = None):
template ='''<lofar:pipeline xsi:type="lofar:AveragingPipelineType">
<topology>%(label)s</topology>
<predecessor_topology>%(predecessor)s</predecessor_topology>
<name>%(name)s</name>
<description>%(name)s: "%(default_template)s"</description>
<currentStatus>
<mom2:%(initial_status)sStatus/>
</currentStatus>
<averagingPipelineAttributes>
<defaultTemplate>%(default_template)s</defaultTemplate>
<duration>%(duration)s</duration>
<startTime>%(start_time)s</startTime>
<endTime></endTime>%(ndppp)s
<flaggingStrategy>%(flagging_strategy)s</flaggingStrategy>
</averagingPipelineAttributes>
<usedDataProducts>%(used_data_products)s
</usedDataProducts>
<resultDataProducts>
<item>
<lofar:uvDataProduct>
<name>%(label)s.dps</name>
<topology>%(label)s.dps</topology>
<status>no_data</status>
</lofar:uvDataProduct>
</item>
</resultDataProducts>
'''
used_data_product_template = '''\n<item>
<lofar:uvDataProduct topology="%(name)s">
<name>%(name)s</name>
</lofar:uvDataProduct>
</item>'''
args = {
'label' : self.label(),
'predecessor' : self.predecessor(),
'name' : self.name,
'default_template' : self.default_template,
'duration' : '',
'start_time' : '',
'flagging_strategy': self.flagging_strategy,
'ndppp' : indent(self.ndppp.xml(), 4),
'used_data_products' : '',
'initial_status': self.initial_status
}
if self.duration_s is not None:
args['duration'] = mom_duration(seconds = self.duration_s)
if self.start_date is not None:
start_date = ephem.Date(self.start_date).tuple()
rounded_start_date = start_date[:-1]+(int(round(start_date[-1])),)
args['start_time'] = mom_timestamp(*rounded_start_date)
if self.input_data is None:
raise ValueError('AveragingPipeline.input_data is None!')
if self.flagging_strategy is None:
args['flagging_strategy'] = self.input_data[0].parent.antenna_set[0:3].upper()+'default'
elif self.flagging_strategy in ['HBAdefault', 'LBAdefault']:
args['flagging_strategy'] = self.flagging_strategy
else:
raise ValueError('lofarobsxml.AverigingPipeline: unknown flagging strategy %r' %
self.flagging_strategy)
args['used_data_products'] = indent(
'\n'.join([
used_data_product_template % {'name' : sap.data_products_label()}
for sap in self.input_data]),
4)
return template % args
def xml_suffix(self, project_name= None):
return '</lofar:pipeline>'