1 | import re |
---|
2 | import math |
---|
3 | import string |
---|
4 | from asap._asap import selector as _selector, srctype |
---|
5 | from asap.utils import unique, _to_list |
---|
6 | |
---|
7 | class selector(_selector): |
---|
8 | """ |
---|
9 | A selection object to be applied to scantables to restrict the |
---|
10 | scantables to specific rows. |
---|
11 | """ |
---|
12 | fields = ["pols", "ifs", "beams", "scans", "cycles", "name", "query", "types", "rows"] |
---|
13 | |
---|
14 | def __init__(self, *args, **kw): |
---|
15 | if len(args) == 1: |
---|
16 | if isinstance(args[0], self.__class__) \ |
---|
17 | or isinstance(args[0], _selector): |
---|
18 | _selector.__init__(self, args[0]) |
---|
19 | else: |
---|
20 | raise TypeError("Argument can only be a selector object") |
---|
21 | else: |
---|
22 | _selector.__init__(self) |
---|
23 | for k,v in kw.items(): |
---|
24 | if k in self.fields: |
---|
25 | func = getattr(self, "set_%s" % k) |
---|
26 | func(v) |
---|
27 | |
---|
28 | def reset(self): |
---|
29 | """ |
---|
30 | Unset all selections. |
---|
31 | """ |
---|
32 | self._reset() |
---|
33 | |
---|
34 | def is_empty(self): |
---|
35 | """ |
---|
36 | Has anything been set? |
---|
37 | """ |
---|
38 | return self._empty() |
---|
39 | |
---|
40 | def set_polarisations(self, pols=[]): |
---|
41 | """ |
---|
42 | Set the polarisations to be selected in the scantable. |
---|
43 | Parameters: |
---|
44 | pols: a list of integers of 0-3, or strings, e.g ["I","Q"]. |
---|
45 | Default [] is no selection |
---|
46 | Example: |
---|
47 | sel = selector() |
---|
48 | # These are equivalent if data is 'linear' |
---|
49 | sel.set_polarisations(["XX","Re(XY)"]) |
---|
50 | sel.set_polarisations([0,2]) |
---|
51 | # reset the polarisation selection |
---|
52 | sel.set_polarisations() |
---|
53 | |
---|
54 | """ |
---|
55 | vec = _to_list(pols, str) or _to_list(pols, int) |
---|
56 | if isinstance(vec, list): # is an empty and/or valid vector |
---|
57 | if len(vec) and isinstance(vec[-1],str): |
---|
58 | self._setpolstrings(vec) |
---|
59 | return |
---|
60 | self._setpols(vec) |
---|
61 | else: |
---|
62 | raise TypeError('Unknown pol type. Please use [0,1...] or ["XX","YY"...]') |
---|
63 | |
---|
64 | # for the americans |
---|
65 | set_polarizations = set_polarisations |
---|
66 | # for the lazy |
---|
67 | set_pols = set_polarisations |
---|
68 | |
---|
69 | def set_ifs(self, ifs=[]): |
---|
70 | """ |
---|
71 | Set a sequence of IF numbers (0-based). |
---|
72 | Parameters: |
---|
73 | ifs: a list of integers. Default [] is to unset the selection. |
---|
74 | """ |
---|
75 | vec = _to_list(ifs, int) |
---|
76 | if isinstance(vec,list): |
---|
77 | self._setifs(vec) |
---|
78 | else: |
---|
79 | raise TypeError('Unknown IFno type. Use lists of integers.') |
---|
80 | |
---|
81 | def set_scans(self, scans=[]): |
---|
82 | """ |
---|
83 | Set a sequence of Scan numbers (0-based). |
---|
84 | Parameters: |
---|
85 | scans: a list of integers. Default [] is to unset the selection. |
---|
86 | """ |
---|
87 | vec = _to_list(scans, int) |
---|
88 | if isinstance(vec,list): |
---|
89 | self._setscans(vec) |
---|
90 | else: |
---|
91 | raise TypeError('Unknown Scan number type. Use lists of integers.') |
---|
92 | |
---|
93 | def set_beams(self, beams=[]): |
---|
94 | """ |
---|
95 | Set a sequence of Beam numbers (0-based). |
---|
96 | Parameters: |
---|
97 | beams: a list of integers. Default [] is to unset the selection. |
---|
98 | """ |
---|
99 | vec = _to_list(beams, int) |
---|
100 | if isinstance(vec,list): |
---|
101 | self._setbeams(vec) |
---|
102 | else: |
---|
103 | raise TypeError('Unknown Beam number type. Use lists of integers.') |
---|
104 | |
---|
105 | def set_cycles(self, cycles=[]): |
---|
106 | """ |
---|
107 | Set a sequence of IF numbers (0-based). |
---|
108 | Parameters: |
---|
109 | cycless: a list of integers. Default [] is to unset the selection. |
---|
110 | """ |
---|
111 | vec = _to_list(cycles, int) |
---|
112 | if isinstance(vec,list): |
---|
113 | self._setcycles(vec) |
---|
114 | else: |
---|
115 | raise TypeError('Unknown Cycle number type. Use lists of integers.') |
---|
116 | |
---|
117 | |
---|
118 | def set_name(self, name): |
---|
119 | """ |
---|
120 | Set a selection based on a name. This can be a unix pattern , e.g. "*_R" |
---|
121 | Parameters: |
---|
122 | name: a string containing a source name or pattern |
---|
123 | Examples: |
---|
124 | # select all reference scans which start with "Orion" |
---|
125 | selection.set_name("Orion*_R") |
---|
126 | """ |
---|
127 | if isinstance(name, str): |
---|
128 | self._setname(name) |
---|
129 | else: |
---|
130 | raise TypeError('name must be a string') |
---|
131 | |
---|
132 | def set_tsys(self, tsysmin=0.0, tsysmax=None): |
---|
133 | """ |
---|
134 | Select by Tsys range. |
---|
135 | Parameters: |
---|
136 | tsysmin: the lower threshold. Default 0.0 |
---|
137 | tsysmax: the upper threshold. Default None. |
---|
138 | Examples: |
---|
139 | # select all spectra with Tsys <= 500.0 |
---|
140 | selection.set_tsys(tsysmax=500.0) |
---|
141 | |
---|
142 | """ |
---|
143 | taql = "SELECT FROM $1 WHERE TSYS[0] >= %f" % (tsysmin) |
---|
144 | if isinstance(tsysmax, float): |
---|
145 | taql = taql + " AND TSYS[0] <= %f" % ( tsysmax) |
---|
146 | self._settaql(taql) |
---|
147 | |
---|
148 | def set_query(self, query): |
---|
149 | """ |
---|
150 | Select by Column query. Power users only! |
---|
151 | Example: |
---|
152 | # select all off scans with integration times over 60 seconds. |
---|
153 | selection.set_query("SRCTYPE == PSOFF AND INTERVAL > 60.0") |
---|
154 | """ |
---|
155 | rx = re.compile("((SRCTYPE *[!=][=] *)([a-zA-Z.]+))", re.I) |
---|
156 | for r in rx.findall(query): |
---|
157 | sval = None |
---|
158 | stype = r[-1].lower() |
---|
159 | if stype.find('srctype.') == -1: |
---|
160 | stype = ".".join(["srctype", stype]) |
---|
161 | try: |
---|
162 | sval = eval(stype) |
---|
163 | sval = "%s%d" % (r[1], sval) |
---|
164 | except: |
---|
165 | continue |
---|
166 | query = query.replace(r[0], sval) |
---|
167 | taql = "SELECT FROM $1 WHERE " + query |
---|
168 | self._settaql(taql) |
---|
169 | |
---|
170 | def set_order(self, order): |
---|
171 | """ |
---|
172 | Set the order the scantable should be sorted by. |
---|
173 | Parameters: |
---|
174 | order: The list of column names to sort by in order |
---|
175 | """ |
---|
176 | self._setorder(order) |
---|
177 | |
---|
178 | def set_rows(self, rows=[]): |
---|
179 | """ |
---|
180 | Set a sequence of row numbers (0-based). Power users Only! |
---|
181 | NOTICE row numbers can be changed easily by sorting, |
---|
182 | prior selection, etc. |
---|
183 | Parameters: |
---|
184 | rows: a list of integers. Default [] is to unset the selection. |
---|
185 | """ |
---|
186 | vec = _to_list(rows, int) |
---|
187 | if isinstance(vec,list): |
---|
188 | self._setrows(vec) |
---|
189 | else: |
---|
190 | raise TypeError('Unknown row number type. Use lists of integers.') |
---|
191 | |
---|
192 | def set_types(self, types=[]): |
---|
193 | """ |
---|
194 | Set a sequence of source types. |
---|
195 | Parameters: |
---|
196 | types: a list of integers. Default [] is to unset the selection. |
---|
197 | """ |
---|
198 | vec = _to_list(types, int) |
---|
199 | if isinstance(vec,list): |
---|
200 | self._settypes(vec) |
---|
201 | else: |
---|
202 | raise TypeError('Unknown row number type. Use lists of integers.') |
---|
203 | |
---|
204 | def set_msselection_field(self, selection): |
---|
205 | """ |
---|
206 | """ |
---|
207 | selection_list = map(string.strip, selection.split(',')) |
---|
208 | query_list = list(self.generate_query(selection_list)) |
---|
209 | query = 'SELECT FROM $1 WHERE ' + ' || '.join(query_list) |
---|
210 | self._settaql(query) |
---|
211 | |
---|
212 | def generate_query(self, selection_list): |
---|
213 | for s in selection_list: |
---|
214 | if s.isdigit() or re.match('^[<>]=?[0-9]*$', s) or \ |
---|
215 | re.match('^[0-9]+~[0-9]+$', s): |
---|
216 | #print '"%s" is ID selection using < or <='%(s) |
---|
217 | a = FieldIdRegexGenerator(s) |
---|
218 | yield '(%s)'%(a.get_regex()) |
---|
219 | else: |
---|
220 | #print '"%s" is UNIX style pattern match'%(s) |
---|
221 | yield '(SRCNAME == pattern(\'%s\'))'%(s) |
---|
222 | |
---|
223 | def get_scans(self): |
---|
224 | return list(self._getscans()) |
---|
225 | def get_cycles(self): |
---|
226 | return list(self._getcycles()) |
---|
227 | def get_beams(self): |
---|
228 | return list(self._getbeams()) |
---|
229 | def get_ifs(self): |
---|
230 | return list(self._getifs()) |
---|
231 | def get_pols(self): |
---|
232 | return list(self._getpols()) |
---|
233 | def get_poltypes(self): |
---|
234 | return list(self._getpoltypes()) |
---|
235 | def get_order(self): |
---|
236 | return list(self._getorder()) |
---|
237 | def get_types(self): |
---|
238 | return list(self._gettypes()) |
---|
239 | def get_rows(self): |
---|
240 | return list(self._getrows()) |
---|
241 | def get_query(self): |
---|
242 | prefix = "SELECT FROM $1 WHERE " |
---|
243 | return self._gettaql().replace(prefix, "") |
---|
244 | |
---|
245 | def get_name(self): |
---|
246 | print "NYI" |
---|
247 | s = self._gettaql() |
---|
248 | return |
---|
249 | def __str__(self): |
---|
250 | out = "" |
---|
251 | d = {"SCANNO": self.get_scans(), |
---|
252 | "CYCLENO": self.get_cycles(), |
---|
253 | "BEAMNO": self.get_beams(), |
---|
254 | "IFNO": self.get_ifs(), |
---|
255 | "Pol Type": self.get_poltypes(), |
---|
256 | "POLNO": self.get_pols(), |
---|
257 | "QUERY": self.get_query(), |
---|
258 | "SRCTYPE": self.get_types(), |
---|
259 | "ROWS": self.get_rows(), |
---|
260 | "Sort Order": self.get_order() |
---|
261 | } |
---|
262 | for k,v in d.iteritems(): |
---|
263 | if v: |
---|
264 | out += "%s: %s\n" % (k, v) |
---|
265 | if len(out): |
---|
266 | return out[:-1] |
---|
267 | else: |
---|
268 | return out |
---|
269 | |
---|
270 | def __add__(self, other): |
---|
271 | """ |
---|
272 | Merge two selections. |
---|
273 | """ |
---|
274 | if self.is_empty(): |
---|
275 | return selector(other) |
---|
276 | elif other.is_empty(): |
---|
277 | return selector(self) |
---|
278 | union = selector() |
---|
279 | gets = [[self._getscans(), other._getscans(), union._setscans], |
---|
280 | [self._getcycles(), other._getcycles(),union._setcycles], |
---|
281 | [self._getbeams(), other._getbeams(), union._setbeams], |
---|
282 | [self._getifs(), other._getifs(), union._setifs], |
---|
283 | [self._getpols(), other._getpols(), union._setpols]] |
---|
284 | for v in gets: |
---|
285 | vec = list(v[0]+v[1]) |
---|
286 | vec.sort() |
---|
287 | v[2](unique(vec)) |
---|
288 | q = other.get_query() |
---|
289 | qs = self.get_query() |
---|
290 | if len(q) and len(qs): |
---|
291 | union.set_query(qs +" AND " + q) |
---|
292 | else: |
---|
293 | if len(q): |
---|
294 | union.set_query(q) |
---|
295 | elif len(qs): |
---|
296 | union.set_query(qs) |
---|
297 | return union |
---|
298 | |
---|
299 | class FieldIdRegexGenerator(object): |
---|
300 | def __init__(self, pattern): |
---|
301 | if pattern.isdigit(): |
---|
302 | self.regex = 'FIELDNAME == regex(\'.+__%s$\')'%(pattern) |
---|
303 | else: |
---|
304 | self.regex = None |
---|
305 | ineq = None |
---|
306 | if pattern.find('<') >= 0: |
---|
307 | ineq = '<' |
---|
308 | s = pattern.strip().lstrip(ineq).lstrip('=') |
---|
309 | if not s.isdigit(): |
---|
310 | raise RuntimeError('Invalid syntax: %s'%(pattern)) |
---|
311 | self.id = int(s) + (-1 if pattern.find('=') < 0 else 0) |
---|
312 | self.template = string.Template('FIELDNAME == regex(\'.+__${reg}$\')') |
---|
313 | elif pattern.find('>') >= 0: |
---|
314 | ineq = '>' |
---|
315 | s = pattern.strip().lstrip(ineq).lstrip('=') |
---|
316 | if not s.isdigit(): |
---|
317 | raise RuntimeError('Invalid syntax: %s'%(pattern)) |
---|
318 | self.id = int(s) + (-1 if pattern.find('=') >= 0 else 0) |
---|
319 | self.template = string.Template('FIELDNAME == regex(\'.+__[0-9]+$\') && FIELDNAME != regex(\'.+__${reg}$\')') |
---|
320 | elif pattern.find('~') >= 0: |
---|
321 | s = map(string.strip, pattern.split('~')) |
---|
322 | if len(s) == 2 and s[0].isdigit() and s[1].isdigit(): |
---|
323 | self.id = [int(s[0])-1,int(s[1])] |
---|
324 | else: |
---|
325 | raise RuntimeError('Invalid syntax: %s'%(pattern)) |
---|
326 | self.template = string.Template('FIELDNAME == regex(\'.+__${reg}$\') && FIELDNAME != regex(\'.+__${optreg}$\')') |
---|
327 | else: |
---|
328 | raise RuntimeError('Invalid syntax: %s'%(pattern)) |
---|
329 | #print 'self.id=',self.id |
---|
330 | |
---|
331 | def get_regex(self): |
---|
332 | if self.regex is not None: |
---|
333 | # 'X' |
---|
334 | return self.regex |
---|
335 | elif isinstance(self.id, list): |
---|
336 | # 'X~Y' |
---|
337 | return self.template.safe_substitute(reg=self.__compile(self.id[1]), |
---|
338 | optreg=self.__compile(self.id[0])) |
---|
339 | else: |
---|
340 | # '<(=)X' or '>(=)X' |
---|
341 | return self.template.safe_substitute(reg=self.__compile(self.id)) |
---|
342 | |
---|
343 | def __compile(self, idx): |
---|
344 | pattern = '' |
---|
345 | if idx >= 0: |
---|
346 | if idx < 10: |
---|
347 | num_digits = 1 |
---|
348 | else: |
---|
349 | num_digits = int(math.log10(idx)) + 1 |
---|
350 | numerics = [] |
---|
351 | modulo = 10 |
---|
352 | divider = 1 |
---|
353 | for i in xrange(num_digits): |
---|
354 | numerics.append(int((idx % modulo) / divider)) |
---|
355 | modulo *= 10 |
---|
356 | divider *= 10 |
---|
357 | #print 'numerics=',numerics |
---|
358 | if num_digits == 1: |
---|
359 | if numerics[0] == 0: |
---|
360 | pattern = '0' |
---|
361 | else: |
---|
362 | pattern = '[0-%s]'%(numerics[0]) |
---|
363 | elif num_digits == 2: |
---|
364 | pattern_list = ['[0-9]'] |
---|
365 | pattern_list.append('[1-%s][0-9]'%(numerics[1]-1)) |
---|
366 | if numerics[0] == 0: |
---|
367 | pattern_list.append('%s%s'%(numerics[1],numerics[0])) |
---|
368 | else: |
---|
369 | pattern_list.append('%s[0-%s]'%(numerics[1],numerics[0])) |
---|
370 | pattern = '(%s)'%('|'.join(pattern_list)) |
---|
371 | elif num_digits == 3: |
---|
372 | pattern_list = ['[0-9]','[1-9][0-9]'] |
---|
373 | if numerics[2] == 2: |
---|
374 | pattern_list.append('1[0-9][0-9]') |
---|
375 | elif numerics[2] > 2: |
---|
376 | pattern_list.append('[1-%s][0-9][0-9]'%(numerics[2]-1)) |
---|
377 | if numerics[1] == 0: |
---|
378 | if numerics[0] == 0: |
---|
379 | pattern_list.append('%s00'%(numerics[2])) |
---|
380 | else: |
---|
381 | pattern_list.append('%s0[0-%s]'%(numerics[2],numerics[0])) |
---|
382 | else: |
---|
383 | pattern_list.append('%s[0-%s][0-9]'%(numerics[2],numerics[1]-1)) |
---|
384 | if numerics[0] == 0: |
---|
385 | pattern_list.append('%s%s%s'%(numerics[2],numerics[1],numerics[0])) |
---|
386 | else: |
---|
387 | pattern_list.append('%s%s[0-%s]'%(numerics[2],numerics[1],numerics[0])) |
---|
388 | pattern = '(%s)'%('|'.join(pattern_list)) |
---|
389 | else: |
---|
390 | raise RuntimeError('ID > 999 is not supported') |
---|
391 | pattern = '' |
---|
392 | else: |
---|
393 | raise RuntimeError('ID must be >= 0') |
---|
394 | return pattern |
---|