"Fossies" - the Fresh Open Source Software Archive

Member "numpy-1.16.4/numpy/ma/mrecords.py" (27 May 2019, 26937 Bytes) of package /linux/misc/numpy-1.16.4.tar.gz:


As a special service "Fossies" has tried to format the requested source page into HTML format using (guessed) Python source code syntax highlighting (style: standard) with prefixed line numbers. Alternatively you can here view or download the uninterpreted source code file. For more information about "mrecords.py" see the Fossies "Dox" file reference documentation and the last Fossies "Diffs" side-by-side code changes report: 1.15.4_vs_1.16.0.

    1 """:mod:`numpy.ma..mrecords`
    2 
    3 Defines the equivalent of :class:`numpy.recarrays` for masked arrays,
    4 where fields can be accessed as attributes.
    5 Note that :class:`numpy.ma.MaskedArray` already supports structured datatypes
    6 and the masking of individual fields.
    7 
    8 .. moduleauthor:: Pierre Gerard-Marchant
    9 
   10 """
   11 from __future__ import division, absolute_import, print_function
   12 
   13 #  We should make sure that no field is called '_mask','mask','_fieldmask',
   14 #  or whatever restricted keywords.  An idea would be to no bother in the
   15 #  first place, and then rename the invalid fields with a trailing
   16 #  underscore. Maybe we could just overload the parser function ?
   17 
   18 import sys
   19 import warnings
   20 
   21 import numpy as np
   22 import numpy.core.numerictypes as ntypes
   23 from numpy.compat import basestring
   24 from numpy import (
   25         bool_, dtype, ndarray, recarray, array as narray
   26         )
   27 from numpy.core.records import (
   28         fromarrays as recfromarrays, fromrecords as recfromrecords
   29         )
   30 
   31 _byteorderconv = np.core.records._byteorderconv
   32 
   33 import numpy.ma as ma
   34 from numpy.ma import (
   35         MAError, MaskedArray, masked, nomask, masked_array, getdata,
   36         getmaskarray, filled
   37         )
   38 
   39 _check_fill_value = ma.core._check_fill_value
   40 
   41 
   42 __all__ = [
   43     'MaskedRecords', 'mrecarray', 'fromarrays', 'fromrecords',
   44     'fromtextfile', 'addfield',
   45     ]
   46 
   47 reserved_fields = ['_data', '_mask', '_fieldmask', 'dtype']
   48 
   49 
   50 def _checknames(descr, names=None):
   51     """
   52     Checks that field names ``descr`` are not reserved keywords.
   53 
   54     If this is the case, a default 'f%i' is substituted.  If the argument
   55     `names` is not None, updates the field names to valid names.
   56 
   57     """
   58     ndescr = len(descr)
   59     default_names = ['f%i' % i for i in range(ndescr)]
   60     if names is None:
   61         new_names = default_names
   62     else:
   63         if isinstance(names, (tuple, list)):
   64             new_names = names
   65         elif isinstance(names, str):
   66             new_names = names.split(',')
   67         else:
   68             raise NameError("illegal input names %s" % repr(names))
   69         nnames = len(new_names)
   70         if nnames < ndescr:
   71             new_names += default_names[nnames:]
   72     ndescr = []
   73     for (n, d, t) in zip(new_names, default_names, descr.descr):
   74         if n in reserved_fields:
   75             if t[0] in reserved_fields:
   76                 ndescr.append((d, t[1]))
   77             else:
   78                 ndescr.append(t)
   79         else:
   80             ndescr.append((n, t[1]))
   81     return np.dtype(ndescr)
   82 
   83 
   84 def _get_fieldmask(self):
   85     mdescr = [(n, '|b1') for n in self.dtype.names]
   86     fdmask = np.empty(self.shape, dtype=mdescr)
   87     fdmask.flat = tuple([False] * len(mdescr))
   88     return fdmask
   89 
   90 
   91 class MaskedRecords(MaskedArray, object):
   92     """
   93 
   94     Attributes
   95     ----------
   96     _data : recarray
   97         Underlying data, as a record array.
   98     _mask : boolean array
   99         Mask of the records. A record is masked when all its fields are
  100         masked.
  101     _fieldmask : boolean recarray
  102         Record array of booleans, setting the mask of each individual field
  103         of each record.
  104     _fill_value : record
  105         Filling values for each field.
  106 
  107     """
  108 
  109     def __new__(cls, shape, dtype=None, buf=None, offset=0, strides=None,
  110                 formats=None, names=None, titles=None,
  111                 byteorder=None, aligned=False,
  112                 mask=nomask, hard_mask=False, fill_value=None, keep_mask=True,
  113                 copy=False,
  114                 **options):
  115 
  116         self = recarray.__new__(cls, shape, dtype=dtype, buf=buf, offset=offset,
  117                                 strides=strides, formats=formats, names=names,
  118                                 titles=titles, byteorder=byteorder,
  119                                 aligned=aligned,)
  120 
  121         mdtype = ma.make_mask_descr(self.dtype)
  122         if mask is nomask or not np.size(mask):
  123             if not keep_mask:
  124                 self._mask = tuple([False] * len(mdtype))
  125         else:
  126             mask = np.array(mask, copy=copy)
  127             if mask.shape != self.shape:
  128                 (nd, nm) = (self.size, mask.size)
  129                 if nm == 1:
  130                     mask = np.resize(mask, self.shape)
  131                 elif nm == nd:
  132                     mask = np.reshape(mask, self.shape)
  133                 else:
  134                     msg = "Mask and data not compatible: data size is %i, " + \
  135                           "mask size is %i."
  136                     raise MAError(msg % (nd, nm))
  137                 copy = True
  138             if not keep_mask:
  139                 self.__setmask__(mask)
  140                 self._sharedmask = True
  141             else:
  142                 if mask.dtype == mdtype:
  143                     _mask = mask
  144                 else:
  145                     _mask = np.array([tuple([m] * len(mdtype)) for m in mask],
  146                                      dtype=mdtype)
  147                 self._mask = _mask
  148         return self
  149 
  150     def __array_finalize__(self, obj):
  151         # Make sure we have a _fieldmask by default
  152         _mask = getattr(obj, '_mask', None)
  153         if _mask is None:
  154             objmask = getattr(obj, '_mask', nomask)
  155             _dtype = ndarray.__getattribute__(self, 'dtype')
  156             if objmask is nomask:
  157                 _mask = ma.make_mask_none(self.shape, dtype=_dtype)
  158             else:
  159                 mdescr = ma.make_mask_descr(_dtype)
  160                 _mask = narray([tuple([m] * len(mdescr)) for m in objmask],
  161                                dtype=mdescr).view(recarray)
  162         # Update some of the attributes
  163         _dict = self.__dict__
  164         _dict.update(_mask=_mask)
  165         self._update_from(obj)
  166         if _dict['_baseclass'] == ndarray:
  167             _dict['_baseclass'] = recarray
  168         return
  169 
  170     def _getdata(self):
  171         """
  172         Returns the data as a recarray.
  173 
  174         """
  175         return ndarray.view(self, recarray)
  176 
  177     _data = property(fget=_getdata)
  178 
  179     def _getfieldmask(self):
  180         """
  181         Alias to mask.
  182 
  183         """
  184         return self._mask
  185 
  186     _fieldmask = property(fget=_getfieldmask)
  187 
  188     def __len__(self):
  189         """
  190         Returns the length
  191 
  192         """
  193         # We have more than one record
  194         if self.ndim:
  195             return len(self._data)
  196         # We have only one record: return the nb of fields
  197         return len(self.dtype)
  198 
  199     def __getattribute__(self, attr):
  200         try:
  201             return object.__getattribute__(self, attr)
  202         except AttributeError:
  203             # attr must be a fieldname
  204             pass
  205         fielddict = ndarray.__getattribute__(self, 'dtype').fields
  206         try:
  207             res = fielddict[attr][:2]
  208         except (TypeError, KeyError):
  209             raise AttributeError("record array has no attribute %s" % attr)
  210         # So far, so good
  211         _localdict = ndarray.__getattribute__(self, '__dict__')
  212         _data = ndarray.view(self, _localdict['_baseclass'])
  213         obj = _data.getfield(*res)
  214         if obj.dtype.fields:
  215             raise NotImplementedError("MaskedRecords is currently limited to"
  216                                       "simple records.")
  217         # Get some special attributes
  218         # Reset the object's mask
  219         hasmasked = False
  220         _mask = _localdict.get('_mask', None)
  221         if _mask is not None:
  222             try:
  223                 _mask = _mask[attr]
  224             except IndexError:
  225                 # Couldn't find a mask: use the default (nomask)
  226                 pass
  227             hasmasked = _mask.view((bool, (len(_mask.dtype) or 1))).any()
  228         if (obj.shape or hasmasked):
  229             obj = obj.view(MaskedArray)
  230             obj._baseclass = ndarray
  231             obj._isfield = True
  232             obj._mask = _mask
  233             # Reset the field values
  234             _fill_value = _localdict.get('_fill_value', None)
  235             if _fill_value is not None:
  236                 try:
  237                     obj._fill_value = _fill_value[attr]
  238                 except ValueError:
  239                     obj._fill_value = None
  240         else:
  241             obj = obj.item()
  242         return obj
  243 
  244     def __setattr__(self, attr, val):
  245         """
  246         Sets the attribute attr to the value val.
  247 
  248         """
  249         # Should we call __setmask__ first ?
  250         if attr in ['mask', 'fieldmask']:
  251             self.__setmask__(val)
  252             return
  253         # Create a shortcut (so that we don't have to call getattr all the time)
  254         _localdict = object.__getattribute__(self, '__dict__')
  255         # Check whether we're creating a new field
  256         newattr = attr not in _localdict
  257         try:
  258             # Is attr a generic attribute ?
  259             ret = object.__setattr__(self, attr, val)
  260         except Exception:
  261             # Not a generic attribute: exit if it's not a valid field
  262             fielddict = ndarray.__getattribute__(self, 'dtype').fields or {}
  263             optinfo = ndarray.__getattribute__(self, '_optinfo') or {}
  264             if not (attr in fielddict or attr in optinfo):
  265                 exctype, value = sys.exc_info()[:2]
  266                 raise exctype(value)
  267         else:
  268             # Get the list of names
  269             fielddict = ndarray.__getattribute__(self, 'dtype').fields or {}
  270             # Check the attribute
  271             if attr not in fielddict:
  272                 return ret
  273             if newattr:
  274                 # We just added this one or this setattr worked on an
  275                 # internal attribute.
  276                 try:
  277                     object.__delattr__(self, attr)
  278                 except Exception:
  279                     return ret
  280         # Let's try to set the field
  281         try:
  282             res = fielddict[attr][:2]
  283         except (TypeError, KeyError):
  284             raise AttributeError("record array has no attribute %s" % attr)
  285 
  286         if val is masked:
  287             _fill_value = _localdict['_fill_value']
  288             if _fill_value is not None:
  289                 dval = _localdict['_fill_value'][attr]
  290             else:
  291                 dval = val
  292             mval = True
  293         else:
  294             dval = filled(val)
  295             mval = getmaskarray(val)
  296         obj = ndarray.__getattribute__(self, '_data').setfield(dval, *res)
  297         _localdict['_mask'].__setitem__(attr, mval)
  298         return obj
  299 
  300     def __getitem__(self, indx):
  301         """
  302         Returns all the fields sharing the same fieldname base.
  303 
  304         The fieldname base is either `_data` or `_mask`.
  305 
  306         """
  307         _localdict = self.__dict__
  308         _mask = ndarray.__getattribute__(self, '_mask')
  309         _data = ndarray.view(self, _localdict['_baseclass'])
  310         # We want a field
  311         if isinstance(indx, basestring):
  312             # Make sure _sharedmask is True to propagate back to _fieldmask
  313             # Don't use _set_mask, there are some copies being made that
  314             # break propagation Don't force the mask to nomask, that wreaks
  315             # easy masking
  316             obj = _data[indx].view(MaskedArray)
  317             obj._mask = _mask[indx]
  318             obj._sharedmask = True
  319             fval = _localdict['_fill_value']
  320             if fval is not None:
  321                 obj._fill_value = fval[indx]
  322             # Force to masked if the mask is True
  323             if not obj.ndim and obj._mask:
  324                 return masked
  325             return obj
  326         # We want some elements.
  327         # First, the data.
  328         obj = np.array(_data[indx], copy=False).view(mrecarray)
  329         obj._mask = np.array(_mask[indx], copy=False).view(recarray)
  330         return obj
  331 
  332     def __setitem__(self, indx, value):
  333         """
  334         Sets the given record to value.
  335 
  336         """
  337         MaskedArray.__setitem__(self, indx, value)
  338         if isinstance(indx, basestring):
  339             self._mask[indx] = ma.getmaskarray(value)
  340 
  341     def __str__(self):
  342         """
  343         Calculates the string representation.
  344 
  345         """
  346         if self.size > 1:
  347             mstr = ["(%s)" % ",".join([str(i) for i in s])
  348                     for s in zip(*[getattr(self, f) for f in self.dtype.names])]
  349             return "[%s]" % ", ".join(mstr)
  350         else:
  351             mstr = ["%s" % ",".join([str(i) for i in s])
  352                     for s in zip([getattr(self, f) for f in self.dtype.names])]
  353             return "(%s)" % ", ".join(mstr)
  354 
  355     def __repr__(self):
  356         """
  357         Calculates the repr representation.
  358 
  359         """
  360         _names = self.dtype.names
  361         fmt = "%%%is : %%s" % (max([len(n) for n in _names]) + 4,)
  362         reprstr = [fmt % (f, getattr(self, f)) for f in self.dtype.names]
  363         reprstr.insert(0, 'masked_records(')
  364         reprstr.extend([fmt % ('    fill_value', self.fill_value),
  365                          '              )'])
  366         return str("\n".join(reprstr))
  367 
  368     def view(self, dtype=None, type=None):
  369         """
  370         Returns a view of the mrecarray.
  371 
  372         """
  373         # OK, basic copy-paste from MaskedArray.view.
  374         if dtype is None:
  375             if type is None:
  376                 output = ndarray.view(self)
  377             else:
  378                 output = ndarray.view(self, type)
  379         # Here again.
  380         elif type is None:
  381             try:
  382                 if issubclass(dtype, ndarray):
  383                     output = ndarray.view(self, dtype)
  384                     dtype = None
  385                 else:
  386                     output = ndarray.view(self, dtype)
  387             # OK, there's the change
  388             except TypeError:
  389                 dtype = np.dtype(dtype)
  390                 # we need to revert to MaskedArray, but keeping the possibility
  391                 # of subclasses (eg, TimeSeriesRecords), so we'll force a type
  392                 # set to the first parent
  393                 if dtype.fields is None:
  394                     basetype = self.__class__.__bases__[0]
  395                     output = self.__array__().view(dtype, basetype)
  396                     output._update_from(self)
  397                 else:
  398                     output = ndarray.view(self, dtype)
  399                 output._fill_value = None
  400         else:
  401             output = ndarray.view(self, dtype, type)
  402         # Update the mask, just like in MaskedArray.view
  403         if (getattr(output, '_mask', nomask) is not nomask):
  404             mdtype = ma.make_mask_descr(output.dtype)
  405             output._mask = self._mask.view(mdtype, ndarray)
  406             output._mask.shape = output.shape
  407         return output
  408 
  409     def harden_mask(self):
  410         """
  411         Forces the mask to hard.
  412 
  413         """
  414         self._hardmask = True
  415 
  416     def soften_mask(self):
  417         """
  418         Forces the mask to soft
  419 
  420         """
  421         self._hardmask = False
  422 
  423     def copy(self):
  424         """
  425         Returns a copy of the masked record.
  426 
  427         """
  428         copied = self._data.copy().view(type(self))
  429         copied._mask = self._mask.copy()
  430         return copied
  431 
  432     def tolist(self, fill_value=None):
  433         """
  434         Return the data portion of the array as a list.
  435 
  436         Data items are converted to the nearest compatible Python type.
  437         Masked values are converted to fill_value. If fill_value is None,
  438         the corresponding entries in the output list will be ``None``.
  439 
  440         """
  441         if fill_value is not None:
  442             return self.filled(fill_value).tolist()
  443         result = narray(self.filled().tolist(), dtype=object)
  444         mask = narray(self._mask.tolist())
  445         result[mask] = None
  446         return result.tolist()
  447 
  448     def __getstate__(self):
  449         """Return the internal state of the masked array.
  450 
  451         This is for pickling.
  452 
  453         """
  454         state = (1,
  455                  self.shape,
  456                  self.dtype,
  457                  self.flags.fnc,
  458                  self._data.tobytes(),
  459                  self._mask.tobytes(),
  460                  self._fill_value,
  461                  )
  462         return state
  463 
  464     def __setstate__(self, state):
  465         """
  466         Restore the internal state of the masked array.
  467 
  468         This is for pickling.  ``state`` is typically the output of the
  469         ``__getstate__`` output, and is a 5-tuple:
  470 
  471         - class name
  472         - a tuple giving the shape of the data
  473         - a typecode for the data
  474         - a binary string for the data
  475         - a binary string for the mask.
  476 
  477         """
  478         (ver, shp, typ, isf, raw, msk, flv) = state
  479         ndarray.__setstate__(self, (shp, typ, isf, raw))
  480         mdtype = dtype([(k, bool_) for (k, _) in self.dtype.descr])
  481         self.__dict__['_mask'].__setstate__((shp, mdtype, isf, msk))
  482         self.fill_value = flv
  483 
  484     def __reduce__(self):
  485         """
  486         Return a 3-tuple for pickling a MaskedArray.
  487 
  488         """
  489         return (_mrreconstruct,
  490                 (self.__class__, self._baseclass, (0,), 'b',),
  491                 self.__getstate__())
  492 
  493 def _mrreconstruct(subtype, baseclass, baseshape, basetype,):
  494     """
  495     Build a new MaskedArray from the information stored in a pickle.
  496 
  497     """
  498     _data = ndarray.__new__(baseclass, baseshape, basetype).view(subtype)
  499     _mask = ndarray.__new__(ndarray, baseshape, 'b1')
  500     return subtype.__new__(subtype, _data, mask=_mask, dtype=basetype,)
  501 
  502 mrecarray = MaskedRecords
  503 
  504 
  505 ###############################################################################
  506 #                             Constructors                                    #
  507 ###############################################################################
  508 
  509 
  510 def fromarrays(arraylist, dtype=None, shape=None, formats=None,
  511                names=None, titles=None, aligned=False, byteorder=None,
  512                fill_value=None):
  513     """
  514     Creates a mrecarray from a (flat) list of masked arrays.
  515 
  516     Parameters
  517     ----------
  518     arraylist : sequence
  519         A list of (masked) arrays. Each element of the sequence is first converted
  520         to a masked array if needed. If a 2D array is passed as argument, it is
  521         processed line by line
  522     dtype : {None, dtype}, optional
  523         Data type descriptor.
  524     shape : {None, integer}, optional
  525         Number of records. If None, shape is defined from the shape of the
  526         first array in the list.
  527     formats : {None, sequence}, optional
  528         Sequence of formats for each individual field. If None, the formats will
  529         be autodetected by inspecting the fields and selecting the highest dtype
  530         possible.
  531     names : {None, sequence}, optional
  532         Sequence of the names of each field.
  533     fill_value : {None, sequence}, optional
  534         Sequence of data to be used as filling values.
  535 
  536     Notes
  537     -----
  538     Lists of tuples should be preferred over lists of lists for faster processing.
  539 
  540     """
  541     datalist = [getdata(x) for x in arraylist]
  542     masklist = [np.atleast_1d(getmaskarray(x)) for x in arraylist]
  543     _array = recfromarrays(datalist,
  544                            dtype=dtype, shape=shape, formats=formats,
  545                            names=names, titles=titles, aligned=aligned,
  546                            byteorder=byteorder).view(mrecarray)
  547     _array._mask.flat = list(zip(*masklist))
  548     if fill_value is not None:
  549         _array.fill_value = fill_value
  550     return _array
  551 
  552 
  553 def fromrecords(reclist, dtype=None, shape=None, formats=None, names=None,
  554                 titles=None, aligned=False, byteorder=None,
  555                 fill_value=None, mask=nomask):
  556     """
  557     Creates a MaskedRecords from a list of records.
  558 
  559     Parameters
  560     ----------
  561     reclist : sequence
  562         A list of records. Each element of the sequence is first converted
  563         to a masked array if needed. If a 2D array is passed as argument, it is
  564         processed line by line
  565     dtype : {None, dtype}, optional
  566         Data type descriptor.
  567     shape : {None,int}, optional
  568         Number of records. If None, ``shape`` is defined from the shape of the
  569         first array in the list.
  570     formats : {None, sequence}, optional
  571         Sequence of formats for each individual field. If None, the formats will
  572         be autodetected by inspecting the fields and selecting the highest dtype
  573         possible.
  574     names : {None, sequence}, optional
  575         Sequence of the names of each field.
  576     fill_value : {None, sequence}, optional
  577         Sequence of data to be used as filling values.
  578     mask : {nomask, sequence}, optional.
  579         External mask to apply on the data.
  580 
  581     Notes
  582     -----
  583     Lists of tuples should be preferred over lists of lists for faster processing.
  584 
  585     """
  586     # Grab the initial _fieldmask, if needed:
  587     _mask = getattr(reclist, '_mask', None)
  588     # Get the list of records.
  589     if isinstance(reclist, ndarray):
  590         # Make sure we don't have some hidden mask
  591         if isinstance(reclist, MaskedArray):
  592             reclist = reclist.filled().view(ndarray)
  593         # Grab the initial dtype, just in case
  594         if dtype is None:
  595             dtype = reclist.dtype
  596         reclist = reclist.tolist()
  597     mrec = recfromrecords(reclist, dtype=dtype, shape=shape, formats=formats,
  598                           names=names, titles=titles,
  599                           aligned=aligned, byteorder=byteorder).view(mrecarray)
  600     # Set the fill_value if needed
  601     if fill_value is not None:
  602         mrec.fill_value = fill_value
  603     # Now, let's deal w/ the mask
  604     if mask is not nomask:
  605         mask = np.array(mask, copy=False)
  606         maskrecordlength = len(mask.dtype)
  607         if maskrecordlength:
  608             mrec._mask.flat = mask
  609         elif mask.ndim == 2:
  610             mrec._mask.flat = [tuple(m) for m in mask]
  611         else:
  612             mrec.__setmask__(mask)
  613     if _mask is not None:
  614         mrec._mask[:] = _mask
  615     return mrec
  616 
  617 
  618 def _guessvartypes(arr):
  619     """
  620     Tries to guess the dtypes of the str_ ndarray `arr`.
  621 
  622     Guesses by testing element-wise conversion. Returns a list of dtypes.
  623     The array is first converted to ndarray. If the array is 2D, the test
  624     is performed on the first line. An exception is raised if the file is
  625     3D or more.
  626 
  627     """
  628     vartypes = []
  629     arr = np.asarray(arr)
  630     if arr.ndim == 2:
  631         arr = arr[0]
  632     elif arr.ndim > 2:
  633         raise ValueError("The array should be 2D at most!")
  634     # Start the conversion loop.
  635     for f in arr:
  636         try:
  637             int(f)
  638         except (ValueError, TypeError):
  639             try:
  640                 float(f)
  641             except (ValueError, TypeError):
  642                 try:
  643                     complex(f)
  644                 except (ValueError, TypeError):
  645                     vartypes.append(arr.dtype)
  646                 else:
  647                     vartypes.append(np.dtype(complex))
  648             else:
  649                 vartypes.append(np.dtype(float))
  650         else:
  651             vartypes.append(np.dtype(int))
  652     return vartypes
  653 
  654 
  655 def openfile(fname):
  656     """
  657     Opens the file handle of file `fname`.
  658 
  659     """
  660     # A file handle
  661     if hasattr(fname, 'readline'):
  662         return fname
  663     # Try to open the file and guess its type
  664     try:
  665         f = open(fname)
  666     except IOError:
  667         raise IOError("No such file: '%s'" % fname)
  668     if f.readline()[:2] != "\\x":
  669         f.seek(0, 0)
  670         return f
  671     f.close()
  672     raise NotImplementedError("Wow, binary file")
  673 
  674 
  675 def fromtextfile(fname, delimitor=None, commentchar='#', missingchar='',
  676                  varnames=None, vartypes=None):
  677     """
  678     Creates a mrecarray from data stored in the file `filename`.
  679 
  680     Parameters
  681     ----------
  682     fname : {file name/handle}
  683         Handle of an opened file.
  684     delimitor : {None, string}, optional
  685         Alphanumeric character used to separate columns in the file.
  686         If None, any (group of) white spacestring(s) will be used.
  687     commentchar : {'#', string}, optional
  688         Alphanumeric character used to mark the start of a comment.
  689     missingchar : {'', string}, optional
  690         String indicating missing data, and used to create the masks.
  691     varnames : {None, sequence}, optional
  692         Sequence of the variable names. If None, a list will be created from
  693         the first non empty line of the file.
  694     vartypes : {None, sequence}, optional
  695         Sequence of the variables dtypes. If None, it will be estimated from
  696         the first non-commented line.
  697 
  698 
  699     Ultra simple: the varnames are in the header, one line"""
  700     # Try to open the file.
  701     ftext = openfile(fname)
  702 
  703     # Get the first non-empty line as the varnames
  704     while True:
  705         line = ftext.readline()
  706         firstline = line[:line.find(commentchar)].strip()
  707         _varnames = firstline.split(delimitor)
  708         if len(_varnames) > 1:
  709             break
  710     if varnames is None:
  711         varnames = _varnames
  712 
  713     # Get the data.
  714     _variables = masked_array([line.strip().split(delimitor) for line in ftext
  715                                if line[0] != commentchar and len(line) > 1])
  716     (_, nfields) = _variables.shape
  717     ftext.close()
  718 
  719     # Try to guess the dtype.
  720     if vartypes is None:
  721         vartypes = _guessvartypes(_variables[0])
  722     else:
  723         vartypes = [np.dtype(v) for v in vartypes]
  724         if len(vartypes) != nfields:
  725             msg = "Attempting to %i dtypes for %i fields!"
  726             msg += " Reverting to default."
  727             warnings.warn(msg % (len(vartypes), nfields), stacklevel=2)
  728             vartypes = _guessvartypes(_variables[0])
  729 
  730     # Construct the descriptor.
  731     mdescr = [(n, f) for (n, f) in zip(varnames, vartypes)]
  732     mfillv = [ma.default_fill_value(f) for f in vartypes]
  733 
  734     # Get the data and the mask.
  735     # We just need a list of masked_arrays. It's easier to create it like that:
  736     _mask = (_variables.T == missingchar)
  737     _datalist = [masked_array(a, mask=m, dtype=t, fill_value=f)
  738                  for (a, m, t, f) in zip(_variables.T, _mask, vartypes, mfillv)]
  739 
  740     return fromarrays(_datalist, dtype=mdescr)
  741 
  742 
  743 def addfield(mrecord, newfield, newfieldname=None):
  744     """Adds a new field to the masked record array
  745 
  746     Uses `newfield` as data and `newfieldname` as name. If `newfieldname`
  747     is None, the new field name is set to 'fi', where `i` is the number of
  748     existing fields.
  749 
  750     """
  751     _data = mrecord._data
  752     _mask = mrecord._mask
  753     if newfieldname is None or newfieldname in reserved_fields:
  754         newfieldname = 'f%i' % len(_data.dtype)
  755     newfield = ma.array(newfield)
  756     # Get the new data.
  757     # Create a new empty recarray
  758     newdtype = np.dtype(_data.dtype.descr + [(newfieldname, newfield.dtype)])
  759     newdata = recarray(_data.shape, newdtype)
  760     # Add the existing field
  761     [newdata.setfield(_data.getfield(*f), *f)
  762          for f in _data.dtype.fields.values()]
  763     # Add the new field
  764     newdata.setfield(newfield._data, *newdata.dtype.fields[newfieldname])
  765     newdata = newdata.view(MaskedRecords)
  766     # Get the new mask
  767     # Create a new empty recarray
  768     newmdtype = np.dtype([(n, bool_) for n in newdtype.names])
  769     newmask = recarray(_data.shape, newmdtype)
  770     # Add the old masks
  771     [newmask.setfield(_mask.getfield(*f), *f)
  772          for f in _mask.dtype.fields.values()]
  773     # Add the mask of the new field
  774     newmask.setfield(getmaskarray(newfield),
  775                      *newmask.dtype.fields[newfieldname])
  776     newdata._mask = newmask
  777     return newdata