import abc
import sys
from copy import deepcopy
import numpy as np
from numpy import ma
from ..units import Unit
from .. import log
from ..utils import OrderedDict, isiterable
from .structhelper import _drop_fields
from .pprint import _pformat_table, _pformat_col, _pformat_col_iter, _more_tabcol
from ..utils.console import color_print
from ..config import ConfigurationItem
from .io_registry import get_reader, get_writer, identify_format
# Python 2 and 3 source compatibility
try:
unicode
except NameError:
unicode = basestring = str
NUMPY_LT_1P5 = [int(x) for x in np.__version__.split('.')[:2]] < [1, 5]
AUTO_COLNAME = ConfigurationItem('auto_colname', 'col{0}',
'The template that determines the name of a column if it cannot be '
'determined. Uses new-style (format method) string formatting')
def _auto_names(n_cols):
return [AUTO_COLNAME().format(i) for i in range(n_cols)]
[docs]class TableColumns(OrderedDict):
"""OrderedDict subclass for a set of columns.
This class enhances item access to provide convenient access to columns
by name or index, including slice access. It also handles renaming
of columns.
The initialization argument ``cols`` can be any structure that is valid
for initializing a Python dict. This includes a dict, list of
(key, val) tuple pairs, list of [key, val] lists, etc.
Parameters
----------
cols : dict, list, tuple; optional
Column objects as data structure that can init dict (see above)
"""
def __init__(self, cols={}):
if isinstance(cols, (list, tuple)):
cols = [(col.name, col) for col in cols]
super(TableColumns, self).__init__(cols)
def __getitem__(self, item):
"""Get items from a TableColumns object.
::
tc = TableColumns(cols=[Column('a'), Column('b'), Column('c')])
tc['a'] # Column('a')
tc[1] # Column('b')
tc['a', 'b'] # <TableColumns names=('a', 'b')>
tc[1:3] # <TableColumns names=('b', 'c')>
"""
if isinstance(item, basestring):
return OrderedDict.__getitem__(self, item)
elif isinstance(item, int):
return self.values()[item]
elif isinstance(item, tuple):
return TableColumns([self[x] for x in item])
elif isinstance(item, slice):
return TableColumns([self[x] for x in self.keys()[item]])
else:
raise IndexError('Illegal key or index value for TableColumns '
'object')
def __repr__(self):
names = ("'{0}'".format(x) for x in self.keys())
return "<TableColumns names=({0})>".format(",".join(names))
def _rename_column(self, name, new_name):
if new_name in self:
raise KeyError("Column {0} already exists".format(new_name))
mapper = {name: new_name}
new_names = [mapper.get(name, name) for name in self]
cols = self.values()
self.clear()
self.update(zip(new_names, cols))
# Define keys and values for Python 2 and 3 source compatibility
[docs] def keys(self):
return list(OrderedDict.keys(self))
[docs] def values(self):
return list(OrderedDict.values(self))
class BaseColumn(object):
__metaclass__ = abc.ABCMeta
def __array_finalize__(self, obj):
# Obj will be none for direct call to Column() creator
if obj is None:
return
# Self was created from template (e.g. obj[slice] or (obj * 2))
# or viewcast e.g. obj.view(Column). In either case we want to
# init Column attributes for self from obj if possible.
self.parent_table = None
for attr in ('name', 'units', 'format', 'description'):
val = getattr(obj, attr, None)
setattr(self, attr, val)
self.meta = deepcopy(getattr(obj, 'meta', {}))
def __array_wrap__(self, out_arr, context=None):
"""
__array_wrap__ is called at the end of every ufunc.
If the output is the same shape as the column then call the standard
ndarray __array_wrap__ which will return a Column object.
If instead the output shape is different (e.g. for reduction ufuncs
like sum() or mean()) then return the output viewed as a standard
np.ndarray. The "[()]" selects everything, but also converts a zero
rank array to a scalar. For some reason np.sum() returns a zero rank
scalar array while np.mean() returns a scalar. So the [()] is needed
for this case.
"""
if self.shape == out_arr.shape:
return np.ndarray.__array_wrap__(self, out_arr, context)
else:
return out_arr.view(np.ndarray)[()]
def _get_name(self):
return self._name
def _set_name(self, val):
if self.parent_table is not None:
table = self.parent_table
table.columns._rename_column(self.name, val)
table._data.dtype.names = table.columns.keys()
self._name = val
name = property(_get_name, _set_name)
@property
def descr(self):
"""Array-interface compliant full description of the column.
This returns a 3-tuple (name, type, shape) that can always be
used in a structured array dtype definition.
"""
return (self.name, self.dtype.str, self.shape[1:])
def __repr__(self):
if self.name:
units = None if self.units is None else str(self.units)
out = "<{0} name={1} units={2} format={3} " \
"description={4}>\n{5}".format(
self.__class__.__name__,
repr(self.name), repr(units),
repr(self.format), repr(self.description), repr(self.data))
else:
out = repr(self.data)
return out
def iter_str_vals(self):
"""
Return an iterator that yields the string-formatted values of this
column.
Returns
-------
str_vals : iterator
Column values formatted as strings
"""
# pprint._pformat_col_iter(col, max_lines, show_name, show_units, outs)
# Iterate over formatted values with no max number of lines, no column
# name, no units, and ignoring the returned header info in outs.
for str_val in _pformat_col_iter(self, -1, False, False, {}):
yield str_val
def attrs_equal(self, col):
"""Compare the column attributes of ``col`` to this object.
The comparison attributes are: name, units, dtype, format, description,
and meta.
Parameters
----------
col: Column
Comparison column
Returns
-------
equal: boolean
True if all attributes are equal
"""
if not isinstance(col, BaseColumn):
raise ValueError('Comparison `col` must be a Column or MaskedColumn object')
attrs = ('name', 'units', 'dtype', 'format', 'description', 'meta')
equal = all(getattr(self, x) == getattr(col, x) for x in attrs)
return equal
def pformat(self, max_lines=None, show_name=True, show_units=False):
"""Return a list of formatted string representation of column values.
If no value of `max_lines` is supplied then the height of the screen
terminal is used to set `max_lines`. If the terminal height cannot be
determined then the default will be determined using the
`astropy.table.pprint.MAX_LINES` configuration item. If a negative
value of `max_lines` is supplied then there is no line limit applied.
Parameters
----------
max_lines : int
Maximum lines of output (header + data rows)
show_name : bool
Include column name (default=True)
show_units : bool
Include a header row for units (default=False)
Returns
-------
lines : list
List of lines with header and formatted column values
"""
lines, n_header = _pformat_col(self, max_lines, show_name, show_units)
return lines
def pprint(self, max_lines=None, show_name=True, show_units=False):
"""Print a formatted string representation of column values.
If no value of `max_lines` is supplied then the height of the screen
terminal is used to set `max_lines`. If the terminal height cannot be
determined then the default will be determined using the
`astropy.table.pprint.MAX_LINES` configuration item. If a negative
value of `max_lines` is supplied then there is no line limit applied.
Parameters
----------
max_lines : int
Maximum number of values in output
show_name : bool
Include column name (default=True)
show_units : bool
Include a header row for units (default=False)
"""
lines, n_header = _pformat_col(self, max_lines, show_name, show_units)
for i, line in enumerate(lines):
if i < n_header:
color_print(line, 'red')
else:
print line
def more(self, max_lines=None, show_name=True, show_units=False):
"""Interactively browse column with a paging interface.
Supported keys::
f, <space> : forward one page
b : back one page
r : refresh same page
n : next row
p : previous row
< : go to beginning
> : go to end
q : quit browsing
h : print this help
Parameters
----------
max_lines : int
Maximum number of lines in table output
show_name : bool
Include a header row for column names (default=True)
show_units : bool
Include a header row for units (default=False)
"""
_more_tabcol(self, max_lines=max_lines, show_name=show_name,
show_units=show_units)
@property
def units(self):
"""
The units associated with this column. May be a string or a
`astropy.units.UnitBase` instance.
Setting the `units` property does not change the values of the
data. To perform a unit conversion, use `convert_units_to`.
"""
return self._units
@units.setter
def units(self, units):
if units is None:
self._units = None
else:
self._units = Unit(units, parse_strict='silent')
@units.deleter
def units(self):
self._units = None
def convert_units_to(self, new_units, equivalencies=[]):
"""
Converts the values of the column in-place from the current
unit to the given unit.
To change the units associated with this column without
actually changing the data values, simply set the `units`
property.
Parameters
----------
new_units : str or `astropy.units.UnitBase` instance
The unit to convert to.
equivalencies : list of equivalence pairs, optional
A list of equivalence pairs to try if the units are not
directly convertible. See :ref:`unit_equivalencies`.
Raises
------
astropy.units.UnitException
If units are inconsistent
"""
if self.units is None:
raise ValueError("No units set on column")
self.data[:] = self.units.to(
new_units, self.data, equivalencies=equivalencies)
self.units = new_units
def __str__(self):
lines, n_header = _pformat_col(self)
return '\n'.join(lines)
[docs]class Column(BaseColumn, np.ndarray):
"""Define a data column for use in a Table object.
Parameters
----------
name : str
Column name and key for reference within Table
data : list, ndarray or None
Column data values
dtype : numpy.dtype compatible value
Data type for column
shape : tuple or ()
Dimensions of a single row element in the column data
length : int or 0
Number of row elements in column data
description : str or None
Full description of column
units : str or None
Physical units
format : str or None
Format string for outputting column values. This can be an
"old-style" (``format % value``) or "new-style" (`str.format`)
format specification string.
meta : dict-like or None
Meta-data associated with the column
Examples
--------
A Column can be created in two different ways:
- Provide a ``data`` value and optionally a ``dtype`` value
Examples::
col = Column('name', data=[1, 2, 3]) # shape=(3,)
col = Column('name', data=[[1, 2], [3, 4]]) # shape=(2, 2)
col = Column('name', data=[1, 2, 3], dtype=float)
col = Column('name', np.array([1, 2, 3]))
col = Column('name', ['hello', 'world'])
The ``dtype`` argument can be any value which is an acceptable
fixed-size data-type initializer for the numpy.dtype() method. See
`<http://docs.scipy.org/doc/numpy/reference/arrays.dtypes.html>`_.
Examples include:
- Python non-string type (float, int, bool)
- Numpy non-string type (e.g. np.float32, np.int64, np.bool)
- Numpy.dtype array-protocol type strings (e.g. 'i4', 'f8', 'S15')
If no ``dtype`` value is provide then the type is inferred using
``np.array(data)``. When ``data`` is provided then the ``shape``
and ``length`` arguments are ignored.
- Provide zero or more of ``dtype``, ``shape``, ``length``
Examples::
col = Column('name')
col = Column('name', dtype=int, length=10, shape=(3,4))
The default ``dtype`` is ``np.float64`` and the default ``length`` is
zero. The ``shape`` argument is the array shape of a single cell in the
column. The default ``shape`` is () which means a single value in each
element.
"""
def __new__(cls, name, data=None,
dtype=None, shape=(), length=0,
description=None, units=None, format=None, meta=None):
if data is None:
dtype = (np.dtype(dtype).str, shape)
self_data = np.zeros(length, dtype=dtype)
elif isinstance(data, Column):
self_data = np.asarray(data.data, dtype=dtype)
if description is None:
description = data.description
if units is None:
units = units or data.units
if format is None:
format = data.format
if meta is None:
meta = deepcopy(data.meta)
elif isinstance(data, MaskedColumn):
raise TypeError("Cannot convert a MaskedColumn to a Column")
else:
self_data = np.asarray(data, dtype=dtype)
self = self_data.view(cls)
self._name = name
self.units = units
self.format = format
self.description = description
self.parent_table = None
self.meta = OrderedDict()
if meta is not None:
self.meta.update(meta)
return self
@property
[docs] def data(self):
return self.view(np.ndarray)
[docs] def copy(self, data=None, copy_data=True):
"""Return a copy of the current Column instance.
"""
if data is None:
data = self.view(np.ndarray)
if copy_data:
data = data.copy()
return Column(self.name, data, units=self.units, format=self.format,
description=self.description, meta=deepcopy(self.meta))
[docs]class MaskedColumn(BaseColumn, ma.MaskedArray):
def __new__(cls, name, data=None, mask=None, fill_value=None,
dtype=None, shape=(), length=0,
description=None, units=None, format=None, meta=None):
if NUMPY_LT_1P5:
raise ValueError('MaskedColumn requires NumPy version 1.5 or later')
if data is None:
dtype = (np.dtype(dtype).str, shape)
self_data = ma.zeros(length, dtype=dtype)
elif isinstance(data, (Column, MaskedColumn)):
self_data = ma.asarray(data.data, dtype=dtype)
if description is None:
description = data.description
if units is None:
units = units or data.units
if format is None:
format = data.format
if meta is None:
meta = deepcopy(data.meta)
else:
self_data = ma.asarray(data, dtype=dtype)
self = self_data.view(MaskedColumn)
if mask is None and hasattr(data, 'mask'):
mask = data.mask
if fill_value is None and hasattr(data, 'fill_value'):
fill_value = data.fill_value
self.mask = mask
self.fill_value = fill_value
self._name = name
self.units = units
self.format = format
self.description = description
self.parent_table = None
self.meta = OrderedDict()
if meta is not None:
self.meta.update(meta)
return self
def __array_finalize__(self, obj):
BaseColumn.__array_finalize__(self, obj)
ma.MaskedArray.__array_finalize__(self, obj)
def _fix_fill_value(self, val):
"""Fix a fill value (if needed) to work around a bug with setting the fill
value of a string array in MaskedArray with Python 3.x. See
https://github.com/numpy/numpy/pull/2733. This mimics the check in
numpy.ma.core._check_fill_value() (version < 1.7) which incorrectly sets
fill_value to a default if self.dtype.char is 'U' (which is the case for Python
3). Here we change the string to a byte string so that in Python 3 the
isinstance(val, basestring) part fails.
"""
if isinstance(val, basestring) and (self.dtype.char not in 'SV'):
val = val.encode()
return val
@property
def fill_value(self):
return self.get_fill_value() # defer to native ma.MaskedArray method
@fill_value.setter
[docs] def fill_value(self, val):
"""Set fill value both in the masked column view and in the parent table
if it exists. Setting one or the other alone doesn't work."""
val = self._fix_fill_value(val)
if self.parent_table:
self.parent_table._data[self._name].fill_value = val
# Yet another ma bug workaround: If the value of fill_value for a string array is
# requested but not yet set then it gets created as 'N/A'. From this point onward
# any new fill_values are truncated to 3 characters. Note that this does not
# occur if the masked array is a structured array (as in the previous block that
# deals with the parent table).
#
# >>> x = ma.array(['xxxx'])
# >>> x.fill_value # fill_value now gets represented as an 'S3' array
# 'N/A'
# >>> x.fill_value='yyyy'
# >>> x.fill_value
# 'yyy'
#
# To handle this we are forced to reset a private variable first:
self._fill_value = None
self.set_fill_value(val) # defer to native ma.MaskedArray method
@property
[docs] def data(self):
out = self.view(ma.MaskedArray)
# The following is necessary because of a bug in Numpy, which was
# fixed in numpy/numpy#2703. The fix should be included in Numpy 1.8.0.
out.fill_value = self.fill_value
return out
[docs] def filled(self, fill_value=None):
"""Return a copy of self, with masked values filled with a given value.
Parameters
----------
fill_value : scalar; optional
The value to use for invalid entries (None by default).
If None, the `fill_value` attribute of the array is used instead.
Returns
-------
filled_column : Column
A copy of ``self`` with masked entries replaced by `fill_value`
(be it the function argument or the attribute of ``self``).
"""
if fill_value is None:
fill_value = self.fill_value
fill_value = self._fix_fill_value(fill_value)
data = super(MaskedColumn, self).filled(fill_value)
out = Column(self.name, data, units=self.units, format=self.format,
description=self.description, meta=deepcopy(self.meta))
return out
[docs] def copy(self, data=None, copy_data=True):
"""
Return a copy of the current MaskedColumn instance.
Parameters
----------
data : array; optional
Data to use when creating MaskedColumn copy. If not supplied the
column data array is used.
copy_data : boolean; optional
Make a copy of input data instead of using a reference (default=True)
Returns
-------
column : MaskedColumn
A copy of ``self``
"""
if data is None:
data = self.view(ma.MaskedArray)
if copy_data:
data = data.copy()
return MaskedColumn(self.name, data, units=self.units, format=self.format,
# Do not include mask=self.mask since `data` has the mask
fill_value=self.fill_value,
description=self.description, meta=deepcopy(self.meta))
[docs]class Row(object):
"""A class to represent one row of a Table object.
A Row object is returned when a Table object is indexed with an integer
or when iterating over a table::
>>> table = Table([(1, 2), (3, 4)], names=('a', 'b'))
>>> row = table[1]
>>> row
<Row 1 of table
values=(2, 4)
dtype=[('a', '<i8'), ('b', '<i8')]>
>>> row['a']
2
>>> row[1]
4
"""
def __init__(self, table, index):
self._table = table
self._index = index
self._data = table._data[index]
# MaskedArray __getitem__ has a strange behavior where if a
# row mask is all False then it returns a np.void which
# has no mask attribute. This makes it impossible to then set
# the mask. Here we recast back to mvoid. This was fixed in
# Numpy following issue numpy/numpy#483, and the fix should be
# included in Numpy 1.8.0.
if self._table.masked and isinstance(self._data, np.void):
self._data = ma.core.mvoid(self._data,
mask=self._table._mask[index])
def __getitem__(self, item):
return self.data[item]
def __setitem__(self, item, val):
self.data[item] = val
def __eq__(self, other):
if self._table.masked:
# Sent bug report to numpy-discussion group on 2012-Oct-21, subject:
# "Comparing rows in a structured masked array raises exception"
# No response, so this is still unresolved.
raise ValueError('Unable to compare rows for masked table due to numpy.ma bug')
return self.data == other
def __ne__(self, other):
if self._table.masked:
raise ValueError('Unable to compare rows for masked table due to numpy.ma bug')
return self.data != other
@property
def _mask(self):
return self._data.mask
def __array__(self, dtype=None):
"""Support converting Row to np.array via np.array(table).
Coercion to a different dtype via np.array(table, dtype) is not
supported and will raise a ValueError.
"""
if dtype is not None:
raise ValueError('Datatype coercion is not allowed')
return np.array(self._data)
def __len__(self):
return len(self._data)
@property
[docs] def table(self):
return self._table
@property
[docs] def index(self):
return self._index
@property
[docs] def data(self):
return self._data
@property
@property
[docs] def columns(self):
return self.table.columns
@property
[docs] def colnames(self):
return self.table.colnames
@property
[docs] def dtype(self):
return self.data.dtype
def __repr__(self):
return "<Row {0} of table\n values={1!r}\n dtype={2}>".format(
self.index, self.data, self.dtype)
[docs]class Table(object):
"""A class to represent tables of heterogeneous data.
`Table` provides a class for heterogeneous tabular data, making use of a
`numpy` structured array internally to store the data values. A key
enhancement provided by the `Table` class is the ability to easily modify
the structure of the table by adding or removing columns, or adding new
rows of data. In addition table and column metadata are fully supported.
`Table` differs from `NDData` by the assumption that the input data
consists of columns of homogeneous data, where each column has a unique
identifier and may contain additional metadata such as the data
units, format, and description.
Parameters
----------
data : numpy ndarray, dict, list, or Table, optional
Data to initialize table.
mask : numpy ndarray, dict, list, optional
The mask to initialize the table
names : list, optional
Specify column names
dtypes : list, optional
Specify column data types
meta : dict, optional
Metadata associated with the table
copy : boolean, optional
Copy the input data (default=True).
"""
def __init__(self, data=None, masked=None, names=None, dtypes=None,
meta=None, copy=True):
# Set up a placeholder empty table
self._data = None
self._set_masked(masked)
self.columns = TableColumns()
self.meta = OrderedDict() if meta is None else deepcopy(meta)
# Must copy if dtypes are changing
if not copy and dtypes is not None:
raise ValueError('Cannot specify dtypes when copy=False')
# Infer the type of the input data and set up the initialization
# function, number of columns, and potentially the default col names
default_names = None
if isinstance(data, (list, tuple)):
init_func = self._init_from_list
n_cols = len(data)
elif isinstance(data, np.ndarray):
if data.dtype.names:
init_func = self._init_from_ndarray # _struct
n_cols = len(data.dtype.names)
default_names = data.dtype.names
else:
init_func = self._init_from_ndarray # _homog
n_cols = data.shape[1]
elif isinstance(data, dict):
init_func = self._init_from_dict
n_cols = len(data.keys())
default_names = data.keys()
elif isinstance(data, Table):
init_func = self._init_from_table
n_cols = len(data.colnames)
default_names = data.colnames
elif data is None:
if names is None:
return # Empty table
else:
init_func = self._init_from_list
n_cols = len(names)
data = [[]] * n_cols
else:
raise ValueError('Data type {0} not allowed to init Table'
.format(type(data)))
# Set up defaults if names and/or dtypes are not specified.
# A value of None means the actual value will be inferred
# within the appropriate initialization routine, either from
# existing specification or auto-generated.
if names is None:
names = default_names or [None] * n_cols
if dtypes is None:
dtypes = [None] * n_cols
self._check_names_dtypes(names, dtypes, n_cols)
# Finally do the real initialization
init_func(data, names, dtypes, n_cols, copy)
# Whatever happens above, the masked property should be set to a boolean
if type(self.masked) != bool:
raise TypeError("masked property has not been set to True or False")
if NUMPY_LT_1P5 and self.masked:
raise ValueError('Masked table requires NumPy version 1.5 or later')
@property
[docs] def mask(self):
return self._data.mask if self.masked else None
@property
def _mask(self):
"""This is needed due to intricacies in numpy.ma, don't remove it."""
return self._data.mask
[docs] def filled(self, fill_value=None):
"""Return a copy of self, with masked values filled.
If input ``fill_value`` supplied then that value is used for all masked entries
in the table. Otherwise the individual ``fill_value`` defined for each
table column is used.
Returns
-------
filled_table : Table
New table with masked values filled
"""
if self.masked:
data = [col.filled(fill_value) for col in self.columns.values()]
else:
data = self
return Table(data, meta=deepcopy(self.meta))
def __array__(self, dtype=None):
"""Support converting Table to np.array via np.array(table).
Coercion to a different dtype via np.array(table, dtype) is not
supported and will raise a ValueError.
"""
if dtype is not None:
raise ValueError('Datatype coercion is not allowed')
# This limitation is because of the following unexpected result that
# should have made a table copy while changing the column names.
#
# >>> d = astropy.table.Table([[1,2],[3,4]])
# >>> np.array(d, dtype=[('a', 'i8'), ('b', 'i8')])
# array([(0, 0), (0, 0)],
# dtype=[('a', '<i8'), ('b', '<i8')])
return self._data.data if self.masked else self._data
def _check_names_dtypes(self, names, dtypes, n_cols):
"""Make sure that names and dtypes are boths iterable and have
the same length as data.
"""
for inp_list, inp_str in ((dtypes, 'dtypes'), (names, 'names')):
if not isiterable(inp_list):
raise ValueError('{0} must be a list or None'.format(inp_str))
if len(names) != n_cols or len(dtypes) != n_cols:
raise ValueError(
'Arguments "names" and "dtypes" must match number of columns'
.format(inp_str))
def _set_masked_from_cols(self, cols):
if self.masked is None:
if any(isinstance(col, (MaskedColumn, ma.MaskedArray)) for col in cols):
self._set_masked(True)
else:
self._set_masked(False)
elif not self.masked:
if any(isinstance(col, (MaskedColumn, ma.MaskedArray)) for col in cols):
self._set_masked(True)
def _init_from_list(self, data, names, dtypes, n_cols, copy):
"""Initialize table from a list of columns. A column can be a
Column object, np.ndarray, or any other iterable object.
"""
if not copy:
raise ValueError('Cannot use copy=False with a list data input')
# Set self.masked appropriately, then get class to create column instances.
self._set_masked_from_cols(data)
cols = []
def_names = _auto_names(n_cols)
for col, name, def_name, dtype in zip(data, names, def_names, dtypes):
if isinstance(col, (Column, MaskedColumn)):
col = self.ColumnClass((name or col.name), col, dtype=dtype)
elif isinstance(col, np.ndarray) or isiterable(col):
col = self.ColumnClass((name or def_name), col, dtype=dtype)
else:
raise ValueError('Elements in list initialization must be '
'either Column or list-like')
cols.append(col)
self._init_from_cols(cols)
def _init_from_ndarray(self, data, names, dtypes, n_cols, copy):
"""Initialize table from an ndarray structured array"""
data_names = data.dtype.names or _auto_names(n_cols)
struct = data.dtype.names is not None
names = [name or data_names[i] for i, name in enumerate(names)]
cols = ([data[name] for name in data_names] if struct else
[data[:, i] for i in range(n_cols)])
# Set self.masked appropriately, then get class to create column instances.
self._set_masked_from_cols(cols)
if copy:
self._init_from_list(cols, names, dtypes, n_cols, copy)
else:
dtypes = [(name, col.dtype) for name, col in zip(names, cols)]
self._data = data.view(dtypes).ravel()
columns = TableColumns()
for name in names:
columns[name] = self.ColumnClass(name, self._data[name])
columns[name].parent_table = self
self.columns = columns
def _init_from_dict(self, data, names, dtypes, n_cols, copy):
"""Initialize table from a dictionary of columns"""
if not copy:
raise ValueError('Cannot use copy=False with a dict data input')
data_list = [data[name] for name in names]
self._init_from_list(data_list, names, dtypes, n_cols, copy)
def _init_from_table(self, data, names, dtypes, n_cols, copy):
"""Initialize table from an existing Table object """
table = data # data is really a Table, rename for clarity
data_names = table.colnames
self.meta = deepcopy(table.meta)
cols = table.columns.values()
# Set self.masked appropriately from cols
self._set_masked_from_cols(cols)
if copy:
self._init_from_list(cols, names, dtypes, n_cols, copy)
else:
names = [vals[0] or vals[1] for vals in zip(names, data_names)]
dtypes = [(name, col.dtype) for name, col in zip(names, cols)]
data = table._data.view(dtypes)
self._update_table_from_cols(self, data, cols, names)
def _init_from_cols(self, cols):
"""Initialize table from a list of Column objects"""
lengths = set(len(col.data) for col in cols)
if len(lengths) != 1:
raise ValueError('Inconsistent data column lengths: {0}'
.format(lengths))
self._set_masked_from_cols(cols)
cols = [self.ColumnClass(col.name, col) for col in cols]
names = [col.name for col in cols]
dtypes = [col.descr for col in cols]
empty_init = ma.empty if self.masked else np.empty
data = empty_init(lengths.pop(), dtype=dtypes)
for col in cols:
data[col.name] = col.data
self._update_table_from_cols(self, data, cols, names)
def _new_from_slice(self, slice_):
"""Create a new table as a referenced slice from self."""
table = Table()
table.meta = deepcopy(self.meta)
cols = self.columns.values()
names = [col.name for col in cols]
data = self._data[slice_]
self._update_table_from_cols(table, data, cols, names)
return table
@staticmethod
def _update_table_from_cols(table, data, cols, names):
"""Update the existing ``table`` so that it represents the given
``data`` (a structured ndarray) with ``cols`` and ``names``."""
columns = TableColumns()
table._data = data
for name, col in zip(names, cols):
newcol = col.copy(data=data[name], copy_data=False)
newcol.name = name
newcol.parent_table = table
columns[name] = newcol
table.columns = columns
def __repr__(self):
names = ("'{0}'".format(x) for x in self.colnames)
s = "<Table rows={0} names=({1})>\n{2}".format(
self.__len__(), ','.join(names), repr(self._data))
return s
def __str__(self):
lines, n_header = _pformat_table(self)
return '\n'.join(lines)
[docs] def pprint(self, max_lines=None, max_width=None, show_name=True,
show_units=False):
"""Print a formatted string representation of the table.
If no value of `max_lines` is supplied then the height of the screen
terminal is used to set `max_lines`. If the terminal height cannot
be determined then the default is taken from the configuration item
`astropy.table.pprint.MAX_LINES`. If a negative value of `max_lines`
is supplied then there is no line limit applied.
The same applies for max_width except the configuration item is
`astropy.table.pprint.MAX_WIDTH`.
Parameters
----------
max_lines : int
Maximum number of lines in table output
max_width : int or None
Maximum character width of output
show_name : bool
Include a header row for column names (default=True)
show_units : bool
Include a header row for units (default=False)
"""
lines, n_header = _pformat_table(self, max_lines, max_width, show_name,
show_units)
for i, line in enumerate(lines):
if i < n_header:
color_print(line, 'red')
else:
print line
[docs] def more(self, max_lines=None, max_width=None, show_name=True,
show_units=False):
"""Interactively browse table with a paging interface.
Supported keys::
f, <space> : forward one page
b : back one page
r : refresh same page
n : next row
p : previous row
< : go to beginning
> : go to end
q : quit browsing
h : print this help
Parameters
----------
max_lines : int
Maximum number of lines in table output
max_width : int or None
Maximum character width of output
show_name : bool
Include a header row for column names (default=True)
show_units : bool
Include a header row for units (default=False)
"""
_more_tabcol(self, max_lines, max_width, show_name,
show_units)
def _repr_html_(self):
lines = self.pformat(html=True)
return ''.join(lines)
def __getitem__(self, item):
if isinstance(item, basestring):
return self.columns[item]
elif isinstance(item, int):
return Row(self, item)
elif isinstance(item, tuple):
if any(x not in set(self.colnames) for x in item):
raise ValueError('Table column slice must contain only valid '
'column names')
return Table([self[x] for x in item], meta=deepcopy(self.meta))
elif (isinstance(item, slice) or isinstance(item, np.ndarray)
or isinstance(item, list)):
return self._new_from_slice(item)
else:
raise ValueError('Illegal type {0} for table item access'
.format(type(item)))
def __setitem__(self, item, value):
try:
self._data[item] = value
except (ValueError, KeyError, TypeError):
raise KeyError("Column {0} does not exist".format(item))
except:
raise
def __delitem__(self, item):
if isinstance(item, basestring):
self.remove_column(item)
elif isinstance(item, tuple):
self.remove_columns(item)
def __iter__(self):
self._iter_index = 0
return self
def __next__(self):
"""Python 3 iterator"""
if self._iter_index < len(self._data):
val = self[self._iter_index]
self._iter_index += 1
return val
else:
raise StopIteration
if sys.version_info[0] < 3: # pragma: py2
next = __next__
[docs] def field(self, item):
"""Return column[item] for recarray compatibility."""
return self.columns[item]
@property
def masked(self):
return self._masked
@masked.setter
[docs] def masked(self, masked):
raise Exception('Masked attribute is read-only (use t = Table(t, masked=True)'
' to convert to a masked table)')
def _set_masked(self, masked):
"""
Set the table masked property.
Parameters
----------
masked : bool
State of table masking (True or False)
"""
if hasattr(self, '_masked'):
# The only allowed change is from None to False or True, or False to True
if self._masked is None and masked in [False, True]:
self._masked = masked
elif self._masked is False and masked is True:
log.info("Upgrading Table to masked Table")
self._masked = masked
elif self._masked is masked:
raise Exception("Masked attribute is already set to {0}".format(masked))
else:
raise Exception("Cannot change masked attribute to {0} once it is set to {1}"
.format(masked, self._masked))
else:
if masked in [True, False, None]:
self._masked = masked
else:
raise ValueError("masked should be one of True, False, None")
if self._masked:
self._column_class = MaskedColumn
else:
self._column_class = Column
@property
[docs] def ColumnClass(self):
if self._column_class is None:
return Column
else:
return self._column_class
@property
[docs] def dtype(self):
return self._data.dtype
@property
[docs] def colnames(self):
return list(self.columns.keys())
[docs] def keys(self):
return list(self.columns.keys())
def __len__(self):
if self._data is None:
return 0
else:
return len(self._data)
[docs] def create_mask(self):
if isinstance(self._data, ma.MaskedArray):
raise Exception("data array is already masked")
else:
self._data = ma.array(self._data)
[docs] def index_column(self, name):
"""
Return the positional index of column ``name``.
Parameters
----------
name : str
column name
Returns
-------
index : int
Positional index of column ``name``.
"""
try:
return self.colnames.index(name)
except ValueError:
raise ValueError("Column {0} does not exist".format(name))
[docs] def add_column(self, col, index=None):
"""
Add a new Column object ``col`` to the table. If ``index``
is supplied then insert column before ``index`` position
in the list of columns, otherwise append column to the end
of the list.
Parameters
----------
col : Column
Column object to add.
index : int or None
Insert column before this position or at end (default)
"""
if index is None:
index = len(self.columns)
self.add_columns([col], [index])
[docs] def add_columns(self, cols, indexes=None):
"""
Add a list of new Column objects ``cols`` to the table. If a
corresponding list of ``indexes`` is supplied then insert column before
each ``index`` position in the *original* list of columns, otherwise
append columns to the end of the list.
Parameters
----------
cols : list of Columns
Column objects to add.
indexes : list of ints or None
Insert column before this position or at end (default)
"""
if indexes is None:
indexes = [len(self.columns)] * len(cols)
elif len(indexes) != len(cols):
raise ValueError('Number of indexes must match number of cols')
if self._data is None:
# No existing table data, init from cols
newcols = cols
else:
newcols = list(self.columns.values())
new_indexes = list(range(len(newcols) + 1))
for col, index in zip(cols, indexes):
i = new_indexes.index(index)
new_indexes.insert(i, None)
newcols.insert(i, col)
self._init_from_cols(newcols)
[docs] def remove_column(self, name):
"""
Remove a column from the table.
This can also be done with::
del table[name]
Parameters
----------
name : str
Name of column to remove
"""
self.remove_columns([name])
[docs] def remove_columns(self, names):
'''
Remove several columns from the table
Parameters
----------
names : list
A list containing the names of the columns to remove
'''
for name in names:
if name not in self.columns:
raise KeyError("Column {0} does not exist".format(name))
for name in names:
self.columns.pop(name)
self._data = _drop_fields(self._data, names)
[docs] def keep_columns(self, names):
'''
Keep only the columns specified (remove the others).
Parameters
----------
names : list
A list containing the names of the columns to keep. All other
columns will be removed.
'''
if isinstance(names, basestring):
names = [names]
for name in names:
if name not in self.columns:
raise KeyError("Column {0} does not exist".format(name))
remove = list(set(self.keys()) - set(names))
self.remove_columns(remove)
[docs] def rename_column(self, name, new_name):
'''
Rename a column.
This can also be done directly with by setting the ``name`` attribute
for a column::
table[name].name = new_name
Parameters
----------
name : str
The current name of the column.
new_name : str
The new name for the column
'''
if name not in self.keys():
raise KeyError("Column {0} does not exist".format(name))
self.columns[name].name = new_name
[docs] def add_row(self, vals=None, mask=None):
"""Add a new row to the end of the table.
The ``vals`` argument can be:
sequence (e.g. tuple or list)
Column values in the same order as table columns.
mapping (e.g. dict)
Keys corresponding to column names. Missing values will be
filled with np.zeros for the column dtype.
None
All values filled with np.zeros for the column dtype.
This method requires that the Table object "owns" the underlying array
data. In particular one cannot add a row to a Table that was
initialized with copy=False from an existing array.
The ``mask`` attribute should give (if desired) the mask for the
values. The type of the mask should match that of the values, i.e. if
``vals`` is an iterable, then ``mask`` should also be an iterable
with the same length, and if ``vals`` is a mapping, then ``mask``
should be a dictionary.
Parameters
----------
vals : tuple, list, dict or None
Use the specified values in the new row
"""
def _is_mapping(obj):
"""Minimal checker for mapping (dict-like) interface for obj"""
attrs = ('__getitem__', '__len__', '__iter__', 'keys', 'values', 'items')
return all(hasattr(obj, attr) for attr in attrs)
newlen = len(self._data) + 1
if mask is not None and not self.masked:
self._set_masked(True)
if self.masked:
self._data = ma.resize(self._data, (newlen,))
else:
self._data.resize((newlen,), refcheck=False)
if _is_mapping(vals):
if mask is not None and not _is_mapping(mask):
raise TypeError("Mismatch between type of vals and mask")
# Now check that the mask is specified for the same keys as the
# values, otherwise things get really confusing.
if mask is not None and set(vals.keys()) != set(mask.keys()):
raise ValueError('keys in mask should match keys in vals')
if self.masked:
# We set the mask to True regardless of whether a mask value
# is specified or not - that is, any cell where a new row
# value is not specified should be treated as missing.
self._data.mask[-1] = (True,) * len(self._data.dtype)
# First we copy the values
for name, val in vals.items():
try:
self._data[name][-1] = val
except IndexError:
raise ValueError("No column {0} in table".format(name))
if mask:
self._data[name].mask[-1] = mask[name]
elif isiterable(vals):
if mask is not None and (not isiterable(mask) or _is_mapping(mask)):
raise TypeError("Mismatch between type of vals and mask")
if len(self.columns) != len(vals):
raise ValueError('Mismatch between number of vals and columns')
if not isinstance(vals, tuple):
vals = tuple(vals)
self._data[-1] = vals
if mask is not None:
if len(self.columns) != len(mask):
raise ValueError('Mismatch between number of masks and columns')
if not isinstance(mask, tuple):
mask = tuple(mask)
self._data.mask[-1] = mask
elif vals is not None:
raise TypeError('Vals must be an iterable or mapping or None')
# Add_row() probably corrupted the Column views of self._data. Rebuild
# self.columns. Col.copy() takes an optional data reference that it
# uses in the copy.
cols = [self.ColumnClass(c.name, c).copy(self._data[c.name]) for c in self.columns.values()]
self.columns = TableColumns(cols)
[docs] def sort(self, keys):
'''
Sort the table according to one or more keys. This operates
on the existing table and does not return a new table.
Parameters
----------
keys : str or list of str
The key(s) to order the table by
'''
if type(keys) is not list:
keys = [keys]
self._data.sort(order=keys)
[docs] def reverse(self):
'''
Reverse the row order of table rows. The table is reversed
in place and there are no function arguments.
'''
self._data[:] = self._data[::-1].copy()
@classmethod
[docs] def read(cls, *args, **kwargs):
'''
Read a table
The arguments passed to this method depend on the format
'''
if 'format' in kwargs:
format = kwargs.pop('format')
else:
format = None
if format is None:
valid_formats = identify_format('read', args, kwargs)
if len(valid_formats) == 0:
raise Exception("Format could not be identified")
elif len(valid_formats) > 1:
raise Exception("Format is ambiguous - options are: {0:s}".format(', '.join(valid_formats)))
else:
format = valid_formats[0]
reader = get_reader(format)
table = reader(*args, **kwargs)
if not isinstance(table, cls):
raise TypeError("reader should return a {0:s} instance".format(cls.__name__))
return table
[docs] def write(self, *args, **kwargs):
'''
Write a table
The arguments passed to this method depend on the format
'''
if 'format' in kwargs:
format = kwargs.pop('format')
else:
format = None
if format is None:
valid_formats = identify_format('write', args, kwargs)
if len(valid_formats) == 0:
raise Exception("Format could not be identified")
elif len(valid_formats) > 1:
raise Exception("Format is ambiguous - options are: {0:s}".format(', '.join(valid_formats)))
else:
format = valid_formats[0]
writer = get_writer(format)
writer(self, *args, **kwargs)