#!/usr/bin/env python # -*- coding: utf8 -*- """ This module contains a collection of utility classes and functions used in agate. """ from collections import OrderedDict, Sequence from functools import wraps import math import string import warnings from slugify import slugify as pslugify from agate.warns import warn_duplicate_column, warn_unnamed_column try: from cdecimal import Decimal, ROUND_FLOOR, ROUND_CEILING, getcontext except ImportError: # pragma: no cover from decimal import Decimal, ROUND_FLOOR, ROUND_CEILING, getcontext import six #: Sentinal for use when `None` is an valid argument value default = object() def memoize(func): """ Dead-simple memoize decorator for instance methods that take no arguments. This is especially useful since so many of our classes are immutable. """ memo = None @wraps(func) def wrapper(self): if memo is not None: return memo return func(self) return wrapper class NullOrder(object): """ Dummy object used for sorting in place of None. Sorts as "greater than everything but other nulls." """ def __lt__(self, other): return False def __gt__(self, other): if other is None: return False return True class Quantiles(Sequence): """ A class representing quantiles (percentiles, quartiles, etc.) for a given column of Number data. """ def __init__(self, quantiles): self._quantiles = quantiles def __getitem__(self, i): return self._quantiles.__getitem__(i) def __iter__(self): return self._quantiles.__iter__() def __len__(self): return self._quantiles.__len__() def __repr__(self): return repr(self._quantiles) def locate(self, value): """ Identify which quantile a given value is part of. """ i = 0 if value < self._quantiles[0]: raise ValueError('Value is less than minimum quantile value.') if value > self._quantiles[-1]: raise ValueError('Value is greater than maximum quantile value.') if value == self._quantiles[-1]: return Decimal(len(self._quantiles) - 1) while value >= self._quantiles[i + 1]: i += 1 return Decimal(i) def median(data_sorted): """ Finds the median value of a given series of values. :param data_sorted: The values to find the median of. Must be sorted. """ length = len(data_sorted) if length % 2 == 1: return data_sorted[((length + 1) // 2) - 1] half = length // 2 a = data_sorted[half - 1] b = data_sorted[half] return (a + b) / 2 def max_precision(values): """ Given a series of values (such as a :class:`.Column`) returns the most significant decimal places present in any value. :param values: The values to analyze. """ max_whole_places = 1 max_decimal_places = 0 precision = getcontext().prec for value in values: if value is None or math.isnan(value): continue sign, digits, exponent = value.normalize().as_tuple() exponent_places = exponent * -1 whole_places = len(digits) - exponent_places if whole_places > max_whole_places: max_whole_places = whole_places if exponent_places > max_decimal_places: max_decimal_places = exponent_places # In Python 2 it was possible for the total digits to exceed the # available context precision. This ensures that can't happen. See #412 if max_whole_places + max_decimal_places > precision: # pragma: no cover max_decimal_places = precision - max_whole_places return max_decimal_places def make_number_formatter(decimal_places, add_ellipsis=False): """ Given a number of decimal places creates a formatting string that will display numbers with that precision. :param decimal_places: The number of decimal places :param add_ellipsis: Optionally add an ellipsis symbol at the end of a number """ fraction = u'0' * decimal_places ellipsis = u'…' if add_ellipsis else u'' return u''.join([u'#,##0.', fraction, ellipsis, u';-#,##0.', fraction, ellipsis]) def round_limits(minimum, maximum): """ Rounds a pair of minimum and maximum values to form reasonable "round" values suitable for use as axis minimum and maximum values. Values are rounded "out": up for maximum and down for minimum, and "off": to one higher than the first significant digit shared by both. See unit tests for examples. """ min_bits = minimum.normalize().as_tuple() max_bits = maximum.normalize().as_tuple() max_digits = max( len(min_bits.digits) + min_bits.exponent, len(max_bits.digits) + max_bits.exponent ) # Whole number rounding if max_digits > 0: multiplier = Decimal('10') ** (max_digits - 1) min_fraction = (minimum / multiplier).to_integral_value(rounding=ROUND_FLOOR) max_fraction = (maximum / multiplier).to_integral_value(rounding=ROUND_CEILING) return ( min_fraction * multiplier, max_fraction * multiplier ) max_exponent = max(min_bits.exponent, max_bits.exponent) # Fractional rounding q = Decimal('10') ** (max_exponent + 1) return ( minimum.quantize(q, rounding=ROUND_FLOOR).normalize(), maximum.quantize(q, rounding=ROUND_CEILING).normalize() ) def letter_name(index): """ Given a column index, assign a "letter" column name equivalent to Excel. For example, index ``4`` would return ``E``. Index ``30`` would return ``EE``. """ letters = string.ascii_lowercase count = len(letters) return letters[index % count] * ((index // count) + 1) def parse_object(obj, path=''): """ Recursively parse JSON-like Python objects as a dictionary of paths/keys and values. Inspired by JSONPipe (https://github.com/dvxhouse/jsonpipe). """ if isinstance(obj, dict): iterator = obj.items() elif isinstance(obj, (list, tuple)): iterator = enumerate(obj) else: return {path.strip('/'): obj} d = OrderedDict() for key, value in iterator: key = six.text_type(key) d.update(parse_object(value, path + key + '/')) return d def issequence(obj): """ Returns :code:`True` if the given object is an instance of :class:`.Sequence` that is not also a string. """ return isinstance(obj, Sequence) and not isinstance(obj, six.string_types) def deduplicate(values, column_names=False, separator='_'): """ Append a unique identifer to duplicate strings in a given sequence of strings. Identifers are an underscore followed by the occurance number of the specific string. ['abc', 'abc', 'cde', 'abc'] -> ['abc', 'abc_2', 'cde', 'abc_3'] :param column_names: If True, values are treated as column names. Warnings will be thrown if column names are None or duplicates. None values will be replaced with letter indices. """ final_values = [] for i, value in enumerate(values): if column_names: if not value: new_value = letter_name(i) warn_unnamed_column(i, new_value) elif isinstance(value, six.string_types): new_value = value else: raise ValueError('Column names must be strings or None.') else: new_value = value final_value = new_value duplicates = 0 while final_value in final_values: final_value = new_value + separator + str(duplicates + 2) duplicates += 1 if column_names and duplicates > 0: warn_duplicate_column(new_value, final_value) final_values.append(final_value) return tuple(final_values) def slugify(values, ensure_unique=False, **kwargs): """ Given a sequence of strings, returns a standardized version of the sequence. If ``ensure_unique`` is True, any duplicate strings will be appended with a unique identifier. agate uses an underscore as a default separator but this can be changed with kwargs. Any kwargs will be passed to the slugify method in python-slugify. See: https://github.com/un33k/python-slugify """ slug_args = {'separator': '_'} slug_args.update(kwargs) if ensure_unique: new_values = tuple(pslugify(value, **slug_args) for value in values) return deduplicate(new_values, separator=slug_args['separator']) else: return tuple(pslugify(value, **slug_args) for value in values)