def classify(fn, M, E):
    """
    Classify SAM or SID on a HSI cube
    Can''t be use with normXCorr
    """
    import pysptools.util as util
    width, height, bands = M.shape
    M = util.convert2d(M)
    cmap = np.zeros(M.shape[0])
    for i in range(M.shape[0]):
        T = M[i]
        floor = np.PINF
        k = 0
        for j in range(E.shape[0]):
            R = E[j]
            result = fn(T, R)
            if result < floor:
                floor = result
                k = j
        cmap[i] = k
    return util.convert3d(cmap, width, height)

def test_adding_bin():
    col = ''mean radius''
    data = cancer_df[col].values
    cib = ConditionalInferenceBinner(''test_dim_{}''.format(col), alpha=0.95)
    cib.fit(data, cancer_target)
 
    cib.add_bin(-1.0, [0.1, 0.9])
 
    np.testing.assert_equal(cib.splits, [-1.0, 11.75, 13.079999923706055, 15.039999961853027, 16.84000015258789, np.PINF, np.NaN])
    np.testing.assert_equal(cib.values,
                            [[0.1, 0.9],
                             [0.02, 0.97999999999999998],
                             [0.086956521739130432, 0.91304347826086951],
                             [0.2878787878787879, 0.71212121212121215],
                             [0.81481481481481477, 0.18518518518518517],
                             [0.99152542372881358, 0.0084745762711864406],
                             [0.37258347978910367, 0.62741652021089633]])

def test_recursion_with_nan():
    col = ''mean area''
    data = cancer_df[col].values
    rand_idx = np.linspace(1, 500, 23).astype(int)
    data[rand_idx] = np.NaN
 
    cib = ConditionalInferenceBinner(''test_dim_{}''.format(col), cancer_target)
 
    np.testing.assert_equal(cib.splits, [471.29998779296875, 555.0999755859375, 693.7000122070312, 880.2000122070312,
                            [[0.030769230769230771, 0.96923076923076923],
                             [0.13414634146341464, 0.86585365853658536],
                             [0.31730769230769229, 0.68269230769230771],
                             [0.83333333333333337, 0.16666666666666666],
                             [0.99145299145299148, 0.0085470085470085479],
                             [0.2608695652173913, 0.73913043478260865]])

def test_recursion_with_nan_and_special_value():
    col = ''mean area''
    data = cancer_df[col].values
    rand_idx = np.linspace(1, 23).astype(int)
    data[rand_idx] = np.NaN
 
    rand_idx_2 = np.linspace(1, 550, 29).astype(int)
    data[rand_idx_2] = -1.0
 
    cib = ConditionalInferenceBinner(''test_dim_{}''.format(col), alpha=0.95, special_values=[-1.0, np.NaN])
    cib.fit(data, 471.29998779296875, 572.2999877929688, 819.7999877929688,
                            [[0.4827586206896552, 0.5172413793103449],
                             [0.032432432432432434, 0.9675675675675676],
                             [0.14432989690721648, 0.8556701030927835],
                             [0.3132530120481928, 0.6867469879518072],
                             [0.8205128205128205, 0.1794871794871795],
                             [1.0, 0.0],
                             [0.23809523809523808, 0.7619047619047619]])

def _init_shapes_and_data(self, data, labels):
 
        self.n_features = data.shape[1]
 
        if isinstance(data, pandas.core.frame.DataFrame):
            self.feature_names = data.columns.tolist()
            data = data.as_matrix()
 
        if self.feature_names is None:
            self.feature_names = [''feature_{}''.format(dim) for dim in range(self.n_features)]
 
        if isinstance(labels, pandas.core.series.Series):
            labels = labels.values
 
        cntr = Counter(labels)
        assert set(cntr.keys()) == {-1, 1}, "Labels must be encoded with -1,1. Cannot contain more classes."
        assert self.n_features is not None, "Number of attributes is None"
 
        self.shapes = {name: ShapeFunction([np.PINF],
                                           [0.0],
                                           name)
                       for name in self.feature_names}
        self.initialized = True
 
        return data, labels

def _recurse_tree(tree, lst, mdlp, node_id=0, depth=0, min_val=np.NINF, max_val=np.PINF):
    left_child = tree.children_left[node_id]
    right_child = tree.children_right[node_id]
 
    if left_child == sklearn.tree._tree.TREE_LEAF:
        lst.append(((min_val, max_val), tree.value[node_id].flatten().tolist()))
        return
    else:
        if mdlp and _check_mdlp_stop(tree, node_id):
            lst.append(((min_val, tree.value[node_id].flatten().tolist()))
            return
        _recurse_tree(tree, left_child, depth=depth + 1, min_val=min_val, max_val=tree.threshold[node_id])
 
    if right_child == sklearn.tree._tree.TREE_LEAF:
        lst.append(((min_val, right_child, min_val=tree.threshold[node_id], max_val=max_val)

def _clipper(self):
        ''''''
        projects the weights to the feasible set
        :return:
        ''''''
        return tf.assign(self.W, tf.clip_by_value(self.W, 0, np.PINF), name="projector")

def _create_partition(lst_of_splits):
    return np.append(lst_of_splits, np.PINF)

def test_func_add_2():
    func1 = ShapeFunction([np.PINF], [0], ''test_1'')
    func2 = ShapeFunction([0, 1, 2], [-1, -2], ''test_1'')
 
    assert func1 != func2
 
    func3 = ShapeFunction([0.0, 1.0, 2.0, np.PINF],
                          [-1.0, -2.0, ''test_1'')
 
    assert func1.add(func2).equals(func3)

def test_recursion():
    binner = tfb.DecisionTreeBinner(''test'', max_leaf_nodes=4)
    binner.fit(data[:, 0], target)
 
    np.testing.assert_equal(binner.splits, [13.094999313354492, 15.045000076293945, 16.924999237060547, np.NaN])
    np.testing.assert_equal(binner.values, [[0.04905660377358491, 0.9509433962264151],
                                            [0.2878787878787879, 0.7121212121212122],
                                            [0.8148148148148148,
                                            [0.9915254237288136, 0.00847457627118644],
                                            [0.37258347978910367, 0.62741652021089633]])

def test_recursion_with_mdlp():
    binner = tfb.DecisionTreeBinner(''test'', mdlp=True)
    binner.fit(data[:, 17.880001068115234,
                                            [0.8533333333333334, 0.14666666666666667],
                                            [1.0, 0.62741652021089633]])

def test_recursion():
    col = ''mean radius''
    data = cancer_df[col].values
    cib = ConditionalInferenceBinner(''test_dim_{}''.format(col), [11.75,
                            [[0.02, 0.62741652021089633]])

def test_adding_bin_with_non_numeric_splits_only():
    cib = ConditionalInferenceBinner(''test'', alpha=0.05)
    cib.splits = [np.PINF, np.NaN]
    cib.values = [[0.1, [0.8, 0.2]]
    cib.is_fit = True
 
    cib.add_bin(-1.0, [0.3, 0.7])
    np.testing.assert_equal(cib.splits, [[0.3, 0.7], 0.2]])

def __init__(self, name, **kwargs):
        self.name = name
 
        self.alpha = kwargs.get(''alpha'', 0.95)
        self.min_samples_split = kwargs.get(''min_samples_split'', 2)
        self.min_samples_leaf = kwargs.get(''min_samples_leaf'', 2)
        self.special_values = kwargs.get(''special_values'', [np.NaN])
 
        self.num_classes = None
 
        self._splits = [np.PINF]
        self._values = list()
        self.nodes = list()
 
        self._is_fit = False

def __init__(self, **kwargs):
        self.name = name
        self._is_fit = False
 
        criterion = kwargs.get(''criterion'', ''gini'')
        splitter = kwargs.get(''splitter'', ''best'')
        max_depth = kwargs.get(''max_depth'', None)
        min_samples_split = kwargs.get(''min_samples_split'', 2)
        min_samples_leaf = kwargs.get(''min_samples_leaf'', 1)
        min_weight_fraction_leaf = kwargs.get(''min_weight_fraction_leaf'', 0.0)
        max_features = kwargs.get(''max_features'', None)
        random_state = kwargs.get(''random_state'', None)
        max_leaf_nodes = kwargs.get(''max_leaf_nodes'', None)
        class_weight = kwargs.get(''class_weight'', None)
        presort = kwargs.get(''presort'', False)
 
        self.mdlp = kwargs.get(''mdlp'', False)
 
        if self.mdlp:
            criterion = ''entropy''
            max_leaf_nodes = None
            max_depth = None
 
        self.dtc = DecisionTreeClassifier(criterion=criterion,
                                          splitter=splitter,
                                          max_depth=max_depth,
                                          min_samples_split=min_samples_split,
                                          min_samples_leaf=min_samples_leaf,
                                          min_weight_fraction_leaf=min_weight_fraction_leaf,
                                          max_features=max_features,
                                          random_state=random_state,
                                          max_leaf_nodes=max_leaf_nodes,
                                          class_weight=class_weight,
                                          presort=presort)
 
        self._splits = [np.PINF]
        self._values = list()

def masked_invalid(a, copy=True):
    """
    Mask an array where invalid values occur (NaNs or infs).
 
    This function is a shortcut to ``masked_where``,with
    `condition` = ~(np.isfinite(a)). Any pre-existing mask is conserved.
    Only applies to arrays with a dtype where NaNs or infs make sense
    (i.e. floating point types),but accepts any array_like object.
 
    See Also
    --------
    masked_where : Mask where a condition is met.
 
    Examples
    --------
    >>> import numpy.ma as ma
    >>> a = np.arange(5,dtype=np.float)
    >>> a[2] = np.NaN
    >>> a[3] = np.PINF
    >>> a
    array([  0.,1.,NaN,Inf,4.])
    >>> ma.masked_invalid(a)
    masked_array(data = [0.0 1.0 -- -- 4.0],
          mask = [False False  True  True False],
          fill_value=1e+20)
 
    """
    a = np.array(a, copy=copy, subok=True)
    mask = getattr(a, ''_mask'', None)
    if mask is not None:
        condition = ~(np.isfinite(getdata(a)))
        if mask is not nomask:
            condition |= mask
        cls = type(a)
    else:
        condition = ~(np.isfinite(a))
        cls = MaskedArray
    result = a.view(cls)
    result._mask = condition
    return result
 
 
###############################################################################
#                            Printing options                                 #
###############################################################################

def describe_1d(data, **kwargs):
    leng = len(data)  # number of observations in the Series
    count = data.count()  # number of non-NaN observations in the Series
 
    # Replace infinite values with NaNs to avoid issues with
    # histograms later.
    data.replace(to_replace=[np.inf, np.NINF, value=np.nan, inplace=True)
 
    n_infinite = count - data.count()  # number of infinte observations in the Series
 
    distinct_count = data.nunique(dropna=False)  # number of unique elements in the Series
    if count > distinct_count > 1:
        mode = data.mode().iloc[0]
    else:
        mode = data[0]
 
    results_data = {''count'': count,
                    ''distinct_count'': distinct_count,
                    ''p_missing'': 1 - count / leng,
                    ''n_missing'': leng - count,
                    ''p_infinite'': n_infinite / leng,
                    ''n_infinite'': n_infinite,
                    ''is_unique'': distinct_count == leng,
                    ''mode'': mode,
                    ''p_unique'': distinct_count / leng}
    try:
        # pandas 0.17 onwards
        results_data[''memorysize''] = data.memory_usage()
    except:
        results_data[''memorysize''] = 0
 
    result = pd.Series(results_data, name=data.name)
 
    vartype = get_vartype(data)
    if vartype == ''CONST'':
        result = result.append(describe_constant_1d(data))
    elif vartype == ''BOOL'':
        result = result.append(describe_boolean_1d(data, **kwargs))
    elif vartype == ''NUM'':
        result = result.append(describe_numeric_1d(data, **kwargs))
    elif vartype == ''DATE'':
        result = result.append(describe_date_1d(data, **kwargs))
    elif vartype == ''UNIQUE'':
        result = result.append(describe_unique_1d(data, **kwargs))
    else:
        result = result.append(describe_categorical_1d(data))
    return result

def masked_invalid(a, None)
    if mask is not None:
        condition = ~(np.isfinite(getdata(a)))
        if mask is not nomask:
            condition |= mask
        cls = type(a)
    else:
        condition = ~(np.isfinite(a))
        cls = MaskedArray
    result = a.view(cls)
    result._mask = condition
    return result
 
 
###############################################################################
#                            Printing options                                 #
###############################################################################

def masked_invalid(a, None)
    if mask is not None:
        condition = ~(np.isfinite(getdata(a)))
        if mask is not nomask:
            condition |= mask
        cls = type(a)
    else:
        condition = ~(np.isfinite(a))
        cls = MaskedArray
    result = a.view(cls)
    result._mask = condition
    return result
 
 
###############################################################################
#                            Printing options                                 #
###############################################################################

def masked_invalid(a, None)
    if mask is not None:
        condition = ~(np.isfinite(getdata(a)))
        if mask is not nomask:
            condition |= mask
        cls = type(a)
    else:
        condition = ~(np.isfinite(a))
        cls = MaskedArray
    result = a.view(cls)
    result._mask = condition
    return result
 
 
###############################################################################
#                            Printing options                                 #
###############################################################################

def masked_invalid(a, None)
    if mask is not None:
        condition = ~(np.isfinite(getdata(a)))
        if mask is not nomask:
            condition |= mask
        cls = type(a)
    else:
        condition = ~(np.isfinite(a))
        cls = MaskedArray
    result = a.view(cls)
    result._mask = condition
    return result
 
 
###############################################################################
#                            Printing options                                 #
###############################################################################

def masked_invalid(a, None)
    if mask is not None:
        condition = ~(np.isfinite(getdata(a)))
        if mask is not nomask:
            condition |= mask
        cls = type(a)
    else:
        condition = ~(np.isfinite(a))
        cls = MaskedArray
    result = a.view(cls)
    result._mask = condition
    return result
 
 
###############################################################################
#                            Printing options                                 #
###############################################################################

def masked_invalid(a, None)
    if mask is not None:
        condition = ~(np.isfinite(getdata(a)))
        if mask is not nomask:
            condition |= mask
        cls = type(a)
    else:
        condition = ~(np.isfinite(a))
        cls = MaskedArray
    result = a.view(cls)
    result._mask = condition
    return result
 
 
###############################################################################
#                            Printing options                                 #
###############################################################################