# SPDX-License-Identifier: BSD-3-Clause
"""Nearest Neighbor Regression
adapted from https://github.com/scikit-learn/scikit-learn/blob/0.21.X/sklearn/neighbors/regression.py"""
# Authors: Jake Vanderplas <vanderplas@astro.washington.edu>
# Fabian Pedregosa <fabian.pedregosa@inria.fr>
# Alexandre Gramfort <alexandre.gramfort@inria.fr>
# Sparseness support by Lars Buitinck
# Multi-output support by Arnaud Joly <a.joly@ulg.ac.be>
# Empty radius support by Andreas Bjerre-Nielsen
# Hubness support by Roman Feldbauer <roman.feldbauer@univie.ac.at>
#
# License: BSD 3 clause (C) INRIA, University of Amsterdam,
# University of Copenhagen
import warnings
import numpy as np
from scipy.sparse import issparse
from sklearn.base import RegressorMixin
from sklearn.utils import check_array
from .base import _get_weights, _check_weights, NeighborsBase, KNeighborsMixin
from .base import RadiusNeighborsMixin, SupervisedFloatMixin
[docs]class KNeighborsRegressor(NeighborsBase, KNeighborsMixin,
SupervisedFloatMixin,
RegressorMixin):
"""Regression based on k-nearest neighbors.
The target is predicted by local interpolation of the targets
associated of the nearest neighbors in the training set.
Read more in the `scikit-learn User Guide <https://scikit-learn.org/stable/modules/neighbors.html#regression>`_.
Parameters
----------
n_neighbors: int, optional (default = 5)
Number of neighbors to use by default for :meth:`kneighbors` queries.
weights: str or callable
weight function used in prediction. Possible values:
- 'uniform': uniform weights. All points in each neighborhood
are weighted equally.
- 'distance': weight points by the inverse of their distance.
in this case, closer neighbors of a query point will have a
greater influence than neighbors which are further away.
- [callable]: a user-defined function which accepts an
array of distances, and returns an array of the same shape
containing the weights.
Uniform weights are used by default.
algorithm : {'auto', 'hnsw', 'lsh', 'falconn_lsh', 'onng', 'rptree', 'ball_tree', 'kd_tree', 'brute'}, optional
Algorithm used to compute the nearest neighbors:
- 'hnsw' will use :class:`HNSW`
- 'lsh' will use :class:`PuffinnLSH`
- 'falconn_lsh' will use :class:`FalconnLSH`
- 'onng' will use :class:`NNG`
- 'rptree' will use :class:`RandomProjectionTree`
- 'ball_tree' will use :class:`BallTree`
- 'kd_tree' will use :class:`KDTree`
- 'brute' will use a brute-force search.
- 'auto' will attempt to decide the most appropriate exact algorithm
based on the values passed to :meth:`fit` method. This will not
select an approximate nearest neighbor algorithm.
Note: fitting on sparse input will override the setting of
this parameter, using brute force.
algorithm_params: dict, optional
Override default parameters of the NN algorithm.
For example, with algorithm='lsh' and algorithm_params={n_candidates: 100}
one hundred approximate neighbors are retrieved with LSH.
If parameter hubness is set, the candidate neighbors are further reordered
with hubness reduction.
Finally, n_neighbors objects are used from the (optionally reordered) candidates.
hubness: {'mutual_proximity', 'local_scaling', 'dis_sim_local', None}, optional
Hubness reduction algorithm
- 'mutual_proximity' or 'mp' will use :class:`MutualProximity`
- 'local_scaling' or 'ls' will use :class:`LocalScaling`
- 'dis_sim_local' or 'dsl' will use :class:`DisSimLocal`
If None, no hubness reduction will be performed (=vanilla kNN).
hubness_params: dict, optional
Override default parameters of the selected hubness reduction algorithm.
For example, with hubness='mp' and hubness_params={'method': 'normal'}
a mutual proximity variant is used, which models distance distributions
with independent Gaussians.
leaf_size: int, optional (default = 30)
Leaf size passed to BallTree or KDTree. This can affect the
speed of the construction and query, as well as the memory
required to store the tree. The optimal value depends on the
nature of the problem.
p: integer, optional (default = 2)
Power parameter for the Minkowski metric. When p = 1, this is
equivalent to using manhattan_distance (l1), and euclidean_distance
(l2) for p = 2. For arbitrary p, minkowski_distance (l_p) is used.
metric: string or callable, default 'minkowski'
the distance metric to use for the tree. The default metric is
minkowski, and with p=2 is equivalent to the standard Euclidean
metric. See the documentation of the DistanceMetric class for a
list of available metrics.
metric_params: dict, optional (default = None)
Additional keyword arguments for the metric function.
n_jobs: int or None, optional (default=None)
The number of parallel jobs to run for neighbors search.
``None`` means 1 unless in a :obj:`joblib.parallel_backend` context.
``-1`` means using all processors. See scikit-learn
`Glossary <https://scikit-learn.org/stable/glossary.html#term-n-jobs>`_
for more details.
Doesn't affect :meth:`fit` method.
Examples
--------
>>> X = [[0], [1], [2], [3]]
>>> y = [0, 0, 1, 1]
>>> from skhubness.neighbors import KNeighborsRegressor
>>> neigh = KNeighborsRegressor(n_neighbors=2)
>>> neigh.fit(X, y) # doctest: +ELLIPSIS
KNeighborsRegressor(...)
>>> print(neigh.predict([[1.5]]))
[0.5]
See also
--------
NearestNeighbors
RadiusNeighborsRegressor
KNeighborsClassifier
RadiusNeighborsClassifier
Notes
-----
See `Nearest Neighbors <https://scikit-learn.org/stable/modules/neighbors.html#neighbors>`_
in the scikit-learn online documentation for a discussion
of the choice of ``algorithm`` and ``leaf_size``.
.. warning::
Regarding the Nearest Neighbors algorithms, if it is found that two
neighbors, neighbor `k+1` and `k`, have identical distances but
different labels, the results will depend on the ordering of the
training data.
https://en.wikipedia.org/wiki/K-nearest_neighbor_algorithm
"""
[docs] def __init__(self, n_neighbors=5, weights='uniform',
algorithm: str = 'auto', algorithm_params: dict = None,
hubness: str = None, hubness_params: dict = None,
leaf_size=30,
p=2, metric='minkowski', metric_params=None, n_jobs=None,
**kwargs):
super().__init__(
n_neighbors=n_neighbors,
algorithm=algorithm,
algorithm_params=algorithm_params,
hubness=hubness,
hubness_params=hubness_params,
leaf_size=leaf_size, metric=metric, p=p,
metric_params=metric_params, n_jobs=n_jobs, **kwargs)
self.weights = _check_weights(weights)
[docs] def predict(self, X):
"""Predict the target for the provided data
Parameters
----------
X: array-like, shape (n_query, n_features), \
or (n_query, n_indexed) if metric == 'precomputed'
Test samples.
Returns
-------
y: array of int, shape = [n_samples] or [n_samples, n_outputs]
Target values
"""
if issparse(X) and self.metric == 'precomputed':
raise ValueError(
"Sparse matrices not supported for prediction with "
"precomputed kernels. Densify your matrix."
)
X = check_array(X, accept_sparse='csr')
neigh_dist, neigh_ind = self.kneighbors(X)
weights = _get_weights(neigh_dist, self.weights)
_y = self._y
if _y.ndim == 1:
_y = _y.reshape((-1, 1))
if weights is None:
y_pred = np.mean(_y[neigh_ind], axis=1)
else:
y_pred = np.empty((X.shape[0], _y.shape[1]), dtype=np.float64)
denom = np.sum(weights, axis=1)
for j in range(_y.shape[1]):
num = np.sum(_y[neigh_ind, j] * weights, axis=1)
y_pred[:, j] = num / denom
if self._y.ndim == 1:
y_pred = y_pred.ravel()
return y_pred
[docs]class RadiusNeighborsRegressor(NeighborsBase, RadiusNeighborsMixin,
SupervisedFloatMixin,
RegressorMixin):
"""Regression based on neighbors within a fixed radius.
The target is predicted by local interpolation of the targets
associated of the nearest neighbors in the training set.
Read more in the `scikit-learn User Guide <https://scikit-learn.org/stable/modules/neighbors.html#regression>`_.
Parameters
----------
radius: float, optional (default = 1.0)
Range of parameter space to use by default for :meth:`radius_neighbors`
queries.
weights: str or callable
weight function used in prediction. Possible values:
- 'uniform': uniform weights. All points in each neighborhood
are weighted equally.
- 'distance': weight points by the inverse of their distance.
in this case, closer neighbors of a query point will have a
greater influence than neighbors which are further away.
- [callable]: a user-defined function which accepts an
array of distances, and returns an array of the same shape
containing the weights.
Uniform weights are used by default.
algorithm: {'auto', 'falconn_lsh', 'ball_tree', 'kd_tree', 'brute'}, optional
Algorithm used to compute the nearest neighbors:
- 'falconn_lsh' will use :class:`FalconnLSH`
- 'ball_tree' will use :class:`BallTree`
- 'kd_tree' will use :class:`KDTree`
- 'brute' will use a brute-force search.
- 'auto' will attempt to decide the most appropriate algorithm
based on the values passed to :meth:`fit` method.
Note: fitting on sparse input will override the setting of
this parameter, using brute force.
algorithm_params: dict, optional
Override default parameters of the NN algorithm.
For example, with algorithm='lsh' and algorithm_params={n_candidates: 100}
one hundred approximate neighbors are retrieved with LSH.
If parameter hubness is set, the candidate neighbors are further reordered
with hubness reduction.
Finally, n_neighbors objects are used from the (optionally reordered) candidates.
hubness: {'mutual_proximity', 'local_scaling', 'dis_sim_local', None}, optional
Hubness reduction algorithm
- 'mutual_proximity' or 'mp' will use :class:`MutualProximity`
- 'local_scaling' or 'ls' will use :class:`LocalScaling`
- 'dis_sim_local' or 'dsl' will use :class:`DisSimLocal`
If None, no hubness reduction will be performed (=vanilla kNN).
hubness_params: dict, optional
Override default parameters of the selected hubness reduction algorithm.
For example, with hubness='mp' and hubness_params={'method': 'normal'}
a mutual proximity variant is used, which models distance distributions
with independent Gaussians.
leaf_size: int, optional (default = 30)
Leaf size passed to BallTree or KDTree. This can affect the
speed of the construction and query, as well as the memory
required to store the tree. The optimal value depends on the
nature of the problem.
p: integer, optional (default = 2)
Power parameter for the Minkowski metric. When p = 1, this is
equivalent to using manhattan_distance (l1), and euclidean_distance
(l2) for p = 2. For arbitrary p, minkowski_distance (l_p) is used.
metric: string or callable, default 'minkowski'
the distance metric to use for the tree. The default metric is
minkowski, and with p=2 is equivalent to the standard Euclidean
metric. See the documentation of the DistanceMetric class for a
list of available metrics.
metric_params: dict, optional (default = None)
Additional keyword arguments for the metric function.
n_jobs: int or None, optional (default=None)
The number of parallel jobs to run for neighbors search.
``None`` means 1 unless in a :obj:`joblib.parallel_backend` context.
``-1`` means using all processors. See scikit-learn
`Glossary <https://scikit-learn.org/stable/glossary.html#term-n-jobs>`_
for more details.
Examples
--------
>>> X = [[0], [1], [2], [3]]
>>> y = [0, 0, 1, 1]
>>> from skhubness.neighbors import RadiusNeighborsRegressor
>>> neigh = RadiusNeighborsRegressor(radius=1.0)
>>> neigh.fit(X, y) # doctest: +ELLIPSIS
RadiusNeighborsRegressor(...)
>>> print(neigh.predict([[1.5]]))
[0.5]
See also
--------
NearestNeighbors
KNeighborsRegressor
KNeighborsClassifier
RadiusNeighborsClassifier
Notes
-----
See `Nearest Neighbors <https://scikit-learn.org/stable/modules/neighbors.html#neighbors>`_
in the scikit-learn online documentation for a discussion
of the choice of ``algorithm`` and ``leaf_size``.
https://en.wikipedia.org/wiki/K-nearest_neighbor_algorithm
"""
[docs] def __init__(self, radius=1.0, weights='uniform',
algorithm: str = 'auto', algorithm_params: dict = None,
hubness: str = None, hubness_params: dict = None,
leaf_size=30, p=2, metric='minkowski', metric_params=None, n_jobs=None,
**kwargs):
super().__init__(
radius=radius,
algorithm=algorithm,
algorithm_params=algorithm_params,
hubness=hubness,
hubness_params=hubness_params,
leaf_size=leaf_size,
p=p, metric=metric, metric_params=metric_params,
n_jobs=n_jobs, **kwargs)
self.weights = _check_weights(weights)
[docs] def predict(self, X):
"""Predict the target for the provided data
Parameters
----------
X: array-like, shape (n_query, n_features), or (n_query, n_indexed) if metric == 'precomputed'
Test samples.
Returns
-------
y: array of float, shape = [n_samples] or [n_samples, n_outputs]
Target values
"""
X = check_array(X, accept_sparse='csr')
neigh_dist, neigh_ind = self.radius_neighbors(X)
weights = _get_weights(neigh_dist, self.weights)
_y = self._y
if _y.ndim == 1:
_y = _y.reshape((-1, 1))
empty_obs = np.full_like(_y[0], np.nan)
if weights is None:
y_pred = np.array([np.mean(_y[ind, :], axis=0)
if len(ind) else empty_obs
for (i, ind) in enumerate(neigh_ind)])
else:
y_pred = np.array([np.average(_y[ind, :], axis=0,
weights=weights[i])
if len(ind) else empty_obs
for (i, ind) in enumerate(neigh_ind)])
if np.max(np.isnan(y_pred)):
empty_warning_msg = ("One or more samples have no neighbors "
"within specified radius; predicting NaN.")
warnings.warn(empty_warning_msg)
if self._y.ndim == 1:
y_pred = y_pred.ravel()
return y_pred