
.. DO NOT EDIT.
.. THIS FILE WAS AUTOMATICALLY GENERATED BY SPHINX-GALLERY.
.. TO MAKE CHANGES, EDIT THE SOURCE PYTHON FILE:
.. "auto_examples/applications/plot_model_complexity_influence.py"
.. LINE NUMBERS ARE GIVEN BELOW.

.. only:: html

    .. note::
        :class: sphx-glr-download-link-note

        Click :ref:`here <sphx_glr_download_auto_examples_applications_plot_model_complexity_influence.py>`
        to download the full example code

.. rst-class:: sphx-glr-example-title

.. _sphx_glr_auto_examples_applications_plot_model_complexity_influence.py:


==========================
Model Complexity Influence
==========================

Demonstrate how model complexity influences both prediction accuracy and
computational performance.

The dataset is the Boston Housing dataset (resp. 20 Newsgroups) for
regression (resp. classification).

For each class of models we make the model complexity vary through the choice
of relevant model parameters and measure the influence on both computational
performance (latency) and predictive power (MSE or Hamming Loss).

.. GENERATED FROM PYTHON SOURCE LINES 16-170


.. rst-class:: sphx-glr-script-out

.. code-block:: pytb

    Traceback (most recent call last):
      File "/build/scikit-learn-ZSX7SD/scikit-learn-0.23.2/examples/applications/plot_model_complexity_influence.py", line 128, in <module>
        classification_data = generate_data('classification', sparse=True)
      File "/build/scikit-learn-ZSX7SD/scikit-learn-0.23.2/examples/applications/plot_model_complexity_influence.py", line 50, in generate_data
        X, y = datasets.fetch_20newsgroups_vectorized(subset='all',
      File "/build/scikit-learn-ZSX7SD/scikit-learn-0.23.2/.pybuild/cpython3_3.10/build/sklearn/utils/validation.py", line 72, in inner_f
        return f(**kwargs)
      File "/build/scikit-learn-ZSX7SD/scikit-learn-0.23.2/.pybuild/cpython3_3.10/build/sklearn/datasets/_twenty_newsgroups.py", line 419, in fetch_20newsgroups_vectorized
        data_train = fetch_20newsgroups(data_home=data_home,
      File "/build/scikit-learn-ZSX7SD/scikit-learn-0.23.2/.pybuild/cpython3_3.10/build/sklearn/utils/validation.py", line 72, in inner_f
        return f(**kwargs)
      File "/build/scikit-learn-ZSX7SD/scikit-learn-0.23.2/.pybuild/cpython3_3.10/build/sklearn/datasets/_twenty_newsgroups.py", line 258, in fetch_20newsgroups
        cache = _download_20newsgroups(target_dir=twenty_home,
      File "/build/scikit-learn-ZSX7SD/scikit-learn-0.23.2/.pybuild/cpython3_3.10/build/sklearn/datasets/_twenty_newsgroups.py", line 74, in _download_20newsgroups
        archive_path = _fetch_remote(ARCHIVE, dirname=target_dir)
      File "/build/scikit-learn-ZSX7SD/scikit-learn-0.23.2/.pybuild/cpython3_3.10/build/sklearn/datasets/_base.py", line 1181, in _fetch_remote
        urlretrieve(remote.url, file_path)
      File "/usr/lib/python3.10/urllib/request.py", line 241, in urlretrieve
        with contextlib.closing(urlopen(url, data)) as fp:
      File "/usr/lib/python3.10/urllib/request.py", line 216, in urlopen
        return opener.open(url, data, timeout)
      File "/usr/lib/python3.10/urllib/request.py", line 519, in open
        response = self._open(req, data)
      File "/usr/lib/python3.10/urllib/request.py", line 536, in _open
        result = self._call_chain(self.handle_open, protocol, protocol +
      File "/usr/lib/python3.10/urllib/request.py", line 496, in _call_chain
        result = func(*args)
      File "/usr/lib/python3.10/urllib/request.py", line 1391, in https_open
        return self.do_open(http.client.HTTPSConnection, req,
      File "/usr/lib/python3.10/urllib/request.py", line 1351, in do_open
        raise URLError(err)
    urllib.error.URLError: <urlopen error [Errno -2] Name or service not known>






|

.. code-block:: default


    print(__doc__)

    # Author: Eustache Diemert <eustache@diemert.fr>
    # License: BSD 3 clause

    import time
    import numpy as np
    import matplotlib.pyplot as plt
    from mpl_toolkits.axes_grid1.parasite_axes import host_subplot
    from mpl_toolkits.axisartist.axislines import Axes
    from scipy.sparse.csr import csr_matrix

    from sklearn import datasets
    from sklearn.utils import shuffle
    from sklearn.metrics import mean_squared_error
    from sklearn.svm import NuSVR
    from sklearn.ensemble import GradientBoostingRegressor
    from sklearn.linear_model import SGDClassifier
    from sklearn.metrics import hamming_loss

    # #############################################################################
    # Routines


    # Initialize random generator
    np.random.seed(0)


    def generate_data(case, sparse=False):
        """Generate regression/classification data."""
        if case == 'regression':
            X, y = datasets.load_boston(return_X_y=True)
        elif case == 'classification':
            X, y = datasets.fetch_20newsgroups_vectorized(subset='all',
                                                          return_X_y=True)
        X, y = shuffle(X, y)
        offset = int(X.shape[0] * 0.8)
        X_train, y_train = X[:offset], y[:offset]
        X_test, y_test = X[offset:], y[offset:]
        if sparse:
            X_train = csr_matrix(X_train)
            X_test = csr_matrix(X_test)
        else:
            X_train = np.array(X_train)
            X_test = np.array(X_test)
        y_test = np.array(y_test)
        y_train = np.array(y_train)
        data = {'X_train': X_train, 'X_test': X_test, 'y_train': y_train,
                'y_test': y_test}
        return data


    def benchmark_influence(conf):
        """
        Benchmark influence of :changing_param: on both MSE and latency.
        """
        prediction_times = []
        prediction_powers = []
        complexities = []
        for param_value in conf['changing_param_values']:
            conf['tuned_params'][conf['changing_param']] = param_value
            estimator = conf['estimator'](**conf['tuned_params'])
            print("Benchmarking %s" % estimator)
            estimator.fit(conf['data']['X_train'], conf['data']['y_train'])
            conf['postfit_hook'](estimator)
            complexity = conf['complexity_computer'](estimator)
            complexities.append(complexity)
            start_time = time.time()
            for _ in range(conf['n_samples']):
                y_pred = estimator.predict(conf['data']['X_test'])
            elapsed_time = (time.time() - start_time) / float(conf['n_samples'])
            prediction_times.append(elapsed_time)
            pred_score = conf['prediction_performance_computer'](
                conf['data']['y_test'], y_pred)
            prediction_powers.append(pred_score)
            print("Complexity: %d | %s: %.4f | Pred. Time: %fs\n" % (
                complexity, conf['prediction_performance_label'], pred_score,
                elapsed_time))
        return prediction_powers, prediction_times, complexities


    def plot_influence(conf, mse_values, prediction_times, complexities):
        """
        Plot influence of model complexity on both accuracy and latency.
        """
        plt.figure(figsize=(12, 6))
        host = host_subplot(111, axes_class=Axes)
        plt.subplots_adjust(right=0.75)
        par1 = host.twinx()
        host.set_xlabel('Model Complexity (%s)' % conf['complexity_label'])
        y1_label = conf['prediction_performance_label']
        y2_label = "Time (s)"
        host.set_ylabel(y1_label)
        par1.set_ylabel(y2_label)
        p1, = host.plot(complexities, mse_values, 'b-', label="prediction error")
        p2, = par1.plot(complexities, prediction_times, 'r-',
                        label="latency")
        host.legend(loc='upper right')
        host.axis["left"].label.set_color(p1.get_color())
        par1.axis["right"].label.set_color(p2.get_color())
        plt.title('Influence of Model Complexity - %s' % conf['estimator'].__name__)
        plt.show()


    def _count_nonzero_coefficients(estimator):
        a = estimator.coef_.toarray()
        return np.count_nonzero(a)

    # #############################################################################
    # Main code
    regression_data = generate_data('regression')
    classification_data = generate_data('classification', sparse=True)
    configurations = [
        {'estimator': SGDClassifier,
         'tuned_params': {'penalty': 'elasticnet', 'alpha': 0.001, 'loss':
                          'modified_huber', 'fit_intercept': True, 'tol': 1e-3},
         'changing_param': 'l1_ratio',
         'changing_param_values': [0.25, 0.5, 0.75, 0.9],
         'complexity_label': 'non_zero coefficients',
         'complexity_computer': _count_nonzero_coefficients,
         'prediction_performance_computer': hamming_loss,
         'prediction_performance_label': 'Hamming Loss (Misclassification Ratio)',
         'postfit_hook': lambda x: x.sparsify(),
         'data': classification_data,
         'n_samples': 30},
        {'estimator': NuSVR,
         'tuned_params': {'C': 1e3, 'gamma': 2 ** -15},
         'changing_param': 'nu',
         'changing_param_values': [0.1, 0.25, 0.5, 0.75, 0.9],
         'complexity_label': 'n_support_vectors',
         'complexity_computer': lambda x: len(x.support_vectors_),
         'data': regression_data,
         'postfit_hook': lambda x: x,
         'prediction_performance_computer': mean_squared_error,
         'prediction_performance_label': 'MSE',
         'n_samples': 30},
        {'estimator': GradientBoostingRegressor,
         'tuned_params': {'loss': 'ls'},
         'changing_param': 'n_estimators',
         'changing_param_values': [10, 50, 100, 200, 500],
         'complexity_label': 'n_trees',
         'complexity_computer': lambda x: x.n_estimators,
         'data': regression_data,
         'postfit_hook': lambda x: x,
         'prediction_performance_computer': mean_squared_error,
         'prediction_performance_label': 'MSE',
         'n_samples': 30},
    ]
    for conf in configurations:
        prediction_performances, prediction_times, complexities = \
            benchmark_influence(conf)
        plot_influence(conf, prediction_performances, prediction_times,
                       complexities)


.. rst-class:: sphx-glr-timing

   **Total running time of the script:** ( 0 minutes  0.107 seconds)


.. _sphx_glr_download_auto_examples_applications_plot_model_complexity_influence.py:


.. only :: html

 .. container:: sphx-glr-footer
    :class: sphx-glr-footer-example



  .. container:: sphx-glr-download sphx-glr-download-python

     :download:`Download Python source code: plot_model_complexity_influence.py <plot_model_complexity_influence.py>`



  .. container:: sphx-glr-download sphx-glr-download-jupyter

     :download:`Download Jupyter notebook: plot_model_complexity_influence.ipynb <plot_model_complexity_influence.ipynb>`


.. only:: html

 .. rst-class:: sphx-glr-signature

    `Gallery generated by Sphinx-Gallery <https://sphinx-gallery.github.io>`_
