def _base_sample_scores(X: np.ndarray, rho: float) -> np.ndarray:
    """Compute the log-likelihood of each sample (log-pdf) under the bivariate
    Gaussian copula model.

    Parameters
    ----------
    X : array-like of shape (n_observations, 2)
        An array of bivariate inputs `(u, v)` where each row represents a
        bivariate observation. Both `u` and `v` must be in the interval `[0, 1]`,
        having been transformed to uniform marginals.

    rho : float
        Gaussian copula parameter.

    Returns
    -------
    density : ndarray of shape (n_observations,)
        The log-likelihood of each sample under the fitted copula.

    Raises
    ------
    ValueError
        If rho is not in (-1, 1)
    """
    if not (-1.0 <= rho <= 1.0):
        raise ValueError("rho must be between -1 and 1.")

    # Inverse CDF (ppf) using stdtrit for better performance
    u_inv, v_inv = sp.ndtri(X).T

    # Using np.log1p to avoid loss of precision
    log_density = -0.5 * np.log1p(-(rho**2)) - rho * (
        0.5 * rho * (u_inv**2 + v_inv**2) - u_inv * v_inv
    ) / (1 - rho**2)
    return log_density

def _neg_log_likelihood(rho: float, X: np.ndarray) -> float:
    """Negative log-likelihood function for optimization.

    Parameters
    ----------
    X : array-like of shape (n_observations, 2)
        An array of bivariate inputs `(u, v)` where each row represents a
        bivariate observation. Both `u` and `v` must be in the interval `[0, 1]`,
        having been transformed to uniform marginals.

    rho : float
        Correlation copula parameter.

    Returns
    -------
    value : float
        The negative log-likelihood value.
    """
    return -np.sum(_base_sample_scores(X=X, rho=rho))


def _base_sample_scores(X: np.ndarray, theta: float) -> np.ndarray:
    r"""Compute the log-likelihood of each sample (log-pdf) under the bivariate Gumbel
    copula.

    Parameters
    ----------
    X : array-like of shape (n_observations, 2)
         Bivariate samples `(u, v)`, with each component in [0,1].

    theta : float
         The dependence parameter (must be greater than 1).

    Returns
    -------
    logpdf : ndarray of shape (n_observations,)
         Log-likelihood values for each observation.
    """
    if theta <= 1:
        raise ValueError("Theta must be greater than 1 for the Gumbel copula.")
    Z = -np.log(X)
    s = np.power(np.power(Z, theta).sum(axis=1), 1 / theta)
    s = np.clip(s, a_min=1e-10, a_max=None)
    log_density = (
        -s
        + np.log(s + theta - 1)
        + (1 - 2 * theta) * np.log(s)
        + (theta - 1) * np.log(Z.prod(axis=1))
        + Z.sum(axis=1)
    )
    return log_density

def _neg_log_likelihood(theta: float, X: np.ndarray) -> float:
    """Negative log-likelihood function for the Gumbel copula.

    Parameters
    ----------
    X : array-like of shape (n_observations, 2)
        An array of bivariate inputs `(u, v)` where each row represents a
        bivariate observation. Both `u` and `v` must be in the interval [0, 1],
        having been transformed to uniform marginals.

    theta : float
        The dependence parameter (must be greater than 1).

    Returns
    -------
    value : float
        The negative log-likelihood value.
    """
    return -np.sum(_base_sample_scores(X=X, theta=theta))

def _apply_copula_rotation(X: npt.ArrayLike, rotation: CopulaRotation) -> np.ndarray:
    r"""Apply a bivariate copula rotation using the standard (clockwise) convention.

    The transformations are defined as follows:

    - `CopulaRotation.R0` (0°): :math:`(u, v) \mapsto (u, v)`
    - `CopulaRotation.R90` (90°): :math:`(u, v) \mapsto (v,\, 1 - u)`
    - `CopulaRotation.R180` (180°): :math:`(u, v) \mapsto (1 - u,\, 1 - v)`
    - `CopulaRotation.R270` (270°): :math:`(u, v) \mapsto (1 - v,\, u)`

    Parameters
    ----------
    X : array-like of shape (n_observations, 2)
        An array of bivariate inputs `(u, v)` where each row represents a
        bivariate observation.

    rotation : CopulaRotation
        The rotation to apply to the copula (default is no rotation).

    Returns
    -------
    rotated_X: ndarray of shape (n_observations, 2)
        The rotated data array.
    """
    match rotation:
        case CopulaRotation.R0:
            # No rotation
            pass
        case CopulaRotation.R90:
            # (u, v) -> (v, 1 - u)
            X = np.column_stack([X[:, 1], 1.0 - X[:, 0]])
        case CopulaRotation.R180:
            # (u, v) -> (1 - u, 1 - v)
            X = 1.0 - X
        case CopulaRotation.R270:
            # (u, v) -> (1 - v, u)
            X = np.column_stack([1.0 - X[:, 1], X[:, 0]])
        case _:
            raise ValueError(f"Unsupported rotation: {rotation}")
    return X

def _apply_margin_swap(X: np.ndarray, first_margin: bool) -> np.ndarray:
    """
    Swap the columns of X if first_margin is False.

    If first_margin is True, X is returned unchanged; otherwise, the columns
    of X are swapped.

    Parameters
    ----------
    X : ndarray of shape (n_observations, 2)
        A 2D array of bivariate inputs (u, v).
    first_margin : bool
        If True, no swap is performed; if False, the columns of X are swapped.

    Returns
    -------
    X_swapped : ndarray of shape (n_observations, 2)
        The data array with columns swapped if first_margin is False.
    """
    assert X.ndim == 2
    assert X.shape[1] == 2
    if first_margin:
        return X[:, [1, 0]]
    return X

def _base_partial_derivative(
    X: np.ndarray, first_margin: bool, theta: float
) -> np.ndarray:
    r"""
    Compute the partial derivative (h-function) for the unrotated Gumbel copula.

    For Gumbel, the copula is defined as:

    .. math::
        C(u,v)=\exp\Bigl(-\Bigl[(-\ln u)^{\theta}+(-\ln v)^{\theta}\Bigr]^{1/\theta}\Bigr).

    The partial derivative with respect to v is:

    .. math::
        \frac{\partial C(u,v)}{\partial v}
          = C(u,v)\,\Bigl[(-\ln u)^{\theta}+(-\ln v)^{\theta}\Bigr]^{\frac{1}{\theta}-1}
            \,(-\ln v)^{\theta-1}\,\frac{1}{v}.

    Parameters
    ----------
    X : array-like of shape (n_observations, 2)
         An array of bivariate inputs `(u, v)` with values in [0, 1].

    first_margin : bool, default=False
         If True, compute with respect to u (by swapping margins); otherwise,
         compute with respect to v.

    theta : float
         The dependence parameter (must be > 1).

    Returns
    -------
    p : ndarray of shape (n_observations,)
         The computed h-function values.
    """
    X = _apply_margin_swap(X, first_margin=first_margin)
    _, v = X.T
    x, y = -np.log(X).T
    p = (
        np.exp(-np.power(np.power(x, theta) + np.power(y, theta), 1.0 / theta))
        * np.power(np.power(x / y, theta) + 1.0, 1.0 / theta - 1.0)
        / v
    )
    return p

def _apply_rotation_partial_derivatives(
    func: Callable,
    X: np.ndarray,
    rotation: CopulaRotation,
    first_margin: bool,
    **kwargs,
) -> np.ndarray:
    """
    Apply a copula rotation to X and compute the corresponding partial derivatives.

    This function rotates the data X using the specified rotation and then computes
    the partial derivative (h-function) using the provided function. The result is then
    adjusted according to the rotation and the margin of interest.

    Parameters
    ----------
    func : Callable
        A function that computes the partial derivative (h-function) given X, the
        margin, and any additional keyword arguments.

    X : ndarray of shape (n_observations, 2)
        A 2D array of bivariate inputs.

    rotation : CopulaRotation
        The rotation to apply.

    first_margin : bool
        If True, compute the partial derivative with respect to the first margin;
        otherwise, compute it with respect to the second margin.

    **kwargs
        Additional keyword arguments to pass to the partial derivative function.

    Returns
    -------
    z : ndarray of shape (n_observations,)
        The transformed partial derivative values after applying the rotation.
    """
    rotated_X = _apply_copula_rotation(X, rotation=rotation)

    match rotation:
        case CopulaRotation.R0:
            z = func(X=rotated_X, first_margin=first_margin, **kwargs)
        case CopulaRotation.R90:
            if first_margin:
                z = func(X=rotated_X, first_margin=not first_margin, **kwargs)
            else:
                z = 1 - func(X=rotated_X, first_margin=not first_margin, **kwargs)
        case CopulaRotation.R180:
            z = 1 - func(X=rotated_X, first_margin=first_margin, **kwargs)
        case CopulaRotation.R270:
            if first_margin:
                z = 1 - func(X=rotated_X, first_margin=not first_margin, **kwargs)
            else:
                z = func(X=rotated_X, first_margin=not first_margin, **kwargs)
        case _:
            raise ValueError(f"Unsupported rotation: {rotation}")
    return z


def _base_sample_scores(X: np.ndarray, theta: float) -> np.ndarray:
    """Compute the log-likelihood of each sample (log-pdf) under the bivariate
    Joe copula model.

    Parameters
    ----------
    X : array-like of shape (n_observations, 2)
        An array of bivariate inputs `(u, v)` where each row represents a
        bivariate observation. Both `u` and `v` must be in the interval `[0, 1]`,
        having been transformed to uniform marginals.

    theta : float
        The dependence parameter (must be greater than 1).

    Returns
    -------
    density : ndarray of shape (n_observations,)
        The log-likelihood of each sample under the fitted copula.

    Raises
    ------
    ValueError
        If rho is not in (-1, 1) or dof is not positive.
    """
    if theta <= 1.0:
        raise ValueError("Theta must be greater than 1 for the Joe copula.")

    # log-space transformation to improve stability near 0  or 1
    x, y = np.log1p(-X).T
    x_y = x + y
    d = np.exp(x * theta) + np.exp(y * theta) - np.exp(x_y * theta)
    log_density = (
        (1.0 / theta - 2.0) * np.log(d) + x_y * (theta - 1.0) + np.log(theta - 1.0 + d)
    )
    return log_density

def _neg_log_likelihood(theta: float, X: np.ndarray) -> float:
    """Negative log-likelihood function for optimization.

    Parameters
    ----------
     X : array-like of shape (n_observations, 2)
         An array of bivariate inputs `(u, v)` where each row represents a
         bivariate observation. Both `u` and `v` must be in the interval `[0, 1]`,
         having been transformed to uniform marginals.

    theta : float
         The dependence parameter (must be greater than 1).

    Returns
    -------
     value : float
         The negative log-likelihood value.
    """
    return -np.sum(_base_sample_scores(X=X, theta=theta))

def _base_partial_derivative(
    X: np.ndarray, first_margin: bool, theta: float
) -> np.ndarray:
    r"""Compute the h-function (partial derivative) for the bivariate unrotated
    Joe copula with respect to a specified margin.

    Parameters
    ----------
    X : array-like of shape (n_observations, 2)
        An array of bivariate inputs `(u, v)` where each row represents a
        bivariate observation. Both `u` and `v` must be in the interval `[0, 1]`,
        having been transformed to uniform marginals.

    first_margin : bool, default=False
        If True, compute the partial derivative with respect to the first
        margin `u`; otherwise, compute the partial derivative with respect to the
        second margin `v`.

    theta : float
        The dependence parameter (must be greater than 1).

    Returns
    -------
      : ndarray of shape (n_observations,)
        h-function values :math:`h(u \mid v) \;=\; p` for each observation in X.
    """
    X = _apply_margin_swap(X, first_margin=first_margin)
    x, y = np.power(1 - X, theta).T
    p = np.power(1 + x / y - x, 1 / theta - 1) * (1.0 - x)
    return p


def _base_sample_scores(X: np.ndarray, theta: float) -> np.ndarray:
    r"""Compute the log-likelihood of each sample (log-pdf) under the bivariate Clayton
    copula.

    Parameters
    ----------
    X : array-like of shape (n_observations, 2)
        Bivariate samples `(u, v)`, with each component in [0,1].

    theta : float
        The dependence parameter (must be greater than 0).

    Returns
    -------
    logpdf : ndarray of shape (n_observations,)
        Log-likelihood values for each observation.

    Raises
    ------
    ValueError
        If theta is not greater than 0.
    """
    if theta <= 0:
        raise ValueError("Theta must be greater than 1 for the Clayton copula.")

    x, y = np.log(X).T

    log_density = (
        np.log1p(theta)
        - (2.0 + 1.0 / theta) * np.log1p(np.expm1(-theta * x) + np.expm1(-theta * y))
        - (1.0 + theta) * (x + y)
    )
    return log_density

def _neg_log_likelihood(theta: float, X: np.ndarray) -> float:
    """Negative log-likelihood function for the Clayton copula.

    Parameters
    ----------
    X : array-like of shape (n_observations, 2)
        An array of bivariate inputs `(u, v)` where each row represents a
        bivariate observation. Both `u` and `v` must be in the interval `[0, 1]`,
        having been transformed to uniform marginals.

    theta : float
        The dependence parameter (must be greater than 0).

    Returns
    -------
    value : float
        The negative log-likelihood value.
    """
    return -np.sum(_base_sample_scores(X=X, theta=theta))


def _sample_scores(X: np.ndarray, rho: float, dof: float) -> np.ndarray:
    """Compute the log-likelihood of each sample (log-pdf) under the bivariate
    Gaussian copula model.

    Parameters
    ----------
    X : array-like of shape (n_observations, 2)
        An array of bivariate inputs `(u, v)` where each row represents a
        bivariate observation. Both `u` and `v` must be in the interval `[0, 1]`,
        having been transformed to uniform marginals.

    rho : float
        Gaussian copula parameter.

    Returns
    -------
    density : ndarray of shape (n_observations,)
        The log-likelihood of each sample under the fitted copula.

    Raises
    ------
    ValueError
        If rho is not in (-1, 1) or dof is not positive.
    """
    if not (-1.0 <= rho <= 1.0):
        raise ValueError("rho must be between -1 and 1.")
    if not 1.0 <= dof <= 50:
        raise ValueError("Degrees of freedom `dof` must be between 1 and 50.")

    # Inverse CDF (ppf) using stdtrit for better performance
    x, y = sp.stdtrit(dof, X).T

    a = 1.0 - rho**2
    log_density = (
        sp.gammaln((dof + 2.0) / 2.0)
        + sp.gammaln(dof / 2.0)
        - 2.0 * sp.gammaln((dof + 1.0) / 2.0)
        - np.log(a) / 2
        + (dof + 1.0) / 2.0 * (np.log1p(x**2 / dof) + np.log1p(y**2 / dof))
        - (dof + 2.0) / 2.0 * np.log1p((x**2 - 2 * rho * x * y + y**2) / a / dof)
    )
    return log_density

def _neg_log_likelihood(dof: float, rho: float, X: np.ndarray) -> float:
    """Negative log-likelihood function for optimization.

    Parameters
    ----------
    X : array-like of shape (n_observations, 2)
        An array of bivariate inputs `(u, v)` where each row represents a
        bivariate observation. Both `u` and `v` must be in the interval `[0, 1]`,
        having been transformed to uniform marginals.

    rho : float
        Correlation copula parameter.

    dof : float
        Degree of freedom copula parameter.

    Returns
    -------
    value : float
        The negative log-likelihood value.
    """
    return -np.sum(_sample_scores(X=X, rho=rho, dof=dof))


def _apply_rotation_cdf(
    func: Callable, X: np.ndarray, rotation: CopulaRotation, **kwargs
) -> np.ndarray:
    """
    Apply a copula rotation to X and compute the corresponding CDF values.

    Parameters
    ----------
    func : Callable
       A function that computes the CDF given data X and additional keyword arguments.

    X : ndarray of shape (n_observations, 2)
       A 2D array of bivariate inputs.

    rotation : CopulaRotation
       The rotation to apply.

    **kwargs
       Additional keyword arguments to pass to the CDF function.

    Returns
    -------
    rotated_cdf : ndarray of shape (n_observations,)
       The transformed CDF values after applying the rotation.
    """
    rotated_X = _apply_copula_rotation(X, rotation=rotation)
    cdf = func(X=rotated_X, **kwargs)

    match rotation:
        case CopulaRotation.R0:
            pass
        case CopulaRotation.R90:
            cdf = X[:, 1] - cdf
        case CopulaRotation.R180:
            cdf = np.sum(X, axis=1) - 1 + cdf
        case CopulaRotation.R270:
            cdf = X[:, 0] - cdf
        case _:
            raise ValueError(f"Unsupported rotation: {rotation}")

    return cdf


def load_gzip_compressed_csv_data(
    data_filename: str,
    data_module: str = DATA_MODULE,
    encoding="utf-8",
    datetime_index: bool = True,
) -> pd.DataFrame:
    """Load gzip-compressed csv files with `importlib.resources`.

    1) Open resource file with `importlib.resources.open_binary`
    2) Decompress csv file with `gzip.open`
    3) Load decompressed data with `pd.read_csv`

    Parameters
    ----------
    data_filename : str
        Name of gzip-compressed csv file  (`'*.csv.gz'`) to be loaded from
        `data_module/data_file_name`. For example `'SPX500.csv.gz'`.

    data_module : str or module, default='skfolio.datasets.data'
        Module where data lives. The default is `'skfolio.datasets.data'`.

    encoding : str, default="utf-8"
        Name of the encoding that the gzip-decompressed file will be
        decoded with. The default is 'utf-8'.

    datetime_index: bool, default=True
        If this is set to True, the DataFrame index is converted to datetime with
        format="%Y-%m-%d".
        The default is `True`.

    Returns
    -------
    df : DataFrame of shape (n_observations, n_assets)
        DataFrame with each row representing one observation and each column
        representing the asset price of a given observation.
    """
    path = resources.files(data_module).joinpath(data_filename)
    with path.open("rb") as compressed_file:
        compressed_file = gzip.open(compressed_file, mode="rt", encoding=encoding)
        df = pd.read_csv(compressed_file, sep=",", index_col=0)
        if datetime_index:
            df.index = pd.to_datetime(df.index, format="%Y-%m-%d")
        return df

def load_sp500_dataset() -> pd.DataFrame:
    """Load the prices of 20 assets from the S&P 500 Index composition.

    This dataset is composed of the daily prices of 20 assets from the S&P 500
    composition starting from 1990-01-02 up to 2022-12-28.

    The data comes from the Yahoo public API.
    The price is the adjusted close which is the closing price after adjustments for
    all applicable splits and dividend distributions.
    The adjustment uses appropriate split and dividend multipliers, adhering to
    the Center for Research in Security Prices (CRSP) standards.

    ==============   ==================
    Observations     8313
    Assets           20
    ==============   ==================

    Returns
    -------
    df : DataFrame of shape (n_observations, n_assets)
        Prices DataFrame

    Examples
    --------
    >>> from skfolio.datasets import load_sp500_dataset
    >>> prices = load_sp500_dataset()
    >>> prices.head()
                    AAPL     AMD       BAC  ...       UNH       WMT      XOM
    1990-01-02  0.332589  4.1250  11.65625  ...  0.382813  5.890625  12.5000
    1990-01-03  0.334821  4.0000  11.75000  ...  0.375000  5.890625  12.3750
    1990-01-04  0.335938  3.9375  11.50000  ...  0.371094  5.859375  12.2500
    1990-01-05  0.337054  3.8125  11.25000  ...  0.355469  5.796875  12.1875
    1990-01-08  0.339286  3.8125  11.31250  ...  0.347656  5.875000  12.3750
    """
    data_filename = "sp500_dataset.csv.gz"
    df = load_gzip_compressed_csv_data(data_filename)
    return df


def get_cumulative_returns(returns: np.ndarray, compounded: bool = False) -> np.ndarray:
    """Compute the cumulative returns from the returns.
    Non-compounded cumulative returns start at 0.
    Compounded cumulative returns are rescaled to start at 1000.

    Parameters
    ----------
    returns : ndarray of shape (n_observations,)
        Vector of returns.

    compounded : bool, default=False
        If this is set to True, the cumulative returns are compounded otherwise they
        are uncompounded.

    Returns
    -------
    values: ndarray of shape (n_observations,)
        Cumulative returns.
    """
    if compounded:
        cumulative_returns = 1000 * np.cumprod(1 + returns)  # Rescaled to start at 1000
    else:
        cumulative_returns = np.cumsum(returns)
    return cumulative_returns

def get_drawdowns(returns: np.ndarray, compounded: bool = False) -> np.ndarray:
    """Compute the drawdowns' series from the returns.

    Parameters
    ----------
    returns : ndarray of shape (n_observations,)
       Vector of returns.

    compounded : bool, default=False
       If this is set to True, the cumulative returns are compounded otherwise they
       are uncompounded.

    Returns
    -------
    values: ndarray of shape (n_observations,)
       Drawdowns.
    """
    cumulative_returns = get_cumulative_returns(returns=returns, compounded=compounded)
    if compounded:
        drawdowns = cumulative_returns / np.maximum.accumulate(cumulative_returns) - 1
    else:
        drawdowns = cumulative_returns - np.maximum.accumulate(cumulative_returns)
    return drawdowns


def _n_splits(n_folds: int, n_test_folds: int) -> int:
    """Number of splits.

    Parameters
    ----------
    n_folds : int
        Number of folds.

    n_test_folds : int
        Number of test folds.

    Returns
    -------
    n_splits : int
        Number of splits
    """
    return math.comb(n_folds, n_test_folds)

def _n_test_paths(n_folds: int, n_test_folds: int) -> int:
    """Number of test paths that can be reconstructed from the train/test
    combinations.

    Parameters
    ----------
    n_folds : int
        Number of folds.

    n_test_folds : int
        Number of test folds.

    Returns
    -------
    n_splits : int
        Number of test paths.
    """
    return (
        _n_splits(n_folds=n_folds, n_test_folds=n_test_folds) * n_test_folds // n_folds
    )

    def n_test_paths(self) -> int:
        """Number of test paths that can be reconstructed from the train/test
        combinations.
        """
        return _n_test_paths(n_folds=self.n_folds, n_test_folds=self.n_test_folds)


def safe_indexing(
    X: npt.ArrayLike | pd.DataFrame, indices: npt.ArrayLike | None, axis: int = 0
):
    """Return rows, items or columns of X using indices.

    Parameters
    ----------
    X : array-like
        Data from which to sample rows.

    indices : array-like, optional
        Indices of rows or columns.
        The default (`None`) is to select the entire data.

    axis : int, default=0
        The axis along which `X` will be sub-sampled. `axis=0` will select
        rows while `axis=1` will select columns.

    Returns
    -------
    subset :
        Subset of X on axis 0.
    """
    if indices is None:
        return X
    if hasattr(X, "iloc"):
        return X.take(indices, axis=axis)
    if axis == 0:
        return X[indices]
    return X[:, indices]

def safe_split(
    X: npt.ArrayLike,
    y: npt.ArrayLike | None = None,
    indices: np.ndarray | None = None,
    axis: int = 0,
):
    """Create subset of dataset.

    Slice X, y according to indices for cross-validation.

    Parameters
    ----------
    X : array-like
        Data to be indexed.

    y : array-like
        Data to be indexed.

    indices : ndarray of int, optional
        Rows or columns to select from X and y.
        The default (`None`) is to select the entire data.

    axis : int, default=0
        The axis along which `X` will be sub-sampled. `axis=0` will select
        rows while `axis=1` will select columns.

    Returns
    -------
    X_subset : array-like
        Indexed data.

    y_subset : array-like
        Indexed targets.
    """
    X_subset = safe_indexing(X, indices=indices, axis=axis)
    if y is not None:
        y_subset = safe_indexing(y, indices=indices, axis=axis)
    else:
        y_subset = None
    return X_subset, y_subset

def cross_val_predict(
    estimator: skb.BaseEstimator,
    X: npt.ArrayLike,
    y: npt.ArrayLike = None,
    cv: sks.BaseCrossValidator | BaseCombinatorialCV | int | None = None,
    n_jobs: int | None = None,
    method: str = "predict",
    verbose: int = 0,
    params: dict | None = None,
    pre_dispatch: str = "2*n_jobs",
    column_indices: np.ndarray | None = None,
    portfolio_params: dict | None = None,
) -> MultiPeriodPortfolio | Population:
    """Generate cross-validated `Portfolios` estimates.

    The data is split according to the `cv` parameter.
    The optimization estimator is fitted on the training set and portfolios are
    predicted on the corresponding test set.

    For non-combinatorial cross-validation like `Kfold`, the output is the predicted
    :class:`~skfolio.portfolio.MultiPeriodPortfolio` where
    each :class:`~skfolio.portfolio.Portfolio` corresponds to the prediction on each
    train/test pair (`k` portfolios for `Kfold`).

    For combinatorial cross-validation
    like :class:`~skfolio.model_selection.CombinatorialPurgedCV`, the output is the
    predicted :class:`~skfolio.population.Population` of multiple
    :class:`~skfolio.portfolio.MultiPeriodPortfolio` (each test outputs are a
    collection of multiple paths instead of one single path).

    Parameters
    ----------
    estimator : BaseOptimization
        :ref:`Optimization estimators <optimization>` use to fit the data.

    X : array-like of shape (n_observations, n_assets)
        Price returns of the assets.

    y : array-like of shape (n_observations, n_targets), optional
        Target data (optional).
        For example, the price returns of the factors.

    cv : int | cross-validation generator, optional
        Determines the cross-validation splitting strategy.
        Possible inputs for cv are:

        * None, to use the default 5-fold cross validation,
        * int, to specify the number of folds in a `(Stratified)KFold`,
        * `CV splitter`,
        * An iterable that generates (train, test) splits as arrays of indices.

    n_jobs : int, optional
        The number of jobs to run in parallel for `fit` of all `estimators`.
        `None` means 1 unless in a `joblib.parallel_backend` context. -1 means
        using all processors.

    method : str
        Invokes the passed method name of the passed estimator.

    verbose : int, default=0
        The verbosity level.

    params : dict, optional
        Parameters to pass to the underlying estimator's ``fit`` and the CV splitter.

    pre_dispatch : int or str, default='2*n_jobs'
        Controls the number of jobs that get dispatched during parallel
        execution. Reducing this number can be useful to avoid an
        explosion of memory consumption when more jobs get dispatched
        than CPUs can process. This parameter can be:

            * None, in which case all the jobs are immediately
              created and spawned. Use this for lightweight and
              fast-running jobs, to avoid delays due to on-demand
              spawning of the jobs

            * An int, giving the exact number of total jobs that are
              spawned

            * A str, giving an expression as a function of n_jobs,
              as in '2*n_jobs'

    column_indices : ndarray, optional
        Indices of the `X` columns to cross-validate on.

    portfolio_params :  dict, optional
        Additional portfolio parameters passed to `MultiPeriodPortfolio`.

    Returns
    -------
    predictions : MultiPeriodPortfolio | Population
        This is the result of calling `predict`
    """
    params = {} if params is None else params

    X, y = safe_split(X, y, indices=column_indices, axis=1)
    X, y = sku.indexable(X, y)

    if _routing_enabled():
        # For estimators, a MetadataRouter is created in get_metadata_routing
        # methods. For these router methods, we create the router to use
        # `process_routing` on it.
        # noinspection PyTypeChecker
        router = (
            skm.MetadataRouter(owner="cross_validate")
            .add(
                splitter=cv,
                method_mapping=skm.MethodMapping().add(caller="fit", callee="split"),
            )
            .add(
                estimator=estimator,
                method_mapping=skm.MethodMapping().add(caller="fit", callee="fit"),
            )
        )
        try:
            routed_params = skm.process_routing(router, "fit", **params)
        except ske.UnsetMetadataPassedError as e:
            # The default exception would mention `fit` since in the above
            # `process_routing` code, we pass `fit` as the caller. However,
            # the user is not calling `fit` directly, so we change the message
            # to make it more suitable for this case.
            unrequested_params = sorted(e.unrequested_params)
            raise ske.UnsetMetadataPassedError(
                message=(
                    f"{unrequested_params} are passed to `cross_val_predict` but are"
                    " not explicitly set as requested or not requested for"
                    f" cross_validate's estimator: {estimator.__class__.__name__} Call"
                    " `.set_fit_request({{metadata}}=True)` on the estimator for"
                    f" each metadata in {unrequested_params} that you want to use and"
                    " `metadata=False` for not using it. See the Metadata Routing User"
                    " guide <https://scikit-learn.org/stable/metadata_routing.html>"
                    " for more information."
                ),
                unrequested_params=e.unrequested_params,
                routed_params=e.routed_params,
            ) from None
    else:
        routed_params = sku.Bunch()
        routed_params.splitter = sku.Bunch(split={})
        routed_params.estimator = sku.Bunch(fit=params)

    cv = sks.check_cv(cv, y)
    splits = list(cv.split(X, y, **routed_params.splitter.split))

    portfolio_params = {} if portfolio_params is None else portfolio_params.copy()

    # We ensure that the folds are not shuffled
    if not isinstance(cv, BaseCombinatorialCV):
        try:
            if cv.shuffle:
                raise ValueError(
                    "`cross_val_predict` only works with cross-validation setting"
                    " `shuffle=False`"
                )
        except AttributeError:
            # If we cannot find the attribute shuffle, we check if the first folds
            # are shuffled
            for fold in splits[0]:
                if not np.all(np.diff(fold) > 0):
                    raise ValueError(
                        "`cross_val_predict` only works with un-shuffled folds"
                    ) from None

    # We clone the estimator to make sure that all the folds are independent
    # and that it is pickle-able.
    parallel = skp.Parallel(n_jobs=n_jobs, verbose=verbose, pre_dispatch=pre_dispatch)
    # TODO remove when https://github.com/joblib/joblib/issues/1071 is fixed
    # noinspection PyCallingNonCallable
    predictions = parallel(
        skp.delayed(fit_and_predict)(
            sk.clone(estimator),
            X,
            y,
            train=train,
            test=test,
            fit_params=routed_params.estimator.fit,
            method=method,
        )
        for train, test in splits
    )

    if isinstance(cv, BaseCombinatorialCV):
        path_ids = cv.get_path_ids()
        path_nb = np.max(path_ids) + 1
        portfolios = [[] for _ in range(path_nb)]
        for i, prediction in enumerate(predictions):
            for j, p in enumerate(prediction):
                path_id = path_ids[i, j]
                portfolios[path_id].append(p)
        name = portfolio_params.pop("name", "path")
        pred = Population(
            [
                MultiPeriodPortfolio(
                    name=f"{name}_{i}", portfolios=portfolios[i], **portfolio_params
                )
                for i in range(path_nb)
            ]
        )
    else:
        # We need to re-order the test folds in case they were un-ordered by the
        # CV generator.
        # Because the tests folds are not shuffled, we use the first index of each
        # fold to order them.
        test_indices = np.concatenate([test for _, test in splits])
        if np.unique(test_indices, axis=0).shape[0] != test_indices.shape[0]:
            raise ValueError(
                "`cross_val_predict` only works with non-duplicated test indices"
            )
        test_indices = [test for _, test in splits]
        sorted_fold_id = np.argsort([x[0] for x in test_indices])
        pred = MultiPeriodPortfolio(
            portfolios=[predictions[fold_id] for fold_id in sorted_fold_id],
            check_observations_order=False,
            **portfolio_params,
        )

    return pred


def get_data_home(data_home: str | Path | None = None) -> str:
    """Return the path of the skfolio data directory.

    This folder is used by some large dataset loaders to avoid downloading the
    data several times.

    By default, the data directory is set to a folder named 'skfolio_data' in the
    user home folder.

    Alternatively, it can be set by the 'SKFOLIO_DATA' environment
    variable or programmatically by giving an explicit folder path. The '~'
    symbol is expanded to the user home folder.

    If the folder does not already exist, it is automatically created.

    Parameters
    ----------
    data_home : str, optional
        The path to skfolio data directory. If `None`, the default path
        is `~/skfolio_data`.

    Returns
    -------
    data_home: str or path-like, optional
        The path to skfolio data directory.
    """
    if data_home is None:
        data_home = os.environ.get("SKFOLIO_DATA", os.path.join("~", "skfolio_data"))
    data_home = os.path.expanduser(data_home)
    os.makedirs(data_home, exist_ok=True)
    return data_home

def download_dataset(
    data_filename: str,
    data_home: str | Path | None = None,
    download_if_missing: bool = True,
) -> pd.DataFrame:
    """Download and save locally a dataset from the remote GitHub dataset folder.

    Parameters
    ----------
    data_filename : str
        Name of gzip-compressed csv file  (`'*.csv.gz'`) to be loaded from a remote
        GitHub dataset folder.

    data_home : str or path-like, optional
        Specify another download and cache folder for the datasets. By default,
        all skfolio data is stored in `~/skfolio_data` sub-folders.

    download_if_missing : bool, default=True
        If False, raise an OSError if the data is not locally available
        instead of trying to download the data from the source site.
        The default is `True`.

    Returns
    -------
    df : DataFrame of shape (n_observations, n_assets)
        DataFrame with each row representing one observation and each column
        representing the asset price of a given observation.
    """
    # Use a CORS proxy when triggering requests from the browser
    url_prefix = "https://corsproxy.io/?" if sys.platform == "emscripten" else ""
    url = url_prefix + (
        f"https://github.com/skfolio/skfolio-datasets/raw/main/"
        f"datasets/{data_filename}.csv.gz"
    )

    data_home = get_data_home(data_home=data_home)
    filepath = os.path.join(data_home, f"{data_filename}.pkz")

    if os.path.exists(filepath):
        return joblib.load(filepath)

    if not download_if_missing:
        raise OSError("Data not found and `download_if_missing` is False")

    archive_path = os.path.join(data_home, os.path.basename(url))
    ur.urlretrieve(url, archive_path)
    df = load_gzip_compressed_csv_data(archive_path)
    joblib.dump(df, filepath, compress=6)
    os.remove(archive_path)
    return df

def load_sp500_implied_vol_dataset(
    data_home=None, download_if_missing=True
) -> pd.DataFrame:
    """Load the 3 months ATM implied volatility of the 20 assets from the
    SP500 dataset.

    This dataset is composed of the 3 months ATM implied volatility of 20 assets
    from the S&P 500 composition starting from 2010-01-04 up to 2022-12-28.

    The data comes from the Yahoo public API option chains.

    ==============   ==================
    Observations     3270
    Assets           20
    ==============   ==================

    Parameters
    ----------
    data_home : str, optional
        Specify another download and cache folder for the datasets.
        By default, all skfolio data is stored in `~/skfolio_data` subfolders.

    download_if_missing : bool, default=True
        If False, raise an OSError if the data is not locally available
        instead of trying to download the data from the source site.

    Returns
    -------
    df : DataFrame of shape (n_observations, n_assets)
        Implied volatility DataFrame

    Examples
    --------
    >>> from skfolio.datasets import load_sp500_implied_vol_dataset
    >>> implied_vol = load_sp500_implied_vol_dataset()
    >>> implied_vol.head()
                    AAPL       AMD       BAC  ...       UNH       WMT       XOM
    Date                                      ...
    2010-01-04  0.364353  0.572056  0.382926  ...  0.362751  0.171737  0.201485
    2010-01-05  0.371865  0.568791  0.374699  ...  0.368504  0.174764  0.203852
    2010-01-06  0.356746  0.558054  0.349220  ...  0.368514  0.171892  0.197475
    2010-01-07  0.361084  0.560475  0.354942  ...  0.355792  0.169083  0.200046
    2010-01-08  0.348085  0.543932  0.360345  ...  0.351130  0.170897  0.204832
    """
    data_filename = "sp500_implied_vol_dataset"
    df = download_dataset(
        data_filename, data_home=data_home, download_if_missing=download_if_missing
    )
    return df


def assert_is_square(x: np.ndarray) -> None:
    """Raises an error if the matrix is not square.

    Parameters
    ----------
    x : ndarray of shape (n, n)
       The matrix.

    Raises
    ------
    ValueError: if the matrix is not square.
    """
    if x.ndim != 2 or x.shape[0] != x.shape[1]:
        raise ValueError("The matrix must be square")

def assert_is_symmetric(x: np.ndarray) -> None:
    """Raises an error if the matrix is not symmetric.

    Parameters
    ----------
    x : ndarray of shape (n, m)
       The matrix.

    Raises
    ------
    ValueError: if the matrix is not symmetric.
    """
    assert_is_square(x)
    if not np.allclose(x, x.T):
        raise ValueError("The matrix must be symmetric")

def assert_is_distance(x: np.ndarray) -> None:
    """Raises an error if the matrix is not a distance matrix.

    Parameters
    ----------
    x : ndarray of shape (n, n)
       The matrix.

    Raises
    ------
    ValueError: if the matrix is a distance matrix.
    """
    assert_is_symmetric(x)
    if not np.allclose(np.diag(x), np.zeros(x.shape[0]), atol=1e-5):
        raise ValueError(
            "The distance matrix must have diagonal elements close to zeros"
        )


    def get_metadata_routing(self):
        # noinspection PyTypeChecker
        router = (
            skm.MetadataRouter(owner=self.__class__.__name__)
            .add(
                mu_estimator=self.mu_estimator,
                method_mapping=skm.MethodMapping().add(caller="fit", callee="fit"),
            )
            .add(
                covariance_estimator=self.covariance_estimator,
                method_mapping=skm.MethodMapping().add(caller="fit", callee="fit"),
            )
        )
        return router

    def fit(self, X: npt.ArrayLike, y=None, **fit_params) -> "EmpiricalPrior":
        """Fit the Empirical Prior estimator.

        Parameters
        ----------
        X : array-like of shape (n_observations, n_assets)
            Price returns of the assets.

        y : Ignored
            Not used, present for API consistency by convention.

        **fit_params : dict
            Parameters to pass to the underlying estimators.
            Only available if `enable_metadata_routing=True`, which can be
            set by using ``sklearn.set_config(enable_metadata_routing=True)``.
            See :ref:`Metadata Routing User Guide <metadata_routing>` for
            more details.

        Returns
        -------
        self : EmpiricalPrior
            Fitted estimator.
        """
        routed_params = skm.process_routing(self, "fit", **fit_params)

        self.mu_estimator_ = check_estimator(
            self.mu_estimator,
            default=EmpiricalMu(),
            check_type=BaseMu,
        )
        self.covariance_estimator_ = check_estimator(
            self.covariance_estimator,
            default=EmpiricalCovariance(),
            check_type=BaseCovariance,
        )
        # fitting estimators
        if not self.is_log_normal:
            if self.investment_horizon is not None:
                raise ValueError(
                    "`investment_horizon` must be `None` when "
                    "`is_log_normal` is `False`"
                )
            # Expected returns
            # noinspection PyArgumentList
            self.mu_estimator_.fit(X, y, **routed_params.mu_estimator.fit)
            mu = self.mu_estimator_.mu_

            # Covariance
            # noinspection PyArgumentList
            self.covariance_estimator_.fit(
                X, y, **routed_params.covariance_estimator.fit
            )
            covariance = self.covariance_estimator_.covariance_
        else:
            if self.investment_horizon is None:
                raise ValueError(
                    "`investment_horizon` must be provided when "
                    "`is_log_normal` is `True`"
                )
            # Convert linear returns to log returns
            X_log = np.log(1 + X)
            y_log = np.log(1 + y) if y is not None else None

            # Estimates the moments on the log returns
            # Expected returns
            # noinspection PyArgumentList
            self.mu_estimator_.fit(X_log, y_log, **routed_params.mu_estimator.fit)
            mu = self.mu_estimator_.mu_

            # Covariance
            # noinspection PyArgumentList
            self.covariance_estimator_.fit(
                X_log, y_log, **routed_params.covariance_estimator.fit
            )
            covariance = self.covariance_estimator_.covariance_

            # Using the property of aggregation across time we scale this distribution
            # to the investment horizon by the “square-root rule”.
            mu *= self.investment_horizon
            covariance *= self.investment_horizon

            # We convert it into a distribution of linear returns over the investment
            # horizon
            mu = np.exp(mu + 0.5 * np.diag(covariance))
            covariance = np.outer(mu, mu) * (np.exp(covariance) - 1)

        # we validate and convert to numpy after all models have been fitted to keep
        # features names information.
        X = skv.validate_data(self, X)
        self.prior_model_ = PriorModel(
            mu=mu,
            covariance=covariance,
            returns=X,
        )
        return self


def load_gzip_compressed_csv_data(
    data_filename: str,
    data_module: str = DATA_MODULE,
    encoding="utf-8",
    datetime_index: bool = True,
) -> pd.DataFrame:
    """Load gzip-compressed csv files with `importlib.resources`.

    1) Open resource file with `importlib.resources.open_binary`
    2) Decompress csv file with `gzip.open`
    3) Load decompressed data with `pd.read_csv`

    Parameters
    ----------
    data_filename : str
        Name of gzip-compressed csv file  (`'*.csv.gz'`) to be loaded from
        `data_module/data_file_name`. For example `'SPX500.csv.gz'`.

    data_module : str or module, default='skfolio.datasets.data'
        Module where data lives. The default is `'skfolio.datasets.data'`.

    encoding : str, default="utf-8"
        Name of the encoding that the gzip-decompressed file will be
        decoded with. The default is 'utf-8'.

    datetime_index: bool, default=True
        If this is set to True, the DataFrame index is converted to datetime with
        format="%Y-%m-%d".
        The default is `True`.

    Returns
    -------
    df : DataFrame of shape (n_observations, n_assets)
        DataFrame with each row representing one observation and each column
        representing the asset price of a given observation.
    """
    path = resources.files(data_module).joinpath(data_filename)
    with path.open("rb") as compressed_file:
        compressed_file = gzip.open(compressed_file, mode="rt", encoding=encoding)
        df = pd.read_csv(compressed_file, sep=",", index_col=0)
        if datetime_index:
            df.index = pd.to_datetime(df.index, format="%Y-%m-%d")
        return df

def load_sp500_dataset() -> pd.DataFrame:
    """Load the prices of 20 assets from the S&P 500 Index composition.

    This dataset is composed of the daily prices of 20 assets from the S&P 500
    composition starting from 1990-01-02 up to 2022-12-28.

    The data comes from the Yahoo public API.
    The price is the adjusted close which is the closing price after adjustments for
    all applicable splits and dividend distributions.
    The adjustment uses appropriate split and dividend multipliers, adhering to
    the Center for Research in Security Prices (CRSP) standards.

    ==============   ==================
    Observations     8313
    Assets           20
    ==============   ==================

    Returns
    -------
    df : DataFrame of shape (n_observations, n_assets)
        Prices DataFrame

    Examples
    --------
    >>> from skfolio.datasets import load_sp500_dataset
    >>> prices = load_sp500_dataset()
    >>> prices.head()
                    AAPL     AMD       BAC  ...       UNH       WMT      XOM
    1990-01-02  0.332589  4.1250  11.65625  ...  0.382813  5.890625  12.5000
    1990-01-03  0.334821  4.0000  11.75000  ...  0.375000  5.890625  12.3750
    1990-01-04  0.335938  3.9375  11.50000  ...  0.371094  5.859375  12.2500
    1990-01-05  0.337054  3.8125  11.25000  ...  0.355469  5.796875  12.1875
    1990-01-08  0.339286  3.8125  11.31250  ...  0.347656  5.875000  12.3750
    """
    data_filename = "sp500_dataset.csv.gz"
    df = load_gzip_compressed_csv_data(data_filename)
    return df

    def get_metadata_routing(self):
        # noinspection PyTypeChecker
        router = skm.MetadataRouter(owner=self.__class__.__name__)
        for name, estimator in self.estimators:
            router.add(
                **{name: estimator},
                method_mapping=skm.MethodMapping().add(caller="fit", callee="fit"),
            )
        return router

    def fit(
        self, X: npt.ArrayLike, y: npt.ArrayLike | None = None, **fit_params
    ) -> "StackingOptimization":
        """Fit the Stacking Optimization estimator.

        Parameters
        ----------
        X : array-like of shape (n_observations, n_assets)
           Price returns of the assets.

        y : array-like of shape (n_observations, n_targets), optional
            Price returns of factors or a target benchmark.
            The default is `None`.

        **fit_params : dict
            Parameters to pass to the underlying estimators.
            Only available if `enable_metadata_routing=True`, which can be
            set by using ``sklearn.set_config(enable_metadata_routing=True)``.
            See :ref:`Metadata Routing User Guide <metadata_routing>` for
            more details.

        Returns
        -------
        self : StackingOptimization
           Fitted estimator.
        """
        routed_params = skm.process_routing(self, "fit", **fit_params)

        names, all_estimators = self._validate_estimators()
        self.final_estimator_ = check_estimator(
            self.final_estimator,
            default=MeanRisk(),
            check_type=BaseOptimization,
        )

        if self.cv == "prefit":
            self.estimators_ = []
            for estimator in all_estimators:
                skv.check_is_fitted(estimator)
                self.estimators_.append(estimator)
        else:
            # Fit the base estimators on the whole training data. Those
            # base estimators will be used to retrieve the inner weights.
            # They are exposed publicly.
            # noinspection PyCallingNonCallable
            self.estimators_ = skp.Parallel(n_jobs=self.n_jobs)(
                skp.delayed(fit_single_estimator)(
                    sk.clone(est), X, y, routed_params[name]["fit"]
                )
                for name, est in zip(names, all_estimators, strict=True)
            )

        self.named_estimators_ = {
            name: estimator
            for name, estimator in zip(names, self.estimators_, strict=True)
        }

        inner_weights = np.array([estimator.weights_ for estimator in self.estimators_])

        # To train the final-estimator using the most data as possible, we use
        # a cross-validation to obtain the output of the stacked estimators.
        # To ensure that the data provided to each estimator are the same,
        # we need to set the random state of the cv if there is one and we
        # need to take a copy.
        if self.cv in ["prefit", "ignore"]:
            X_pred = np.array(
                [estimator.predict(X) for estimator in self.estimators_]
            ).T
        else:
            cv = sks.check_cv(self.cv)
            if hasattr(cv, "random_state") and cv.random_state is None:
                cv.random_state = np.random.RandomState()
            # noinspection PyCallingNonCallable
            cv_predictions = skp.Parallel(n_jobs=self.n_jobs)(
                skp.delayed(cross_val_predict)(
                    sk.clone(est),
                    X,
                    y,
                    cv=deepcopy(cv),
                    method="predict",
                    n_jobs=self.n_jobs,
                    params=routed_params[name]["fit"],
                    verbose=self.verbose,
                )
                for name, est in zip(names, all_estimators, strict=True)
            )

            # We validate and convert to numpy array only after base-estimator fitting
            # to keep the assets names in case they are used in the estimator.
            if y is not None:
                _, y = skv.validate_data(self, X, y, multi_output=True)
            else:
                _ = skv.validate_data(self, X)

            if isinstance(self.cv, BaseCombinatorialCV):
                X_pred = np.array(
                    [
                        pred.quantile(measure=self.quantile_measure, q=self.quantile)
                        for pred in cv_predictions
                    ]
                ).T
            else:
                X_pred = np.array(cv_predictions).T
                if y is not None:
                    test_indices = np.sort(
                        np.concatenate([test for _, test in cv.split(X, y)])
                    )
                    y = y[test_indices]

        fit_single_estimator(self.final_estimator_, X_pred, y, {})
        outer_weights = self.final_estimator_.weights_
        self.weights_ = outer_weights @ inner_weights
        return self


def load_factors_dataset() -> pd.DataFrame:
    """Load the prices of 5 factor ETFs.

    This dataset is composed of the daily prices of 5 ETF representing common factors
    starting from 2014-01-02 up to 2022-12-28.

    The factors are:

        * "MTUM": Momentum
        * "QUAL": Quality
        * "SIZE": Size
        * "VLUE": Value
        * "USMV": low volatility

    The data comes from the Yahoo public API.
    The price is the adjusted close which is the closing price after adjustments for
    all applicable splits and dividend distributions.
    The adjustment uses appropriate split and dividend multipliers, adhering to
    the Center for Research in Security Prices (CRSP) standards.

    ==============   ==================
    Observations     2264
    Assets           5
    ==============   ==================

    Returns
    -------
    df : DataFrame of shape (n_observations, n_assets)
        Prices DataFrame

    Examples
    --------
    >>> from skfolio.datasets import load_factors_dataset
    >>> prices = load_factors_dataset()
    >>> prices.head()
                  MTUM    QUAL    SIZE    USMV    VLUE
    Date
    2014-01-02  52.704  48.351  48.986  29.338  47.054
    2014-01-03  52.792  48.256  48.722  29.330  46.999
    2014-01-06  52.677  48.067  48.722  29.263  46.991
    2014-01-07  53.112  48.455  48.731  29.430  47.253
    2014-01-08  53.502  48.437  48.731  29.422  47.253
    """
    data_filename = "factors_dataset.csv.gz"
    df = load_gzip_compressed_csv_data(data_filename)
    return df


def semi_variance(
    returns: np.ndarray, min_acceptable_return: float | None = None
) -> float:
    """Compute the semi-variance (second lower partial moment).

    The semi-variance is the variance of the returns below a minimum acceptable return.

    Parameters
    ----------
    returns : ndarray of shape (n_observations,)
        Vector of returns

    min_acceptable_return : float, optional
        Minimum acceptable return. It is the return target to distinguish "downside" and
        "upside" returns.
        The default (`None`) is to use the mean.

    Returns
    -------
    value : float
        Semi-variance.
    """
    if min_acceptable_return is None:
        min_acceptable_return = np.mean(returns, axis=0)
    return np.sum(np.power(np.minimum(0, returns - min_acceptable_return), 2)) / (
        len(returns) - 1
    )

def semi_deviation(
    returns: np.ndarray, min_acceptable_return: float | None = None
) -> float:
    """Compute the semi standard-deviation (semi-deviation) (square root of the second lower
    partial moment).

    Parameters
    ----------
    returns : ndarray of shape (n_observations,)
        Vector of returns.

    min_acceptable_return : float, optional
        Minimum acceptable return. It is the return target to distinguish "downside" and
        "upside" returns.
        The default (`None`) is to use the returns mean.

    Returns
    -------
    value : float
        Semi-standard-deviation.
    """
    return np.sqrt(
        semi_variance(returns=returns, min_acceptable_return=min_acceptable_return)
    )


def variance(returns: np.ndarray) -> float:
    """Compute the variance (second moment).

    Parameters
    ----------
    returns : ndarray of shape (n_observations,)
        Vector of returns.

    Returns
    -------
    value : float
        Variance.
    """
    return returns.var(ddof=1)

def standard_deviation(returns: np.ndarray) -> float:
    """Compute the standard-deviation (square root of the second moment).

    Parameters
    ----------
    returns : ndarray of shape (n_observations,)
        Vector of returns.

    Returns
    -------
    value : float
        Standard-deviation.
    """
    return np.sqrt(variance(returns=returns))


def _validate_groups(groups: npt.ArrayLike, name: str = "groups") -> np.ndarray:
    """Validate groups by checking its dim and if group names don't appear in multiple
    levels and convert to numpy array.

    Parameters
    ----------
    groups : array-like of shape (n_groups, n_assets)
        2D-array of strings.

    Returns
    -------
    groups : ndarray of shape (n_groups, n_assets)
        2D-array of strings.
    """
    groups = np.asarray(groups)
    if groups.ndim != 2:
        raise ValueError(
            f"`{name} must be a 2D array, got {groups.ndim}D array instead."
        )
    n = len(groups)
    group_sets = [set(groups[i]) for i in range(n)]
    for i in range(n - 1):
        for e in group_sets[i]:
            for j in range(i + 1, n):
                if e in group_sets[j]:
                    raise DuplicateGroupsError(
                        f"'{e}' appear in two levels: {list(groups[i])} "
                        f"and {list(groups[i])}. "
                        f"{name} must be in only one level."
                    )

    return groups

def equations_to_matrix(
    groups: npt.ArrayLike,
    equations: npt.ArrayLike,
    sum_to_one: bool = False,
    raise_if_group_missing: bool = False,
    names: tuple[str, str] = ("groups", "equations"),
) -> tuple[np.ndarray, np.ndarray, np.ndarray, np.ndarray]:
    """Convert a list of linear equations into the left and right matrices of the
    inequality A <= B and equality A == B.

    Parameters
    ----------
    groups : array-like of shape (n_groups, n_assets)
        2D array of assets groups.

        For example:

             groups = np.array(
                [
                    ["SPX", "SX5E", "NKY", "TLT"],
                    ["Equity", "Equity", "Equity", "Bond"],
                    ["US", "Europe", "Japan", "US"],
                ]
            )

    equations : array-like of shape (n_equations,)
         1D array of equations.

         Example of valid equation patterns:
            * "number_1 * group_1 + number_3 <= number_4 * group_3 + number_5"
            * "group_1 == number * group_2"
            * "group_1 <= number"
            * "group_1 == number"

        "group_1" and "group_2" are the group names defined in `groups`.
        The second expression means that the sum of all assets in "group_1" should be
        less or equal to "number" times the sum of all assets in "group_2".

        For example:

             equations = [
                "Equity <= 3 * Bond",
                "US >= 1.5",
                "Europe >= 0.5 * Japan",
                "Japan == 1",
                "3*SPX + 5*SX5E == 2*TLT + 3",
            ]

    sum_to_one : bool
        If this is set to True, all elements in a group sum to one (used in the `views`
        of the Black-Litterman model).

    raise_if_group_missing : bool, default=False
        If this is set to True, an error is raised when a group is not found in the
        groups, otherwise only a warning is shown.
        The default is False.

    names : tuple[str, str], default=('groups', 'equations')
        The group and equation names used in error messages.
        The default is `('groups', 'equations')`.

    Returns
    -------
    left_equality: ndarray of shape (n_equations_equality, n_assets)
    right_equality: ndarray of shape (n_equations_equality,)
        The left and right matrices of the inequality A <= B.

    left_inequality: ndarray of shape (n_equations_inequality, n_assets)
    right_inequality: ndarray of shape (n_equations_inequality,)
        The left and right matrices of the equality A == B.
    """
    groups = _validate_groups(groups, name=names[0])
    equations = _validate_equations(equations, name=names[1])

    a_equality = []
    b_equality = []

    a_inequality = []
    b_inequality = []

    for string in equations:
        try:
            left, right, is_inequality = _string_to_equation(
                groups=groups,
                string=string,
                sum_to_one=sum_to_one,
            )
            if is_inequality:
                a_inequality.append(left)
                b_inequality.append(right)
            else:
                a_equality.append(left)
                b_equality.append(right)
        except GroupNotFoundError as e:
            if raise_if_group_missing:
                raise
            warnings.warn(str(e), stacklevel=2)
    return (
        np.array(a_equality),
        np.array(b_equality),
        np.array(a_inequality),
        np.array(b_inequality),
    )


    def central(self) -> bool:
        """Determine whether this node is considered central.
        It is inherited from the associated edge's centrality.

        Returns
        -------
        central: bool
           True if the node is central; otherwise, False.
        """
        if self._central is None:
            self._central = self.ref.strongly_central
        return self._central

    def weakly_central(self) -> bool:
        """Determine if the edge is weakly central.
        An edge is weakly central if at least one of its two nodes is central.
        """
        return self.node1.central or self.node2.central

    def set_edges_from_mst(self, dependence_method: DependenceMethod) -> None:
        """Construct the Maximum Spanning Tree (MST) from the current nodes using
        the specified dependence method.

        The MST is built based on pairwise dependence measures computed between nodes.
        If any edge is (weakly) central, a central factor is added to the dependence
        measure to favor edges connected to central nodes.

        Parameters
        ----------
        dependence_method : DependenceMethod
            The method used to compute the dependence measure between nodes (e.g.,
            Kendall's tau).

        Returns
        -------
        None
        """
        n = len(self.nodes)
        dependence_matrix = np.zeros((n, n))
        eligible_edges = {}
        central = False
        for i, j in combinations(range(n), 2):
            node1 = self.nodes[i]
            node2 = self.nodes[j]
            if self.level == 0 or node1.ref.share_one_node(node2.ref):
                edge = Edge(
                    node1=node1, node2=node2, dependence_method=dependence_method
                )
                if not central and edge.weakly_central:
                    central = True
                # Negate the matrix to use minimum_spanning_tree for maximum spanning
                # Add a cst to ensure that even if dep is 0, we still build a valid MST
                dep = abs(edge.dependence) + 1e-5
                dependence_matrix[i, j] = dep
                eligible_edges[(i, j)] = edge

        if np.any(np.isnan(dependence_matrix)):
            raise RuntimeError("dependence_matrix contains NaNs")

        if central:
            max_dep = np.max(dependence_matrix)
            for (i, j), edge in eligible_edges.items():
                if edge.weakly_central:
                    if edge.strongly_central:
                        central_factor = 3 * max_dep
                    else:
                        central_factor = 2 * max_dep
                    dep = dependence_matrix[i, j] + central_factor
                    dependence_matrix[i, j] = dep

        # Compute the minimum spanning tree
        mst = ssc.minimum_spanning_tree(-dependence_matrix, overwrite=True)

        edges = []
        # Extract the indices of the non-zero entries (edges)
        for i, j in zip(*mst.nonzero(), strict=True):
            edge = eligible_edges[(i, j)]
            # connect Nodes to Edges
            edge.ref_to_nodes()
            edges.append(edge)

        self.edges = edges


def _split_equation_string(string: str) -> list[str]:
    """Split an equation strings by operators."""
    comp_pattern = "(?=" + "|".join([".+\\" + e for e in _COMPARISON_OPERATORS]) + ")"
    if not bool(re.match(comp_pattern, string)):
        raise EquationToMatrixError(
            f"The string must contains a comparison operator: "
            f"{list(_COMPARISON_OPERATORS)}"
        )

    # Regex to match only '>' and '<' but not '<=' or '>='
    invalid_pattern = r"(?<!<)(?<!<=)>(?!=)|(?<!>)<(?!=)"
    invalid_matches = re.findall(invalid_pattern, string)

    if len(invalid_matches) > 0:
        raise EquationToMatrixError(
            f"{invalid_matches[0]} is an invalid comparison operator. "
            f"Valid comparison operators are: {list(_COMPARISON_OPERATORS)}"
        )

    # '==' needs to be before '='
    operators = sorted(_OPERATORS, reverse=True)
    pattern = "((?:" + "|".join(["\\" + e for e in operators]) + "))"
    res = [x.strip() for x in re.split(pattern, string)]
    res = [x for x in res if x != ""]
    return res

def _string_to_equation(
    groups: np.ndarray,
    string: str,
    sum_to_one: bool,
) -> tuple[np.ndarray, float, bool]:
    """Convert a string to a left 1D-array and right float of the form:
    `groups @ left <= right` or `groups @ left == right` and return whether it's an
    equality or inequality.

    Parameters
    ----------
    groups : ndarray of shape (n_groups, n_assets)
        Groups 2D-array

    string : str
        String to convert

    sum_to_one : bool
        If this is set to True, the 1D-array is scaled to have a sum of one.

    Returns
    -------
    left : 1D-array of shape (n_assets,)
    right : float
    is_inequality : bool
    """
    n = groups.shape[1]
    err_msg = f"Wrong pattern encountered while converting the string '{string}'"

    iterator = iter(_split_equation_string(string))
    group_names = set(groups.flatten())

    def is_group(name: str) -> bool:
        return name in group_names

    left = np.zeros(n)
    right = 0
    main_sign = 1
    comparison_sign = None
    is_inequality = None
    e = next(iterator, None)
    i = 0
    while True:
        i += 1
        if i > 1e6:
            raise RecursionError(err_msg)
        if e is None:
            break
        sign = 1
        if e in _COMPARISON_OPERATORS:
            if e in _INEQUALITY_OPERATORS:
                is_inequality = True
            else:
                is_inequality = False
            main_sign = -1
            comparison_sign = _comparison_operator_sign(e)
            e = next(iterator, None)
            if e in _SUB_ADD_OPERATORS:
                sign *= _sub_add_operator_sign(e)
                e = next(iterator, None)
        elif e in _SUB_ADD_OPERATORS:
            sign *= _sub_add_operator_sign(e)
            e = next(iterator, None)
        elif e in _MUL_OPERATORS:
            raise EquationToMatrixError(
                f"{err_msg}: the character '{e}' is wrongly positioned"
            )
        sign *= main_sign
        # next can only be a number or a group
        if e is None or e in _OPERATORS:
            raise EquationToMatrixError(
                f"{err_msg}: the character '{e}' is wrongly positioned"
            )
        if is_group(e):
            arr = _matching_array(values=groups, key=e, sum_to_one=sum_to_one)
            # next can only be a '*' or an ['-', '+', '>=', '<=', '==', '='] or None
            e = next(iterator, None)
            if e is None or e in _NON_MUL_OPERATORS:
                left += sign * arr
            elif e in _MUL_OPERATORS:
                # next can only a number
                e = next(iterator, None)
                try:
                    number = float(e)
                except ValueError:
                    raise GroupNotFoundError(
                        f"{err_msg}: the group '{e}' is missing from the groups"
                        f" {groups}"
                    ) from None

                left += number * sign * arr
                e = next(iterator, None)
            else:
                raise EquationToMatrixError(
                    f"{err_msg}: the character '{e}' is wrongly positioned"
                )
        else:
            try:
                number = float(e)
            except ValueError:
                raise GroupNotFoundError(
                    f"{err_msg}: the group '{e}' is missing from the groups {groups}"
                ) from None
            # next can only be a '*' or an operator or None
            e = next(iterator, None)
            if e in _MUL_OPERATORS:
                # next can only a group
                e = next(iterator, None)
                if not is_group(e):
                    raise EquationToMatrixError(
                        f"{err_msg}: the character '{e}' is wrongly positioned"
                    )
                arr = _matching_array(values=groups, key=e, sum_to_one=sum_to_one)
                left += number * sign * arr
                e = next(iterator, None)
            elif e is None or e in _NON_MUL_OPERATORS:
                right += number * sign
            else:
                raise EquationToMatrixError(
                    f"{err_msg}: the character '{e}' is wrongly positioned"
                )

    left *= comparison_sign
    right *= -comparison_sign

    return left, right, is_inequality


def load_gzip_compressed_csv_data(
    data_filename: str,
    data_module: str = DATA_MODULE,
    encoding="utf-8",
    datetime_index: bool = True,
) -> pd.DataFrame:
    """Load gzip-compressed csv files with `importlib.resources`.

    1) Open resource file with `importlib.resources.open_binary`
    2) Decompress csv file with `gzip.open`
    3) Load decompressed data with `pd.read_csv`

    Parameters
    ----------
    data_filename : str
        Name of gzip-compressed csv file  (`'*.csv.gz'`) to be loaded from
        `data_module/data_file_name`. For example `'SPX500.csv.gz'`.

    data_module : str or module, default='skfolio.datasets.data'
        Module where data lives. The default is `'skfolio.datasets.data'`.

    encoding : str, default="utf-8"
        Name of the encoding that the gzip-decompressed file will be
        decoded with. The default is 'utf-8'.

    datetime_index: bool, default=True
        If this is set to True, the DataFrame index is converted to datetime with
        format="%Y-%m-%d".
        The default is `True`.

    Returns
    -------
    df : DataFrame of shape (n_observations, n_assets)
        DataFrame with each row representing one observation and each column
        representing the asset price of a given observation.
    """
    path = resources.files(data_module).joinpath(data_filename)
    with path.open("rb") as compressed_file:
        compressed_file = gzip.open(compressed_file, mode="rt", encoding=encoding)
        df = pd.read_csv(compressed_file, sep=",", index_col=0)
        if datetime_index:
            df.index = pd.to_datetime(df.index, format="%Y-%m-%d")
        return df

def load_sp500_dataset() -> pd.DataFrame:
    """Load the prices of 20 assets from the S&P 500 Index composition.

    This dataset is composed of the daily prices of 20 assets from the S&P 500
    composition starting from 1990-01-02 up to 2022-12-28.

    The data comes from the Yahoo public API.
    The price is the adjusted close which is the closing price after adjustments for
    all applicable splits and dividend distributions.
    The adjustment uses appropriate split and dividend multipliers, adhering to
    the Center for Research in Security Prices (CRSP) standards.

    ==============   ==================
    Observations     8313
    Assets           20
    ==============   ==================

    Returns
    -------
    df : DataFrame of shape (n_observations, n_assets)
        Prices DataFrame

    Examples
    --------
    >>> from skfolio.datasets import load_sp500_dataset
    >>> prices = load_sp500_dataset()
    >>> prices.head()
                    AAPL     AMD       BAC  ...       UNH       WMT      XOM
    1990-01-02  0.332589  4.1250  11.65625  ...  0.382813  5.890625  12.5000
    1990-01-03  0.334821  4.0000  11.75000  ...  0.375000  5.890625  12.3750
    1990-01-04  0.335938  3.9375  11.50000  ...  0.371094  5.859375  12.2500
    1990-01-05  0.337054  3.8125  11.25000  ...  0.355469  5.796875  12.1875
    1990-01-08  0.339286  3.8125  11.31250  ...  0.347656  5.875000  12.3750
    """
    data_filename = "sp500_dataset.csv.gz"
    df = load_gzip_compressed_csv_data(data_filename)
    return df

def get_data_home(data_home: str | Path | None = None) -> str:
    """Return the path of the skfolio data directory.

    This folder is used by some large dataset loaders to avoid downloading the
    data several times.

    By default, the data directory is set to a folder named 'skfolio_data' in the
    user home folder.

    Alternatively, it can be set by the 'SKFOLIO_DATA' environment
    variable or programmatically by giving an explicit folder path. The '~'
    symbol is expanded to the user home folder.

    If the folder does not already exist, it is automatically created.

    Parameters
    ----------
    data_home : str, optional
        The path to skfolio data directory. If `None`, the default path
        is `~/skfolio_data`.

    Returns
    -------
    data_home: str or path-like, optional
        The path to skfolio data directory.
    """
    if data_home is None:
        data_home = os.environ.get("SKFOLIO_DATA", os.path.join("~", "skfolio_data"))
    data_home = os.path.expanduser(data_home)
    os.makedirs(data_home, exist_ok=True)
    return data_home

def download_dataset(
    data_filename: str,
    data_home: str | Path | None = None,
    download_if_missing: bool = True,
) -> pd.DataFrame:
    """Download and save locally a dataset from the remote GitHub dataset folder.

    Parameters
    ----------
    data_filename : str
        Name of gzip-compressed csv file  (`'*.csv.gz'`) to be loaded from a remote
        GitHub dataset folder.

    data_home : str or path-like, optional
        Specify another download and cache folder for the datasets. By default,
        all skfolio data is stored in `~/skfolio_data` sub-folders.

    download_if_missing : bool, default=True
        If False, raise an OSError if the data is not locally available
        instead of trying to download the data from the source site.
        The default is `True`.

    Returns
    -------
    df : DataFrame of shape (n_observations, n_assets)
        DataFrame with each row representing one observation and each column
        representing the asset price of a given observation.
    """
    # Use a CORS proxy when triggering requests from the browser
    url_prefix = "https://corsproxy.io/?" if sys.platform == "emscripten" else ""
    url = url_prefix + (
        f"https://github.com/skfolio/skfolio-datasets/raw/main/"
        f"datasets/{data_filename}.csv.gz"
    )

    data_home = get_data_home(data_home=data_home)
    filepath = os.path.join(data_home, f"{data_filename}.pkz")

    if os.path.exists(filepath):
        return joblib.load(filepath)

    if not download_if_missing:
        raise OSError("Data not found and `download_if_missing` is False")

    archive_path = os.path.join(data_home, os.path.basename(url))
    ur.urlretrieve(url, archive_path)
    df = load_gzip_compressed_csv_data(archive_path)
    joblib.dump(df, filepath, compress=6)
    os.remove(archive_path)
    return df

def load_nasdaq_dataset(data_home=None, download_if_missing=True) -> pd.DataFrame:
    """Load the prices of 1455 assets from the NASDAQ Composite Index.

    This dataset is composed of the daily prices of 1455 assets from the NASDAQ
    Composite starting from 2018-01-02 up to 2023-05-31.

    The data comes from the Yahoo public API.
    The price is the adjusted close which is the closing price after adjustments for
    all applicable splits and dividend distributions.
    The adjustment uses appropriate split and dividend multipliers, adhering to
    the Center for Research in Security Prices (CRSP) standards.

    ==============   ==================
    Observations     1362
    Assets           1455
    ==============   ==================

    Parameters
    ----------
    data_home : str, optional
        Specify another download and cache folder for the datasets.
        By default, all skfolio data is stored in `~/skfolio_data` subfolders.

    download_if_missing : bool, default=True
        If False, raise an OSError if the data is not locally available
        instead of trying to download the data from the source site.

    Returns
    -------
    df : DataFrame of shape (n_observations, n_assets)
        Prices DataFrame

    Examples
    --------
    >>> from skfolio.datasets import load_nasdaq_dataset
    >>> prices = load_nasdaq_dataset()
    >>> prices.head()
                   AAL   AAOI    AAON    AAPL  ...  ZVRA   ZYME    ZYNE   ZYXI
    Date                                       ...
    2018-01-02  51.648  37.91  35.621  41.310  ...  66.4  7.933  12.995  2.922
    2018-01-03  51.014  37.89  36.247  41.303  ...  72.8  7.965  13.460  2.913
    2018-01-04  51.336  38.38  36.103  41.495  ...  78.4  8.430  12.700  2.869
    2018-01-05  51.316  38.89  36.681  41.967  ...  77.6  8.400  12.495  2.780
    2018-01-08  50.809  38.37  36.103  41.811  ...  82.4  8.310  12.550  2.825
    """
    data_filename = "nasdaq_dataset"
    df = download_dataset(
        data_filename, data_home=data_home, download_if_missing=download_if_missing
    )
    return df


def optimal_rounding_decimals(x: float) -> int:
    """Return the optimal rounding decimal number for a user-friendly formatting.

    Parameters
    ----------
    x : float
        Number to round.

    Returns
    -------
    n : int
        Rounding decimal number.
    """
    return min(6, max(int(-np.log10(abs(x))) + 2, 2))

def format_measure(x: float, percent: bool = False) -> str:
    """Format a measure number into a user-friendly string.

    Parameters
    ----------
    x : float
        Number to format.

    percent : bool, default=False
        If this is set to True, the number is formatted in percentage.

    Returns
    -------
    formatted : str
        Formatted string.
    """
    if np.isnan(x):
        return str(x)
    if percent:
        xn = x * 100
        f = "%"
    else:
        xn = x
        f = "f"
    if xn == 0:
        n = 0
    else:
        n = optimal_rounding_decimals(xn)
    return "{value:{fmt}}".format(value=x, fmt=f".{n}{f}")

def safe_indexing(
    X: npt.ArrayLike | pd.DataFrame, indices: npt.ArrayLike | None, axis: int = 0
):
    """Return rows, items or columns of X using indices.

    Parameters
    ----------
    X : array-like
        Data from which to sample rows.

    indices : array-like, optional
        Indices of rows or columns.
        The default (`None`) is to select the entire data.

    axis : int, default=0
        The axis along which `X` will be sub-sampled. `axis=0` will select
        rows while `axis=1` will select columns.

    Returns
    -------
    subset :
        Subset of X on axis 0.
    """
    if indices is None:
        return X
    if hasattr(X, "iloc"):
        return X.take(indices, axis=axis)
    if axis == 0:
        return X[indices]
    return X[:, indices]

def safe_split(
    X: npt.ArrayLike,
    y: npt.ArrayLike | None = None,
    indices: np.ndarray | None = None,
    axis: int = 0,
):
    """Create subset of dataset.

    Slice X, y according to indices for cross-validation.

    Parameters
    ----------
    X : array-like
        Data to be indexed.

    y : array-like
        Data to be indexed.

    indices : ndarray of int, optional
        Rows or columns to select from X and y.
        The default (`None`) is to select the entire data.

    axis : int, default=0
        The axis along which `X` will be sub-sampled. `axis=0` will select
        rows while `axis=1` will select columns.

    Returns
    -------
    X_subset : array-like
        Indexed data.

    y_subset : array-like
        Indexed targets.
    """
    X_subset = safe_indexing(X, indices=indices, axis=axis)
    if y is not None:
        y_subset = safe_indexing(y, indices=indices, axis=axis)
    else:
        y_subset = None
    return X_subset, y_subset


    def __call__(self, episode: EpisodeBase, index: int) -> Transition:
        _validate_index(episode, index)

        observation = retrieve_observation(episode.observations, index)
        is_terminal = episode.terminated and index == episode.size() - 1
        if is_terminal:
            next_observation = create_zero_observation(observation)
            next_action = np.zeros_like(episode.actions[index])
        else:
            next_observation = retrieve_observation(
                episode.observations, index + 1
            )
            next_action = episode.actions[index + 1]

        return Transition(
            observation=observation,
            action=episode.actions[index],
            reward=episode.rewards[index],
            next_observation=next_observation,
            next_action=next_action,
            terminal=float(is_terminal),
            interval=1,
            rewards_to_go=episode.rewards[index:],
        )

def make_batches(
    episode: EpisodeBase,
    window_size: int,
    transition_picker: TransitionPickerProtocol,
) -> Iterator[TransitionMiniBatch]:
    n_batches = len(episode) // window_size
    if len(episode) % window_size != 0:
        n_batches += 1
    for i in range(n_batches):
        head_index = i * window_size
        last_index = min(head_index + window_size, episode.transition_count)
        transitions = [
            transition_picker(episode, index)
            for index in range(head_index, last_index)
        ]
        batch = TransitionMiniBatch.from_transitions(transitions)
        yield batch

    def __call__(
        self,
        algo: QLearningAlgoProtocol,
        dataset: ReplayBufferBase,
    ) -> float:
        total_errors = []
        episodes = self._episodes if self._episodes else dataset.episodes
        for episode in episodes:
            for batch in make_batches(
                episode, WINDOW_SIZE, dataset.transition_picker
            ):
                # estimate values for current observations
                values = algo.predict_value(batch.observations, batch.actions)

                # estimate values for next observations
                next_actions = algo.predict(batch.next_observations)
                next_values = algo.predict_value(
                    batch.next_observations, next_actions
                )

                # calculate td errors
                mask = (1.0 - batch.terminals).reshape(-1)
                rewards = np.asarray(batch.rewards).reshape(-1)
                if algo.reward_scaler:
                    rewards = algo.reward_scaler.transform_numpy(rewards)
                y = rewards + algo.gamma * next_values * mask
                total_errors += ((values - y) ** 2).tolist()

        return float(np.mean(total_errors))

    def create(self, observation_shape: Shape) -> Encoder:
        factory: Union[PixelEncoderFactory, VectorEncoderFactory]
        if len(observation_shape) == 3:
            factory = PixelEncoderFactory(
                activation=self.activation,
                use_batch_norm=self.use_batch_norm,
                dropout_rate=self.dropout_rate,
            )
        else:
            factory = VectorEncoderFactory(
                activation=self.activation,
                use_batch_norm=self.use_batch_norm,
                dropout_rate=self.dropout_rate,
            )
        return factory.create(observation_shape)

def create_discrete_q_function(
    observation_shape: Shape,
    action_size: int,
    encoder_factory: EncoderFactory,
    q_func_factory: QFunctionFactory,
    device: str,
    enable_ddp: bool,
    n_ensembles: int = 1,
) -> tuple[nn.ModuleList, DiscreteEnsembleQFunctionForwarder]:
    if q_func_factory.share_encoder:
        encoder = encoder_factory.create(observation_shape)
        hidden_size = compute_output_size([observation_shape], encoder)
        # normalize gradient scale by ensemble size
        for p in cast(nn.Module, encoder).parameters():
            p.register_hook(lambda grad: grad / n_ensembles)

    q_funcs = []
    forwarders = []
    for _ in range(n_ensembles):
        if not q_func_factory.share_encoder:
            encoder = encoder_factory.create(observation_shape)
            hidden_size = compute_output_size([observation_shape], encoder)
        q_func, forwarder = q_func_factory.create_discrete(
            encoder, hidden_size, action_size
        )
        q_func.to(device)
        if enable_ddp:
            q_func = wrap_model_by_ddp(q_func)
            forwarder.set_q_func(q_func)
        q_funcs.append(q_func)
        forwarders.append(forwarder)
    q_func_modules = nn.ModuleList(q_funcs)
    ensemble_forwarder = DiscreteEnsembleQFunctionForwarder(
        forwarders, action_size
    )
    return q_func_modules, ensemble_forwarder


    def create_with_action(
        self,
        observation_shape: Shape,
        action_size: int,
        discrete_action: bool = False,
    ) -> EncoderWithAction:
        factory: Union[PixelEncoderFactory, VectorEncoderFactory]
        if len(observation_shape) == 3:
            factory = PixelEncoderFactory(
                activation=self.activation,
                use_batch_norm=self.use_batch_norm,
                dropout_rate=self.dropout_rate,
            )
        else:
            factory = VectorEncoderFactory(
                activation=self.activation,
                use_batch_norm=self.use_batch_norm,
                dropout_rate=self.dropout_rate,
            )
        return factory.create_with_action(
            observation_shape, action_size, discrete_action
        )

def create_continuous_q_function(
    observation_shape: Shape,
    action_size: int,
    encoder_factory: EncoderFactory,
    q_func_factory: QFunctionFactory,
    device: str,
    enable_ddp: bool,
    n_ensembles: int = 1,
) -> tuple[nn.ModuleList, ContinuousEnsembleQFunctionForwarder]:
    if q_func_factory.share_encoder:
        encoder = encoder_factory.create_with_action(
            observation_shape, action_size
        )
        hidden_size = compute_output_size(
            [observation_shape, (action_size,)], encoder
        )
        # normalize gradient scale by ensemble size
        for p in cast(nn.Module, encoder).parameters():
            p.register_hook(lambda grad: grad / n_ensembles)

    q_funcs = []
    forwarders = []
    for _ in range(n_ensembles):
        if not q_func_factory.share_encoder:
            encoder = encoder_factory.create_with_action(
                observation_shape, action_size
            )
            hidden_size = compute_output_size(
                [observation_shape, (action_size,)], encoder
            )
        q_func, forwarder = q_func_factory.create_continuous(
            encoder, hidden_size
        )
        q_func.to(device)
        if enable_ddp:
            q_func = wrap_model_by_ddp(q_func)
            forwarder.set_q_func(q_func)
        q_funcs.append(q_func)
        forwarders.append(forwarder)
    q_func_modules = nn.ModuleList(q_funcs)
    ensemble_forwarder = ContinuousEnsembleQFunctionForwarder(
        forwarders, action_size
    )
    return q_func_modules, ensemble_forwarder


def _gather_quantiles_by_indices(
    y: torch.Tensor, indices: torch.Tensor
) -> torch.Tensor:
    # TODO: implement this in general case
    if y.dim() == 3:
        # (N, batch, n_quantiles) -> (batch, n_quantiles)
        return y.transpose(0, 1)[torch.arange(y.shape[1]), indices]
    elif y.dim() == 4:
        # (N, batch, action, n_quantiles) -> (batch, action, N, n_quantiles)
        transposed_y = y.transpose(0, 1).transpose(1, 2)
        # (batch, action, N, n_quantiles) -> (batch * action, N, n_quantiles)
        flat_y = transposed_y.reshape(-1, y.shape[0], y.shape[3])
        head_indices = torch.arange(y.shape[1] * y.shape[2])
        # (batch * action, N, n_quantiles) -> (batch * action, n_quantiles)
        gathered_y = flat_y[head_indices, indices.view(-1)]
        # (batch * action, n_quantiles) -> (batch, action, n_quantiles)
        return gathered_y.view(y.shape[1], y.shape[2], -1)
    raise ValueError

def _reduce_quantile_ensemble(
    y: torch.Tensor, reduction: str = "min", dim: int = 0, lam: float = 0.75
) -> torch.Tensor:
    # reduction beased on expectation
    mean = y.mean(dim=-1)
    if reduction == "min":
        indices = mean.min(dim=dim).indices
        return _gather_quantiles_by_indices(y, indices)
    elif reduction == "max":
        indices = mean.max(dim=dim).indices
        return _gather_quantiles_by_indices(y, indices)
    elif reduction == "none":
        return y
    elif reduction == "mix":
        min_indices = mean.min(dim=dim).indices
        max_indices = mean.max(dim=dim).indices
        min_values = _gather_quantiles_by_indices(y, min_indices)
        max_values = _gather_quantiles_by_indices(y, max_indices)
        return lam * min_values + (1.0 - lam) * max_values
    raise ValueError

    def compute_error(
        self,
        observations: TorchObservation,
        actions: torch.Tensor,
        rewards: torch.Tensor,
        target: torch.Tensor,
        terminals: torch.Tensor,
        gamma: Union[float, torch.Tensor] = 0.99,
        reduction: str = "mean",
    ) -> torch.Tensor:
        one_hot = F.one_hot(actions.view(-1), num_classes=self._action_size)
        value = (self._q_func(observations).q_value * one_hot.float()).sum(
            dim=1, keepdim=True
        )
        y = rewards + gamma * target * (1 - terminals)
        loss = compute_huber_loss(value, y)
        return compute_reduce(loss, reduction)

def compute_ensemble_q_function_error(
    forwarders: Union[
        Sequence[DiscreteQFunctionForwarder],
        Sequence[ContinuousQFunctionForwarder],
    ],
    observations: TorchObservation,
    actions: torch.Tensor,
    rewards: torch.Tensor,
    target: torch.Tensor,
    terminals: torch.Tensor,
    gamma: Union[float, torch.Tensor] = 0.99,
    masks: Optional[torch.Tensor] = None,
) -> torch.Tensor:
    assert target.ndim == 2
    td_sum = torch.tensor(
        0.0,
        dtype=torch.float32,
        device=get_device(observations),
    )
    for forwarder in forwarders:
        loss = forwarder.compute_error(
            observations=observations,
            actions=actions,
            rewards=rewards,
            target=target,
            terminals=terminals,
            gamma=gamma,
            reduction="none",
        )
        if masks is not None:
            loss = loss * masks
        td_sum += loss.mean()
    return td_sum

    def compute_error(
        self,
        observations: TorchObservation,
        actions: torch.Tensor,
        rewards: torch.Tensor,
        target: torch.Tensor,
        terminals: torch.Tensor,
        gamma: Union[float, torch.Tensor] = 0.99,
        masks: Optional[torch.Tensor] = None,
    ) -> torch.Tensor:
        return compute_ensemble_q_function_error(
            forwarders=self._forwarders,
            observations=observations,
            actions=actions,
            rewards=rewards,
            target=target,
            terminals=terminals,
            gamma=gamma,
            masks=masks,
        )


    def fit_with_transition_picker(
        self,
        episodes: Sequence[EpisodeBase],
        transition_picker: TransitionPickerProtocol,
    ) -> None:
        assert not self.built
        rewards = []
        for episode in episodes:
            for i in range(episode.transition_count):
                transition = transition_picker(episode, i)
                rewards.append(transition.reward)
        self.minimum = float(np.min(rewards))
        self.maximum = float(np.max(rewards))


    def ensure_cached(self) -> None:
        if self._catalog is None:
            raise RuntimeError(
                "cannot download file to cache because catalog is not setup"
            )
        client = self._catalog.get_client(self.source)
        client.download(self, callback=self._download_cb)

    def open(self, mode: Literal["rb", "r"] = "rb") -> Iterator[Any]:
        """Open the file and return a file object."""
        if self.location:
            with VFileRegistry.resolve(self, self.location) as f:  # type: ignore[arg-type]
                yield f

        else:
            if self._caching_enabled:
                self.ensure_cached()
            client: Client = self._catalog.get_client(self.source)
            with client.open_object(
                self, use_cache=self._caching_enabled, cb=self._download_cb
            ) as f:
                yield io.TextIOWrapper(f) if mode == "r" else f

    def _symlink_to(self, destination: str):
        if self.location:
            raise OSError(errno.ENOTSUP, "Symlinking virtual file is not supported")

        if self._caching_enabled:
            self.ensure_cached()
            source = self.get_local_path()
            assert source, "File was not cached"
        elif self.source.startswith("file://"):
            source = self.get_path()
        else:
            raise OSError(errno.EXDEV, "can't link across filesystems")

        return os.symlink(source, destination)


    def _get_flat_tree(
        self, tree: dict, prefix: list[str], depth: int
    ) -> Iterator[tuple[list[str], DataType, bool, int]]:
        for name, (type_, substree) in tree.items():
            suffix = name.split(".")
            new_prefix = prefix + suffix
            has_subtree = substree is not None
            yield new_prefix, type_, has_subtree, depth
            if substree is not None:
                yield from self._get_flat_tree(substree, new_prefix, depth + 1)

    def get_column_type(self, col_name: str, with_subtree: bool = False) -> DataType:
        """
        Returns column type by column name.

        If `with_subtree` is True, then it will return the type of the column
        even if it has a subtree (e.g. model with nested fields), otherwise it will
        return the type of the column (standard type field, not the model).

        If column is not found, raises `SignalResolvingError`.
        """
        for path, _type, has_subtree, _ in self.get_flat_tree():
            if (with_subtree or not has_subtree) and DEFAULT_DELIMITER.join(
                path
            ) == col_name:
                return _type
        raise SignalResolvingError([col_name], "is not found")

    def mutate(self, args_map: dict) -> "SignalSchema":
        new_values = self.values.copy()

        for name, value in args_map.items():
            if isinstance(value, Column) and value.name in self.values:
                # renaming existing signal
                del new_values[value.name]
                new_values[name] = self.values[value.name]
                continue
            if isinstance(value, Column):
                # adding new signal from existing signal field
                try:
                    new_values[name] = self.get_column_type(
                        value.name, with_subtree=True
                    )
                    continue
                except SignalResolvingError:
                    pass
            if isinstance(value, Func):
                # adding new signal with function
                new_values[name] = value.get_result_type(self)
                continue
            if isinstance(value, ColumnElement):
                # adding new signal
                new_values[name] = sql_to_python(value)
                continue
            new_values[name] = value

        return SignalSchema(new_values)


    def add(self, file: tarfile.TarInfo):
        fstream = File(path=file.name)
        ext = fstream.get_file_ext()
        stem = fstream.get_file_stem()

        if self.state.stem is not None and self.state.stem != stem:
            raise StopIteration

        if self.state.stem is None:
            self.state.stem = stem

        if ext in self._core_extensions:
            if self.state.core_file is not None:
                raise CoreFileDuplicationError(
                    self._tar_stream, file.name, self.state.core_file.name
                )
            self.state.core_file = file
        elif ext in self.state.data:
            raise WDSError(
                self._tar_stream,
                f"file with extension '.{ext}' already exists in the archive",
            )
        else:
            type_ = self._get_type(ext)
            if type_ is None:
                raise UnknownFileExtensionError(self._tar_stream, fstream.name, ext)

            if issubclass(type_, WDSReadableSubclass):
                reader = type_._reader
            else:
                reader = self.DEFAULT_TYPES_READERS.get(type_, None)

            if reader is None:
                raise WDSError(
                    self._tar_stream,
                    f"unable to find a reader for type {type_}, extension .{ext}",
                )
            self.state.data[ext] = reader(self, file)

def get_tar_groups(stream, tar, core_extensions, spec, encoding="utf-8"):
    builder = Builder(stream, core_extensions, spec, tar, encoding)

    for item in sorted(tar.getmembers(), key=lambda m: Path(m.name).stem):
        if not item.isfile():
            continue
        try:
            builder.add(item)
        except StopIteration:
            yield builder.produce()
            builder.add(item)
    if builder.state.stem is not None:
        yield builder.produce()


    def resolve_value(self) -> Optional[Any]:
        """Resolve the secret to an atomic value. The semantics of the value is secret-dependent."""
        out = None
        for env_var in self._env_vars:
            value = os.getenv(env_var)
            if value is not None:
                out = value
                break
        if out is None and self._strict:
            raise ValueError(f"None of the following authentication environment variables are set: {self._env_vars}")
        return out

    def warm_up(self):
        """
        Initializes the component.
        """
        if self.model is None:
            self.model = AutoModelForSequenceClassification.from_pretrained(
                self.model_name_or_path, token=self.token.resolve_value() if self.token else None, **self.model_kwargs
            )
            self.tokenizer = AutoTokenizer.from_pretrained(
                self.model_name_or_path,
                token=self.token.resolve_value() if self.token else None,
                **self.tokenizer_kwargs,
            )
            self.device = ComponentDevice.from_multiple(device_map=DeviceMap.from_hf(self.model.hf_device_map))