crossval

class VectorModelCrossValidationData(trained_models: Optional[List[sensai.evaluation.crossval.TModel]], eval_data_list: List[sensai.evaluation.crossval.TEvalData], predicted_var_names: List[str], test_indices_list=None)[source]

Bases: abc.ABC, Generic[sensai.evaluation.crossval.TModel, sensai.evaluation.crossval.TEvalData, sensai.evaluation.crossval.TEvalStats, sensai.evaluation.crossval.TEvalStatsCollection]

__init__(trained_models: Optional[List[sensai.evaluation.crossval.TModel]], eval_data_list: List[sensai.evaluation.crossval.TEvalData], predicted_var_names: List[str], test_indices_list=None)
property model_name
get_eval_stats_collection(predicted_var_name=None) → sensai.evaluation.crossval.TEvalStatsCollection
iter_input_output_ground_truth_tuples(predicted_var_name=None) → Generator[Tuple[sensai.util.typing.PandasNamedTuple, Any, Any], None, None]
track_metrics(tracking_context: sensai.tracking.tracking_base.TrackingContext)
class CrossValidationSplitter[source]

Bases: abc.ABC

Defines a mechanism with which to generate data splits for cross-validation

abstract create_folds(data: sensai.data.InputOutputData, num_folds: int) → List[Tuple[Sequence[int], Sequence[int]]]
Parameters

  • data – the data from which to obtain the folds

  • num_folds – the number of splits/folds

Returns

a list containing numFolds tuples (t, e) where t and e are sequences of data point indices to use for training and evaluation respectively

class CrossValidationSplitterDefault(shuffle=True, random_seed=42)[source]

Bases: sensai.evaluation.crossval.CrossValidationSplitter

__init__(shuffle=True, random_seed=42)
create_folds(data: sensai.data.InputOutputData, num_splits: int) → List[Tuple[Sequence[int], Sequence[int]]]
Parameters

  • data – the data from which to obtain the folds

  • num_folds – the number of splits/folds

Returns

a list containing numFolds tuples (t, e) where t and e are sequences of data point indices to use for training and evaluation respectively

class CrossValidationSplitterNested(test_fraction: float)[source]

Bases: sensai.evaluation.crossval.CrossValidationSplitter

A data splitter for nested cross-validation (which is useful, in particular, for time series prediction problems)

__init__(test_fraction: float)
create_folds(data: sensai.data.InputOutputData, num_folds: int) → List[Tuple[Sequence[int], Sequence[int]]]
Parameters

  • data – the data from which to obtain the folds

  • num_folds – the number of splits/folds

Returns

a list containing numFolds tuples (t, e) where t and e are sequences of data point indices to use for training and evaluation respectively

class VectorModelCrossValidatorParams(folds: int = 5, splitter: Optional[sensai.evaluation.crossval.CrossValidationSplitter] = None, return_trained_models=False, evaluator_params: Optional[Union[sensai.evaluation.evaluator.RegressionEvaluatorParams, sensai.evaluation.evaluator.ClassificationEvaluatorParams]] = None, default_splitter_random_seed=42, default_splitter_shuffle=True)[source]

Bases: object

__init__(folds: int = 5, splitter: Optional[sensai.evaluation.crossval.CrossValidationSplitter] = None, return_trained_models=False, evaluator_params: Optional[Union[sensai.evaluation.evaluator.RegressionEvaluatorParams, sensai.evaluation.evaluator.ClassificationEvaluatorParams]] = None, default_splitter_random_seed=42, default_splitter_shuffle=True)
Parameters

  • folds – the number of folds

  • splitter – the splitter to use in order to generate the folds; if None, use default split (using parameters for random seed and shuffling below)

  • return_trained_models – whether to create a copy of the model for each fold and return each of the models (requires that models can be deep-copied); if False, the model that is passed to evalModel is fitted several times

  • evaluator_params – the model evaluator parameters

  • default_splitter_random_seed – [if splitter is None] the random seed to use for splits

  • default_splitter_shuffle – [if splitter is None] whether to shuffle the data (using randomSeed) before creating the folds

class VectorModelCrossValidator(data: sensai.data.InputOutputData, params: sensai.evaluation.crossval.VectorModelCrossValidatorParams)[source]

Bases: sensai.evaluation.evaluator.MetricsDictProvider, Generic[sensai.evaluation.crossval.TCrossValData], abc.ABC

__init__(data: sensai.data.InputOutputData, params: sensai.evaluation.crossval.VectorModelCrossValidatorParams)
Parameters

  • data – the data set

  • params – parameters

static for_model(model: sensai.vector_model.VectorModel, data: sensai.data.InputOutputData, params: sensai.evaluation.crossval.VectorModelCrossValidatorParams) → Union[sensai.evaluation.crossval.VectorClassificationModelCrossValidator, sensai.evaluation.crossval.VectorRegressionModelCrossValidator]
eval_model(model: sensai.vector_model.VectorModel, track: bool = True)
Parameters

  • model – the model to evaluate

  • track – whether tracking shall be enabled for the case where a tracked experiment is set on this object

Returns

cross-validation results

create_metrics_dict_provider(predicted_var_name: Optional[str]) → sensai.evaluation.evaluator.MetricsDictProvider

Creates a metrics dictionary provider, e.g. for use in hyperparameter optimisation

Parameters

predicted_var_name – the name of the predicted variable for which to obtain evaluation metrics; may be None only if the model outputs but a single predicted variable

Returns

a metrics dictionary provider instance for the given variable

class VectorRegressionModelCrossValidationData(trained_models: Optional[List[sensai.evaluation.crossval.TModel]], eval_data_list: List[sensai.evaluation.crossval.TEvalData], predicted_var_names: List[str], test_indices_list=None)[source]

Bases: sensai.evaluation.crossval.VectorModelCrossValidationData[sensai.vector_model.VectorRegressionModel, sensai.evaluation.evaluator.VectorRegressionModelEvaluationData, sensai.evaluation.eval_stats.eval_stats_regression.RegressionEvalStats, sensai.evaluation.eval_stats.eval_stats_regression.RegressionEvalStatsCollection]

class VectorRegressionModelCrossValidator(data: sensai.data.InputOutputData, params: sensai.evaluation.crossval.VectorModelCrossValidatorParams)[source]

Bases: sensai.evaluation.crossval.VectorModelCrossValidator[sensai.evaluation.crossval.VectorRegressionModelCrossValidationData]

class VectorClassificationModelCrossValidationData(trained_models: Optional[List[sensai.evaluation.crossval.TModel]], eval_data_list: List[sensai.evaluation.crossval.TEvalData], predicted_var_names: List[str], test_indices_list=None)[source]

Bases: sensai.evaluation.crossval.VectorModelCrossValidationData[sensai.vector_model.VectorClassificationModel, sensai.evaluation.evaluator.VectorClassificationModelEvaluationData, sensai.evaluation.eval_stats.eval_stats_classification.ClassificationEvalStats, sensai.evaluation.eval_stats.eval_stats_classification.ClassificationEvalStatsCollection]

class VectorClassificationModelCrossValidator(data: sensai.data.InputOutputData, params: sensai.evaluation.crossval.VectorModelCrossValidatorParams)[source]

Bases: sensai.evaluation.crossval.VectorModelCrossValidator[sensai.evaluation.crossval.VectorClassificationModelCrossValidationData]