Config & YAML

Declarative suite loading and the custom-test registry.

modeltest.config

Declarative suite definition: load a ModelSuite from YAML.

The YAML format mirrors the built-in scenarios with a type and params:

.. code-block:: yaml

suite:
  name: "Credit Scoring Model"
  tests:
    - type: minimum_accuracy
      params:
        threshold: 0.85
    - type: group_performance
      params:
        metric: accuracy
        threshold: 0.8
        group_col: "gender"
    - type: robustness
      params: {noise_std: 0.01, max_drop: 0.03}
    - type: data_drift
      params: {features: [age, income], max_psi: 0.15}
    - type: equal_opportunity
      params: {protected_col: "gender", max_diff: 0.1}
    - type: statistical_parity
      params: {protected_col: "gender", max_diff: 0.1, min_ratio: 0.8}
    - type: feature_dominance
      params: {max_top_share: 0.9}
    - type: top_features
      params: {expected_features: [income, age], k: 2}
    - type: confidence_threshold
      params: {metric: accuracy, threshold: 0.85, n_boot: 1000, alpha: 0.05}
    - type: data_invariant
      params: {expected_columns: [age, income], max_null_ratio: 0.02}

load_suite_yaml(path)

Build a :class:ModelSuite from a YAML file.

Raises ValueError for unknown test types or malformed structure.

Source code in modeltest/config.py

def load_suite_yaml(path: str) -> ModelSuite:
    """Build a :class:`ModelSuite` from a YAML file.

    Raises ``ValueError`` for unknown test types or malformed structure.
    """
    with open(path) as fh:
        doc = yaml.safe_load(fh)

    suite_cfg = doc.get("suite")
    if not isinstance(suite_cfg, dict):
        raise ValueError("YAML must contain a top-level `suite:` mapping")

    name = suite_cfg.get("name", "suite")
    suite = ModelSuite(name=name)

    for raw in suite_cfg.get("tests", []):
        suite.add_test(_build_test(raw))

    return suite

dump_suite_yaml(suite, path)

Serialise a suite back to YAML (best-effort; only built-in tests).

Source code in modeltest/config.py

def dump_suite_yaml(suite: ModelSuite, path: str) -> None:
    """Serialise a suite back to YAML (best-effort; only built-in tests)."""
    import inspect

    doc = {"suite": {"name": suite.name, "tests": []}}
    for test in suite.tests:
        params = {
            k: v
            for k, v in vars(test).items()
            if not k.startswith("_")
            and k in inspect.signature(type(test).__init__).parameters
        }
        doc["suite"]["tests"].append({"type": _inverse_name(test), "params": params})
    with open(path, "w") as fh:
        yaml.safe_dump(doc, fh, sort_keys=False)

register(type_name, cls)

Map a YAML type string to a custom test class.

Custom tests must subclass :class:modeltest.ModelTest. Registering before :func:load_suite_yaml lets a suite reference your test by name::

from modeltest.config import register
register("my_error_check", MyErrorCheck)

# suite.yaml
# suite:
#   tests:
#     - type: my_error_check
#       params: {max_errors: 5}

Source code in modeltest/config.py

def register(type_name: str, cls: type) -> None:
    """Map a YAML ``type`` string to a custom test class.

    Custom tests must subclass :class:`modeltest.ModelTest`. Registering
    before :func:`load_suite_yaml` lets a suite reference your test by name::

        from modeltest.config import register
        register("my_error_check", MyErrorCheck)

        # suite.yaml
        # suite:
        #   tests:
        #     - type: my_error_check
        #       params: {max_errors: 5}
    """
    if not (isinstance(cls, type) and issubclass(cls, ModelTest)):
        raise TypeError(f"register() expects a ModelTest subclass, got {cls!r}")
    _REGISTRY[type_name] = cls

unregister(type_name)

Remove a previously registered type (built-ins included).

Source code in modeltest/config.py

def unregister(type_name: str) -> None:
    """Remove a previously registered type (built-ins included)."""
    _REGISTRY.pop(type_name, None)

modeltest.cli

Command-line interface: modeltest validate.

main(argv=None)

Entry point for the modeltest console script.

Args: argv: Command-line arguments (defaults to sys.argv[1:]).

Returns: Process exit code: 0 when the validated suite passes, 1 when any test fails.

Source code in modeltest/cli.py

def main(argv: Optional[list] = None) -> int:
    """Entry point for the ``modeltest`` console script.

    Args:
        argv: Command-line arguments (defaults to ``sys.argv[1:]``).

    Returns:
        Process exit code: ``0`` when the validated suite passes, ``1``
        when any test fails.
    """
    parser = argparse.ArgumentParser(
        prog="modeltest", description="Unit tests for machine learning models."
    )
    sub = parser.add_subparsers(dest="command", required=True)

    validate = sub.add_parser("validate", help="Run a test suite against a model.")
    validate.add_argument(
        "--suite",
        required=True,
        help="Path to suite.py (defining `suite`) or suite.yaml (declarative).",
    )
    validate.add_argument("--model", required=True, help="Path to model (pickle).")
    validate.add_argument("--data", required=True, help="Path to validation CSV.")
    validate.add_argument(
        "--target", default="target", help="Name of the target column."
    )
    validate.add_argument(
        "--output", default=None, help="Write JUnit XML to this path."
    )
    validate.add_argument(
        "--train-data", default=None, help="Optional training CSV (for drift tests)."
    )
    validate.set_defaults(func=_run_validate)

    args = parser.parse_args(argv)
    return args.func(args)