pydantic-evals 2.6.0
“Agent Framework / shim to use Pydantic with LLMs”
setup() call followed by other codeBundled Python dependency library path
SHA-2565345625469dfe969720567ca2e9a1ad16e1b974a6178e7d70dcb0d514ea847d6
MaleculeO(As)H(Db)
Evidence
115:16… erence is None:
116 return None
117 try:
118 trace_id = int(span_reference.trace_id, 16)
119 span_id = int(span_reference.span_id, 16)
120 except ValueError: # pragma: no cover — d …
1"""Online evaluation — attach evaluators to live functions for automatic background evaluation.
2
3This module provides the infrastructure for running evaluators on production (or staging) traffic.
4The same `Evaluator` instances used with `Dataset.evaluate()` work here, the difference is in how
5they are wired up (decorator vs dataset) rather than what they are.
6
7Example:
8```python
9from dataclasses import dataclass
10
11from pydantic_evals.evaluators import Evaluator, EvaluatorContext
12from pydantic_evals.online import evaluate
13
14
⋯12 lines
27from __future__ import annotations
28
29import asyncio
30import functools
31import inspect
32import threading
33from collections.abc import Awaitable, Callable, Generator, Iterable, Sequence
34from contextlib import contextmanager
35from dataclasses import dataclass
36from typing import Any, Literal, Protocol
37
38import anyio
39from opentelemetry import trace
40from typing_extensions import LiteralString, ParamSpec, TypeVar
41
42from . import _online as _online_internal, _task_run
43from ._online import CallbackSink, EvaluationSink, SinkPayload
44from ._utils import UNSET, Unset, logfire_span
45from .evaluators._run_evaluator import run_evaluator
46from .evaluators.context import EvaluatorContext
47from .evaluators.evaluator import EvaluationResult, Evaluator, EvaluatorFailure
48
195:67… heuristic should
196 run on 100% of traffic while an expensive LLM judge might run on only 1%.
197 """
198
⋯5 lines
323:48… s):
324 tg.start_soon(_run, i, evaluator)
325
326 for i in range(len(evaluators)):
327 result = results_by_index[i]
328 if isinstance(result, EvaluatorFailure):
329 all_failures.ap …
381:47… following `@logfire.instrument`'s convention."""
382 qualname = getattr(func, '__qualname__', getattr(func, '__name__', repr(func)))
383 module = inspect.getmodule(func)
384 module_name = getattr(mod …
No evidence locations were recorded for this file. Raw result