diff --git a/langfuse/api/__init__.py b/langfuse/api/__init__.py index ebb0bea02..06204d072 100644 --- a/langfuse/api/__init__.py +++ b/langfuse/api/__init__.py @@ -14,6 +14,10 @@ dataset_items, dataset_run_items, datasets, + evaluation_commons, + evaluation_errors, + evaluation_rules, + evaluators, experiments, feedback, health, @@ -157,6 +161,117 @@ PaginatedDatasetRuns, PaginatedDatasets, ) + from .evaluation_commons import ( + ArrayOptionsEvaluationRuleFilter, + BooleanEvaluationRuleFilter, + CategoryOptionsEvaluationRuleFilter, + CodeEvaluatorSourceCodeLanguage, + DateTimeEvaluationRuleFilter, + EvaluationRuleArrayOptionsFilterOperator, + EvaluationRuleBooleanFilterOperator, + EvaluationRuleFilter, + EvaluationRuleFilter_ArrayOptions, + EvaluationRuleFilter_Boolean, + EvaluationRuleFilter_CategoryOptions, + EvaluationRuleFilter_Datetime, + EvaluationRuleFilter_Null, + EvaluationRuleFilter_Number, + EvaluationRuleFilter_NumberObject, + EvaluationRuleFilter_String, + EvaluationRuleFilter_StringObject, + EvaluationRuleFilter_StringOptions, + EvaluationRuleNullFilterOperator, + EvaluationRuleNumberFilterOperator, + EvaluationRuleOptionsFilterOperator, + EvaluationRuleReadFilter, + EvaluationRuleReadFilterBase, + EvaluationRuleReadFilterWithKey, + EvaluationRuleStringFilterOperator, + EvaluatorOutputDefinition, + EvaluatorOutputDefinitionBase, + EvaluatorOutputDefinition_Boolean, + EvaluatorOutputDefinition_Categorical, + EvaluatorOutputDefinition_Numeric, + EvaluatorOutputScoreType, + EvaluatorType, + LegacyEvaluationObject, + LegacyPromptVariableMapping, + NullEvaluationRuleFilter, + NumberEvaluationRuleFilter, + NumberObjectEvaluationRuleFilter, + PromptVariableMapping, + PromptVariableMappingInput, + PromptVariableMappingRead, + PromptVariableMappingSource, + PublicEvaluatorBooleanScore, + PublicEvaluatorCategoricalScore, + PublicEvaluatorNumericScore, + PublicEvaluatorOutputDefinition, + PublicEvaluatorOutputDefinition_Boolean, + PublicEvaluatorOutputDefinition_Categorical, + PublicEvaluatorOutputDefinition_Numeric, + StringEvaluationRuleFilter, + StringObjectEvaluationRuleFilter, + StringOptionsEvaluationRuleFilter, + ) + from .evaluation_errors import ( + BadRequestError, + ConflictError, + EvaluationAccessDeniedError, + EvaluationMethodNotAllowedError, + EvaluationNotFoundError, + EvaluationUnauthorizedError, + InternalServerError, + PreconditionFailedError, + PublicApiError, + PublicApiErrorCode, + PublicApiErrorDetails, + PublicApiValidationIssue, + TooManyRequestsError, + ) + from .evaluation_rules import ( + CreateEvaluationRuleRequest, + DeletedEvaluationRule, + EvaluationRule, + EvaluationRuleEvaluatorAssignmentInput, + EvaluationRulesPage, + EvaluatorAssignment, + UpdateEvaluationRuleRequest, + ) + from .evaluators import ( + CodeEvaluator, + CodeEvaluatorVersion, + CreateCodeEvaluatorRequest, + CreateEvaluatorRequest, + CreateEvaluatorRequest_Code, + CreateEvaluatorRequest_LlmAsJudge, + CreateLlmAsJudgeEvaluatorRequest, + Creator, + CursorMeta, + DeletedEvaluator, + EvaluationRuleAssignment, + Evaluator, + EvaluatorBase, + EvaluatorChatMessage, + EvaluatorChatPrompt, + EvaluatorChatPromptInput, + EvaluatorModelConfig, + EvaluatorStatus, + EvaluatorVersion, + EvaluatorVersionBase, + EvaluatorVersion_Code, + EvaluatorVersion_LlmAsJudge, + EvaluatorVersionsPage, + Evaluator_Code, + Evaluator_LlmAsJudge, + EvaluatorsPage, + LlmAsJudgeEvaluator, + LlmAsJudgeEvaluatorVersion, + UpdateCodeEvaluatorRequest, + UpdateEvaluatorMetadataRequest, + UpdateEvaluatorRequest, + UpdateLlmAsJudgeEvaluatorRequest, + ) from .experiments import ( Experiment, ExperimentItem, @@ -365,8 +480,10 @@ "ApiKeyList": ".projects", "ApiKeyResponse": ".projects", "ApiKeySummary": ".projects", + "ArrayOptionsEvaluationRuleFilter": ".evaluation_commons", "AsyncLangfuseAPI": ".client", "AuthenticationScheme": ".scim", + "BadRequestError": ".evaluation_errors", "BaseEvent": ".ingestion", "BasePrompt": ".prompts", "BaseScore": ".commons", @@ -384,6 +501,7 @@ "BlobStorageIntegrationType": ".blob_storage_integrations", "BlobStorageIntegrationsResponse": ".blob_storage_integrations", "BlobStorageSyncStatus": ".blob_storage_integrations", + "BooleanEvaluationRuleFilter": ".evaluation_commons", "BooleanScore": ".commons", "BooleanScoreV1": ".commons", "BooleanScoreV3": ".scores_v3", @@ -391,13 +509,18 @@ "CategoricalScore": ".commons", "CategoricalScoreV1": ".commons", "CategoricalScoreV3": ".scores_v3", + "CategoryOptionsEvaluationRuleFilter": ".evaluation_commons", "ChatMessage": ".prompts", "ChatMessageType": ".prompts", "ChatMessageWithPlaceholders": ".prompts", "ChatPrompt": ".prompts", + "CodeEvaluator": ".evaluators", + "CodeEvaluatorSourceCodeLanguage": ".evaluation_commons", + "CodeEvaluatorVersion": ".evaluators", "Comment": ".commons", "CommentObjectType": ".commons", "ConfigCategory": ".commons", + "ConflictError": ".evaluation_errors", "CorrectionScore": ".commons", "CorrectionScoreV3": ".scores_v3", "CreateAnnotationQueueAssignmentResponse": ".annotation_queues", @@ -406,15 +529,21 @@ "CreateBlobStorageIntegrationRequest": ".blob_storage_integrations", "CreateChatPromptRequest": ".prompts", "CreateChatPromptType": ".prompts", + "CreateCodeEvaluatorRequest": ".evaluators", "CreateCommentRequest": ".comments", "CreateCommentResponse": ".comments", "CreateDatasetItemRequest": ".dataset_items", "CreateDatasetRequest": ".datasets", "CreateDatasetRunItemRequest": ".dataset_run_items", + "CreateEvaluationRuleRequest": ".evaluation_rules", + "CreateEvaluatorRequest": ".evaluators", + "CreateEvaluatorRequest_Code": ".evaluators", + "CreateEvaluatorRequest_LlmAsJudge": ".evaluators", "CreateEventBody": ".ingestion", "CreateEventEvent": ".ingestion", "CreateGenerationBody": ".ingestion", "CreateGenerationEvent": ".ingestion", + "CreateLlmAsJudgeEvaluatorRequest": ".evaluators", "CreateModelRequest": ".models", "CreateObservationEvent": ".ingestion", "CreatePromptRequest": ".prompts", @@ -427,6 +556,8 @@ "CreateSpanEvent": ".ingestion", "CreateTextPromptRequest": ".prompts", "CreateTextPromptType": ".prompts", + "Creator": ".evaluators", + "CursorMeta": ".evaluators", "Dataset": ".commons", "DatasetItem": ".commons", "DatasetItemMediaReference": ".commons", @@ -436,6 +567,7 @@ "DatasetRunItem": ".commons", "DatasetRunWithItems": ".commons", "DatasetStatus": ".commons", + "DateTimeEvaluationRuleFilter": ".evaluation_commons", "DeleteAnnotationQueueAssignmentResponse": ".annotation_queues", "DeleteAnnotationQueueItemResponse": ".annotation_queues", "DeleteDatasetItemResponse": ".dataset_items", @@ -443,9 +575,62 @@ "DeleteLlmConnectionResponse": ".llm_connections", "DeleteMembershipRequest": ".organizations", "DeleteTraceResponse": ".trace", + "DeletedEvaluationRule": ".evaluation_rules", + "DeletedEvaluator": ".evaluators", "Deprecation": ".commons", "EmptyResponse": ".scim", "Error": ".commons", + "EvaluationAccessDeniedError": ".evaluation_errors", + "EvaluationMethodNotAllowedError": ".evaluation_errors", + "EvaluationNotFoundError": ".evaluation_errors", + "EvaluationRule": ".evaluation_rules", + "EvaluationRuleArrayOptionsFilterOperator": ".evaluation_commons", + "EvaluationRuleAssignment": ".evaluators", + "EvaluationRuleBooleanFilterOperator": ".evaluation_commons", + "EvaluationRuleEvaluatorAssignmentInput": ".evaluation_rules", + "EvaluationRuleFilter": ".evaluation_commons", + "EvaluationRuleFilter_ArrayOptions": ".evaluation_commons", + "EvaluationRuleFilter_Boolean": ".evaluation_commons", + "EvaluationRuleFilter_CategoryOptions": ".evaluation_commons", + "EvaluationRuleFilter_Datetime": ".evaluation_commons", + "EvaluationRuleFilter_Null": ".evaluation_commons", + "EvaluationRuleFilter_Number": ".evaluation_commons", + "EvaluationRuleFilter_NumberObject": ".evaluation_commons", + "EvaluationRuleFilter_String": ".evaluation_commons", + "EvaluationRuleFilter_StringObject": ".evaluation_commons", + "EvaluationRuleFilter_StringOptions": ".evaluation_commons", + "EvaluationRuleNullFilterOperator": ".evaluation_commons", + "EvaluationRuleNumberFilterOperator": ".evaluation_commons", + "EvaluationRuleOptionsFilterOperator": ".evaluation_commons", + "EvaluationRuleReadFilter": ".evaluation_commons", + "EvaluationRuleReadFilterBase": ".evaluation_commons", + "EvaluationRuleReadFilterWithKey": ".evaluation_commons", + "EvaluationRuleStringFilterOperator": ".evaluation_commons", + "EvaluationRulesPage": ".evaluation_rules", + "EvaluationUnauthorizedError": ".evaluation_errors", + "Evaluator": ".evaluators", + "EvaluatorAssignment": ".evaluation_rules", + "EvaluatorBase": ".evaluators", + "EvaluatorChatMessage": ".evaluators", + "EvaluatorChatPrompt": ".evaluators", + "EvaluatorChatPromptInput": ".evaluators", + "EvaluatorModelConfig": ".evaluators", + "EvaluatorOutputDefinition": ".evaluation_commons", + "EvaluatorOutputDefinitionBase": ".evaluation_commons", + "EvaluatorOutputDefinition_Boolean": ".evaluation_commons", + "EvaluatorOutputDefinition_Categorical": ".evaluation_commons", + "EvaluatorOutputDefinition_Numeric": ".evaluation_commons", + "EvaluatorOutputScoreType": ".evaluation_commons", + "EvaluatorStatus": ".evaluators", + "EvaluatorType": ".evaluation_commons", + "EvaluatorVersion": ".evaluators", + "EvaluatorVersionBase": ".evaluators", + "EvaluatorVersion_Code": ".evaluators", + "EvaluatorVersion_LlmAsJudge": ".evaluators", + "EvaluatorVersionsPage": ".evaluators", + "Evaluator_Code": ".evaluators", + "Evaluator_LlmAsJudge": ".evaluators", + "EvaluatorsPage": ".evaluators", "Experiment": ".experiments", "ExperimentItem": ".experiments", "ExperimentItemsResponse": ".experiments", @@ -489,8 +674,13 @@ "IngestionResponse": ".ingestion", "IngestionSuccess": ".ingestion", "IngestionUsage": ".ingestion", + "InternalServerError": ".evaluation_errors", "LangfuseAPI": ".client", + "LegacyEvaluationObject": ".evaluation_commons", + "LegacyPromptVariableMapping": ".evaluation_commons", "LlmAdapter": ".llm_connections", + "LlmAsJudgeEvaluator": ".evaluators", + "LlmAsJudgeEvaluatorVersion": ".evaluators", "LlmConnection": ".llm_connections", "MapValue": ".commons", "MediaContentType": ".media", @@ -506,6 +696,9 @@ "ModelTokenizerId": ".models", "ModelUsageUnit": ".commons", "NotFoundError": ".commons", + "NullEvaluationRuleFilter": ".evaluation_commons", + "NumberEvaluationRuleFilter": ".evaluation_commons", + "NumberObjectEvaluationRuleFilter": ".evaluation_commons", "NumericScore": ".commons", "NumericScoreV1": ".commons", "NumericScoreV3": ".scores_v3", @@ -547,6 +740,7 @@ "PatchMediaBody": ".media", "PlaceholderMessage": ".prompts", "PlaceholderMessageType": ".prompts", + "PreconditionFailedError": ".evaluation_errors", "PricingTier": ".commons", "PricingTierAttributeCondition": ".commons", "PricingTierAttributeSource": ".commons", @@ -563,8 +757,23 @@ "PromptMeta": ".prompts", "PromptMetaListResponse": ".prompts", "PromptType": ".prompts", + "PromptVariableMapping": ".evaluation_commons", + "PromptVariableMappingInput": ".evaluation_commons", + "PromptVariableMappingRead": ".evaluation_commons", + "PromptVariableMappingSource": ".evaluation_commons", "Prompt_Chat": ".prompts", "Prompt_Text": ".prompts", + "PublicApiError": ".evaluation_errors", + "PublicApiErrorCode": ".evaluation_errors", + "PublicApiErrorDetails": ".evaluation_errors", + "PublicApiValidationIssue": ".evaluation_errors", + "PublicEvaluatorBooleanScore": ".evaluation_commons", + "PublicEvaluatorCategoricalScore": ".evaluation_commons", + "PublicEvaluatorNumericScore": ".evaluation_commons", + "PublicEvaluatorOutputDefinition": ".evaluation_commons", + "PublicEvaluatorOutputDefinition_Boolean": ".evaluation_commons", + "PublicEvaluatorOutputDefinition_Categorical": ".evaluation_commons", + "PublicEvaluatorOutputDefinition_Numeric": ".evaluation_commons", "ResourceMeta": ".scim", "ResourceType": ".scim", "ResourceTypesResponse": ".scim", @@ -616,12 +825,16 @@ "Session": ".commons", "SessionWithTraces": ".commons", "Sort": ".trace", + "StringEvaluationRuleFilter": ".evaluation_commons", + "StringObjectEvaluationRuleFilter": ".evaluation_commons", + "StringOptionsEvaluationRuleFilter": ".evaluation_commons", "SubmitFeedbackRequest": ".feedback", "SubmitFeedbackResponse": ".feedback", "TextPrompt": ".prompts", "TextScore": ".commons", "TextScoreV1": ".commons", "TextScoreV3": ".scores_v3", + "TooManyRequestsError": ".evaluation_errors", "Trace": ".commons", "TraceBody": ".ingestion", "TraceEvent": ".ingestion", @@ -630,9 +843,14 @@ "Traces": ".trace", "UnauthorizedError": ".commons", "UpdateAnnotationQueueItemRequest": ".annotation_queues", + "UpdateCodeEvaluatorRequest": ".evaluators", + "UpdateEvaluationRuleRequest": ".evaluation_rules", + "UpdateEvaluatorMetadataRequest": ".evaluators", + "UpdateEvaluatorRequest": ".evaluators", "UpdateEventBody": ".ingestion", "UpdateGenerationBody": ".ingestion", "UpdateGenerationEvent": ".ingestion", + "UpdateLlmAsJudgeEvaluatorRequest": ".evaluators", "UpdateObservationEvent": ".ingestion", "UpdateScoreConfigRequest": ".score_configs", "UpdateSpanBody": ".ingestion", @@ -648,6 +866,10 @@ "dataset_items": ".dataset_items", "dataset_run_items": ".dataset_run_items", "datasets": ".datasets", + "evaluation_commons": ".evaluation_commons", + "evaluation_errors": ".evaluation_errors", + "evaluation_rules": ".evaluation_rules", + "evaluators": ".evaluators", "experiments": ".experiments", "feedback": ".feedback", "health": ".health", @@ -712,8 +934,10 @@ def __dir__(): "ApiKeyList", "ApiKeyResponse", "ApiKeySummary", + "ArrayOptionsEvaluationRuleFilter", "AsyncLangfuseAPI", "AuthenticationScheme", + "BadRequestError", "BaseEvent", "BasePrompt", "BaseScore", @@ -731,6 +955,7 @@ def __dir__(): "BlobStorageIntegrationType", "BlobStorageIntegrationsResponse", "BlobStorageSyncStatus", + "BooleanEvaluationRuleFilter", "BooleanScore", "BooleanScoreV1", "BooleanScoreV3", @@ -738,13 +963,18 @@ def __dir__(): "CategoricalScore", "CategoricalScoreV1", "CategoricalScoreV3", + "CategoryOptionsEvaluationRuleFilter", "ChatMessage", "ChatMessageType", "ChatMessageWithPlaceholders", "ChatPrompt", + "CodeEvaluator", + "CodeEvaluatorSourceCodeLanguage", + "CodeEvaluatorVersion", "Comment", "CommentObjectType", "ConfigCategory", + "ConflictError", "CorrectionScore", "CorrectionScoreV3", "CreateAnnotationQueueAssignmentResponse", @@ -753,15 +983,21 @@ def __dir__(): "CreateBlobStorageIntegrationRequest", "CreateChatPromptRequest", "CreateChatPromptType", + "CreateCodeEvaluatorRequest", "CreateCommentRequest", "CreateCommentResponse", "CreateDatasetItemRequest", "CreateDatasetRequest", "CreateDatasetRunItemRequest", + "CreateEvaluationRuleRequest", + "CreateEvaluatorRequest", + "CreateEvaluatorRequest_Code", + "CreateEvaluatorRequest_LlmAsJudge", "CreateEventBody", "CreateEventEvent", "CreateGenerationBody", "CreateGenerationEvent", + "CreateLlmAsJudgeEvaluatorRequest", "CreateModelRequest", "CreateObservationEvent", "CreatePromptRequest", @@ -774,6 +1010,8 @@ def __dir__(): "CreateSpanEvent", "CreateTextPromptRequest", "CreateTextPromptType", + "Creator", + "CursorMeta", "Dataset", "DatasetItem", "DatasetItemMediaReference", @@ -783,6 +1021,7 @@ def __dir__(): "DatasetRunItem", "DatasetRunWithItems", "DatasetStatus", + "DateTimeEvaluationRuleFilter", "DeleteAnnotationQueueAssignmentResponse", "DeleteAnnotationQueueItemResponse", "DeleteDatasetItemResponse", @@ -790,9 +1029,62 @@ def __dir__(): "DeleteLlmConnectionResponse", "DeleteMembershipRequest", "DeleteTraceResponse", + "DeletedEvaluationRule", + "DeletedEvaluator", "Deprecation", "EmptyResponse", "Error", + "EvaluationAccessDeniedError", + "EvaluationMethodNotAllowedError", + "EvaluationNotFoundError", + "EvaluationRule", + "EvaluationRuleArrayOptionsFilterOperator", + "EvaluationRuleAssignment", + "EvaluationRuleBooleanFilterOperator", + "EvaluationRuleEvaluatorAssignmentInput", + "EvaluationRuleFilter", + "EvaluationRuleFilter_ArrayOptions", + "EvaluationRuleFilter_Boolean", + "EvaluationRuleFilter_CategoryOptions", + "EvaluationRuleFilter_Datetime", + "EvaluationRuleFilter_Null", + "EvaluationRuleFilter_Number", + "EvaluationRuleFilter_NumberObject", + "EvaluationRuleFilter_String", + "EvaluationRuleFilter_StringObject", + "EvaluationRuleFilter_StringOptions", + "EvaluationRuleNullFilterOperator", + "EvaluationRuleNumberFilterOperator", + "EvaluationRuleOptionsFilterOperator", + "EvaluationRuleReadFilter", + "EvaluationRuleReadFilterBase", + "EvaluationRuleReadFilterWithKey", + "EvaluationRuleStringFilterOperator", + "EvaluationRulesPage", + "EvaluationUnauthorizedError", + "Evaluator", + "EvaluatorAssignment", + "EvaluatorBase", + "EvaluatorChatMessage", + "EvaluatorChatPrompt", + "EvaluatorChatPromptInput", + "EvaluatorModelConfig", + "EvaluatorOutputDefinition", + "EvaluatorOutputDefinitionBase", + "EvaluatorOutputDefinition_Boolean", + "EvaluatorOutputDefinition_Categorical", + "EvaluatorOutputDefinition_Numeric", + "EvaluatorOutputScoreType", + "EvaluatorStatus", + "EvaluatorType", + "EvaluatorVersion", + "EvaluatorVersionBase", + "EvaluatorVersion_Code", + "EvaluatorVersion_LlmAsJudge", + "EvaluatorVersionsPage", + "Evaluator_Code", + "Evaluator_LlmAsJudge", + "EvaluatorsPage", "Experiment", "ExperimentItem", "ExperimentItemsResponse", @@ -836,8 +1128,13 @@ def __dir__(): "IngestionResponse", "IngestionSuccess", "IngestionUsage", + "InternalServerError", "LangfuseAPI", + "LegacyEvaluationObject", + "LegacyPromptVariableMapping", "LlmAdapter", + "LlmAsJudgeEvaluator", + "LlmAsJudgeEvaluatorVersion", "LlmConnection", "MapValue", "MediaContentType", @@ -853,6 +1150,9 @@ def __dir__(): "ModelTokenizerId", "ModelUsageUnit", "NotFoundError", + "NullEvaluationRuleFilter", + "NumberEvaluationRuleFilter", + "NumberObjectEvaluationRuleFilter", "NumericScore", "NumericScoreV1", "NumericScoreV3", @@ -894,6 +1194,7 @@ def __dir__(): "PatchMediaBody", "PlaceholderMessage", "PlaceholderMessageType", + "PreconditionFailedError", "PricingTier", "PricingTierAttributeCondition", "PricingTierAttributeSource", @@ -910,8 +1211,23 @@ def __dir__(): "PromptMeta", "PromptMetaListResponse", "PromptType", + "PromptVariableMapping", + "PromptVariableMappingInput", + "PromptVariableMappingRead", + "PromptVariableMappingSource", "Prompt_Chat", "Prompt_Text", + "PublicApiError", + "PublicApiErrorCode", + "PublicApiErrorDetails", + "PublicApiValidationIssue", + "PublicEvaluatorBooleanScore", + "PublicEvaluatorCategoricalScore", + "PublicEvaluatorNumericScore", + "PublicEvaluatorOutputDefinition", + "PublicEvaluatorOutputDefinition_Boolean", + "PublicEvaluatorOutputDefinition_Categorical", + "PublicEvaluatorOutputDefinition_Numeric", "ResourceMeta", "ResourceType", "ResourceTypesResponse", @@ -963,12 +1279,16 @@ def __dir__(): "Session", "SessionWithTraces", "Sort", + "StringEvaluationRuleFilter", + "StringObjectEvaluationRuleFilter", + "StringOptionsEvaluationRuleFilter", "SubmitFeedbackRequest", "SubmitFeedbackResponse", "TextPrompt", "TextScore", "TextScoreV1", "TextScoreV3", + "TooManyRequestsError", "Trace", "TraceBody", "TraceEvent", @@ -977,9 +1297,14 @@ def __dir__(): "Traces", "UnauthorizedError", "UpdateAnnotationQueueItemRequest", + "UpdateCodeEvaluatorRequest", + "UpdateEvaluationRuleRequest", + "UpdateEvaluatorMetadataRequest", + "UpdateEvaluatorRequest", "UpdateEventBody", "UpdateGenerationBody", "UpdateGenerationEvent", + "UpdateLlmAsJudgeEvaluatorRequest", "UpdateObservationEvent", "UpdateScoreConfigRequest", "UpdateSpanBody", @@ -995,6 +1320,10 @@ def __dir__(): "dataset_items", "dataset_run_items", "datasets", + "evaluation_commons", + "evaluation_errors", + "evaluation_rules", + "evaluators", "experiments", "feedback", "health", diff --git a/langfuse/api/client.py b/langfuse/api/client.py index 7fc58ca40..3781b4fcc 100644 --- a/langfuse/api/client.py +++ b/langfuse/api/client.py @@ -23,6 +23,11 @@ DatasetRunItemsClient, ) from .datasets.client import AsyncDatasetsClient, DatasetsClient + from .evaluation_rules.client import ( + AsyncEvaluationRulesClient, + EvaluationRulesClient, + ) + from .evaluators.client import AsyncEvaluatorsClient, EvaluatorsClient from .experiments.client import AsyncExperimentsClient, ExperimentsClient from .feedback.client import AsyncFeedbackClient, FeedbackClient from .health.client import AsyncHealthClient, HealthClient @@ -133,6 +138,8 @@ def __init__( self._dataset_items: typing.Optional[DatasetItemsClient] = None self._dataset_run_items: typing.Optional[DatasetRunItemsClient] = None self._datasets: typing.Optional[DatasetsClient] = None + self._evaluation_rules: typing.Optional[EvaluationRulesClient] = None + self._evaluators: typing.Optional[EvaluatorsClient] = None self._experiments: typing.Optional[ExperimentsClient] = None self._feedback: typing.Optional[FeedbackClient] = None self._health: typing.Optional[HealthClient] = None @@ -212,6 +219,24 @@ def datasets(self): self._datasets = DatasetsClient(client_wrapper=self._client_wrapper) return self._datasets + @property + def evaluation_rules(self): + if self._evaluation_rules is None: + from .evaluation_rules.client import EvaluationRulesClient # noqa: E402 + + self._evaluation_rules = EvaluationRulesClient( + client_wrapper=self._client_wrapper + ) + return self._evaluation_rules + + @property + def evaluators(self): + if self._evaluators is None: + from .evaluators.client import EvaluatorsClient # noqa: E402 + + self._evaluators = EvaluatorsClient(client_wrapper=self._client_wrapper) + return self._evaluators + @property def experiments(self): if self._experiments is None: @@ -485,6 +510,8 @@ def __init__( self._dataset_items: typing.Optional[AsyncDatasetItemsClient] = None self._dataset_run_items: typing.Optional[AsyncDatasetRunItemsClient] = None self._datasets: typing.Optional[AsyncDatasetsClient] = None + self._evaluation_rules: typing.Optional[AsyncEvaluationRulesClient] = None + self._evaluators: typing.Optional[AsyncEvaluatorsClient] = None self._experiments: typing.Optional[AsyncExperimentsClient] = None self._feedback: typing.Optional[AsyncFeedbackClient] = None self._health: typing.Optional[AsyncHealthClient] = None @@ -566,6 +593,26 @@ def datasets(self): self._datasets = AsyncDatasetsClient(client_wrapper=self._client_wrapper) return self._datasets + @property + def evaluation_rules(self): + if self._evaluation_rules is None: + from .evaluation_rules.client import AsyncEvaluationRulesClient # noqa: E402 + + self._evaluation_rules = AsyncEvaluationRulesClient( + client_wrapper=self._client_wrapper + ) + return self._evaluation_rules + + @property + def evaluators(self): + if self._evaluators is None: + from .evaluators.client import AsyncEvaluatorsClient # noqa: E402 + + self._evaluators = AsyncEvaluatorsClient( + client_wrapper=self._client_wrapper + ) + return self._evaluators + @property def experiments(self): if self._experiments is None: diff --git a/langfuse/api/evaluation_commons/__init__.py b/langfuse/api/evaluation_commons/__init__.py new file mode 100644 index 000000000..00912e7ce --- /dev/null +++ b/langfuse/api/evaluation_commons/__init__.py @@ -0,0 +1,196 @@ +# This file was auto-generated by Fern from our API Definition. + +# isort: skip_file + +import typing +from importlib import import_module + +if typing.TYPE_CHECKING: + from .types import ( + ArrayOptionsEvaluationRuleFilter, + BooleanEvaluationRuleFilter, + CategoryOptionsEvaluationRuleFilter, + CodeEvaluatorSourceCodeLanguage, + DateTimeEvaluationRuleFilter, + EvaluationRuleArrayOptionsFilterOperator, + EvaluationRuleBooleanFilterOperator, + EvaluationRuleFilter, + EvaluationRuleFilter_ArrayOptions, + EvaluationRuleFilter_Boolean, + EvaluationRuleFilter_CategoryOptions, + EvaluationRuleFilter_Datetime, + EvaluationRuleFilter_Null, + EvaluationRuleFilter_Number, + EvaluationRuleFilter_NumberObject, + EvaluationRuleFilter_String, + EvaluationRuleFilter_StringObject, + EvaluationRuleFilter_StringOptions, + EvaluationRuleNullFilterOperator, + EvaluationRuleNumberFilterOperator, + EvaluationRuleOptionsFilterOperator, + EvaluationRuleReadFilter, + EvaluationRuleReadFilterBase, + EvaluationRuleReadFilterWithKey, + EvaluationRuleStringFilterOperator, + EvaluatorOutputDefinition, + EvaluatorOutputDefinitionBase, + EvaluatorOutputDefinition_Boolean, + EvaluatorOutputDefinition_Categorical, + EvaluatorOutputDefinition_Numeric, + EvaluatorOutputScoreType, + EvaluatorType, + LegacyEvaluationObject, + LegacyPromptVariableMapping, + NullEvaluationRuleFilter, + NumberEvaluationRuleFilter, + NumberObjectEvaluationRuleFilter, + PromptVariableMapping, + PromptVariableMappingInput, + PromptVariableMappingRead, + PromptVariableMappingSource, + PublicEvaluatorBooleanScore, + PublicEvaluatorCategoricalScore, + PublicEvaluatorNumericScore, + PublicEvaluatorOutputDefinition, + PublicEvaluatorOutputDefinition_Boolean, + PublicEvaluatorOutputDefinition_Categorical, + PublicEvaluatorOutputDefinition_Numeric, + StringEvaluationRuleFilter, + StringObjectEvaluationRuleFilter, + StringOptionsEvaluationRuleFilter, + ) +_dynamic_imports: typing.Dict[str, str] = { + "ArrayOptionsEvaluationRuleFilter": ".types", + "BooleanEvaluationRuleFilter": ".types", + "CategoryOptionsEvaluationRuleFilter": ".types", + "CodeEvaluatorSourceCodeLanguage": ".types", + "DateTimeEvaluationRuleFilter": ".types", + "EvaluationRuleArrayOptionsFilterOperator": ".types", + "EvaluationRuleBooleanFilterOperator": ".types", + "EvaluationRuleFilter": ".types", + "EvaluationRuleFilter_ArrayOptions": ".types", + "EvaluationRuleFilter_Boolean": ".types", + "EvaluationRuleFilter_CategoryOptions": ".types", + "EvaluationRuleFilter_Datetime": ".types", + "EvaluationRuleFilter_Null": ".types", + "EvaluationRuleFilter_Number": ".types", + "EvaluationRuleFilter_NumberObject": ".types", + "EvaluationRuleFilter_String": ".types", + "EvaluationRuleFilter_StringObject": ".types", + "EvaluationRuleFilter_StringOptions": ".types", + "EvaluationRuleNullFilterOperator": ".types", + "EvaluationRuleNumberFilterOperator": ".types", + "EvaluationRuleOptionsFilterOperator": ".types", + "EvaluationRuleReadFilter": ".types", + "EvaluationRuleReadFilterBase": ".types", + "EvaluationRuleReadFilterWithKey": ".types", + "EvaluationRuleStringFilterOperator": ".types", + "EvaluatorOutputDefinition": ".types", + "EvaluatorOutputDefinitionBase": ".types", + "EvaluatorOutputDefinition_Boolean": ".types", + "EvaluatorOutputDefinition_Categorical": ".types", + "EvaluatorOutputDefinition_Numeric": ".types", + "EvaluatorOutputScoreType": ".types", + "EvaluatorType": ".types", + "LegacyEvaluationObject": ".types", + "LegacyPromptVariableMapping": ".types", + "NullEvaluationRuleFilter": ".types", + "NumberEvaluationRuleFilter": ".types", + "NumberObjectEvaluationRuleFilter": ".types", + "PromptVariableMapping": ".types", + "PromptVariableMappingInput": ".types", + "PromptVariableMappingRead": ".types", + "PromptVariableMappingSource": ".types", + "PublicEvaluatorBooleanScore": ".types", + "PublicEvaluatorCategoricalScore": ".types", + "PublicEvaluatorNumericScore": ".types", + "PublicEvaluatorOutputDefinition": ".types", + "PublicEvaluatorOutputDefinition_Boolean": ".types", + "PublicEvaluatorOutputDefinition_Categorical": ".types", + "PublicEvaluatorOutputDefinition_Numeric": ".types", + "StringEvaluationRuleFilter": ".types", + "StringObjectEvaluationRuleFilter": ".types", + "StringOptionsEvaluationRuleFilter": ".types", +} + + +def __getattr__(attr_name: str) -> typing.Any: + module_name = _dynamic_imports.get(attr_name) + if module_name is None: + raise AttributeError( + f"No {attr_name} found in _dynamic_imports for module name -> {__name__}" + ) + try: + module = import_module(module_name, __package__) + if module_name == f".{attr_name}": + return module + else: + return getattr(module, attr_name) + except ImportError as e: + raise ImportError( + f"Failed to import {attr_name} from {module_name}: {e}" + ) from e + except AttributeError as e: + raise AttributeError( + f"Failed to get {attr_name} from {module_name}: {e}" + ) from e + + +def __dir__(): + lazy_attrs = list(_dynamic_imports.keys()) + return sorted(lazy_attrs) + + +__all__ = [ + "ArrayOptionsEvaluationRuleFilter", + "BooleanEvaluationRuleFilter", + "CategoryOptionsEvaluationRuleFilter", + "CodeEvaluatorSourceCodeLanguage", + "DateTimeEvaluationRuleFilter", + "EvaluationRuleArrayOptionsFilterOperator", + "EvaluationRuleBooleanFilterOperator", + "EvaluationRuleFilter", + "EvaluationRuleFilter_ArrayOptions", + "EvaluationRuleFilter_Boolean", + "EvaluationRuleFilter_CategoryOptions", + "EvaluationRuleFilter_Datetime", + "EvaluationRuleFilter_Null", + "EvaluationRuleFilter_Number", + "EvaluationRuleFilter_NumberObject", + "EvaluationRuleFilter_String", + "EvaluationRuleFilter_StringObject", + "EvaluationRuleFilter_StringOptions", + "EvaluationRuleNullFilterOperator", + "EvaluationRuleNumberFilterOperator", + "EvaluationRuleOptionsFilterOperator", + "EvaluationRuleReadFilter", + "EvaluationRuleReadFilterBase", + "EvaluationRuleReadFilterWithKey", + "EvaluationRuleStringFilterOperator", + "EvaluatorOutputDefinition", + "EvaluatorOutputDefinitionBase", + "EvaluatorOutputDefinition_Boolean", + "EvaluatorOutputDefinition_Categorical", + "EvaluatorOutputDefinition_Numeric", + "EvaluatorOutputScoreType", + "EvaluatorType", + "LegacyEvaluationObject", + "LegacyPromptVariableMapping", + "NullEvaluationRuleFilter", + "NumberEvaluationRuleFilter", + "NumberObjectEvaluationRuleFilter", + "PromptVariableMapping", + "PromptVariableMappingInput", + "PromptVariableMappingRead", + "PromptVariableMappingSource", + "PublicEvaluatorBooleanScore", + "PublicEvaluatorCategoricalScore", + "PublicEvaluatorNumericScore", + "PublicEvaluatorOutputDefinition", + "PublicEvaluatorOutputDefinition_Boolean", + "PublicEvaluatorOutputDefinition_Categorical", + "PublicEvaluatorOutputDefinition_Numeric", + "StringEvaluationRuleFilter", + "StringObjectEvaluationRuleFilter", + "StringOptionsEvaluationRuleFilter", +] diff --git a/langfuse/api/evaluation_commons/types/__init__.py b/langfuse/api/evaluation_commons/types/__init__.py new file mode 100644 index 000000000..d04011e88 --- /dev/null +++ b/langfuse/api/evaluation_commons/types/__init__.py @@ -0,0 +1,212 @@ +# This file was auto-generated by Fern from our API Definition. + +# isort: skip_file + +import typing +from importlib import import_module + +if typing.TYPE_CHECKING: + from .array_options_evaluation_rule_filter import ArrayOptionsEvaluationRuleFilter + from .boolean_evaluation_rule_filter import BooleanEvaluationRuleFilter + from .category_options_evaluation_rule_filter import ( + CategoryOptionsEvaluationRuleFilter, + ) + from .code_evaluator_source_code_language import CodeEvaluatorSourceCodeLanguage + from .date_time_evaluation_rule_filter import DateTimeEvaluationRuleFilter + from .evaluation_rule_array_options_filter_operator import ( + EvaluationRuleArrayOptionsFilterOperator, + ) + from .evaluation_rule_boolean_filter_operator import ( + EvaluationRuleBooleanFilterOperator, + ) + from .evaluation_rule_filter import ( + EvaluationRuleFilter, + EvaluationRuleFilter_ArrayOptions, + EvaluationRuleFilter_Boolean, + EvaluationRuleFilter_CategoryOptions, + EvaluationRuleFilter_Datetime, + EvaluationRuleFilter_Null, + EvaluationRuleFilter_Number, + EvaluationRuleFilter_NumberObject, + EvaluationRuleFilter_String, + EvaluationRuleFilter_StringObject, + EvaluationRuleFilter_StringOptions, + ) + from .evaluation_rule_null_filter_operator import EvaluationRuleNullFilterOperator + from .evaluation_rule_number_filter_operator import ( + EvaluationRuleNumberFilterOperator, + ) + from .evaluation_rule_options_filter_operator import ( + EvaluationRuleOptionsFilterOperator, + ) + from .evaluation_rule_read_filter import EvaluationRuleReadFilter + from .evaluation_rule_read_filter_base import EvaluationRuleReadFilterBase + from .evaluation_rule_read_filter_with_key import EvaluationRuleReadFilterWithKey + from .evaluation_rule_string_filter_operator import ( + EvaluationRuleStringFilterOperator, + ) + from .evaluator_output_definition import ( + EvaluatorOutputDefinition, + EvaluatorOutputDefinition_Boolean, + EvaluatorOutputDefinition_Categorical, + EvaluatorOutputDefinition_Numeric, + ) + from .evaluator_output_definition_base import EvaluatorOutputDefinitionBase + from .evaluator_output_score_type import EvaluatorOutputScoreType + from .evaluator_type import EvaluatorType + from .legacy_evaluation_object import LegacyEvaluationObject + from .legacy_prompt_variable_mapping import LegacyPromptVariableMapping + from .null_evaluation_rule_filter import NullEvaluationRuleFilter + from .number_evaluation_rule_filter import NumberEvaluationRuleFilter + from .number_object_evaluation_rule_filter import NumberObjectEvaluationRuleFilter + from .prompt_variable_mapping import PromptVariableMapping + from .prompt_variable_mapping_input import PromptVariableMappingInput + from .prompt_variable_mapping_read import PromptVariableMappingRead + from .prompt_variable_mapping_source import PromptVariableMappingSource + from .public_evaluator_boolean_score import PublicEvaluatorBooleanScore + from .public_evaluator_categorical_score import PublicEvaluatorCategoricalScore + from .public_evaluator_numeric_score import PublicEvaluatorNumericScore + from .public_evaluator_output_definition import ( + PublicEvaluatorOutputDefinition, + PublicEvaluatorOutputDefinition_Boolean, + PublicEvaluatorOutputDefinition_Categorical, + PublicEvaluatorOutputDefinition_Numeric, + ) + from .string_evaluation_rule_filter import StringEvaluationRuleFilter + from .string_object_evaluation_rule_filter import StringObjectEvaluationRuleFilter + from .string_options_evaluation_rule_filter import StringOptionsEvaluationRuleFilter +_dynamic_imports: typing.Dict[str, str] = { + "ArrayOptionsEvaluationRuleFilter": ".array_options_evaluation_rule_filter", + "BooleanEvaluationRuleFilter": ".boolean_evaluation_rule_filter", + "CategoryOptionsEvaluationRuleFilter": ".category_options_evaluation_rule_filter", + "CodeEvaluatorSourceCodeLanguage": ".code_evaluator_source_code_language", + "DateTimeEvaluationRuleFilter": ".date_time_evaluation_rule_filter", + "EvaluationRuleArrayOptionsFilterOperator": ".evaluation_rule_array_options_filter_operator", + "EvaluationRuleBooleanFilterOperator": ".evaluation_rule_boolean_filter_operator", + "EvaluationRuleFilter": ".evaluation_rule_filter", + "EvaluationRuleFilter_ArrayOptions": ".evaluation_rule_filter", + "EvaluationRuleFilter_Boolean": ".evaluation_rule_filter", + "EvaluationRuleFilter_CategoryOptions": ".evaluation_rule_filter", + "EvaluationRuleFilter_Datetime": ".evaluation_rule_filter", + "EvaluationRuleFilter_Null": ".evaluation_rule_filter", + "EvaluationRuleFilter_Number": ".evaluation_rule_filter", + "EvaluationRuleFilter_NumberObject": ".evaluation_rule_filter", + "EvaluationRuleFilter_String": ".evaluation_rule_filter", + "EvaluationRuleFilter_StringObject": ".evaluation_rule_filter", + "EvaluationRuleFilter_StringOptions": ".evaluation_rule_filter", + "EvaluationRuleNullFilterOperator": ".evaluation_rule_null_filter_operator", + "EvaluationRuleNumberFilterOperator": ".evaluation_rule_number_filter_operator", + "EvaluationRuleOptionsFilterOperator": ".evaluation_rule_options_filter_operator", + "EvaluationRuleReadFilter": ".evaluation_rule_read_filter", + "EvaluationRuleReadFilterBase": ".evaluation_rule_read_filter_base", + "EvaluationRuleReadFilterWithKey": ".evaluation_rule_read_filter_with_key", + "EvaluationRuleStringFilterOperator": ".evaluation_rule_string_filter_operator", + "EvaluatorOutputDefinition": ".evaluator_output_definition", + "EvaluatorOutputDefinitionBase": ".evaluator_output_definition_base", + "EvaluatorOutputDefinition_Boolean": ".evaluator_output_definition", + "EvaluatorOutputDefinition_Categorical": ".evaluator_output_definition", + "EvaluatorOutputDefinition_Numeric": ".evaluator_output_definition", + "EvaluatorOutputScoreType": ".evaluator_output_score_type", + "EvaluatorType": ".evaluator_type", + "LegacyEvaluationObject": ".legacy_evaluation_object", + "LegacyPromptVariableMapping": ".legacy_prompt_variable_mapping", + "NullEvaluationRuleFilter": ".null_evaluation_rule_filter", + "NumberEvaluationRuleFilter": ".number_evaluation_rule_filter", + "NumberObjectEvaluationRuleFilter": ".number_object_evaluation_rule_filter", + "PromptVariableMapping": ".prompt_variable_mapping", + "PromptVariableMappingInput": ".prompt_variable_mapping_input", + "PromptVariableMappingRead": ".prompt_variable_mapping_read", + "PromptVariableMappingSource": ".prompt_variable_mapping_source", + "PublicEvaluatorBooleanScore": ".public_evaluator_boolean_score", + "PublicEvaluatorCategoricalScore": ".public_evaluator_categorical_score", + "PublicEvaluatorNumericScore": ".public_evaluator_numeric_score", + "PublicEvaluatorOutputDefinition": ".public_evaluator_output_definition", + "PublicEvaluatorOutputDefinition_Boolean": ".public_evaluator_output_definition", + "PublicEvaluatorOutputDefinition_Categorical": ".public_evaluator_output_definition", + "PublicEvaluatorOutputDefinition_Numeric": ".public_evaluator_output_definition", + "StringEvaluationRuleFilter": ".string_evaluation_rule_filter", + "StringObjectEvaluationRuleFilter": ".string_object_evaluation_rule_filter", + "StringOptionsEvaluationRuleFilter": ".string_options_evaluation_rule_filter", +} + + +def __getattr__(attr_name: str) -> typing.Any: + module_name = _dynamic_imports.get(attr_name) + if module_name is None: + raise AttributeError( + f"No {attr_name} found in _dynamic_imports for module name -> {__name__}" + ) + try: + module = import_module(module_name, __package__) + if module_name == f".{attr_name}": + return module + else: + return getattr(module, attr_name) + except ImportError as e: + raise ImportError( + f"Failed to import {attr_name} from {module_name}: {e}" + ) from e + except AttributeError as e: + raise AttributeError( + f"Failed to get {attr_name} from {module_name}: {e}" + ) from e + + +def __dir__(): + lazy_attrs = list(_dynamic_imports.keys()) + return sorted(lazy_attrs) + + +__all__ = [ + "ArrayOptionsEvaluationRuleFilter", + "BooleanEvaluationRuleFilter", + "CategoryOptionsEvaluationRuleFilter", + "CodeEvaluatorSourceCodeLanguage", + "DateTimeEvaluationRuleFilter", + "EvaluationRuleArrayOptionsFilterOperator", + "EvaluationRuleBooleanFilterOperator", + "EvaluationRuleFilter", + "EvaluationRuleFilter_ArrayOptions", + "EvaluationRuleFilter_Boolean", + "EvaluationRuleFilter_CategoryOptions", + "EvaluationRuleFilter_Datetime", + "EvaluationRuleFilter_Null", + "EvaluationRuleFilter_Number", + "EvaluationRuleFilter_NumberObject", + "EvaluationRuleFilter_String", + "EvaluationRuleFilter_StringObject", + "EvaluationRuleFilter_StringOptions", + "EvaluationRuleNullFilterOperator", + "EvaluationRuleNumberFilterOperator", + "EvaluationRuleOptionsFilterOperator", + "EvaluationRuleReadFilter", + "EvaluationRuleReadFilterBase", + "EvaluationRuleReadFilterWithKey", + "EvaluationRuleStringFilterOperator", + "EvaluatorOutputDefinition", + "EvaluatorOutputDefinitionBase", + "EvaluatorOutputDefinition_Boolean", + "EvaluatorOutputDefinition_Categorical", + "EvaluatorOutputDefinition_Numeric", + "EvaluatorOutputScoreType", + "EvaluatorType", + "LegacyEvaluationObject", + "LegacyPromptVariableMapping", + "NullEvaluationRuleFilter", + "NumberEvaluationRuleFilter", + "NumberObjectEvaluationRuleFilter", + "PromptVariableMapping", + "PromptVariableMappingInput", + "PromptVariableMappingRead", + "PromptVariableMappingSource", + "PublicEvaluatorBooleanScore", + "PublicEvaluatorCategoricalScore", + "PublicEvaluatorNumericScore", + "PublicEvaluatorOutputDefinition", + "PublicEvaluatorOutputDefinition_Boolean", + "PublicEvaluatorOutputDefinition_Categorical", + "PublicEvaluatorOutputDefinition_Numeric", + "StringEvaluationRuleFilter", + "StringObjectEvaluationRuleFilter", + "StringOptionsEvaluationRuleFilter", +] diff --git a/langfuse/api/evaluation_commons/types/array_options_evaluation_rule_filter.py b/langfuse/api/evaluation_commons/types/array_options_evaluation_rule_filter.py new file mode 100644 index 000000000..9d87c0c9f --- /dev/null +++ b/langfuse/api/evaluation_commons/types/array_options_evaluation_rule_filter.py @@ -0,0 +1,26 @@ +# This file was auto-generated by Fern from our API Definition. + +import typing + +import pydantic +from ...core.pydantic_utilities import UniversalBaseModel +from .evaluation_rule_array_options_filter_operator import ( + EvaluationRuleArrayOptionsFilterOperator, +) + + +class ArrayOptionsEvaluationRuleFilter(UniversalBaseModel): + column: str = pydantic.Field() + """ + Column to filter on. + """ + + operator: EvaluationRuleArrayOptionsFilterOperator + value: typing.List[str] = pydantic.Field() + """ + One or more array elements to match. + """ + + model_config: typing.ClassVar[pydantic.ConfigDict] = pydantic.ConfigDict( + extra="allow", frozen=True + ) diff --git a/langfuse/api/evaluation_commons/types/boolean_evaluation_rule_filter.py b/langfuse/api/evaluation_commons/types/boolean_evaluation_rule_filter.py new file mode 100644 index 000000000..f3c6dd25f --- /dev/null +++ b/langfuse/api/evaluation_commons/types/boolean_evaluation_rule_filter.py @@ -0,0 +1,21 @@ +# This file was auto-generated by Fern from our API Definition. + +import typing + +import pydantic +from ...core.pydantic_utilities import UniversalBaseModel +from .evaluation_rule_boolean_filter_operator import EvaluationRuleBooleanFilterOperator + + +class BooleanEvaluationRuleFilter(UniversalBaseModel): + column: str = pydantic.Field() + """ + Column to filter on. + """ + + operator: EvaluationRuleBooleanFilterOperator + value: bool + + model_config: typing.ClassVar[pydantic.ConfigDict] = pydantic.ConfigDict( + extra="allow", frozen=True + ) diff --git a/langfuse/api/evaluation_commons/types/category_options_evaluation_rule_filter.py b/langfuse/api/evaluation_commons/types/category_options_evaluation_rule_filter.py new file mode 100644 index 000000000..18cc111a8 --- /dev/null +++ b/langfuse/api/evaluation_commons/types/category_options_evaluation_rule_filter.py @@ -0,0 +1,26 @@ +# This file was auto-generated by Fern from our API Definition. + +import typing + +import pydantic +from ...core.pydantic_utilities import UniversalBaseModel +from .evaluation_rule_options_filter_operator import EvaluationRuleOptionsFilterOperator + + +class CategoryOptionsEvaluationRuleFilter(UniversalBaseModel): + column: str = pydantic.Field() + """ + Object-valued column to filter on. + """ + + key: str = pydantic.Field() + """ + Key inside the object-valued column to filter on. + """ + + operator: EvaluationRuleOptionsFilterOperator + value: typing.List[str] + + model_config: typing.ClassVar[pydantic.ConfigDict] = pydantic.ConfigDict( + extra="allow", frozen=True + ) diff --git a/langfuse/api/evaluation_commons/types/code_evaluator_source_code_language.py b/langfuse/api/evaluation_commons/types/code_evaluator_source_code_language.py new file mode 100644 index 000000000..a34761538 --- /dev/null +++ b/langfuse/api/evaluation_commons/types/code_evaluator_source_code_language.py @@ -0,0 +1,26 @@ +# This file was auto-generated by Fern from our API Definition. + +import typing + +from ...core import enum + +T_Result = typing.TypeVar("T_Result") + + +class CodeEvaluatorSourceCodeLanguage(enum.StrEnum): + """ + Code evaluator runtime language. + """ + + PYTHON = "PYTHON" + TYPESCRIPT = "TYPESCRIPT" + + def visit( + self, + python: typing.Callable[[], T_Result], + typescript: typing.Callable[[], T_Result], + ) -> T_Result: + if self is CodeEvaluatorSourceCodeLanguage.PYTHON: + return python() + if self is CodeEvaluatorSourceCodeLanguage.TYPESCRIPT: + return typescript() diff --git a/langfuse/api/evaluation_commons/types/date_time_evaluation_rule_filter.py b/langfuse/api/evaluation_commons/types/date_time_evaluation_rule_filter.py new file mode 100644 index 000000000..c3acab66e --- /dev/null +++ b/langfuse/api/evaluation_commons/types/date_time_evaluation_rule_filter.py @@ -0,0 +1,29 @@ +# This file was auto-generated by Fern from our API Definition. + +import datetime as dt +import typing + +import pydantic +from ...core.pydantic_utilities import UniversalBaseModel +from .evaluation_rule_number_filter_operator import EvaluationRuleNumberFilterOperator + + +class DateTimeEvaluationRuleFilter(UniversalBaseModel): + column: str = pydantic.Field() + """ + Column to filter on. + """ + + operator: EvaluationRuleNumberFilterOperator = pydantic.Field() + """ + Comparison operator for datetime values. + """ + + value: dt.datetime = pydantic.Field() + """ + Datetime value to compare against. + """ + + model_config: typing.ClassVar[pydantic.ConfigDict] = pydantic.ConfigDict( + extra="allow", frozen=True + ) diff --git a/langfuse/api/evaluation_commons/types/evaluation_rule_array_options_filter_operator.py b/langfuse/api/evaluation_commons/types/evaluation_rule_array_options_filter_operator.py new file mode 100644 index 000000000..1c90b50f8 --- /dev/null +++ b/langfuse/api/evaluation_commons/types/evaluation_rule_array_options_filter_operator.py @@ -0,0 +1,26 @@ +# This file was auto-generated by Fern from our API Definition. + +import typing + +from ...core import enum + +T_Result = typing.TypeVar("T_Result") + + +class EvaluationRuleArrayOptionsFilterOperator(enum.StrEnum): + ANY_OF = "any of" + NONE_OF = "none of" + ALL_OF = "all of" + + def visit( + self, + any_of: typing.Callable[[], T_Result], + none_of: typing.Callable[[], T_Result], + all_of: typing.Callable[[], T_Result], + ) -> T_Result: + if self is EvaluationRuleArrayOptionsFilterOperator.ANY_OF: + return any_of() + if self is EvaluationRuleArrayOptionsFilterOperator.NONE_OF: + return none_of() + if self is EvaluationRuleArrayOptionsFilterOperator.ALL_OF: + return all_of() diff --git a/langfuse/api/evaluation_commons/types/evaluation_rule_boolean_filter_operator.py b/langfuse/api/evaluation_commons/types/evaluation_rule_boolean_filter_operator.py new file mode 100644 index 000000000..8e47e8a5f --- /dev/null +++ b/langfuse/api/evaluation_commons/types/evaluation_rule_boolean_filter_operator.py @@ -0,0 +1,22 @@ +# This file was auto-generated by Fern from our API Definition. + +import typing + +from ...core import enum + +T_Result = typing.TypeVar("T_Result") + + +class EvaluationRuleBooleanFilterOperator(enum.StrEnum): + EQUALS = "=" + NOT_EQUALS = "<>" + + def visit( + self, + equals: typing.Callable[[], T_Result], + not_equals: typing.Callable[[], T_Result], + ) -> T_Result: + if self is EvaluationRuleBooleanFilterOperator.EQUALS: + return equals() + if self is EvaluationRuleBooleanFilterOperator.NOT_EQUALS: + return not_equals() diff --git a/langfuse/api/evaluation_commons/types/evaluation_rule_filter.py b/langfuse/api/evaluation_commons/types/evaluation_rule_filter.py new file mode 100644 index 000000000..f281dbb17 --- /dev/null +++ b/langfuse/api/evaluation_commons/types/evaluation_rule_filter.py @@ -0,0 +1,680 @@ +# This file was auto-generated by Fern from our API Definition. + +from __future__ import annotations + +import datetime as dt +import typing + +import pydantic +import typing_extensions +from ...core.pydantic_utilities import UniversalBaseModel +from .evaluation_rule_array_options_filter_operator import ( + EvaluationRuleArrayOptionsFilterOperator, +) +from .evaluation_rule_boolean_filter_operator import EvaluationRuleBooleanFilterOperator +from .evaluation_rule_null_filter_operator import EvaluationRuleNullFilterOperator +from .evaluation_rule_number_filter_operator import EvaluationRuleNumberFilterOperator +from .evaluation_rule_options_filter_operator import EvaluationRuleOptionsFilterOperator +from .evaluation_rule_string_filter_operator import EvaluationRuleStringFilterOperator + + +class EvaluationRuleFilter_Datetime(UniversalBaseModel): + """ + One filter condition used to decide whether a live-ingested observation should be evaluated. + + All filters must be satisfied for the rule to run. Experiment scope is expressed with filters rather than a separate target field: + - `isExperimentItemRootSpan = true` limits execution to experiment item roots. + - `datasetId` limits execution to experiments for the selected datasets. Use dataset IDs from `GET /api/public/v2/datasets`. + + Pick the filter `type` first, as it determines the required fields and value shape. Use `key` only for object filters such as `metadata`. + + Operator quick reference by filter `type`: + - `string`: `"="`, `contains`, `does not contain`, `starts with`, `ends with` + - `number`: `"="`, `">"`, `"<"`, `">="`, `"<="` + - `datetime`: `"="`, `">"`, `"<"`, `">="`, `"<="` + - `stringOptions`: `any of`, `none of` + - `arrayOptions`: `any of`, `none of`, `all of` + - `stringObject`: same operators as `string` + - `boolean`: `"="`, `"<>"` + - `null`: `is null`, `is not null` + + Supported columns. Each column accepts exactly one filter `type`; sending a column with a different `type` is rejected with `400`. + - `type`: `stringOptions`, values `SPAN`, `EVENT`, `GENERATION`, `AGENT`, `TOOL`, `CHAIN`, `RETRIEVER`, `EVALUATOR`, `EMBEDDING`, `GUARDRAIL` + - `name`: `stringOptions` + - `environment`: `stringOptions` + - `level`: `stringOptions`, values `DEBUG`, `DEFAULT`, `WARNING`, `ERROR` + - `version`: `string` + - `traceName`: `stringOptions` + - `userId`: `string` + - `sessionId`: `string` + - `tags`: `arrayOptions` + - `metadata`: `stringObject`, requires `key` + - `isRootObservation`: `boolean`; true when the observation has no parent or is explicitly marked as an application root + - `parentObservationId`: `null` + - `experimentId`: `stringOptions` + - `isExperimentItemRootSpan`: `boolean` + - `calledToolNames`: `arrayOptions` + - `toolCalls`: `number`, the number of tool calls on the observation + - `datasetId`: `stringOptions`. Use dataset `id` values from `GET /api/public/v2/datasets`, not dataset names. + + Examples + -------- + from langfuse.evaluation_commons import ( + EvaluationRuleFilter_StringOptions, + EvaluationRuleOptionsFilterOperator, + ) + + EvaluationRuleFilter_StringOptions( + column="type", + operator=EvaluationRuleOptionsFilterOperator.ANY_OF, + value=["GENERATION"], + ) + """ + + type: typing.Literal["datetime"] = "datetime" + column: str + operator: EvaluationRuleNumberFilterOperator + value: dt.datetime + + model_config: typing.ClassVar[pydantic.ConfigDict] = pydantic.ConfigDict( + extra="allow", frozen=True + ) + + +class EvaluationRuleFilter_String(UniversalBaseModel): + """ + One filter condition used to decide whether a live-ingested observation should be evaluated. + + All filters must be satisfied for the rule to run. Experiment scope is expressed with filters rather than a separate target field: + - `isExperimentItemRootSpan = true` limits execution to experiment item roots. + - `datasetId` limits execution to experiments for the selected datasets. Use dataset IDs from `GET /api/public/v2/datasets`. + + Pick the filter `type` first, as it determines the required fields and value shape. Use `key` only for object filters such as `metadata`. + + Operator quick reference by filter `type`: + - `string`: `"="`, `contains`, `does not contain`, `starts with`, `ends with` + - `number`: `"="`, `">"`, `"<"`, `">="`, `"<="` + - `datetime`: `"="`, `">"`, `"<"`, `">="`, `"<="` + - `stringOptions`: `any of`, `none of` + - `arrayOptions`: `any of`, `none of`, `all of` + - `stringObject`: same operators as `string` + - `boolean`: `"="`, `"<>"` + - `null`: `is null`, `is not null` + + Supported columns. Each column accepts exactly one filter `type`; sending a column with a different `type` is rejected with `400`. + - `type`: `stringOptions`, values `SPAN`, `EVENT`, `GENERATION`, `AGENT`, `TOOL`, `CHAIN`, `RETRIEVER`, `EVALUATOR`, `EMBEDDING`, `GUARDRAIL` + - `name`: `stringOptions` + - `environment`: `stringOptions` + - `level`: `stringOptions`, values `DEBUG`, `DEFAULT`, `WARNING`, `ERROR` + - `version`: `string` + - `traceName`: `stringOptions` + - `userId`: `string` + - `sessionId`: `string` + - `tags`: `arrayOptions` + - `metadata`: `stringObject`, requires `key` + - `isRootObservation`: `boolean`; true when the observation has no parent or is explicitly marked as an application root + - `parentObservationId`: `null` + - `experimentId`: `stringOptions` + - `isExperimentItemRootSpan`: `boolean` + - `calledToolNames`: `arrayOptions` + - `toolCalls`: `number`, the number of tool calls on the observation + - `datasetId`: `stringOptions`. Use dataset `id` values from `GET /api/public/v2/datasets`, not dataset names. + + Examples + -------- + from langfuse.evaluation_commons import ( + EvaluationRuleFilter_StringOptions, + EvaluationRuleOptionsFilterOperator, + ) + + EvaluationRuleFilter_StringOptions( + column="type", + operator=EvaluationRuleOptionsFilterOperator.ANY_OF, + value=["GENERATION"], + ) + """ + + type: typing.Literal["string"] = "string" + column: str + operator: EvaluationRuleStringFilterOperator + value: str + + model_config: typing.ClassVar[pydantic.ConfigDict] = pydantic.ConfigDict( + extra="allow", frozen=True + ) + + +class EvaluationRuleFilter_Number(UniversalBaseModel): + """ + One filter condition used to decide whether a live-ingested observation should be evaluated. + + All filters must be satisfied for the rule to run. Experiment scope is expressed with filters rather than a separate target field: + - `isExperimentItemRootSpan = true` limits execution to experiment item roots. + - `datasetId` limits execution to experiments for the selected datasets. Use dataset IDs from `GET /api/public/v2/datasets`. + + Pick the filter `type` first, as it determines the required fields and value shape. Use `key` only for object filters such as `metadata`. + + Operator quick reference by filter `type`: + - `string`: `"="`, `contains`, `does not contain`, `starts with`, `ends with` + - `number`: `"="`, `">"`, `"<"`, `">="`, `"<="` + - `datetime`: `"="`, `">"`, `"<"`, `">="`, `"<="` + - `stringOptions`: `any of`, `none of` + - `arrayOptions`: `any of`, `none of`, `all of` + - `stringObject`: same operators as `string` + - `boolean`: `"="`, `"<>"` + - `null`: `is null`, `is not null` + + Supported columns. Each column accepts exactly one filter `type`; sending a column with a different `type` is rejected with `400`. + - `type`: `stringOptions`, values `SPAN`, `EVENT`, `GENERATION`, `AGENT`, `TOOL`, `CHAIN`, `RETRIEVER`, `EVALUATOR`, `EMBEDDING`, `GUARDRAIL` + - `name`: `stringOptions` + - `environment`: `stringOptions` + - `level`: `stringOptions`, values `DEBUG`, `DEFAULT`, `WARNING`, `ERROR` + - `version`: `string` + - `traceName`: `stringOptions` + - `userId`: `string` + - `sessionId`: `string` + - `tags`: `arrayOptions` + - `metadata`: `stringObject`, requires `key` + - `isRootObservation`: `boolean`; true when the observation has no parent or is explicitly marked as an application root + - `parentObservationId`: `null` + - `experimentId`: `stringOptions` + - `isExperimentItemRootSpan`: `boolean` + - `calledToolNames`: `arrayOptions` + - `toolCalls`: `number`, the number of tool calls on the observation + - `datasetId`: `stringOptions`. Use dataset `id` values from `GET /api/public/v2/datasets`, not dataset names. + + Examples + -------- + from langfuse.evaluation_commons import ( + EvaluationRuleFilter_StringOptions, + EvaluationRuleOptionsFilterOperator, + ) + + EvaluationRuleFilter_StringOptions( + column="type", + operator=EvaluationRuleOptionsFilterOperator.ANY_OF, + value=["GENERATION"], + ) + """ + + type: typing.Literal["number"] = "number" + column: str + operator: EvaluationRuleNumberFilterOperator + value: float + + model_config: typing.ClassVar[pydantic.ConfigDict] = pydantic.ConfigDict( + extra="allow", frozen=True + ) + + +class EvaluationRuleFilter_StringOptions(UniversalBaseModel): + """ + One filter condition used to decide whether a live-ingested observation should be evaluated. + + All filters must be satisfied for the rule to run. Experiment scope is expressed with filters rather than a separate target field: + - `isExperimentItemRootSpan = true` limits execution to experiment item roots. + - `datasetId` limits execution to experiments for the selected datasets. Use dataset IDs from `GET /api/public/v2/datasets`. + + Pick the filter `type` first, as it determines the required fields and value shape. Use `key` only for object filters such as `metadata`. + + Operator quick reference by filter `type`: + - `string`: `"="`, `contains`, `does not contain`, `starts with`, `ends with` + - `number`: `"="`, `">"`, `"<"`, `">="`, `"<="` + - `datetime`: `"="`, `">"`, `"<"`, `">="`, `"<="` + - `stringOptions`: `any of`, `none of` + - `arrayOptions`: `any of`, `none of`, `all of` + - `stringObject`: same operators as `string` + - `boolean`: `"="`, `"<>"` + - `null`: `is null`, `is not null` + + Supported columns. Each column accepts exactly one filter `type`; sending a column with a different `type` is rejected with `400`. + - `type`: `stringOptions`, values `SPAN`, `EVENT`, `GENERATION`, `AGENT`, `TOOL`, `CHAIN`, `RETRIEVER`, `EVALUATOR`, `EMBEDDING`, `GUARDRAIL` + - `name`: `stringOptions` + - `environment`: `stringOptions` + - `level`: `stringOptions`, values `DEBUG`, `DEFAULT`, `WARNING`, `ERROR` + - `version`: `string` + - `traceName`: `stringOptions` + - `userId`: `string` + - `sessionId`: `string` + - `tags`: `arrayOptions` + - `metadata`: `stringObject`, requires `key` + - `isRootObservation`: `boolean`; true when the observation has no parent or is explicitly marked as an application root + - `parentObservationId`: `null` + - `experimentId`: `stringOptions` + - `isExperimentItemRootSpan`: `boolean` + - `calledToolNames`: `arrayOptions` + - `toolCalls`: `number`, the number of tool calls on the observation + - `datasetId`: `stringOptions`. Use dataset `id` values from `GET /api/public/v2/datasets`, not dataset names. + + Examples + -------- + from langfuse.evaluation_commons import ( + EvaluationRuleFilter_StringOptions, + EvaluationRuleOptionsFilterOperator, + ) + + EvaluationRuleFilter_StringOptions( + column="type", + operator=EvaluationRuleOptionsFilterOperator.ANY_OF, + value=["GENERATION"], + ) + """ + + type: typing.Literal["stringOptions"] = "stringOptions" + column: str + operator: EvaluationRuleOptionsFilterOperator + value: typing.List[str] + + model_config: typing.ClassVar[pydantic.ConfigDict] = pydantic.ConfigDict( + extra="allow", frozen=True + ) + + +class EvaluationRuleFilter_CategoryOptions(UniversalBaseModel): + """ + One filter condition used to decide whether a live-ingested observation should be evaluated. + + All filters must be satisfied for the rule to run. Experiment scope is expressed with filters rather than a separate target field: + - `isExperimentItemRootSpan = true` limits execution to experiment item roots. + - `datasetId` limits execution to experiments for the selected datasets. Use dataset IDs from `GET /api/public/v2/datasets`. + + Pick the filter `type` first, as it determines the required fields and value shape. Use `key` only for object filters such as `metadata`. + + Operator quick reference by filter `type`: + - `string`: `"="`, `contains`, `does not contain`, `starts with`, `ends with` + - `number`: `"="`, `">"`, `"<"`, `">="`, `"<="` + - `datetime`: `"="`, `">"`, `"<"`, `">="`, `"<="` + - `stringOptions`: `any of`, `none of` + - `arrayOptions`: `any of`, `none of`, `all of` + - `stringObject`: same operators as `string` + - `boolean`: `"="`, `"<>"` + - `null`: `is null`, `is not null` + + Supported columns. Each column accepts exactly one filter `type`; sending a column with a different `type` is rejected with `400`. + - `type`: `stringOptions`, values `SPAN`, `EVENT`, `GENERATION`, `AGENT`, `TOOL`, `CHAIN`, `RETRIEVER`, `EVALUATOR`, `EMBEDDING`, `GUARDRAIL` + - `name`: `stringOptions` + - `environment`: `stringOptions` + - `level`: `stringOptions`, values `DEBUG`, `DEFAULT`, `WARNING`, `ERROR` + - `version`: `string` + - `traceName`: `stringOptions` + - `userId`: `string` + - `sessionId`: `string` + - `tags`: `arrayOptions` + - `metadata`: `stringObject`, requires `key` + - `isRootObservation`: `boolean`; true when the observation has no parent or is explicitly marked as an application root + - `parentObservationId`: `null` + - `experimentId`: `stringOptions` + - `isExperimentItemRootSpan`: `boolean` + - `calledToolNames`: `arrayOptions` + - `toolCalls`: `number`, the number of tool calls on the observation + - `datasetId`: `stringOptions`. Use dataset `id` values from `GET /api/public/v2/datasets`, not dataset names. + + Examples + -------- + from langfuse.evaluation_commons import ( + EvaluationRuleFilter_StringOptions, + EvaluationRuleOptionsFilterOperator, + ) + + EvaluationRuleFilter_StringOptions( + column="type", + operator=EvaluationRuleOptionsFilterOperator.ANY_OF, + value=["GENERATION"], + ) + """ + + type: typing.Literal["categoryOptions"] = "categoryOptions" + column: str + key: str + operator: EvaluationRuleOptionsFilterOperator + value: typing.List[str] + + model_config: typing.ClassVar[pydantic.ConfigDict] = pydantic.ConfigDict( + extra="allow", frozen=True + ) + + +class EvaluationRuleFilter_ArrayOptions(UniversalBaseModel): + """ + One filter condition used to decide whether a live-ingested observation should be evaluated. + + All filters must be satisfied for the rule to run. Experiment scope is expressed with filters rather than a separate target field: + - `isExperimentItemRootSpan = true` limits execution to experiment item roots. + - `datasetId` limits execution to experiments for the selected datasets. Use dataset IDs from `GET /api/public/v2/datasets`. + + Pick the filter `type` first, as it determines the required fields and value shape. Use `key` only for object filters such as `metadata`. + + Operator quick reference by filter `type`: + - `string`: `"="`, `contains`, `does not contain`, `starts with`, `ends with` + - `number`: `"="`, `">"`, `"<"`, `">="`, `"<="` + - `datetime`: `"="`, `">"`, `"<"`, `">="`, `"<="` + - `stringOptions`: `any of`, `none of` + - `arrayOptions`: `any of`, `none of`, `all of` + - `stringObject`: same operators as `string` + - `boolean`: `"="`, `"<>"` + - `null`: `is null`, `is not null` + + Supported columns. Each column accepts exactly one filter `type`; sending a column with a different `type` is rejected with `400`. + - `type`: `stringOptions`, values `SPAN`, `EVENT`, `GENERATION`, `AGENT`, `TOOL`, `CHAIN`, `RETRIEVER`, `EVALUATOR`, `EMBEDDING`, `GUARDRAIL` + - `name`: `stringOptions` + - `environment`: `stringOptions` + - `level`: `stringOptions`, values `DEBUG`, `DEFAULT`, `WARNING`, `ERROR` + - `version`: `string` + - `traceName`: `stringOptions` + - `userId`: `string` + - `sessionId`: `string` + - `tags`: `arrayOptions` + - `metadata`: `stringObject`, requires `key` + - `isRootObservation`: `boolean`; true when the observation has no parent or is explicitly marked as an application root + - `parentObservationId`: `null` + - `experimentId`: `stringOptions` + - `isExperimentItemRootSpan`: `boolean` + - `calledToolNames`: `arrayOptions` + - `toolCalls`: `number`, the number of tool calls on the observation + - `datasetId`: `stringOptions`. Use dataset `id` values from `GET /api/public/v2/datasets`, not dataset names. + + Examples + -------- + from langfuse.evaluation_commons import ( + EvaluationRuleFilter_StringOptions, + EvaluationRuleOptionsFilterOperator, + ) + + EvaluationRuleFilter_StringOptions( + column="type", + operator=EvaluationRuleOptionsFilterOperator.ANY_OF, + value=["GENERATION"], + ) + """ + + type: typing.Literal["arrayOptions"] = "arrayOptions" + column: str + operator: EvaluationRuleArrayOptionsFilterOperator + value: typing.List[str] + + model_config: typing.ClassVar[pydantic.ConfigDict] = pydantic.ConfigDict( + extra="allow", frozen=True + ) + + +class EvaluationRuleFilter_StringObject(UniversalBaseModel): + """ + One filter condition used to decide whether a live-ingested observation should be evaluated. + + All filters must be satisfied for the rule to run. Experiment scope is expressed with filters rather than a separate target field: + - `isExperimentItemRootSpan = true` limits execution to experiment item roots. + - `datasetId` limits execution to experiments for the selected datasets. Use dataset IDs from `GET /api/public/v2/datasets`. + + Pick the filter `type` first, as it determines the required fields and value shape. Use `key` only for object filters such as `metadata`. + + Operator quick reference by filter `type`: + - `string`: `"="`, `contains`, `does not contain`, `starts with`, `ends with` + - `number`: `"="`, `">"`, `"<"`, `">="`, `"<="` + - `datetime`: `"="`, `">"`, `"<"`, `">="`, `"<="` + - `stringOptions`: `any of`, `none of` + - `arrayOptions`: `any of`, `none of`, `all of` + - `stringObject`: same operators as `string` + - `boolean`: `"="`, `"<>"` + - `null`: `is null`, `is not null` + + Supported columns. Each column accepts exactly one filter `type`; sending a column with a different `type` is rejected with `400`. + - `type`: `stringOptions`, values `SPAN`, `EVENT`, `GENERATION`, `AGENT`, `TOOL`, `CHAIN`, `RETRIEVER`, `EVALUATOR`, `EMBEDDING`, `GUARDRAIL` + - `name`: `stringOptions` + - `environment`: `stringOptions` + - `level`: `stringOptions`, values `DEBUG`, `DEFAULT`, `WARNING`, `ERROR` + - `version`: `string` + - `traceName`: `stringOptions` + - `userId`: `string` + - `sessionId`: `string` + - `tags`: `arrayOptions` + - `metadata`: `stringObject`, requires `key` + - `isRootObservation`: `boolean`; true when the observation has no parent or is explicitly marked as an application root + - `parentObservationId`: `null` + - `experimentId`: `stringOptions` + - `isExperimentItemRootSpan`: `boolean` + - `calledToolNames`: `arrayOptions` + - `toolCalls`: `number`, the number of tool calls on the observation + - `datasetId`: `stringOptions`. Use dataset `id` values from `GET /api/public/v2/datasets`, not dataset names. + + Examples + -------- + from langfuse.evaluation_commons import ( + EvaluationRuleFilter_StringOptions, + EvaluationRuleOptionsFilterOperator, + ) + + EvaluationRuleFilter_StringOptions( + column="type", + operator=EvaluationRuleOptionsFilterOperator.ANY_OF, + value=["GENERATION"], + ) + """ + + type: typing.Literal["stringObject"] = "stringObject" + column: str + key: str + operator: EvaluationRuleStringFilterOperator + value: str + + model_config: typing.ClassVar[pydantic.ConfigDict] = pydantic.ConfigDict( + extra="allow", frozen=True + ) + + +class EvaluationRuleFilter_NumberObject(UniversalBaseModel): + """ + One filter condition used to decide whether a live-ingested observation should be evaluated. + + All filters must be satisfied for the rule to run. Experiment scope is expressed with filters rather than a separate target field: + - `isExperimentItemRootSpan = true` limits execution to experiment item roots. + - `datasetId` limits execution to experiments for the selected datasets. Use dataset IDs from `GET /api/public/v2/datasets`. + + Pick the filter `type` first, as it determines the required fields and value shape. Use `key` only for object filters such as `metadata`. + + Operator quick reference by filter `type`: + - `string`: `"="`, `contains`, `does not contain`, `starts with`, `ends with` + - `number`: `"="`, `">"`, `"<"`, `">="`, `"<="` + - `datetime`: `"="`, `">"`, `"<"`, `">="`, `"<="` + - `stringOptions`: `any of`, `none of` + - `arrayOptions`: `any of`, `none of`, `all of` + - `stringObject`: same operators as `string` + - `boolean`: `"="`, `"<>"` + - `null`: `is null`, `is not null` + + Supported columns. Each column accepts exactly one filter `type`; sending a column with a different `type` is rejected with `400`. + - `type`: `stringOptions`, values `SPAN`, `EVENT`, `GENERATION`, `AGENT`, `TOOL`, `CHAIN`, `RETRIEVER`, `EVALUATOR`, `EMBEDDING`, `GUARDRAIL` + - `name`: `stringOptions` + - `environment`: `stringOptions` + - `level`: `stringOptions`, values `DEBUG`, `DEFAULT`, `WARNING`, `ERROR` + - `version`: `string` + - `traceName`: `stringOptions` + - `userId`: `string` + - `sessionId`: `string` + - `tags`: `arrayOptions` + - `metadata`: `stringObject`, requires `key` + - `isRootObservation`: `boolean`; true when the observation has no parent or is explicitly marked as an application root + - `parentObservationId`: `null` + - `experimentId`: `stringOptions` + - `isExperimentItemRootSpan`: `boolean` + - `calledToolNames`: `arrayOptions` + - `toolCalls`: `number`, the number of tool calls on the observation + - `datasetId`: `stringOptions`. Use dataset `id` values from `GET /api/public/v2/datasets`, not dataset names. + + Examples + -------- + from langfuse.evaluation_commons import ( + EvaluationRuleFilter_StringOptions, + EvaluationRuleOptionsFilterOperator, + ) + + EvaluationRuleFilter_StringOptions( + column="type", + operator=EvaluationRuleOptionsFilterOperator.ANY_OF, + value=["GENERATION"], + ) + """ + + type: typing.Literal["numberObject"] = "numberObject" + column: str + key: str + operator: EvaluationRuleNumberFilterOperator + value: float + + model_config: typing.ClassVar[pydantic.ConfigDict] = pydantic.ConfigDict( + extra="allow", frozen=True + ) + + +class EvaluationRuleFilter_Boolean(UniversalBaseModel): + """ + One filter condition used to decide whether a live-ingested observation should be evaluated. + + All filters must be satisfied for the rule to run. Experiment scope is expressed with filters rather than a separate target field: + - `isExperimentItemRootSpan = true` limits execution to experiment item roots. + - `datasetId` limits execution to experiments for the selected datasets. Use dataset IDs from `GET /api/public/v2/datasets`. + + Pick the filter `type` first, as it determines the required fields and value shape. Use `key` only for object filters such as `metadata`. + + Operator quick reference by filter `type`: + - `string`: `"="`, `contains`, `does not contain`, `starts with`, `ends with` + - `number`: `"="`, `">"`, `"<"`, `">="`, `"<="` + - `datetime`: `"="`, `">"`, `"<"`, `">="`, `"<="` + - `stringOptions`: `any of`, `none of` + - `arrayOptions`: `any of`, `none of`, `all of` + - `stringObject`: same operators as `string` + - `boolean`: `"="`, `"<>"` + - `null`: `is null`, `is not null` + + Supported columns. Each column accepts exactly one filter `type`; sending a column with a different `type` is rejected with `400`. + - `type`: `stringOptions`, values `SPAN`, `EVENT`, `GENERATION`, `AGENT`, `TOOL`, `CHAIN`, `RETRIEVER`, `EVALUATOR`, `EMBEDDING`, `GUARDRAIL` + - `name`: `stringOptions` + - `environment`: `stringOptions` + - `level`: `stringOptions`, values `DEBUG`, `DEFAULT`, `WARNING`, `ERROR` + - `version`: `string` + - `traceName`: `stringOptions` + - `userId`: `string` + - `sessionId`: `string` + - `tags`: `arrayOptions` + - `metadata`: `stringObject`, requires `key` + - `isRootObservation`: `boolean`; true when the observation has no parent or is explicitly marked as an application root + - `parentObservationId`: `null` + - `experimentId`: `stringOptions` + - `isExperimentItemRootSpan`: `boolean` + - `calledToolNames`: `arrayOptions` + - `toolCalls`: `number`, the number of tool calls on the observation + - `datasetId`: `stringOptions`. Use dataset `id` values from `GET /api/public/v2/datasets`, not dataset names. + + Examples + -------- + from langfuse.evaluation_commons import ( + EvaluationRuleFilter_StringOptions, + EvaluationRuleOptionsFilterOperator, + ) + + EvaluationRuleFilter_StringOptions( + column="type", + operator=EvaluationRuleOptionsFilterOperator.ANY_OF, + value=["GENERATION"], + ) + """ + + type: typing.Literal["boolean"] = "boolean" + column: str + operator: EvaluationRuleBooleanFilterOperator + value: bool + + model_config: typing.ClassVar[pydantic.ConfigDict] = pydantic.ConfigDict( + extra="allow", frozen=True + ) + + +class EvaluationRuleFilter_Null(UniversalBaseModel): + """ + One filter condition used to decide whether a live-ingested observation should be evaluated. + + All filters must be satisfied for the rule to run. Experiment scope is expressed with filters rather than a separate target field: + - `isExperimentItemRootSpan = true` limits execution to experiment item roots. + - `datasetId` limits execution to experiments for the selected datasets. Use dataset IDs from `GET /api/public/v2/datasets`. + + Pick the filter `type` first, as it determines the required fields and value shape. Use `key` only for object filters such as `metadata`. + + Operator quick reference by filter `type`: + - `string`: `"="`, `contains`, `does not contain`, `starts with`, `ends with` + - `number`: `"="`, `">"`, `"<"`, `">="`, `"<="` + - `datetime`: `"="`, `">"`, `"<"`, `">="`, `"<="` + - `stringOptions`: `any of`, `none of` + - `arrayOptions`: `any of`, `none of`, `all of` + - `stringObject`: same operators as `string` + - `boolean`: `"="`, `"<>"` + - `null`: `is null`, `is not null` + + Supported columns. Each column accepts exactly one filter `type`; sending a column with a different `type` is rejected with `400`. + - `type`: `stringOptions`, values `SPAN`, `EVENT`, `GENERATION`, `AGENT`, `TOOL`, `CHAIN`, `RETRIEVER`, `EVALUATOR`, `EMBEDDING`, `GUARDRAIL` + - `name`: `stringOptions` + - `environment`: `stringOptions` + - `level`: `stringOptions`, values `DEBUG`, `DEFAULT`, `WARNING`, `ERROR` + - `version`: `string` + - `traceName`: `stringOptions` + - `userId`: `string` + - `sessionId`: `string` + - `tags`: `arrayOptions` + - `metadata`: `stringObject`, requires `key` + - `isRootObservation`: `boolean`; true when the observation has no parent or is explicitly marked as an application root + - `parentObservationId`: `null` + - `experimentId`: `stringOptions` + - `isExperimentItemRootSpan`: `boolean` + - `calledToolNames`: `arrayOptions` + - `toolCalls`: `number`, the number of tool calls on the observation + - `datasetId`: `stringOptions`. Use dataset `id` values from `GET /api/public/v2/datasets`, not dataset names. + + Examples + -------- + from langfuse.evaluation_commons import ( + EvaluationRuleFilter_StringOptions, + EvaluationRuleOptionsFilterOperator, + ) + + EvaluationRuleFilter_StringOptions( + column="type", + operator=EvaluationRuleOptionsFilterOperator.ANY_OF, + value=["GENERATION"], + ) + """ + + type: typing.Literal["null"] = "null" + column: str + operator: EvaluationRuleNullFilterOperator + value: typing.Literal[""] = "" + + model_config: typing.ClassVar[pydantic.ConfigDict] = pydantic.ConfigDict( + extra="allow", frozen=True + ) + + +""" +from langfuse.evaluation_commons import ( + EvaluationRuleFilter_StringOptions, + EvaluationRuleOptionsFilterOperator, +) + +EvaluationRuleFilter_StringOptions( + column="type", + operator=EvaluationRuleOptionsFilterOperator.ANY_OF, + value=["GENERATION"], +) +""" +EvaluationRuleFilter = typing_extensions.Annotated[ + typing.Union[ + EvaluationRuleFilter_Datetime, + EvaluationRuleFilter_String, + EvaluationRuleFilter_Number, + EvaluationRuleFilter_StringOptions, + EvaluationRuleFilter_CategoryOptions, + EvaluationRuleFilter_ArrayOptions, + EvaluationRuleFilter_StringObject, + EvaluationRuleFilter_NumberObject, + EvaluationRuleFilter_Boolean, + EvaluationRuleFilter_Null, + ], + pydantic.Field(discriminator="type"), +] diff --git a/langfuse/api/evaluation_commons/types/evaluation_rule_null_filter_operator.py b/langfuse/api/evaluation_commons/types/evaluation_rule_null_filter_operator.py new file mode 100644 index 000000000..080213b8a --- /dev/null +++ b/langfuse/api/evaluation_commons/types/evaluation_rule_null_filter_operator.py @@ -0,0 +1,22 @@ +# This file was auto-generated by Fern from our API Definition. + +import typing + +from ...core import enum + +T_Result = typing.TypeVar("T_Result") + + +class EvaluationRuleNullFilterOperator(enum.StrEnum): + IS_NULL = "is null" + IS_NOT_NULL = "is not null" + + def visit( + self, + is_null: typing.Callable[[], T_Result], + is_not_null: typing.Callable[[], T_Result], + ) -> T_Result: + if self is EvaluationRuleNullFilterOperator.IS_NULL: + return is_null() + if self is EvaluationRuleNullFilterOperator.IS_NOT_NULL: + return is_not_null() diff --git a/langfuse/api/evaluation_commons/types/evaluation_rule_number_filter_operator.py b/langfuse/api/evaluation_commons/types/evaluation_rule_number_filter_operator.py new file mode 100644 index 000000000..b359688a7 --- /dev/null +++ b/langfuse/api/evaluation_commons/types/evaluation_rule_number_filter_operator.py @@ -0,0 +1,34 @@ +# This file was auto-generated by Fern from our API Definition. + +import typing + +from ...core import enum + +T_Result = typing.TypeVar("T_Result") + + +class EvaluationRuleNumberFilterOperator(enum.StrEnum): + EQUALS = "=" + GREATER_THAN = ">" + LESS_THAN = "<" + GREATER_THAN_OR_EQUAL = ">=" + LESS_THAN_OR_EQUAL = "<=" + + def visit( + self, + equals: typing.Callable[[], T_Result], + greater_than: typing.Callable[[], T_Result], + less_than: typing.Callable[[], T_Result], + greater_than_or_equal: typing.Callable[[], T_Result], + less_than_or_equal: typing.Callable[[], T_Result], + ) -> T_Result: + if self is EvaluationRuleNumberFilterOperator.EQUALS: + return equals() + if self is EvaluationRuleNumberFilterOperator.GREATER_THAN: + return greater_than() + if self is EvaluationRuleNumberFilterOperator.LESS_THAN: + return less_than() + if self is EvaluationRuleNumberFilterOperator.GREATER_THAN_OR_EQUAL: + return greater_than_or_equal() + if self is EvaluationRuleNumberFilterOperator.LESS_THAN_OR_EQUAL: + return less_than_or_equal() diff --git a/langfuse/api/evaluation_commons/types/evaluation_rule_options_filter_operator.py b/langfuse/api/evaluation_commons/types/evaluation_rule_options_filter_operator.py new file mode 100644 index 000000000..c057f2cc1 --- /dev/null +++ b/langfuse/api/evaluation_commons/types/evaluation_rule_options_filter_operator.py @@ -0,0 +1,22 @@ +# This file was auto-generated by Fern from our API Definition. + +import typing + +from ...core import enum + +T_Result = typing.TypeVar("T_Result") + + +class EvaluationRuleOptionsFilterOperator(enum.StrEnum): + ANY_OF = "any of" + NONE_OF = "none of" + + def visit( + self, + any_of: typing.Callable[[], T_Result], + none_of: typing.Callable[[], T_Result], + ) -> T_Result: + if self is EvaluationRuleOptionsFilterOperator.ANY_OF: + return any_of() + if self is EvaluationRuleOptionsFilterOperator.NONE_OF: + return none_of() diff --git a/langfuse/api/evaluation_commons/types/evaluation_rule_read_filter.py b/langfuse/api/evaluation_commons/types/evaluation_rule_read_filter.py new file mode 100644 index 000000000..23ef95445 --- /dev/null +++ b/langfuse/api/evaluation_commons/types/evaluation_rule_read_filter.py @@ -0,0 +1,10 @@ +# This file was auto-generated by Fern from our API Definition. + +import typing + +from .evaluation_rule_read_filter_base import EvaluationRuleReadFilterBase +from .evaluation_rule_read_filter_with_key import EvaluationRuleReadFilterWithKey + +EvaluationRuleReadFilter = typing.Union[ + EvaluationRuleReadFilterWithKey, EvaluationRuleReadFilterBase +] diff --git a/langfuse/api/evaluation_commons/types/evaluation_rule_read_filter_base.py b/langfuse/api/evaluation_commons/types/evaluation_rule_read_filter_base.py new file mode 100644 index 000000000..28d53b889 --- /dev/null +++ b/langfuse/api/evaluation_commons/types/evaluation_rule_read_filter_base.py @@ -0,0 +1,28 @@ +# This file was auto-generated by Fern from our API Definition. + +import typing + +import pydantic +from ...core.pydantic_utilities import UniversalBaseModel + + +class EvaluationRuleReadFilterBase(UniversalBaseModel): + """ + Stored filter returned verbatim by evaluation-rule read endpoints. + """ + + type: str = pydantic.Field() + """ + Stored filter type. This is not broken down into separate public response types. + """ + + column: str + operator: str + value: typing.Optional[typing.Any] = pydantic.Field(default=None) + """ + Stored filter value. Its shape depends on the filter type. + """ + + model_config: typing.ClassVar[pydantic.ConfigDict] = pydantic.ConfigDict( + extra="allow", frozen=True + ) diff --git a/langfuse/api/evaluation_commons/types/evaluation_rule_read_filter_with_key.py b/langfuse/api/evaluation_commons/types/evaluation_rule_read_filter_with_key.py new file mode 100644 index 000000000..5f0956674 --- /dev/null +++ b/langfuse/api/evaluation_commons/types/evaluation_rule_read_filter_with_key.py @@ -0,0 +1,17 @@ +# This file was auto-generated by Fern from our API Definition. + +import typing + +import pydantic +from .evaluation_rule_read_filter_base import EvaluationRuleReadFilterBase + + +class EvaluationRuleReadFilterWithKey(EvaluationRuleReadFilterBase): + key: str = pydantic.Field() + """ + Stored object or legacy filter key. + """ + + model_config: typing.ClassVar[pydantic.ConfigDict] = pydantic.ConfigDict( + extra="allow", frozen=True + ) diff --git a/langfuse/api/evaluation_commons/types/evaluation_rule_string_filter_operator.py b/langfuse/api/evaluation_commons/types/evaluation_rule_string_filter_operator.py new file mode 100644 index 000000000..b19dd323f --- /dev/null +++ b/langfuse/api/evaluation_commons/types/evaluation_rule_string_filter_operator.py @@ -0,0 +1,34 @@ +# This file was auto-generated by Fern from our API Definition. + +import typing + +from ...core import enum + +T_Result = typing.TypeVar("T_Result") + + +class EvaluationRuleStringFilterOperator(enum.StrEnum): + EQUALS = "=" + CONTAINS = "contains" + DOES_NOT_CONTAIN = "does not contain" + STARTS_WITH = "starts with" + ENDS_WITH = "ends with" + + def visit( + self, + equals: typing.Callable[[], T_Result], + contains: typing.Callable[[], T_Result], + does_not_contain: typing.Callable[[], T_Result], + starts_with: typing.Callable[[], T_Result], + ends_with: typing.Callable[[], T_Result], + ) -> T_Result: + if self is EvaluationRuleStringFilterOperator.EQUALS: + return equals() + if self is EvaluationRuleStringFilterOperator.CONTAINS: + return contains() + if self is EvaluationRuleStringFilterOperator.DOES_NOT_CONTAIN: + return does_not_contain() + if self is EvaluationRuleStringFilterOperator.STARTS_WITH: + return starts_with() + if self is EvaluationRuleStringFilterOperator.ENDS_WITH: + return ends_with() diff --git a/langfuse/api/evaluation_commons/types/evaluator_output_definition.py b/langfuse/api/evaluation_commons/types/evaluator_output_definition.py new file mode 100644 index 000000000..68d8cefc2 --- /dev/null +++ b/langfuse/api/evaluation_commons/types/evaluator_output_definition.py @@ -0,0 +1,143 @@ +# This file was auto-generated by Fern from our API Definition. + +from __future__ import annotations + +import typing + +import pydantic +import typing_extensions +from ...core.pydantic_utilities import UniversalBaseModel +from ...core.serialization import FieldMetadata + + +class EvaluatorOutputDefinition_Numeric(UniversalBaseModel): + """ + Flat structured output definition used when creating or updating an evaluator. + + - `dataType` is required. + - `scoreReasoningInstructions` and `scoreValueInstructions` are optional instructions. + - `minValue` and `maxValue` apply only to `NUMERIC` outputs. If both are set, `minValue` must not exceed `maxValue`. + - `categories` and `shouldAllowMultipleMatches` apply only to `CATEGORICAL` outputs. + - Do not send `version`; that is an internal storage detail. + + Examples + -------- + from langfuse.evaluation_commons import EvaluatorOutputDefinition_Numeric + + EvaluatorOutputDefinition_Numeric( + min_value=0.0, + max_value=1.0, + ) + """ + + data_type: typing_extensions.Annotated[ + typing.Literal["NUMERIC"], FieldMetadata(alias="dataType") + ] = "NUMERIC" + min_value: typing_extensions.Annotated[ + typing.Optional[float], FieldMetadata(alias="minValue") + ] = None + max_value: typing_extensions.Annotated[ + typing.Optional[float], FieldMetadata(alias="maxValue") + ] = None + score_reasoning_instructions: typing_extensions.Annotated[ + typing.Optional[str], FieldMetadata(alias="scoreReasoningInstructions") + ] = None + score_value_instructions: typing_extensions.Annotated[ + typing.Optional[str], FieldMetadata(alias="scoreValueInstructions") + ] = None + + model_config: typing.ClassVar[pydantic.ConfigDict] = pydantic.ConfigDict( + extra="allow", frozen=True + ) + + +class EvaluatorOutputDefinition_Boolean(UniversalBaseModel): + """ + Flat structured output definition used when creating or updating an evaluator. + + - `dataType` is required. + - `scoreReasoningInstructions` and `scoreValueInstructions` are optional instructions. + - `minValue` and `maxValue` apply only to `NUMERIC` outputs. If both are set, `minValue` must not exceed `maxValue`. + - `categories` and `shouldAllowMultipleMatches` apply only to `CATEGORICAL` outputs. + - Do not send `version`; that is an internal storage detail. + + Examples + -------- + from langfuse.evaluation_commons import EvaluatorOutputDefinition_Numeric + + EvaluatorOutputDefinition_Numeric( + min_value=0.0, + max_value=1.0, + ) + """ + + data_type: typing_extensions.Annotated[ + typing.Literal["BOOLEAN"], FieldMetadata(alias="dataType") + ] = "BOOLEAN" + score_reasoning_instructions: typing_extensions.Annotated[ + typing.Optional[str], FieldMetadata(alias="scoreReasoningInstructions") + ] = None + score_value_instructions: typing_extensions.Annotated[ + typing.Optional[str], FieldMetadata(alias="scoreValueInstructions") + ] = None + + model_config: typing.ClassVar[pydantic.ConfigDict] = pydantic.ConfigDict( + extra="allow", frozen=True + ) + + +class EvaluatorOutputDefinition_Categorical(UniversalBaseModel): + """ + Flat structured output definition used when creating or updating an evaluator. + + - `dataType` is required. + - `scoreReasoningInstructions` and `scoreValueInstructions` are optional instructions. + - `minValue` and `maxValue` apply only to `NUMERIC` outputs. If both are set, `minValue` must not exceed `maxValue`. + - `categories` and `shouldAllowMultipleMatches` apply only to `CATEGORICAL` outputs. + - Do not send `version`; that is an internal storage detail. + + Examples + -------- + from langfuse.evaluation_commons import EvaluatorOutputDefinition_Numeric + + EvaluatorOutputDefinition_Numeric( + min_value=0.0, + max_value=1.0, + ) + """ + + data_type: typing_extensions.Annotated[ + typing.Literal["CATEGORICAL"], FieldMetadata(alias="dataType") + ] = "CATEGORICAL" + categories: typing.List[str] + should_allow_multiple_matches: typing_extensions.Annotated[ + bool, FieldMetadata(alias="shouldAllowMultipleMatches") + ] + score_reasoning_instructions: typing_extensions.Annotated[ + typing.Optional[str], FieldMetadata(alias="scoreReasoningInstructions") + ] = None + score_value_instructions: typing_extensions.Annotated[ + typing.Optional[str], FieldMetadata(alias="scoreValueInstructions") + ] = None + + model_config: typing.ClassVar[pydantic.ConfigDict] = pydantic.ConfigDict( + extra="allow", frozen=True + ) + + +""" +from langfuse.evaluation_commons import EvaluatorOutputDefinition_Numeric + +EvaluatorOutputDefinition_Numeric( + min_value=0.0, + max_value=1.0, +) +""" +EvaluatorOutputDefinition = typing_extensions.Annotated[ + typing.Union[ + EvaluatorOutputDefinition_Numeric, + EvaluatorOutputDefinition_Boolean, + EvaluatorOutputDefinition_Categorical, + ], + pydantic.Field(discriminator="data_type"), +] diff --git a/langfuse/api/evaluation_commons/types/evaluator_output_definition_base.py b/langfuse/api/evaluation_commons/types/evaluator_output_definition_base.py new file mode 100644 index 000000000..e2debe185 --- /dev/null +++ b/langfuse/api/evaluation_commons/types/evaluator_output_definition_base.py @@ -0,0 +1,28 @@ +# This file was auto-generated by Fern from our API Definition. + +import typing + +import pydantic +import typing_extensions +from ...core.pydantic_utilities import UniversalBaseModel +from ...core.serialization import FieldMetadata + + +class EvaluatorOutputDefinitionBase(UniversalBaseModel): + score_reasoning_instructions: typing_extensions.Annotated[ + typing.Optional[str], FieldMetadata(alias="scoreReasoningInstructions") + ] = pydantic.Field(default=None) + """ + Optional instructions for deriving the reasoning returned with the score. + """ + + score_value_instructions: typing_extensions.Annotated[ + typing.Optional[str], FieldMetadata(alias="scoreValueInstructions") + ] = pydantic.Field(default=None) + """ + Optional instructions for deriving the score value. + """ + + model_config: typing.ClassVar[pydantic.ConfigDict] = pydantic.ConfigDict( + extra="allow", frozen=True + ) diff --git a/langfuse/api/evaluation_commons/types/evaluator_output_score_type.py b/langfuse/api/evaluation_commons/types/evaluator_output_score_type.py new file mode 100644 index 000000000..b98ff4cdb --- /dev/null +++ b/langfuse/api/evaluation_commons/types/evaluator_output_score_type.py @@ -0,0 +1,35 @@ +# This file was auto-generated by Fern from our API Definition. + +import typing + +from ...core import enum + +T_Result = typing.TypeVar("T_Result") + + +class EvaluatorOutputScoreType(enum.StrEnum): + """ + Structured score type returned by an evaluator. + + This controls the type of score value Langfuse stores for evaluation results: + - `NUMERIC`: a numeric score such as `0.82` + - `BOOLEAN`: a boolean score such as `true` + - `CATEGORICAL`: one or more category labels from a fixed list + """ + + NUMERIC = "NUMERIC" + BOOLEAN = "BOOLEAN" + CATEGORICAL = "CATEGORICAL" + + def visit( + self, + numeric: typing.Callable[[], T_Result], + boolean: typing.Callable[[], T_Result], + categorical: typing.Callable[[], T_Result], + ) -> T_Result: + if self is EvaluatorOutputScoreType.NUMERIC: + return numeric() + if self is EvaluatorOutputScoreType.BOOLEAN: + return boolean() + if self is EvaluatorOutputScoreType.CATEGORICAL: + return categorical() diff --git a/langfuse/api/evaluation_commons/types/evaluator_type.py b/langfuse/api/evaluation_commons/types/evaluator_type.py new file mode 100644 index 000000000..49d1bf3b1 --- /dev/null +++ b/langfuse/api/evaluation_commons/types/evaluator_type.py @@ -0,0 +1,28 @@ +# This file was auto-generated by Fern from our API Definition. + +import typing + +from ...core import enum + +T_Result = typing.TypeVar("T_Result") + + +class EvaluatorType(enum.StrEnum): + """ + The evaluator type. + + The public API supports LLM-as-a-judge and code evaluators. + """ + + LLM_AS_JUDGE = "llm_as_judge" + CODE = "code" + + def visit( + self, + llm_as_judge: typing.Callable[[], T_Result], + code: typing.Callable[[], T_Result], + ) -> T_Result: + if self is EvaluatorType.LLM_AS_JUDGE: + return llm_as_judge() + if self is EvaluatorType.CODE: + return code() diff --git a/langfuse/api/evaluation_commons/types/legacy_evaluation_object.py b/langfuse/api/evaluation_commons/types/legacy_evaluation_object.py new file mode 100644 index 000000000..d9bdf81fd --- /dev/null +++ b/langfuse/api/evaluation_commons/types/legacy_evaluation_object.py @@ -0,0 +1,66 @@ +# This file was auto-generated by Fern from our API Definition. + +import typing + +from ...core import enum + +T_Result = typing.TypeVar("T_Result") + + +class LegacyEvaluationObject(enum.StrEnum): + """ + **Deprecated:** Legacy Langfuse object kind used by trace and dataset evaluation rules. + """ + + TRACE = "trace" + SPAN = "span" + GENERATION = "generation" + EVENT = "event" + AGENT = "agent" + TOOL = "tool" + CHAIN = "chain" + RETRIEVER = "retriever" + EVALUATOR = "evaluator" + EMBEDDING = "embedding" + GUARDRAIL = "guardrail" + DATASET_ITEM = "dataset_item" + + def visit( + self, + trace: typing.Callable[[], T_Result], + span: typing.Callable[[], T_Result], + generation: typing.Callable[[], T_Result], + event: typing.Callable[[], T_Result], + agent: typing.Callable[[], T_Result], + tool: typing.Callable[[], T_Result], + chain: typing.Callable[[], T_Result], + retriever: typing.Callable[[], T_Result], + evaluator: typing.Callable[[], T_Result], + embedding: typing.Callable[[], T_Result], + guardrail: typing.Callable[[], T_Result], + dataset_item: typing.Callable[[], T_Result], + ) -> T_Result: + if self is LegacyEvaluationObject.TRACE: + return trace() + if self is LegacyEvaluationObject.SPAN: + return span() + if self is LegacyEvaluationObject.GENERATION: + return generation() + if self is LegacyEvaluationObject.EVENT: + return event() + if self is LegacyEvaluationObject.AGENT: + return agent() + if self is LegacyEvaluationObject.TOOL: + return tool() + if self is LegacyEvaluationObject.CHAIN: + return chain() + if self is LegacyEvaluationObject.RETRIEVER: + return retriever() + if self is LegacyEvaluationObject.EVALUATOR: + return evaluator() + if self is LegacyEvaluationObject.EMBEDDING: + return embedding() + if self is LegacyEvaluationObject.GUARDRAIL: + return guardrail() + if self is LegacyEvaluationObject.DATASET_ITEM: + return dataset_item() diff --git a/langfuse/api/evaluation_commons/types/legacy_prompt_variable_mapping.py b/langfuse/api/evaluation_commons/types/legacy_prompt_variable_mapping.py new file mode 100644 index 000000000..f84aa1eea --- /dev/null +++ b/langfuse/api/evaluation_commons/types/legacy_prompt_variable_mapping.py @@ -0,0 +1,72 @@ +# This file was auto-generated by Fern from our API Definition. + +import typing + +import pydantic +import typing_extensions +from ...core.pydantic_utilities import UniversalBaseModel +from ...core.serialization import FieldMetadata +from .legacy_evaluation_object import LegacyEvaluationObject + + +class LegacyPromptVariableMapping(UniversalBaseModel): + """ + **Deprecated:** Connects one prompt variable to data from a legacy trace or dataset evaluation rule. + + `langfuseObject` selects the object kind. `objectName` separately selects a named observation and is `null` for trace and dataset-item mappings. + + Examples + -------- + from langfuse.evaluation_commons import ( + LegacyEvaluationObject, + LegacyPromptVariableMapping, + ) + + LegacyPromptVariableMapping( + variable="input", + langfuse_object=LegacyEvaluationObject.TRACE, + source="input", + ) + """ + + mapping_type: typing_extensions.Annotated[ + typing.Literal["legacy"], FieldMetadata(alias="mappingType") + ] = pydantic.Field(default="legacy") + """ + Explicitly marks this as a legacy mapping. + """ + + variable: str = pydantic.Field() + """ + Prompt variable name without braces. + """ + + langfuse_object: typing_extensions.Annotated[ + LegacyEvaluationObject, FieldMetadata(alias="langfuseObject") + ] = pydantic.Field() + """ + Legacy object kind selected as the mapping source. + """ + + object_name: typing_extensions.Annotated[ + typing.Optional[str], FieldMetadata(alias="objectName") + ] = pydantic.Field(default=None) + """ + Observation name to match, or `null` when `langfuseObject` is `trace` or `dataset_item`. + """ + + source: str = pydantic.Field() + """ + Field selected from the legacy object. + """ + + json_path: typing_extensions.Annotated[ + typing.Optional[str], FieldMetadata(alias="jsonPath") + ] = pydantic.Field(default=None) + """ + Optional JSONPath selector applied to the selected field. + """ + + model_config: typing.ClassVar[pydantic.ConfigDict] = pydantic.ConfigDict( + extra="allow", frozen=True + ) diff --git a/langfuse/api/evaluation_commons/types/null_evaluation_rule_filter.py b/langfuse/api/evaluation_commons/types/null_evaluation_rule_filter.py new file mode 100644 index 000000000..17343d87b --- /dev/null +++ b/langfuse/api/evaluation_commons/types/null_evaluation_rule_filter.py @@ -0,0 +1,24 @@ +# This file was auto-generated by Fern from our API Definition. + +import typing + +import pydantic +from ...core.pydantic_utilities import UniversalBaseModel +from .evaluation_rule_null_filter_operator import EvaluationRuleNullFilterOperator + + +class NullEvaluationRuleFilter(UniversalBaseModel): + column: str = pydantic.Field() + """ + Column to filter on, for example `parentObservationId`. + """ + + operator: EvaluationRuleNullFilterOperator + value: typing.Literal[""] = pydantic.Field(default="") + """ + Required empty-string placeholder used by the runtime filter contract. + """ + + model_config: typing.ClassVar[pydantic.ConfigDict] = pydantic.ConfigDict( + extra="allow", frozen=True + ) diff --git a/langfuse/api/evaluation_commons/types/number_evaluation_rule_filter.py b/langfuse/api/evaluation_commons/types/number_evaluation_rule_filter.py new file mode 100644 index 000000000..af38ad6a5 --- /dev/null +++ b/langfuse/api/evaluation_commons/types/number_evaluation_rule_filter.py @@ -0,0 +1,21 @@ +# This file was auto-generated by Fern from our API Definition. + +import typing + +import pydantic +from ...core.pydantic_utilities import UniversalBaseModel +from .evaluation_rule_number_filter_operator import EvaluationRuleNumberFilterOperator + + +class NumberEvaluationRuleFilter(UniversalBaseModel): + column: str = pydantic.Field() + """ + Column to filter on. + """ + + operator: EvaluationRuleNumberFilterOperator + value: float + + model_config: typing.ClassVar[pydantic.ConfigDict] = pydantic.ConfigDict( + extra="allow", frozen=True + ) diff --git a/langfuse/api/evaluation_commons/types/number_object_evaluation_rule_filter.py b/langfuse/api/evaluation_commons/types/number_object_evaluation_rule_filter.py new file mode 100644 index 000000000..130028c62 --- /dev/null +++ b/langfuse/api/evaluation_commons/types/number_object_evaluation_rule_filter.py @@ -0,0 +1,26 @@ +# This file was auto-generated by Fern from our API Definition. + +import typing + +import pydantic +from ...core.pydantic_utilities import UniversalBaseModel +from .evaluation_rule_number_filter_operator import EvaluationRuleNumberFilterOperator + + +class NumberObjectEvaluationRuleFilter(UniversalBaseModel): + column: str = pydantic.Field() + """ + Object-valued column to filter on. + """ + + key: str = pydantic.Field() + """ + Key inside the object-valued column to filter on. + """ + + operator: EvaluationRuleNumberFilterOperator + value: float + + model_config: typing.ClassVar[pydantic.ConfigDict] = pydantic.ConfigDict( + extra="allow", frozen=True + ) diff --git a/langfuse/api/evaluation_commons/types/prompt_variable_mapping.py b/langfuse/api/evaluation_commons/types/prompt_variable_mapping.py new file mode 100644 index 000000000..63dad0bb1 --- /dev/null +++ b/langfuse/api/evaluation_commons/types/prompt_variable_mapping.py @@ -0,0 +1,10 @@ +# This file was auto-generated by Fern from our API Definition. + +import typing + +from .legacy_prompt_variable_mapping import LegacyPromptVariableMapping +from .prompt_variable_mapping_read import PromptVariableMappingRead + +PromptVariableMapping = typing.Union[ + PromptVariableMappingRead, LegacyPromptVariableMapping +] diff --git a/langfuse/api/evaluation_commons/types/prompt_variable_mapping_input.py b/langfuse/api/evaluation_commons/types/prompt_variable_mapping_input.py new file mode 100644 index 000000000..0db6ff195 --- /dev/null +++ b/langfuse/api/evaluation_commons/types/prompt_variable_mapping_input.py @@ -0,0 +1,70 @@ +# This file was auto-generated by Fern from our API Definition. + +import typing + +import pydantic +import typing_extensions +from ...core.pydantic_utilities import UniversalBaseModel +from ...core.serialization import FieldMetadata +from .prompt_variable_mapping_source import PromptVariableMappingSource + + +class PromptVariableMappingInput(UniversalBaseModel): + """ + Connects one prompt variable to data from an observation or experiment. + + Manual mappings are used for `llm_as_judge` evaluators. `code` evaluators use a fixed runtime mapping managed by Langfuse. + + How to build a valid mapping list: + 1. Create the evaluator or fetch it with `GET /evaluators/{id}`. + 2. Read the evaluator `variables` array. + 3. Add exactly one mapping object for each variable in that array. + 4. Use the variable name exactly as returned, without braces such as `{{` or `}}`. + 5. Choose the source field that should populate the variable. + + `jsonPath` is optional. Use it only when the selected source is a JSON object and you want to extract one nested field before inserting it into the evaluator prompt. + + Invalid, missing, or duplicate mappings return a validation error. Malformed JSONPath expressions are also rejected. + + Examples + -------- + from langfuse.evaluation_commons import ( + PromptVariableMappingInput, + PromptVariableMappingSource, + ) + + PromptVariableMappingInput( + variable="input", + source=PromptVariableMappingSource.INPUT, + ) + """ + + variable: str = pydantic.Field() + """ + Prompt variable name without braces. + + Example: for the prompt `Judge {{input}} against {{output}}`, use `input` and `output`. + """ + + source: PromptVariableMappingSource = pydantic.Field() + """ + Source field that should populate the prompt variable. + + Available sources are `input`, `output`, `metadata`, `tool_calls`, `expected_output`, and `experiment_item_metadata`. + """ + + json_path: typing_extensions.Annotated[ + typing.Optional[str], FieldMetadata(alias="jsonPath") + ] = pydantic.Field(default=None) + """ + Optional JSONPath selector applied to the selected source before it is passed to the evaluator prompt. + + Requirements: + - Must start with `$` + - Must be a syntactically valid JSONPath expression + - Most useful with `source=metadata` + """ + + model_config: typing.ClassVar[pydantic.ConfigDict] = pydantic.ConfigDict( + extra="allow", frozen=True + ) diff --git a/langfuse/api/evaluation_commons/types/prompt_variable_mapping_read.py b/langfuse/api/evaluation_commons/types/prompt_variable_mapping_read.py new file mode 100644 index 000000000..9a0476073 --- /dev/null +++ b/langfuse/api/evaluation_commons/types/prompt_variable_mapping_read.py @@ -0,0 +1,46 @@ +# This file was auto-generated by Fern from our API Definition. + +import typing + +import pydantic +import typing_extensions +from ...core.pydantic_utilities import UniversalBaseModel +from ...core.serialization import FieldMetadata + + +class PromptVariableMappingRead(UniversalBaseModel): + """ + Connects one prompt variable to source data. + + `source` is `null` when the mapping is incomplete. + + Examples + -------- + from langfuse.evaluation_commons import PromptVariableMappingRead + + PromptVariableMappingRead( + variable="input", + source="input", + ) + """ + + variable: str = pydantic.Field() + """ + Prompt variable name without braces. + """ + + source: typing.Optional[str] = pydantic.Field(default=None) + """ + Stored source field populating the variable, or `null` when the mapping is incomplete. + """ + + json_path: typing_extensions.Annotated[ + typing.Optional[str], FieldMetadata(alias="jsonPath") + ] = pydantic.Field(default=None) + """ + Optional JSONPath selector applied to the selected source. + """ + + model_config: typing.ClassVar[pydantic.ConfigDict] = pydantic.ConfigDict( + extra="allow", frozen=True + ) diff --git a/langfuse/api/evaluation_commons/types/prompt_variable_mapping_source.py b/langfuse/api/evaluation_commons/types/prompt_variable_mapping_source.py new file mode 100644 index 000000000..3283f204f --- /dev/null +++ b/langfuse/api/evaluation_commons/types/prompt_variable_mapping_source.py @@ -0,0 +1,52 @@ +# This file was auto-generated by Fern from our API Definition. + +import typing + +from ...core import enum + +T_Result = typing.TypeVar("T_Result") + + +class PromptVariableMappingSource(enum.StrEnum): + """ + Source field used to populate a prompt variable. + + Use these values when mapping evaluator prompt variables to live data. + + Source semantics: + - `input`: the observation input payload + - `output`: the observation output payload + - `metadata`: the observation metadata object. Combine with `jsonPath` when you need one nested field instead of the whole object. + - `tool_calls`: the tool calls recorded on the observation, as an array of `{id, name, arguments, type, index}` objects in the order the model emitted them. Combine with `jsonPath` (for example `$[*].name`) to select parts of each call. + - `expected_output`: the experiment item's expected output when the observation belongs to an experiment. + - `experiment_item_metadata`: the experiment item's metadata when the observation belongs to an experiment. + """ + + INPUT = "input" + OUTPUT = "output" + METADATA = "metadata" + TOOL_CALLS = "tool_calls" + EXPECTED_OUTPUT = "expected_output" + EXPERIMENT_ITEM_METADATA = "experiment_item_metadata" + + def visit( + self, + input: typing.Callable[[], T_Result], + output: typing.Callable[[], T_Result], + metadata: typing.Callable[[], T_Result], + tool_calls: typing.Callable[[], T_Result], + expected_output: typing.Callable[[], T_Result], + experiment_item_metadata: typing.Callable[[], T_Result], + ) -> T_Result: + if self is PromptVariableMappingSource.INPUT: + return input() + if self is PromptVariableMappingSource.OUTPUT: + return output() + if self is PromptVariableMappingSource.METADATA: + return metadata() + if self is PromptVariableMappingSource.TOOL_CALLS: + return tool_calls() + if self is PromptVariableMappingSource.EXPECTED_OUTPUT: + return expected_output() + if self is PromptVariableMappingSource.EXPERIMENT_ITEM_METADATA: + return experiment_item_metadata() diff --git a/langfuse/api/evaluation_commons/types/public_evaluator_boolean_score.py b/langfuse/api/evaluation_commons/types/public_evaluator_boolean_score.py new file mode 100644 index 000000000..1aa825b82 --- /dev/null +++ b/langfuse/api/evaluation_commons/types/public_evaluator_boolean_score.py @@ -0,0 +1,21 @@ +# This file was auto-generated by Fern from our API Definition. + +import typing + +import pydantic +import typing_extensions +from ...core.serialization import FieldMetadata +from .evaluator_output_definition_base import EvaluatorOutputDefinitionBase + + +class PublicEvaluatorBooleanScore(EvaluatorOutputDefinitionBase): + data_type: typing_extensions.Annotated[ + typing.Literal["BOOLEAN"], FieldMetadata(alias="dataType") + ] = pydantic.Field(default="BOOLEAN") + """ + Boolean score output. + """ + + model_config: typing.ClassVar[pydantic.ConfigDict] = pydantic.ConfigDict( + extra="allow", frozen=True + ) diff --git a/langfuse/api/evaluation_commons/types/public_evaluator_categorical_score.py b/langfuse/api/evaluation_commons/types/public_evaluator_categorical_score.py new file mode 100644 index 000000000..7ffb7ecb0 --- /dev/null +++ b/langfuse/api/evaluation_commons/types/public_evaluator_categorical_score.py @@ -0,0 +1,33 @@ +# This file was auto-generated by Fern from our API Definition. + +import typing + +import pydantic +import typing_extensions +from ...core.serialization import FieldMetadata +from .evaluator_output_definition_base import EvaluatorOutputDefinitionBase + + +class PublicEvaluatorCategoricalScore(EvaluatorOutputDefinitionBase): + data_type: typing_extensions.Annotated[ + typing.Literal["CATEGORICAL"], FieldMetadata(alias="dataType") + ] = pydantic.Field(default="CATEGORICAL") + """ + Categorical score output. + """ + + categories: typing.List[str] = pydantic.Field() + """ + Allowed category values. At least two unique values are required. + """ + + should_allow_multiple_matches: typing_extensions.Annotated[ + bool, FieldMetadata(alias="shouldAllowMultipleMatches") + ] = pydantic.Field() + """ + Whether the evaluator may return more than one category. + """ + + model_config: typing.ClassVar[pydantic.ConfigDict] = pydantic.ConfigDict( + extra="allow", frozen=True + ) diff --git a/langfuse/api/evaluation_commons/types/public_evaluator_numeric_score.py b/langfuse/api/evaluation_commons/types/public_evaluator_numeric_score.py new file mode 100644 index 000000000..6a734f569 --- /dev/null +++ b/langfuse/api/evaluation_commons/types/public_evaluator_numeric_score.py @@ -0,0 +1,35 @@ +# This file was auto-generated by Fern from our API Definition. + +import typing + +import pydantic +import typing_extensions +from ...core.serialization import FieldMetadata +from .evaluator_output_definition_base import EvaluatorOutputDefinitionBase + + +class PublicEvaluatorNumericScore(EvaluatorOutputDefinitionBase): + data_type: typing_extensions.Annotated[ + typing.Literal["NUMERIC"], FieldMetadata(alias="dataType") + ] = pydantic.Field(default="NUMERIC") + """ + Numeric score output. + """ + + min_value: typing_extensions.Annotated[ + typing.Optional[float], FieldMetadata(alias="minValue") + ] = pydantic.Field(default=None) + """ + Optional inclusive minimum value. + """ + + max_value: typing_extensions.Annotated[ + typing.Optional[float], FieldMetadata(alias="maxValue") + ] = pydantic.Field(default=None) + """ + Optional inclusive maximum value. + """ + + model_config: typing.ClassVar[pydantic.ConfigDict] = pydantic.ConfigDict( + extra="allow", frozen=True + ) diff --git a/langfuse/api/evaluation_commons/types/public_evaluator_output_definition.py b/langfuse/api/evaluation_commons/types/public_evaluator_output_definition.py new file mode 100644 index 000000000..01b662def --- /dev/null +++ b/langfuse/api/evaluation_commons/types/public_evaluator_output_definition.py @@ -0,0 +1,134 @@ +# This file was auto-generated by Fern from our API Definition. + +from __future__ import annotations + +import typing + +import pydantic +import typing_extensions +from ...core.pydantic_utilities import UniversalBaseModel +from ...core.serialization import FieldMetadata + + +class PublicEvaluatorOutputDefinition_Numeric(UniversalBaseModel): + """ + Flat evaluator output definition returned by the public API. + + This response always includes `dataType` and never includes an internal output-definition `version`. + Optional empty descriptions from legacy definitions are omitted. + + Examples + -------- + from langfuse.evaluation_commons import PublicEvaluatorOutputDefinition_Numeric + + PublicEvaluatorOutputDefinition_Numeric( + min_value=0.0, + max_value=1.0, + ) + """ + + data_type: typing_extensions.Annotated[ + typing.Literal["NUMERIC"], FieldMetadata(alias="dataType") + ] = "NUMERIC" + min_value: typing_extensions.Annotated[ + typing.Optional[float], FieldMetadata(alias="minValue") + ] = None + max_value: typing_extensions.Annotated[ + typing.Optional[float], FieldMetadata(alias="maxValue") + ] = None + score_reasoning_instructions: typing_extensions.Annotated[ + typing.Optional[str], FieldMetadata(alias="scoreReasoningInstructions") + ] = None + score_value_instructions: typing_extensions.Annotated[ + typing.Optional[str], FieldMetadata(alias="scoreValueInstructions") + ] = None + + model_config: typing.ClassVar[pydantic.ConfigDict] = pydantic.ConfigDict( + extra="allow", frozen=True + ) + + +class PublicEvaluatorOutputDefinition_Boolean(UniversalBaseModel): + """ + Flat evaluator output definition returned by the public API. + + This response always includes `dataType` and never includes an internal output-definition `version`. + Optional empty descriptions from legacy definitions are omitted. + + Examples + -------- + from langfuse.evaluation_commons import PublicEvaluatorOutputDefinition_Numeric + + PublicEvaluatorOutputDefinition_Numeric( + min_value=0.0, + max_value=1.0, + ) + """ + + data_type: typing_extensions.Annotated[ + typing.Literal["BOOLEAN"], FieldMetadata(alias="dataType") + ] = "BOOLEAN" + score_reasoning_instructions: typing_extensions.Annotated[ + typing.Optional[str], FieldMetadata(alias="scoreReasoningInstructions") + ] = None + score_value_instructions: typing_extensions.Annotated[ + typing.Optional[str], FieldMetadata(alias="scoreValueInstructions") + ] = None + + model_config: typing.ClassVar[pydantic.ConfigDict] = pydantic.ConfigDict( + extra="allow", frozen=True + ) + + +class PublicEvaluatorOutputDefinition_Categorical(UniversalBaseModel): + """ + Flat evaluator output definition returned by the public API. + + This response always includes `dataType` and never includes an internal output-definition `version`. + Optional empty descriptions from legacy definitions are omitted. + + Examples + -------- + from langfuse.evaluation_commons import PublicEvaluatorOutputDefinition_Numeric + + PublicEvaluatorOutputDefinition_Numeric( + min_value=0.0, + max_value=1.0, + ) + """ + + data_type: typing_extensions.Annotated[ + typing.Literal["CATEGORICAL"], FieldMetadata(alias="dataType") + ] = "CATEGORICAL" + categories: typing.List[str] + should_allow_multiple_matches: typing_extensions.Annotated[ + bool, FieldMetadata(alias="shouldAllowMultipleMatches") + ] + score_reasoning_instructions: typing_extensions.Annotated[ + typing.Optional[str], FieldMetadata(alias="scoreReasoningInstructions") + ] = None + score_value_instructions: typing_extensions.Annotated[ + typing.Optional[str], FieldMetadata(alias="scoreValueInstructions") + ] = None + + model_config: typing.ClassVar[pydantic.ConfigDict] = pydantic.ConfigDict( + extra="allow", frozen=True + ) + + +""" +from langfuse.evaluation_commons import PublicEvaluatorOutputDefinition_Numeric + +PublicEvaluatorOutputDefinition_Numeric( + min_value=0.0, + max_value=1.0, +) +""" +PublicEvaluatorOutputDefinition = typing_extensions.Annotated[ + typing.Union[ + PublicEvaluatorOutputDefinition_Numeric, + PublicEvaluatorOutputDefinition_Boolean, + PublicEvaluatorOutputDefinition_Categorical, + ], + pydantic.Field(discriminator="data_type"), +] diff --git a/langfuse/api/evaluation_commons/types/string_evaluation_rule_filter.py b/langfuse/api/evaluation_commons/types/string_evaluation_rule_filter.py new file mode 100644 index 000000000..942a9ad61 --- /dev/null +++ b/langfuse/api/evaluation_commons/types/string_evaluation_rule_filter.py @@ -0,0 +1,21 @@ +# This file was auto-generated by Fern from our API Definition. + +import typing + +import pydantic +from ...core.pydantic_utilities import UniversalBaseModel +from .evaluation_rule_string_filter_operator import EvaluationRuleStringFilterOperator + + +class StringEvaluationRuleFilter(UniversalBaseModel): + column: str = pydantic.Field() + """ + Column to filter on. + """ + + operator: EvaluationRuleStringFilterOperator + value: str + + model_config: typing.ClassVar[pydantic.ConfigDict] = pydantic.ConfigDict( + extra="allow", frozen=True + ) diff --git a/langfuse/api/evaluation_commons/types/string_object_evaluation_rule_filter.py b/langfuse/api/evaluation_commons/types/string_object_evaluation_rule_filter.py new file mode 100644 index 000000000..d291153f0 --- /dev/null +++ b/langfuse/api/evaluation_commons/types/string_object_evaluation_rule_filter.py @@ -0,0 +1,26 @@ +# This file was auto-generated by Fern from our API Definition. + +import typing + +import pydantic +from ...core.pydantic_utilities import UniversalBaseModel +from .evaluation_rule_string_filter_operator import EvaluationRuleStringFilterOperator + + +class StringObjectEvaluationRuleFilter(UniversalBaseModel): + column: str = pydantic.Field() + """ + Object-valued column to filter on. Currently only `metadata` is supported. + """ + + key: str = pydantic.Field() + """ + Top-level key inside the object-valued column to filter on. + """ + + operator: EvaluationRuleStringFilterOperator + value: str + + model_config: typing.ClassVar[pydantic.ConfigDict] = pydantic.ConfigDict( + extra="allow", frozen=True + ) diff --git a/langfuse/api/evaluation_commons/types/string_options_evaluation_rule_filter.py b/langfuse/api/evaluation_commons/types/string_options_evaluation_rule_filter.py new file mode 100644 index 000000000..31bda7301 --- /dev/null +++ b/langfuse/api/evaluation_commons/types/string_options_evaluation_rule_filter.py @@ -0,0 +1,24 @@ +# This file was auto-generated by Fern from our API Definition. + +import typing + +import pydantic +from ...core.pydantic_utilities import UniversalBaseModel +from .evaluation_rule_options_filter_operator import EvaluationRuleOptionsFilterOperator + + +class StringOptionsEvaluationRuleFilter(UniversalBaseModel): + column: str = pydantic.Field() + """ + Column to filter on. + """ + + operator: EvaluationRuleOptionsFilterOperator + value: typing.List[str] = pydantic.Field() + """ + One or more allowed string values. + """ + + model_config: typing.ClassVar[pydantic.ConfigDict] = pydantic.ConfigDict( + extra="allow", frozen=True + ) diff --git a/langfuse/api/evaluation_errors/__init__.py b/langfuse/api/evaluation_errors/__init__.py new file mode 100644 index 000000000..073193082 --- /dev/null +++ b/langfuse/api/evaluation_errors/__init__.py @@ -0,0 +1,84 @@ +# This file was auto-generated by Fern from our API Definition. + +# isort: skip_file + +import typing +from importlib import import_module + +if typing.TYPE_CHECKING: + from .types import ( + PublicApiError, + PublicApiErrorCode, + PublicApiErrorDetails, + PublicApiValidationIssue, + ) + from .errors import ( + BadRequestError, + ConflictError, + EvaluationAccessDeniedError, + EvaluationMethodNotAllowedError, + EvaluationNotFoundError, + EvaluationUnauthorizedError, + InternalServerError, + PreconditionFailedError, + TooManyRequestsError, + ) +_dynamic_imports: typing.Dict[str, str] = { + "BadRequestError": ".errors", + "ConflictError": ".errors", + "EvaluationAccessDeniedError": ".errors", + "EvaluationMethodNotAllowedError": ".errors", + "EvaluationNotFoundError": ".errors", + "EvaluationUnauthorizedError": ".errors", + "InternalServerError": ".errors", + "PreconditionFailedError": ".errors", + "PublicApiError": ".types", + "PublicApiErrorCode": ".types", + "PublicApiErrorDetails": ".types", + "PublicApiValidationIssue": ".types", + "TooManyRequestsError": ".errors", +} + + +def __getattr__(attr_name: str) -> typing.Any: + module_name = _dynamic_imports.get(attr_name) + if module_name is None: + raise AttributeError( + f"No {attr_name} found in _dynamic_imports for module name -> {__name__}" + ) + try: + module = import_module(module_name, __package__) + if module_name == f".{attr_name}": + return module + else: + return getattr(module, attr_name) + except ImportError as e: + raise ImportError( + f"Failed to import {attr_name} from {module_name}: {e}" + ) from e + except AttributeError as e: + raise AttributeError( + f"Failed to get {attr_name} from {module_name}: {e}" + ) from e + + +def __dir__(): + lazy_attrs = list(_dynamic_imports.keys()) + return sorted(lazy_attrs) + + +__all__ = [ + "BadRequestError", + "ConflictError", + "EvaluationAccessDeniedError", + "EvaluationMethodNotAllowedError", + "EvaluationNotFoundError", + "EvaluationUnauthorizedError", + "InternalServerError", + "PreconditionFailedError", + "PublicApiError", + "PublicApiErrorCode", + "PublicApiErrorDetails", + "PublicApiValidationIssue", + "TooManyRequestsError", +] diff --git a/langfuse/api/evaluation_errors/errors/__init__.py b/langfuse/api/evaluation_errors/errors/__init__.py new file mode 100644 index 000000000..210831480 --- /dev/null +++ b/langfuse/api/evaluation_errors/errors/__init__.py @@ -0,0 +1,68 @@ +# This file was auto-generated by Fern from our API Definition. + +# isort: skip_file + +import typing +from importlib import import_module + +if typing.TYPE_CHECKING: + from .bad_request_error import BadRequestError + from .conflict_error import ConflictError + from .evaluation_access_denied_error import EvaluationAccessDeniedError + from .evaluation_method_not_allowed_error import EvaluationMethodNotAllowedError + from .evaluation_not_found_error import EvaluationNotFoundError + from .evaluation_unauthorized_error import EvaluationUnauthorizedError + from .internal_server_error import InternalServerError + from .precondition_failed_error import PreconditionFailedError + from .too_many_requests_error import TooManyRequestsError +_dynamic_imports: typing.Dict[str, str] = { + "BadRequestError": ".bad_request_error", + "ConflictError": ".conflict_error", + "EvaluationAccessDeniedError": ".evaluation_access_denied_error", + "EvaluationMethodNotAllowedError": ".evaluation_method_not_allowed_error", + "EvaluationNotFoundError": ".evaluation_not_found_error", + "EvaluationUnauthorizedError": ".evaluation_unauthorized_error", + "InternalServerError": ".internal_server_error", + "PreconditionFailedError": ".precondition_failed_error", + "TooManyRequestsError": ".too_many_requests_error", +} + + +def __getattr__(attr_name: str) -> typing.Any: + module_name = _dynamic_imports.get(attr_name) + if module_name is None: + raise AttributeError( + f"No {attr_name} found in _dynamic_imports for module name -> {__name__}" + ) + try: + module = import_module(module_name, __package__) + if module_name == f".{attr_name}": + return module + else: + return getattr(module, attr_name) + except ImportError as e: + raise ImportError( + f"Failed to import {attr_name} from {module_name}: {e}" + ) from e + except AttributeError as e: + raise AttributeError( + f"Failed to get {attr_name} from {module_name}: {e}" + ) from e + + +def __dir__(): + lazy_attrs = list(_dynamic_imports.keys()) + return sorted(lazy_attrs) + + +__all__ = [ + "BadRequestError", + "ConflictError", + "EvaluationAccessDeniedError", + "EvaluationMethodNotAllowedError", + "EvaluationNotFoundError", + "EvaluationUnauthorizedError", + "InternalServerError", + "PreconditionFailedError", + "TooManyRequestsError", +] diff --git a/langfuse/api/evaluation_errors/errors/bad_request_error.py b/langfuse/api/evaluation_errors/errors/bad_request_error.py new file mode 100644 index 000000000..c78332bf3 --- /dev/null +++ b/langfuse/api/evaluation_errors/errors/bad_request_error.py @@ -0,0 +1,15 @@ +# This file was auto-generated by Fern from our API Definition. + +import typing + +from ...core.api_error import ApiError +from ..types.public_api_error import PublicApiError + + +class BadRequestError(ApiError): + def __init__( + self, + body: PublicApiError, + headers: typing.Optional[typing.Dict[str, str]] = None, + ): + super().__init__(status_code=400, headers=headers, body=body) diff --git a/langfuse/api/evaluation_errors/errors/conflict_error.py b/langfuse/api/evaluation_errors/errors/conflict_error.py new file mode 100644 index 000000000..5ff6835ce --- /dev/null +++ b/langfuse/api/evaluation_errors/errors/conflict_error.py @@ -0,0 +1,15 @@ +# This file was auto-generated by Fern from our API Definition. + +import typing + +from ...core.api_error import ApiError +from ..types.public_api_error import PublicApiError + + +class ConflictError(ApiError): + def __init__( + self, + body: PublicApiError, + headers: typing.Optional[typing.Dict[str, str]] = None, + ): + super().__init__(status_code=409, headers=headers, body=body) diff --git a/langfuse/api/evaluation_errors/errors/evaluation_access_denied_error.py b/langfuse/api/evaluation_errors/errors/evaluation_access_denied_error.py new file mode 100644 index 000000000..6aaa266ce --- /dev/null +++ b/langfuse/api/evaluation_errors/errors/evaluation_access_denied_error.py @@ -0,0 +1,15 @@ +# This file was auto-generated by Fern from our API Definition. + +import typing + +from ...core.api_error import ApiError +from ..types.public_api_error import PublicApiError + + +class EvaluationAccessDeniedError(ApiError): + def __init__( + self, + body: PublicApiError, + headers: typing.Optional[typing.Dict[str, str]] = None, + ): + super().__init__(status_code=403, headers=headers, body=body) diff --git a/langfuse/api/evaluation_errors/errors/evaluation_method_not_allowed_error.py b/langfuse/api/evaluation_errors/errors/evaluation_method_not_allowed_error.py new file mode 100644 index 000000000..8c2b8980e --- /dev/null +++ b/langfuse/api/evaluation_errors/errors/evaluation_method_not_allowed_error.py @@ -0,0 +1,15 @@ +# This file was auto-generated by Fern from our API Definition. + +import typing + +from ...core.api_error import ApiError +from ..types.public_api_error import PublicApiError + + +class EvaluationMethodNotAllowedError(ApiError): + def __init__( + self, + body: PublicApiError, + headers: typing.Optional[typing.Dict[str, str]] = None, + ): + super().__init__(status_code=405, headers=headers, body=body) diff --git a/langfuse/api/evaluation_errors/errors/evaluation_not_found_error.py b/langfuse/api/evaluation_errors/errors/evaluation_not_found_error.py new file mode 100644 index 000000000..9ab507b9a --- /dev/null +++ b/langfuse/api/evaluation_errors/errors/evaluation_not_found_error.py @@ -0,0 +1,15 @@ +# This file was auto-generated by Fern from our API Definition. + +import typing + +from ...core.api_error import ApiError +from ..types.public_api_error import PublicApiError + + +class EvaluationNotFoundError(ApiError): + def __init__( + self, + body: PublicApiError, + headers: typing.Optional[typing.Dict[str, str]] = None, + ): + super().__init__(status_code=404, headers=headers, body=body) diff --git a/langfuse/api/evaluation_errors/errors/evaluation_unauthorized_error.py b/langfuse/api/evaluation_errors/errors/evaluation_unauthorized_error.py new file mode 100644 index 000000000..0200f426f --- /dev/null +++ b/langfuse/api/evaluation_errors/errors/evaluation_unauthorized_error.py @@ -0,0 +1,15 @@ +# This file was auto-generated by Fern from our API Definition. + +import typing + +from ...core.api_error import ApiError +from ..types.public_api_error import PublicApiError + + +class EvaluationUnauthorizedError(ApiError): + def __init__( + self, + body: PublicApiError, + headers: typing.Optional[typing.Dict[str, str]] = None, + ): + super().__init__(status_code=401, headers=headers, body=body) diff --git a/langfuse/api/evaluation_errors/errors/internal_server_error.py b/langfuse/api/evaluation_errors/errors/internal_server_error.py new file mode 100644 index 000000000..da88d1fa2 --- /dev/null +++ b/langfuse/api/evaluation_errors/errors/internal_server_error.py @@ -0,0 +1,15 @@ +# This file was auto-generated by Fern from our API Definition. + +import typing + +from ...core.api_error import ApiError +from ..types.public_api_error import PublicApiError + + +class InternalServerError(ApiError): + def __init__( + self, + body: PublicApiError, + headers: typing.Optional[typing.Dict[str, str]] = None, + ): + super().__init__(status_code=500, headers=headers, body=body) diff --git a/langfuse/api/evaluation_errors/errors/precondition_failed_error.py b/langfuse/api/evaluation_errors/errors/precondition_failed_error.py new file mode 100644 index 000000000..d6d5e00c4 --- /dev/null +++ b/langfuse/api/evaluation_errors/errors/precondition_failed_error.py @@ -0,0 +1,15 @@ +# This file was auto-generated by Fern from our API Definition. + +import typing + +from ...core.api_error import ApiError +from ..types.public_api_error import PublicApiError + + +class PreconditionFailedError(ApiError): + def __init__( + self, + body: PublicApiError, + headers: typing.Optional[typing.Dict[str, str]] = None, + ): + super().__init__(status_code=412, headers=headers, body=body) diff --git a/langfuse/api/evaluation_errors/errors/too_many_requests_error.py b/langfuse/api/evaluation_errors/errors/too_many_requests_error.py new file mode 100644 index 000000000..6f74b3c06 --- /dev/null +++ b/langfuse/api/evaluation_errors/errors/too_many_requests_error.py @@ -0,0 +1,15 @@ +# This file was auto-generated by Fern from our API Definition. + +import typing + +from ...core.api_error import ApiError +from ..types.public_api_error import PublicApiError + + +class TooManyRequestsError(ApiError): + def __init__( + self, + body: PublicApiError, + headers: typing.Optional[typing.Dict[str, str]] = None, + ): + super().__init__(status_code=429, headers=headers, body=body) diff --git a/langfuse/api/evaluation_errors/types/__init__.py b/langfuse/api/evaluation_errors/types/__init__.py new file mode 100644 index 000000000..fd016304e --- /dev/null +++ b/langfuse/api/evaluation_errors/types/__init__.py @@ -0,0 +1,53 @@ +# This file was auto-generated by Fern from our API Definition. + +# isort: skip_file + +import typing +from importlib import import_module + +if typing.TYPE_CHECKING: + from .public_api_error import PublicApiError + from .public_api_error_code import PublicApiErrorCode + from .public_api_error_details import PublicApiErrorDetails + from .public_api_validation_issue import PublicApiValidationIssue +_dynamic_imports: typing.Dict[str, str] = { + "PublicApiError": ".public_api_error", + "PublicApiErrorCode": ".public_api_error_code", + "PublicApiErrorDetails": ".public_api_error_details", + "PublicApiValidationIssue": ".public_api_validation_issue", +} + + +def __getattr__(attr_name: str) -> typing.Any: + module_name = _dynamic_imports.get(attr_name) + if module_name is None: + raise AttributeError( + f"No {attr_name} found in _dynamic_imports for module name -> {__name__}" + ) + try: + module = import_module(module_name, __package__) + if module_name == f".{attr_name}": + return module + else: + return getattr(module, attr_name) + except ImportError as e: + raise ImportError( + f"Failed to import {attr_name} from {module_name}: {e}" + ) from e + except AttributeError as e: + raise AttributeError( + f"Failed to get {attr_name} from {module_name}: {e}" + ) from e + + +def __dir__(): + lazy_attrs = list(_dynamic_imports.keys()) + return sorted(lazy_attrs) + + +__all__ = [ + "PublicApiError", + "PublicApiErrorCode", + "PublicApiErrorDetails", + "PublicApiValidationIssue", +] diff --git a/langfuse/api/evaluation_errors/types/public_api_error.py b/langfuse/api/evaluation_errors/types/public_api_error.py new file mode 100644 index 000000000..62e587559 --- /dev/null +++ b/langfuse/api/evaluation_errors/types/public_api_error.py @@ -0,0 +1,58 @@ +# This file was auto-generated by Fern from our API Definition. + +import typing + +import pydantic +from ...core.pydantic_utilities import UniversalBaseModel +from .public_api_error_code import PublicApiErrorCode +from .public_api_error_details import PublicApiErrorDetails + + +class PublicApiError(UniversalBaseModel): + """ + Standard error response for the stable evaluators and evaluation-rules API. + + Use the HTTP status for the broad failure class and `code` for programmatic handling. `details` is included when field-level validation or retry information is available. + + Examples + -------- + from langfuse.evaluation_errors import ( + PublicApiError, + PublicApiErrorCode, + PublicApiErrorDetails, + PublicApiValidationIssue, + ) + + PublicApiError( + message="Invalid request body", + code=PublicApiErrorCode.INVALID_BODY, + details=PublicApiErrorDetails( + issues=[ + PublicApiValidationIssue( + code="invalid_type", + message="Invalid input", + path=["definition"], + ) + ], + ), + ) + """ + + message: str = pydantic.Field() + """ + Human-readable description of the failure. + """ + + code: PublicApiErrorCode = pydantic.Field() + """ + Stable machine-readable error code. + """ + + details: typing.Optional[PublicApiErrorDetails] = pydantic.Field(default=None) + """ + Optional structured validation or rate-limit context. + """ + + model_config: typing.ClassVar[pydantic.ConfigDict] = pydantic.ConfigDict( + extra="allow", frozen=True + ) diff --git a/langfuse/api/evaluation_errors/types/public_api_error_code.py b/langfuse/api/evaluation_errors/types/public_api_error_code.py new file mode 100644 index 000000000..828fb2df5 --- /dev/null +++ b/langfuse/api/evaluation_errors/types/public_api_error_code.py @@ -0,0 +1,58 @@ +# This file was auto-generated by Fern from our API Definition. + +import typing + +from ...core import enum + +T_Result = typing.TypeVar("T_Result") + + +class PublicApiErrorCode(enum.StrEnum): + """ + Stable machine-readable error code returned by the evaluators and evaluation-rules API. + """ + + AUTHENTICATION_FAILED = "authentication_failed" + ACCESS_DENIED = "access_denied" + INVALID_REQUEST = "invalid_request" + INVALID_QUERY = "invalid_query" + INVALID_BODY = "invalid_body" + RESOURCE_NOT_FOUND = "resource_not_found" + CONFLICT = "conflict" + RATE_LIMITED = "rate_limited" + METHOD_NOT_ALLOWED = "method_not_allowed" + INTERNAL_ERROR = "internal_error" + + def visit( + self, + authentication_failed: typing.Callable[[], T_Result], + access_denied: typing.Callable[[], T_Result], + invalid_request: typing.Callable[[], T_Result], + invalid_query: typing.Callable[[], T_Result], + invalid_body: typing.Callable[[], T_Result], + resource_not_found: typing.Callable[[], T_Result], + conflict: typing.Callable[[], T_Result], + rate_limited: typing.Callable[[], T_Result], + method_not_allowed: typing.Callable[[], T_Result], + internal_error: typing.Callable[[], T_Result], + ) -> T_Result: + if self is PublicApiErrorCode.AUTHENTICATION_FAILED: + return authentication_failed() + if self is PublicApiErrorCode.ACCESS_DENIED: + return access_denied() + if self is PublicApiErrorCode.INVALID_REQUEST: + return invalid_request() + if self is PublicApiErrorCode.INVALID_QUERY: + return invalid_query() + if self is PublicApiErrorCode.INVALID_BODY: + return invalid_body() + if self is PublicApiErrorCode.RESOURCE_NOT_FOUND: + return resource_not_found() + if self is PublicApiErrorCode.CONFLICT: + return conflict() + if self is PublicApiErrorCode.RATE_LIMITED: + return rate_limited() + if self is PublicApiErrorCode.METHOD_NOT_ALLOWED: + return method_not_allowed() + if self is PublicApiErrorCode.INTERNAL_ERROR: + return internal_error() diff --git a/langfuse/api/evaluation_errors/types/public_api_error_details.py b/langfuse/api/evaluation_errors/types/public_api_error_details.py new file mode 100644 index 000000000..9a4a1daa5 --- /dev/null +++ b/langfuse/api/evaluation_errors/types/public_api_error_details.py @@ -0,0 +1,51 @@ +# This file was auto-generated by Fern from our API Definition. + +import datetime as dt +import typing + +import pydantic +import typing_extensions +from ...core.pydantic_utilities import UniversalBaseModel +from ...core.serialization import FieldMetadata +from .public_api_validation_issue import PublicApiValidationIssue + + +class PublicApiErrorDetails(UniversalBaseModel): + """ + Optional structured context for validation and rate-limit errors. + """ + + issues: typing.Optional[typing.List[PublicApiValidationIssue]] = pydantic.Field( + default=None + ) + """ + Validation issues for an invalid request body or query. + """ + + retry_after_seconds: typing_extensions.Annotated[ + typing.Optional[int], FieldMetadata(alias="retryAfterSeconds") + ] = pydantic.Field(default=None) + """ + Number of seconds to wait before retrying a rate-limited request. + """ + + limit: typing.Optional[int] = pydantic.Field(default=None) + """ + Rate-limit request allowance. + """ + + remaining: typing.Optional[int] = pydantic.Field(default=None) + """ + Remaining requests in the current rate-limit window. + """ + + reset_at: typing_extensions.Annotated[ + typing.Optional[dt.datetime], FieldMetadata(alias="resetAt") + ] = pydantic.Field(default=None) + """ + Time when the current rate-limit window resets. + """ + + model_config: typing.ClassVar[pydantic.ConfigDict] = pydantic.ConfigDict( + extra="allow", frozen=True + ) diff --git a/langfuse/api/evaluation_errors/types/public_api_validation_issue.py b/langfuse/api/evaluation_errors/types/public_api_validation_issue.py new file mode 100644 index 000000000..eb17a3851 --- /dev/null +++ b/langfuse/api/evaluation_errors/types/public_api_validation_issue.py @@ -0,0 +1,31 @@ +# This file was auto-generated by Fern from our API Definition. + +import typing + +import pydantic +from ...core.pydantic_utilities import UniversalBaseModel + + +class PublicApiValidationIssue(UniversalBaseModel): + """ + One request validation issue. + """ + + code: str = pydantic.Field() + """ + Validator issue code. + """ + + message: str = pydantic.Field() + """ + Human-readable explanation. + """ + + path: typing.List[typing.Any] = pydantic.Field() + """ + Path to the invalid request field. + """ + + model_config: typing.ClassVar[pydantic.ConfigDict] = pydantic.ConfigDict( + extra="allow", frozen=True + ) diff --git a/langfuse/api/evaluation_rules/__init__.py b/langfuse/api/evaluation_rules/__init__.py new file mode 100644 index 000000000..6573f1bb5 --- /dev/null +++ b/langfuse/api/evaluation_rules/__init__.py @@ -0,0 +1,64 @@ +# This file was auto-generated by Fern from our API Definition. + +# isort: skip_file + +import typing +from importlib import import_module + +if typing.TYPE_CHECKING: + from .types import ( + CreateEvaluationRuleRequest, + DeletedEvaluationRule, + EvaluationRule, + EvaluationRuleEvaluatorAssignmentInput, + EvaluationRulesPage, + EvaluatorAssignment, + UpdateEvaluationRuleRequest, + ) +_dynamic_imports: typing.Dict[str, str] = { + "CreateEvaluationRuleRequest": ".types", + "DeletedEvaluationRule": ".types", + "EvaluationRule": ".types", + "EvaluationRuleEvaluatorAssignmentInput": ".types", + "EvaluationRulesPage": ".types", + "EvaluatorAssignment": ".types", + "UpdateEvaluationRuleRequest": ".types", +} + + +def __getattr__(attr_name: str) -> typing.Any: + module_name = _dynamic_imports.get(attr_name) + if module_name is None: + raise AttributeError( + f"No {attr_name} found in _dynamic_imports for module name -> {__name__}" + ) + try: + module = import_module(module_name, __package__) + if module_name == f".{attr_name}": + return module + else: + return getattr(module, attr_name) + except ImportError as e: + raise ImportError( + f"Failed to import {attr_name} from {module_name}: {e}" + ) from e + except AttributeError as e: + raise AttributeError( + f"Failed to get {attr_name} from {module_name}: {e}" + ) from e + + +def __dir__(): + lazy_attrs = list(_dynamic_imports.keys()) + return sorted(lazy_attrs) + + +__all__ = [ + "CreateEvaluationRuleRequest", + "DeletedEvaluationRule", + "EvaluationRule", + "EvaluationRuleEvaluatorAssignmentInput", + "EvaluationRulesPage", + "EvaluatorAssignment", + "UpdateEvaluationRuleRequest", +] diff --git a/langfuse/api/evaluation_rules/client.py b/langfuse/api/evaluation_rules/client.py new file mode 100644 index 000000000..21dcd1852 --- /dev/null +++ b/langfuse/api/evaluation_rules/client.py @@ -0,0 +1,671 @@ +# This file was auto-generated by Fern from our API Definition. + +import typing + +from ..core.client_wrapper import AsyncClientWrapper, SyncClientWrapper +from ..core.request_options import RequestOptions +from ..evaluation_commons.types.evaluation_rule_filter import EvaluationRuleFilter +from .raw_client import AsyncRawEvaluationRulesClient, RawEvaluationRulesClient +from .types.deleted_evaluation_rule import DeletedEvaluationRule +from .types.evaluation_rule import EvaluationRule +from .types.evaluation_rule_evaluator_assignment_input import ( + EvaluationRuleEvaluatorAssignmentInput, +) +from .types.evaluation_rules_page import EvaluationRulesPage + +# this is used as the default value for optional parameters +OMIT = typing.cast(typing.Any, ...) + + +class EvaluationRulesClient: + def __init__(self, *, client_wrapper: SyncClientWrapper): + self._raw_client = RawEvaluationRulesClient(client_wrapper=client_wrapper) + + @property + def with_raw_response(self) -> RawEvaluationRulesClient: + """ + Retrieves a raw implementation of this client that returns raw responses. + + Returns + ------- + RawEvaluationRulesClient + """ + return self._raw_client + + def create( + self, + *, + name: str, + enabled: bool, + evaluator_assignments: typing.Sequence[EvaluationRuleEvaluatorAssignmentInput], + sampling: typing.Optional[float] = OMIT, + filter: typing.Optional[typing.Sequence[EvaluationRuleFilter]] = OMIT, + request_options: typing.Optional[RequestOptions] = None, + ) -> EvaluationRule: + """ + Create an evaluation rule using stable evaluator identifiers. + + An evaluation rule defines **which** incoming observations should be evaluated and how prompt variables should be populated. + + Key behavior: + - rule names are not identifiers and do not need to be unique + - rules always use the latest version of each associated evaluator + - an enabled rule requires at least one evaluator assignment + - omit `sampling` to evaluate every matching observation + - omit `filter` to match every incoming observation + - `datasetId` is the public filter name for selecting experiment datasets + + Parameters + ---------- + name : str + Human-readable rule name. Names are not identifiers and do not need to be unique. + + enabled : bool + Whether live execution should start immediately. Enabled rules require at least one evaluator assignment. + + evaluator_assignments : typing.Sequence[EvaluationRuleEvaluatorAssignmentInput] + Evaluators to attach to this rule. Disabled rules may use an empty list as a draft. + + sampling : typing.Optional[float] + Fraction of matching observations to evaluate. Omit this field to use the default of `1`, which evaluates every match. `null` is not accepted. + + filter : typing.Optional[typing.Sequence[EvaluationRuleFilter]] + Conditions used to select observations. Defaults to an empty list, which matches every incoming observation. + + request_options : typing.Optional[RequestOptions] + Request-specific configuration. + + Returns + ------- + EvaluationRule + + Examples + -------- + from langfuse import LangfuseAPI + from langfuse.evaluation_rules import EvaluationRuleEvaluatorAssignmentInput + + client = LangfuseAPI( + x_langfuse_sdk_name="YOUR_X_LANGFUSE_SDK_NAME", + x_langfuse_sdk_version="YOUR_X_LANGFUSE_SDK_VERSION", + x_langfuse_public_key="YOUR_X_LANGFUSE_PUBLIC_KEY", + username="YOUR_USERNAME", + password="YOUR_PASSWORD", + base_url="https://yourhost.com/path/to/api", + ) + client.evaluation_rules.create( + name="name", + enabled=True, + evaluator_assignments=[ + EvaluationRuleEvaluatorAssignmentInput( + evaluator_id="evaluatorId", + ), + EvaluationRuleEvaluatorAssignmentInput( + evaluator_id="evaluatorId", + ), + ], + ) + """ + _response = self._raw_client.create( + name=name, + enabled=enabled, + evaluator_assignments=evaluator_assignments, + sampling=sampling, + filter=filter, + request_options=request_options, + ) + return _response.data + + def list( + self, + *, + limit: typing.Optional[int] = None, + cursor: typing.Optional[str] = None, + request_options: typing.Optional[RequestOptions] = None, + ) -> EvaluationRulesPage: + """ + List evaluation rules in newest-first creation order. + + This includes legacy trace and dataset rules so they can be inspected and migrated. Treat the cursor as opaque and return it unchanged. + + Parameters + ---------- + limit : typing.Optional[int] + Maximum number of items to return. Defaults to `50` and cannot exceed `100`. + + cursor : typing.Optional[str] + Opaque cursor returned by the previous page. + + request_options : typing.Optional[RequestOptions] + Request-specific configuration. + + Returns + ------- + EvaluationRulesPage + + Examples + -------- + from langfuse import LangfuseAPI + + client = LangfuseAPI( + x_langfuse_sdk_name="YOUR_X_LANGFUSE_SDK_NAME", + x_langfuse_sdk_version="YOUR_X_LANGFUSE_SDK_VERSION", + x_langfuse_public_key="YOUR_X_LANGFUSE_PUBLIC_KEY", + username="YOUR_USERNAME", + password="YOUR_PASSWORD", + base_url="https://yourhost.com/path/to/api", + ) + client.evaluation_rules.list() + """ + _response = self._raw_client.list( + limit=limit, cursor=cursor, request_options=request_options + ) + return _response.data + + def get( + self, + evaluation_rule_id: str, + *, + request_options: typing.Optional[RequestOptions] = None, + ) -> EvaluationRule: + """ + Get one evaluation rule, including a legacy trace or dataset rule, by its stable identifier. + + Parameters + ---------- + evaluation_rule_id : str + Stable evaluation-rule identifier returned by the evaluation-rule endpoints. + + request_options : typing.Optional[RequestOptions] + Request-specific configuration. + + Returns + ------- + EvaluationRule + + Examples + -------- + from langfuse import LangfuseAPI + + client = LangfuseAPI( + x_langfuse_sdk_name="YOUR_X_LANGFUSE_SDK_NAME", + x_langfuse_sdk_version="YOUR_X_LANGFUSE_SDK_VERSION", + x_langfuse_public_key="YOUR_X_LANGFUSE_PUBLIC_KEY", + username="YOUR_USERNAME", + password="YOUR_PASSWORD", + base_url="https://yourhost.com/path/to/api", + ) + client.evaluation_rules.get( + evaluation_rule_id="evaluationRuleId", + ) + """ + _response = self._raw_client.get( + evaluation_rule_id, request_options=request_options + ) + return _response.data + + def update( + self, + evaluation_rule_id: str, + *, + name: typing.Optional[str] = OMIT, + enabled: typing.Optional[bool] = OMIT, + sampling: typing.Optional[float] = OMIT, + filter: typing.Optional[typing.Sequence[EvaluationRuleFilter]] = OMIT, + evaluator_assignments: typing.Optional[ + typing.Sequence[EvaluationRuleEvaluatorAssignmentInput] + ] = OMIT, + request_options: typing.Optional[RequestOptions] = None, + ) -> EvaluationRule: + """ + Update an evaluation rule by its stable identifier. + + Provide only the fields to change. Providing `evaluatorAssignments` replaces the complete assignment list. Replacing the list with an empty array disables the rule. Setting `enabled=true` is rejected when the resulting assignment list is empty, including when both fields are sent in the same request. + + Legacy trace and dataset rules follow the existing lifecycle restrictions: they can be deactivated with `enabled=false`, but their name, filters, sampling, and evaluator assignments cannot be changed. + + Parameters + ---------- + evaluation_rule_id : str + Stable evaluation-rule identifier returned by the evaluation-rule endpoints. + + name : typing.Optional[str] + New human-readable rule name. + + enabled : typing.Optional[bool] + New desired live-execution state. + + sampling : typing.Optional[float] + New fraction of matching observations to evaluate. Omit to keep the current value. + + filter : typing.Optional[typing.Sequence[EvaluationRuleFilter]] + Complete replacement filter list. An empty list matches every incoming observation. + + evaluator_assignments : typing.Optional[typing.Sequence[EvaluationRuleEvaluatorAssignmentInput]] + Complete replacement assignment list. An empty list disables the rule. + + request_options : typing.Optional[RequestOptions] + Request-specific configuration. + + Returns + ------- + EvaluationRule + + Examples + -------- + from langfuse import LangfuseAPI + + client = LangfuseAPI( + x_langfuse_sdk_name="YOUR_X_LANGFUSE_SDK_NAME", + x_langfuse_sdk_version="YOUR_X_LANGFUSE_SDK_VERSION", + x_langfuse_public_key="YOUR_X_LANGFUSE_PUBLIC_KEY", + username="YOUR_USERNAME", + password="YOUR_PASSWORD", + base_url="https://yourhost.com/path/to/api", + ) + client.evaluation_rules.update( + evaluation_rule_id="evaluationRuleId", + ) + """ + _response = self._raw_client.update( + evaluation_rule_id, + name=name, + enabled=enabled, + sampling=sampling, + filter=filter, + evaluator_assignments=evaluator_assignments, + request_options=request_options, + ) + return _response.data + + def delete( + self, + evaluation_rule_id: str, + *, + request_options: typing.Optional[RequestOptions] = None, + ) -> DeletedEvaluationRule: + """ + Delete an evaluation rule. + + This removes the live-ingestion rule only. It does not delete associated evaluators or scores already produced by them. + + Legacy trace and dataset rules can also be deleted. Their evaluators and previously produced scores are preserved. + + Parameters + ---------- + evaluation_rule_id : str + Stable evaluation-rule identifier returned by the evaluation-rule endpoints. + + request_options : typing.Optional[RequestOptions] + Request-specific configuration. + + Returns + ------- + DeletedEvaluationRule + + Examples + -------- + from langfuse import LangfuseAPI + + client = LangfuseAPI( + x_langfuse_sdk_name="YOUR_X_LANGFUSE_SDK_NAME", + x_langfuse_sdk_version="YOUR_X_LANGFUSE_SDK_VERSION", + x_langfuse_public_key="YOUR_X_LANGFUSE_PUBLIC_KEY", + username="YOUR_USERNAME", + password="YOUR_PASSWORD", + base_url="https://yourhost.com/path/to/api", + ) + client.evaluation_rules.delete( + evaluation_rule_id="evaluationRuleId", + ) + """ + _response = self._raw_client.delete( + evaluation_rule_id, request_options=request_options + ) + return _response.data + + +class AsyncEvaluationRulesClient: + def __init__(self, *, client_wrapper: AsyncClientWrapper): + self._raw_client = AsyncRawEvaluationRulesClient(client_wrapper=client_wrapper) + + @property + def with_raw_response(self) -> AsyncRawEvaluationRulesClient: + """ + Retrieves a raw implementation of this client that returns raw responses. + + Returns + ------- + AsyncRawEvaluationRulesClient + """ + return self._raw_client + + async def create( + self, + *, + name: str, + enabled: bool, + evaluator_assignments: typing.Sequence[EvaluationRuleEvaluatorAssignmentInput], + sampling: typing.Optional[float] = OMIT, + filter: typing.Optional[typing.Sequence[EvaluationRuleFilter]] = OMIT, + request_options: typing.Optional[RequestOptions] = None, + ) -> EvaluationRule: + """ + Create an evaluation rule using stable evaluator identifiers. + + An evaluation rule defines **which** incoming observations should be evaluated and how prompt variables should be populated. + + Key behavior: + - rule names are not identifiers and do not need to be unique + - rules always use the latest version of each associated evaluator + - an enabled rule requires at least one evaluator assignment + - omit `sampling` to evaluate every matching observation + - omit `filter` to match every incoming observation + - `datasetId` is the public filter name for selecting experiment datasets + + Parameters + ---------- + name : str + Human-readable rule name. Names are not identifiers and do not need to be unique. + + enabled : bool + Whether live execution should start immediately. Enabled rules require at least one evaluator assignment. + + evaluator_assignments : typing.Sequence[EvaluationRuleEvaluatorAssignmentInput] + Evaluators to attach to this rule. Disabled rules may use an empty list as a draft. + + sampling : typing.Optional[float] + Fraction of matching observations to evaluate. Omit this field to use the default of `1`, which evaluates every match. `null` is not accepted. + + filter : typing.Optional[typing.Sequence[EvaluationRuleFilter]] + Conditions used to select observations. Defaults to an empty list, which matches every incoming observation. + + request_options : typing.Optional[RequestOptions] + Request-specific configuration. + + Returns + ------- + EvaluationRule + + Examples + -------- + import asyncio + + from langfuse import AsyncLangfuseAPI + from langfuse.evaluation_rules import EvaluationRuleEvaluatorAssignmentInput + + client = AsyncLangfuseAPI( + x_langfuse_sdk_name="YOUR_X_LANGFUSE_SDK_NAME", + x_langfuse_sdk_version="YOUR_X_LANGFUSE_SDK_VERSION", + x_langfuse_public_key="YOUR_X_LANGFUSE_PUBLIC_KEY", + username="YOUR_USERNAME", + password="YOUR_PASSWORD", + base_url="https://yourhost.com/path/to/api", + ) + + + async def main() -> None: + await client.evaluation_rules.create( + name="name", + enabled=True, + evaluator_assignments=[ + EvaluationRuleEvaluatorAssignmentInput( + evaluator_id="evaluatorId", + ), + EvaluationRuleEvaluatorAssignmentInput( + evaluator_id="evaluatorId", + ), + ], + ) + + + asyncio.run(main()) + """ + _response = await self._raw_client.create( + name=name, + enabled=enabled, + evaluator_assignments=evaluator_assignments, + sampling=sampling, + filter=filter, + request_options=request_options, + ) + return _response.data + + async def list( + self, + *, + limit: typing.Optional[int] = None, + cursor: typing.Optional[str] = None, + request_options: typing.Optional[RequestOptions] = None, + ) -> EvaluationRulesPage: + """ + List evaluation rules in newest-first creation order. + + This includes legacy trace and dataset rules so they can be inspected and migrated. Treat the cursor as opaque and return it unchanged. + + Parameters + ---------- + limit : typing.Optional[int] + Maximum number of items to return. Defaults to `50` and cannot exceed `100`. + + cursor : typing.Optional[str] + Opaque cursor returned by the previous page. + + request_options : typing.Optional[RequestOptions] + Request-specific configuration. + + Returns + ------- + EvaluationRulesPage + + Examples + -------- + import asyncio + + from langfuse import AsyncLangfuseAPI + + client = AsyncLangfuseAPI( + x_langfuse_sdk_name="YOUR_X_LANGFUSE_SDK_NAME", + x_langfuse_sdk_version="YOUR_X_LANGFUSE_SDK_VERSION", + x_langfuse_public_key="YOUR_X_LANGFUSE_PUBLIC_KEY", + username="YOUR_USERNAME", + password="YOUR_PASSWORD", + base_url="https://yourhost.com/path/to/api", + ) + + + async def main() -> None: + await client.evaluation_rules.list() + + + asyncio.run(main()) + """ + _response = await self._raw_client.list( + limit=limit, cursor=cursor, request_options=request_options + ) + return _response.data + + async def get( + self, + evaluation_rule_id: str, + *, + request_options: typing.Optional[RequestOptions] = None, + ) -> EvaluationRule: + """ + Get one evaluation rule, including a legacy trace or dataset rule, by its stable identifier. + + Parameters + ---------- + evaluation_rule_id : str + Stable evaluation-rule identifier returned by the evaluation-rule endpoints. + + request_options : typing.Optional[RequestOptions] + Request-specific configuration. + + Returns + ------- + EvaluationRule + + Examples + -------- + import asyncio + + from langfuse import AsyncLangfuseAPI + + client = AsyncLangfuseAPI( + x_langfuse_sdk_name="YOUR_X_LANGFUSE_SDK_NAME", + x_langfuse_sdk_version="YOUR_X_LANGFUSE_SDK_VERSION", + x_langfuse_public_key="YOUR_X_LANGFUSE_PUBLIC_KEY", + username="YOUR_USERNAME", + password="YOUR_PASSWORD", + base_url="https://yourhost.com/path/to/api", + ) + + + async def main() -> None: + await client.evaluation_rules.get( + evaluation_rule_id="evaluationRuleId", + ) + + + asyncio.run(main()) + """ + _response = await self._raw_client.get( + evaluation_rule_id, request_options=request_options + ) + return _response.data + + async def update( + self, + evaluation_rule_id: str, + *, + name: typing.Optional[str] = OMIT, + enabled: typing.Optional[bool] = OMIT, + sampling: typing.Optional[float] = OMIT, + filter: typing.Optional[typing.Sequence[EvaluationRuleFilter]] = OMIT, + evaluator_assignments: typing.Optional[ + typing.Sequence[EvaluationRuleEvaluatorAssignmentInput] + ] = OMIT, + request_options: typing.Optional[RequestOptions] = None, + ) -> EvaluationRule: + """ + Update an evaluation rule by its stable identifier. + + Provide only the fields to change. Providing `evaluatorAssignments` replaces the complete assignment list. Replacing the list with an empty array disables the rule. Setting `enabled=true` is rejected when the resulting assignment list is empty, including when both fields are sent in the same request. + + Legacy trace and dataset rules follow the existing lifecycle restrictions: they can be deactivated with `enabled=false`, but their name, filters, sampling, and evaluator assignments cannot be changed. + + Parameters + ---------- + evaluation_rule_id : str + Stable evaluation-rule identifier returned by the evaluation-rule endpoints. + + name : typing.Optional[str] + New human-readable rule name. + + enabled : typing.Optional[bool] + New desired live-execution state. + + sampling : typing.Optional[float] + New fraction of matching observations to evaluate. Omit to keep the current value. + + filter : typing.Optional[typing.Sequence[EvaluationRuleFilter]] + Complete replacement filter list. An empty list matches every incoming observation. + + evaluator_assignments : typing.Optional[typing.Sequence[EvaluationRuleEvaluatorAssignmentInput]] + Complete replacement assignment list. An empty list disables the rule. + + request_options : typing.Optional[RequestOptions] + Request-specific configuration. + + Returns + ------- + EvaluationRule + + Examples + -------- + import asyncio + + from langfuse import AsyncLangfuseAPI + + client = AsyncLangfuseAPI( + x_langfuse_sdk_name="YOUR_X_LANGFUSE_SDK_NAME", + x_langfuse_sdk_version="YOUR_X_LANGFUSE_SDK_VERSION", + x_langfuse_public_key="YOUR_X_LANGFUSE_PUBLIC_KEY", + username="YOUR_USERNAME", + password="YOUR_PASSWORD", + base_url="https://yourhost.com/path/to/api", + ) + + + async def main() -> None: + await client.evaluation_rules.update( + evaluation_rule_id="evaluationRuleId", + ) + + + asyncio.run(main()) + """ + _response = await self._raw_client.update( + evaluation_rule_id, + name=name, + enabled=enabled, + sampling=sampling, + filter=filter, + evaluator_assignments=evaluator_assignments, + request_options=request_options, + ) + return _response.data + + async def delete( + self, + evaluation_rule_id: str, + *, + request_options: typing.Optional[RequestOptions] = None, + ) -> DeletedEvaluationRule: + """ + Delete an evaluation rule. + + This removes the live-ingestion rule only. It does not delete associated evaluators or scores already produced by them. + + Legacy trace and dataset rules can also be deleted. Their evaluators and previously produced scores are preserved. + + Parameters + ---------- + evaluation_rule_id : str + Stable evaluation-rule identifier returned by the evaluation-rule endpoints. + + request_options : typing.Optional[RequestOptions] + Request-specific configuration. + + Returns + ------- + DeletedEvaluationRule + + Examples + -------- + import asyncio + + from langfuse import AsyncLangfuseAPI + + client = AsyncLangfuseAPI( + x_langfuse_sdk_name="YOUR_X_LANGFUSE_SDK_NAME", + x_langfuse_sdk_version="YOUR_X_LANGFUSE_SDK_VERSION", + x_langfuse_public_key="YOUR_X_LANGFUSE_PUBLIC_KEY", + username="YOUR_USERNAME", + password="YOUR_PASSWORD", + base_url="https://yourhost.com/path/to/api", + ) + + + async def main() -> None: + await client.evaluation_rules.delete( + evaluation_rule_id="evaluationRuleId", + ) + + + asyncio.run(main()) + """ + _response = await self._raw_client.delete( + evaluation_rule_id, request_options=request_options + ) + return _response.data diff --git a/langfuse/api/evaluation_rules/raw_client.py b/langfuse/api/evaluation_rules/raw_client.py new file mode 100644 index 000000000..059b22c07 --- /dev/null +++ b/langfuse/api/evaluation_rules/raw_client.py @@ -0,0 +1,2038 @@ +# This file was auto-generated by Fern from our API Definition. + +import typing +from json.decoder import JSONDecodeError + +from ..commons.errors.access_denied_error import AccessDeniedError +from ..commons.errors.error import Error +from ..commons.errors.method_not_allowed_error import MethodNotAllowedError +from ..commons.errors.not_found_error import NotFoundError +from ..commons.errors.unauthorized_error import UnauthorizedError +from ..core.api_error import ApiError +from ..core.client_wrapper import AsyncClientWrapper, SyncClientWrapper +from ..core.http_response import AsyncHttpResponse, HttpResponse +from ..core.jsonable_encoder import jsonable_encoder +from ..core.pydantic_utilities import parse_obj_as +from ..core.request_options import RequestOptions +from ..core.serialization import convert_and_respect_annotation_metadata +from ..evaluation_commons.types.evaluation_rule_filter import EvaluationRuleFilter +from ..evaluation_errors.errors.bad_request_error import BadRequestError +from ..evaluation_errors.errors.conflict_error import ConflictError +from ..evaluation_errors.errors.evaluation_access_denied_error import ( + EvaluationAccessDeniedError, +) +from ..evaluation_errors.errors.evaluation_method_not_allowed_error import ( + EvaluationMethodNotAllowedError, +) +from ..evaluation_errors.errors.evaluation_not_found_error import ( + EvaluationNotFoundError, +) +from ..evaluation_errors.errors.evaluation_unauthorized_error import ( + EvaluationUnauthorizedError, +) +from ..evaluation_errors.errors.internal_server_error import InternalServerError +from ..evaluation_errors.errors.too_many_requests_error import TooManyRequestsError +from ..evaluation_errors.types.public_api_error import PublicApiError +from .types.deleted_evaluation_rule import DeletedEvaluationRule +from .types.evaluation_rule import EvaluationRule +from .types.evaluation_rule_evaluator_assignment_input import ( + EvaluationRuleEvaluatorAssignmentInput, +) +from .types.evaluation_rules_page import EvaluationRulesPage + +# this is used as the default value for optional parameters +OMIT = typing.cast(typing.Any, ...) + + +class RawEvaluationRulesClient: + def __init__(self, *, client_wrapper: SyncClientWrapper): + self._client_wrapper = client_wrapper + + def create( + self, + *, + name: str, + enabled: bool, + evaluator_assignments: typing.Sequence[EvaluationRuleEvaluatorAssignmentInput], + sampling: typing.Optional[float] = OMIT, + filter: typing.Optional[typing.Sequence[EvaluationRuleFilter]] = OMIT, + request_options: typing.Optional[RequestOptions] = None, + ) -> HttpResponse[EvaluationRule]: + """ + Create an evaluation rule using stable evaluator identifiers. + + An evaluation rule defines **which** incoming observations should be evaluated and how prompt variables should be populated. + + Key behavior: + - rule names are not identifiers and do not need to be unique + - rules always use the latest version of each associated evaluator + - an enabled rule requires at least one evaluator assignment + - omit `sampling` to evaluate every matching observation + - omit `filter` to match every incoming observation + - `datasetId` is the public filter name for selecting experiment datasets + + Parameters + ---------- + name : str + Human-readable rule name. Names are not identifiers and do not need to be unique. + + enabled : bool + Whether live execution should start immediately. Enabled rules require at least one evaluator assignment. + + evaluator_assignments : typing.Sequence[EvaluationRuleEvaluatorAssignmentInput] + Evaluators to attach to this rule. Disabled rules may use an empty list as a draft. + + sampling : typing.Optional[float] + Fraction of matching observations to evaluate. Omit this field to use the default of `1`, which evaluates every match. `null` is not accepted. + + filter : typing.Optional[typing.Sequence[EvaluationRuleFilter]] + Conditions used to select observations. Defaults to an empty list, which matches every incoming observation. + + request_options : typing.Optional[RequestOptions] + Request-specific configuration. + + Returns + ------- + HttpResponse[EvaluationRule] + """ + _response = self._client_wrapper.httpx_client.request( + "api/public/v2/evaluation-rules", + method="POST", + json={ + "name": name, + "enabled": enabled, + "sampling": sampling, + "filter": convert_and_respect_annotation_metadata( + object_=filter, + annotation=typing.Sequence[EvaluationRuleFilter], + direction="write", + ), + "evaluatorAssignments": convert_and_respect_annotation_metadata( + object_=evaluator_assignments, + annotation=typing.Sequence[EvaluationRuleEvaluatorAssignmentInput], + direction="write", + ), + }, + request_options=request_options, + omit=OMIT, + ) + try: + if 200 <= _response.status_code < 300: + _data = typing.cast( + EvaluationRule, + parse_obj_as( + type_=EvaluationRule, # type: ignore + object_=_response.json(), + ), + ) + return HttpResponse(response=_response, data=_data) + if _response.status_code == 400: + raise BadRequestError( + headers=dict(_response.headers), + body=typing.cast( + PublicApiError, + parse_obj_as( + type_=PublicApiError, # type: ignore + object_=_response.json(), + ), + ), + ) + if _response.status_code == 401: + raise EvaluationUnauthorizedError( + headers=dict(_response.headers), + body=typing.cast( + PublicApiError, + parse_obj_as( + type_=PublicApiError, # type: ignore + object_=_response.json(), + ), + ), + ) + if _response.status_code == 403: + raise EvaluationAccessDeniedError( + headers=dict(_response.headers), + body=typing.cast( + PublicApiError, + parse_obj_as( + type_=PublicApiError, # type: ignore + object_=_response.json(), + ), + ), + ) + if _response.status_code == 404: + raise EvaluationNotFoundError( + headers=dict(_response.headers), + body=typing.cast( + PublicApiError, + parse_obj_as( + type_=PublicApiError, # type: ignore + object_=_response.json(), + ), + ), + ) + if _response.status_code == 405: + raise EvaluationMethodNotAllowedError( + headers=dict(_response.headers), + body=typing.cast( + PublicApiError, + parse_obj_as( + type_=PublicApiError, # type: ignore + object_=_response.json(), + ), + ), + ) + if _response.status_code == 409: + raise ConflictError( + headers=dict(_response.headers), + body=typing.cast( + PublicApiError, + parse_obj_as( + type_=PublicApiError, # type: ignore + object_=_response.json(), + ), + ), + ) + if _response.status_code == 429: + raise TooManyRequestsError( + headers=dict(_response.headers), + body=typing.cast( + PublicApiError, + parse_obj_as( + type_=PublicApiError, # type: ignore + object_=_response.json(), + ), + ), + ) + if _response.status_code == 500: + raise InternalServerError( + headers=dict(_response.headers), + body=typing.cast( + PublicApiError, + parse_obj_as( + type_=PublicApiError, # type: ignore + object_=_response.json(), + ), + ), + ) + if _response.status_code == 400: + raise Error( + headers=dict(_response.headers), + body=typing.cast( + typing.Any, + parse_obj_as( + type_=typing.Any, # type: ignore + object_=_response.json(), + ), + ), + ) + if _response.status_code == 401: + raise UnauthorizedError( + headers=dict(_response.headers), + body=typing.cast( + typing.Any, + parse_obj_as( + type_=typing.Any, # type: ignore + object_=_response.json(), + ), + ), + ) + if _response.status_code == 403: + raise AccessDeniedError( + headers=dict(_response.headers), + body=typing.cast( + typing.Any, + parse_obj_as( + type_=typing.Any, # type: ignore + object_=_response.json(), + ), + ), + ) + if _response.status_code == 405: + raise MethodNotAllowedError( + headers=dict(_response.headers), + body=typing.cast( + typing.Any, + parse_obj_as( + type_=typing.Any, # type: ignore + object_=_response.json(), + ), + ), + ) + if _response.status_code == 404: + raise NotFoundError( + headers=dict(_response.headers), + body=typing.cast( + typing.Any, + parse_obj_as( + type_=typing.Any, # type: ignore + object_=_response.json(), + ), + ), + ) + _response_json = _response.json() + except JSONDecodeError: + raise ApiError( + status_code=_response.status_code, + headers=dict(_response.headers), + body=_response.text, + ) + raise ApiError( + status_code=_response.status_code, + headers=dict(_response.headers), + body=_response_json, + ) + + def list( + self, + *, + limit: typing.Optional[int] = None, + cursor: typing.Optional[str] = None, + request_options: typing.Optional[RequestOptions] = None, + ) -> HttpResponse[EvaluationRulesPage]: + """ + List evaluation rules in newest-first creation order. + + This includes legacy trace and dataset rules so they can be inspected and migrated. Treat the cursor as opaque and return it unchanged. + + Parameters + ---------- + limit : typing.Optional[int] + Maximum number of items to return. Defaults to `50` and cannot exceed `100`. + + cursor : typing.Optional[str] + Opaque cursor returned by the previous page. + + request_options : typing.Optional[RequestOptions] + Request-specific configuration. + + Returns + ------- + HttpResponse[EvaluationRulesPage] + """ + _response = self._client_wrapper.httpx_client.request( + "api/public/v2/evaluation-rules", + method="GET", + params={ + "limit": limit, + "cursor": cursor, + }, + request_options=request_options, + ) + try: + if 200 <= _response.status_code < 300: + _data = typing.cast( + EvaluationRulesPage, + parse_obj_as( + type_=EvaluationRulesPage, # type: ignore + object_=_response.json(), + ), + ) + return HttpResponse(response=_response, data=_data) + if _response.status_code == 400: + raise BadRequestError( + headers=dict(_response.headers), + body=typing.cast( + PublicApiError, + parse_obj_as( + type_=PublicApiError, # type: ignore + object_=_response.json(), + ), + ), + ) + if _response.status_code == 401: + raise EvaluationUnauthorizedError( + headers=dict(_response.headers), + body=typing.cast( + PublicApiError, + parse_obj_as( + type_=PublicApiError, # type: ignore + object_=_response.json(), + ), + ), + ) + if _response.status_code == 403: + raise EvaluationAccessDeniedError( + headers=dict(_response.headers), + body=typing.cast( + PublicApiError, + parse_obj_as( + type_=PublicApiError, # type: ignore + object_=_response.json(), + ), + ), + ) + if _response.status_code == 405: + raise EvaluationMethodNotAllowedError( + headers=dict(_response.headers), + body=typing.cast( + PublicApiError, + parse_obj_as( + type_=PublicApiError, # type: ignore + object_=_response.json(), + ), + ), + ) + if _response.status_code == 429: + raise TooManyRequestsError( + headers=dict(_response.headers), + body=typing.cast( + PublicApiError, + parse_obj_as( + type_=PublicApiError, # type: ignore + object_=_response.json(), + ), + ), + ) + if _response.status_code == 500: + raise InternalServerError( + headers=dict(_response.headers), + body=typing.cast( + PublicApiError, + parse_obj_as( + type_=PublicApiError, # type: ignore + object_=_response.json(), + ), + ), + ) + if _response.status_code == 400: + raise Error( + headers=dict(_response.headers), + body=typing.cast( + typing.Any, + parse_obj_as( + type_=typing.Any, # type: ignore + object_=_response.json(), + ), + ), + ) + if _response.status_code == 401: + raise UnauthorizedError( + headers=dict(_response.headers), + body=typing.cast( + typing.Any, + parse_obj_as( + type_=typing.Any, # type: ignore + object_=_response.json(), + ), + ), + ) + if _response.status_code == 403: + raise AccessDeniedError( + headers=dict(_response.headers), + body=typing.cast( + typing.Any, + parse_obj_as( + type_=typing.Any, # type: ignore + object_=_response.json(), + ), + ), + ) + if _response.status_code == 405: + raise MethodNotAllowedError( + headers=dict(_response.headers), + body=typing.cast( + typing.Any, + parse_obj_as( + type_=typing.Any, # type: ignore + object_=_response.json(), + ), + ), + ) + if _response.status_code == 404: + raise NotFoundError( + headers=dict(_response.headers), + body=typing.cast( + typing.Any, + parse_obj_as( + type_=typing.Any, # type: ignore + object_=_response.json(), + ), + ), + ) + _response_json = _response.json() + except JSONDecodeError: + raise ApiError( + status_code=_response.status_code, + headers=dict(_response.headers), + body=_response.text, + ) + raise ApiError( + status_code=_response.status_code, + headers=dict(_response.headers), + body=_response_json, + ) + + def get( + self, + evaluation_rule_id: str, + *, + request_options: typing.Optional[RequestOptions] = None, + ) -> HttpResponse[EvaluationRule]: + """ + Get one evaluation rule, including a legacy trace or dataset rule, by its stable identifier. + + Parameters + ---------- + evaluation_rule_id : str + Stable evaluation-rule identifier returned by the evaluation-rule endpoints. + + request_options : typing.Optional[RequestOptions] + Request-specific configuration. + + Returns + ------- + HttpResponse[EvaluationRule] + """ + _response = self._client_wrapper.httpx_client.request( + f"api/public/v2/evaluation-rules/{jsonable_encoder(evaluation_rule_id)}", + method="GET", + request_options=request_options, + ) + try: + if 200 <= _response.status_code < 300: + _data = typing.cast( + EvaluationRule, + parse_obj_as( + type_=EvaluationRule, # type: ignore + object_=_response.json(), + ), + ) + return HttpResponse(response=_response, data=_data) + if _response.status_code == 401: + raise EvaluationUnauthorizedError( + headers=dict(_response.headers), + body=typing.cast( + PublicApiError, + parse_obj_as( + type_=PublicApiError, # type: ignore + object_=_response.json(), + ), + ), + ) + if _response.status_code == 403: + raise EvaluationAccessDeniedError( + headers=dict(_response.headers), + body=typing.cast( + PublicApiError, + parse_obj_as( + type_=PublicApiError, # type: ignore + object_=_response.json(), + ), + ), + ) + if _response.status_code == 404: + raise EvaluationNotFoundError( + headers=dict(_response.headers), + body=typing.cast( + PublicApiError, + parse_obj_as( + type_=PublicApiError, # type: ignore + object_=_response.json(), + ), + ), + ) + if _response.status_code == 405: + raise EvaluationMethodNotAllowedError( + headers=dict(_response.headers), + body=typing.cast( + PublicApiError, + parse_obj_as( + type_=PublicApiError, # type: ignore + object_=_response.json(), + ), + ), + ) + if _response.status_code == 429: + raise TooManyRequestsError( + headers=dict(_response.headers), + body=typing.cast( + PublicApiError, + parse_obj_as( + type_=PublicApiError, # type: ignore + object_=_response.json(), + ), + ), + ) + if _response.status_code == 500: + raise InternalServerError( + headers=dict(_response.headers), + body=typing.cast( + PublicApiError, + parse_obj_as( + type_=PublicApiError, # type: ignore + object_=_response.json(), + ), + ), + ) + if _response.status_code == 400: + raise Error( + headers=dict(_response.headers), + body=typing.cast( + typing.Any, + parse_obj_as( + type_=typing.Any, # type: ignore + object_=_response.json(), + ), + ), + ) + if _response.status_code == 401: + raise UnauthorizedError( + headers=dict(_response.headers), + body=typing.cast( + typing.Any, + parse_obj_as( + type_=typing.Any, # type: ignore + object_=_response.json(), + ), + ), + ) + if _response.status_code == 403: + raise AccessDeniedError( + headers=dict(_response.headers), + body=typing.cast( + typing.Any, + parse_obj_as( + type_=typing.Any, # type: ignore + object_=_response.json(), + ), + ), + ) + if _response.status_code == 405: + raise MethodNotAllowedError( + headers=dict(_response.headers), + body=typing.cast( + typing.Any, + parse_obj_as( + type_=typing.Any, # type: ignore + object_=_response.json(), + ), + ), + ) + if _response.status_code == 404: + raise NotFoundError( + headers=dict(_response.headers), + body=typing.cast( + typing.Any, + parse_obj_as( + type_=typing.Any, # type: ignore + object_=_response.json(), + ), + ), + ) + _response_json = _response.json() + except JSONDecodeError: + raise ApiError( + status_code=_response.status_code, + headers=dict(_response.headers), + body=_response.text, + ) + raise ApiError( + status_code=_response.status_code, + headers=dict(_response.headers), + body=_response_json, + ) + + def update( + self, + evaluation_rule_id: str, + *, + name: typing.Optional[str] = OMIT, + enabled: typing.Optional[bool] = OMIT, + sampling: typing.Optional[float] = OMIT, + filter: typing.Optional[typing.Sequence[EvaluationRuleFilter]] = OMIT, + evaluator_assignments: typing.Optional[ + typing.Sequence[EvaluationRuleEvaluatorAssignmentInput] + ] = OMIT, + request_options: typing.Optional[RequestOptions] = None, + ) -> HttpResponse[EvaluationRule]: + """ + Update an evaluation rule by its stable identifier. + + Provide only the fields to change. Providing `evaluatorAssignments` replaces the complete assignment list. Replacing the list with an empty array disables the rule. Setting `enabled=true` is rejected when the resulting assignment list is empty, including when both fields are sent in the same request. + + Legacy trace and dataset rules follow the existing lifecycle restrictions: they can be deactivated with `enabled=false`, but their name, filters, sampling, and evaluator assignments cannot be changed. + + Parameters + ---------- + evaluation_rule_id : str + Stable evaluation-rule identifier returned by the evaluation-rule endpoints. + + name : typing.Optional[str] + New human-readable rule name. + + enabled : typing.Optional[bool] + New desired live-execution state. + + sampling : typing.Optional[float] + New fraction of matching observations to evaluate. Omit to keep the current value. + + filter : typing.Optional[typing.Sequence[EvaluationRuleFilter]] + Complete replacement filter list. An empty list matches every incoming observation. + + evaluator_assignments : typing.Optional[typing.Sequence[EvaluationRuleEvaluatorAssignmentInput]] + Complete replacement assignment list. An empty list disables the rule. + + request_options : typing.Optional[RequestOptions] + Request-specific configuration. + + Returns + ------- + HttpResponse[EvaluationRule] + """ + _response = self._client_wrapper.httpx_client.request( + f"api/public/v2/evaluation-rules/{jsonable_encoder(evaluation_rule_id)}", + method="PATCH", + json={ + "name": name, + "enabled": enabled, + "sampling": sampling, + "filter": convert_and_respect_annotation_metadata( + object_=filter, + annotation=typing.Sequence[EvaluationRuleFilter], + direction="write", + ), + "evaluatorAssignments": convert_and_respect_annotation_metadata( + object_=evaluator_assignments, + annotation=typing.Sequence[EvaluationRuleEvaluatorAssignmentInput], + direction="write", + ), + }, + request_options=request_options, + omit=OMIT, + ) + try: + if 200 <= _response.status_code < 300: + _data = typing.cast( + EvaluationRule, + parse_obj_as( + type_=EvaluationRule, # type: ignore + object_=_response.json(), + ), + ) + return HttpResponse(response=_response, data=_data) + if _response.status_code == 400: + raise BadRequestError( + headers=dict(_response.headers), + body=typing.cast( + PublicApiError, + parse_obj_as( + type_=PublicApiError, # type: ignore + object_=_response.json(), + ), + ), + ) + if _response.status_code == 401: + raise EvaluationUnauthorizedError( + headers=dict(_response.headers), + body=typing.cast( + PublicApiError, + parse_obj_as( + type_=PublicApiError, # type: ignore + object_=_response.json(), + ), + ), + ) + if _response.status_code == 403: + raise EvaluationAccessDeniedError( + headers=dict(_response.headers), + body=typing.cast( + PublicApiError, + parse_obj_as( + type_=PublicApiError, # type: ignore + object_=_response.json(), + ), + ), + ) + if _response.status_code == 404: + raise EvaluationNotFoundError( + headers=dict(_response.headers), + body=typing.cast( + PublicApiError, + parse_obj_as( + type_=PublicApiError, # type: ignore + object_=_response.json(), + ), + ), + ) + if _response.status_code == 405: + raise EvaluationMethodNotAllowedError( + headers=dict(_response.headers), + body=typing.cast( + PublicApiError, + parse_obj_as( + type_=PublicApiError, # type: ignore + object_=_response.json(), + ), + ), + ) + if _response.status_code == 409: + raise ConflictError( + headers=dict(_response.headers), + body=typing.cast( + PublicApiError, + parse_obj_as( + type_=PublicApiError, # type: ignore + object_=_response.json(), + ), + ), + ) + if _response.status_code == 429: + raise TooManyRequestsError( + headers=dict(_response.headers), + body=typing.cast( + PublicApiError, + parse_obj_as( + type_=PublicApiError, # type: ignore + object_=_response.json(), + ), + ), + ) + if _response.status_code == 500: + raise InternalServerError( + headers=dict(_response.headers), + body=typing.cast( + PublicApiError, + parse_obj_as( + type_=PublicApiError, # type: ignore + object_=_response.json(), + ), + ), + ) + if _response.status_code == 400: + raise Error( + headers=dict(_response.headers), + body=typing.cast( + typing.Any, + parse_obj_as( + type_=typing.Any, # type: ignore + object_=_response.json(), + ), + ), + ) + if _response.status_code == 401: + raise UnauthorizedError( + headers=dict(_response.headers), + body=typing.cast( + typing.Any, + parse_obj_as( + type_=typing.Any, # type: ignore + object_=_response.json(), + ), + ), + ) + if _response.status_code == 403: + raise AccessDeniedError( + headers=dict(_response.headers), + body=typing.cast( + typing.Any, + parse_obj_as( + type_=typing.Any, # type: ignore + object_=_response.json(), + ), + ), + ) + if _response.status_code == 405: + raise MethodNotAllowedError( + headers=dict(_response.headers), + body=typing.cast( + typing.Any, + parse_obj_as( + type_=typing.Any, # type: ignore + object_=_response.json(), + ), + ), + ) + if _response.status_code == 404: + raise NotFoundError( + headers=dict(_response.headers), + body=typing.cast( + typing.Any, + parse_obj_as( + type_=typing.Any, # type: ignore + object_=_response.json(), + ), + ), + ) + _response_json = _response.json() + except JSONDecodeError: + raise ApiError( + status_code=_response.status_code, + headers=dict(_response.headers), + body=_response.text, + ) + raise ApiError( + status_code=_response.status_code, + headers=dict(_response.headers), + body=_response_json, + ) + + def delete( + self, + evaluation_rule_id: str, + *, + request_options: typing.Optional[RequestOptions] = None, + ) -> HttpResponse[DeletedEvaluationRule]: + """ + Delete an evaluation rule. + + This removes the live-ingestion rule only. It does not delete associated evaluators or scores already produced by them. + + Legacy trace and dataset rules can also be deleted. Their evaluators and previously produced scores are preserved. + + Parameters + ---------- + evaluation_rule_id : str + Stable evaluation-rule identifier returned by the evaluation-rule endpoints. + + request_options : typing.Optional[RequestOptions] + Request-specific configuration. + + Returns + ------- + HttpResponse[DeletedEvaluationRule] + """ + _response = self._client_wrapper.httpx_client.request( + f"api/public/v2/evaluation-rules/{jsonable_encoder(evaluation_rule_id)}", + method="DELETE", + request_options=request_options, + ) + try: + if 200 <= _response.status_code < 300: + _data = typing.cast( + DeletedEvaluationRule, + parse_obj_as( + type_=DeletedEvaluationRule, # type: ignore + object_=_response.json(), + ), + ) + return HttpResponse(response=_response, data=_data) + if _response.status_code == 401: + raise EvaluationUnauthorizedError( + headers=dict(_response.headers), + body=typing.cast( + PublicApiError, + parse_obj_as( + type_=PublicApiError, # type: ignore + object_=_response.json(), + ), + ), + ) + if _response.status_code == 403: + raise EvaluationAccessDeniedError( + headers=dict(_response.headers), + body=typing.cast( + PublicApiError, + parse_obj_as( + type_=PublicApiError, # type: ignore + object_=_response.json(), + ), + ), + ) + if _response.status_code == 404: + raise EvaluationNotFoundError( + headers=dict(_response.headers), + body=typing.cast( + PublicApiError, + parse_obj_as( + type_=PublicApiError, # type: ignore + object_=_response.json(), + ), + ), + ) + if _response.status_code == 405: + raise EvaluationMethodNotAllowedError( + headers=dict(_response.headers), + body=typing.cast( + PublicApiError, + parse_obj_as( + type_=PublicApiError, # type: ignore + object_=_response.json(), + ), + ), + ) + if _response.status_code == 429: + raise TooManyRequestsError( + headers=dict(_response.headers), + body=typing.cast( + PublicApiError, + parse_obj_as( + type_=PublicApiError, # type: ignore + object_=_response.json(), + ), + ), + ) + if _response.status_code == 500: + raise InternalServerError( + headers=dict(_response.headers), + body=typing.cast( + PublicApiError, + parse_obj_as( + type_=PublicApiError, # type: ignore + object_=_response.json(), + ), + ), + ) + if _response.status_code == 400: + raise Error( + headers=dict(_response.headers), + body=typing.cast( + typing.Any, + parse_obj_as( + type_=typing.Any, # type: ignore + object_=_response.json(), + ), + ), + ) + if _response.status_code == 401: + raise UnauthorizedError( + headers=dict(_response.headers), + body=typing.cast( + typing.Any, + parse_obj_as( + type_=typing.Any, # type: ignore + object_=_response.json(), + ), + ), + ) + if _response.status_code == 403: + raise AccessDeniedError( + headers=dict(_response.headers), + body=typing.cast( + typing.Any, + parse_obj_as( + type_=typing.Any, # type: ignore + object_=_response.json(), + ), + ), + ) + if _response.status_code == 405: + raise MethodNotAllowedError( + headers=dict(_response.headers), + body=typing.cast( + typing.Any, + parse_obj_as( + type_=typing.Any, # type: ignore + object_=_response.json(), + ), + ), + ) + if _response.status_code == 404: + raise NotFoundError( + headers=dict(_response.headers), + body=typing.cast( + typing.Any, + parse_obj_as( + type_=typing.Any, # type: ignore + object_=_response.json(), + ), + ), + ) + _response_json = _response.json() + except JSONDecodeError: + raise ApiError( + status_code=_response.status_code, + headers=dict(_response.headers), + body=_response.text, + ) + raise ApiError( + status_code=_response.status_code, + headers=dict(_response.headers), + body=_response_json, + ) + + +class AsyncRawEvaluationRulesClient: + def __init__(self, *, client_wrapper: AsyncClientWrapper): + self._client_wrapper = client_wrapper + + async def create( + self, + *, + name: str, + enabled: bool, + evaluator_assignments: typing.Sequence[EvaluationRuleEvaluatorAssignmentInput], + sampling: typing.Optional[float] = OMIT, + filter: typing.Optional[typing.Sequence[EvaluationRuleFilter]] = OMIT, + request_options: typing.Optional[RequestOptions] = None, + ) -> AsyncHttpResponse[EvaluationRule]: + """ + Create an evaluation rule using stable evaluator identifiers. + + An evaluation rule defines **which** incoming observations should be evaluated and how prompt variables should be populated. + + Key behavior: + - rule names are not identifiers and do not need to be unique + - rules always use the latest version of each associated evaluator + - an enabled rule requires at least one evaluator assignment + - omit `sampling` to evaluate every matching observation + - omit `filter` to match every incoming observation + - `datasetId` is the public filter name for selecting experiment datasets + + Parameters + ---------- + name : str + Human-readable rule name. Names are not identifiers and do not need to be unique. + + enabled : bool + Whether live execution should start immediately. Enabled rules require at least one evaluator assignment. + + evaluator_assignments : typing.Sequence[EvaluationRuleEvaluatorAssignmentInput] + Evaluators to attach to this rule. Disabled rules may use an empty list as a draft. + + sampling : typing.Optional[float] + Fraction of matching observations to evaluate. Omit this field to use the default of `1`, which evaluates every match. `null` is not accepted. + + filter : typing.Optional[typing.Sequence[EvaluationRuleFilter]] + Conditions used to select observations. Defaults to an empty list, which matches every incoming observation. + + request_options : typing.Optional[RequestOptions] + Request-specific configuration. + + Returns + ------- + AsyncHttpResponse[EvaluationRule] + """ + _response = await self._client_wrapper.httpx_client.request( + "api/public/v2/evaluation-rules", + method="POST", + json={ + "name": name, + "enabled": enabled, + "sampling": sampling, + "filter": convert_and_respect_annotation_metadata( + object_=filter, + annotation=typing.Sequence[EvaluationRuleFilter], + direction="write", + ), + "evaluatorAssignments": convert_and_respect_annotation_metadata( + object_=evaluator_assignments, + annotation=typing.Sequence[EvaluationRuleEvaluatorAssignmentInput], + direction="write", + ), + }, + request_options=request_options, + omit=OMIT, + ) + try: + if 200 <= _response.status_code < 300: + _data = typing.cast( + EvaluationRule, + parse_obj_as( + type_=EvaluationRule, # type: ignore + object_=_response.json(), + ), + ) + return AsyncHttpResponse(response=_response, data=_data) + if _response.status_code == 400: + raise BadRequestError( + headers=dict(_response.headers), + body=typing.cast( + PublicApiError, + parse_obj_as( + type_=PublicApiError, # type: ignore + object_=_response.json(), + ), + ), + ) + if _response.status_code == 401: + raise EvaluationUnauthorizedError( + headers=dict(_response.headers), + body=typing.cast( + PublicApiError, + parse_obj_as( + type_=PublicApiError, # type: ignore + object_=_response.json(), + ), + ), + ) + if _response.status_code == 403: + raise EvaluationAccessDeniedError( + headers=dict(_response.headers), + body=typing.cast( + PublicApiError, + parse_obj_as( + type_=PublicApiError, # type: ignore + object_=_response.json(), + ), + ), + ) + if _response.status_code == 404: + raise EvaluationNotFoundError( + headers=dict(_response.headers), + body=typing.cast( + PublicApiError, + parse_obj_as( + type_=PublicApiError, # type: ignore + object_=_response.json(), + ), + ), + ) + if _response.status_code == 405: + raise EvaluationMethodNotAllowedError( + headers=dict(_response.headers), + body=typing.cast( + PublicApiError, + parse_obj_as( + type_=PublicApiError, # type: ignore + object_=_response.json(), + ), + ), + ) + if _response.status_code == 409: + raise ConflictError( + headers=dict(_response.headers), + body=typing.cast( + PublicApiError, + parse_obj_as( + type_=PublicApiError, # type: ignore + object_=_response.json(), + ), + ), + ) + if _response.status_code == 429: + raise TooManyRequestsError( + headers=dict(_response.headers), + body=typing.cast( + PublicApiError, + parse_obj_as( + type_=PublicApiError, # type: ignore + object_=_response.json(), + ), + ), + ) + if _response.status_code == 500: + raise InternalServerError( + headers=dict(_response.headers), + body=typing.cast( + PublicApiError, + parse_obj_as( + type_=PublicApiError, # type: ignore + object_=_response.json(), + ), + ), + ) + if _response.status_code == 400: + raise Error( + headers=dict(_response.headers), + body=typing.cast( + typing.Any, + parse_obj_as( + type_=typing.Any, # type: ignore + object_=_response.json(), + ), + ), + ) + if _response.status_code == 401: + raise UnauthorizedError( + headers=dict(_response.headers), + body=typing.cast( + typing.Any, + parse_obj_as( + type_=typing.Any, # type: ignore + object_=_response.json(), + ), + ), + ) + if _response.status_code == 403: + raise AccessDeniedError( + headers=dict(_response.headers), + body=typing.cast( + typing.Any, + parse_obj_as( + type_=typing.Any, # type: ignore + object_=_response.json(), + ), + ), + ) + if _response.status_code == 405: + raise MethodNotAllowedError( + headers=dict(_response.headers), + body=typing.cast( + typing.Any, + parse_obj_as( + type_=typing.Any, # type: ignore + object_=_response.json(), + ), + ), + ) + if _response.status_code == 404: + raise NotFoundError( + headers=dict(_response.headers), + body=typing.cast( + typing.Any, + parse_obj_as( + type_=typing.Any, # type: ignore + object_=_response.json(), + ), + ), + ) + _response_json = _response.json() + except JSONDecodeError: + raise ApiError( + status_code=_response.status_code, + headers=dict(_response.headers), + body=_response.text, + ) + raise ApiError( + status_code=_response.status_code, + headers=dict(_response.headers), + body=_response_json, + ) + + async def list( + self, + *, + limit: typing.Optional[int] = None, + cursor: typing.Optional[str] = None, + request_options: typing.Optional[RequestOptions] = None, + ) -> AsyncHttpResponse[EvaluationRulesPage]: + """ + List evaluation rules in newest-first creation order. + + This includes legacy trace and dataset rules so they can be inspected and migrated. Treat the cursor as opaque and return it unchanged. + + Parameters + ---------- + limit : typing.Optional[int] + Maximum number of items to return. Defaults to `50` and cannot exceed `100`. + + cursor : typing.Optional[str] + Opaque cursor returned by the previous page. + + request_options : typing.Optional[RequestOptions] + Request-specific configuration. + + Returns + ------- + AsyncHttpResponse[EvaluationRulesPage] + """ + _response = await self._client_wrapper.httpx_client.request( + "api/public/v2/evaluation-rules", + method="GET", + params={ + "limit": limit, + "cursor": cursor, + }, + request_options=request_options, + ) + try: + if 200 <= _response.status_code < 300: + _data = typing.cast( + EvaluationRulesPage, + parse_obj_as( + type_=EvaluationRulesPage, # type: ignore + object_=_response.json(), + ), + ) + return AsyncHttpResponse(response=_response, data=_data) + if _response.status_code == 400: + raise BadRequestError( + headers=dict(_response.headers), + body=typing.cast( + PublicApiError, + parse_obj_as( + type_=PublicApiError, # type: ignore + object_=_response.json(), + ), + ), + ) + if _response.status_code == 401: + raise EvaluationUnauthorizedError( + headers=dict(_response.headers), + body=typing.cast( + PublicApiError, + parse_obj_as( + type_=PublicApiError, # type: ignore + object_=_response.json(), + ), + ), + ) + if _response.status_code == 403: + raise EvaluationAccessDeniedError( + headers=dict(_response.headers), + body=typing.cast( + PublicApiError, + parse_obj_as( + type_=PublicApiError, # type: ignore + object_=_response.json(), + ), + ), + ) + if _response.status_code == 405: + raise EvaluationMethodNotAllowedError( + headers=dict(_response.headers), + body=typing.cast( + PublicApiError, + parse_obj_as( + type_=PublicApiError, # type: ignore + object_=_response.json(), + ), + ), + ) + if _response.status_code == 429: + raise TooManyRequestsError( + headers=dict(_response.headers), + body=typing.cast( + PublicApiError, + parse_obj_as( + type_=PublicApiError, # type: ignore + object_=_response.json(), + ), + ), + ) + if _response.status_code == 500: + raise InternalServerError( + headers=dict(_response.headers), + body=typing.cast( + PublicApiError, + parse_obj_as( + type_=PublicApiError, # type: ignore + object_=_response.json(), + ), + ), + ) + if _response.status_code == 400: + raise Error( + headers=dict(_response.headers), + body=typing.cast( + typing.Any, + parse_obj_as( + type_=typing.Any, # type: ignore + object_=_response.json(), + ), + ), + ) + if _response.status_code == 401: + raise UnauthorizedError( + headers=dict(_response.headers), + body=typing.cast( + typing.Any, + parse_obj_as( + type_=typing.Any, # type: ignore + object_=_response.json(), + ), + ), + ) + if _response.status_code == 403: + raise AccessDeniedError( + headers=dict(_response.headers), + body=typing.cast( + typing.Any, + parse_obj_as( + type_=typing.Any, # type: ignore + object_=_response.json(), + ), + ), + ) + if _response.status_code == 405: + raise MethodNotAllowedError( + headers=dict(_response.headers), + body=typing.cast( + typing.Any, + parse_obj_as( + type_=typing.Any, # type: ignore + object_=_response.json(), + ), + ), + ) + if _response.status_code == 404: + raise NotFoundError( + headers=dict(_response.headers), + body=typing.cast( + typing.Any, + parse_obj_as( + type_=typing.Any, # type: ignore + object_=_response.json(), + ), + ), + ) + _response_json = _response.json() + except JSONDecodeError: + raise ApiError( + status_code=_response.status_code, + headers=dict(_response.headers), + body=_response.text, + ) + raise ApiError( + status_code=_response.status_code, + headers=dict(_response.headers), + body=_response_json, + ) + + async def get( + self, + evaluation_rule_id: str, + *, + request_options: typing.Optional[RequestOptions] = None, + ) -> AsyncHttpResponse[EvaluationRule]: + """ + Get one evaluation rule, including a legacy trace or dataset rule, by its stable identifier. + + Parameters + ---------- + evaluation_rule_id : str + Stable evaluation-rule identifier returned by the evaluation-rule endpoints. + + request_options : typing.Optional[RequestOptions] + Request-specific configuration. + + Returns + ------- + AsyncHttpResponse[EvaluationRule] + """ + _response = await self._client_wrapper.httpx_client.request( + f"api/public/v2/evaluation-rules/{jsonable_encoder(evaluation_rule_id)}", + method="GET", + request_options=request_options, + ) + try: + if 200 <= _response.status_code < 300: + _data = typing.cast( + EvaluationRule, + parse_obj_as( + type_=EvaluationRule, # type: ignore + object_=_response.json(), + ), + ) + return AsyncHttpResponse(response=_response, data=_data) + if _response.status_code == 401: + raise EvaluationUnauthorizedError( + headers=dict(_response.headers), + body=typing.cast( + PublicApiError, + parse_obj_as( + type_=PublicApiError, # type: ignore + object_=_response.json(), + ), + ), + ) + if _response.status_code == 403: + raise EvaluationAccessDeniedError( + headers=dict(_response.headers), + body=typing.cast( + PublicApiError, + parse_obj_as( + type_=PublicApiError, # type: ignore + object_=_response.json(), + ), + ), + ) + if _response.status_code == 404: + raise EvaluationNotFoundError( + headers=dict(_response.headers), + body=typing.cast( + PublicApiError, + parse_obj_as( + type_=PublicApiError, # type: ignore + object_=_response.json(), + ), + ), + ) + if _response.status_code == 405: + raise EvaluationMethodNotAllowedError( + headers=dict(_response.headers), + body=typing.cast( + PublicApiError, + parse_obj_as( + type_=PublicApiError, # type: ignore + object_=_response.json(), + ), + ), + ) + if _response.status_code == 429: + raise TooManyRequestsError( + headers=dict(_response.headers), + body=typing.cast( + PublicApiError, + parse_obj_as( + type_=PublicApiError, # type: ignore + object_=_response.json(), + ), + ), + ) + if _response.status_code == 500: + raise InternalServerError( + headers=dict(_response.headers), + body=typing.cast( + PublicApiError, + parse_obj_as( + type_=PublicApiError, # type: ignore + object_=_response.json(), + ), + ), + ) + if _response.status_code == 400: + raise Error( + headers=dict(_response.headers), + body=typing.cast( + typing.Any, + parse_obj_as( + type_=typing.Any, # type: ignore + object_=_response.json(), + ), + ), + ) + if _response.status_code == 401: + raise UnauthorizedError( + headers=dict(_response.headers), + body=typing.cast( + typing.Any, + parse_obj_as( + type_=typing.Any, # type: ignore + object_=_response.json(), + ), + ), + ) + if _response.status_code == 403: + raise AccessDeniedError( + headers=dict(_response.headers), + body=typing.cast( + typing.Any, + parse_obj_as( + type_=typing.Any, # type: ignore + object_=_response.json(), + ), + ), + ) + if _response.status_code == 405: + raise MethodNotAllowedError( + headers=dict(_response.headers), + body=typing.cast( + typing.Any, + parse_obj_as( + type_=typing.Any, # type: ignore + object_=_response.json(), + ), + ), + ) + if _response.status_code == 404: + raise NotFoundError( + headers=dict(_response.headers), + body=typing.cast( + typing.Any, + parse_obj_as( + type_=typing.Any, # type: ignore + object_=_response.json(), + ), + ), + ) + _response_json = _response.json() + except JSONDecodeError: + raise ApiError( + status_code=_response.status_code, + headers=dict(_response.headers), + body=_response.text, + ) + raise ApiError( + status_code=_response.status_code, + headers=dict(_response.headers), + body=_response_json, + ) + + async def update( + self, + evaluation_rule_id: str, + *, + name: typing.Optional[str] = OMIT, + enabled: typing.Optional[bool] = OMIT, + sampling: typing.Optional[float] = OMIT, + filter: typing.Optional[typing.Sequence[EvaluationRuleFilter]] = OMIT, + evaluator_assignments: typing.Optional[ + typing.Sequence[EvaluationRuleEvaluatorAssignmentInput] + ] = OMIT, + request_options: typing.Optional[RequestOptions] = None, + ) -> AsyncHttpResponse[EvaluationRule]: + """ + Update an evaluation rule by its stable identifier. + + Provide only the fields to change. Providing `evaluatorAssignments` replaces the complete assignment list. Replacing the list with an empty array disables the rule. Setting `enabled=true` is rejected when the resulting assignment list is empty, including when both fields are sent in the same request. + + Legacy trace and dataset rules follow the existing lifecycle restrictions: they can be deactivated with `enabled=false`, but their name, filters, sampling, and evaluator assignments cannot be changed. + + Parameters + ---------- + evaluation_rule_id : str + Stable evaluation-rule identifier returned by the evaluation-rule endpoints. + + name : typing.Optional[str] + New human-readable rule name. + + enabled : typing.Optional[bool] + New desired live-execution state. + + sampling : typing.Optional[float] + New fraction of matching observations to evaluate. Omit to keep the current value. + + filter : typing.Optional[typing.Sequence[EvaluationRuleFilter]] + Complete replacement filter list. An empty list matches every incoming observation. + + evaluator_assignments : typing.Optional[typing.Sequence[EvaluationRuleEvaluatorAssignmentInput]] + Complete replacement assignment list. An empty list disables the rule. + + request_options : typing.Optional[RequestOptions] + Request-specific configuration. + + Returns + ------- + AsyncHttpResponse[EvaluationRule] + """ + _response = await self._client_wrapper.httpx_client.request( + f"api/public/v2/evaluation-rules/{jsonable_encoder(evaluation_rule_id)}", + method="PATCH", + json={ + "name": name, + "enabled": enabled, + "sampling": sampling, + "filter": convert_and_respect_annotation_metadata( + object_=filter, + annotation=typing.Sequence[EvaluationRuleFilter], + direction="write", + ), + "evaluatorAssignments": convert_and_respect_annotation_metadata( + object_=evaluator_assignments, + annotation=typing.Sequence[EvaluationRuleEvaluatorAssignmentInput], + direction="write", + ), + }, + request_options=request_options, + omit=OMIT, + ) + try: + if 200 <= _response.status_code < 300: + _data = typing.cast( + EvaluationRule, + parse_obj_as( + type_=EvaluationRule, # type: ignore + object_=_response.json(), + ), + ) + return AsyncHttpResponse(response=_response, data=_data) + if _response.status_code == 400: + raise BadRequestError( + headers=dict(_response.headers), + body=typing.cast( + PublicApiError, + parse_obj_as( + type_=PublicApiError, # type: ignore + object_=_response.json(), + ), + ), + ) + if _response.status_code == 401: + raise EvaluationUnauthorizedError( + headers=dict(_response.headers), + body=typing.cast( + PublicApiError, + parse_obj_as( + type_=PublicApiError, # type: ignore + object_=_response.json(), + ), + ), + ) + if _response.status_code == 403: + raise EvaluationAccessDeniedError( + headers=dict(_response.headers), + body=typing.cast( + PublicApiError, + parse_obj_as( + type_=PublicApiError, # type: ignore + object_=_response.json(), + ), + ), + ) + if _response.status_code == 404: + raise EvaluationNotFoundError( + headers=dict(_response.headers), + body=typing.cast( + PublicApiError, + parse_obj_as( + type_=PublicApiError, # type: ignore + object_=_response.json(), + ), + ), + ) + if _response.status_code == 405: + raise EvaluationMethodNotAllowedError( + headers=dict(_response.headers), + body=typing.cast( + PublicApiError, + parse_obj_as( + type_=PublicApiError, # type: ignore + object_=_response.json(), + ), + ), + ) + if _response.status_code == 409: + raise ConflictError( + headers=dict(_response.headers), + body=typing.cast( + PublicApiError, + parse_obj_as( + type_=PublicApiError, # type: ignore + object_=_response.json(), + ), + ), + ) + if _response.status_code == 429: + raise TooManyRequestsError( + headers=dict(_response.headers), + body=typing.cast( + PublicApiError, + parse_obj_as( + type_=PublicApiError, # type: ignore + object_=_response.json(), + ), + ), + ) + if _response.status_code == 500: + raise InternalServerError( + headers=dict(_response.headers), + body=typing.cast( + PublicApiError, + parse_obj_as( + type_=PublicApiError, # type: ignore + object_=_response.json(), + ), + ), + ) + if _response.status_code == 400: + raise Error( + headers=dict(_response.headers), + body=typing.cast( + typing.Any, + parse_obj_as( + type_=typing.Any, # type: ignore + object_=_response.json(), + ), + ), + ) + if _response.status_code == 401: + raise UnauthorizedError( + headers=dict(_response.headers), + body=typing.cast( + typing.Any, + parse_obj_as( + type_=typing.Any, # type: ignore + object_=_response.json(), + ), + ), + ) + if _response.status_code == 403: + raise AccessDeniedError( + headers=dict(_response.headers), + body=typing.cast( + typing.Any, + parse_obj_as( + type_=typing.Any, # type: ignore + object_=_response.json(), + ), + ), + ) + if _response.status_code == 405: + raise MethodNotAllowedError( + headers=dict(_response.headers), + body=typing.cast( + typing.Any, + parse_obj_as( + type_=typing.Any, # type: ignore + object_=_response.json(), + ), + ), + ) + if _response.status_code == 404: + raise NotFoundError( + headers=dict(_response.headers), + body=typing.cast( + typing.Any, + parse_obj_as( + type_=typing.Any, # type: ignore + object_=_response.json(), + ), + ), + ) + _response_json = _response.json() + except JSONDecodeError: + raise ApiError( + status_code=_response.status_code, + headers=dict(_response.headers), + body=_response.text, + ) + raise ApiError( + status_code=_response.status_code, + headers=dict(_response.headers), + body=_response_json, + ) + + async def delete( + self, + evaluation_rule_id: str, + *, + request_options: typing.Optional[RequestOptions] = None, + ) -> AsyncHttpResponse[DeletedEvaluationRule]: + """ + Delete an evaluation rule. + + This removes the live-ingestion rule only. It does not delete associated evaluators or scores already produced by them. + + Legacy trace and dataset rules can also be deleted. Their evaluators and previously produced scores are preserved. + + Parameters + ---------- + evaluation_rule_id : str + Stable evaluation-rule identifier returned by the evaluation-rule endpoints. + + request_options : typing.Optional[RequestOptions] + Request-specific configuration. + + Returns + ------- + AsyncHttpResponse[DeletedEvaluationRule] + """ + _response = await self._client_wrapper.httpx_client.request( + f"api/public/v2/evaluation-rules/{jsonable_encoder(evaluation_rule_id)}", + method="DELETE", + request_options=request_options, + ) + try: + if 200 <= _response.status_code < 300: + _data = typing.cast( + DeletedEvaluationRule, + parse_obj_as( + type_=DeletedEvaluationRule, # type: ignore + object_=_response.json(), + ), + ) + return AsyncHttpResponse(response=_response, data=_data) + if _response.status_code == 401: + raise EvaluationUnauthorizedError( + headers=dict(_response.headers), + body=typing.cast( + PublicApiError, + parse_obj_as( + type_=PublicApiError, # type: ignore + object_=_response.json(), + ), + ), + ) + if _response.status_code == 403: + raise EvaluationAccessDeniedError( + headers=dict(_response.headers), + body=typing.cast( + PublicApiError, + parse_obj_as( + type_=PublicApiError, # type: ignore + object_=_response.json(), + ), + ), + ) + if _response.status_code == 404: + raise EvaluationNotFoundError( + headers=dict(_response.headers), + body=typing.cast( + PublicApiError, + parse_obj_as( + type_=PublicApiError, # type: ignore + object_=_response.json(), + ), + ), + ) + if _response.status_code == 405: + raise EvaluationMethodNotAllowedError( + headers=dict(_response.headers), + body=typing.cast( + PublicApiError, + parse_obj_as( + type_=PublicApiError, # type: ignore + object_=_response.json(), + ), + ), + ) + if _response.status_code == 429: + raise TooManyRequestsError( + headers=dict(_response.headers), + body=typing.cast( + PublicApiError, + parse_obj_as( + type_=PublicApiError, # type: ignore + object_=_response.json(), + ), + ), + ) + if _response.status_code == 500: + raise InternalServerError( + headers=dict(_response.headers), + body=typing.cast( + PublicApiError, + parse_obj_as( + type_=PublicApiError, # type: ignore + object_=_response.json(), + ), + ), + ) + if _response.status_code == 400: + raise Error( + headers=dict(_response.headers), + body=typing.cast( + typing.Any, + parse_obj_as( + type_=typing.Any, # type: ignore + object_=_response.json(), + ), + ), + ) + if _response.status_code == 401: + raise UnauthorizedError( + headers=dict(_response.headers), + body=typing.cast( + typing.Any, + parse_obj_as( + type_=typing.Any, # type: ignore + object_=_response.json(), + ), + ), + ) + if _response.status_code == 403: + raise AccessDeniedError( + headers=dict(_response.headers), + body=typing.cast( + typing.Any, + parse_obj_as( + type_=typing.Any, # type: ignore + object_=_response.json(), + ), + ), + ) + if _response.status_code == 405: + raise MethodNotAllowedError( + headers=dict(_response.headers), + body=typing.cast( + typing.Any, + parse_obj_as( + type_=typing.Any, # type: ignore + object_=_response.json(), + ), + ), + ) + if _response.status_code == 404: + raise NotFoundError( + headers=dict(_response.headers), + body=typing.cast( + typing.Any, + parse_obj_as( + type_=typing.Any, # type: ignore + object_=_response.json(), + ), + ), + ) + _response_json = _response.json() + except JSONDecodeError: + raise ApiError( + status_code=_response.status_code, + headers=dict(_response.headers), + body=_response.text, + ) + raise ApiError( + status_code=_response.status_code, + headers=dict(_response.headers), + body=_response_json, + ) diff --git a/langfuse/api/evaluation_rules/types/__init__.py b/langfuse/api/evaluation_rules/types/__init__.py new file mode 100644 index 000000000..c068e9efd --- /dev/null +++ b/langfuse/api/evaluation_rules/types/__init__.py @@ -0,0 +1,64 @@ +# This file was auto-generated by Fern from our API Definition. + +# isort: skip_file + +import typing +from importlib import import_module + +if typing.TYPE_CHECKING: + from .create_evaluation_rule_request import CreateEvaluationRuleRequest + from .deleted_evaluation_rule import DeletedEvaluationRule + from .evaluation_rule import EvaluationRule + from .evaluation_rule_evaluator_assignment_input import ( + EvaluationRuleEvaluatorAssignmentInput, + ) + from .evaluation_rules_page import EvaluationRulesPage + from .evaluator_assignment import EvaluatorAssignment + from .update_evaluation_rule_request import UpdateEvaluationRuleRequest +_dynamic_imports: typing.Dict[str, str] = { + "CreateEvaluationRuleRequest": ".create_evaluation_rule_request", + "DeletedEvaluationRule": ".deleted_evaluation_rule", + "EvaluationRule": ".evaluation_rule", + "EvaluationRuleEvaluatorAssignmentInput": ".evaluation_rule_evaluator_assignment_input", + "EvaluationRulesPage": ".evaluation_rules_page", + "EvaluatorAssignment": ".evaluator_assignment", + "UpdateEvaluationRuleRequest": ".update_evaluation_rule_request", +} + + +def __getattr__(attr_name: str) -> typing.Any: + module_name = _dynamic_imports.get(attr_name) + if module_name is None: + raise AttributeError( + f"No {attr_name} found in _dynamic_imports for module name -> {__name__}" + ) + try: + module = import_module(module_name, __package__) + if module_name == f".{attr_name}": + return module + else: + return getattr(module, attr_name) + except ImportError as e: + raise ImportError( + f"Failed to import {attr_name} from {module_name}: {e}" + ) from e + except AttributeError as e: + raise AttributeError( + f"Failed to get {attr_name} from {module_name}: {e}" + ) from e + + +def __dir__(): + lazy_attrs = list(_dynamic_imports.keys()) + return sorted(lazy_attrs) + + +__all__ = [ + "CreateEvaluationRuleRequest", + "DeletedEvaluationRule", + "EvaluationRule", + "EvaluationRuleEvaluatorAssignmentInput", + "EvaluationRulesPage", + "EvaluatorAssignment", + "UpdateEvaluationRuleRequest", +] diff --git a/langfuse/api/evaluation_rules/types/create_evaluation_rule_request.py b/langfuse/api/evaluation_rules/types/create_evaluation_rule_request.py new file mode 100644 index 000000000..b494ae2d2 --- /dev/null +++ b/langfuse/api/evaluation_rules/types/create_evaluation_rule_request.py @@ -0,0 +1,48 @@ +# This file was auto-generated by Fern from our API Definition. + +import typing + +import pydantic +import typing_extensions +from ...core.pydantic_utilities import UniversalBaseModel +from ...core.serialization import FieldMetadata +from ...evaluation_commons.types.evaluation_rule_filter import EvaluationRuleFilter +from .evaluation_rule_evaluator_assignment_input import ( + EvaluationRuleEvaluatorAssignmentInput, +) + + +class CreateEvaluationRuleRequest(UniversalBaseModel): + name: str = pydantic.Field() + """ + Human-readable rule name. Names are not identifiers and do not need to be unique. + """ + + enabled: bool = pydantic.Field() + """ + Whether live execution should start immediately. Enabled rules require at least one evaluator assignment. + """ + + sampling: typing.Optional[float] = pydantic.Field(default=None) + """ + Fraction of matching observations to evaluate. Omit this field to use the default of `1`, which evaluates every match. `null` is not accepted. + """ + + filter: typing.Optional[typing.List[EvaluationRuleFilter]] = pydantic.Field( + default=None + ) + """ + Conditions used to select observations. Defaults to an empty list, which matches every incoming observation. + """ + + evaluator_assignments: typing_extensions.Annotated[ + typing.List[EvaluationRuleEvaluatorAssignmentInput], + FieldMetadata(alias="evaluatorAssignments"), + ] = pydantic.Field() + """ + Evaluators to attach to this rule. Disabled rules may use an empty list as a draft. + """ + + model_config: typing.ClassVar[pydantic.ConfigDict] = pydantic.ConfigDict( + extra="allow", frozen=True + ) diff --git a/langfuse/api/evaluation_rules/types/deleted_evaluation_rule.py b/langfuse/api/evaluation_rules/types/deleted_evaluation_rule.py new file mode 100644 index 000000000..a08d27a57 --- /dev/null +++ b/langfuse/api/evaluation_rules/types/deleted_evaluation_rule.py @@ -0,0 +1,21 @@ +# This file was auto-generated by Fern from our API Definition. + +import typing + +import pydantic +from ...core.pydantic_utilities import UniversalBaseModel + + +class DeletedEvaluationRule(UniversalBaseModel): + """ + Confirmation returned after successful evaluation-rule deletion. + """ + + id: str = pydantic.Field() + """ + Identifier of the deleted evaluation rule. + """ + + model_config: typing.ClassVar[pydantic.ConfigDict] = pydantic.ConfigDict( + extra="allow", frozen=True + ) diff --git a/langfuse/api/evaluation_rules/types/evaluation_rule.py b/langfuse/api/evaluation_rules/types/evaluation_rule.py new file mode 100644 index 000000000..8286d6065 --- /dev/null +++ b/langfuse/api/evaluation_rules/types/evaluation_rule.py @@ -0,0 +1,83 @@ +# This file was auto-generated by Fern from our API Definition. + +import datetime as dt +import typing + +import pydantic +import typing_extensions +from ...core.pydantic_utilities import UniversalBaseModel +from ...core.serialization import FieldMetadata +from ...evaluation_commons.types.evaluation_rule_read_filter import ( + EvaluationRuleReadFilter, +) +from ...evaluators.types.creator import Creator +from .evaluator_assignment import EvaluatorAssignment + + +class EvaluationRule(UniversalBaseModel): + """ + Live evaluation rule for incoming observations. + + A rule determines which evaluators should be used, which observations should trigger scoring, how often scoring should run, and which observation fields should populate prompt variables. + """ + + id: str = pydantic.Field() + """ + Stable evaluation-rule identifier. + """ + + name: str = pydantic.Field() + """ + Human-readable rule name. This is independent from evaluator names and does not need to be unique. + """ + + created_by: typing_extensions.Annotated[ + typing.Optional[Creator], FieldMetadata(alias="createdBy") + ] = pydantic.Field(default=None) + """ + User who created this rule, or `null` when no user can be resolved. + """ + + enabled: bool = pydantic.Field() + """ + Whether live execution is enabled for this rule. + """ + + sampling: float = pydantic.Field() + """ + Fraction of matching observations that should be evaluated. + + Must be between `0` and `1`. + - `1` evaluates every matching observation. + - `0.25` evaluates approximately 25% of matching observations. + """ + + filter: typing.List[EvaluationRuleReadFilter] = pydantic.Field() + """ + List of stored filter conditions returned verbatim. Filters with a `key` use the keyed response shape; all others use the base shape. These response shapes are not broken down by internal filter type. An empty list matches every incoming object. + """ + + evaluator_assignments: typing_extensions.Annotated[ + typing.List[EvaluatorAssignment], FieldMetadata(alias="evaluatorAssignments") + ] = pydantic.Field() + """ + Evaluators attached to this rule in deterministic assignment order. + """ + + created_at: typing_extensions.Annotated[ + dt.datetime, FieldMetadata(alias="createdAt") + ] = pydantic.Field() + """ + Timestamp when the evaluation rule was created. + """ + + updated_at: typing_extensions.Annotated[ + dt.datetime, FieldMetadata(alias="updatedAt") + ] = pydantic.Field() + """ + Timestamp when the evaluation rule was last updated. + """ + + model_config: typing.ClassVar[pydantic.ConfigDict] = pydantic.ConfigDict( + extra="allow", frozen=True + ) diff --git a/langfuse/api/evaluation_rules/types/evaluation_rule_evaluator_assignment_input.py b/langfuse/api/evaluation_rules/types/evaluation_rule_evaluator_assignment_input.py new file mode 100644 index 000000000..fd2a33089 --- /dev/null +++ b/langfuse/api/evaluation_rules/types/evaluation_rule_evaluator_assignment_input.py @@ -0,0 +1,34 @@ +# This file was auto-generated by Fern from our API Definition. + +import typing + +import pydantic +import typing_extensions +from ...core.pydantic_utilities import UniversalBaseModel +from ...core.serialization import FieldMetadata +from ...evaluation_commons.types.prompt_variable_mapping_input import ( + PromptVariableMappingInput, +) + + +class EvaluationRuleEvaluatorAssignmentInput(UniversalBaseModel): + evaluator_id: typing_extensions.Annotated[ + str, FieldMetadata(alias="evaluatorId") + ] = pydantic.Field() + """ + Stable evaluator identifier. The rule automatically uses that evaluator's latest version. + """ + + variable_mapping: typing_extensions.Annotated[ + typing.Optional[typing.List[PromptVariableMappingInput]], + FieldMetadata(alias="variableMapping"), + ] = pydantic.Field(default=None) + """ + Rule-specific prompt-variable mapping. + + Set to `null` or omit to inherit the evaluator's latest default mapping. Code evaluators use the fixed runtime mapping and should use `null`. + """ + + model_config: typing.ClassVar[pydantic.ConfigDict] = pydantic.ConfigDict( + extra="allow", frozen=True + ) diff --git a/langfuse/api/evaluation_rules/types/evaluation_rules_page.py b/langfuse/api/evaluation_rules/types/evaluation_rules_page.py new file mode 100644 index 000000000..79c6c2092 --- /dev/null +++ b/langfuse/api/evaluation_rules/types/evaluation_rules_page.py @@ -0,0 +1,24 @@ +# This file was auto-generated by Fern from our API Definition. + +import typing + +import pydantic +from ...core.pydantic_utilities import UniversalBaseModel +from ...evaluators.types.cursor_meta import CursorMeta +from .evaluation_rule import EvaluationRule + + +class EvaluationRulesPage(UniversalBaseModel): + data: typing.List[EvaluationRule] = pydantic.Field() + """ + Evaluation rules for this page. + """ + + meta: CursorMeta = pydantic.Field() + """ + Cursor pagination metadata. + """ + + model_config: typing.ClassVar[pydantic.ConfigDict] = pydantic.ConfigDict( + extra="allow", frozen=True + ) diff --git a/langfuse/api/evaluation_rules/types/evaluator_assignment.py b/langfuse/api/evaluation_rules/types/evaluator_assignment.py new file mode 100644 index 000000000..d5e8912b2 --- /dev/null +++ b/langfuse/api/evaluation_rules/types/evaluator_assignment.py @@ -0,0 +1,30 @@ +# This file was auto-generated by Fern from our API Definition. + +import typing + +import pydantic +import typing_extensions +from ...core.pydantic_utilities import UniversalBaseModel +from ...core.serialization import FieldMetadata +from ...evaluation_commons.types.prompt_variable_mapping import PromptVariableMapping + + +class EvaluatorAssignment(UniversalBaseModel): + evaluator_id: typing_extensions.Annotated[ + str, FieldMetadata(alias="evaluatorId") + ] = pydantic.Field() + """ + Stable identifier of the evaluator associated with this rule. + """ + + variable_mapping: typing_extensions.Annotated[ + typing.Optional[typing.List[PromptVariableMapping]], + FieldMetadata(alias="variableMapping"), + ] = pydantic.Field(default=None) + """ + Stored rule-specific override, or `null` when the evaluator's latest default mapping is inherited. Legacy mappings are returned as an explicit `legacy` variant. + """ + + model_config: typing.ClassVar[pydantic.ConfigDict] = pydantic.ConfigDict( + extra="allow", frozen=True + ) diff --git a/langfuse/api/evaluation_rules/types/update_evaluation_rule_request.py b/langfuse/api/evaluation_rules/types/update_evaluation_rule_request.py new file mode 100644 index 000000000..9b80bae7c --- /dev/null +++ b/langfuse/api/evaluation_rules/types/update_evaluation_rule_request.py @@ -0,0 +1,52 @@ +# This file was auto-generated by Fern from our API Definition. + +import typing + +import pydantic +import typing_extensions +from ...core.pydantic_utilities import UniversalBaseModel +from ...core.serialization import FieldMetadata +from ...evaluation_commons.types.evaluation_rule_filter import EvaluationRuleFilter +from .evaluation_rule_evaluator_assignment_input import ( + EvaluationRuleEvaluatorAssignmentInput, +) + + +class UpdateEvaluationRuleRequest(UniversalBaseModel): + """ + At least one field must be provided. + """ + + name: typing.Optional[str] = pydantic.Field(default=None) + """ + New human-readable rule name. + """ + + enabled: typing.Optional[bool] = pydantic.Field(default=None) + """ + New desired live-execution state. + """ + + sampling: typing.Optional[float] = pydantic.Field(default=None) + """ + New fraction of matching observations to evaluate. Omit to keep the current value. + """ + + filter: typing.Optional[typing.List[EvaluationRuleFilter]] = pydantic.Field( + default=None + ) + """ + Complete replacement filter list. An empty list matches every incoming observation. + """ + + evaluator_assignments: typing_extensions.Annotated[ + typing.Optional[typing.List[EvaluationRuleEvaluatorAssignmentInput]], + FieldMetadata(alias="evaluatorAssignments"), + ] = pydantic.Field(default=None) + """ + Complete replacement assignment list. An empty list disables the rule. + """ + + model_config: typing.ClassVar[pydantic.ConfigDict] = pydantic.ConfigDict( + extra="allow", frozen=True + ) diff --git a/langfuse/api/evaluators/__init__.py b/langfuse/api/evaluators/__init__.py new file mode 100644 index 000000000..587354a54 --- /dev/null +++ b/langfuse/api/evaluators/__init__.py @@ -0,0 +1,139 @@ +# This file was auto-generated by Fern from our API Definition. + +# isort: skip_file + +import typing +from importlib import import_module + +if typing.TYPE_CHECKING: + from .types import ( + CodeEvaluator, + CodeEvaluatorVersion, + CreateCodeEvaluatorRequest, + CreateEvaluatorRequest, + CreateEvaluatorRequest_Code, + CreateEvaluatorRequest_LlmAsJudge, + CreateLlmAsJudgeEvaluatorRequest, + Creator, + CursorMeta, + DeletedEvaluator, + EvaluationRuleAssignment, + Evaluator, + EvaluatorBase, + EvaluatorChatMessage, + EvaluatorChatPrompt, + EvaluatorChatPromptInput, + EvaluatorModelConfig, + EvaluatorStatus, + EvaluatorVersion, + EvaluatorVersionBase, + EvaluatorVersion_Code, + EvaluatorVersion_LlmAsJudge, + EvaluatorVersionsPage, + Evaluator_Code, + Evaluator_LlmAsJudge, + EvaluatorsPage, + LlmAsJudgeEvaluator, + LlmAsJudgeEvaluatorVersion, + UpdateCodeEvaluatorRequest, + UpdateEvaluatorMetadataRequest, + UpdateEvaluatorRequest, + UpdateLlmAsJudgeEvaluatorRequest, + ) +_dynamic_imports: typing.Dict[str, str] = { + "CodeEvaluator": ".types", + "CodeEvaluatorVersion": ".types", + "CreateCodeEvaluatorRequest": ".types", + "CreateEvaluatorRequest": ".types", + "CreateEvaluatorRequest_Code": ".types", + "CreateEvaluatorRequest_LlmAsJudge": ".types", + "CreateLlmAsJudgeEvaluatorRequest": ".types", + "Creator": ".types", + "CursorMeta": ".types", + "DeletedEvaluator": ".types", + "EvaluationRuleAssignment": ".types", + "Evaluator": ".types", + "EvaluatorBase": ".types", + "EvaluatorChatMessage": ".types", + "EvaluatorChatPrompt": ".types", + "EvaluatorChatPromptInput": ".types", + "EvaluatorModelConfig": ".types", + "EvaluatorStatus": ".types", + "EvaluatorVersion": ".types", + "EvaluatorVersionBase": ".types", + "EvaluatorVersion_Code": ".types", + "EvaluatorVersion_LlmAsJudge": ".types", + "EvaluatorVersionsPage": ".types", + "Evaluator_Code": ".types", + "Evaluator_LlmAsJudge": ".types", + "EvaluatorsPage": ".types", + "LlmAsJudgeEvaluator": ".types", + "LlmAsJudgeEvaluatorVersion": ".types", + "UpdateCodeEvaluatorRequest": ".types", + "UpdateEvaluatorMetadataRequest": ".types", + "UpdateEvaluatorRequest": ".types", + "UpdateLlmAsJudgeEvaluatorRequest": ".types", +} + + +def __getattr__(attr_name: str) -> typing.Any: + module_name = _dynamic_imports.get(attr_name) + if module_name is None: + raise AttributeError( + f"No {attr_name} found in _dynamic_imports for module name -> {__name__}" + ) + try: + module = import_module(module_name, __package__) + if module_name == f".{attr_name}": + return module + else: + return getattr(module, attr_name) + except ImportError as e: + raise ImportError( + f"Failed to import {attr_name} from {module_name}: {e}" + ) from e + except AttributeError as e: + raise AttributeError( + f"Failed to get {attr_name} from {module_name}: {e}" + ) from e + + +def __dir__(): + lazy_attrs = list(_dynamic_imports.keys()) + return sorted(lazy_attrs) + + +__all__ = [ + "CodeEvaluator", + "CodeEvaluatorVersion", + "CreateCodeEvaluatorRequest", + "CreateEvaluatorRequest", + "CreateEvaluatorRequest_Code", + "CreateEvaluatorRequest_LlmAsJudge", + "CreateLlmAsJudgeEvaluatorRequest", + "Creator", + "CursorMeta", + "DeletedEvaluator", + "EvaluationRuleAssignment", + "Evaluator", + "EvaluatorBase", + "EvaluatorChatMessage", + "EvaluatorChatPrompt", + "EvaluatorChatPromptInput", + "EvaluatorModelConfig", + "EvaluatorStatus", + "EvaluatorVersion", + "EvaluatorVersionBase", + "EvaluatorVersion_Code", + "EvaluatorVersion_LlmAsJudge", + "EvaluatorVersionsPage", + "Evaluator_Code", + "Evaluator_LlmAsJudge", + "EvaluatorsPage", + "LlmAsJudgeEvaluator", + "LlmAsJudgeEvaluatorVersion", + "UpdateCodeEvaluatorRequest", + "UpdateEvaluatorMetadataRequest", + "UpdateEvaluatorRequest", + "UpdateLlmAsJudgeEvaluatorRequest", +] diff --git a/langfuse/api/evaluators/client.py b/langfuse/api/evaluators/client.py new file mode 100644 index 000000000..9ff89ceba --- /dev/null +++ b/langfuse/api/evaluators/client.py @@ -0,0 +1,674 @@ +# This file was auto-generated by Fern from our API Definition. + +import typing + +from ..core.client_wrapper import AsyncClientWrapper, SyncClientWrapper +from ..core.request_options import RequestOptions +from .raw_client import AsyncRawEvaluatorsClient, RawEvaluatorsClient +from .types.create_evaluator_request import CreateEvaluatorRequest +from .types.deleted_evaluator import DeletedEvaluator +from .types.evaluator import Evaluator +from .types.evaluator_versions_page import EvaluatorVersionsPage +from .types.evaluators_page import EvaluatorsPage +from .types.update_evaluator_request import UpdateEvaluatorRequest + +# this is used as the default value for optional parameters +OMIT = typing.cast(typing.Any, ...) + + +class EvaluatorsClient: + def __init__(self, *, client_wrapper: SyncClientWrapper): + self._raw_client = RawEvaluatorsClient(client_wrapper=client_wrapper) + + @property + def with_raw_response(self) -> RawEvaluatorsClient: + """ + Retrieves a raw implementation of this client that returns raw responses. + + Returns + ------- + RawEvaluatorsClient + """ + return self._raw_client + + def create( + self, + *, + request: CreateEvaluatorRequest, + request_options: typing.Optional[RequestOptions] = None, + ) -> Evaluator: + """ + Create an evaluator in the authenticated project. + + An evaluator defines **how** Langfuse should score data. LLM-as-a-judge evaluators define a prompt, expected structured output, and optional model configuration. Code evaluators define source code and a runtime language. + + This always creates a new evaluator with version `1`. Names are not identifiers and do not need to be unique. + + Parameters + ---------- + request : CreateEvaluatorRequest + + request_options : typing.Optional[RequestOptions] + Request-specific configuration. + + Returns + ------- + Evaluator + + Examples + -------- + from langfuse import LangfuseAPI + from langfuse.evaluation_commons import EvaluatorOutputDefinition_Numeric + from langfuse.evaluators import CreateEvaluatorRequest_LlmAsJudge + + client = LangfuseAPI( + x_langfuse_sdk_name="YOUR_X_LANGFUSE_SDK_NAME", + x_langfuse_sdk_version="YOUR_X_LANGFUSE_SDK_VERSION", + x_langfuse_public_key="YOUR_X_LANGFUSE_PUBLIC_KEY", + username="YOUR_USERNAME", + password="YOUR_PASSWORD", + base_url="https://yourhost.com/path/to/api", + ) + client.evaluators.create( + request=CreateEvaluatorRequest_LlmAsJudge( + name="name", + prompt="prompt", + output_definition=EvaluatorOutputDefinition_Numeric(), + ), + ) + """ + _response = self._raw_client.create( + request=request, request_options=request_options + ) + return _response.data + + def list( + self, + *, + limit: typing.Optional[int] = None, + cursor: typing.Optional[str] = None, + request_options: typing.Optional[RequestOptions] = None, + ) -> EvaluatorsPage: + """ + List evaluators in newest-first creation order. + + Every evaluator includes its latest definition and version metadata flattened into the evaluator object, plus associated evaluation rules. Treat the cursor as opaque and return it unchanged. + + Parameters + ---------- + limit : typing.Optional[int] + Maximum number of items to return. Defaults to `50` and cannot exceed `100`. + + cursor : typing.Optional[str] + Opaque cursor returned by the previous page. + + request_options : typing.Optional[RequestOptions] + Request-specific configuration. + + Returns + ------- + EvaluatorsPage + + Examples + -------- + from langfuse import LangfuseAPI + + client = LangfuseAPI( + x_langfuse_sdk_name="YOUR_X_LANGFUSE_SDK_NAME", + x_langfuse_sdk_version="YOUR_X_LANGFUSE_SDK_VERSION", + x_langfuse_public_key="YOUR_X_LANGFUSE_PUBLIC_KEY", + username="YOUR_USERNAME", + password="YOUR_PASSWORD", + base_url="https://yourhost.com/path/to/api", + ) + client.evaluators.list() + """ + _response = self._raw_client.list( + limit=limit, cursor=cursor, request_options=request_options + ) + return _response.data + + def get( + self, + evaluator_id: str, + *, + request_options: typing.Optional[RequestOptions] = None, + ) -> Evaluator: + """ + Get one evaluator by its stable identifier. + + The response includes the evaluator's latest definition and version metadata flattened into the evaluator object, plus associated evaluation rules. Use the version-history endpoint when older definitions are needed. + + Parameters + ---------- + evaluator_id : str + Stable evaluator identifier returned by the evaluator endpoints. + + request_options : typing.Optional[RequestOptions] + Request-specific configuration. + + Returns + ------- + Evaluator + + Examples + -------- + from langfuse import LangfuseAPI + + client = LangfuseAPI( + x_langfuse_sdk_name="YOUR_X_LANGFUSE_SDK_NAME", + x_langfuse_sdk_version="YOUR_X_LANGFUSE_SDK_VERSION", + x_langfuse_public_key="YOUR_X_LANGFUSE_PUBLIC_KEY", + username="YOUR_USERNAME", + password="YOUR_PASSWORD", + base_url="https://yourhost.com/path/to/api", + ) + client.evaluators.get( + evaluator_id="evaluatorId", + ) + """ + _response = self._raw_client.get(evaluator_id, request_options=request_options) + return _response.data + + def update( + self, + evaluator_id: str, + *, + request: UpdateEvaluatorRequest, + request_options: typing.Optional[RequestOptions] = None, + ) -> Evaluator: + """ + Update an evaluator by its stable identifier. + + Provide only the top-level fields to change. Metadata-only changes do not create a version. Evaluator type cannot change. + + Definition fields are flattened into the request. To replace a definition, include `type` and every definition field for that type. Definition fields are replaced as a complete unit rather than merged. For LLM-as-a-judge evaluators, omitting or setting `modelConfig` to `null` selects the project's default evaluation model. + + Replacing a definition automatically returns an evaluator paused by an invalid or missing model configuration to `active`, while preserving its id and evaluation-rule assignments. Pauses caused by provider authentication, billing, connectivity, account state, or an unknown legacy reason remain paused until the explicit reactivation check succeeds. + + Parameters + ---------- + evaluator_id : str + Stable evaluator identifier returned by the evaluator endpoints. + + request : UpdateEvaluatorRequest + + request_options : typing.Optional[RequestOptions] + Request-specific configuration. + + Returns + ------- + Evaluator + + Examples + -------- + from langfuse import LangfuseAPI + from langfuse.evaluators import UpdateEvaluatorMetadataRequest + + client = LangfuseAPI( + x_langfuse_sdk_name="YOUR_X_LANGFUSE_SDK_NAME", + x_langfuse_sdk_version="YOUR_X_LANGFUSE_SDK_VERSION", + x_langfuse_public_key="YOUR_X_LANGFUSE_PUBLIC_KEY", + username="YOUR_USERNAME", + password="YOUR_PASSWORD", + base_url="https://yourhost.com/path/to/api", + ) + client.evaluators.update( + evaluator_id="evaluatorId", + request=UpdateEvaluatorMetadataRequest(), + ) + """ + _response = self._raw_client.update( + evaluator_id, request=request, request_options=request_options + ) + return _response.data + + def delete( + self, + evaluator_id: str, + *, + request_options: typing.Optional[RequestOptions] = None, + ) -> DeletedEvaluator: + """ + Delete an evaluator and all of its stored versions. + + Associated evaluation-rule assignments are also removed. Scores already produced by the evaluator are preserved. + + Parameters + ---------- + evaluator_id : str + Stable evaluator identifier returned by the evaluator endpoints. + + request_options : typing.Optional[RequestOptions] + Request-specific configuration. + + Returns + ------- + DeletedEvaluator + + Examples + -------- + from langfuse import LangfuseAPI + + client = LangfuseAPI( + x_langfuse_sdk_name="YOUR_X_LANGFUSE_SDK_NAME", + x_langfuse_sdk_version="YOUR_X_LANGFUSE_SDK_VERSION", + x_langfuse_public_key="YOUR_X_LANGFUSE_PUBLIC_KEY", + username="YOUR_USERNAME", + password="YOUR_PASSWORD", + base_url="https://yourhost.com/path/to/api", + ) + client.evaluators.delete( + evaluator_id="evaluatorId", + ) + """ + _response = self._raw_client.delete( + evaluator_id, request_options=request_options + ) + return _response.data + + def list_versions( + self, + evaluator_id: str, + *, + limit: typing.Optional[int] = None, + cursor: typing.Optional[str] = None, + request_options: typing.Optional[RequestOptions] = None, + ) -> EvaluatorVersionsPage: + """ + List an evaluator's version history in newest-first order. + + This endpoint is intended for history and audit use cases. Ordinary clients can use the flattened `version` and definition fields on the evaluator response. + + Parameters + ---------- + evaluator_id : str + Stable evaluator identifier returned by the evaluator endpoints. + + limit : typing.Optional[int] + Maximum number of versions to return. Defaults to `50` and cannot exceed `100`. + + cursor : typing.Optional[str] + Opaque cursor returned by the previous page. + + request_options : typing.Optional[RequestOptions] + Request-specific configuration. + + Returns + ------- + EvaluatorVersionsPage + + Examples + -------- + from langfuse import LangfuseAPI + + client = LangfuseAPI( + x_langfuse_sdk_name="YOUR_X_LANGFUSE_SDK_NAME", + x_langfuse_sdk_version="YOUR_X_LANGFUSE_SDK_VERSION", + x_langfuse_public_key="YOUR_X_LANGFUSE_PUBLIC_KEY", + username="YOUR_USERNAME", + password="YOUR_PASSWORD", + base_url="https://yourhost.com/path/to/api", + ) + client.evaluators.list_versions( + evaluator_id="evaluatorId", + ) + """ + _response = self._raw_client.list_versions( + evaluator_id, limit=limit, cursor=cursor, request_options=request_options + ) + return _response.data + + +class AsyncEvaluatorsClient: + def __init__(self, *, client_wrapper: AsyncClientWrapper): + self._raw_client = AsyncRawEvaluatorsClient(client_wrapper=client_wrapper) + + @property + def with_raw_response(self) -> AsyncRawEvaluatorsClient: + """ + Retrieves a raw implementation of this client that returns raw responses. + + Returns + ------- + AsyncRawEvaluatorsClient + """ + return self._raw_client + + async def create( + self, + *, + request: CreateEvaluatorRequest, + request_options: typing.Optional[RequestOptions] = None, + ) -> Evaluator: + """ + Create an evaluator in the authenticated project. + + An evaluator defines **how** Langfuse should score data. LLM-as-a-judge evaluators define a prompt, expected structured output, and optional model configuration. Code evaluators define source code and a runtime language. + + This always creates a new evaluator with version `1`. Names are not identifiers and do not need to be unique. + + Parameters + ---------- + request : CreateEvaluatorRequest + + request_options : typing.Optional[RequestOptions] + Request-specific configuration. + + Returns + ------- + Evaluator + + Examples + -------- + import asyncio + + from langfuse import AsyncLangfuseAPI + from langfuse.evaluation_commons import EvaluatorOutputDefinition_Numeric + from langfuse.evaluators import CreateEvaluatorRequest_LlmAsJudge + + client = AsyncLangfuseAPI( + x_langfuse_sdk_name="YOUR_X_LANGFUSE_SDK_NAME", + x_langfuse_sdk_version="YOUR_X_LANGFUSE_SDK_VERSION", + x_langfuse_public_key="YOUR_X_LANGFUSE_PUBLIC_KEY", + username="YOUR_USERNAME", + password="YOUR_PASSWORD", + base_url="https://yourhost.com/path/to/api", + ) + + + async def main() -> None: + await client.evaluators.create( + request=CreateEvaluatorRequest_LlmAsJudge( + name="name", + prompt="prompt", + output_definition=EvaluatorOutputDefinition_Numeric(), + ), + ) + + + asyncio.run(main()) + """ + _response = await self._raw_client.create( + request=request, request_options=request_options + ) + return _response.data + + async def list( + self, + *, + limit: typing.Optional[int] = None, + cursor: typing.Optional[str] = None, + request_options: typing.Optional[RequestOptions] = None, + ) -> EvaluatorsPage: + """ + List evaluators in newest-first creation order. + + Every evaluator includes its latest definition and version metadata flattened into the evaluator object, plus associated evaluation rules. Treat the cursor as opaque and return it unchanged. + + Parameters + ---------- + limit : typing.Optional[int] + Maximum number of items to return. Defaults to `50` and cannot exceed `100`. + + cursor : typing.Optional[str] + Opaque cursor returned by the previous page. + + request_options : typing.Optional[RequestOptions] + Request-specific configuration. + + Returns + ------- + EvaluatorsPage + + Examples + -------- + import asyncio + + from langfuse import AsyncLangfuseAPI + + client = AsyncLangfuseAPI( + x_langfuse_sdk_name="YOUR_X_LANGFUSE_SDK_NAME", + x_langfuse_sdk_version="YOUR_X_LANGFUSE_SDK_VERSION", + x_langfuse_public_key="YOUR_X_LANGFUSE_PUBLIC_KEY", + username="YOUR_USERNAME", + password="YOUR_PASSWORD", + base_url="https://yourhost.com/path/to/api", + ) + + + async def main() -> None: + await client.evaluators.list() + + + asyncio.run(main()) + """ + _response = await self._raw_client.list( + limit=limit, cursor=cursor, request_options=request_options + ) + return _response.data + + async def get( + self, + evaluator_id: str, + *, + request_options: typing.Optional[RequestOptions] = None, + ) -> Evaluator: + """ + Get one evaluator by its stable identifier. + + The response includes the evaluator's latest definition and version metadata flattened into the evaluator object, plus associated evaluation rules. Use the version-history endpoint when older definitions are needed. + + Parameters + ---------- + evaluator_id : str + Stable evaluator identifier returned by the evaluator endpoints. + + request_options : typing.Optional[RequestOptions] + Request-specific configuration. + + Returns + ------- + Evaluator + + Examples + -------- + import asyncio + + from langfuse import AsyncLangfuseAPI + + client = AsyncLangfuseAPI( + x_langfuse_sdk_name="YOUR_X_LANGFUSE_SDK_NAME", + x_langfuse_sdk_version="YOUR_X_LANGFUSE_SDK_VERSION", + x_langfuse_public_key="YOUR_X_LANGFUSE_PUBLIC_KEY", + username="YOUR_USERNAME", + password="YOUR_PASSWORD", + base_url="https://yourhost.com/path/to/api", + ) + + + async def main() -> None: + await client.evaluators.get( + evaluator_id="evaluatorId", + ) + + + asyncio.run(main()) + """ + _response = await self._raw_client.get( + evaluator_id, request_options=request_options + ) + return _response.data + + async def update( + self, + evaluator_id: str, + *, + request: UpdateEvaluatorRequest, + request_options: typing.Optional[RequestOptions] = None, + ) -> Evaluator: + """ + Update an evaluator by its stable identifier. + + Provide only the top-level fields to change. Metadata-only changes do not create a version. Evaluator type cannot change. + + Definition fields are flattened into the request. To replace a definition, include `type` and every definition field for that type. Definition fields are replaced as a complete unit rather than merged. For LLM-as-a-judge evaluators, omitting or setting `modelConfig` to `null` selects the project's default evaluation model. + + Replacing a definition automatically returns an evaluator paused by an invalid or missing model configuration to `active`, while preserving its id and evaluation-rule assignments. Pauses caused by provider authentication, billing, connectivity, account state, or an unknown legacy reason remain paused until the explicit reactivation check succeeds. + + Parameters + ---------- + evaluator_id : str + Stable evaluator identifier returned by the evaluator endpoints. + + request : UpdateEvaluatorRequest + + request_options : typing.Optional[RequestOptions] + Request-specific configuration. + + Returns + ------- + Evaluator + + Examples + -------- + import asyncio + + from langfuse import AsyncLangfuseAPI + from langfuse.evaluators import UpdateEvaluatorMetadataRequest + + client = AsyncLangfuseAPI( + x_langfuse_sdk_name="YOUR_X_LANGFUSE_SDK_NAME", + x_langfuse_sdk_version="YOUR_X_LANGFUSE_SDK_VERSION", + x_langfuse_public_key="YOUR_X_LANGFUSE_PUBLIC_KEY", + username="YOUR_USERNAME", + password="YOUR_PASSWORD", + base_url="https://yourhost.com/path/to/api", + ) + + + async def main() -> None: + await client.evaluators.update( + evaluator_id="evaluatorId", + request=UpdateEvaluatorMetadataRequest(), + ) + + + asyncio.run(main()) + """ + _response = await self._raw_client.update( + evaluator_id, request=request, request_options=request_options + ) + return _response.data + + async def delete( + self, + evaluator_id: str, + *, + request_options: typing.Optional[RequestOptions] = None, + ) -> DeletedEvaluator: + """ + Delete an evaluator and all of its stored versions. + + Associated evaluation-rule assignments are also removed. Scores already produced by the evaluator are preserved. + + Parameters + ---------- + evaluator_id : str + Stable evaluator identifier returned by the evaluator endpoints. + + request_options : typing.Optional[RequestOptions] + Request-specific configuration. + + Returns + ------- + DeletedEvaluator + + Examples + -------- + import asyncio + + from langfuse import AsyncLangfuseAPI + + client = AsyncLangfuseAPI( + x_langfuse_sdk_name="YOUR_X_LANGFUSE_SDK_NAME", + x_langfuse_sdk_version="YOUR_X_LANGFUSE_SDK_VERSION", + x_langfuse_public_key="YOUR_X_LANGFUSE_PUBLIC_KEY", + username="YOUR_USERNAME", + password="YOUR_PASSWORD", + base_url="https://yourhost.com/path/to/api", + ) + + + async def main() -> None: + await client.evaluators.delete( + evaluator_id="evaluatorId", + ) + + + asyncio.run(main()) + """ + _response = await self._raw_client.delete( + evaluator_id, request_options=request_options + ) + return _response.data + + async def list_versions( + self, + evaluator_id: str, + *, + limit: typing.Optional[int] = None, + cursor: typing.Optional[str] = None, + request_options: typing.Optional[RequestOptions] = None, + ) -> EvaluatorVersionsPage: + """ + List an evaluator's version history in newest-first order. + + This endpoint is intended for history and audit use cases. Ordinary clients can use the flattened `version` and definition fields on the evaluator response. + + Parameters + ---------- + evaluator_id : str + Stable evaluator identifier returned by the evaluator endpoints. + + limit : typing.Optional[int] + Maximum number of versions to return. Defaults to `50` and cannot exceed `100`. + + cursor : typing.Optional[str] + Opaque cursor returned by the previous page. + + request_options : typing.Optional[RequestOptions] + Request-specific configuration. + + Returns + ------- + EvaluatorVersionsPage + + Examples + -------- + import asyncio + + from langfuse import AsyncLangfuseAPI + + client = AsyncLangfuseAPI( + x_langfuse_sdk_name="YOUR_X_LANGFUSE_SDK_NAME", + x_langfuse_sdk_version="YOUR_X_LANGFUSE_SDK_VERSION", + x_langfuse_public_key="YOUR_X_LANGFUSE_PUBLIC_KEY", + username="YOUR_USERNAME", + password="YOUR_PASSWORD", + base_url="https://yourhost.com/path/to/api", + ) + + + async def main() -> None: + await client.evaluators.list_versions( + evaluator_id="evaluatorId", + ) + + + asyncio.run(main()) + """ + _response = await self._raw_client.list_versions( + evaluator_id, limit=limit, cursor=cursor, request_options=request_options + ) + return _response.data diff --git a/langfuse/api/evaluators/raw_client.py b/langfuse/api/evaluators/raw_client.py new file mode 100644 index 000000000..bc4a8059b --- /dev/null +++ b/langfuse/api/evaluators/raw_client.py @@ -0,0 +1,2300 @@ +# This file was auto-generated by Fern from our API Definition. + +import typing +from json.decoder import JSONDecodeError + +from ..commons.errors.access_denied_error import AccessDeniedError +from ..commons.errors.error import Error +from ..commons.errors.method_not_allowed_error import MethodNotAllowedError +from ..commons.errors.not_found_error import NotFoundError +from ..commons.errors.unauthorized_error import UnauthorizedError +from ..core.api_error import ApiError +from ..core.client_wrapper import AsyncClientWrapper, SyncClientWrapper +from ..core.http_response import AsyncHttpResponse, HttpResponse +from ..core.jsonable_encoder import jsonable_encoder +from ..core.pydantic_utilities import parse_obj_as +from ..core.request_options import RequestOptions +from ..core.serialization import convert_and_respect_annotation_metadata +from ..evaluation_errors.errors.bad_request_error import BadRequestError +from ..evaluation_errors.errors.conflict_error import ConflictError +from ..evaluation_errors.errors.evaluation_access_denied_error import ( + EvaluationAccessDeniedError, +) +from ..evaluation_errors.errors.evaluation_method_not_allowed_error import ( + EvaluationMethodNotAllowedError, +) +from ..evaluation_errors.errors.evaluation_not_found_error import ( + EvaluationNotFoundError, +) +from ..evaluation_errors.errors.evaluation_unauthorized_error import ( + EvaluationUnauthorizedError, +) +from ..evaluation_errors.errors.internal_server_error import InternalServerError +from ..evaluation_errors.errors.precondition_failed_error import PreconditionFailedError +from ..evaluation_errors.errors.too_many_requests_error import TooManyRequestsError +from ..evaluation_errors.types.public_api_error import PublicApiError +from .types.create_evaluator_request import CreateEvaluatorRequest +from .types.deleted_evaluator import DeletedEvaluator +from .types.evaluator import Evaluator +from .types.evaluator_versions_page import EvaluatorVersionsPage +from .types.evaluators_page import EvaluatorsPage +from .types.update_evaluator_request import UpdateEvaluatorRequest + +# this is used as the default value for optional parameters +OMIT = typing.cast(typing.Any, ...) + + +class RawEvaluatorsClient: + def __init__(self, *, client_wrapper: SyncClientWrapper): + self._client_wrapper = client_wrapper + + def create( + self, + *, + request: CreateEvaluatorRequest, + request_options: typing.Optional[RequestOptions] = None, + ) -> HttpResponse[Evaluator]: + """ + Create an evaluator in the authenticated project. + + An evaluator defines **how** Langfuse should score data. LLM-as-a-judge evaluators define a prompt, expected structured output, and optional model configuration. Code evaluators define source code and a runtime language. + + This always creates a new evaluator with version `1`. Names are not identifiers and do not need to be unique. + + Parameters + ---------- + request : CreateEvaluatorRequest + + request_options : typing.Optional[RequestOptions] + Request-specific configuration. + + Returns + ------- + HttpResponse[Evaluator] + """ + _response = self._client_wrapper.httpx_client.request( + "api/public/v2/evaluators", + method="POST", + json=convert_and_respect_annotation_metadata( + object_=request, annotation=CreateEvaluatorRequest, direction="write" + ), + request_options=request_options, + omit=OMIT, + ) + try: + if 200 <= _response.status_code < 300: + _data = typing.cast( + Evaluator, + parse_obj_as( + type_=Evaluator, # type: ignore + object_=_response.json(), + ), + ) + return HttpResponse(response=_response, data=_data) + if _response.status_code == 400: + raise BadRequestError( + headers=dict(_response.headers), + body=typing.cast( + PublicApiError, + parse_obj_as( + type_=PublicApiError, # type: ignore + object_=_response.json(), + ), + ), + ) + if _response.status_code == 401: + raise EvaluationUnauthorizedError( + headers=dict(_response.headers), + body=typing.cast( + PublicApiError, + parse_obj_as( + type_=PublicApiError, # type: ignore + object_=_response.json(), + ), + ), + ) + if _response.status_code == 403: + raise EvaluationAccessDeniedError( + headers=dict(_response.headers), + body=typing.cast( + PublicApiError, + parse_obj_as( + type_=PublicApiError, # type: ignore + object_=_response.json(), + ), + ), + ) + if _response.status_code == 405: + raise EvaluationMethodNotAllowedError( + headers=dict(_response.headers), + body=typing.cast( + PublicApiError, + parse_obj_as( + type_=PublicApiError, # type: ignore + object_=_response.json(), + ), + ), + ) + if _response.status_code == 412: + raise PreconditionFailedError( + headers=dict(_response.headers), + body=typing.cast( + PublicApiError, + parse_obj_as( + type_=PublicApiError, # type: ignore + object_=_response.json(), + ), + ), + ) + if _response.status_code == 429: + raise TooManyRequestsError( + headers=dict(_response.headers), + body=typing.cast( + PublicApiError, + parse_obj_as( + type_=PublicApiError, # type: ignore + object_=_response.json(), + ), + ), + ) + if _response.status_code == 500: + raise InternalServerError( + headers=dict(_response.headers), + body=typing.cast( + PublicApiError, + parse_obj_as( + type_=PublicApiError, # type: ignore + object_=_response.json(), + ), + ), + ) + if _response.status_code == 400: + raise Error( + headers=dict(_response.headers), + body=typing.cast( + typing.Any, + parse_obj_as( + type_=typing.Any, # type: ignore + object_=_response.json(), + ), + ), + ) + if _response.status_code == 401: + raise UnauthorizedError( + headers=dict(_response.headers), + body=typing.cast( + typing.Any, + parse_obj_as( + type_=typing.Any, # type: ignore + object_=_response.json(), + ), + ), + ) + if _response.status_code == 403: + raise AccessDeniedError( + headers=dict(_response.headers), + body=typing.cast( + typing.Any, + parse_obj_as( + type_=typing.Any, # type: ignore + object_=_response.json(), + ), + ), + ) + if _response.status_code == 405: + raise MethodNotAllowedError( + headers=dict(_response.headers), + body=typing.cast( + typing.Any, + parse_obj_as( + type_=typing.Any, # type: ignore + object_=_response.json(), + ), + ), + ) + if _response.status_code == 404: + raise NotFoundError( + headers=dict(_response.headers), + body=typing.cast( + typing.Any, + parse_obj_as( + type_=typing.Any, # type: ignore + object_=_response.json(), + ), + ), + ) + _response_json = _response.json() + except JSONDecodeError: + raise ApiError( + status_code=_response.status_code, + headers=dict(_response.headers), + body=_response.text, + ) + raise ApiError( + status_code=_response.status_code, + headers=dict(_response.headers), + body=_response_json, + ) + + def list( + self, + *, + limit: typing.Optional[int] = None, + cursor: typing.Optional[str] = None, + request_options: typing.Optional[RequestOptions] = None, + ) -> HttpResponse[EvaluatorsPage]: + """ + List evaluators in newest-first creation order. + + Every evaluator includes its latest definition and version metadata flattened into the evaluator object, plus associated evaluation rules. Treat the cursor as opaque and return it unchanged. + + Parameters + ---------- + limit : typing.Optional[int] + Maximum number of items to return. Defaults to `50` and cannot exceed `100`. + + cursor : typing.Optional[str] + Opaque cursor returned by the previous page. + + request_options : typing.Optional[RequestOptions] + Request-specific configuration. + + Returns + ------- + HttpResponse[EvaluatorsPage] + """ + _response = self._client_wrapper.httpx_client.request( + "api/public/v2/evaluators", + method="GET", + params={ + "limit": limit, + "cursor": cursor, + }, + request_options=request_options, + ) + try: + if 200 <= _response.status_code < 300: + _data = typing.cast( + EvaluatorsPage, + parse_obj_as( + type_=EvaluatorsPage, # type: ignore + object_=_response.json(), + ), + ) + return HttpResponse(response=_response, data=_data) + if _response.status_code == 400: + raise BadRequestError( + headers=dict(_response.headers), + body=typing.cast( + PublicApiError, + parse_obj_as( + type_=PublicApiError, # type: ignore + object_=_response.json(), + ), + ), + ) + if _response.status_code == 401: + raise EvaluationUnauthorizedError( + headers=dict(_response.headers), + body=typing.cast( + PublicApiError, + parse_obj_as( + type_=PublicApiError, # type: ignore + object_=_response.json(), + ), + ), + ) + if _response.status_code == 403: + raise EvaluationAccessDeniedError( + headers=dict(_response.headers), + body=typing.cast( + PublicApiError, + parse_obj_as( + type_=PublicApiError, # type: ignore + object_=_response.json(), + ), + ), + ) + if _response.status_code == 405: + raise EvaluationMethodNotAllowedError( + headers=dict(_response.headers), + body=typing.cast( + PublicApiError, + parse_obj_as( + type_=PublicApiError, # type: ignore + object_=_response.json(), + ), + ), + ) + if _response.status_code == 429: + raise TooManyRequestsError( + headers=dict(_response.headers), + body=typing.cast( + PublicApiError, + parse_obj_as( + type_=PublicApiError, # type: ignore + object_=_response.json(), + ), + ), + ) + if _response.status_code == 500: + raise InternalServerError( + headers=dict(_response.headers), + body=typing.cast( + PublicApiError, + parse_obj_as( + type_=PublicApiError, # type: ignore + object_=_response.json(), + ), + ), + ) + if _response.status_code == 400: + raise Error( + headers=dict(_response.headers), + body=typing.cast( + typing.Any, + parse_obj_as( + type_=typing.Any, # type: ignore + object_=_response.json(), + ), + ), + ) + if _response.status_code == 401: + raise UnauthorizedError( + headers=dict(_response.headers), + body=typing.cast( + typing.Any, + parse_obj_as( + type_=typing.Any, # type: ignore + object_=_response.json(), + ), + ), + ) + if _response.status_code == 403: + raise AccessDeniedError( + headers=dict(_response.headers), + body=typing.cast( + typing.Any, + parse_obj_as( + type_=typing.Any, # type: ignore + object_=_response.json(), + ), + ), + ) + if _response.status_code == 405: + raise MethodNotAllowedError( + headers=dict(_response.headers), + body=typing.cast( + typing.Any, + parse_obj_as( + type_=typing.Any, # type: ignore + object_=_response.json(), + ), + ), + ) + if _response.status_code == 404: + raise NotFoundError( + headers=dict(_response.headers), + body=typing.cast( + typing.Any, + parse_obj_as( + type_=typing.Any, # type: ignore + object_=_response.json(), + ), + ), + ) + _response_json = _response.json() + except JSONDecodeError: + raise ApiError( + status_code=_response.status_code, + headers=dict(_response.headers), + body=_response.text, + ) + raise ApiError( + status_code=_response.status_code, + headers=dict(_response.headers), + body=_response_json, + ) + + def get( + self, + evaluator_id: str, + *, + request_options: typing.Optional[RequestOptions] = None, + ) -> HttpResponse[Evaluator]: + """ + Get one evaluator by its stable identifier. + + The response includes the evaluator's latest definition and version metadata flattened into the evaluator object, plus associated evaluation rules. Use the version-history endpoint when older definitions are needed. + + Parameters + ---------- + evaluator_id : str + Stable evaluator identifier returned by the evaluator endpoints. + + request_options : typing.Optional[RequestOptions] + Request-specific configuration. + + Returns + ------- + HttpResponse[Evaluator] + """ + _response = self._client_wrapper.httpx_client.request( + f"api/public/v2/evaluators/{jsonable_encoder(evaluator_id)}", + method="GET", + request_options=request_options, + ) + try: + if 200 <= _response.status_code < 300: + _data = typing.cast( + Evaluator, + parse_obj_as( + type_=Evaluator, # type: ignore + object_=_response.json(), + ), + ) + return HttpResponse(response=_response, data=_data) + if _response.status_code == 401: + raise EvaluationUnauthorizedError( + headers=dict(_response.headers), + body=typing.cast( + PublicApiError, + parse_obj_as( + type_=PublicApiError, # type: ignore + object_=_response.json(), + ), + ), + ) + if _response.status_code == 403: + raise EvaluationAccessDeniedError( + headers=dict(_response.headers), + body=typing.cast( + PublicApiError, + parse_obj_as( + type_=PublicApiError, # type: ignore + object_=_response.json(), + ), + ), + ) + if _response.status_code == 404: + raise EvaluationNotFoundError( + headers=dict(_response.headers), + body=typing.cast( + PublicApiError, + parse_obj_as( + type_=PublicApiError, # type: ignore + object_=_response.json(), + ), + ), + ) + if _response.status_code == 405: + raise EvaluationMethodNotAllowedError( + headers=dict(_response.headers), + body=typing.cast( + PublicApiError, + parse_obj_as( + type_=PublicApiError, # type: ignore + object_=_response.json(), + ), + ), + ) + if _response.status_code == 429: + raise TooManyRequestsError( + headers=dict(_response.headers), + body=typing.cast( + PublicApiError, + parse_obj_as( + type_=PublicApiError, # type: ignore + object_=_response.json(), + ), + ), + ) + if _response.status_code == 500: + raise InternalServerError( + headers=dict(_response.headers), + body=typing.cast( + PublicApiError, + parse_obj_as( + type_=PublicApiError, # type: ignore + object_=_response.json(), + ), + ), + ) + if _response.status_code == 400: + raise Error( + headers=dict(_response.headers), + body=typing.cast( + typing.Any, + parse_obj_as( + type_=typing.Any, # type: ignore + object_=_response.json(), + ), + ), + ) + if _response.status_code == 401: + raise UnauthorizedError( + headers=dict(_response.headers), + body=typing.cast( + typing.Any, + parse_obj_as( + type_=typing.Any, # type: ignore + object_=_response.json(), + ), + ), + ) + if _response.status_code == 403: + raise AccessDeniedError( + headers=dict(_response.headers), + body=typing.cast( + typing.Any, + parse_obj_as( + type_=typing.Any, # type: ignore + object_=_response.json(), + ), + ), + ) + if _response.status_code == 405: + raise MethodNotAllowedError( + headers=dict(_response.headers), + body=typing.cast( + typing.Any, + parse_obj_as( + type_=typing.Any, # type: ignore + object_=_response.json(), + ), + ), + ) + if _response.status_code == 404: + raise NotFoundError( + headers=dict(_response.headers), + body=typing.cast( + typing.Any, + parse_obj_as( + type_=typing.Any, # type: ignore + object_=_response.json(), + ), + ), + ) + _response_json = _response.json() + except JSONDecodeError: + raise ApiError( + status_code=_response.status_code, + headers=dict(_response.headers), + body=_response.text, + ) + raise ApiError( + status_code=_response.status_code, + headers=dict(_response.headers), + body=_response_json, + ) + + def update( + self, + evaluator_id: str, + *, + request: UpdateEvaluatorRequest, + request_options: typing.Optional[RequestOptions] = None, + ) -> HttpResponse[Evaluator]: + """ + Update an evaluator by its stable identifier. + + Provide only the top-level fields to change. Metadata-only changes do not create a version. Evaluator type cannot change. + + Definition fields are flattened into the request. To replace a definition, include `type` and every definition field for that type. Definition fields are replaced as a complete unit rather than merged. For LLM-as-a-judge evaluators, omitting or setting `modelConfig` to `null` selects the project's default evaluation model. + + Replacing a definition automatically returns an evaluator paused by an invalid or missing model configuration to `active`, while preserving its id and evaluation-rule assignments. Pauses caused by provider authentication, billing, connectivity, account state, or an unknown legacy reason remain paused until the explicit reactivation check succeeds. + + Parameters + ---------- + evaluator_id : str + Stable evaluator identifier returned by the evaluator endpoints. + + request : UpdateEvaluatorRequest + + request_options : typing.Optional[RequestOptions] + Request-specific configuration. + + Returns + ------- + HttpResponse[Evaluator] + """ + _response = self._client_wrapper.httpx_client.request( + f"api/public/v2/evaluators/{jsonable_encoder(evaluator_id)}", + method="PATCH", + json=convert_and_respect_annotation_metadata( + object_=request, annotation=UpdateEvaluatorRequest, direction="write" + ), + request_options=request_options, + omit=OMIT, + ) + try: + if 200 <= _response.status_code < 300: + _data = typing.cast( + Evaluator, + parse_obj_as( + type_=Evaluator, # type: ignore + object_=_response.json(), + ), + ) + return HttpResponse(response=_response, data=_data) + if _response.status_code == 400: + raise BadRequestError( + headers=dict(_response.headers), + body=typing.cast( + PublicApiError, + parse_obj_as( + type_=PublicApiError, # type: ignore + object_=_response.json(), + ), + ), + ) + if _response.status_code == 401: + raise EvaluationUnauthorizedError( + headers=dict(_response.headers), + body=typing.cast( + PublicApiError, + parse_obj_as( + type_=PublicApiError, # type: ignore + object_=_response.json(), + ), + ), + ) + if _response.status_code == 403: + raise EvaluationAccessDeniedError( + headers=dict(_response.headers), + body=typing.cast( + PublicApiError, + parse_obj_as( + type_=PublicApiError, # type: ignore + object_=_response.json(), + ), + ), + ) + if _response.status_code == 404: + raise EvaluationNotFoundError( + headers=dict(_response.headers), + body=typing.cast( + PublicApiError, + parse_obj_as( + type_=PublicApiError, # type: ignore + object_=_response.json(), + ), + ), + ) + if _response.status_code == 405: + raise EvaluationMethodNotAllowedError( + headers=dict(_response.headers), + body=typing.cast( + PublicApiError, + parse_obj_as( + type_=PublicApiError, # type: ignore + object_=_response.json(), + ), + ), + ) + if _response.status_code == 409: + raise ConflictError( + headers=dict(_response.headers), + body=typing.cast( + PublicApiError, + parse_obj_as( + type_=PublicApiError, # type: ignore + object_=_response.json(), + ), + ), + ) + if _response.status_code == 412: + raise PreconditionFailedError( + headers=dict(_response.headers), + body=typing.cast( + PublicApiError, + parse_obj_as( + type_=PublicApiError, # type: ignore + object_=_response.json(), + ), + ), + ) + if _response.status_code == 429: + raise TooManyRequestsError( + headers=dict(_response.headers), + body=typing.cast( + PublicApiError, + parse_obj_as( + type_=PublicApiError, # type: ignore + object_=_response.json(), + ), + ), + ) + if _response.status_code == 500: + raise InternalServerError( + headers=dict(_response.headers), + body=typing.cast( + PublicApiError, + parse_obj_as( + type_=PublicApiError, # type: ignore + object_=_response.json(), + ), + ), + ) + if _response.status_code == 400: + raise Error( + headers=dict(_response.headers), + body=typing.cast( + typing.Any, + parse_obj_as( + type_=typing.Any, # type: ignore + object_=_response.json(), + ), + ), + ) + if _response.status_code == 401: + raise UnauthorizedError( + headers=dict(_response.headers), + body=typing.cast( + typing.Any, + parse_obj_as( + type_=typing.Any, # type: ignore + object_=_response.json(), + ), + ), + ) + if _response.status_code == 403: + raise AccessDeniedError( + headers=dict(_response.headers), + body=typing.cast( + typing.Any, + parse_obj_as( + type_=typing.Any, # type: ignore + object_=_response.json(), + ), + ), + ) + if _response.status_code == 405: + raise MethodNotAllowedError( + headers=dict(_response.headers), + body=typing.cast( + typing.Any, + parse_obj_as( + type_=typing.Any, # type: ignore + object_=_response.json(), + ), + ), + ) + if _response.status_code == 404: + raise NotFoundError( + headers=dict(_response.headers), + body=typing.cast( + typing.Any, + parse_obj_as( + type_=typing.Any, # type: ignore + object_=_response.json(), + ), + ), + ) + _response_json = _response.json() + except JSONDecodeError: + raise ApiError( + status_code=_response.status_code, + headers=dict(_response.headers), + body=_response.text, + ) + raise ApiError( + status_code=_response.status_code, + headers=dict(_response.headers), + body=_response_json, + ) + + def delete( + self, + evaluator_id: str, + *, + request_options: typing.Optional[RequestOptions] = None, + ) -> HttpResponse[DeletedEvaluator]: + """ + Delete an evaluator and all of its stored versions. + + Associated evaluation-rule assignments are also removed. Scores already produced by the evaluator are preserved. + + Parameters + ---------- + evaluator_id : str + Stable evaluator identifier returned by the evaluator endpoints. + + request_options : typing.Optional[RequestOptions] + Request-specific configuration. + + Returns + ------- + HttpResponse[DeletedEvaluator] + """ + _response = self._client_wrapper.httpx_client.request( + f"api/public/v2/evaluators/{jsonable_encoder(evaluator_id)}", + method="DELETE", + request_options=request_options, + ) + try: + if 200 <= _response.status_code < 300: + _data = typing.cast( + DeletedEvaluator, + parse_obj_as( + type_=DeletedEvaluator, # type: ignore + object_=_response.json(), + ), + ) + return HttpResponse(response=_response, data=_data) + if _response.status_code == 401: + raise EvaluationUnauthorizedError( + headers=dict(_response.headers), + body=typing.cast( + PublicApiError, + parse_obj_as( + type_=PublicApiError, # type: ignore + object_=_response.json(), + ), + ), + ) + if _response.status_code == 403: + raise EvaluationAccessDeniedError( + headers=dict(_response.headers), + body=typing.cast( + PublicApiError, + parse_obj_as( + type_=PublicApiError, # type: ignore + object_=_response.json(), + ), + ), + ) + if _response.status_code == 404: + raise EvaluationNotFoundError( + headers=dict(_response.headers), + body=typing.cast( + PublicApiError, + parse_obj_as( + type_=PublicApiError, # type: ignore + object_=_response.json(), + ), + ), + ) + if _response.status_code == 405: + raise EvaluationMethodNotAllowedError( + headers=dict(_response.headers), + body=typing.cast( + PublicApiError, + parse_obj_as( + type_=PublicApiError, # type: ignore + object_=_response.json(), + ), + ), + ) + if _response.status_code == 429: + raise TooManyRequestsError( + headers=dict(_response.headers), + body=typing.cast( + PublicApiError, + parse_obj_as( + type_=PublicApiError, # type: ignore + object_=_response.json(), + ), + ), + ) + if _response.status_code == 500: + raise InternalServerError( + headers=dict(_response.headers), + body=typing.cast( + PublicApiError, + parse_obj_as( + type_=PublicApiError, # type: ignore + object_=_response.json(), + ), + ), + ) + if _response.status_code == 400: + raise Error( + headers=dict(_response.headers), + body=typing.cast( + typing.Any, + parse_obj_as( + type_=typing.Any, # type: ignore + object_=_response.json(), + ), + ), + ) + if _response.status_code == 401: + raise UnauthorizedError( + headers=dict(_response.headers), + body=typing.cast( + typing.Any, + parse_obj_as( + type_=typing.Any, # type: ignore + object_=_response.json(), + ), + ), + ) + if _response.status_code == 403: + raise AccessDeniedError( + headers=dict(_response.headers), + body=typing.cast( + typing.Any, + parse_obj_as( + type_=typing.Any, # type: ignore + object_=_response.json(), + ), + ), + ) + if _response.status_code == 405: + raise MethodNotAllowedError( + headers=dict(_response.headers), + body=typing.cast( + typing.Any, + parse_obj_as( + type_=typing.Any, # type: ignore + object_=_response.json(), + ), + ), + ) + if _response.status_code == 404: + raise NotFoundError( + headers=dict(_response.headers), + body=typing.cast( + typing.Any, + parse_obj_as( + type_=typing.Any, # type: ignore + object_=_response.json(), + ), + ), + ) + _response_json = _response.json() + except JSONDecodeError: + raise ApiError( + status_code=_response.status_code, + headers=dict(_response.headers), + body=_response.text, + ) + raise ApiError( + status_code=_response.status_code, + headers=dict(_response.headers), + body=_response_json, + ) + + def list_versions( + self, + evaluator_id: str, + *, + limit: typing.Optional[int] = None, + cursor: typing.Optional[str] = None, + request_options: typing.Optional[RequestOptions] = None, + ) -> HttpResponse[EvaluatorVersionsPage]: + """ + List an evaluator's version history in newest-first order. + + This endpoint is intended for history and audit use cases. Ordinary clients can use the flattened `version` and definition fields on the evaluator response. + + Parameters + ---------- + evaluator_id : str + Stable evaluator identifier returned by the evaluator endpoints. + + limit : typing.Optional[int] + Maximum number of versions to return. Defaults to `50` and cannot exceed `100`. + + cursor : typing.Optional[str] + Opaque cursor returned by the previous page. + + request_options : typing.Optional[RequestOptions] + Request-specific configuration. + + Returns + ------- + HttpResponse[EvaluatorVersionsPage] + """ + _response = self._client_wrapper.httpx_client.request( + f"api/public/v2/evaluators/{jsonable_encoder(evaluator_id)}/versions", + method="GET", + params={ + "limit": limit, + "cursor": cursor, + }, + request_options=request_options, + ) + try: + if 200 <= _response.status_code < 300: + _data = typing.cast( + EvaluatorVersionsPage, + parse_obj_as( + type_=EvaluatorVersionsPage, # type: ignore + object_=_response.json(), + ), + ) + return HttpResponse(response=_response, data=_data) + if _response.status_code == 400: + raise BadRequestError( + headers=dict(_response.headers), + body=typing.cast( + PublicApiError, + parse_obj_as( + type_=PublicApiError, # type: ignore + object_=_response.json(), + ), + ), + ) + if _response.status_code == 401: + raise EvaluationUnauthorizedError( + headers=dict(_response.headers), + body=typing.cast( + PublicApiError, + parse_obj_as( + type_=PublicApiError, # type: ignore + object_=_response.json(), + ), + ), + ) + if _response.status_code == 403: + raise EvaluationAccessDeniedError( + headers=dict(_response.headers), + body=typing.cast( + PublicApiError, + parse_obj_as( + type_=PublicApiError, # type: ignore + object_=_response.json(), + ), + ), + ) + if _response.status_code == 404: + raise EvaluationNotFoundError( + headers=dict(_response.headers), + body=typing.cast( + PublicApiError, + parse_obj_as( + type_=PublicApiError, # type: ignore + object_=_response.json(), + ), + ), + ) + if _response.status_code == 405: + raise EvaluationMethodNotAllowedError( + headers=dict(_response.headers), + body=typing.cast( + PublicApiError, + parse_obj_as( + type_=PublicApiError, # type: ignore + object_=_response.json(), + ), + ), + ) + if _response.status_code == 429: + raise TooManyRequestsError( + headers=dict(_response.headers), + body=typing.cast( + PublicApiError, + parse_obj_as( + type_=PublicApiError, # type: ignore + object_=_response.json(), + ), + ), + ) + if _response.status_code == 500: + raise InternalServerError( + headers=dict(_response.headers), + body=typing.cast( + PublicApiError, + parse_obj_as( + type_=PublicApiError, # type: ignore + object_=_response.json(), + ), + ), + ) + if _response.status_code == 400: + raise Error( + headers=dict(_response.headers), + body=typing.cast( + typing.Any, + parse_obj_as( + type_=typing.Any, # type: ignore + object_=_response.json(), + ), + ), + ) + if _response.status_code == 401: + raise UnauthorizedError( + headers=dict(_response.headers), + body=typing.cast( + typing.Any, + parse_obj_as( + type_=typing.Any, # type: ignore + object_=_response.json(), + ), + ), + ) + if _response.status_code == 403: + raise AccessDeniedError( + headers=dict(_response.headers), + body=typing.cast( + typing.Any, + parse_obj_as( + type_=typing.Any, # type: ignore + object_=_response.json(), + ), + ), + ) + if _response.status_code == 405: + raise MethodNotAllowedError( + headers=dict(_response.headers), + body=typing.cast( + typing.Any, + parse_obj_as( + type_=typing.Any, # type: ignore + object_=_response.json(), + ), + ), + ) + if _response.status_code == 404: + raise NotFoundError( + headers=dict(_response.headers), + body=typing.cast( + typing.Any, + parse_obj_as( + type_=typing.Any, # type: ignore + object_=_response.json(), + ), + ), + ) + _response_json = _response.json() + except JSONDecodeError: + raise ApiError( + status_code=_response.status_code, + headers=dict(_response.headers), + body=_response.text, + ) + raise ApiError( + status_code=_response.status_code, + headers=dict(_response.headers), + body=_response_json, + ) + + +class AsyncRawEvaluatorsClient: + def __init__(self, *, client_wrapper: AsyncClientWrapper): + self._client_wrapper = client_wrapper + + async def create( + self, + *, + request: CreateEvaluatorRequest, + request_options: typing.Optional[RequestOptions] = None, + ) -> AsyncHttpResponse[Evaluator]: + """ + Create an evaluator in the authenticated project. + + An evaluator defines **how** Langfuse should score data. LLM-as-a-judge evaluators define a prompt, expected structured output, and optional model configuration. Code evaluators define source code and a runtime language. + + This always creates a new evaluator with version `1`. Names are not identifiers and do not need to be unique. + + Parameters + ---------- + request : CreateEvaluatorRequest + + request_options : typing.Optional[RequestOptions] + Request-specific configuration. + + Returns + ------- + AsyncHttpResponse[Evaluator] + """ + _response = await self._client_wrapper.httpx_client.request( + "api/public/v2/evaluators", + method="POST", + json=convert_and_respect_annotation_metadata( + object_=request, annotation=CreateEvaluatorRequest, direction="write" + ), + request_options=request_options, + omit=OMIT, + ) + try: + if 200 <= _response.status_code < 300: + _data = typing.cast( + Evaluator, + parse_obj_as( + type_=Evaluator, # type: ignore + object_=_response.json(), + ), + ) + return AsyncHttpResponse(response=_response, data=_data) + if _response.status_code == 400: + raise BadRequestError( + headers=dict(_response.headers), + body=typing.cast( + PublicApiError, + parse_obj_as( + type_=PublicApiError, # type: ignore + object_=_response.json(), + ), + ), + ) + if _response.status_code == 401: + raise EvaluationUnauthorizedError( + headers=dict(_response.headers), + body=typing.cast( + PublicApiError, + parse_obj_as( + type_=PublicApiError, # type: ignore + object_=_response.json(), + ), + ), + ) + if _response.status_code == 403: + raise EvaluationAccessDeniedError( + headers=dict(_response.headers), + body=typing.cast( + PublicApiError, + parse_obj_as( + type_=PublicApiError, # type: ignore + object_=_response.json(), + ), + ), + ) + if _response.status_code == 405: + raise EvaluationMethodNotAllowedError( + headers=dict(_response.headers), + body=typing.cast( + PublicApiError, + parse_obj_as( + type_=PublicApiError, # type: ignore + object_=_response.json(), + ), + ), + ) + if _response.status_code == 412: + raise PreconditionFailedError( + headers=dict(_response.headers), + body=typing.cast( + PublicApiError, + parse_obj_as( + type_=PublicApiError, # type: ignore + object_=_response.json(), + ), + ), + ) + if _response.status_code == 429: + raise TooManyRequestsError( + headers=dict(_response.headers), + body=typing.cast( + PublicApiError, + parse_obj_as( + type_=PublicApiError, # type: ignore + object_=_response.json(), + ), + ), + ) + if _response.status_code == 500: + raise InternalServerError( + headers=dict(_response.headers), + body=typing.cast( + PublicApiError, + parse_obj_as( + type_=PublicApiError, # type: ignore + object_=_response.json(), + ), + ), + ) + if _response.status_code == 400: + raise Error( + headers=dict(_response.headers), + body=typing.cast( + typing.Any, + parse_obj_as( + type_=typing.Any, # type: ignore + object_=_response.json(), + ), + ), + ) + if _response.status_code == 401: + raise UnauthorizedError( + headers=dict(_response.headers), + body=typing.cast( + typing.Any, + parse_obj_as( + type_=typing.Any, # type: ignore + object_=_response.json(), + ), + ), + ) + if _response.status_code == 403: + raise AccessDeniedError( + headers=dict(_response.headers), + body=typing.cast( + typing.Any, + parse_obj_as( + type_=typing.Any, # type: ignore + object_=_response.json(), + ), + ), + ) + if _response.status_code == 405: + raise MethodNotAllowedError( + headers=dict(_response.headers), + body=typing.cast( + typing.Any, + parse_obj_as( + type_=typing.Any, # type: ignore + object_=_response.json(), + ), + ), + ) + if _response.status_code == 404: + raise NotFoundError( + headers=dict(_response.headers), + body=typing.cast( + typing.Any, + parse_obj_as( + type_=typing.Any, # type: ignore + object_=_response.json(), + ), + ), + ) + _response_json = _response.json() + except JSONDecodeError: + raise ApiError( + status_code=_response.status_code, + headers=dict(_response.headers), + body=_response.text, + ) + raise ApiError( + status_code=_response.status_code, + headers=dict(_response.headers), + body=_response_json, + ) + + async def list( + self, + *, + limit: typing.Optional[int] = None, + cursor: typing.Optional[str] = None, + request_options: typing.Optional[RequestOptions] = None, + ) -> AsyncHttpResponse[EvaluatorsPage]: + """ + List evaluators in newest-first creation order. + + Every evaluator includes its latest definition and version metadata flattened into the evaluator object, plus associated evaluation rules. Treat the cursor as opaque and return it unchanged. + + Parameters + ---------- + limit : typing.Optional[int] + Maximum number of items to return. Defaults to `50` and cannot exceed `100`. + + cursor : typing.Optional[str] + Opaque cursor returned by the previous page. + + request_options : typing.Optional[RequestOptions] + Request-specific configuration. + + Returns + ------- + AsyncHttpResponse[EvaluatorsPage] + """ + _response = await self._client_wrapper.httpx_client.request( + "api/public/v2/evaluators", + method="GET", + params={ + "limit": limit, + "cursor": cursor, + }, + request_options=request_options, + ) + try: + if 200 <= _response.status_code < 300: + _data = typing.cast( + EvaluatorsPage, + parse_obj_as( + type_=EvaluatorsPage, # type: ignore + object_=_response.json(), + ), + ) + return AsyncHttpResponse(response=_response, data=_data) + if _response.status_code == 400: + raise BadRequestError( + headers=dict(_response.headers), + body=typing.cast( + PublicApiError, + parse_obj_as( + type_=PublicApiError, # type: ignore + object_=_response.json(), + ), + ), + ) + if _response.status_code == 401: + raise EvaluationUnauthorizedError( + headers=dict(_response.headers), + body=typing.cast( + PublicApiError, + parse_obj_as( + type_=PublicApiError, # type: ignore + object_=_response.json(), + ), + ), + ) + if _response.status_code == 403: + raise EvaluationAccessDeniedError( + headers=dict(_response.headers), + body=typing.cast( + PublicApiError, + parse_obj_as( + type_=PublicApiError, # type: ignore + object_=_response.json(), + ), + ), + ) + if _response.status_code == 405: + raise EvaluationMethodNotAllowedError( + headers=dict(_response.headers), + body=typing.cast( + PublicApiError, + parse_obj_as( + type_=PublicApiError, # type: ignore + object_=_response.json(), + ), + ), + ) + if _response.status_code == 429: + raise TooManyRequestsError( + headers=dict(_response.headers), + body=typing.cast( + PublicApiError, + parse_obj_as( + type_=PublicApiError, # type: ignore + object_=_response.json(), + ), + ), + ) + if _response.status_code == 500: + raise InternalServerError( + headers=dict(_response.headers), + body=typing.cast( + PublicApiError, + parse_obj_as( + type_=PublicApiError, # type: ignore + object_=_response.json(), + ), + ), + ) + if _response.status_code == 400: + raise Error( + headers=dict(_response.headers), + body=typing.cast( + typing.Any, + parse_obj_as( + type_=typing.Any, # type: ignore + object_=_response.json(), + ), + ), + ) + if _response.status_code == 401: + raise UnauthorizedError( + headers=dict(_response.headers), + body=typing.cast( + typing.Any, + parse_obj_as( + type_=typing.Any, # type: ignore + object_=_response.json(), + ), + ), + ) + if _response.status_code == 403: + raise AccessDeniedError( + headers=dict(_response.headers), + body=typing.cast( + typing.Any, + parse_obj_as( + type_=typing.Any, # type: ignore + object_=_response.json(), + ), + ), + ) + if _response.status_code == 405: + raise MethodNotAllowedError( + headers=dict(_response.headers), + body=typing.cast( + typing.Any, + parse_obj_as( + type_=typing.Any, # type: ignore + object_=_response.json(), + ), + ), + ) + if _response.status_code == 404: + raise NotFoundError( + headers=dict(_response.headers), + body=typing.cast( + typing.Any, + parse_obj_as( + type_=typing.Any, # type: ignore + object_=_response.json(), + ), + ), + ) + _response_json = _response.json() + except JSONDecodeError: + raise ApiError( + status_code=_response.status_code, + headers=dict(_response.headers), + body=_response.text, + ) + raise ApiError( + status_code=_response.status_code, + headers=dict(_response.headers), + body=_response_json, + ) + + async def get( + self, + evaluator_id: str, + *, + request_options: typing.Optional[RequestOptions] = None, + ) -> AsyncHttpResponse[Evaluator]: + """ + Get one evaluator by its stable identifier. + + The response includes the evaluator's latest definition and version metadata flattened into the evaluator object, plus associated evaluation rules. Use the version-history endpoint when older definitions are needed. + + Parameters + ---------- + evaluator_id : str + Stable evaluator identifier returned by the evaluator endpoints. + + request_options : typing.Optional[RequestOptions] + Request-specific configuration. + + Returns + ------- + AsyncHttpResponse[Evaluator] + """ + _response = await self._client_wrapper.httpx_client.request( + f"api/public/v2/evaluators/{jsonable_encoder(evaluator_id)}", + method="GET", + request_options=request_options, + ) + try: + if 200 <= _response.status_code < 300: + _data = typing.cast( + Evaluator, + parse_obj_as( + type_=Evaluator, # type: ignore + object_=_response.json(), + ), + ) + return AsyncHttpResponse(response=_response, data=_data) + if _response.status_code == 401: + raise EvaluationUnauthorizedError( + headers=dict(_response.headers), + body=typing.cast( + PublicApiError, + parse_obj_as( + type_=PublicApiError, # type: ignore + object_=_response.json(), + ), + ), + ) + if _response.status_code == 403: + raise EvaluationAccessDeniedError( + headers=dict(_response.headers), + body=typing.cast( + PublicApiError, + parse_obj_as( + type_=PublicApiError, # type: ignore + object_=_response.json(), + ), + ), + ) + if _response.status_code == 404: + raise EvaluationNotFoundError( + headers=dict(_response.headers), + body=typing.cast( + PublicApiError, + parse_obj_as( + type_=PublicApiError, # type: ignore + object_=_response.json(), + ), + ), + ) + if _response.status_code == 405: + raise EvaluationMethodNotAllowedError( + headers=dict(_response.headers), + body=typing.cast( + PublicApiError, + parse_obj_as( + type_=PublicApiError, # type: ignore + object_=_response.json(), + ), + ), + ) + if _response.status_code == 429: + raise TooManyRequestsError( + headers=dict(_response.headers), + body=typing.cast( + PublicApiError, + parse_obj_as( + type_=PublicApiError, # type: ignore + object_=_response.json(), + ), + ), + ) + if _response.status_code == 500: + raise InternalServerError( + headers=dict(_response.headers), + body=typing.cast( + PublicApiError, + parse_obj_as( + type_=PublicApiError, # type: ignore + object_=_response.json(), + ), + ), + ) + if _response.status_code == 400: + raise Error( + headers=dict(_response.headers), + body=typing.cast( + typing.Any, + parse_obj_as( + type_=typing.Any, # type: ignore + object_=_response.json(), + ), + ), + ) + if _response.status_code == 401: + raise UnauthorizedError( + headers=dict(_response.headers), + body=typing.cast( + typing.Any, + parse_obj_as( + type_=typing.Any, # type: ignore + object_=_response.json(), + ), + ), + ) + if _response.status_code == 403: + raise AccessDeniedError( + headers=dict(_response.headers), + body=typing.cast( + typing.Any, + parse_obj_as( + type_=typing.Any, # type: ignore + object_=_response.json(), + ), + ), + ) + if _response.status_code == 405: + raise MethodNotAllowedError( + headers=dict(_response.headers), + body=typing.cast( + typing.Any, + parse_obj_as( + type_=typing.Any, # type: ignore + object_=_response.json(), + ), + ), + ) + if _response.status_code == 404: + raise NotFoundError( + headers=dict(_response.headers), + body=typing.cast( + typing.Any, + parse_obj_as( + type_=typing.Any, # type: ignore + object_=_response.json(), + ), + ), + ) + _response_json = _response.json() + except JSONDecodeError: + raise ApiError( + status_code=_response.status_code, + headers=dict(_response.headers), + body=_response.text, + ) + raise ApiError( + status_code=_response.status_code, + headers=dict(_response.headers), + body=_response_json, + ) + + async def update( + self, + evaluator_id: str, + *, + request: UpdateEvaluatorRequest, + request_options: typing.Optional[RequestOptions] = None, + ) -> AsyncHttpResponse[Evaluator]: + """ + Update an evaluator by its stable identifier. + + Provide only the top-level fields to change. Metadata-only changes do not create a version. Evaluator type cannot change. + + Definition fields are flattened into the request. To replace a definition, include `type` and every definition field for that type. Definition fields are replaced as a complete unit rather than merged. For LLM-as-a-judge evaluators, omitting or setting `modelConfig` to `null` selects the project's default evaluation model. + + Replacing a definition automatically returns an evaluator paused by an invalid or missing model configuration to `active`, while preserving its id and evaluation-rule assignments. Pauses caused by provider authentication, billing, connectivity, account state, or an unknown legacy reason remain paused until the explicit reactivation check succeeds. + + Parameters + ---------- + evaluator_id : str + Stable evaluator identifier returned by the evaluator endpoints. + + request : UpdateEvaluatorRequest + + request_options : typing.Optional[RequestOptions] + Request-specific configuration. + + Returns + ------- + AsyncHttpResponse[Evaluator] + """ + _response = await self._client_wrapper.httpx_client.request( + f"api/public/v2/evaluators/{jsonable_encoder(evaluator_id)}", + method="PATCH", + json=convert_and_respect_annotation_metadata( + object_=request, annotation=UpdateEvaluatorRequest, direction="write" + ), + request_options=request_options, + omit=OMIT, + ) + try: + if 200 <= _response.status_code < 300: + _data = typing.cast( + Evaluator, + parse_obj_as( + type_=Evaluator, # type: ignore + object_=_response.json(), + ), + ) + return AsyncHttpResponse(response=_response, data=_data) + if _response.status_code == 400: + raise BadRequestError( + headers=dict(_response.headers), + body=typing.cast( + PublicApiError, + parse_obj_as( + type_=PublicApiError, # type: ignore + object_=_response.json(), + ), + ), + ) + if _response.status_code == 401: + raise EvaluationUnauthorizedError( + headers=dict(_response.headers), + body=typing.cast( + PublicApiError, + parse_obj_as( + type_=PublicApiError, # type: ignore + object_=_response.json(), + ), + ), + ) + if _response.status_code == 403: + raise EvaluationAccessDeniedError( + headers=dict(_response.headers), + body=typing.cast( + PublicApiError, + parse_obj_as( + type_=PublicApiError, # type: ignore + object_=_response.json(), + ), + ), + ) + if _response.status_code == 404: + raise EvaluationNotFoundError( + headers=dict(_response.headers), + body=typing.cast( + PublicApiError, + parse_obj_as( + type_=PublicApiError, # type: ignore + object_=_response.json(), + ), + ), + ) + if _response.status_code == 405: + raise EvaluationMethodNotAllowedError( + headers=dict(_response.headers), + body=typing.cast( + PublicApiError, + parse_obj_as( + type_=PublicApiError, # type: ignore + object_=_response.json(), + ), + ), + ) + if _response.status_code == 409: + raise ConflictError( + headers=dict(_response.headers), + body=typing.cast( + PublicApiError, + parse_obj_as( + type_=PublicApiError, # type: ignore + object_=_response.json(), + ), + ), + ) + if _response.status_code == 412: + raise PreconditionFailedError( + headers=dict(_response.headers), + body=typing.cast( + PublicApiError, + parse_obj_as( + type_=PublicApiError, # type: ignore + object_=_response.json(), + ), + ), + ) + if _response.status_code == 429: + raise TooManyRequestsError( + headers=dict(_response.headers), + body=typing.cast( + PublicApiError, + parse_obj_as( + type_=PublicApiError, # type: ignore + object_=_response.json(), + ), + ), + ) + if _response.status_code == 500: + raise InternalServerError( + headers=dict(_response.headers), + body=typing.cast( + PublicApiError, + parse_obj_as( + type_=PublicApiError, # type: ignore + object_=_response.json(), + ), + ), + ) + if _response.status_code == 400: + raise Error( + headers=dict(_response.headers), + body=typing.cast( + typing.Any, + parse_obj_as( + type_=typing.Any, # type: ignore + object_=_response.json(), + ), + ), + ) + if _response.status_code == 401: + raise UnauthorizedError( + headers=dict(_response.headers), + body=typing.cast( + typing.Any, + parse_obj_as( + type_=typing.Any, # type: ignore + object_=_response.json(), + ), + ), + ) + if _response.status_code == 403: + raise AccessDeniedError( + headers=dict(_response.headers), + body=typing.cast( + typing.Any, + parse_obj_as( + type_=typing.Any, # type: ignore + object_=_response.json(), + ), + ), + ) + if _response.status_code == 405: + raise MethodNotAllowedError( + headers=dict(_response.headers), + body=typing.cast( + typing.Any, + parse_obj_as( + type_=typing.Any, # type: ignore + object_=_response.json(), + ), + ), + ) + if _response.status_code == 404: + raise NotFoundError( + headers=dict(_response.headers), + body=typing.cast( + typing.Any, + parse_obj_as( + type_=typing.Any, # type: ignore + object_=_response.json(), + ), + ), + ) + _response_json = _response.json() + except JSONDecodeError: + raise ApiError( + status_code=_response.status_code, + headers=dict(_response.headers), + body=_response.text, + ) + raise ApiError( + status_code=_response.status_code, + headers=dict(_response.headers), + body=_response_json, + ) + + async def delete( + self, + evaluator_id: str, + *, + request_options: typing.Optional[RequestOptions] = None, + ) -> AsyncHttpResponse[DeletedEvaluator]: + """ + Delete an evaluator and all of its stored versions. + + Associated evaluation-rule assignments are also removed. Scores already produced by the evaluator are preserved. + + Parameters + ---------- + evaluator_id : str + Stable evaluator identifier returned by the evaluator endpoints. + + request_options : typing.Optional[RequestOptions] + Request-specific configuration. + + Returns + ------- + AsyncHttpResponse[DeletedEvaluator] + """ + _response = await self._client_wrapper.httpx_client.request( + f"api/public/v2/evaluators/{jsonable_encoder(evaluator_id)}", + method="DELETE", + request_options=request_options, + ) + try: + if 200 <= _response.status_code < 300: + _data = typing.cast( + DeletedEvaluator, + parse_obj_as( + type_=DeletedEvaluator, # type: ignore + object_=_response.json(), + ), + ) + return AsyncHttpResponse(response=_response, data=_data) + if _response.status_code == 401: + raise EvaluationUnauthorizedError( + headers=dict(_response.headers), + body=typing.cast( + PublicApiError, + parse_obj_as( + type_=PublicApiError, # type: ignore + object_=_response.json(), + ), + ), + ) + if _response.status_code == 403: + raise EvaluationAccessDeniedError( + headers=dict(_response.headers), + body=typing.cast( + PublicApiError, + parse_obj_as( + type_=PublicApiError, # type: ignore + object_=_response.json(), + ), + ), + ) + if _response.status_code == 404: + raise EvaluationNotFoundError( + headers=dict(_response.headers), + body=typing.cast( + PublicApiError, + parse_obj_as( + type_=PublicApiError, # type: ignore + object_=_response.json(), + ), + ), + ) + if _response.status_code == 405: + raise EvaluationMethodNotAllowedError( + headers=dict(_response.headers), + body=typing.cast( + PublicApiError, + parse_obj_as( + type_=PublicApiError, # type: ignore + object_=_response.json(), + ), + ), + ) + if _response.status_code == 429: + raise TooManyRequestsError( + headers=dict(_response.headers), + body=typing.cast( + PublicApiError, + parse_obj_as( + type_=PublicApiError, # type: ignore + object_=_response.json(), + ), + ), + ) + if _response.status_code == 500: + raise InternalServerError( + headers=dict(_response.headers), + body=typing.cast( + PublicApiError, + parse_obj_as( + type_=PublicApiError, # type: ignore + object_=_response.json(), + ), + ), + ) + if _response.status_code == 400: + raise Error( + headers=dict(_response.headers), + body=typing.cast( + typing.Any, + parse_obj_as( + type_=typing.Any, # type: ignore + object_=_response.json(), + ), + ), + ) + if _response.status_code == 401: + raise UnauthorizedError( + headers=dict(_response.headers), + body=typing.cast( + typing.Any, + parse_obj_as( + type_=typing.Any, # type: ignore + object_=_response.json(), + ), + ), + ) + if _response.status_code == 403: + raise AccessDeniedError( + headers=dict(_response.headers), + body=typing.cast( + typing.Any, + parse_obj_as( + type_=typing.Any, # type: ignore + object_=_response.json(), + ), + ), + ) + if _response.status_code == 405: + raise MethodNotAllowedError( + headers=dict(_response.headers), + body=typing.cast( + typing.Any, + parse_obj_as( + type_=typing.Any, # type: ignore + object_=_response.json(), + ), + ), + ) + if _response.status_code == 404: + raise NotFoundError( + headers=dict(_response.headers), + body=typing.cast( + typing.Any, + parse_obj_as( + type_=typing.Any, # type: ignore + object_=_response.json(), + ), + ), + ) + _response_json = _response.json() + except JSONDecodeError: + raise ApiError( + status_code=_response.status_code, + headers=dict(_response.headers), + body=_response.text, + ) + raise ApiError( + status_code=_response.status_code, + headers=dict(_response.headers), + body=_response_json, + ) + + async def list_versions( + self, + evaluator_id: str, + *, + limit: typing.Optional[int] = None, + cursor: typing.Optional[str] = None, + request_options: typing.Optional[RequestOptions] = None, + ) -> AsyncHttpResponse[EvaluatorVersionsPage]: + """ + List an evaluator's version history in newest-first order. + + This endpoint is intended for history and audit use cases. Ordinary clients can use the flattened `version` and definition fields on the evaluator response. + + Parameters + ---------- + evaluator_id : str + Stable evaluator identifier returned by the evaluator endpoints. + + limit : typing.Optional[int] + Maximum number of versions to return. Defaults to `50` and cannot exceed `100`. + + cursor : typing.Optional[str] + Opaque cursor returned by the previous page. + + request_options : typing.Optional[RequestOptions] + Request-specific configuration. + + Returns + ------- + AsyncHttpResponse[EvaluatorVersionsPage] + """ + _response = await self._client_wrapper.httpx_client.request( + f"api/public/v2/evaluators/{jsonable_encoder(evaluator_id)}/versions", + method="GET", + params={ + "limit": limit, + "cursor": cursor, + }, + request_options=request_options, + ) + try: + if 200 <= _response.status_code < 300: + _data = typing.cast( + EvaluatorVersionsPage, + parse_obj_as( + type_=EvaluatorVersionsPage, # type: ignore + object_=_response.json(), + ), + ) + return AsyncHttpResponse(response=_response, data=_data) + if _response.status_code == 400: + raise BadRequestError( + headers=dict(_response.headers), + body=typing.cast( + PublicApiError, + parse_obj_as( + type_=PublicApiError, # type: ignore + object_=_response.json(), + ), + ), + ) + if _response.status_code == 401: + raise EvaluationUnauthorizedError( + headers=dict(_response.headers), + body=typing.cast( + PublicApiError, + parse_obj_as( + type_=PublicApiError, # type: ignore + object_=_response.json(), + ), + ), + ) + if _response.status_code == 403: + raise EvaluationAccessDeniedError( + headers=dict(_response.headers), + body=typing.cast( + PublicApiError, + parse_obj_as( + type_=PublicApiError, # type: ignore + object_=_response.json(), + ), + ), + ) + if _response.status_code == 404: + raise EvaluationNotFoundError( + headers=dict(_response.headers), + body=typing.cast( + PublicApiError, + parse_obj_as( + type_=PublicApiError, # type: ignore + object_=_response.json(), + ), + ), + ) + if _response.status_code == 405: + raise EvaluationMethodNotAllowedError( + headers=dict(_response.headers), + body=typing.cast( + PublicApiError, + parse_obj_as( + type_=PublicApiError, # type: ignore + object_=_response.json(), + ), + ), + ) + if _response.status_code == 429: + raise TooManyRequestsError( + headers=dict(_response.headers), + body=typing.cast( + PublicApiError, + parse_obj_as( + type_=PublicApiError, # type: ignore + object_=_response.json(), + ), + ), + ) + if _response.status_code == 500: + raise InternalServerError( + headers=dict(_response.headers), + body=typing.cast( + PublicApiError, + parse_obj_as( + type_=PublicApiError, # type: ignore + object_=_response.json(), + ), + ), + ) + if _response.status_code == 400: + raise Error( + headers=dict(_response.headers), + body=typing.cast( + typing.Any, + parse_obj_as( + type_=typing.Any, # type: ignore + object_=_response.json(), + ), + ), + ) + if _response.status_code == 401: + raise UnauthorizedError( + headers=dict(_response.headers), + body=typing.cast( + typing.Any, + parse_obj_as( + type_=typing.Any, # type: ignore + object_=_response.json(), + ), + ), + ) + if _response.status_code == 403: + raise AccessDeniedError( + headers=dict(_response.headers), + body=typing.cast( + typing.Any, + parse_obj_as( + type_=typing.Any, # type: ignore + object_=_response.json(), + ), + ), + ) + if _response.status_code == 405: + raise MethodNotAllowedError( + headers=dict(_response.headers), + body=typing.cast( + typing.Any, + parse_obj_as( + type_=typing.Any, # type: ignore + object_=_response.json(), + ), + ), + ) + if _response.status_code == 404: + raise NotFoundError( + headers=dict(_response.headers), + body=typing.cast( + typing.Any, + parse_obj_as( + type_=typing.Any, # type: ignore + object_=_response.json(), + ), + ), + ) + _response_json = _response.json() + except JSONDecodeError: + raise ApiError( + status_code=_response.status_code, + headers=dict(_response.headers), + body=_response.text, + ) + raise ApiError( + status_code=_response.status_code, + headers=dict(_response.headers), + body=_response_json, + ) diff --git a/langfuse/api/evaluators/types/__init__.py b/langfuse/api/evaluators/types/__init__.py new file mode 100644 index 000000000..5f172d62a --- /dev/null +++ b/langfuse/api/evaluators/types/__init__.py @@ -0,0 +1,139 @@ +# This file was auto-generated by Fern from our API Definition. + +# isort: skip_file + +import typing +from importlib import import_module + +if typing.TYPE_CHECKING: + from .code_evaluator import CodeEvaluator + from .code_evaluator_version import CodeEvaluatorVersion + from .create_code_evaluator_request import CreateCodeEvaluatorRequest + from .create_evaluator_request import ( + CreateEvaluatorRequest, + CreateEvaluatorRequest_Code, + CreateEvaluatorRequest_LlmAsJudge, + ) + from .create_llm_as_judge_evaluator_request import CreateLlmAsJudgeEvaluatorRequest + from .creator import Creator + from .cursor_meta import CursorMeta + from .deleted_evaluator import DeletedEvaluator + from .evaluation_rule_assignment import EvaluationRuleAssignment + from .evaluator import Evaluator, Evaluator_Code, Evaluator_LlmAsJudge + from .evaluator_base import EvaluatorBase + from .evaluator_chat_message import EvaluatorChatMessage + from .evaluator_chat_prompt import EvaluatorChatPrompt + from .evaluator_chat_prompt_input import EvaluatorChatPromptInput + from .evaluator_model_config import EvaluatorModelConfig + from .evaluator_status import EvaluatorStatus + from .evaluator_version import ( + EvaluatorVersion, + EvaluatorVersion_Code, + EvaluatorVersion_LlmAsJudge, + ) + from .evaluator_version_base import EvaluatorVersionBase + from .evaluator_versions_page import EvaluatorVersionsPage + from .evaluators_page import EvaluatorsPage + from .llm_as_judge_evaluator import LlmAsJudgeEvaluator + from .llm_as_judge_evaluator_version import LlmAsJudgeEvaluatorVersion + from .update_code_evaluator_request import UpdateCodeEvaluatorRequest + from .update_evaluator_metadata_request import UpdateEvaluatorMetadataRequest + from .update_evaluator_request import UpdateEvaluatorRequest + from .update_llm_as_judge_evaluator_request import UpdateLlmAsJudgeEvaluatorRequest +_dynamic_imports: typing.Dict[str, str] = { + "CodeEvaluator": ".code_evaluator", + "CodeEvaluatorVersion": ".code_evaluator_version", + "CreateCodeEvaluatorRequest": ".create_code_evaluator_request", + "CreateEvaluatorRequest": ".create_evaluator_request", + "CreateEvaluatorRequest_Code": ".create_evaluator_request", + "CreateEvaluatorRequest_LlmAsJudge": ".create_evaluator_request", + "CreateLlmAsJudgeEvaluatorRequest": ".create_llm_as_judge_evaluator_request", + "Creator": ".creator", + "CursorMeta": ".cursor_meta", + "DeletedEvaluator": ".deleted_evaluator", + "EvaluationRuleAssignment": ".evaluation_rule_assignment", + "Evaluator": ".evaluator", + "EvaluatorBase": ".evaluator_base", + "EvaluatorChatMessage": ".evaluator_chat_message", + "EvaluatorChatPrompt": ".evaluator_chat_prompt", + "EvaluatorChatPromptInput": ".evaluator_chat_prompt_input", + "EvaluatorModelConfig": ".evaluator_model_config", + "EvaluatorStatus": ".evaluator_status", + "EvaluatorVersion": ".evaluator_version", + "EvaluatorVersionBase": ".evaluator_version_base", + "EvaluatorVersion_Code": ".evaluator_version", + "EvaluatorVersion_LlmAsJudge": ".evaluator_version", + "EvaluatorVersionsPage": ".evaluator_versions_page", + "Evaluator_Code": ".evaluator", + "Evaluator_LlmAsJudge": ".evaluator", + "EvaluatorsPage": ".evaluators_page", + "LlmAsJudgeEvaluator": ".llm_as_judge_evaluator", + "LlmAsJudgeEvaluatorVersion": ".llm_as_judge_evaluator_version", + "UpdateCodeEvaluatorRequest": ".update_code_evaluator_request", + "UpdateEvaluatorMetadataRequest": ".update_evaluator_metadata_request", + "UpdateEvaluatorRequest": ".update_evaluator_request", + "UpdateLlmAsJudgeEvaluatorRequest": ".update_llm_as_judge_evaluator_request", +} + + +def __getattr__(attr_name: str) -> typing.Any: + module_name = _dynamic_imports.get(attr_name) + if module_name is None: + raise AttributeError( + f"No {attr_name} found in _dynamic_imports for module name -> {__name__}" + ) + try: + module = import_module(module_name, __package__) + if module_name == f".{attr_name}": + return module + else: + return getattr(module, attr_name) + except ImportError as e: + raise ImportError( + f"Failed to import {attr_name} from {module_name}: {e}" + ) from e + except AttributeError as e: + raise AttributeError( + f"Failed to get {attr_name} from {module_name}: {e}" + ) from e + + +def __dir__(): + lazy_attrs = list(_dynamic_imports.keys()) + return sorted(lazy_attrs) + + +__all__ = [ + "CodeEvaluator", + "CodeEvaluatorVersion", + "CreateCodeEvaluatorRequest", + "CreateEvaluatorRequest", + "CreateEvaluatorRequest_Code", + "CreateEvaluatorRequest_LlmAsJudge", + "CreateLlmAsJudgeEvaluatorRequest", + "Creator", + "CursorMeta", + "DeletedEvaluator", + "EvaluationRuleAssignment", + "Evaluator", + "EvaluatorBase", + "EvaluatorChatMessage", + "EvaluatorChatPrompt", + "EvaluatorChatPromptInput", + "EvaluatorModelConfig", + "EvaluatorStatus", + "EvaluatorVersion", + "EvaluatorVersionBase", + "EvaluatorVersion_Code", + "EvaluatorVersion_LlmAsJudge", + "EvaluatorVersionsPage", + "Evaluator_Code", + "Evaluator_LlmAsJudge", + "EvaluatorsPage", + "LlmAsJudgeEvaluator", + "LlmAsJudgeEvaluatorVersion", + "UpdateCodeEvaluatorRequest", + "UpdateEvaluatorMetadataRequest", + "UpdateEvaluatorRequest", + "UpdateLlmAsJudgeEvaluatorRequest", +] diff --git a/langfuse/api/evaluators/types/code_evaluator.py b/langfuse/api/evaluators/types/code_evaluator.py new file mode 100644 index 000000000..892a877f0 --- /dev/null +++ b/langfuse/api/evaluators/types/code_evaluator.py @@ -0,0 +1,36 @@ +# This file was auto-generated by Fern from our API Definition. + +import typing + +import pydantic +import typing_extensions +from ...core.serialization import FieldMetadata +from ...evaluation_commons.types.code_evaluator_source_code_language import ( + CodeEvaluatorSourceCodeLanguage, +) +from .evaluator_base import EvaluatorBase + + +class CodeEvaluator(EvaluatorBase): + type: typing.Literal["code"] = pydantic.Field(default="code") + """ + Evaluator type. + """ + + source_code: typing_extensions.Annotated[str, FieldMetadata(alias="sourceCode")] = ( + pydantic.Field() + ) + """ + Source code executed by the latest evaluator version. + """ + + source_code_language: typing_extensions.Annotated[ + CodeEvaluatorSourceCodeLanguage, FieldMetadata(alias="sourceCodeLanguage") + ] = pydantic.Field() + """ + Runtime language used to execute the latest source code. + """ + + model_config: typing.ClassVar[pydantic.ConfigDict] = pydantic.ConfigDict( + extra="allow", frozen=True + ) diff --git a/langfuse/api/evaluators/types/code_evaluator_version.py b/langfuse/api/evaluators/types/code_evaluator_version.py new file mode 100644 index 000000000..5c4994bda --- /dev/null +++ b/langfuse/api/evaluators/types/code_evaluator_version.py @@ -0,0 +1,36 @@ +# This file was auto-generated by Fern from our API Definition. + +import typing + +import pydantic +import typing_extensions +from ...core.serialization import FieldMetadata +from ...evaluation_commons.types.code_evaluator_source_code_language import ( + CodeEvaluatorSourceCodeLanguage, +) +from .evaluator_version_base import EvaluatorVersionBase + + +class CodeEvaluatorVersion(EvaluatorVersionBase): + type: typing.Literal["code"] = pydantic.Field(default="code") + """ + Evaluator type. + """ + + source_code: typing_extensions.Annotated[str, FieldMetadata(alias="sourceCode")] = ( + pydantic.Field() + ) + """ + Source code executed for each matched observation. + """ + + source_code_language: typing_extensions.Annotated[ + CodeEvaluatorSourceCodeLanguage, FieldMetadata(alias="sourceCodeLanguage") + ] = pydantic.Field() + """ + Runtime language used to execute the source code. + """ + + model_config: typing.ClassVar[pydantic.ConfigDict] = pydantic.ConfigDict( + extra="allow", frozen=True + ) diff --git a/langfuse/api/evaluators/types/create_code_evaluator_request.py b/langfuse/api/evaluators/types/create_code_evaluator_request.py new file mode 100644 index 000000000..7cd5ee5e8 --- /dev/null +++ b/langfuse/api/evaluators/types/create_code_evaluator_request.py @@ -0,0 +1,46 @@ +# This file was auto-generated by Fern from our API Definition. + +import typing + +import pydantic +import typing_extensions +from ...core.pydantic_utilities import UniversalBaseModel +from ...core.serialization import FieldMetadata +from ...evaluation_commons.types.code_evaluator_source_code_language import ( + CodeEvaluatorSourceCodeLanguage, +) + + +class CreateCodeEvaluatorRequest(UniversalBaseModel): + name: str = pydantic.Field() + """ + Human-readable evaluator name. Names are not identifiers and do not need to be unique. + """ + + description: typing.Optional[str] = pydantic.Field(default=None) + """ + Optional human-readable evaluator description. + """ + + type: typing.Literal["code"] = pydantic.Field(default="code") + """ + Evaluator type. + """ + + source_code: typing_extensions.Annotated[str, FieldMetadata(alias="sourceCode")] = ( + pydantic.Field() + ) + """ + Source code executed for each matched observation. + """ + + source_code_language: typing_extensions.Annotated[ + CodeEvaluatorSourceCodeLanguage, FieldMetadata(alias="sourceCodeLanguage") + ] = pydantic.Field() + """ + Runtime language used to execute the source code. + """ + + model_config: typing.ClassVar[pydantic.ConfigDict] = pydantic.ConfigDict( + extra="allow", frozen=True + ) diff --git a/langfuse/api/evaluators/types/create_evaluator_request.py b/langfuse/api/evaluators/types/create_evaluator_request.py new file mode 100644 index 000000000..42cf32ce2 --- /dev/null +++ b/langfuse/api/evaluators/types/create_evaluator_request.py @@ -0,0 +1,70 @@ +# This file was auto-generated by Fern from our API Definition. + +from __future__ import annotations + +import typing + +import pydantic +import typing_extensions +from ...core.pydantic_utilities import UniversalBaseModel +from ...core.serialization import FieldMetadata +from ...evaluation_commons.types.code_evaluator_source_code_language import ( + CodeEvaluatorSourceCodeLanguage, +) +from ...evaluation_commons.types.evaluator_output_definition import ( + EvaluatorOutputDefinition, +) +from ...evaluation_commons.types.prompt_variable_mapping_input import ( + PromptVariableMappingInput, +) +from .evaluator_chat_prompt_input import EvaluatorChatPromptInput +from .evaluator_model_config import EvaluatorModelConfig + + +class CreateEvaluatorRequest_LlmAsJudge(UniversalBaseModel): + """ + Initial evaluator definition with metadata and definition fields at the same level. The returned evaluator starts at version `1`. + """ + + type: typing.Literal["llm_as_judge"] = "llm_as_judge" + name: str + description: typing.Optional[str] = None + prompt: EvaluatorChatPromptInput + model_config_: typing_extensions.Annotated[ + typing.Optional[EvaluatorModelConfig], FieldMetadata(alias="modelConfig") + ] = None + variable_mapping: typing_extensions.Annotated[ + typing.Optional[typing.List[PromptVariableMappingInput]], + FieldMetadata(alias="variableMapping"), + ] = None + output_definition: typing_extensions.Annotated[ + EvaluatorOutputDefinition, FieldMetadata(alias="outputDefinition") + ] + + model_config: typing.ClassVar[pydantic.ConfigDict] = pydantic.ConfigDict( + extra="allow", frozen=True + ) + + +class CreateEvaluatorRequest_Code(UniversalBaseModel): + """ + Initial evaluator definition with metadata and definition fields at the same level. The returned evaluator starts at version `1`. + """ + + type: typing.Literal["code"] = "code" + name: str + description: typing.Optional[str] = None + source_code: typing_extensions.Annotated[str, FieldMetadata(alias="sourceCode")] + source_code_language: typing_extensions.Annotated[ + CodeEvaluatorSourceCodeLanguage, FieldMetadata(alias="sourceCodeLanguage") + ] + + model_config: typing.ClassVar[pydantic.ConfigDict] = pydantic.ConfigDict( + extra="allow", frozen=True + ) + + +CreateEvaluatorRequest = typing_extensions.Annotated[ + typing.Union[CreateEvaluatorRequest_LlmAsJudge, CreateEvaluatorRequest_Code], + pydantic.Field(discriminator="type"), +] diff --git a/langfuse/api/evaluators/types/create_llm_as_judge_evaluator_request.py b/langfuse/api/evaluators/types/create_llm_as_judge_evaluator_request.py new file mode 100644 index 000000000..57ab718cd --- /dev/null +++ b/langfuse/api/evaluators/types/create_llm_as_judge_evaluator_request.py @@ -0,0 +1,64 @@ +# This file was auto-generated by Fern from our API Definition. + +import typing + +import pydantic +import typing_extensions +from ...core.pydantic_utilities import UniversalBaseModel +from ...core.serialization import FieldMetadata +from ...evaluation_commons.types.evaluator_output_definition import ( + EvaluatorOutputDefinition, +) +from ...evaluation_commons.types.prompt_variable_mapping_input import ( + PromptVariableMappingInput, +) +from .evaluator_chat_prompt_input import EvaluatorChatPromptInput +from .evaluator_model_config import EvaluatorModelConfig + + +class CreateLlmAsJudgeEvaluatorRequest(UniversalBaseModel): + name: str = pydantic.Field() + """ + Human-readable evaluator name. Names are not identifiers and do not need to be unique. + """ + + description: typing.Optional[str] = pydantic.Field(default=None) + """ + Optional human-readable evaluator description. + """ + + type: typing.Literal["llm_as_judge"] = pydantic.Field(default="llm_as_judge") + """ + Evaluator type. + """ + + prompt: EvaluatorChatPromptInput = pydantic.Field() + """ + User prompt string shortcut or a list containing exactly one user chat message. + """ + + model_config_: typing_extensions.Annotated[ + typing.Optional[EvaluatorModelConfig], FieldMetadata(alias="modelConfig") + ] = pydantic.Field(default=None) + """ + Explicit model configuration. Set to `null` or omit to use the project's default evaluation model. + """ + + variable_mapping: typing_extensions.Annotated[ + typing.Optional[typing.List[PromptVariableMappingInput]], + FieldMetadata(alias="variableMapping"), + ] = pydantic.Field(default=None) + """ + Default prompt-variable mapping, or `null` when no default is configured. + """ + + output_definition: typing_extensions.Annotated[ + EvaluatorOutputDefinition, FieldMetadata(alias="outputDefinition") + ] = pydantic.Field() + """ + Structured output schema returned by this evaluator. Do not include an internal persistence version. + """ + + model_config: typing.ClassVar[pydantic.ConfigDict] = pydantic.ConfigDict( + extra="allow", frozen=True + ) diff --git a/langfuse/api/evaluators/types/creator.py b/langfuse/api/evaluators/types/creator.py new file mode 100644 index 000000000..8268ea60b --- /dev/null +++ b/langfuse/api/evaluators/types/creator.py @@ -0,0 +1,26 @@ +# This file was auto-generated by Fern from our API Definition. + +import typing + +import pydantic +from ...core.pydantic_utilities import UniversalBaseModel + + +class Creator(UniversalBaseModel): + """ + User who created the resource, or `null` when no user can be resolved. + """ + + id: str = pydantic.Field() + """ + User identifier. + """ + + name: typing.Optional[str] = pydantic.Field(default=None) + """ + User name, or `null` when unavailable. + """ + + model_config: typing.ClassVar[pydantic.ConfigDict] = pydantic.ConfigDict( + extra="allow", frozen=True + ) diff --git a/langfuse/api/evaluators/types/cursor_meta.py b/langfuse/api/evaluators/types/cursor_meta.py new file mode 100644 index 000000000..3ee6e1a0a --- /dev/null +++ b/langfuse/api/evaluators/types/cursor_meta.py @@ -0,0 +1,17 @@ +# This file was auto-generated by Fern from our API Definition. + +import typing + +import pydantic +from ...core.pydantic_utilities import UniversalBaseModel + + +class CursorMeta(UniversalBaseModel): + cursor: typing.Optional[str] = pydantic.Field(default=None) + """ + Opaque cursor for the next page. Omitted when there is no next page. + """ + + model_config: typing.ClassVar[pydantic.ConfigDict] = pydantic.ConfigDict( + extra="allow", frozen=True + ) diff --git a/langfuse/api/evaluators/types/deleted_evaluator.py b/langfuse/api/evaluators/types/deleted_evaluator.py new file mode 100644 index 000000000..a82e056f7 --- /dev/null +++ b/langfuse/api/evaluators/types/deleted_evaluator.py @@ -0,0 +1,21 @@ +# This file was auto-generated by Fern from our API Definition. + +import typing + +import pydantic +from ...core.pydantic_utilities import UniversalBaseModel + + +class DeletedEvaluator(UniversalBaseModel): + """ + Confirmation returned after successful evaluator deletion. + """ + + id: str = pydantic.Field() + """ + Identifier of the deleted evaluator. + """ + + model_config: typing.ClassVar[pydantic.ConfigDict] = pydantic.ConfigDict( + extra="allow", frozen=True + ) diff --git a/langfuse/api/evaluators/types/evaluation_rule_assignment.py b/langfuse/api/evaluators/types/evaluation_rule_assignment.py new file mode 100644 index 000000000..9888d797b --- /dev/null +++ b/langfuse/api/evaluators/types/evaluation_rule_assignment.py @@ -0,0 +1,34 @@ +# This file was auto-generated by Fern from our API Definition. + +import typing + +import pydantic +import typing_extensions +from ...core.pydantic_utilities import UniversalBaseModel +from ...core.serialization import FieldMetadata +from ...evaluation_commons.types.prompt_variable_mapping import PromptVariableMapping + + +class EvaluationRuleAssignment(UniversalBaseModel): + """ + Evaluation-rule assignment associated with this evaluator. + """ + + evaluation_rule_id: typing_extensions.Annotated[ + str, FieldMetadata(alias="evaluationRuleId") + ] = pydantic.Field() + """ + Stable identifier of the assigned evaluation rule. + """ + + variable_mapping_override: typing_extensions.Annotated[ + typing.Optional[typing.List[PromptVariableMapping]], + FieldMetadata(alias="variableMappingOverride"), + ] = pydantic.Field(default=None) + """ + Rule-specific variable mapping override. Omitted when the evaluator's latest default mapping is inherited. Legacy mappings use the explicit `legacy` variant. + """ + + model_config: typing.ClassVar[pydantic.ConfigDict] = pydantic.ConfigDict( + extra="allow", frozen=True + ) diff --git a/langfuse/api/evaluators/types/evaluator.py b/langfuse/api/evaluators/types/evaluator.py new file mode 100644 index 000000000..ba5cc1d28 --- /dev/null +++ b/langfuse/api/evaluators/types/evaluator.py @@ -0,0 +1,143 @@ +# This file was auto-generated by Fern from our API Definition. + +from __future__ import annotations + +import datetime as dt +import typing + +import pydantic +import typing_extensions +from ...core.pydantic_utilities import UniversalBaseModel +from ...core.serialization import FieldMetadata +from ...evaluation_commons.types.code_evaluator_source_code_language import ( + CodeEvaluatorSourceCodeLanguage, +) +from ...evaluation_commons.types.prompt_variable_mapping_read import ( + PromptVariableMappingRead, +) +from ...evaluation_commons.types.public_evaluator_output_definition import ( + PublicEvaluatorOutputDefinition, +) +from .creator import Creator +from .evaluation_rule_assignment import EvaluationRuleAssignment +from .evaluator_chat_prompt import EvaluatorChatPrompt +from .evaluator_model_config import EvaluatorModelConfig +from .evaluator_status import EvaluatorStatus + + +class Evaluator_LlmAsJudge(UniversalBaseModel): + """ + One evaluator that can be used for scoring. + + An evaluator describes **how** to score data. Associated evaluation rules describe **which** live objects should be evaluated. + """ + + type: typing.Literal["llm_as_judge"] = "llm_as_judge" + prompt: EvaluatorChatPrompt + variables: typing.List[str] + variable_mapping: typing_extensions.Annotated[ + typing.Optional[typing.List[PromptVariableMappingRead]], + FieldMetadata(alias="variableMapping"), + ] = None + model_config_: typing_extensions.Annotated[ + typing.Optional[EvaluatorModelConfig], FieldMetadata(alias="modelConfig") + ] = None + output_definition: typing_extensions.Annotated[ + PublicEvaluatorOutputDefinition, FieldMetadata(alias="outputDefinition") + ] + id: str + name: str + description: typing.Optional[str] = None + created_by: typing_extensions.Annotated[ + typing.Optional[Creator], FieldMetadata(alias="createdBy") + ] = None + status: EvaluatorStatus + paused_at: typing_extensions.Annotated[ + typing.Optional[dt.datetime], FieldMetadata(alias="pausedAt") + ] = None + paused_reason: typing_extensions.Annotated[ + typing.Optional[str], FieldMetadata(alias="pausedReason") + ] = None + paused_message: typing_extensions.Annotated[ + typing.Optional[str], FieldMetadata(alias="pausedMessage") + ] = None + evaluation_rule_assignments: typing_extensions.Annotated[ + typing.List[EvaluationRuleAssignment], + FieldMetadata(alias="evaluationRuleAssignments"), + ] + created_at: typing_extensions.Annotated[ + dt.datetime, FieldMetadata(alias="createdAt") + ] + updated_at: typing_extensions.Annotated[ + dt.datetime, FieldMetadata(alias="updatedAt") + ] + version_id: typing_extensions.Annotated[str, FieldMetadata(alias="versionId")] + version: int + version_created_at: typing_extensions.Annotated[ + dt.datetime, FieldMetadata(alias="versionCreatedAt") + ] + version_created_by: typing_extensions.Annotated[ + typing.Optional[Creator], FieldMetadata(alias="versionCreatedBy") + ] = None + + model_config: typing.ClassVar[pydantic.ConfigDict] = pydantic.ConfigDict( + extra="allow", frozen=True + ) + + +class Evaluator_Code(UniversalBaseModel): + """ + One evaluator that can be used for scoring. + + An evaluator describes **how** to score data. Associated evaluation rules describe **which** live objects should be evaluated. + """ + + type: typing.Literal["code"] = "code" + source_code: typing_extensions.Annotated[str, FieldMetadata(alias="sourceCode")] + source_code_language: typing_extensions.Annotated[ + CodeEvaluatorSourceCodeLanguage, FieldMetadata(alias="sourceCodeLanguage") + ] + id: str + name: str + description: typing.Optional[str] = None + created_by: typing_extensions.Annotated[ + typing.Optional[Creator], FieldMetadata(alias="createdBy") + ] = None + status: EvaluatorStatus + paused_at: typing_extensions.Annotated[ + typing.Optional[dt.datetime], FieldMetadata(alias="pausedAt") + ] = None + paused_reason: typing_extensions.Annotated[ + typing.Optional[str], FieldMetadata(alias="pausedReason") + ] = None + paused_message: typing_extensions.Annotated[ + typing.Optional[str], FieldMetadata(alias="pausedMessage") + ] = None + evaluation_rule_assignments: typing_extensions.Annotated[ + typing.List[EvaluationRuleAssignment], + FieldMetadata(alias="evaluationRuleAssignments"), + ] + created_at: typing_extensions.Annotated[ + dt.datetime, FieldMetadata(alias="createdAt") + ] + updated_at: typing_extensions.Annotated[ + dt.datetime, FieldMetadata(alias="updatedAt") + ] + version_id: typing_extensions.Annotated[str, FieldMetadata(alias="versionId")] + version: int + version_created_at: typing_extensions.Annotated[ + dt.datetime, FieldMetadata(alias="versionCreatedAt") + ] + version_created_by: typing_extensions.Annotated[ + typing.Optional[Creator], FieldMetadata(alias="versionCreatedBy") + ] = None + + model_config: typing.ClassVar[pydantic.ConfigDict] = pydantic.ConfigDict( + extra="allow", frozen=True + ) + + +Evaluator = typing_extensions.Annotated[ + typing.Union[Evaluator_LlmAsJudge, Evaluator_Code], + pydantic.Field(discriminator="type"), +] diff --git a/langfuse/api/evaluators/types/evaluator_base.py b/langfuse/api/evaluators/types/evaluator_base.py new file mode 100644 index 000000000..010feb402 --- /dev/null +++ b/langfuse/api/evaluators/types/evaluator_base.py @@ -0,0 +1,114 @@ +# This file was auto-generated by Fern from our API Definition. + +import datetime as dt +import typing + +import pydantic +import typing_extensions +from ...core.pydantic_utilities import UniversalBaseModel +from ...core.serialization import FieldMetadata +from .creator import Creator +from .evaluation_rule_assignment import EvaluationRuleAssignment +from .evaluator_status import EvaluatorStatus + + +class EvaluatorBase(UniversalBaseModel): + id: str = pydantic.Field() + """ + Stable identifier of this evaluator across all versions. + """ + + name: str = pydantic.Field() + """ + Human-readable evaluator name. Names are not identifiers and do not need to be unique. + """ + + description: typing.Optional[str] = pydantic.Field(default=None) + """ + Optional human-readable evaluator description. + """ + + created_by: typing_extensions.Annotated[ + typing.Optional[Creator], FieldMetadata(alias="createdBy") + ] = pydantic.Field(default=None) + """ + User who created this evaluator, or `null` when no user can be resolved. + """ + + status: EvaluatorStatus = pydantic.Field() + """ + Effective evaluator status after Langfuse validates its runtime configuration. + """ + + paused_at: typing_extensions.Annotated[ + typing.Optional[dt.datetime], FieldMetadata(alias="pausedAt") + ] = pydantic.Field(default=None) + """ + Timestamp when the evaluator was paused, otherwise `null`. + """ + + paused_reason: typing_extensions.Annotated[ + typing.Optional[str], FieldMetadata(alias="pausedReason") + ] = pydantic.Field(default=None) + """ + Machine-readable reason when `status=paused`, otherwise `null`. + """ + + paused_message: typing_extensions.Annotated[ + typing.Optional[str], FieldMetadata(alias="pausedMessage") + ] = pydantic.Field(default=None) + """ + Human-readable explanation when `status=paused`, otherwise `null`. + """ + + evaluation_rule_assignments: typing_extensions.Annotated[ + typing.List[EvaluationRuleAssignment], + FieldMetadata(alias="evaluationRuleAssignments"), + ] = pydantic.Field() + """ + All modern and legacy evaluation-rule assignments in newest-assignment-first order. Rule-specific mappings are exposed as `variableMappingOverride`; inherited defaults are omitted. + """ + + created_at: typing_extensions.Annotated[ + dt.datetime, FieldMetadata(alias="createdAt") + ] = pydantic.Field() + """ + Timestamp when the evaluator was created. + """ + + updated_at: typing_extensions.Annotated[ + dt.datetime, FieldMetadata(alias="updatedAt") + ] = pydantic.Field() + """ + Timestamp when the evaluator was last updated. + """ + + version_id: typing_extensions.Annotated[str, FieldMetadata(alias="versionId")] = ( + pydantic.Field() + ) + """ + Stable identifier of the latest evaluator version. + """ + + version: int = pydantic.Field() + """ + Monotonically increasing latest evaluator version number. + """ + + version_created_at: typing_extensions.Annotated[ + dt.datetime, FieldMetadata(alias="versionCreatedAt") + ] = pydantic.Field() + """ + Timestamp when the latest evaluator version was created. + """ + + version_created_by: typing_extensions.Annotated[ + typing.Optional[Creator], FieldMetadata(alias="versionCreatedBy") + ] = pydantic.Field(default=None) + """ + User who created the latest evaluator version, or `null` when no user can be resolved. + """ + + model_config: typing.ClassVar[pydantic.ConfigDict] = pydantic.ConfigDict( + extra="allow", frozen=True + ) diff --git a/langfuse/api/evaluators/types/evaluator_chat_message.py b/langfuse/api/evaluators/types/evaluator_chat_message.py new file mode 100644 index 000000000..f116c3af9 --- /dev/null +++ b/langfuse/api/evaluators/types/evaluator_chat_message.py @@ -0,0 +1,22 @@ +# This file was auto-generated by Fern from our API Definition. + +import typing + +import pydantic +from ...core.pydantic_utilities import UniversalBaseModel + + +class EvaluatorChatMessage(UniversalBaseModel): + """ + One user chat message in an evaluator prompt. + """ + + role: typing.Literal["user"] = "user" + content: str = pydantic.Field() + """ + Message content. Evaluator variables use `{{variable}}` syntax. + """ + + model_config: typing.ClassVar[pydantic.ConfigDict] = pydantic.ConfigDict( + extra="allow", frozen=True + ) diff --git a/langfuse/api/evaluators/types/evaluator_chat_prompt.py b/langfuse/api/evaluators/types/evaluator_chat_prompt.py new file mode 100644 index 000000000..80a0c74ec --- /dev/null +++ b/langfuse/api/evaluators/types/evaluator_chat_prompt.py @@ -0,0 +1,7 @@ +# This file was auto-generated by Fern from our API Definition. + +import typing + +from .evaluator_chat_message import EvaluatorChatMessage + +EvaluatorChatPrompt = typing.List[EvaluatorChatMessage] diff --git a/langfuse/api/evaluators/types/evaluator_chat_prompt_input.py b/langfuse/api/evaluators/types/evaluator_chat_prompt_input.py new file mode 100644 index 000000000..8222936ea --- /dev/null +++ b/langfuse/api/evaluators/types/evaluator_chat_prompt_input.py @@ -0,0 +1,7 @@ +# This file was auto-generated by Fern from our API Definition. + +import typing + +from .evaluator_chat_prompt import EvaluatorChatPrompt + +EvaluatorChatPromptInput = typing.Union[str, EvaluatorChatPrompt] diff --git a/langfuse/api/evaluators/types/evaluator_model_config.py b/langfuse/api/evaluators/types/evaluator_model_config.py new file mode 100644 index 000000000..4d42de75f --- /dev/null +++ b/langfuse/api/evaluators/types/evaluator_model_config.py @@ -0,0 +1,40 @@ +# This file was auto-generated by Fern from our API Definition. + +import typing + +import pydantic +from ...core.pydantic_utilities import UniversalBaseModel + + +class EvaluatorModelConfig(UniversalBaseModel): + """ + Explicit model configuration for an evaluator. + + If the complete `modelConfig` is `null`, Langfuse uses the project's default evaluation model. + If provided, the model must be available to the project when the evaluator or evaluation rule is enabled. + + Examples + -------- + from langfuse.evaluators import EvaluatorModelConfig + + EvaluatorModelConfig( + provider="openai", + model="gpt-4.1-mini", + ) + """ + + provider: str = pydantic.Field() + """ + Provider identifier, for example `openai` or `anthropic`. + + To discover valid values for the current project, call `GET /api/public/llm-connections` and use one of the returned `provider` values. + """ + + model: str = pydantic.Field() + """ + Model identifier exposed by the provider, for example `gpt-4.1-mini`. + """ + + model_config: typing.ClassVar[pydantic.ConfigDict] = pydantic.ConfigDict( + extra="allow", frozen=True + ) diff --git a/langfuse/api/evaluators/types/evaluator_status.py b/langfuse/api/evaluators/types/evaluator_status.py new file mode 100644 index 000000000..cc8f5bfc2 --- /dev/null +++ b/langfuse/api/evaluators/types/evaluator_status.py @@ -0,0 +1,29 @@ +# This file was auto-generated by Fern from our API Definition. + +import typing + +from ...core import enum + +T_Result = typing.TypeVar("T_Result") + + +class EvaluatorStatus(enum.StrEnum): + """ + Effective evaluator runtime status. + + - `active`: the evaluator can run. + - `paused`: Langfuse paused execution until the underlying issue is resolved. + """ + + ACTIVE = "active" + PAUSED = "paused" + + def visit( + self, + active: typing.Callable[[], T_Result], + paused: typing.Callable[[], T_Result], + ) -> T_Result: + if self is EvaluatorStatus.ACTIVE: + return active() + if self is EvaluatorStatus.PAUSED: + return paused() diff --git a/langfuse/api/evaluators/types/evaluator_version.py b/langfuse/api/evaluators/types/evaluator_version.py new file mode 100644 index 000000000..399985b7c --- /dev/null +++ b/langfuse/api/evaluators/types/evaluator_version.py @@ -0,0 +1,77 @@ +# This file was auto-generated by Fern from our API Definition. + +from __future__ import annotations + +import datetime as dt +import typing + +import pydantic +import typing_extensions +from ...core.pydantic_utilities import UniversalBaseModel +from ...core.serialization import FieldMetadata +from ...evaluation_commons.types.code_evaluator_source_code_language import ( + CodeEvaluatorSourceCodeLanguage, +) +from ...evaluation_commons.types.prompt_variable_mapping_read import ( + PromptVariableMappingRead, +) +from ...evaluation_commons.types.public_evaluator_output_definition import ( + PublicEvaluatorOutputDefinition, +) +from .creator import Creator +from .evaluator_chat_prompt import EvaluatorChatPrompt +from .evaluator_model_config import EvaluatorModelConfig + + +class EvaluatorVersion_LlmAsJudge(UniversalBaseModel): + type: typing.Literal["llm_as_judge"] = "llm_as_judge" + prompt: EvaluatorChatPrompt + variables: typing.List[str] + variable_mapping: typing_extensions.Annotated[ + typing.Optional[typing.List[PromptVariableMappingRead]], + FieldMetadata(alias="variableMapping"), + ] = None + model_config_: typing_extensions.Annotated[ + typing.Optional[EvaluatorModelConfig], FieldMetadata(alias="modelConfig") + ] = None + output_definition: typing_extensions.Annotated[ + PublicEvaluatorOutputDefinition, FieldMetadata(alias="outputDefinition") + ] + id: str + version: int + created_at: typing_extensions.Annotated[ + dt.datetime, FieldMetadata(alias="createdAt") + ] + created_by: typing_extensions.Annotated[ + typing.Optional[Creator], FieldMetadata(alias="createdBy") + ] = None + + model_config: typing.ClassVar[pydantic.ConfigDict] = pydantic.ConfigDict( + extra="allow", frozen=True + ) + + +class EvaluatorVersion_Code(UniversalBaseModel): + type: typing.Literal["code"] = "code" + source_code: typing_extensions.Annotated[str, FieldMetadata(alias="sourceCode")] + source_code_language: typing_extensions.Annotated[ + CodeEvaluatorSourceCodeLanguage, FieldMetadata(alias="sourceCodeLanguage") + ] + id: str + version: int + created_at: typing_extensions.Annotated[ + dt.datetime, FieldMetadata(alias="createdAt") + ] + created_by: typing_extensions.Annotated[ + typing.Optional[Creator], FieldMetadata(alias="createdBy") + ] = None + + model_config: typing.ClassVar[pydantic.ConfigDict] = pydantic.ConfigDict( + extra="allow", frozen=True + ) + + +EvaluatorVersion = typing_extensions.Annotated[ + typing.Union[EvaluatorVersion_LlmAsJudge, EvaluatorVersion_Code], + pydantic.Field(discriminator="type"), +] diff --git a/langfuse/api/evaluators/types/evaluator_version_base.py b/langfuse/api/evaluators/types/evaluator_version_base.py new file mode 100644 index 000000000..a6f3695bc --- /dev/null +++ b/langfuse/api/evaluators/types/evaluator_version_base.py @@ -0,0 +1,40 @@ +# This file was auto-generated by Fern from our API Definition. + +import datetime as dt +import typing + +import pydantic +import typing_extensions +from ...core.pydantic_utilities import UniversalBaseModel +from ...core.serialization import FieldMetadata +from .creator import Creator + + +class EvaluatorVersionBase(UniversalBaseModel): + id: str = pydantic.Field() + """ + Stable identifier of this evaluator version. + """ + + version: int = pydantic.Field() + """ + Monotonically increasing evaluator version number. + """ + + created_at: typing_extensions.Annotated[ + dt.datetime, FieldMetadata(alias="createdAt") + ] = pydantic.Field() + """ + Timestamp when this evaluator version was created. + """ + + created_by: typing_extensions.Annotated[ + typing.Optional[Creator], FieldMetadata(alias="createdBy") + ] = pydantic.Field(default=None) + """ + User who created this version, or `null` when no user can be resolved. + """ + + model_config: typing.ClassVar[pydantic.ConfigDict] = pydantic.ConfigDict( + extra="allow", frozen=True + ) diff --git a/langfuse/api/evaluators/types/evaluator_versions_page.py b/langfuse/api/evaluators/types/evaluator_versions_page.py new file mode 100644 index 000000000..38971c429 --- /dev/null +++ b/langfuse/api/evaluators/types/evaluator_versions_page.py @@ -0,0 +1,24 @@ +# This file was auto-generated by Fern from our API Definition. + +import typing + +import pydantic +from ...core.pydantic_utilities import UniversalBaseModel +from .cursor_meta import CursorMeta +from .evaluator_version import EvaluatorVersion + + +class EvaluatorVersionsPage(UniversalBaseModel): + data: typing.List[EvaluatorVersion] = pydantic.Field() + """ + Evaluator versions for this page in newest-first order. + """ + + meta: CursorMeta = pydantic.Field() + """ + Cursor pagination metadata. + """ + + model_config: typing.ClassVar[pydantic.ConfigDict] = pydantic.ConfigDict( + extra="allow", frozen=True + ) diff --git a/langfuse/api/evaluators/types/evaluators_page.py b/langfuse/api/evaluators/types/evaluators_page.py new file mode 100644 index 000000000..ad8f488f1 --- /dev/null +++ b/langfuse/api/evaluators/types/evaluators_page.py @@ -0,0 +1,24 @@ +# This file was auto-generated by Fern from our API Definition. + +import typing + +import pydantic +from ...core.pydantic_utilities import UniversalBaseModel +from .cursor_meta import CursorMeta +from .evaluator import Evaluator + + +class EvaluatorsPage(UniversalBaseModel): + data: typing.List[Evaluator] = pydantic.Field() + """ + Evaluators for this page. + """ + + meta: CursorMeta = pydantic.Field() + """ + Cursor pagination metadata. + """ + + model_config: typing.ClassVar[pydantic.ConfigDict] = pydantic.ConfigDict( + extra="allow", frozen=True + ) diff --git a/langfuse/api/evaluators/types/llm_as_judge_evaluator.py b/langfuse/api/evaluators/types/llm_as_judge_evaluator.py new file mode 100644 index 000000000..bb6e72f32 --- /dev/null +++ b/langfuse/api/evaluators/types/llm_as_judge_evaluator.py @@ -0,0 +1,59 @@ +# This file was auto-generated by Fern from our API Definition. + +import typing + +import pydantic +import typing_extensions +from ...core.serialization import FieldMetadata +from ...evaluation_commons.types.prompt_variable_mapping_read import ( + PromptVariableMappingRead, +) +from ...evaluation_commons.types.public_evaluator_output_definition import ( + PublicEvaluatorOutputDefinition, +) +from .evaluator_base import EvaluatorBase +from .evaluator_chat_prompt import EvaluatorChatPrompt +from .evaluator_model_config import EvaluatorModelConfig + + +class LlmAsJudgeEvaluator(EvaluatorBase): + type: typing.Literal["llm_as_judge"] = pydantic.Field(default="llm_as_judge") + """ + Evaluator type. + """ + + prompt: EvaluatorChatPrompt = pydantic.Field() + """ + The single user chat message used by the latest evaluator version. + """ + + variables: typing.List[str] = pydantic.Field() + """ + Variables extracted from the latest prompt and available for evaluation-rule mappings. + """ + + variable_mapping: typing_extensions.Annotated[ + typing.Optional[typing.List[PromptVariableMappingRead]], + FieldMetadata(alias="variableMapping"), + ] = pydantic.Field(default=None) + """ + Default variable mapping for the latest version, or `null` when no default is configured. + """ + + model_config_: typing_extensions.Annotated[ + typing.Optional[EvaluatorModelConfig], FieldMetadata(alias="modelConfig") + ] = pydantic.Field(default=None) + """ + Explicit model configuration for the latest version, or `null` when the project's default evaluation model is used. + """ + + output_definition: typing_extensions.Annotated[ + PublicEvaluatorOutputDefinition, FieldMetadata(alias="outputDefinition") + ] = pydantic.Field() + """ + Structured output schema returned by the latest evaluator version. + """ + + model_config: typing.ClassVar[pydantic.ConfigDict] = pydantic.ConfigDict( + extra="allow", frozen=True + ) diff --git a/langfuse/api/evaluators/types/llm_as_judge_evaluator_version.py b/langfuse/api/evaluators/types/llm_as_judge_evaluator_version.py new file mode 100644 index 000000000..23d7bdef2 --- /dev/null +++ b/langfuse/api/evaluators/types/llm_as_judge_evaluator_version.py @@ -0,0 +1,63 @@ +# This file was auto-generated by Fern from our API Definition. + +import typing + +import pydantic +import typing_extensions +from ...core.serialization import FieldMetadata +from ...evaluation_commons.types.prompt_variable_mapping_read import ( + PromptVariableMappingRead, +) +from ...evaluation_commons.types.public_evaluator_output_definition import ( + PublicEvaluatorOutputDefinition, +) +from .evaluator_chat_prompt import EvaluatorChatPrompt +from .evaluator_model_config import EvaluatorModelConfig +from .evaluator_version_base import EvaluatorVersionBase + + +class LlmAsJudgeEvaluatorVersion(EvaluatorVersionBase): + type: typing.Literal["llm_as_judge"] = pydantic.Field(default="llm_as_judge") + """ + Evaluator type. + """ + + prompt: EvaluatorChatPrompt = pydantic.Field() + """ + The single user chat message used during evaluation. + """ + + variables: typing.List[str] = pydantic.Field() + """ + Variables extracted from the prompt and available for evaluation-rule mappings. + + Every variable must be mapped exactly once when a rule provides an explicit mapping. + """ + + variable_mapping: typing_extensions.Annotated[ + typing.Optional[typing.List[PromptVariableMappingRead]], + FieldMetadata(alias="variableMapping"), + ] = pydantic.Field(default=None) + """ + Default variable mapping for this evaluator version, or `null` when no default is configured. + """ + + model_config_: typing_extensions.Annotated[ + typing.Optional[EvaluatorModelConfig], FieldMetadata(alias="modelConfig") + ] = pydantic.Field(default=None) + """ + Explicit model configuration, or `null` when the project's default evaluation model is used. + """ + + output_definition: typing_extensions.Annotated[ + PublicEvaluatorOutputDefinition, FieldMetadata(alias="outputDefinition") + ] = pydantic.Field() + """ + Structured output schema returned by this evaluator. + + Responses include `dataType` and omit the internal output-definition persistence version. + """ + + model_config: typing.ClassVar[pydantic.ConfigDict] = pydantic.ConfigDict( + extra="allow", frozen=True + ) diff --git a/langfuse/api/evaluators/types/update_code_evaluator_request.py b/langfuse/api/evaluators/types/update_code_evaluator_request.py new file mode 100644 index 000000000..89f47368f --- /dev/null +++ b/langfuse/api/evaluators/types/update_code_evaluator_request.py @@ -0,0 +1,46 @@ +# This file was auto-generated by Fern from our API Definition. + +import typing + +import pydantic +import typing_extensions +from ...core.pydantic_utilities import UniversalBaseModel +from ...core.serialization import FieldMetadata +from ...evaluation_commons.types.code_evaluator_source_code_language import ( + CodeEvaluatorSourceCodeLanguage, +) + + +class UpdateCodeEvaluatorRequest(UniversalBaseModel): + name: typing.Optional[str] = pydantic.Field(default=None) + """ + New human-readable evaluator name. + """ + + description: typing.Optional[str] = pydantic.Field(default=None) + """ + New description. Set to `null` to clear it. + """ + + type: typing.Literal["code"] = pydantic.Field(default="code") + """ + Evaluator type. The type of an existing evaluator cannot change. + """ + + source_code: typing_extensions.Annotated[str, FieldMetadata(alias="sourceCode")] = ( + pydantic.Field() + ) + """ + Complete replacement source code. + """ + + source_code_language: typing_extensions.Annotated[ + CodeEvaluatorSourceCodeLanguage, FieldMetadata(alias="sourceCodeLanguage") + ] = pydantic.Field() + """ + Runtime language used to execute the source code. + """ + + model_config: typing.ClassVar[pydantic.ConfigDict] = pydantic.ConfigDict( + extra="allow", frozen=True + ) diff --git a/langfuse/api/evaluators/types/update_evaluator_metadata_request.py b/langfuse/api/evaluators/types/update_evaluator_metadata_request.py new file mode 100644 index 000000000..2c7700edf --- /dev/null +++ b/langfuse/api/evaluators/types/update_evaluator_metadata_request.py @@ -0,0 +1,26 @@ +# This file was auto-generated by Fern from our API Definition. + +import typing + +import pydantic +from ...core.pydantic_utilities import UniversalBaseModel + + +class UpdateEvaluatorMetadataRequest(UniversalBaseModel): + """ + Metadata-only update. At least one field must be provided. + """ + + name: typing.Optional[str] = pydantic.Field(default=None) + """ + New human-readable evaluator name. + """ + + description: typing.Optional[str] = pydantic.Field(default=None) + """ + New description. Set to `null` to clear it. + """ + + model_config: typing.ClassVar[pydantic.ConfigDict] = pydantic.ConfigDict( + extra="allow", frozen=True + ) diff --git a/langfuse/api/evaluators/types/update_evaluator_request.py b/langfuse/api/evaluators/types/update_evaluator_request.py new file mode 100644 index 000000000..e4f1b3a34 --- /dev/null +++ b/langfuse/api/evaluators/types/update_evaluator_request.py @@ -0,0 +1,13 @@ +# This file was auto-generated by Fern from our API Definition. + +import typing + +from .update_code_evaluator_request import UpdateCodeEvaluatorRequest +from .update_evaluator_metadata_request import UpdateEvaluatorMetadataRequest +from .update_llm_as_judge_evaluator_request import UpdateLlmAsJudgeEvaluatorRequest + +UpdateEvaluatorRequest = typing.Union[ + UpdateEvaluatorMetadataRequest, + UpdateLlmAsJudgeEvaluatorRequest, + UpdateCodeEvaluatorRequest, +] diff --git a/langfuse/api/evaluators/types/update_llm_as_judge_evaluator_request.py b/langfuse/api/evaluators/types/update_llm_as_judge_evaluator_request.py new file mode 100644 index 000000000..140dec573 --- /dev/null +++ b/langfuse/api/evaluators/types/update_llm_as_judge_evaluator_request.py @@ -0,0 +1,64 @@ +# This file was auto-generated by Fern from our API Definition. + +import typing + +import pydantic +import typing_extensions +from ...core.pydantic_utilities import UniversalBaseModel +from ...core.serialization import FieldMetadata +from ...evaluation_commons.types.evaluator_output_definition import ( + EvaluatorOutputDefinition, +) +from ...evaluation_commons.types.prompt_variable_mapping_input import ( + PromptVariableMappingInput, +) +from .evaluator_chat_prompt_input import EvaluatorChatPromptInput +from .evaluator_model_config import EvaluatorModelConfig + + +class UpdateLlmAsJudgeEvaluatorRequest(UniversalBaseModel): + name: typing.Optional[str] = pydantic.Field(default=None) + """ + New human-readable evaluator name. + """ + + description: typing.Optional[str] = pydantic.Field(default=None) + """ + New description. Set to `null` to clear it. + """ + + type: typing.Literal["llm_as_judge"] = pydantic.Field(default="llm_as_judge") + """ + Evaluator type. The type of an existing evaluator cannot change. + """ + + prompt: EvaluatorChatPromptInput = pydantic.Field() + """ + Complete replacement user prompt string or a list containing exactly one user chat message. + """ + + model_config_: typing_extensions.Annotated[ + typing.Optional[EvaluatorModelConfig], FieldMetadata(alias="modelConfig") + ] = pydantic.Field(default=None) + """ + Explicit model configuration. Set to `null` or omit to use the project's default evaluation model. + """ + + variable_mapping: typing_extensions.Annotated[ + typing.Optional[typing.List[PromptVariableMappingInput]], + FieldMetadata(alias="variableMapping"), + ] = pydantic.Field(default=None) + """ + Complete replacement default variable mapping, or `null` when no default is configured. + """ + + output_definition: typing_extensions.Annotated[ + EvaluatorOutputDefinition, FieldMetadata(alias="outputDefinition") + ] = pydantic.Field() + """ + Complete replacement output schema. Do not include an internal persistence version. + """ + + model_config: typing.ClassVar[pydantic.ConfigDict] = pydantic.ConfigDict( + extra="allow", frozen=True + ) diff --git a/langfuse/api/unstable/__init__.py b/langfuse/api/unstable/__init__.py index 8ac8c3c43..84a0cbfa7 100644 --- a/langfuse/api/unstable/__init__.py +++ b/langfuse/api/unstable/__init__.py @@ -48,12 +48,9 @@ EvaluationRuleFilter_String, EvaluationRuleFilter_StringObject, EvaluationRuleFilter_StringOptions, - EvaluationRuleMapping, - EvaluationRuleMappingSource, EvaluationRuleNullFilterOperator, EvaluationRuleNumberFilterOperator, EvaluationRuleOptionsFilterOperator, - EvaluationRuleReadMapping, EvaluationRuleStatus, EvaluationRuleStringFilterOperator, EvaluationRuleTarget, @@ -68,6 +65,9 @@ NullEvaluationRuleFilter, NumberEvaluationRuleFilter, NumberObjectEvaluationRuleFilter, + PromptVariableMappingInput, + PromptVariableMappingRead, + PromptVariableMappingSource, PublicBooleanEvaluatorOutputDefinition, PublicCategoricalEvaluatorOutputDefinition, PublicCategoricalEvaluatorOutputScoreDefinition, @@ -138,8 +138,8 @@ LegacyEvaluationObject, LegacyEvaluationRule, LegacyEvaluationRuleEvaluatorAssignment, - LegacyEvaluationRuleMapping, LegacyEvaluationRuleTarget, + LegacyPromptVariableMapping, LlmAsJudgeEvaluationRuleEvaluatorReference, LlmAsJudgeEvaluatorType, ReadableEvaluationRule, @@ -231,12 +231,9 @@ "EvaluationRuleFilter_String": ".commons", "EvaluationRuleFilter_StringObject": ".commons", "EvaluationRuleFilter_StringOptions": ".commons", - "EvaluationRuleMapping": ".commons", - "EvaluationRuleMappingSource": ".commons", "EvaluationRuleNullFilterOperator": ".commons", "EvaluationRuleNumberFilterOperator": ".commons", "EvaluationRuleOptionsFilterOperator": ".commons", - "EvaluationRuleReadMapping": ".commons", "EvaluationRuleStatus": ".commons", "EvaluationRuleStringFilterOperator": ".commons", "EvaluationRuleTarget": ".commons", @@ -259,8 +256,8 @@ "LegacyEvaluationObject": ".evaluation_rules", "LegacyEvaluationRule": ".evaluation_rules", "LegacyEvaluationRuleEvaluatorAssignment": ".evaluation_rules", - "LegacyEvaluationRuleMapping": ".evaluation_rules", "LegacyEvaluationRuleTarget": ".evaluation_rules", + "LegacyPromptVariableMapping": ".evaluation_rules", "LlmAsJudgeEvaluationRuleEvaluatorReference": ".evaluation_rules", "LlmAsJudgeEvaluator": ".evaluators", "LlmAsJudgeEvaluatorType": ".evaluation_rules", @@ -270,6 +267,9 @@ "NumberEvaluationRuleFilter": ".commons", "NumberObjectEvaluationRuleFilter": ".commons", "PresetPlacement": ".dashboards", + "PromptVariableMappingInput": ".commons", + "PromptVariableMappingRead": ".commons", + "PromptVariableMappingSource": ".commons", "PublicApiError": ".errors", "PublicApiErrorCode": ".errors", "PublicApiErrorDetails": ".errors", @@ -402,12 +402,9 @@ def __dir__(): "EvaluationRuleFilter_String", "EvaluationRuleFilter_StringObject", "EvaluationRuleFilter_StringOptions", - "EvaluationRuleMapping", - "EvaluationRuleMappingSource", "EvaluationRuleNullFilterOperator", "EvaluationRuleNumberFilterOperator", "EvaluationRuleOptionsFilterOperator", - "EvaluationRuleReadMapping", "EvaluationRuleStatus", "EvaluationRuleStringFilterOperator", "EvaluationRuleTarget", @@ -430,8 +427,8 @@ def __dir__(): "LegacyEvaluationObject", "LegacyEvaluationRule", "LegacyEvaluationRuleEvaluatorAssignment", - "LegacyEvaluationRuleMapping", "LegacyEvaluationRuleTarget", + "LegacyPromptVariableMapping", "LlmAsJudgeEvaluationRuleEvaluatorReference", "LlmAsJudgeEvaluator", "LlmAsJudgeEvaluatorType", @@ -441,6 +438,9 @@ def __dir__(): "NumberEvaluationRuleFilter", "NumberObjectEvaluationRuleFilter", "PresetPlacement", + "PromptVariableMappingInput", + "PromptVariableMappingRead", + "PromptVariableMappingSource", "PublicApiError", "PublicApiErrorCode", "PublicApiErrorDetails", diff --git a/langfuse/api/unstable/commons/__init__.py b/langfuse/api/unstable/commons/__init__.py index 748c1d8cd..61d7a2a87 100644 --- a/langfuse/api/unstable/commons/__init__.py +++ b/langfuse/api/unstable/commons/__init__.py @@ -25,12 +25,9 @@ EvaluationRuleFilter_String, EvaluationRuleFilter_StringObject, EvaluationRuleFilter_StringOptions, - EvaluationRuleMapping, - EvaluationRuleMappingSource, EvaluationRuleNullFilterOperator, EvaluationRuleNumberFilterOperator, EvaluationRuleOptionsFilterOperator, - EvaluationRuleReadMapping, EvaluationRuleStatus, EvaluationRuleStringFilterOperator, EvaluationRuleTarget, @@ -45,6 +42,9 @@ NullEvaluationRuleFilter, NumberEvaluationRuleFilter, NumberObjectEvaluationRuleFilter, + PromptVariableMappingInput, + PromptVariableMappingRead, + PromptVariableMappingSource, PublicBooleanEvaluatorOutputDefinition, PublicCategoricalEvaluatorOutputDefinition, PublicCategoricalEvaluatorOutputScoreDefinition, @@ -77,12 +77,9 @@ "EvaluationRuleFilter_String": ".types", "EvaluationRuleFilter_StringObject": ".types", "EvaluationRuleFilter_StringOptions": ".types", - "EvaluationRuleMapping": ".types", - "EvaluationRuleMappingSource": ".types", "EvaluationRuleNullFilterOperator": ".types", "EvaluationRuleNumberFilterOperator": ".types", "EvaluationRuleOptionsFilterOperator": ".types", - "EvaluationRuleReadMapping": ".types", "EvaluationRuleStatus": ".types", "EvaluationRuleStringFilterOperator": ".types", "EvaluationRuleTarget": ".types", @@ -97,6 +94,9 @@ "NullEvaluationRuleFilter": ".types", "NumberEvaluationRuleFilter": ".types", "NumberObjectEvaluationRuleFilter": ".types", + "PromptVariableMappingInput": ".types", + "PromptVariableMappingRead": ".types", + "PromptVariableMappingSource": ".types", "PublicBooleanEvaluatorOutputDefinition": ".types", "PublicCategoricalEvaluatorOutputDefinition": ".types", "PublicCategoricalEvaluatorOutputScoreDefinition": ".types", @@ -158,12 +158,9 @@ def __dir__(): "EvaluationRuleFilter_String", "EvaluationRuleFilter_StringObject", "EvaluationRuleFilter_StringOptions", - "EvaluationRuleMapping", - "EvaluationRuleMappingSource", "EvaluationRuleNullFilterOperator", "EvaluationRuleNumberFilterOperator", "EvaluationRuleOptionsFilterOperator", - "EvaluationRuleReadMapping", "EvaluationRuleStatus", "EvaluationRuleStringFilterOperator", "EvaluationRuleTarget", @@ -178,6 +175,9 @@ def __dir__(): "NullEvaluationRuleFilter", "NumberEvaluationRuleFilter", "NumberObjectEvaluationRuleFilter", + "PromptVariableMappingInput", + "PromptVariableMappingRead", + "PromptVariableMappingSource", "PublicBooleanEvaluatorOutputDefinition", "PublicCategoricalEvaluatorOutputDefinition", "PublicCategoricalEvaluatorOutputScoreDefinition", diff --git a/langfuse/api/unstable/commons/types/__init__.py b/langfuse/api/unstable/commons/types/__init__.py index 1b60a87c1..894b1d731 100644 --- a/langfuse/api/unstable/commons/types/__init__.py +++ b/langfuse/api/unstable/commons/types/__init__.py @@ -32,8 +32,6 @@ EvaluationRuleFilter_StringObject, EvaluationRuleFilter_StringOptions, ) - from .evaluation_rule_mapping import EvaluationRuleMapping - from .evaluation_rule_mapping_source import EvaluationRuleMappingSource from .evaluation_rule_null_filter_operator import EvaluationRuleNullFilterOperator from .evaluation_rule_number_filter_operator import ( EvaluationRuleNumberFilterOperator, @@ -41,7 +39,6 @@ from .evaluation_rule_options_filter_operator import ( EvaluationRuleOptionsFilterOperator, ) - from .evaluation_rule_read_mapping import EvaluationRuleReadMapping from .evaluation_rule_status import EvaluationRuleStatus from .evaluation_rule_string_filter_operator import ( EvaluationRuleStringFilterOperator, @@ -60,6 +57,9 @@ from .null_evaluation_rule_filter import NullEvaluationRuleFilter from .number_evaluation_rule_filter import NumberEvaluationRuleFilter from .number_object_evaluation_rule_filter import NumberObjectEvaluationRuleFilter + from .prompt_variable_mapping_input import PromptVariableMappingInput + from .prompt_variable_mapping_read import PromptVariableMappingRead + from .prompt_variable_mapping_source import PromptVariableMappingSource from .public_boolean_evaluator_output_definition import ( PublicBooleanEvaluatorOutputDefinition, ) @@ -103,12 +103,9 @@ "EvaluationRuleFilter_String": ".evaluation_rule_filter", "EvaluationRuleFilter_StringObject": ".evaluation_rule_filter", "EvaluationRuleFilter_StringOptions": ".evaluation_rule_filter", - "EvaluationRuleMapping": ".evaluation_rule_mapping", - "EvaluationRuleMappingSource": ".evaluation_rule_mapping_source", "EvaluationRuleNullFilterOperator": ".evaluation_rule_null_filter_operator", "EvaluationRuleNumberFilterOperator": ".evaluation_rule_number_filter_operator", "EvaluationRuleOptionsFilterOperator": ".evaluation_rule_options_filter_operator", - "EvaluationRuleReadMapping": ".evaluation_rule_read_mapping", "EvaluationRuleStatus": ".evaluation_rule_status", "EvaluationRuleStringFilterOperator": ".evaluation_rule_string_filter_operator", "EvaluationRuleTarget": ".evaluation_rule_target", @@ -123,6 +120,9 @@ "NullEvaluationRuleFilter": ".null_evaluation_rule_filter", "NumberEvaluationRuleFilter": ".number_evaluation_rule_filter", "NumberObjectEvaluationRuleFilter": ".number_object_evaluation_rule_filter", + "PromptVariableMappingInput": ".prompt_variable_mapping_input", + "PromptVariableMappingRead": ".prompt_variable_mapping_read", + "PromptVariableMappingSource": ".prompt_variable_mapping_source", "PublicBooleanEvaluatorOutputDefinition": ".public_boolean_evaluator_output_definition", "PublicCategoricalEvaluatorOutputDefinition": ".public_categorical_evaluator_output_definition", "PublicCategoricalEvaluatorOutputScoreDefinition": ".public_categorical_evaluator_output_score_definition", @@ -184,12 +184,9 @@ def __dir__(): "EvaluationRuleFilter_String", "EvaluationRuleFilter_StringObject", "EvaluationRuleFilter_StringOptions", - "EvaluationRuleMapping", - "EvaluationRuleMappingSource", "EvaluationRuleNullFilterOperator", "EvaluationRuleNumberFilterOperator", "EvaluationRuleOptionsFilterOperator", - "EvaluationRuleReadMapping", "EvaluationRuleStatus", "EvaluationRuleStringFilterOperator", "EvaluationRuleTarget", @@ -204,6 +201,9 @@ def __dir__(): "NullEvaluationRuleFilter", "NumberEvaluationRuleFilter", "NumberObjectEvaluationRuleFilter", + "PromptVariableMappingInput", + "PromptVariableMappingRead", + "PromptVariableMappingSource", "PublicBooleanEvaluatorOutputDefinition", "PublicCategoricalEvaluatorOutputDefinition", "PublicCategoricalEvaluatorOutputScoreDefinition", diff --git a/langfuse/api/unstable/commons/types/evaluator_type.py b/langfuse/api/unstable/commons/types/evaluator_type.py index f219fb7e1..633eb30ff 100644 --- a/langfuse/api/unstable/commons/types/evaluator_type.py +++ b/langfuse/api/unstable/commons/types/evaluator_type.py @@ -9,7 +9,7 @@ class EvaluatorType(enum.StrEnum): """ - The evaluator engine type. + The evaluator type. The unstable public API supports LLM-as-a-judge and code evaluators. """ diff --git a/langfuse/api/unstable/commons/types/evaluation_rule_mapping.py b/langfuse/api/unstable/commons/types/prompt_variable_mapping_input.py similarity index 80% rename from langfuse/api/unstable/commons/types/evaluation_rule_mapping.py rename to langfuse/api/unstable/commons/types/prompt_variable_mapping_input.py index 3e69a9986..478eabe52 100644 --- a/langfuse/api/unstable/commons/types/evaluation_rule_mapping.py +++ b/langfuse/api/unstable/commons/types/prompt_variable_mapping_input.py @@ -6,12 +6,12 @@ import typing_extensions from ....core.pydantic_utilities import UniversalBaseModel from ....core.serialization import FieldMetadata -from .evaluation_rule_mapping_source import EvaluationRuleMappingSource +from .prompt_variable_mapping_source import PromptVariableMappingSource -class EvaluationRuleMapping(UniversalBaseModel): +class PromptVariableMappingInput(UniversalBaseModel): """ - Maps one evaluator variable to one source field from the target object. + Connects one prompt variable to the available evaluation data. Manual mappings are used for `llm_as_judge` evaluators. `code` evaluators use a fixed runtime mapping managed by Langfuse. @@ -26,20 +26,20 @@ class EvaluationRuleMapping(UniversalBaseModel): Recovery guidance: - `invalid_variable_mapping`: the variable name is unknown for this evaluator, or the selected `source` is not valid for the chosen `target` - - `missing_variable_mapping`: one or more LLM-as-judge evaluator variables are not mapped yet - - `duplicate_variable_mapping`: the same evaluator variable appears more than once + - `missing_variable_mapping`: one or more LLM-as-judge prompt variables are not mapped yet + - `duplicate_variable_mapping`: the same prompt variable appears more than once - `invalid_json_path`: the JSONPath expression is malformed. Remove it or correct it. Examples -------- from langfuse.unstable.commons import ( - EvaluationRuleMapping, - EvaluationRuleMappingSource, + PromptVariableMappingInput, + PromptVariableMappingSource, ) - EvaluationRuleMapping( + PromptVariableMappingInput( variable="input", - source=EvaluationRuleMappingSource.INPUT, + source=PromptVariableMappingSource.INPUT, ) """ @@ -50,7 +50,7 @@ class EvaluationRuleMapping(UniversalBaseModel): Example: for the prompt `Judge {{input}} against {{output}}`, use `input` and `output`. """ - source: EvaluationRuleMappingSource = pydantic.Field() + source: PromptVariableMappingSource = pydantic.Field() """ Source field that should populate the prompt variable. diff --git a/langfuse/api/unstable/commons/types/evaluation_rule_read_mapping.py b/langfuse/api/unstable/commons/types/prompt_variable_mapping_read.py similarity index 50% rename from langfuse/api/unstable/commons/types/evaluation_rule_read_mapping.py rename to langfuse/api/unstable/commons/types/prompt_variable_mapping_read.py index 20cb403a7..370903955 100644 --- a/langfuse/api/unstable/commons/types/evaluation_rule_read_mapping.py +++ b/langfuse/api/unstable/commons/types/prompt_variable_mapping_read.py @@ -6,31 +6,25 @@ import typing_extensions from ....core.pydantic_utilities import UniversalBaseModel from ....core.serialization import FieldMetadata -from .evaluation_rule_mapping_source import EvaluationRuleMappingSource +from .prompt_variable_mapping_source import PromptVariableMappingSource -class EvaluationRuleReadMapping(UniversalBaseModel): +class PromptVariableMappingRead(UniversalBaseModel): """ - Maps an evaluator variable to target data, as returned by read endpoints. + Connects one prompt variable to data from a target. - Reads are more permissive than writes in two ways, so that data configured before a - migration stays readable and repairable: - - `source` is `null` when a mapping was never fully configured. Set a concrete `source` to fix it. - - every `source` value is allowed regardless of the rule `target`, because an evaluator's - default mapping is stored independently of the rules that use it. - - Requests always require a concrete `source` that is valid for the selected `target`. + `source` is `null` when the mapping is incomplete. Examples -------- from langfuse.unstable.commons import ( - EvaluationRuleMappingSource, - EvaluationRuleReadMapping, + PromptVariableMappingRead, + PromptVariableMappingSource, ) - EvaluationRuleReadMapping( + PromptVariableMappingRead( variable="input", - source=EvaluationRuleMappingSource.INPUT, + source=PromptVariableMappingSource.INPUT, ) """ @@ -39,7 +33,7 @@ class EvaluationRuleReadMapping(UniversalBaseModel): Prompt variable name without braces. """ - source: typing.Optional[EvaluationRuleMappingSource] = pydantic.Field(default=None) + source: typing.Optional[PromptVariableMappingSource] = pydantic.Field(default=None) """ Source field populating the variable, or `null` when the mapping is incomplete. """ diff --git a/langfuse/api/unstable/commons/types/evaluation_rule_mapping_source.py b/langfuse/api/unstable/commons/types/prompt_variable_mapping_source.py similarity index 83% rename from langfuse/api/unstable/commons/types/evaluation_rule_mapping_source.py rename to langfuse/api/unstable/commons/types/prompt_variable_mapping_source.py index 8e98605e9..37e5e19c4 100644 --- a/langfuse/api/unstable/commons/types/evaluation_rule_mapping_source.py +++ b/langfuse/api/unstable/commons/types/prompt_variable_mapping_source.py @@ -7,7 +7,7 @@ T_Result = typing.TypeVar("T_Result") -class EvaluationRuleMappingSource(enum.StrEnum): +class PromptVariableMappingSource(enum.StrEnum): """ Source field used to populate a prompt variable. @@ -42,15 +42,15 @@ def visit( expected_output: typing.Callable[[], T_Result], experiment_item_metadata: typing.Callable[[], T_Result], ) -> T_Result: - if self is EvaluationRuleMappingSource.INPUT: + if self is PromptVariableMappingSource.INPUT: return input() - if self is EvaluationRuleMappingSource.OUTPUT: + if self is PromptVariableMappingSource.OUTPUT: return output() - if self is EvaluationRuleMappingSource.METADATA: + if self is PromptVariableMappingSource.METADATA: return metadata() - if self is EvaluationRuleMappingSource.TOOL_CALLS: + if self is PromptVariableMappingSource.TOOL_CALLS: return tool_calls() - if self is EvaluationRuleMappingSource.EXPECTED_OUTPUT: + if self is PromptVariableMappingSource.EXPECTED_OUTPUT: return expected_output() - if self is EvaluationRuleMappingSource.EXPERIMENT_ITEM_METADATA: + if self is PromptVariableMappingSource.EXPERIMENT_ITEM_METADATA: return experiment_item_metadata() diff --git a/langfuse/api/unstable/evaluation_rules/__init__.py b/langfuse/api/unstable/evaluation_rules/__init__.py index f10039b83..478b15189 100644 --- a/langfuse/api/unstable/evaluation_rules/__init__.py +++ b/langfuse/api/unstable/evaluation_rules/__init__.py @@ -25,8 +25,8 @@ LegacyEvaluationObject, LegacyEvaluationRule, LegacyEvaluationRuleEvaluatorAssignment, - LegacyEvaluationRuleMapping, LegacyEvaluationRuleTarget, + LegacyPromptVariableMapping, LlmAsJudgeEvaluationRuleEvaluatorReference, LlmAsJudgeEvaluatorType, ReadableEvaluationRule, @@ -51,8 +51,8 @@ "LegacyEvaluationObject": ".types", "LegacyEvaluationRule": ".types", "LegacyEvaluationRuleEvaluatorAssignment": ".types", - "LegacyEvaluationRuleMapping": ".types", "LegacyEvaluationRuleTarget": ".types", + "LegacyPromptVariableMapping": ".types", "LlmAsJudgeEvaluationRuleEvaluatorReference": ".types", "LlmAsJudgeEvaluatorType": ".types", "ReadableEvaluationRule": ".types", @@ -106,8 +106,8 @@ def __dir__(): "LegacyEvaluationObject", "LegacyEvaluationRule", "LegacyEvaluationRuleEvaluatorAssignment", - "LegacyEvaluationRuleMapping", "LegacyEvaluationRuleTarget", + "LegacyPromptVariableMapping", "LlmAsJudgeEvaluationRuleEvaluatorReference", "LlmAsJudgeEvaluatorType", "ReadableEvaluationRule", diff --git a/langfuse/api/unstable/evaluation_rules/client.py b/langfuse/api/unstable/evaluation_rules/client.py index 598481a40..35a6fdd07 100644 --- a/langfuse/api/unstable/evaluation_rules/client.py +++ b/langfuse/api/unstable/evaluation_rules/client.py @@ -2,11 +2,12 @@ import typing +import typing_extensions from ...core.client_wrapper import AsyncClientWrapper, SyncClientWrapper from ...core.request_options import RequestOptions from ..commons.types.evaluation_rule_filter import EvaluationRuleFilter -from ..commons.types.evaluation_rule_mapping import EvaluationRuleMapping from ..commons.types.evaluation_rule_target import EvaluationRuleTarget +from ..commons.types.prompt_variable_mapping_input import PromptVariableMappingInput from .raw_client import AsyncRawEvaluationRulesClient, RawEvaluationRulesClient from .types.create_evaluation_rule_evaluator_assignment import ( CreateEvaluationRuleEvaluatorAssignment, @@ -37,6 +38,10 @@ def with_raw_response(self) -> RawEvaluationRulesClient: """ return self._raw_client + @typing_extensions.deprecated( + "On Langfuse Cloud, this unstable endpoint is deprecated and will be removed on November 16, 2026. Use the stable `/api/public/v2/evaluation-rules` API instead. Self-hosted deployments are unaffected by this date; the endpoint becomes unavailable when they upgrade to Langfuse v4.", + category=None, + ) def create( self, *, @@ -99,10 +104,10 @@ def create( from langfuse import LangfuseAPI from langfuse.unstable.commons import ( EvaluationRuleFilter_StringOptions, - EvaluationRuleMapping, - EvaluationRuleMappingSource, EvaluationRuleOptionsFilterOperator, EvaluationRuleTarget, + PromptVariableMappingInput, + PromptVariableMappingSource, ) from langfuse.unstable.evaluation_rules import ( CreateLlmAsJudgeEvaluationRuleRequest, @@ -136,13 +141,13 @@ def create( ) ], mapping=[ - EvaluationRuleMapping( + PromptVariableMappingInput( variable="input", - source=EvaluationRuleMappingSource.INPUT, + source=PromptVariableMappingSource.INPUT, ), - EvaluationRuleMapping( + PromptVariableMappingInput( variable="output", - source=EvaluationRuleMappingSource.OUTPUT, + source=PromptVariableMappingSource.OUTPUT, ), ], ), @@ -153,6 +158,10 @@ def create( ) return _response.data + @typing_extensions.deprecated( + "On Langfuse Cloud, this unstable endpoint is deprecated and will be removed on November 16, 2026. Use the stable `/api/public/v2/evaluation-rules` API instead. Self-hosted deployments are unaffected by this date; the endpoint becomes unavailable when they upgrade to Langfuse v4.", + category=None, + ) def list( self, *, @@ -199,6 +208,10 @@ def list( ) return _response.data + @typing_extensions.deprecated( + "On Langfuse Cloud, this unstable endpoint is deprecated and will be removed on November 16, 2026. Use the stable `/api/public/v2/evaluation-rules` API instead. Self-hosted deployments are unaffected by this date; the endpoint becomes unavailable when they upgrade to Langfuse v4.", + category=None, + ) def get( self, evaluation_rule_id: str, @@ -243,6 +256,10 @@ def get( ) return _response.data + @typing_extensions.deprecated( + "On Langfuse Cloud, this unstable endpoint is deprecated and will be removed on November 16, 2026. Use the stable `/api/public/v2/evaluation-rules` API instead. Self-hosted deployments are unaffected by this date; the endpoint becomes unavailable when they upgrade to Langfuse v4.", + category=None, + ) def update( self, evaluation_rule_id: str, @@ -256,7 +273,7 @@ def update( enabled: typing.Optional[bool] = OMIT, sampling: typing.Optional[float] = OMIT, filter: typing.Optional[typing.Sequence[EvaluationRuleFilter]] = OMIT, - mapping: typing.Optional[typing.Sequence[EvaluationRuleMapping]] = OMIT, + mapping: typing.Optional[typing.Sequence[PromptVariableMappingInput]] = OMIT, request_options: typing.Optional[RequestOptions] = None, ) -> EvaluationRule: """ @@ -316,7 +333,7 @@ def update( For `target=experiment`, `column=datasetId` expects dataset `id` values from `GET /api/public/v2/datasets`, not dataset names. - mapping : typing.Optional[typing.Sequence[EvaluationRuleMapping]] + mapping : typing.Optional[typing.Sequence[PromptVariableMappingInput]] Updated LLM-as-judge variable mappings. Do not send this field for code evaluator rules. Langfuse stores the fixed code runtime mapping automatically and returns it in the response. @@ -358,6 +375,10 @@ def update( ) return _response.data + @typing_extensions.deprecated( + "On Langfuse Cloud, this unstable endpoint is deprecated and will be removed on November 16, 2026. Use the stable `/api/public/v2/evaluation-rules` API instead. Self-hosted deployments are unaffected by this date; the endpoint becomes unavailable when they upgrade to Langfuse v4.", + category=None, + ) def delete( self, evaluation_rule_id: str, @@ -418,6 +439,10 @@ def with_raw_response(self) -> AsyncRawEvaluationRulesClient: """ return self._raw_client + @typing_extensions.deprecated( + "On Langfuse Cloud, this unstable endpoint is deprecated and will be removed on November 16, 2026. Use the stable `/api/public/v2/evaluation-rules` API instead. Self-hosted deployments are unaffected by this date; the endpoint becomes unavailable when they upgrade to Langfuse v4.", + category=None, + ) async def create( self, *, @@ -482,10 +507,10 @@ async def create( from langfuse import AsyncLangfuseAPI from langfuse.unstable.commons import ( EvaluationRuleFilter_StringOptions, - EvaluationRuleMapping, - EvaluationRuleMappingSource, EvaluationRuleOptionsFilterOperator, EvaluationRuleTarget, + PromptVariableMappingInput, + PromptVariableMappingSource, ) from langfuse.unstable.evaluation_rules import ( CreateLlmAsJudgeEvaluationRuleRequest, @@ -522,13 +547,13 @@ async def main() -> None: ) ], mapping=[ - EvaluationRuleMapping( + PromptVariableMappingInput( variable="input", - source=EvaluationRuleMappingSource.INPUT, + source=PromptVariableMappingSource.INPUT, ), - EvaluationRuleMapping( + PromptVariableMappingInput( variable="output", - source=EvaluationRuleMappingSource.OUTPUT, + source=PromptVariableMappingSource.OUTPUT, ), ], ), @@ -542,6 +567,10 @@ async def main() -> None: ) return _response.data + @typing_extensions.deprecated( + "On Langfuse Cloud, this unstable endpoint is deprecated and will be removed on November 16, 2026. Use the stable `/api/public/v2/evaluation-rules` API instead. Self-hosted deployments are unaffected by this date; the endpoint becomes unavailable when they upgrade to Langfuse v4.", + category=None, + ) async def list( self, *, @@ -596,6 +625,10 @@ async def main() -> None: ) return _response.data + @typing_extensions.deprecated( + "On Langfuse Cloud, this unstable endpoint is deprecated and will be removed on November 16, 2026. Use the stable `/api/public/v2/evaluation-rules` API instead. Self-hosted deployments are unaffected by this date; the endpoint becomes unavailable when they upgrade to Langfuse v4.", + category=None, + ) async def get( self, evaluation_rule_id: str, @@ -648,6 +681,10 @@ async def main() -> None: ) return _response.data + @typing_extensions.deprecated( + "On Langfuse Cloud, this unstable endpoint is deprecated and will be removed on November 16, 2026. Use the stable `/api/public/v2/evaluation-rules` API instead. Self-hosted deployments are unaffected by this date; the endpoint becomes unavailable when they upgrade to Langfuse v4.", + category=None, + ) async def update( self, evaluation_rule_id: str, @@ -661,7 +698,7 @@ async def update( enabled: typing.Optional[bool] = OMIT, sampling: typing.Optional[float] = OMIT, filter: typing.Optional[typing.Sequence[EvaluationRuleFilter]] = OMIT, - mapping: typing.Optional[typing.Sequence[EvaluationRuleMapping]] = OMIT, + mapping: typing.Optional[typing.Sequence[PromptVariableMappingInput]] = OMIT, request_options: typing.Optional[RequestOptions] = None, ) -> EvaluationRule: """ @@ -721,7 +758,7 @@ async def update( For `target=experiment`, `column=datasetId` expects dataset `id` values from `GET /api/public/v2/datasets`, not dataset names. - mapping : typing.Optional[typing.Sequence[EvaluationRuleMapping]] + mapping : typing.Optional[typing.Sequence[PromptVariableMappingInput]] Updated LLM-as-judge variable mappings. Do not send this field for code evaluator rules. Langfuse stores the fixed code runtime mapping automatically and returns it in the response. @@ -771,6 +808,10 @@ async def main() -> None: ) return _response.data + @typing_extensions.deprecated( + "On Langfuse Cloud, this unstable endpoint is deprecated and will be removed on November 16, 2026. Use the stable `/api/public/v2/evaluation-rules` API instead. Self-hosted deployments are unaffected by this date; the endpoint becomes unavailable when they upgrade to Langfuse v4.", + category=None, + ) async def delete( self, evaluation_rule_id: str, diff --git a/langfuse/api/unstable/evaluation_rules/raw_client.py b/langfuse/api/unstable/evaluation_rules/raw_client.py index 607f87f3c..a445b891a 100644 --- a/langfuse/api/unstable/evaluation_rules/raw_client.py +++ b/langfuse/api/unstable/evaluation_rules/raw_client.py @@ -1,6 +1,7 @@ # This file was auto-generated by Fern from our API Definition. import typing +import typing_extensions from json.decoder import JSONDecodeError from ...commons.errors.access_denied_error import ( @@ -24,8 +25,8 @@ from ...core.request_options import RequestOptions from ...core.serialization import convert_and_respect_annotation_metadata from ..commons.types.evaluation_rule_filter import EvaluationRuleFilter -from ..commons.types.evaluation_rule_mapping import EvaluationRuleMapping from ..commons.types.evaluation_rule_target import EvaluationRuleTarget +from ..commons.types.prompt_variable_mapping_input import PromptVariableMappingInput from ..errors.errors.access_denied_error import ( AccessDeniedError as unstable_errors_errors_access_denied_error_AccessDeniedError, ) @@ -62,6 +63,10 @@ class RawEvaluationRulesClient: def __init__(self, *, client_wrapper: SyncClientWrapper): self._client_wrapper = client_wrapper + @typing_extensions.deprecated( + "On Langfuse Cloud, this unstable endpoint is deprecated and will be removed on November 16, 2026. Use the stable `/api/public/v2/evaluation-rules` API instead. Self-hosted deployments are unaffected by this date; the endpoint becomes unavailable when they upgrade to Langfuse v4.", + category=None, + ) def create( self, *, @@ -307,6 +312,10 @@ def create( body=_response_json, ) + @typing_extensions.deprecated( + "On Langfuse Cloud, this unstable endpoint is deprecated and will be removed on November 16, 2026. Use the stable `/api/public/v2/evaluation-rules` API instead. Self-hosted deployments are unaffected by this date; the endpoint becomes unavailable when they upgrade to Langfuse v4.", + category=None, + ) def list( self, *, @@ -487,6 +496,10 @@ def list( body=_response_json, ) + @typing_extensions.deprecated( + "On Langfuse Cloud, this unstable endpoint is deprecated and will be removed on November 16, 2026. Use the stable `/api/public/v2/evaluation-rules` API instead. Self-hosted deployments are unaffected by this date; the endpoint becomes unavailable when they upgrade to Langfuse v4.", + category=None, + ) def get( self, evaluation_rule_id: str, @@ -670,6 +683,10 @@ def get( body=_response_json, ) + @typing_extensions.deprecated( + "On Langfuse Cloud, this unstable endpoint is deprecated and will be removed on November 16, 2026. Use the stable `/api/public/v2/evaluation-rules` API instead. Self-hosted deployments are unaffected by this date; the endpoint becomes unavailable when they upgrade to Langfuse v4.", + category=None, + ) def update( self, evaluation_rule_id: str, @@ -683,7 +700,7 @@ def update( enabled: typing.Optional[bool] = OMIT, sampling: typing.Optional[float] = OMIT, filter: typing.Optional[typing.Sequence[EvaluationRuleFilter]] = OMIT, - mapping: typing.Optional[typing.Sequence[EvaluationRuleMapping]] = OMIT, + mapping: typing.Optional[typing.Sequence[PromptVariableMappingInput]] = OMIT, request_options: typing.Optional[RequestOptions] = None, ) -> HttpResponse[EvaluationRule]: """ @@ -743,7 +760,7 @@ def update( For `target=experiment`, `column=datasetId` expects dataset `id` values from `GET /api/public/v2/datasets`, not dataset names. - mapping : typing.Optional[typing.Sequence[EvaluationRuleMapping]] + mapping : typing.Optional[typing.Sequence[PromptVariableMappingInput]] Updated LLM-as-judge variable mappings. Do not send this field for code evaluator rules. Langfuse stores the fixed code runtime mapping automatically and returns it in the response. @@ -780,7 +797,7 @@ def update( ), "mapping": convert_and_respect_annotation_metadata( object_=mapping, - annotation=typing.Sequence[EvaluationRuleMapping], + annotation=typing.Sequence[PromptVariableMappingInput], direction="write", ), }, @@ -953,6 +970,10 @@ def update( body=_response_json, ) + @typing_extensions.deprecated( + "On Langfuse Cloud, this unstable endpoint is deprecated and will be removed on November 16, 2026. Use the stable `/api/public/v2/evaluation-rules` API instead. Self-hosted deployments are unaffected by this date; the endpoint becomes unavailable when they upgrade to Langfuse v4.", + category=None, + ) def delete( self, evaluation_rule_id: str, @@ -1141,6 +1162,10 @@ class AsyncRawEvaluationRulesClient: def __init__(self, *, client_wrapper: AsyncClientWrapper): self._client_wrapper = client_wrapper + @typing_extensions.deprecated( + "On Langfuse Cloud, this unstable endpoint is deprecated and will be removed on November 16, 2026. Use the stable `/api/public/v2/evaluation-rules` API instead. Self-hosted deployments are unaffected by this date; the endpoint becomes unavailable when they upgrade to Langfuse v4.", + category=None, + ) async def create( self, *, @@ -1386,6 +1411,10 @@ async def create( body=_response_json, ) + @typing_extensions.deprecated( + "On Langfuse Cloud, this unstable endpoint is deprecated and will be removed on November 16, 2026. Use the stable `/api/public/v2/evaluation-rules` API instead. Self-hosted deployments are unaffected by this date; the endpoint becomes unavailable when they upgrade to Langfuse v4.", + category=None, + ) async def list( self, *, @@ -1566,6 +1595,10 @@ async def list( body=_response_json, ) + @typing_extensions.deprecated( + "On Langfuse Cloud, this unstable endpoint is deprecated and will be removed on November 16, 2026. Use the stable `/api/public/v2/evaluation-rules` API instead. Self-hosted deployments are unaffected by this date; the endpoint becomes unavailable when they upgrade to Langfuse v4.", + category=None, + ) async def get( self, evaluation_rule_id: str, @@ -1749,6 +1782,10 @@ async def get( body=_response_json, ) + @typing_extensions.deprecated( + "On Langfuse Cloud, this unstable endpoint is deprecated and will be removed on November 16, 2026. Use the stable `/api/public/v2/evaluation-rules` API instead. Self-hosted deployments are unaffected by this date; the endpoint becomes unavailable when they upgrade to Langfuse v4.", + category=None, + ) async def update( self, evaluation_rule_id: str, @@ -1762,7 +1799,7 @@ async def update( enabled: typing.Optional[bool] = OMIT, sampling: typing.Optional[float] = OMIT, filter: typing.Optional[typing.Sequence[EvaluationRuleFilter]] = OMIT, - mapping: typing.Optional[typing.Sequence[EvaluationRuleMapping]] = OMIT, + mapping: typing.Optional[typing.Sequence[PromptVariableMappingInput]] = OMIT, request_options: typing.Optional[RequestOptions] = None, ) -> AsyncHttpResponse[EvaluationRule]: """ @@ -1822,7 +1859,7 @@ async def update( For `target=experiment`, `column=datasetId` expects dataset `id` values from `GET /api/public/v2/datasets`, not dataset names. - mapping : typing.Optional[typing.Sequence[EvaluationRuleMapping]] + mapping : typing.Optional[typing.Sequence[PromptVariableMappingInput]] Updated LLM-as-judge variable mappings. Do not send this field for code evaluator rules. Langfuse stores the fixed code runtime mapping automatically and returns it in the response. @@ -1859,7 +1896,7 @@ async def update( ), "mapping": convert_and_respect_annotation_metadata( object_=mapping, - annotation=typing.Sequence[EvaluationRuleMapping], + annotation=typing.Sequence[PromptVariableMappingInput], direction="write", ), }, @@ -2032,6 +2069,10 @@ async def update( body=_response_json, ) + @typing_extensions.deprecated( + "On Langfuse Cloud, this unstable endpoint is deprecated and will be removed on November 16, 2026. Use the stable `/api/public/v2/evaluation-rules` API instead. Self-hosted deployments are unaffected by this date; the endpoint becomes unavailable when they upgrade to Langfuse v4.", + category=None, + ) async def delete( self, evaluation_rule_id: str, diff --git a/langfuse/api/unstable/evaluation_rules/types/__init__.py b/langfuse/api/unstable/evaluation_rules/types/__init__.py index c8198e458..d45705039 100644 --- a/langfuse/api/unstable/evaluation_rules/types/__init__.py +++ b/langfuse/api/unstable/evaluation_rules/types/__init__.py @@ -36,8 +36,8 @@ from .legacy_evaluation_rule_evaluator_assignment import ( LegacyEvaluationRuleEvaluatorAssignment, ) - from .legacy_evaluation_rule_mapping import LegacyEvaluationRuleMapping from .legacy_evaluation_rule_target import LegacyEvaluationRuleTarget + from .legacy_prompt_variable_mapping import LegacyPromptVariableMapping from .llm_as_judge_evaluation_rule_evaluator_reference import ( LlmAsJudgeEvaluationRuleEvaluatorReference, ) @@ -63,8 +63,8 @@ "LegacyEvaluationObject": ".legacy_evaluation_object", "LegacyEvaluationRule": ".legacy_evaluation_rule", "LegacyEvaluationRuleEvaluatorAssignment": ".legacy_evaluation_rule_evaluator_assignment", - "LegacyEvaluationRuleMapping": ".legacy_evaluation_rule_mapping", "LegacyEvaluationRuleTarget": ".legacy_evaluation_rule_target", + "LegacyPromptVariableMapping": ".legacy_prompt_variable_mapping", "LlmAsJudgeEvaluationRuleEvaluatorReference": ".llm_as_judge_evaluation_rule_evaluator_reference", "LlmAsJudgeEvaluatorType": ".llm_as_judge_evaluator_type", "ReadableEvaluationRule": ".readable_evaluation_rule", @@ -118,8 +118,8 @@ def __dir__(): "LegacyEvaluationObject", "LegacyEvaluationRule", "LegacyEvaluationRuleEvaluatorAssignment", - "LegacyEvaluationRuleMapping", "LegacyEvaluationRuleTarget", + "LegacyPromptVariableMapping", "LlmAsJudgeEvaluationRuleEvaluatorReference", "LlmAsJudgeEvaluatorType", "ReadableEvaluationRule", diff --git a/langfuse/api/unstable/evaluation_rules/types/create_evaluation_rule_evaluator_assignment.py b/langfuse/api/unstable/evaluation_rules/types/create_evaluation_rule_evaluator_assignment.py index a48961404..a69c59072 100644 --- a/langfuse/api/unstable/evaluation_rules/types/create_evaluation_rule_evaluator_assignment.py +++ b/langfuse/api/unstable/evaluation_rules/types/create_evaluation_rule_evaluator_assignment.py @@ -4,7 +4,7 @@ import pydantic from ....core.pydantic_utilities import UniversalBaseModel -from ...commons.types.evaluation_rule_mapping import EvaluationRuleMapping +from ...commons.types.prompt_variable_mapping_input import PromptVariableMappingInput from .create_evaluation_rule_evaluator_reference import ( CreateEvaluationRuleEvaluatorReference, ) @@ -12,7 +12,7 @@ class CreateEvaluationRuleEvaluatorAssignment(UniversalBaseModel): evaluator: CreateEvaluationRuleEvaluatorReference - mapping: typing.Optional[typing.List[EvaluationRuleMapping]] = pydantic.Field( + mapping: typing.Optional[typing.List[PromptVariableMappingInput]] = pydantic.Field( default=None ) """ diff --git a/langfuse/api/unstable/evaluation_rules/types/create_llm_as_judge_evaluation_rule_request.py b/langfuse/api/unstable/evaluation_rules/types/create_llm_as_judge_evaluation_rule_request.py index 715df7765..b8347bc30 100644 --- a/langfuse/api/unstable/evaluation_rules/types/create_llm_as_judge_evaluation_rule_request.py +++ b/langfuse/api/unstable/evaluation_rules/types/create_llm_as_judge_evaluation_rule_request.py @@ -5,8 +5,8 @@ import pydantic from ....core.pydantic_utilities import UniversalBaseModel from ...commons.types.evaluation_rule_filter import EvaluationRuleFilter -from ...commons.types.evaluation_rule_mapping import EvaluationRuleMapping from ...commons.types.evaluation_rule_target import EvaluationRuleTarget +from ...commons.types.prompt_variable_mapping_input import PromptVariableMappingInput from .llm_as_judge_evaluation_rule_evaluator_reference import ( LlmAsJudgeEvaluationRuleEvaluatorReference, ) @@ -52,12 +52,12 @@ class CreateLlmAsJudgeEvaluationRuleRequest(UniversalBaseModel): For `target=experiment`, `column=datasetId` expects dataset `id` values from `GET /api/public/v2/datasets`, not dataset names. """ - mapping: typing.List[EvaluationRuleMapping] = pydantic.Field() + mapping: typing.List[PromptVariableMappingInput] = pydantic.Field() """ LLM-as-judge variable mappings. - Every evaluator variable must appear exactly once. - Build this list from the evaluator `variables` array returned by the evaluator endpoints. + Every prompt variable must appear exactly once. + Build this list from the evaluator's `variables` array. """ model_config: typing.ClassVar[pydantic.ConfigDict] = pydantic.ConfigDict( diff --git a/langfuse/api/unstable/evaluation_rules/types/evaluation_rule.py b/langfuse/api/unstable/evaluation_rules/types/evaluation_rule.py index 6fabadd4e..7d48501f3 100644 --- a/langfuse/api/unstable/evaluation_rules/types/evaluation_rule.py +++ b/langfuse/api/unstable/evaluation_rules/types/evaluation_rule.py @@ -4,8 +4,8 @@ import pydantic from ...commons.types.evaluation_rule_filter import EvaluationRuleFilter -from ...commons.types.evaluation_rule_read_mapping import EvaluationRuleReadMapping from ...commons.types.evaluation_rule_target import EvaluationRuleTarget +from ...commons.types.prompt_variable_mapping_read import PromptVariableMappingRead from .evaluation_rule_base import EvaluationRuleBase from .evaluation_rule_evaluator_assignment import EvaluationRuleEvaluatorAssignment @@ -18,12 +18,12 @@ class EvaluationRule(EvaluationRuleBase): from langfuse.unstable.commons import ( EvaluationRuleFilter_StringOptions, - EvaluationRuleMappingSource, EvaluationRuleOptionsFilterOperator, - EvaluationRuleReadMapping, EvaluationRuleStatus, EvaluationRuleTarget, EvaluatorType, + PromptVariableMappingRead, + PromptVariableMappingSource, ) from langfuse.unstable.evaluation_rules import ( EvaluationRule, @@ -47,13 +47,13 @@ class EvaluationRule(EvaluationRuleBase): type=EvaluatorType.LLM_AS_JUDGE, ), mapping=[ - EvaluationRuleReadMapping( + PromptVariableMappingRead( variable="input", - source=EvaluationRuleMappingSource.INPUT, + source=PromptVariableMappingSource.INPUT, ), - EvaluationRuleReadMapping( + PromptVariableMappingRead( variable="output", - source=EvaluationRuleMappingSource.OUTPUT, + source=PromptVariableMappingSource.OUTPUT, ), ], ) @@ -70,13 +70,13 @@ class EvaluationRule(EvaluationRuleBase): ) ], mapping=[ - EvaluationRuleReadMapping( + PromptVariableMappingRead( variable="input", - source=EvaluationRuleMappingSource.INPUT, + source=PromptVariableMappingSource.INPUT, ), - EvaluationRuleReadMapping( + PromptVariableMappingRead( variable="output", - source=EvaluationRuleMappingSource.OUTPUT, + source=PromptVariableMappingSource.OUTPUT, ), ], created_at=datetime.datetime.fromisoformat( @@ -103,7 +103,7 @@ class EvaluationRule(EvaluationRuleBase): List of filter conditions used to decide whether a target should be evaluated. """ - mapping: typing.List[EvaluationRuleReadMapping] = pydantic.Field() + mapping: typing.List[PromptVariableMappingRead] = pydantic.Field() """ Deprecated compatibility alias containing the effective mapping for `evaluators[0]`. """ diff --git a/langfuse/api/unstable/evaluation_rules/types/evaluation_rule_base.py b/langfuse/api/unstable/evaluation_rules/types/evaluation_rule_base.py index 95ec471c9..9ee19d860 100644 --- a/langfuse/api/unstable/evaluation_rules/types/evaluation_rule_base.py +++ b/langfuse/api/unstable/evaluation_rules/types/evaluation_rule_base.py @@ -19,7 +19,7 @@ class EvaluationRuleBase(UniversalBaseModel): - which evaluator should be used - which target objects should trigger scoring - how often scoring should run - - which target fields should populate each evaluator variable + - which target fields should populate each prompt variable - whether the deployment is active, inactive, or paused Important status semantics: diff --git a/langfuse/api/unstable/evaluation_rules/types/evaluation_rule_evaluator.py b/langfuse/api/unstable/evaluation_rules/types/evaluation_rule_evaluator.py index a2f36d885..57f8378ee 100644 --- a/langfuse/api/unstable/evaluation_rules/types/evaluation_rule_evaluator.py +++ b/langfuse/api/unstable/evaluation_rules/types/evaluation_rule_evaluator.py @@ -27,7 +27,7 @@ class EvaluationRuleEvaluator(UniversalBaseModel): type: EvaluatorType = pydantic.Field() """ - Evaluator engine type. + Evaluator type. """ model_config: typing.ClassVar[pydantic.ConfigDict] = pydantic.ConfigDict( diff --git a/langfuse/api/unstable/evaluation_rules/types/evaluation_rule_evaluator_assignment.py b/langfuse/api/unstable/evaluation_rules/types/evaluation_rule_evaluator_assignment.py index 2062dcbe3..b299fd5a9 100644 --- a/langfuse/api/unstable/evaluation_rules/types/evaluation_rule_evaluator_assignment.py +++ b/langfuse/api/unstable/evaluation_rules/types/evaluation_rule_evaluator_assignment.py @@ -4,13 +4,13 @@ import pydantic from ....core.pydantic_utilities import UniversalBaseModel -from ...commons.types.evaluation_rule_read_mapping import EvaluationRuleReadMapping +from ...commons.types.prompt_variable_mapping_read import PromptVariableMappingRead from .evaluation_rule_evaluator import EvaluationRuleEvaluator class EvaluationRuleEvaluatorAssignment(UniversalBaseModel): evaluator: EvaluationRuleEvaluator - mapping: typing.Optional[typing.List[EvaluationRuleReadMapping]] = pydantic.Field( + mapping: typing.Optional[typing.List[PromptVariableMappingRead]] = pydantic.Field( default=None ) """ diff --git a/langfuse/api/unstable/evaluation_rules/types/legacy_evaluation_object.py b/langfuse/api/unstable/evaluation_rules/types/legacy_evaluation_object.py index 0cf92f67e..7f952d7b9 100644 --- a/langfuse/api/unstable/evaluation_rules/types/legacy_evaluation_object.py +++ b/langfuse/api/unstable/evaluation_rules/types/legacy_evaluation_object.py @@ -8,6 +8,10 @@ class LegacyEvaluationObject(enum.StrEnum): + """ + **Deprecated:** Legacy Langfuse object used by a trace or dataset evaluation rule. + """ + TRACE = "trace" SPAN = "span" GENERATION = "generation" diff --git a/langfuse/api/unstable/evaluation_rules/types/legacy_evaluation_rule.py b/langfuse/api/unstable/evaluation_rules/types/legacy_evaluation_rule.py index c2aeb3a3d..7ce85a386 100644 --- a/langfuse/api/unstable/evaluation_rules/types/legacy_evaluation_rule.py +++ b/langfuse/api/unstable/evaluation_rules/types/legacy_evaluation_rule.py @@ -11,15 +11,15 @@ from .legacy_evaluation_rule_evaluator_assignment import ( LegacyEvaluationRuleEvaluatorAssignment, ) -from .legacy_evaluation_rule_mapping import LegacyEvaluationRuleMapping from .legacy_evaluation_rule_target import LegacyEvaluationRuleTarget +from .legacy_prompt_variable_mapping import LegacyPromptVariableMapping class LegacyEvaluationRule(EvaluationRuleBase): """ - Legacy trace- or dataset-level evaluation rule returned by list and get for migration. + **Deprecated:** Legacy trace- or dataset-level evaluation rule returned by list and get for migration. - This resource is read-only through the unstable public API. Its mapping preserves the trace, dataset item, or named observation selected for each evaluator variable. + This resource is read-only through the unstable public API. Its mapping preserves the trace, dataset item, or named observation selected for each prompt variable. """ evaluators: typing.List[LegacyEvaluationRuleEvaluatorAssignment] = pydantic.Field() @@ -45,7 +45,7 @@ class LegacyEvaluationRule(EvaluationRuleBase): Stored filters used by the legacy trace or dataset rule. """ - mapping: typing.List[LegacyEvaluationRuleMapping] = pydantic.Field() + mapping: typing.List[LegacyPromptVariableMapping] = pydantic.Field() """ Stored variable mappings, including the trace, dataset item, or named observation selected for each variable. """ diff --git a/langfuse/api/unstable/evaluation_rules/types/legacy_evaluation_rule_evaluator_assignment.py b/langfuse/api/unstable/evaluation_rules/types/legacy_evaluation_rule_evaluator_assignment.py index 146f49f7c..00233e0c8 100644 --- a/langfuse/api/unstable/evaluation_rules/types/legacy_evaluation_rule_evaluator_assignment.py +++ b/langfuse/api/unstable/evaluation_rules/types/legacy_evaluation_rule_evaluator_assignment.py @@ -5,12 +5,16 @@ import pydantic from ....core.pydantic_utilities import UniversalBaseModel from .evaluation_rule_evaluator import EvaluationRuleEvaluator -from .legacy_evaluation_rule_mapping import LegacyEvaluationRuleMapping +from .legacy_prompt_variable_mapping import LegacyPromptVariableMapping class LegacyEvaluationRuleEvaluatorAssignment(UniversalBaseModel): + """ + **Deprecated:** Evaluator assignment on a legacy trace or dataset rule. + """ + evaluator: EvaluationRuleEvaluator - mapping: typing.Optional[typing.List[LegacyEvaluationRuleMapping]] = None + mapping: typing.Optional[typing.List[LegacyPromptVariableMapping]] = None model_config: typing.ClassVar[pydantic.ConfigDict] = pydantic.ConfigDict( extra="allow", frozen=True diff --git a/langfuse/api/unstable/evaluation_rules/types/legacy_evaluation_rule_target.py b/langfuse/api/unstable/evaluation_rules/types/legacy_evaluation_rule_target.py index c90d59399..24603666d 100644 --- a/langfuse/api/unstable/evaluation_rules/types/legacy_evaluation_rule_target.py +++ b/langfuse/api/unstable/evaluation_rules/types/legacy_evaluation_rule_target.py @@ -8,6 +8,10 @@ class LegacyEvaluationRuleTarget(enum.StrEnum): + """ + **Deprecated:** Legacy trace or dataset evaluation target. + """ + TRACE = "trace" DATASET = "dataset" diff --git a/langfuse/api/unstable/evaluation_rules/types/legacy_evaluation_rule_mapping.py b/langfuse/api/unstable/evaluation_rules/types/legacy_prompt_variable_mapping.py similarity index 84% rename from langfuse/api/unstable/evaluation_rules/types/legacy_evaluation_rule_mapping.py rename to langfuse/api/unstable/evaluation_rules/types/legacy_prompt_variable_mapping.py index b740b0e29..238694d9e 100644 --- a/langfuse/api/unstable/evaluation_rules/types/legacy_evaluation_rule_mapping.py +++ b/langfuse/api/unstable/evaluation_rules/types/legacy_prompt_variable_mapping.py @@ -9,9 +9,9 @@ from .legacy_evaluation_object import LegacyEvaluationObject -class LegacyEvaluationRuleMapping(UniversalBaseModel): +class LegacyPromptVariableMapping(UniversalBaseModel): """ - Maps one evaluator variable to a trace, dataset item, or field on a named observation in a legacy rule. + **Deprecated:** Connects one prompt variable to data from a trace, dataset item, or named observation in a legacy rule. """ variable: str diff --git a/langfuse/api/unstable/evaluation_rules/types/llm_as_judge_evaluation_rule_evaluator_reference.py b/langfuse/api/unstable/evaluation_rules/types/llm_as_judge_evaluation_rule_evaluator_reference.py index 7747b7b10..14062c1d3 100644 --- a/langfuse/api/unstable/evaluation_rules/types/llm_as_judge_evaluation_rule_evaluator_reference.py +++ b/langfuse/api/unstable/evaluation_rules/types/llm_as_judge_evaluation_rule_evaluator_reference.py @@ -19,7 +19,7 @@ class LlmAsJudgeEvaluationRuleEvaluatorReference(UniversalBaseModel): type: typing.Optional[LlmAsJudgeEvaluatorType] = pydantic.Field(default=None) """ - Evaluator engine type. Defaults to `llm_as_judge` when omitted. + Evaluator type. Defaults to `llm_as_judge` when omitted. """ model_config: typing.ClassVar[pydantic.ConfigDict] = pydantic.ConfigDict( diff --git a/langfuse/api/unstable/evaluation_rules/types/update_evaluation_rule_request.py b/langfuse/api/unstable/evaluation_rules/types/update_evaluation_rule_request.py index e81c52c99..567ab59f3 100644 --- a/langfuse/api/unstable/evaluation_rules/types/update_evaluation_rule_request.py +++ b/langfuse/api/unstable/evaluation_rules/types/update_evaluation_rule_request.py @@ -5,8 +5,8 @@ import pydantic from ....core.pydantic_utilities import UniversalBaseModel from ...commons.types.evaluation_rule_filter import EvaluationRuleFilter -from ...commons.types.evaluation_rule_mapping import EvaluationRuleMapping from ...commons.types.evaluation_rule_target import EvaluationRuleTarget +from ...commons.types.prompt_variable_mapping_input import PromptVariableMappingInput from .create_evaluation_rule_evaluator_assignment import ( CreateEvaluationRuleEvaluatorAssignment, ) @@ -24,7 +24,7 @@ class UpdateEvaluationRuleRequest(UniversalBaseModel): - If you only want to rename the rule or change sampling, send just those fields. - To add, remove, or remap evaluators, send `evaluators`. It replaces the whole assignment set, so include every evaluator the rule should keep. - `evaluators` cannot be combined with the deprecated `evaluator`/`mapping` pair, which only ever addressed the first assignment. - - If you change to an LLM-as-judge `evaluator`, send a fresh `mapping` unless you are certain the existing mapping still matches the evaluator variables. + - If you change to an LLM-as-judge `evaluator`, send a fresh `mapping` unless you are certain the existing mapping still matches the prompt variables. - If you change `target` for an LLM-as-judge rule, usually send both `filter` and `mapping` in the same request. - For code evaluator rules, omit `mapping`; Langfuse stores the fixed code runtime mapping automatically. - If you change an experiment `datasetId` filter, call `GET /api/public/v2/datasets` and use dataset `id` values from that response. @@ -79,7 +79,7 @@ class UpdateEvaluationRuleRequest(UniversalBaseModel): For `target=experiment`, `column=datasetId` expects dataset `id` values from `GET /api/public/v2/datasets`, not dataset names. """ - mapping: typing.Optional[typing.List[EvaluationRuleMapping]] = pydantic.Field( + mapping: typing.Optional[typing.List[PromptVariableMappingInput]] = pydantic.Field( default=None ) """ diff --git a/langfuse/api/unstable/evaluators/client.py b/langfuse/api/unstable/evaluators/client.py index 272792a5e..a4d74c3b0 100644 --- a/langfuse/api/unstable/evaluators/client.py +++ b/langfuse/api/unstable/evaluators/client.py @@ -2,6 +2,7 @@ import typing +import typing_extensions from ...core.client_wrapper import AsyncClientWrapper, SyncClientWrapper from ...core.request_options import RequestOptions from .raw_client import AsyncRawEvaluatorsClient, RawEvaluatorsClient @@ -29,6 +30,10 @@ def with_raw_response(self) -> RawEvaluatorsClient: """ return self._raw_client + @typing_extensions.deprecated( + "On Langfuse Cloud, this unstable endpoint is deprecated and will be removed on November 16, 2026. Use the stable `/api/public/v2/evaluators` API instead. Self-hosted deployments are unaffected by this date; the endpoint becomes unavailable when they upgrade to Langfuse v4.", + category=None, + ) def create( self, *, @@ -82,12 +87,12 @@ def create( -------- from langfuse import LangfuseAPI from langfuse.unstable.commons import ( - EvaluationRuleMapping, - EvaluationRuleMappingSource, EvaluatorModelConfig, EvaluatorOutputDataType, EvaluatorOutputDefinition_Numeric, EvaluatorOutputFieldDefinition, + PromptVariableMappingInput, + PromptVariableMappingSource, PublicNumericEvaluatorOutputScoreDefinition, ) from langfuse.unstable.evaluators import CreateEvaluatorRequest_LlmAsJudge @@ -118,13 +123,13 @@ def create( model="gpt-4.1-mini", ), mapping=[ - EvaluationRuleMapping( + PromptVariableMappingInput( variable="input", - source=EvaluationRuleMappingSource.INPUT, + source=PromptVariableMappingSource.INPUT, ), - EvaluationRuleMapping( + PromptVariableMappingInput( variable="output", - source=EvaluationRuleMappingSource.OUTPUT, + source=PromptVariableMappingSource.OUTPUT, ), ], ), @@ -135,6 +140,10 @@ def create( ) return _response.data + @typing_extensions.deprecated( + "On Langfuse Cloud, this unstable endpoint is deprecated and will be removed on November 16, 2026. Use the stable `/api/public/v2/evaluators` API instead. Self-hosted deployments are unaffected by this date; the endpoint becomes unavailable when they upgrade to Langfuse v4.", + category=None, + ) def list( self, *, @@ -183,6 +192,10 @@ def list( ) return _response.data + @typing_extensions.deprecated( + "On Langfuse Cloud, this unstable endpoint is deprecated and will be removed on November 16, 2026. Use the stable `/api/public/v2/evaluators` API instead. Self-hosted deployments are unaffected by this date; the endpoint becomes unavailable when they upgrade to Langfuse v4.", + category=None, + ) def get( self, evaluator_id: str, @@ -225,6 +238,10 @@ def get( _response = self._raw_client.get(evaluator_id, request_options=request_options) return _response.data + @typing_extensions.deprecated( + "On Langfuse Cloud, this unstable endpoint is deprecated and will be removed on November 16, 2026. Use the stable `/api/public/v2/evaluators` API instead. Self-hosted deployments are unaffected by this date; the endpoint becomes unavailable when they upgrade to Langfuse v4.", + category=None, + ) def delete( self, evaluator_id: str, @@ -288,6 +305,10 @@ def with_raw_response(self) -> AsyncRawEvaluatorsClient: """ return self._raw_client + @typing_extensions.deprecated( + "On Langfuse Cloud, this unstable endpoint is deprecated and will be removed on November 16, 2026. Use the stable `/api/public/v2/evaluators` API instead. Self-hosted deployments are unaffected by this date; the endpoint becomes unavailable when they upgrade to Langfuse v4.", + category=None, + ) async def create( self, *, @@ -343,12 +364,12 @@ async def create( from langfuse import AsyncLangfuseAPI from langfuse.unstable.commons import ( - EvaluationRuleMapping, - EvaluationRuleMappingSource, EvaluatorModelConfig, EvaluatorOutputDataType, EvaluatorOutputDefinition_Numeric, EvaluatorOutputFieldDefinition, + PromptVariableMappingInput, + PromptVariableMappingSource, PublicNumericEvaluatorOutputScoreDefinition, ) from langfuse.unstable.evaluators import CreateEvaluatorRequest_LlmAsJudge @@ -382,13 +403,13 @@ async def main() -> None: model="gpt-4.1-mini", ), mapping=[ - EvaluationRuleMapping( + PromptVariableMappingInput( variable="input", - source=EvaluationRuleMappingSource.INPUT, + source=PromptVariableMappingSource.INPUT, ), - EvaluationRuleMapping( + PromptVariableMappingInput( variable="output", - source=EvaluationRuleMappingSource.OUTPUT, + source=PromptVariableMappingSource.OUTPUT, ), ], ), @@ -402,6 +423,10 @@ async def main() -> None: ) return _response.data + @typing_extensions.deprecated( + "On Langfuse Cloud, this unstable endpoint is deprecated and will be removed on November 16, 2026. Use the stable `/api/public/v2/evaluators` API instead. Self-hosted deployments are unaffected by this date; the endpoint becomes unavailable when they upgrade to Langfuse v4.", + category=None, + ) async def list( self, *, @@ -458,6 +483,10 @@ async def main() -> None: ) return _response.data + @typing_extensions.deprecated( + "On Langfuse Cloud, this unstable endpoint is deprecated and will be removed on November 16, 2026. Use the stable `/api/public/v2/evaluators` API instead. Self-hosted deployments are unaffected by this date; the endpoint becomes unavailable when they upgrade to Langfuse v4.", + category=None, + ) async def get( self, evaluator_id: str, @@ -510,6 +539,10 @@ async def main() -> None: ) return _response.data + @typing_extensions.deprecated( + "On Langfuse Cloud, this unstable endpoint is deprecated and will be removed on November 16, 2026. Use the stable `/api/public/v2/evaluators` API instead. Self-hosted deployments are unaffected by this date; the endpoint becomes unavailable when they upgrade to Langfuse v4.", + category=None, + ) async def delete( self, evaluator_id: str, diff --git a/langfuse/api/unstable/evaluators/raw_client.py b/langfuse/api/unstable/evaluators/raw_client.py index 8e08ee8e4..e0110306a 100644 --- a/langfuse/api/unstable/evaluators/raw_client.py +++ b/langfuse/api/unstable/evaluators/raw_client.py @@ -1,6 +1,7 @@ # This file was auto-generated by Fern from our API Definition. import typing +import typing_extensions from json.decoder import JSONDecodeError from ...commons.errors.access_denied_error import ( @@ -54,6 +55,10 @@ class RawEvaluatorsClient: def __init__(self, *, client_wrapper: SyncClientWrapper): self._client_wrapper = client_wrapper + @typing_extensions.deprecated( + "On Langfuse Cloud, this unstable endpoint is deprecated and will be removed on November 16, 2026. Use the stable `/api/public/v2/evaluators` API instead. Self-hosted deployments are unaffected by this date; the endpoint becomes unavailable when they upgrade to Langfuse v4.", + category=None, + ) def create( self, *, @@ -278,6 +283,10 @@ def create( body=_response_json, ) + @typing_extensions.deprecated( + "On Langfuse Cloud, this unstable endpoint is deprecated and will be removed on November 16, 2026. Use the stable `/api/public/v2/evaluators` API instead. Self-hosted deployments are unaffected by this date; the endpoint becomes unavailable when they upgrade to Langfuse v4.", + category=None, + ) def list( self, *, @@ -460,6 +469,10 @@ def list( body=_response_json, ) + @typing_extensions.deprecated( + "On Langfuse Cloud, this unstable endpoint is deprecated and will be removed on November 16, 2026. Use the stable `/api/public/v2/evaluators` API instead. Self-hosted deployments are unaffected by this date; the endpoint becomes unavailable when they upgrade to Langfuse v4.", + category=None, + ) def get( self, evaluator_id: str, @@ -643,6 +656,10 @@ def get( body=_response_json, ) + @typing_extensions.deprecated( + "On Langfuse Cloud, this unstable endpoint is deprecated and will be removed on November 16, 2026. Use the stable `/api/public/v2/evaluators` API instead. Self-hosted deployments are unaffected by this date; the endpoint becomes unavailable when they upgrade to Langfuse v4.", + category=None, + ) def delete( self, evaluator_id: str, @@ -834,6 +851,10 @@ class AsyncRawEvaluatorsClient: def __init__(self, *, client_wrapper: AsyncClientWrapper): self._client_wrapper = client_wrapper + @typing_extensions.deprecated( + "On Langfuse Cloud, this unstable endpoint is deprecated and will be removed on November 16, 2026. Use the stable `/api/public/v2/evaluators` API instead. Self-hosted deployments are unaffected by this date; the endpoint becomes unavailable when they upgrade to Langfuse v4.", + category=None, + ) async def create( self, *, @@ -1058,6 +1079,10 @@ async def create( body=_response_json, ) + @typing_extensions.deprecated( + "On Langfuse Cloud, this unstable endpoint is deprecated and will be removed on November 16, 2026. Use the stable `/api/public/v2/evaluators` API instead. Self-hosted deployments are unaffected by this date; the endpoint becomes unavailable when they upgrade to Langfuse v4.", + category=None, + ) async def list( self, *, @@ -1240,6 +1265,10 @@ async def list( body=_response_json, ) + @typing_extensions.deprecated( + "On Langfuse Cloud, this unstable endpoint is deprecated and will be removed on November 16, 2026. Use the stable `/api/public/v2/evaluators` API instead. Self-hosted deployments are unaffected by this date; the endpoint becomes unavailable when they upgrade to Langfuse v4.", + category=None, + ) async def get( self, evaluator_id: str, @@ -1423,6 +1452,10 @@ async def get( body=_response_json, ) + @typing_extensions.deprecated( + "On Langfuse Cloud, this unstable endpoint is deprecated and will be removed on November 16, 2026. Use the stable `/api/public/v2/evaluators` API instead. Self-hosted deployments are unaffected by this date; the endpoint becomes unavailable when they upgrade to Langfuse v4.", + category=None, + ) async def delete( self, evaluator_id: str, diff --git a/langfuse/api/unstable/evaluators/types/create_evaluator_request.py b/langfuse/api/unstable/evaluators/types/create_evaluator_request.py index 94b7ad620..ea7e3c23c 100644 --- a/langfuse/api/unstable/evaluators/types/create_evaluator_request.py +++ b/langfuse/api/unstable/evaluators/types/create_evaluator_request.py @@ -11,9 +11,9 @@ from ...commons.types.code_evaluator_source_code_language import ( CodeEvaluatorSourceCodeLanguage, ) -from ...commons.types.evaluation_rule_mapping import EvaluationRuleMapping from ...commons.types.evaluator_model_config import EvaluatorModelConfig from ...commons.types.evaluator_output_definition import EvaluatorOutputDefinition +from ...commons.types.prompt_variable_mapping_input import PromptVariableMappingInput class CreateEvaluatorRequest_LlmAsJudge(UniversalBaseModel): @@ -34,7 +34,7 @@ class CreateEvaluatorRequest_LlmAsJudge(UniversalBaseModel): model_config_: typing_extensions.Annotated[ typing.Optional[EvaluatorModelConfig], FieldMetadata(alias="modelConfig") ] = None - mapping: typing.Optional[typing.List[EvaluationRuleMapping]] = None + mapping: typing.Optional[typing.List[PromptVariableMappingInput]] = None model_config: typing.ClassVar[pydantic.ConfigDict] = pydantic.ConfigDict( extra="allow", frozen=True diff --git a/langfuse/api/unstable/evaluators/types/create_llm_as_judge_evaluator_request.py b/langfuse/api/unstable/evaluators/types/create_llm_as_judge_evaluator_request.py index 456a142f9..a90f71c72 100644 --- a/langfuse/api/unstable/evaluators/types/create_llm_as_judge_evaluator_request.py +++ b/langfuse/api/unstable/evaluators/types/create_llm_as_judge_evaluator_request.py @@ -6,9 +6,9 @@ import typing_extensions from ....core.pydantic_utilities import UniversalBaseModel from ....core.serialization import FieldMetadata -from ...commons.types.evaluation_rule_mapping import EvaluationRuleMapping from ...commons.types.evaluator_model_config import EvaluatorModelConfig from ...commons.types.evaluator_output_definition import EvaluatorOutputDefinition +from ...commons.types.prompt_variable_mapping_input import PromptVariableMappingInput class CreateLlmAsJudgeEvaluatorRequest(UniversalBaseModel): @@ -39,7 +39,7 @@ class CreateLlmAsJudgeEvaluatorRequest(UniversalBaseModel): Optional explicit model configuration. Omit or set to `null` to use the project default evaluation model. """ - mapping: typing.Optional[typing.List[EvaluationRuleMapping]] = pydantic.Field( + mapping: typing.Optional[typing.List[PromptVariableMappingInput]] = pydantic.Field( default=None ) """ diff --git a/langfuse/api/unstable/evaluators/types/evaluator.py b/langfuse/api/unstable/evaluators/types/evaluator.py index 61995dada..2b8387bc2 100644 --- a/langfuse/api/unstable/evaluators/types/evaluator.py +++ b/langfuse/api/unstable/evaluators/types/evaluator.py @@ -12,8 +12,8 @@ from ...commons.types.code_evaluator_source_code_language import ( CodeEvaluatorSourceCodeLanguage, ) -from ...commons.types.evaluation_rule_read_mapping import EvaluationRuleReadMapping from ...commons.types.evaluator_model_config import EvaluatorModelConfig +from ...commons.types.prompt_variable_mapping_read import PromptVariableMappingRead from ...commons.types.public_evaluator_output_definition import ( PublicEvaluatorOutputDefinition, ) @@ -50,7 +50,7 @@ class Evaluator_LlmAsJudge(UniversalBaseModel): name: str version: int variables: typing.List[str] - mapping: typing.Optional[typing.List[EvaluationRuleReadMapping]] = None + mapping: typing.Optional[typing.List[PromptVariableMappingRead]] = None evaluation_rule_count: typing_extensions.Annotated[ int, FieldMetadata(alias="evaluationRuleCount") ] @@ -94,7 +94,7 @@ class Evaluator_Code(UniversalBaseModel): name: str version: int variables: typing.List[str] - mapping: typing.Optional[typing.List[EvaluationRuleReadMapping]] = None + mapping: typing.Optional[typing.List[PromptVariableMappingRead]] = None evaluation_rule_count: typing_extensions.Annotated[ int, FieldMetadata(alias="evaluationRuleCount") ] diff --git a/langfuse/api/unstable/evaluators/types/evaluator_base.py b/langfuse/api/unstable/evaluators/types/evaluator_base.py index 00ff20a11..6bf6484cd 100644 --- a/langfuse/api/unstable/evaluators/types/evaluator_base.py +++ b/langfuse/api/unstable/evaluators/types/evaluator_base.py @@ -7,7 +7,7 @@ import typing_extensions from ....core.pydantic_utilities import UniversalBaseModel from ....core.serialization import FieldMetadata -from ...commons.types.evaluation_rule_read_mapping import EvaluationRuleReadMapping +from ...commons.types.prompt_variable_mapping_read import PromptVariableMappingRead class EvaluatorBase(UniversalBaseModel): @@ -33,7 +33,7 @@ class EvaluatorBase(UniversalBaseModel): LLM evaluators require every variable to be mapped exactly once. Code evaluators always expose the fixed runtime payload fields and Langfuse maps them automatically. """ - mapping: typing.Optional[typing.List[EvaluationRuleReadMapping]] = pydantic.Field( + mapping: typing.Optional[typing.List[PromptVariableMappingRead]] = pydantic.Field( default=None ) """