diff --git a/.generator/schemas/v2/openapi.yaml b/.generator/schemas/v2/openapi.yaml index 52c5ef104a4d..1f302592b0b8 100644 --- a/.generator/schemas/v2/openapi.yaml +++ b/.generator/schemas/v2/openapi.yaml @@ -47690,18 +47690,34 @@ components: description: Name of the experiment. example: "My Experiment v1" type: string + parent_experiment_id: + description: Identifier of the parent (baseline) experiment this experiment is run against. + example: "3fd6b5e0-8910-4b1c-a7d0-5b84de329012" + type: string project_id: description: Identifier of the project this experiment belongs to. example: "a33671aa-24fd-4dcd-9b33-a8ec7dde7751" type: string + run_count: + description: Number of runs configured for this experiment. + format: int32 + maximum: 2147483647 + type: integer required: - project_id - - dataset_id - name type: object LLMObsExperimentDataAttributesResponse: description: Attributes of an LLM Observability experiment. properties: + aggregate_data: + additionalProperties: {} + description: >- + Pre-computed aggregate metrics for this experiment run, including eval score distributions, token costs, and error rates. + nullable: true + type: object + author: + $ref: "#/components/schemas/LLMObsExperimentUser" config: additionalProperties: {} description: Configuration parameters for the experiment. @@ -47716,11 +47732,34 @@ components: description: Identifier of the dataset used in this experiment. example: "9f64e5c7-dc5a-45c8-a17c-1b85f0bec97d" type: string + dataset_name: + description: |- + Name of the dataset used in this experiment. + Only present when `include[dataset_names]` is `true`. + nullable: true + type: string + dataset_version: + description: Version of the dataset used in this experiment. + format: int64 + type: integer + deleted_at: + description: Timestamp when the experiment was soft-deleted, if applicable. + format: date-time + nullable: true + type: string description: description: Description of the experiment. example: "" nullable: true type: string + error: + description: Error message describing why the experiment failed, if applicable. + nullable: true + type: string + experiment: + description: Logical name of the experiment, shared across all runs of the same pipeline. + example: "my-pipeline" + type: string metadata: additionalProperties: {} description: Arbitrary metadata associated with the experiment. @@ -47730,10 +47769,22 @@ components: description: Name of the experiment. example: "My Experiment v1" type: string + parent_experiment_id: + description: Identifier of the parent (baseline) experiment this experiment was run against, if any. + example: "3fd6b5e0-8910-4b1c-a7d0-5b84de329012" + nullable: true + type: string project_id: description: Identifier of the project this experiment belongs to. example: "a33671aa-24fd-4dcd-9b33-a8ec7dde7751" type: string + run_count: + description: Expected number of runs for this experiment. + format: int32 + maximum: 2147483647 + type: integer + status: + $ref: "#/components/schemas/LLMObsExperimentStatus" updated_at: description: Timestamp when the experiment was last updated. example: "2024-01-15T10:30:00Z" @@ -48088,6 +48139,22 @@ components: - duration - status type: object + LLMObsExperimentSpanDataResponse: + description: JSON:API data item wrapping a single experiment span with evaluations. + properties: + attributes: + $ref: "#/components/schemas/LLMObsExperimentSpanWithEvals" + id: + description: Unique identifier of the span. + example: "00000000-0000-0000-0000-000000000001" + type: string + type: + $ref: "#/components/schemas/LLMObsExperimentSpanType" + required: + - id + - type + - attributes + type: object LLMObsExperimentSpanError: description: Error details for an experiment span. properties: @@ -48128,6 +48195,14 @@ components: x-enum-varnames: - OK - ERROR + LLMObsExperimentSpanType: + description: Resource type for a span item in an experiment spans response. + enum: + - experiments + example: experiments + type: string + x-enum-varnames: + - EXPERIMENTS_SPAN LLMObsExperimentSpanWithEvals: description: An experiment span enriched with its associated evaluation metrics. properties: @@ -48185,6 +48260,32 @@ components: example: "abc123def456" type: string type: object + LLMObsExperimentSpansResponse: + description: >- + Response for listing experiment spans (v1). Returns only spans with their evaluation metrics. No summary metrics or pagination are included. Deprecated in favor of `ListLLMObsExperimentEventsV3`. + properties: + data: + description: List of experiment spans with their evaluation metrics. + items: + $ref: "#/components/schemas/LLMObsExperimentSpanDataResponse" + type: array + required: + - data + type: object + LLMObsExperimentStatus: + description: Execution status of an LLM Observability experiment. + enum: + - running + - completed + - failed + - interrupted + example: completed + type: string + x-enum-varnames: + - RUNNING + - COMPLETED + - FAILED + - INTERRUPTED LLMObsExperimentType: description: Resource type of an LLM Observability experiment. enum: @@ -48196,12 +48297,25 @@ components: LLMObsExperimentUpdateDataAttributesRequest: description: Attributes for updating an LLM Observability experiment. properties: + dataset_id: + description: Updated identifier of the dataset used in this experiment. + example: "9f64e5c7-dc5a-45c8-a17c-1b85f0bec97d" + type: string description: description: Updated description of the experiment. type: string + error: + description: Error message describing why the experiment failed, if applicable. + type: string + metadata: + additionalProperties: {} + description: Updated arbitrary metadata associated with the experiment. + type: object name: description: Updated name of the experiment. type: string + status: + $ref: "#/components/schemas/LLMObsExperimentStatus" type: object LLMObsExperimentUpdateDataRequest: description: Data object for updating an LLM Observability experiment. @@ -48222,6 +48336,30 @@ components: required: - data type: object + LLMObsExperimentUser: + description: User data for the author of an experiment. Only present when `include[user_data]` is `true`. + properties: + email: + description: Email address of the user. + example: "jane.doe@example.com" + type: string + handle: + description: Username or handle associated with the user's Datadog account. + example: "jane.doe@example.com" + type: string + icon: + description: URL of the user's icon. + example: "https://example.com/icon.png" + type: string + id: + description: Unique identifier of the user. + example: "00000000-0000-0000-0000-000000000010" + type: string + name: + description: Display name of the user. + example: "Jane Doe" + type: string + type: object LLMObsExperimentationAnalyticsAggregate: description: Analytics aggregation parameters. properties: @@ -132311,16 +132449,59 @@ paths: name: filter[id] schema: type: string - - description: Use the Pagination cursor to retrieve the next page of results. + - description: Filter experiments by their exact run name. + in: query + name: filter[name] + schema: + type: string + - description: >- + Filter by logical experiment name. This is the `name` field set when creating an experiment through `POST /experiments`. Returns all experiment runs that share the same name, enabling cross-commit and cross-branch comparisons. + in: query + name: filter[experiment] + schema: + type: string + - description: |- + Filter by JSONB metadata containment. Provide a JSON object string where + experiments whose metadata contains all specified key-value pairs are returned. + For example: `{"commit":"abc123","branch":"main"}`. + in: query + name: filter[metadata] + schema: + type: string + - description: >- + Filter experiments by the ID of their parent (baseline) experiment. Returns all experiments that were run against the given baseline. Can be specified multiple times. + in: query + name: filter[parent_experiment_id] + schema: + type: string + - description: When `true`, return only soft-deleted experiments. Defaults to `false`. + in: query + name: filter[is_deleted] + schema: + type: boolean + - description: When `true`, enrich each experiment with its author's user data in the `author` field. + in: query + name: include[user_data] + schema: + type: boolean + - description: When `true`, enrich each experiment with its dataset name in the `dataset_name` field. + in: query + name: include[dataset_names] + schema: + type: boolean + - description: Use the pagination cursor returned in `meta.after` to retrieve the next page of results. in: query name: page[cursor] schema: type: string - - description: Maximum number of results to return per page. + - description: |- + Maximum number of results to return per page. Values above 5000 are clamped + to 5000. Defaults to 5000. in: query name: page[limit] schema: format: int64 + maximum: 5000 type: integer responses: "200": @@ -132591,6 +132772,76 @@ paths: **Note**: This endpoint is in preview and is subject to change. If you have any feedback, contact [Datadog support](https://docs.datadoghq.com/help/). /api/v2/llm-obs/v1/experiments/{experiment_id}/events: + get: + deprecated: true + description: >- + Retrieve spans with their evaluation metrics for a given experiment. Returns spans only, with no summary metrics and no pagination. Deprecated in favor of `ListLLMObsExperimentEventsV3`. + operationId: ListLLMObsExperimentEventsV1 + parameters: + - $ref: "#/components/parameters/LLMObsExperimentIDPathParameter" + responses: + "200": + content: + application/json: + examples: + default: + value: + data: + - attributes: + duration: 1500000000.0 + eval_metrics: [] + name: llm_call + span_id: span-7a1b2c3d + start_ns: 1705314600000000000 + status: ok + tags: [] + trace_id: abc123def456 + id: 00000000-0000-0000-0000-000000000001 + type: experiments + schema: + $ref: "#/components/schemas/LLMObsExperimentSpansResponse" + description: OK + "400": + content: + application/json: + schema: + $ref: "#/components/schemas/JSONAPIErrorResponse" + description: Bad Request + "401": + content: + application/json: + schema: + $ref: "#/components/schemas/JSONAPIErrorResponse" + description: Unauthorized + "403": + content: + application/json: + schema: + $ref: "#/components/schemas/JSONAPIErrorResponse" + description: Forbidden + "404": + content: + application/json: + schema: + $ref: "#/components/schemas/JSONAPIErrorResponse" + description: Not Found + "429": + $ref: "#/components/responses/TooManyRequestsResponse" + "500": + content: + application/json: + schema: + $ref: "#/components/schemas/JSONAPIErrorResponse" + description: Internal Server Error + security: + - apiKeyAuth: [] + appKeyAuth: [] + summary: List LLM Observability experiment spans (v1) + tags: + - LLM Observability + x-unstable: |- + **Note**: This endpoint is in preview and is subject to change. + If you have any feedback, contact [Datadog support](https://docs.datadoghq.com/help/). post: description: Push spans and metrics for an LLM Observability experiment. operationId: CreateLLMObsExperimentEvents @@ -134676,6 +134927,80 @@ paths: x-unstable: |- **Note**: This endpoint is in preview and is subject to change. If you have any feedback, contact [Datadog support](https://docs.datadoghq.com/help/). + /api/v2/llm-obs/v2/experiments/{experiment_id}/events: + get: + deprecated: true + description: >- + Retrieve spans and experiment-level summary metrics for a given experiment. Returns the full events payload without pagination. Deprecated: use `ListLLMObsExperimentEventsV3` instead. + operationId: ListLLMObsExperimentEventsV2 + parameters: + - $ref: "#/components/parameters/LLMObsExperimentIDPathParameter" + responses: + "200": + content: + application/json: + examples: + default: + value: + data: + attributes: + spans: + - duration: 1500000000.0 + eval_metrics: [] + id: 00000000-0000-0000-0000-000000000002 + name: llm_call + span_id: span-7a1b2c3d + start_ns: 1705314600000000000 + status: ok + tags: [] + trace_id: abc123def456 + summary_metrics: [] + id: 00000000-0000-0000-0000-000000000001 + type: experiment_events + schema: + $ref: "#/components/schemas/LLMObsExperimentEventsV2Response" + description: OK + "400": + content: + application/json: + schema: + $ref: "#/components/schemas/JSONAPIErrorResponse" + description: Bad Request + "401": + content: + application/json: + schema: + $ref: "#/components/schemas/JSONAPIErrorResponse" + description: Unauthorized + "403": + content: + application/json: + schema: + $ref: "#/components/schemas/JSONAPIErrorResponse" + description: Forbidden + "404": + content: + application/json: + schema: + $ref: "#/components/schemas/JSONAPIErrorResponse" + description: Not Found + "429": + $ref: "#/components/responses/TooManyRequestsResponse" + "500": + content: + application/json: + schema: + $ref: "#/components/schemas/JSONAPIErrorResponse" + description: Internal Server Error + security: + - apiKeyAuth: [] + appKeyAuth: [] + summary: List LLM Observability experiment events (v2) + tags: + - LLM Observability + x-unstable: |- + **Note**: This endpoint is in preview and is subject to change. + If you have any feedback, contact [Datadog support](https://docs.datadoghq.com/help/). /api/v2/llm-obs/v2/{project_id}/datasets/{dataset_id}/records/upload: post: description: |- diff --git a/examples/v2/llm-observability/CreateLLMObsExperiment.rb b/examples/v2/llm-observability/CreateLLMObsExperiment.rb index d2bdfa15d6d4..49e7eab5a6fa 100644 --- a/examples/v2/llm-observability/CreateLLMObsExperiment.rb +++ b/examples/v2/llm-observability/CreateLLMObsExperiment.rb @@ -11,6 +11,7 @@ attributes: DatadogAPIClient::V2::LLMObsExperimentDataAttributesRequest.new({ dataset_id: "9f64e5c7-dc5a-45c8-a17c-1b85f0bec97d", name: "My Experiment v1", + parent_experiment_id: "3fd6b5e0-8910-4b1c-a7d0-5b84de329012", project_id: "a33671aa-24fd-4dcd-9b33-a8ec7dde7751", }), type: DatadogAPIClient::V2::LLMObsExperimentType::EXPERIMENTS, diff --git a/examples/v2/llm-observability/ListLLMObsExperimentEventsV1.rb b/examples/v2/llm-observability/ListLLMObsExperimentEventsV1.rb new file mode 100644 index 000000000000..879b89f02735 --- /dev/null +++ b/examples/v2/llm-observability/ListLLMObsExperimentEventsV1.rb @@ -0,0 +1,8 @@ +# List LLM Observability experiment spans (v1) returns "OK" response + +require "datadog_api_client" +DatadogAPIClient.configure do |config| + config.unstable_operations["v2.list_llm_obs_experiment_events_v1".to_sym] = true +end +api_instance = DatadogAPIClient::V2::LLMObservabilityAPI.new +p api_instance.list_llm_obs_experiment_events_v1("experiment_id") diff --git a/examples/v2/llm-observability/ListLLMObsExperimentEventsV2.rb b/examples/v2/llm-observability/ListLLMObsExperimentEventsV2.rb new file mode 100644 index 000000000000..025e23853b7d --- /dev/null +++ b/examples/v2/llm-observability/ListLLMObsExperimentEventsV2.rb @@ -0,0 +1,8 @@ +# List LLM Observability experiment events (v2) returns "OK" response + +require "datadog_api_client" +DatadogAPIClient.configure do |config| + config.unstable_operations["v2.list_llm_obs_experiment_events_v2".to_sym] = true +end +api_instance = DatadogAPIClient::V2::LLMObservabilityAPI.new +p api_instance.list_llm_obs_experiment_events_v2("experiment_id") diff --git a/examples/v2/llm-observability/UpdateLLMObsExperiment.rb b/examples/v2/llm-observability/UpdateLLMObsExperiment.rb index ddfbd759c679..6ca2de0640ae 100644 --- a/examples/v2/llm-observability/UpdateLLMObsExperiment.rb +++ b/examples/v2/llm-observability/UpdateLLMObsExperiment.rb @@ -8,7 +8,10 @@ body = DatadogAPIClient::V2::LLMObsExperimentUpdateRequest.new({ data: DatadogAPIClient::V2::LLMObsExperimentUpdateDataRequest.new({ - attributes: DatadogAPIClient::V2::LLMObsExperimentUpdateDataAttributesRequest.new({}), + attributes: DatadogAPIClient::V2::LLMObsExperimentUpdateDataAttributesRequest.new({ + dataset_id: "9f64e5c7-dc5a-45c8-a17c-1b85f0bec97d", + status: DatadogAPIClient::V2::LLMObsExperimentStatus::COMPLETED, + }), type: DatadogAPIClient::V2::LLMObsExperimentType::EXPERIMENTS, }), }) diff --git a/features/scenarios_model_mapping.rb b/features/scenarios_model_mapping.rb index c3e7e7142da3..a4b0df539af2 100644 --- a/features/scenarios_model_mapping.rb +++ b/features/scenarios_model_mapping.rb @@ -972,6 +972,13 @@ "filter_project_id" => "String", "filter_dataset_id" => "String", "filter_id" => "String", + "filter_name" => "String", + "filter_experiment" => "String", + "filter_metadata" => "String", + "filter_parent_experiment_id" => "String", + "filter_is_deleted" => "Boolean", + "include_user_data" => "Boolean", + "include_dataset_names" => "Boolean", "page_cursor" => "String", "page_limit" => "Integer", }, @@ -985,6 +992,9 @@ "experiment_id" => "String", "body" => "LLMObsExperimentUpdateRequest", }, + "v2.ListLLMObsExperimentEventsV1" => { + "experiment_id" => "String", + }, "v2.CreateLLMObsExperimentEvents" => { "experiment_id" => "String", "body" => "LLMObsExperimentEventsRequest", @@ -1113,6 +1123,9 @@ "project_id" => "String", "dataset_id" => "String", }, + "v2.ListLLMObsExperimentEventsV2" => { + "experiment_id" => "String", + }, "v2.UploadLLMObsDatasetRecordsFile" => { "project_id" => "String", "dataset_id" => "String", diff --git a/features/v2/llm_observability.feature b/features/v2/llm_observability.feature index ea384fa8ed44..36976da92ebb 100644 --- a/features/v2/llm_observability.feature +++ b/features/v2/llm_observability.feature @@ -235,7 +235,7 @@ Feature: LLM Observability Scenario: Create an LLM Observability experiment returns "Bad Request" response Given operation "CreateLLMObsExperiment" enabled And new "CreateLLMObsExperiment" request - And body with value {"data": {"attributes": {"dataset_id": "9f64e5c7-dc5a-45c8-a17c-1b85f0bec97d", "name": "My Experiment v1", "project_id": "a33671aa-24fd-4dcd-9b33-a8ec7dde7751"}, "type": "experiments"}} + And body with value {"data": {"attributes": {"dataset_id": "9f64e5c7-dc5a-45c8-a17c-1b85f0bec97d", "name": "My Experiment v1", "parent_experiment_id": "3fd6b5e0-8910-4b1c-a7d0-5b84de329012", "project_id": "a33671aa-24fd-4dcd-9b33-a8ec7dde7751"}, "type": "experiments"}} When the request is sent Then the response status is 400 Bad Request @@ -243,7 +243,7 @@ Feature: LLM Observability Scenario: Create an LLM Observability experiment returns "Created" response Given operation "CreateLLMObsExperiment" enabled And new "CreateLLMObsExperiment" request - And body with value {"data": {"attributes": {"dataset_id": "9f64e5c7-dc5a-45c8-a17c-1b85f0bec97d", "name": "My Experiment v1", "project_id": "a33671aa-24fd-4dcd-9b33-a8ec7dde7751"}, "type": "experiments"}} + And body with value {"data": {"attributes": {"dataset_id": "9f64e5c7-dc5a-45c8-a17c-1b85f0bec97d", "name": "My Experiment v1", "parent_experiment_id": "3fd6b5e0-8910-4b1c-a7d0-5b84de329012", "project_id": "a33671aa-24fd-4dcd-9b33-a8ec7dde7751"}, "type": "experiments"}} When the request is sent Then the response status is 201 Created @@ -251,7 +251,7 @@ Feature: LLM Observability Scenario: Create an LLM Observability experiment returns "OK" response Given operation "CreateLLMObsExperiment" enabled And new "CreateLLMObsExperiment" request - And body with value {"data": {"attributes": {"dataset_id": "9f64e5c7-dc5a-45c8-a17c-1b85f0bec97d", "name": "My Experiment v1", "project_id": "a33671aa-24fd-4dcd-9b33-a8ec7dde7751"}, "type": "experiments"}} + And body with value {"data": {"attributes": {"dataset_id": "9f64e5c7-dc5a-45c8-a17c-1b85f0bec97d", "name": "My Experiment v1", "parent_experiment_id": "3fd6b5e0-8910-4b1c-a7d0-5b84de329012", "project_id": "a33671aa-24fd-4dcd-9b33-a8ec7dde7751"}, "type": "experiments"}} When the request is sent Then the response status is 200 OK @@ -713,6 +713,54 @@ Feature: LLM Observability When the request is sent Then the response status is 200 OK + @generated @skip @team:DataDog/ml-observability + Scenario: List LLM Observability experiment events (v2) returns "Bad Request" response + Given operation "ListLLMObsExperimentEventsV2" enabled + And new "ListLLMObsExperimentEventsV2" request + And request contains "experiment_id" parameter from "REPLACE.ME" + When the request is sent + Then the response status is 400 Bad Request + + @generated @skip @team:DataDog/ml-observability + Scenario: List LLM Observability experiment events (v2) returns "Not Found" response + Given operation "ListLLMObsExperimentEventsV2" enabled + And new "ListLLMObsExperimentEventsV2" request + And request contains "experiment_id" parameter from "REPLACE.ME" + When the request is sent + Then the response status is 404 Not Found + + @generated @skip @team:DataDog/ml-observability + Scenario: List LLM Observability experiment events (v2) returns "OK" response + Given operation "ListLLMObsExperimentEventsV2" enabled + And new "ListLLMObsExperimentEventsV2" request + And request contains "experiment_id" parameter from "REPLACE.ME" + When the request is sent + Then the response status is 200 OK + + @generated @skip @team:DataDog/ml-observability + Scenario: List LLM Observability experiment spans (v1) returns "Bad Request" response + Given operation "ListLLMObsExperimentEventsV1" enabled + And new "ListLLMObsExperimentEventsV1" request + And request contains "experiment_id" parameter from "REPLACE.ME" + When the request is sent + Then the response status is 400 Bad Request + + @generated @skip @team:DataDog/ml-observability + Scenario: List LLM Observability experiment spans (v1) returns "Not Found" response + Given operation "ListLLMObsExperimentEventsV1" enabled + And new "ListLLMObsExperimentEventsV1" request + And request contains "experiment_id" parameter from "REPLACE.ME" + When the request is sent + Then the response status is 404 Not Found + + @generated @skip @team:DataDog/ml-observability + Scenario: List LLM Observability experiment spans (v1) returns "OK" response + Given operation "ListLLMObsExperimentEventsV1" enabled + And new "ListLLMObsExperimentEventsV1" request + And request contains "experiment_id" parameter from "REPLACE.ME" + When the request is sent + Then the response status is 200 OK + @generated @skip @team:DataDog/ml-observability Scenario: List LLM Observability experiments returns "Bad Request" response Given operation "ListLLMObsExperiments" enabled @@ -1092,7 +1140,7 @@ Feature: LLM Observability Given operation "UpdateLLMObsExperiment" enabled And new "UpdateLLMObsExperiment" request And request contains "experiment_id" parameter from "REPLACE.ME" - And body with value {"data": {"attributes": {}, "type": "experiments"}} + And body with value {"data": {"attributes": {"dataset_id": "9f64e5c7-dc5a-45c8-a17c-1b85f0bec97d", "status": "completed"}, "type": "experiments"}} When the request is sent Then the response status is 400 Bad Request @@ -1101,7 +1149,7 @@ Feature: LLM Observability Given operation "UpdateLLMObsExperiment" enabled And new "UpdateLLMObsExperiment" request And request contains "experiment_id" parameter from "REPLACE.ME" - And body with value {"data": {"attributes": {}, "type": "experiments"}} + And body with value {"data": {"attributes": {"dataset_id": "9f64e5c7-dc5a-45c8-a17c-1b85f0bec97d", "status": "completed"}, "type": "experiments"}} When the request is sent Then the response status is 404 Not Found @@ -1110,7 +1158,7 @@ Feature: LLM Observability Given operation "UpdateLLMObsExperiment" enabled And new "UpdateLLMObsExperiment" request And request contains "experiment_id" parameter from "REPLACE.ME" - And body with value {"data": {"attributes": {}, "type": "experiments"}} + And body with value {"data": {"attributes": {"dataset_id": "9f64e5c7-dc5a-45c8-a17c-1b85f0bec97d", "status": "completed"}, "type": "experiments"}} When the request is sent Then the response status is 200 OK diff --git a/features/v2/undo.json b/features/v2/undo.json index f60cf1a2f731..09cc27161f37 100644 --- a/features/v2/undo.json +++ b/features/v2/undo.json @@ -4090,6 +4090,12 @@ "type": "idempotent" } }, + "ListLLMObsExperimentEventsV1": { + "tag": "LLM Observability", + "undo": { + "type": "safe" + } + }, "CreateLLMObsExperimentEvents": { "tag": "LLM Observability", "undo": { @@ -4270,6 +4276,12 @@ "type": "safe" } }, + "ListLLMObsExperimentEventsV2": { + "tag": "LLM Observability", + "undo": { + "type": "safe" + } + }, "UploadLLMObsDatasetRecordsFile": { "tag": "LLM Observability", "undo": { diff --git a/lib/datadog_api_client/configuration.rb b/lib/datadog_api_client/configuration.rb index 108c82400aa9..21505bdf26aa 100644 --- a/lib/datadog_api_client/configuration.rb +++ b/lib/datadog_api_client/configuration.rb @@ -237,6 +237,8 @@ def initialize "v2.list_llm_obs_datasets": false, "v2.list_llm_obs_dataset_versions": false, "v2.list_llm_obs_experiment_events": false, + "v2.list_llm_obs_experiment_events_v1": false, + "v2.list_llm_obs_experiment_events_v2": false, "v2.list_llm_obs_experiments": false, "v2.list_llm_obs_integration_accounts": false, "v2.list_llm_obs_integration_models": false, diff --git a/lib/datadog_api_client/inflector.rb b/lib/datadog_api_client/inflector.rb index 838c1450b433..c657bb69d575 100644 --- a/lib/datadog_api_client/inflector.rb +++ b/lib/datadog_api_client/inflector.rb @@ -4095,15 +4095,20 @@ def overrides "v2.llm_obs_experiment_response" => "LLMObsExperimentResponse", "v2.llm_obs_experiment_run_data_response" => "LLMObsExperimentRunDataResponse", "v2.llm_obs_experiment_span" => "LLMObsExperimentSpan", + "v2.llm_obs_experiment_span_data_response" => "LLMObsExperimentSpanDataResponse", "v2.llm_obs_experiment_span_error" => "LLMObsExperimentSpanError", "v2.llm_obs_experiment_span_meta" => "LLMObsExperimentSpanMeta", + "v2.llm_obs_experiment_spans_response" => "LLMObsExperimentSpansResponse", "v2.llm_obs_experiment_span_status" => "LLMObsExperimentSpanStatus", + "v2.llm_obs_experiment_span_type" => "LLMObsExperimentSpanType", "v2.llm_obs_experiment_span_with_evals" => "LLMObsExperimentSpanWithEvals", "v2.llm_obs_experiments_response" => "LLMObsExperimentsResponse", + "v2.llm_obs_experiment_status" => "LLMObsExperimentStatus", "v2.llm_obs_experiment_type" => "LLMObsExperimentType", "v2.llm_obs_experiment_update_data_attributes_request" => "LLMObsExperimentUpdateDataAttributesRequest", "v2.llm_obs_experiment_update_data_request" => "LLMObsExperimentUpdateDataRequest", "v2.llm_obs_experiment_update_request" => "LLMObsExperimentUpdateRequest", + "v2.llm_obs_experiment_user" => "LLMObsExperimentUser", "v2.llm_obs_inference_code" => "LLMObsInferenceCode", "v2.llm_obs_inference_content" => "LLMObsInferenceContent", "v2.llm_obs_inference_content_value" => "LLMObsInferenceContentValue", diff --git a/lib/datadog_api_client/v2/api/llm_observability_api.rb b/lib/datadog_api_client/v2/api/llm_observability_api.rb index 4e8a60da520b..daec334b5ba6 100644 --- a/lib/datadog_api_client/v2/api/llm_observability_api.rb +++ b/lib/datadog_api_client/v2/api/llm_observability_api.rb @@ -2350,6 +2350,154 @@ def list_llm_obs_experiment_events_with_http_info(experiment_id, opts = {}) return data, status_code, headers end + # List LLM Observability experiment spans (v1). + # + # @see #list_llm_obs_experiment_events_v1_with_http_info + def list_llm_obs_experiment_events_v1(experiment_id, opts = {}) + data, _status_code, _headers = list_llm_obs_experiment_events_v1_with_http_info(experiment_id, opts) + data + end + + # List LLM Observability experiment spans (v1). + # + # Retrieve spans with their evaluation metrics for a given experiment. Returns spans only, with no summary metrics and no pagination. Deprecated in favor of `ListLLMObsExperimentEventsV3`. + # + # @deprecated This API is deprecated. + # + # @param experiment_id [String] The ID of the LLM Observability experiment. + # @param opts [Hash] the optional parameters + # @return [Array<(LLMObsExperimentSpansResponse, Integer, Hash)>] LLMObsExperimentSpansResponse data, response status code and response headers + def list_llm_obs_experiment_events_v1_with_http_info(experiment_id, opts = {}) + warn "[DEPRECATION] `ListLLMObsExperimentEventsV1` is deprecated." + unstable_enabled = @api_client.config.unstable_operations["v2.list_llm_obs_experiment_events_v1".to_sym] + if unstable_enabled + @api_client.config.logger.warn format("Using unstable operation '%s'", "v2.list_llm_obs_experiment_events_v1") + else + raise DatadogAPIClient::APIError.new(message: format("Unstable operation '%s' is disabled", "v2.list_llm_obs_experiment_events_v1")) + end + + if @api_client.config.debugging + @api_client.config.logger.debug 'Calling API: LLMObservabilityAPI.list_llm_obs_experiment_events_v1 ...' + end + # verify the required parameter 'experiment_id' is set + if @api_client.config.client_side_validation && experiment_id.nil? + fail ArgumentError, "Missing the required parameter 'experiment_id' when calling LLMObservabilityAPI.list_llm_obs_experiment_events_v1" + end + # resource path + local_var_path = '/api/v2/llm-obs/v1/experiments/{experiment_id}/events'.sub('{experiment_id}', CGI.escape(experiment_id.to_s).gsub('%2F', '/')) + + # query parameters + query_params = opts[:query_params] || {} + + # header parameters + header_params = opts[:header_params] || {} + # HTTP header 'Accept' (if needed) + header_params['Accept'] = @api_client.select_header_accept(['application/json']) + + # form parameters + form_params = opts[:form_params] || {} + + # http body (model) + post_body = opts[:debug_body] + + # return_type + return_type = opts[:debug_return_type] || 'LLMObsExperimentSpansResponse' + + # auth_names + auth_names = opts[:debug_auth_names] || [:apiKeyAuth, :appKeyAuth] + + new_options = opts.merge( + :operation => :list_llm_obs_experiment_events_v1, + :header_params => header_params, + :query_params => query_params, + :form_params => form_params, + :body => post_body, + :auth_names => auth_names, + :return_type => return_type, + :api_version => "V2" + ) + + data, status_code, headers = @api_client.call_api(Net::HTTP::Get, local_var_path, new_options) + if @api_client.config.debugging + @api_client.config.logger.debug "API called: LLMObservabilityAPI#list_llm_obs_experiment_events_v1\nData: #{data.inspect}\nStatus code: #{status_code}\nHeaders: #{headers}" + end + return data, status_code, headers + end + + # List LLM Observability experiment events (v2). + # + # @see #list_llm_obs_experiment_events_v2_with_http_info + def list_llm_obs_experiment_events_v2(experiment_id, opts = {}) + data, _status_code, _headers = list_llm_obs_experiment_events_v2_with_http_info(experiment_id, opts) + data + end + + # List LLM Observability experiment events (v2). + # + # Retrieve spans and experiment-level summary metrics for a given experiment. Returns the full events payload without pagination. Deprecated: use `ListLLMObsExperimentEventsV3` instead. + # + # @deprecated This API is deprecated. + # + # @param experiment_id [String] The ID of the LLM Observability experiment. + # @param opts [Hash] the optional parameters + # @return [Array<(LLMObsExperimentEventsV2Response, Integer, Hash)>] LLMObsExperimentEventsV2Response data, response status code and response headers + def list_llm_obs_experiment_events_v2_with_http_info(experiment_id, opts = {}) + warn "[DEPRECATION] `ListLLMObsExperimentEventsV2` is deprecated." + unstable_enabled = @api_client.config.unstable_operations["v2.list_llm_obs_experiment_events_v2".to_sym] + if unstable_enabled + @api_client.config.logger.warn format("Using unstable operation '%s'", "v2.list_llm_obs_experiment_events_v2") + else + raise DatadogAPIClient::APIError.new(message: format("Unstable operation '%s' is disabled", "v2.list_llm_obs_experiment_events_v2")) + end + + if @api_client.config.debugging + @api_client.config.logger.debug 'Calling API: LLMObservabilityAPI.list_llm_obs_experiment_events_v2 ...' + end + # verify the required parameter 'experiment_id' is set + if @api_client.config.client_side_validation && experiment_id.nil? + fail ArgumentError, "Missing the required parameter 'experiment_id' when calling LLMObservabilityAPI.list_llm_obs_experiment_events_v2" + end + # resource path + local_var_path = '/api/v2/llm-obs/v2/experiments/{experiment_id}/events'.sub('{experiment_id}', CGI.escape(experiment_id.to_s).gsub('%2F', '/')) + + # query parameters + query_params = opts[:query_params] || {} + + # header parameters + header_params = opts[:header_params] || {} + # HTTP header 'Accept' (if needed) + header_params['Accept'] = @api_client.select_header_accept(['application/json']) + + # form parameters + form_params = opts[:form_params] || {} + + # http body (model) + post_body = opts[:debug_body] + + # return_type + return_type = opts[:debug_return_type] || 'LLMObsExperimentEventsV2Response' + + # auth_names + auth_names = opts[:debug_auth_names] || [:apiKeyAuth, :appKeyAuth] + + new_options = opts.merge( + :operation => :list_llm_obs_experiment_events_v2, + :header_params => header_params, + :query_params => query_params, + :form_params => form_params, + :body => post_body, + :auth_names => auth_names, + :return_type => return_type, + :api_version => "V2" + ) + + data, status_code, headers = @api_client.call_api(Net::HTTP::Get, local_var_path, new_options) + if @api_client.config.debugging + @api_client.config.logger.debug "API called: LLMObservabilityAPI#list_llm_obs_experiment_events_v2\nData: #{data.inspect}\nStatus code: #{status_code}\nHeaders: #{headers}" + end + return data, status_code, headers + end + # List LLM Observability experiments. # # @see #list_llm_obs_experiments_with_http_info @@ -2366,8 +2514,15 @@ def list_llm_obs_experiments(opts = {}) # @option opts [String] :filter_project_id Filter experiments by project ID. Required if `filter[dataset_id]` is not provided. # @option opts [String] :filter_dataset_id Filter experiments by dataset ID. # @option opts [String] :filter_id Filter experiments by experiment ID. Can be specified multiple times. - # @option opts [String] :page_cursor Use the Pagination cursor to retrieve the next page of results. - # @option opts [Integer] :page_limit Maximum number of results to return per page. + # @option opts [String] :filter_name Filter experiments by their exact run name. + # @option opts [String] :filter_experiment Filter by logical experiment name. This is the `name` field set when creating an experiment through `POST /experiments`. Returns all experiment runs that share the same name, enabling cross-commit and cross-branch comparisons. + # @option opts [String] :filter_metadata Filter by JSONB metadata containment. Provide a JSON object string where experiments whose metadata contains all specified key-value pairs are returned. For example: `{"commit":"abc123","branch":"main"}`. + # @option opts [String] :filter_parent_experiment_id Filter experiments by the ID of their parent (baseline) experiment. Returns all experiments that were run against the given baseline. Can be specified multiple times. + # @option opts [Boolean] :filter_is_deleted When `true`, return only soft-deleted experiments. Defaults to `false`. + # @option opts [Boolean] :include_user_data When `true`, enrich each experiment with its author's user data in the `author` field. + # @option opts [Boolean] :include_dataset_names When `true`, enrich each experiment with its dataset name in the `dataset_name` field. + # @option opts [String] :page_cursor Use the pagination cursor returned in `meta.after` to retrieve the next page of results. + # @option opts [Integer] :page_limit Maximum number of results to return per page. Values above 5000 are clamped to 5000. Defaults to 5000. # @return [Array<(LLMObsExperimentsResponse, Integer, Hash)>] LLMObsExperimentsResponse data, response status code and response headers def list_llm_obs_experiments_with_http_info(opts = {}) unstable_enabled = @api_client.config.unstable_operations["v2.list_llm_obs_experiments".to_sym] @@ -2380,6 +2535,9 @@ def list_llm_obs_experiments_with_http_info(opts = {}) if @api_client.config.debugging @api_client.config.logger.debug 'Calling API: LLMObservabilityAPI.list_llm_obs_experiments ...' end + if @api_client.config.client_side_validation && !opts[:'page_limit'].nil? && opts[:'page_limit'] > 5000 + fail ArgumentError, 'invalid value for "opts[:"page_limit"]" when calling LLMObservabilityAPI.list_llm_obs_experiments, must be smaller than or equal to 5000.' + end # resource path local_var_path = '/api/v2/llm-obs/v1/experiments' @@ -2388,6 +2546,13 @@ def list_llm_obs_experiments_with_http_info(opts = {}) query_params[:'filter[project_id]'] = opts[:'filter_project_id'] if !opts[:'filter_project_id'].nil? query_params[:'filter[dataset_id]'] = opts[:'filter_dataset_id'] if !opts[:'filter_dataset_id'].nil? query_params[:'filter[id]'] = opts[:'filter_id'] if !opts[:'filter_id'].nil? + query_params[:'filter[name]'] = opts[:'filter_name'] if !opts[:'filter_name'].nil? + query_params[:'filter[experiment]'] = opts[:'filter_experiment'] if !opts[:'filter_experiment'].nil? + query_params[:'filter[metadata]'] = opts[:'filter_metadata'] if !opts[:'filter_metadata'].nil? + query_params[:'filter[parent_experiment_id]'] = opts[:'filter_parent_experiment_id'] if !opts[:'filter_parent_experiment_id'].nil? + query_params[:'filter[is_deleted]'] = opts[:'filter_is_deleted'] if !opts[:'filter_is_deleted'].nil? + query_params[:'include[user_data]'] = opts[:'include_user_data'] if !opts[:'include_user_data'].nil? + query_params[:'include[dataset_names]'] = opts[:'include_dataset_names'] if !opts[:'include_dataset_names'].nil? query_params[:'page[cursor]'] = opts[:'page_cursor'] if !opts[:'page_cursor'].nil? query_params[:'page[limit]'] = opts[:'page_limit'] if !opts[:'page_limit'].nil? diff --git a/lib/datadog_api_client/v2/models/llm_obs_experiment_data_attributes_request.rb b/lib/datadog_api_client/v2/models/llm_obs_experiment_data_attributes_request.rb index e46de8fd97ce..386751963605 100644 --- a/lib/datadog_api_client/v2/models/llm_obs_experiment_data_attributes_request.rb +++ b/lib/datadog_api_client/v2/models/llm_obs_experiment_data_attributes_request.rb @@ -25,7 +25,7 @@ class LLMObsExperimentDataAttributesRequest attr_accessor :config # Identifier of the dataset used in this experiment. - attr_reader :dataset_id + attr_accessor :dataset_id # Version of the dataset to use. Defaults to the current version if not specified. attr_accessor :dataset_version @@ -42,9 +42,15 @@ class LLMObsExperimentDataAttributesRequest # Name of the experiment. attr_reader :name + # Identifier of the parent (baseline) experiment this experiment is run against. + attr_accessor :parent_experiment_id + # Identifier of the project this experiment belongs to. attr_reader :project_id + # Number of runs configured for this experiment. + attr_reader :run_count + attr_accessor :additional_properties # Attribute mapping from ruby-style variable name to JSON key. @@ -58,7 +64,9 @@ def self.attribute_map :'ensure_unique' => :'ensure_unique', :'metadata' => :'metadata', :'name' => :'name', - :'project_id' => :'project_id' + :'parent_experiment_id' => :'parent_experiment_id', + :'project_id' => :'project_id', + :'run_count' => :'run_count' } end @@ -73,7 +81,9 @@ def self.openapi_types :'ensure_unique' => :'Boolean', :'metadata' => :'Hash', :'name' => :'String', - :'project_id' => :'String' + :'parent_experiment_id' => :'String', + :'project_id' => :'String', + :'run_count' => :'Integer' } end @@ -123,31 +133,29 @@ def initialize(attributes = {}) self.name = attributes[:'name'] end + if attributes.key?(:'parent_experiment_id') + self.parent_experiment_id = attributes[:'parent_experiment_id'] + end + if attributes.key?(:'project_id') self.project_id = attributes[:'project_id'] end + + if attributes.key?(:'run_count') + self.run_count = attributes[:'run_count'] + end end # Check to see if the all the properties in the model are valid # @return true if the model is valid # @!visibility private def valid? - return false if @dataset_id.nil? return false if @name.nil? return false if @project_id.nil? + return false if !@run_count.nil? && @run_count > 2147483647 true end - # Custom attribute writer method with validation - # @param dataset_id [Object] Object to be assigned - # @!visibility private - def dataset_id=(dataset_id) - if dataset_id.nil? - fail ArgumentError, 'invalid value for "dataset_id", dataset_id cannot be nil.' - end - @dataset_id = dataset_id - end - # Custom attribute writer method with validation # @param name [Object] Object to be assigned # @!visibility private @@ -168,6 +176,16 @@ def project_id=(project_id) @project_id = project_id end + # Custom attribute writer method with validation + # @param run_count [Object] Object to be assigned + # @!visibility private + def run_count=(run_count) + if !run_count.nil? && run_count > 2147483647 + fail ArgumentError, 'invalid value for "run_count", must be smaller than or equal to 2147483647.' + end + @run_count = run_count + end + # Returns the object in the form of hash, with additionalProperties support. # @return [Hash] Returns the object in the form of hash # @!visibility private @@ -201,7 +219,9 @@ def ==(o) ensure_unique == o.ensure_unique && metadata == o.metadata && name == o.name && + parent_experiment_id == o.parent_experiment_id && project_id == o.project_id && + run_count == o.run_count && additional_properties == o.additional_properties end @@ -209,7 +229,7 @@ def ==(o) # @return [Integer] Hash code # @!visibility private def hash - [config, dataset_id, dataset_version, description, ensure_unique, metadata, name, project_id, additional_properties].hash + [config, dataset_id, dataset_version, description, ensure_unique, metadata, name, parent_experiment_id, project_id, run_count, additional_properties].hash end end end diff --git a/lib/datadog_api_client/v2/models/llm_obs_experiment_data_attributes_response.rb b/lib/datadog_api_client/v2/models/llm_obs_experiment_data_attributes_response.rb index 6a3f256b3ce4..12595c04149c 100644 --- a/lib/datadog_api_client/v2/models/llm_obs_experiment_data_attributes_response.rb +++ b/lib/datadog_api_client/v2/models/llm_obs_experiment_data_attributes_response.rb @@ -21,6 +21,12 @@ module DatadogAPIClient::V2 class LLMObsExperimentDataAttributesResponse include BaseGenericModel + # Pre-computed aggregate metrics for this experiment run, including eval score distributions, token costs, and error rates. + attr_accessor :aggregate_data + + # User data for the author of an experiment. Only present when `include[user_data]` is `true`. + attr_accessor :author + # Configuration parameters for the experiment. attr_accessor :config @@ -30,18 +36,43 @@ class LLMObsExperimentDataAttributesResponse # Identifier of the dataset used in this experiment. attr_reader :dataset_id + # Name of the dataset used in this experiment. + # Only present when `include[dataset_names]` is `true`. + attr_accessor :dataset_name + + # Version of the dataset used in this experiment. + attr_accessor :dataset_version + + # Timestamp when the experiment was soft-deleted, if applicable. + attr_accessor :deleted_at + # Description of the experiment. attr_accessor :description + # Error message describing why the experiment failed, if applicable. + attr_accessor :error + + # Logical name of the experiment, shared across all runs of the same pipeline. + attr_accessor :experiment + # Arbitrary metadata associated with the experiment. attr_accessor :metadata # Name of the experiment. attr_reader :name + # Identifier of the parent (baseline) experiment this experiment was run against, if any. + attr_accessor :parent_experiment_id + # Identifier of the project this experiment belongs to. attr_reader :project_id + # Expected number of runs for this experiment. + attr_reader :run_count + + # Execution status of an LLM Observability experiment. + attr_accessor :status + # Timestamp when the experiment was last updated. attr_reader :updated_at @@ -51,13 +82,23 @@ class LLMObsExperimentDataAttributesResponse # @!visibility private def self.attribute_map { + :'aggregate_data' => :'aggregate_data', + :'author' => :'author', :'config' => :'config', :'created_at' => :'created_at', :'dataset_id' => :'dataset_id', + :'dataset_name' => :'dataset_name', + :'dataset_version' => :'dataset_version', + :'deleted_at' => :'deleted_at', :'description' => :'description', + :'error' => :'error', + :'experiment' => :'experiment', :'metadata' => :'metadata', :'name' => :'name', + :'parent_experiment_id' => :'parent_experiment_id', :'project_id' => :'project_id', + :'run_count' => :'run_count', + :'status' => :'status', :'updated_at' => :'updated_at' } end @@ -66,13 +107,23 @@ def self.attribute_map # @!visibility private def self.openapi_types { + :'aggregate_data' => :'Hash', + :'author' => :'LLMObsExperimentUser', :'config' => :'Hash', :'created_at' => :'Time', :'dataset_id' => :'String', + :'dataset_name' => :'String', + :'dataset_version' => :'Integer', + :'deleted_at' => :'Time', :'description' => :'String', + :'error' => :'String', + :'experiment' => :'String', :'metadata' => :'Hash', :'name' => :'String', + :'parent_experiment_id' => :'String', :'project_id' => :'String', + :'run_count' => :'Integer', + :'status' => :'LLMObsExperimentStatus', :'updated_at' => :'Time' } end @@ -81,9 +132,14 @@ def self.openapi_types # @!visibility private def self.openapi_nullable Set.new([ + :'aggregate_data', :'config', + :'dataset_name', + :'deleted_at', :'description', + :'error', :'metadata', + :'parent_experiment_id', ]) end @@ -105,6 +161,14 @@ def initialize(attributes = {}) end } + if attributes.key?(:'aggregate_data') + self.aggregate_data = attributes[:'aggregate_data'] + end + + if attributes.key?(:'author') + self.author = attributes[:'author'] + end + if attributes.key?(:'config') self.config = attributes[:'config'] end @@ -117,10 +181,30 @@ def initialize(attributes = {}) self.dataset_id = attributes[:'dataset_id'] end + if attributes.key?(:'dataset_name') + self.dataset_name = attributes[:'dataset_name'] + end + + if attributes.key?(:'dataset_version') + self.dataset_version = attributes[:'dataset_version'] + end + + if attributes.key?(:'deleted_at') + self.deleted_at = attributes[:'deleted_at'] + end + if attributes.key?(:'description') self.description = attributes[:'description'] end + if attributes.key?(:'error') + self.error = attributes[:'error'] + end + + if attributes.key?(:'experiment') + self.experiment = attributes[:'experiment'] + end + if attributes.key?(:'metadata') self.metadata = attributes[:'metadata'] end @@ -129,10 +213,22 @@ def initialize(attributes = {}) self.name = attributes[:'name'] end + if attributes.key?(:'parent_experiment_id') + self.parent_experiment_id = attributes[:'parent_experiment_id'] + end + if attributes.key?(:'project_id') self.project_id = attributes[:'project_id'] end + if attributes.key?(:'run_count') + self.run_count = attributes[:'run_count'] + end + + if attributes.key?(:'status') + self.status = attributes[:'status'] + end + if attributes.key?(:'updated_at') self.updated_at = attributes[:'updated_at'] end @@ -146,6 +242,7 @@ def valid? return false if @dataset_id.nil? return false if @name.nil? return false if @project_id.nil? + return false if !@run_count.nil? && @run_count > 2147483647 return false if @updated_at.nil? true end @@ -190,6 +287,16 @@ def project_id=(project_id) @project_id = project_id end + # Custom attribute writer method with validation + # @param run_count [Object] Object to be assigned + # @!visibility private + def run_count=(run_count) + if !run_count.nil? && run_count > 2147483647 + fail ArgumentError, 'invalid value for "run_count", must be smaller than or equal to 2147483647.' + end + @run_count = run_count + end + # Custom attribute writer method with validation # @param updated_at [Object] Object to be assigned # @!visibility private @@ -226,13 +333,23 @@ def to_hash def ==(o) return true if self.equal?(o) self.class == o.class && + aggregate_data == o.aggregate_data && + author == o.author && config == o.config && created_at == o.created_at && dataset_id == o.dataset_id && + dataset_name == o.dataset_name && + dataset_version == o.dataset_version && + deleted_at == o.deleted_at && description == o.description && + error == o.error && + experiment == o.experiment && metadata == o.metadata && name == o.name && + parent_experiment_id == o.parent_experiment_id && project_id == o.project_id && + run_count == o.run_count && + status == o.status && updated_at == o.updated_at && additional_properties == o.additional_properties end @@ -241,7 +358,7 @@ def ==(o) # @return [Integer] Hash code # @!visibility private def hash - [config, created_at, dataset_id, description, metadata, name, project_id, updated_at, additional_properties].hash + [aggregate_data, author, config, created_at, dataset_id, dataset_name, dataset_version, deleted_at, description, error, experiment, metadata, name, parent_experiment_id, project_id, run_count, status, updated_at, additional_properties].hash end end end diff --git a/lib/datadog_api_client/v2/models/llm_obs_experiment_span_data_response.rb b/lib/datadog_api_client/v2/models/llm_obs_experiment_span_data_response.rb new file mode 100644 index 000000000000..bf5f87f6fdd3 --- /dev/null +++ b/lib/datadog_api_client/v2/models/llm_obs_experiment_span_data_response.rb @@ -0,0 +1,165 @@ +=begin +#Datadog API V2 Collection + +#Collection of all Datadog Public endpoints. + +The version of the OpenAPI document: 1.0 +Contact: support@datadoghq.com +Generated by: https://github.com/DataDog/datadog-api-client-ruby/tree/master/.generator + + Unless explicitly stated otherwise all files in this repository are licensed under the Apache-2.0 License. + This product includes software developed at Datadog (https://www.datadoghq.com/). + Copyright 2020-Present Datadog, Inc. + +=end + +require 'date' +require 'time' + +module DatadogAPIClient::V2 + # JSON:API data item wrapping a single experiment span with evaluations. + class LLMObsExperimentSpanDataResponse + include BaseGenericModel + + # An experiment span enriched with its associated evaluation metrics. + attr_reader :attributes + + # Unique identifier of the span. + attr_reader :id + + # Resource type for a span item in an experiment spans response. + attr_reader :type + + attr_accessor :additional_properties + + # Attribute mapping from ruby-style variable name to JSON key. + # @!visibility private + def self.attribute_map + { + :'attributes' => :'attributes', + :'id' => :'id', + :'type' => :'type' + } + end + + # Attribute type mapping. + # @!visibility private + def self.openapi_types + { + :'attributes' => :'LLMObsExperimentSpanWithEvals', + :'id' => :'String', + :'type' => :'LLMObsExperimentSpanType' + } + end + + # Initializes the object + # @param attributes [Hash] Model attributes in the form of hash + # @!visibility private + def initialize(attributes = {}) + if (!attributes.is_a?(Hash)) + fail ArgumentError, "The input argument (attributes) must be a hash in `DatadogAPIClient::V2::LLMObsExperimentSpanDataResponse` initialize method" + end + + self.additional_properties = {} + # check to see if the attribute exists and convert string to symbol for hash key + attributes = attributes.each_with_object({}) { |(k, v), h| + if (!self.class.attribute_map.key?(k.to_sym)) + self.additional_properties[k.to_sym] = v + else + h[k.to_sym] = v + end + } + + if attributes.key?(:'attributes') + self.attributes = attributes[:'attributes'] + end + + if attributes.key?(:'id') + self.id = attributes[:'id'] + end + + if attributes.key?(:'type') + self.type = attributes[:'type'] + end + end + + # Check to see if the all the properties in the model are valid + # @return true if the model is valid + # @!visibility private + def valid? + return false if @attributes.nil? + return false if @id.nil? + return false if @type.nil? + true + end + + # Custom attribute writer method with validation + # @param attributes [Object] Object to be assigned + # @!visibility private + def attributes=(attributes) + if attributes.nil? + fail ArgumentError, 'invalid value for "attributes", attributes cannot be nil.' + end + @attributes = attributes + end + + # Custom attribute writer method with validation + # @param id [Object] Object to be assigned + # @!visibility private + def id=(id) + if id.nil? + fail ArgumentError, 'invalid value for "id", id cannot be nil.' + end + @id = id + end + + # Custom attribute writer method with validation + # @param type [Object] Object to be assigned + # @!visibility private + def type=(type) + if type.nil? + fail ArgumentError, 'invalid value for "type", type cannot be nil.' + end + @type = type + end + + # Returns the object in the form of hash, with additionalProperties support. + # @return [Hash] Returns the object in the form of hash + # @!visibility private + def to_hash + hash = {} + self.class.attribute_map.each_pair do |attr, param| + value = self.send(attr) + if value.nil? + is_nullable = self.class.openapi_nullable.include?(attr) + next if !is_nullable || (is_nullable && !instance_variable_defined?(:"@#{attr}")) + end + + hash[param] = _to_hash(value) + end + self.additional_properties.each_pair do |attr, value| + hash[attr] = value + end + hash + end + + # Checks equality by comparing each attribute. + # @param o [Object] Object to be compared + # @!visibility private + def ==(o) + return true if self.equal?(o) + self.class == o.class && + attributes == o.attributes && + id == o.id && + type == o.type && + additional_properties == o.additional_properties + end + + # Calculates hash code according to all attributes. + # @return [Integer] Hash code + # @!visibility private + def hash + [attributes, id, type, additional_properties].hash + end + end +end diff --git a/lib/datadog_api_client/v2/models/llm_obs_experiment_span_type.rb b/lib/datadog_api_client/v2/models/llm_obs_experiment_span_type.rb new file mode 100644 index 000000000000..cc1a3c7974cf --- /dev/null +++ b/lib/datadog_api_client/v2/models/llm_obs_experiment_span_type.rb @@ -0,0 +1,26 @@ +=begin +#Datadog API V2 Collection + +#Collection of all Datadog Public endpoints. + +The version of the OpenAPI document: 1.0 +Contact: support@datadoghq.com +Generated by: https://github.com/DataDog/datadog-api-client-ruby/tree/master/.generator + + Unless explicitly stated otherwise all files in this repository are licensed under the Apache-2.0 License. + This product includes software developed at Datadog (https://www.datadoghq.com/). + Copyright 2020-Present Datadog, Inc. + +=end + +require 'date' +require 'time' + +module DatadogAPIClient::V2 + # Resource type for a span item in an experiment spans response. + class LLMObsExperimentSpanType + include BaseEnumModel + + EXPERIMENTS_SPAN = "experiments".freeze + end +end diff --git a/lib/datadog_api_client/v2/models/llm_obs_experiment_spans_response.rb b/lib/datadog_api_client/v2/models/llm_obs_experiment_spans_response.rb new file mode 100644 index 000000000000..1dab203af948 --- /dev/null +++ b/lib/datadog_api_client/v2/models/llm_obs_experiment_spans_response.rb @@ -0,0 +1,125 @@ +=begin +#Datadog API V2 Collection + +#Collection of all Datadog Public endpoints. + +The version of the OpenAPI document: 1.0 +Contact: support@datadoghq.com +Generated by: https://github.com/DataDog/datadog-api-client-ruby/tree/master/.generator + + Unless explicitly stated otherwise all files in this repository are licensed under the Apache-2.0 License. + This product includes software developed at Datadog (https://www.datadoghq.com/). + Copyright 2020-Present Datadog, Inc. + +=end + +require 'date' +require 'time' + +module DatadogAPIClient::V2 + # Response for listing experiment spans (v1). Returns only spans with their evaluation metrics. No summary metrics or pagination are included. Deprecated in favor of `ListLLMObsExperimentEventsV3`. + class LLMObsExperimentSpansResponse + include BaseGenericModel + + # List of experiment spans with their evaluation metrics. + attr_reader :data + + attr_accessor :additional_properties + + # Attribute mapping from ruby-style variable name to JSON key. + # @!visibility private + def self.attribute_map + { + :'data' => :'data' + } + end + + # Attribute type mapping. + # @!visibility private + def self.openapi_types + { + :'data' => :'Array' + } + end + + # Initializes the object + # @param attributes [Hash] Model attributes in the form of hash + # @!visibility private + def initialize(attributes = {}) + if (!attributes.is_a?(Hash)) + fail ArgumentError, "The input argument (attributes) must be a hash in `DatadogAPIClient::V2::LLMObsExperimentSpansResponse` initialize method" + end + + self.additional_properties = {} + # check to see if the attribute exists and convert string to symbol for hash key + attributes = attributes.each_with_object({}) { |(k, v), h| + if (!self.class.attribute_map.key?(k.to_sym)) + self.additional_properties[k.to_sym] = v + else + h[k.to_sym] = v + end + } + + if attributes.key?(:'data') + if (value = attributes[:'data']).is_a?(Array) + self.data = value + end + end + end + + # Check to see if the all the properties in the model are valid + # @return true if the model is valid + # @!visibility private + def valid? + return false if @data.nil? + true + end + + # Custom attribute writer method with validation + # @param data [Object] Object to be assigned + # @!visibility private + def data=(data) + if data.nil? + fail ArgumentError, 'invalid value for "data", data cannot be nil.' + end + @data = data + end + + # Returns the object in the form of hash, with additionalProperties support. + # @return [Hash] Returns the object in the form of hash + # @!visibility private + def to_hash + hash = {} + self.class.attribute_map.each_pair do |attr, param| + value = self.send(attr) + if value.nil? + is_nullable = self.class.openapi_nullable.include?(attr) + next if !is_nullable || (is_nullable && !instance_variable_defined?(:"@#{attr}")) + end + + hash[param] = _to_hash(value) + end + self.additional_properties.each_pair do |attr, value| + hash[attr] = value + end + hash + end + + # Checks equality by comparing each attribute. + # @param o [Object] Object to be compared + # @!visibility private + def ==(o) + return true if self.equal?(o) + self.class == o.class && + data == o.data && + additional_properties == o.additional_properties + end + + # Calculates hash code according to all attributes. + # @return [Integer] Hash code + # @!visibility private + def hash + [data, additional_properties].hash + end + end +end diff --git a/lib/datadog_api_client/v2/models/llm_obs_experiment_status.rb b/lib/datadog_api_client/v2/models/llm_obs_experiment_status.rb new file mode 100644 index 000000000000..06d544977b71 --- /dev/null +++ b/lib/datadog_api_client/v2/models/llm_obs_experiment_status.rb @@ -0,0 +1,29 @@ +=begin +#Datadog API V2 Collection + +#Collection of all Datadog Public endpoints. + +The version of the OpenAPI document: 1.0 +Contact: support@datadoghq.com +Generated by: https://github.com/DataDog/datadog-api-client-ruby/tree/master/.generator + + Unless explicitly stated otherwise all files in this repository are licensed under the Apache-2.0 License. + This product includes software developed at Datadog (https://www.datadoghq.com/). + Copyright 2020-Present Datadog, Inc. + +=end + +require 'date' +require 'time' + +module DatadogAPIClient::V2 + # Execution status of an LLM Observability experiment. + class LLMObsExperimentStatus + include BaseEnumModel + + RUNNING = "running".freeze + COMPLETED = "completed".freeze + FAILED = "failed".freeze + INTERRUPTED = "interrupted".freeze + end +end diff --git a/lib/datadog_api_client/v2/models/llm_obs_experiment_update_data_attributes_request.rb b/lib/datadog_api_client/v2/models/llm_obs_experiment_update_data_attributes_request.rb index 1b3f52147c75..3673d4f596ad 100644 --- a/lib/datadog_api_client/v2/models/llm_obs_experiment_update_data_attributes_request.rb +++ b/lib/datadog_api_client/v2/models/llm_obs_experiment_update_data_attributes_request.rb @@ -21,20 +21,36 @@ module DatadogAPIClient::V2 class LLMObsExperimentUpdateDataAttributesRequest include BaseGenericModel + # Updated identifier of the dataset used in this experiment. + attr_accessor :dataset_id + # Updated description of the experiment. attr_accessor :description + # Error message describing why the experiment failed, if applicable. + attr_accessor :error + + # Updated arbitrary metadata associated with the experiment. + attr_accessor :metadata + # Updated name of the experiment. attr_accessor :name + # Execution status of an LLM Observability experiment. + attr_accessor :status + attr_accessor :additional_properties # Attribute mapping from ruby-style variable name to JSON key. # @!visibility private def self.attribute_map { + :'dataset_id' => :'dataset_id', :'description' => :'description', - :'name' => :'name' + :'error' => :'error', + :'metadata' => :'metadata', + :'name' => :'name', + :'status' => :'status' } end @@ -42,8 +58,12 @@ def self.attribute_map # @!visibility private def self.openapi_types { + :'dataset_id' => :'String', :'description' => :'String', - :'name' => :'String' + :'error' => :'String', + :'metadata' => :'Hash', + :'name' => :'String', + :'status' => :'LLMObsExperimentStatus' } end @@ -65,13 +85,29 @@ def initialize(attributes = {}) end } + if attributes.key?(:'dataset_id') + self.dataset_id = attributes[:'dataset_id'] + end + if attributes.key?(:'description') self.description = attributes[:'description'] end + if attributes.key?(:'error') + self.error = attributes[:'error'] + end + + if attributes.key?(:'metadata') + self.metadata = attributes[:'metadata'] + end + if attributes.key?(:'name') self.name = attributes[:'name'] end + + if attributes.key?(:'status') + self.status = attributes[:'status'] + end end # Returns the object in the form of hash, with additionalProperties support. @@ -100,8 +136,12 @@ def to_hash def ==(o) return true if self.equal?(o) self.class == o.class && + dataset_id == o.dataset_id && description == o.description && + error == o.error && + metadata == o.metadata && name == o.name && + status == o.status && additional_properties == o.additional_properties end @@ -109,7 +149,7 @@ def ==(o) # @return [Integer] Hash code # @!visibility private def hash - [description, name, additional_properties].hash + [dataset_id, description, error, metadata, name, status, additional_properties].hash end end end diff --git a/lib/datadog_api_client/v2/models/llm_obs_experiment_user.rb b/lib/datadog_api_client/v2/models/llm_obs_experiment_user.rb new file mode 100644 index 000000000000..86aa01bca344 --- /dev/null +++ b/lib/datadog_api_client/v2/models/llm_obs_experiment_user.rb @@ -0,0 +1,145 @@ +=begin +#Datadog API V2 Collection + +#Collection of all Datadog Public endpoints. + +The version of the OpenAPI document: 1.0 +Contact: support@datadoghq.com +Generated by: https://github.com/DataDog/datadog-api-client-ruby/tree/master/.generator + + Unless explicitly stated otherwise all files in this repository are licensed under the Apache-2.0 License. + This product includes software developed at Datadog (https://www.datadoghq.com/). + Copyright 2020-Present Datadog, Inc. + +=end + +require 'date' +require 'time' + +module DatadogAPIClient::V2 + # User data for the author of an experiment. Only present when `include[user_data]` is `true`. + class LLMObsExperimentUser + include BaseGenericModel + + # Email address of the user. + attr_accessor :email + + # Username or handle associated with the user's Datadog account. + attr_accessor :handle + + # URL of the user's icon. + attr_accessor :icon + + # Unique identifier of the user. + attr_accessor :id + + # Display name of the user. + attr_accessor :name + + attr_accessor :additional_properties + + # Attribute mapping from ruby-style variable name to JSON key. + # @!visibility private + def self.attribute_map + { + :'email' => :'email', + :'handle' => :'handle', + :'icon' => :'icon', + :'id' => :'id', + :'name' => :'name' + } + end + + # Attribute type mapping. + # @!visibility private + def self.openapi_types + { + :'email' => :'String', + :'handle' => :'String', + :'icon' => :'String', + :'id' => :'String', + :'name' => :'String' + } + end + + # Initializes the object + # @param attributes [Hash] Model attributes in the form of hash + # @!visibility private + def initialize(attributes = {}) + if (!attributes.is_a?(Hash)) + fail ArgumentError, "The input argument (attributes) must be a hash in `DatadogAPIClient::V2::LLMObsExperimentUser` initialize method" + end + + self.additional_properties = {} + # check to see if the attribute exists and convert string to symbol for hash key + attributes = attributes.each_with_object({}) { |(k, v), h| + if (!self.class.attribute_map.key?(k.to_sym)) + self.additional_properties[k.to_sym] = v + else + h[k.to_sym] = v + end + } + + if attributes.key?(:'email') + self.email = attributes[:'email'] + end + + if attributes.key?(:'handle') + self.handle = attributes[:'handle'] + end + + if attributes.key?(:'icon') + self.icon = attributes[:'icon'] + end + + if attributes.key?(:'id') + self.id = attributes[:'id'] + end + + if attributes.key?(:'name') + self.name = attributes[:'name'] + end + end + + # Returns the object in the form of hash, with additionalProperties support. + # @return [Hash] Returns the object in the form of hash + # @!visibility private + def to_hash + hash = {} + self.class.attribute_map.each_pair do |attr, param| + value = self.send(attr) + if value.nil? + is_nullable = self.class.openapi_nullable.include?(attr) + next if !is_nullable || (is_nullable && !instance_variable_defined?(:"@#{attr}")) + end + + hash[param] = _to_hash(value) + end + self.additional_properties.each_pair do |attr, value| + hash[attr] = value + end + hash + end + + # Checks equality by comparing each attribute. + # @param o [Object] Object to be compared + # @!visibility private + def ==(o) + return true if self.equal?(o) + self.class == o.class && + email == o.email && + handle == o.handle && + icon == o.icon && + id == o.id && + name == o.name && + additional_properties == o.additional_properties + end + + # Calculates hash code according to all attributes. + # @return [Integer] Hash code + # @!visibility private + def hash + [email, handle, icon, id, name, additional_properties].hash + end + end +end