diff --git a/.codegen/_openapi_sha b/.codegen/_openapi_sha index 6e816b0ebb8..0373270428a 100644 --- a/.codegen/_openapi_sha +++ b/.codegen/_openapi_sha @@ -1 +1 @@ -a499dda0f7802e37868d3f3076f62639165475d8 \ No newline at end of file +0555d6a59265799ed8ea12f355eee662e739430d \ No newline at end of file diff --git a/.gitattributes b/.gitattributes index 44638c51278..595b0dba8c2 100644 --- a/.gitattributes +++ b/.gitattributes @@ -54,6 +54,7 @@ cmd/workspace/apps-settings/apps-settings.go linguist-generated=true cmd/workspace/apps/apps.go linguist-generated=true cmd/workspace/artifact-allowlists/artifact-allowlists.go linguist-generated=true cmd/workspace/automatic-cluster-update/automatic-cluster-update.go linguist-generated=true +cmd/workspace/bundle/bundle.go linguist-generated=true cmd/workspace/catalogs/catalogs.go linguist-generated=true cmd/workspace/clean-room-asset-revisions/clean-room-asset-revisions.go linguist-generated=true cmd/workspace/clean-room-assets/clean-room-assets.go linguist-generated=true diff --git a/NEXT_CHANGELOG.md b/NEXT_CHANGELOG.md index b7372336591..004cd4da4c0 100644 --- a/NEXT_CHANGELOG.md +++ b/NEXT_CHANGELOG.md @@ -10,4 +10,6 @@ * The error reported when a direct-only resource (catalogs, external locations, vector search endpoints) is used with the terraform engine now also suggests setting `bundle.engine: direct` in `databricks.yml`, in addition to the `DATABRICKS_BUNDLE_ENGINE` environment variable ([#5295](https://github.com/databricks/cli/pull/5295)). ### Dependency updates + * Bump Go toolchain to 1.26.3 ([#5302](https://github.com/databricks/cli/pull/5302)). +* Bump `github.com/databricks/databricks-sdk-go` from v0.132.0 to v0.136.0. diff --git a/acceptance/bundle/refschema/out.fields.txt b/acceptance/bundle/refschema/out.fields.txt index 42a2026c5eb..f433389916f 100644 --- a/acceptance/bundle/refschema/out.fields.txt +++ b/acceptance/bundle/refschema/out.fields.txt @@ -1232,7 +1232,15 @@ resources.jobs.*.tasks[*].for_each_task.task.notification_settings.no_alert_for_ resources.jobs.*.tasks[*].for_each_task.task.notification_settings.no_alert_for_skipped_runs bool ALL resources.jobs.*.tasks[*].for_each_task.task.pipeline_task *jobs.PipelineTask ALL resources.jobs.*.tasks[*].for_each_task.task.pipeline_task.full_refresh bool ALL +resources.jobs.*.tasks[*].for_each_task.task.pipeline_task.full_refresh_selection []string ALL +resources.jobs.*.tasks[*].for_each_task.task.pipeline_task.full_refresh_selection[*] string ALL resources.jobs.*.tasks[*].for_each_task.task.pipeline_task.pipeline_id string ALL +resources.jobs.*.tasks[*].for_each_task.task.pipeline_task.refresh_flow_selection []string ALL +resources.jobs.*.tasks[*].for_each_task.task.pipeline_task.refresh_flow_selection[*] string ALL +resources.jobs.*.tasks[*].for_each_task.task.pipeline_task.refresh_selection []string ALL +resources.jobs.*.tasks[*].for_each_task.task.pipeline_task.refresh_selection[*] string ALL +resources.jobs.*.tasks[*].for_each_task.task.pipeline_task.reset_checkpoint_selection []string ALL +resources.jobs.*.tasks[*].for_each_task.task.pipeline_task.reset_checkpoint_selection[*] string ALL resources.jobs.*.tasks[*].for_each_task.task.power_bi_task *jobs.PowerBiTask ALL resources.jobs.*.tasks[*].for_each_task.task.power_bi_task.connection_resource_name string ALL resources.jobs.*.tasks[*].for_each_task.task.power_bi_task.power_bi_model *jobs.PowerBiModel ALL @@ -1249,6 +1257,12 @@ resources.jobs.*.tasks[*].for_each_task.task.power_bi_task.tables[*].name string resources.jobs.*.tasks[*].for_each_task.task.power_bi_task.tables[*].schema string ALL resources.jobs.*.tasks[*].for_each_task.task.power_bi_task.tables[*].storage_mode jobs.StorageMode ALL resources.jobs.*.tasks[*].for_each_task.task.power_bi_task.warehouse_id string ALL +resources.jobs.*.tasks[*].for_each_task.task.python_operator_task *jobs.PythonOperatorTask ALL +resources.jobs.*.tasks[*].for_each_task.task.python_operator_task.main string ALL +resources.jobs.*.tasks[*].for_each_task.task.python_operator_task.parameters []jobs.PythonOperatorTaskParameter ALL +resources.jobs.*.tasks[*].for_each_task.task.python_operator_task.parameters[*] jobs.PythonOperatorTaskParameter ALL +resources.jobs.*.tasks[*].for_each_task.task.python_operator_task.parameters[*].name string ALL +resources.jobs.*.tasks[*].for_each_task.task.python_operator_task.parameters[*].value string ALL resources.jobs.*.tasks[*].for_each_task.task.python_wheel_task *jobs.PythonWheelTask ALL resources.jobs.*.tasks[*].for_each_task.task.python_wheel_task.entry_point string ALL resources.jobs.*.tasks[*].for_each_task.task.python_wheel_task.named_parameters map[string]string ALL @@ -1270,6 +1284,14 @@ resources.jobs.*.tasks[*].for_each_task.task.run_job_task.notebook_params map[st resources.jobs.*.tasks[*].for_each_task.task.run_job_task.notebook_params.* string ALL resources.jobs.*.tasks[*].for_each_task.task.run_job_task.pipeline_params *jobs.PipelineParams ALL resources.jobs.*.tasks[*].for_each_task.task.run_job_task.pipeline_params.full_refresh bool ALL +resources.jobs.*.tasks[*].for_each_task.task.run_job_task.pipeline_params.full_refresh_selection []string ALL +resources.jobs.*.tasks[*].for_each_task.task.run_job_task.pipeline_params.full_refresh_selection[*] string ALL +resources.jobs.*.tasks[*].for_each_task.task.run_job_task.pipeline_params.refresh_flow_selection []string ALL +resources.jobs.*.tasks[*].for_each_task.task.run_job_task.pipeline_params.refresh_flow_selection[*] string ALL +resources.jobs.*.tasks[*].for_each_task.task.run_job_task.pipeline_params.refresh_selection []string ALL +resources.jobs.*.tasks[*].for_each_task.task.run_job_task.pipeline_params.refresh_selection[*] string ALL +resources.jobs.*.tasks[*].for_each_task.task.run_job_task.pipeline_params.reset_checkpoint_selection []string ALL +resources.jobs.*.tasks[*].for_each_task.task.run_job_task.pipeline_params.reset_checkpoint_selection[*] string ALL resources.jobs.*.tasks[*].for_each_task.task.run_job_task.python_named_params map[string]string ALL resources.jobs.*.tasks[*].for_each_task.task.run_job_task.python_named_params.* string ALL resources.jobs.*.tasks[*].for_each_task.task.run_job_task.python_params []string ALL @@ -1493,7 +1515,15 @@ resources.jobs.*.tasks[*].notification_settings.no_alert_for_canceled_runs bool resources.jobs.*.tasks[*].notification_settings.no_alert_for_skipped_runs bool ALL resources.jobs.*.tasks[*].pipeline_task *jobs.PipelineTask ALL resources.jobs.*.tasks[*].pipeline_task.full_refresh bool ALL +resources.jobs.*.tasks[*].pipeline_task.full_refresh_selection []string ALL +resources.jobs.*.tasks[*].pipeline_task.full_refresh_selection[*] string ALL resources.jobs.*.tasks[*].pipeline_task.pipeline_id string ALL +resources.jobs.*.tasks[*].pipeline_task.refresh_flow_selection []string ALL +resources.jobs.*.tasks[*].pipeline_task.refresh_flow_selection[*] string ALL +resources.jobs.*.tasks[*].pipeline_task.refresh_selection []string ALL +resources.jobs.*.tasks[*].pipeline_task.refresh_selection[*] string ALL +resources.jobs.*.tasks[*].pipeline_task.reset_checkpoint_selection []string ALL +resources.jobs.*.tasks[*].pipeline_task.reset_checkpoint_selection[*] string ALL resources.jobs.*.tasks[*].power_bi_task *jobs.PowerBiTask ALL resources.jobs.*.tasks[*].power_bi_task.connection_resource_name string ALL resources.jobs.*.tasks[*].power_bi_task.power_bi_model *jobs.PowerBiModel ALL @@ -1510,6 +1540,12 @@ resources.jobs.*.tasks[*].power_bi_task.tables[*].name string ALL resources.jobs.*.tasks[*].power_bi_task.tables[*].schema string ALL resources.jobs.*.tasks[*].power_bi_task.tables[*].storage_mode jobs.StorageMode ALL resources.jobs.*.tasks[*].power_bi_task.warehouse_id string ALL +resources.jobs.*.tasks[*].python_operator_task *jobs.PythonOperatorTask ALL +resources.jobs.*.tasks[*].python_operator_task.main string ALL +resources.jobs.*.tasks[*].python_operator_task.parameters []jobs.PythonOperatorTaskParameter ALL +resources.jobs.*.tasks[*].python_operator_task.parameters[*] jobs.PythonOperatorTaskParameter ALL +resources.jobs.*.tasks[*].python_operator_task.parameters[*].name string ALL +resources.jobs.*.tasks[*].python_operator_task.parameters[*].value string ALL resources.jobs.*.tasks[*].python_wheel_task *jobs.PythonWheelTask ALL resources.jobs.*.tasks[*].python_wheel_task.entry_point string ALL resources.jobs.*.tasks[*].python_wheel_task.named_parameters map[string]string ALL @@ -1531,6 +1567,14 @@ resources.jobs.*.tasks[*].run_job_task.notebook_params map[string]string ALL resources.jobs.*.tasks[*].run_job_task.notebook_params.* string ALL resources.jobs.*.tasks[*].run_job_task.pipeline_params *jobs.PipelineParams ALL resources.jobs.*.tasks[*].run_job_task.pipeline_params.full_refresh bool ALL +resources.jobs.*.tasks[*].run_job_task.pipeline_params.full_refresh_selection []string ALL +resources.jobs.*.tasks[*].run_job_task.pipeline_params.full_refresh_selection[*] string ALL +resources.jobs.*.tasks[*].run_job_task.pipeline_params.refresh_flow_selection []string ALL +resources.jobs.*.tasks[*].run_job_task.pipeline_params.refresh_flow_selection[*] string ALL +resources.jobs.*.tasks[*].run_job_task.pipeline_params.refresh_selection []string ALL +resources.jobs.*.tasks[*].run_job_task.pipeline_params.refresh_selection[*] string ALL +resources.jobs.*.tasks[*].run_job_task.pipeline_params.reset_checkpoint_selection []string ALL +resources.jobs.*.tasks[*].run_job_task.pipeline_params.reset_checkpoint_selection[*] string ALL resources.jobs.*.tasks[*].run_job_task.python_named_params map[string]string ALL resources.jobs.*.tasks[*].run_job_task.python_named_params.* string ALL resources.jobs.*.tasks[*].run_job_task.python_params []string ALL @@ -2367,6 +2411,30 @@ resources.pipelines.*.ingestion_definition.objects[*].schema.connector_options.g resources.pipelines.*.ingestion_definition.objects[*].schema.connector_options.jira_options *pipelines.JiraConnectorOptions ALL resources.pipelines.*.ingestion_definition.objects[*].schema.connector_options.jira_options.include_jira_spaces []string ALL resources.pipelines.*.ingestion_definition.objects[*].schema.connector_options.jira_options.include_jira_spaces[*] string ALL +resources.pipelines.*.ingestion_definition.objects[*].schema.connector_options.kafka_options *pipelines.KafkaOptions ALL +resources.pipelines.*.ingestion_definition.objects[*].schema.connector_options.kafka_options.client_config map[string]string ALL +resources.pipelines.*.ingestion_definition.objects[*].schema.connector_options.kafka_options.client_config.* string ALL +resources.pipelines.*.ingestion_definition.objects[*].schema.connector_options.kafka_options.key_transformer *pipelines.Transformer ALL +resources.pipelines.*.ingestion_definition.objects[*].schema.connector_options.kafka_options.key_transformer.format pipelines.TransformerFormat ALL +resources.pipelines.*.ingestion_definition.objects[*].schema.connector_options.kafka_options.key_transformer.json_options *pipelines.JsonTransformerOptions ALL +resources.pipelines.*.ingestion_definition.objects[*].schema.connector_options.kafka_options.key_transformer.json_options.as_variant bool ALL +resources.pipelines.*.ingestion_definition.objects[*].schema.connector_options.kafka_options.key_transformer.json_options.schema string ALL +resources.pipelines.*.ingestion_definition.objects[*].schema.connector_options.kafka_options.key_transformer.json_options.schema_evolution_mode pipelines.FileIngestionOptionsSchemaEvolutionMode ALL +resources.pipelines.*.ingestion_definition.objects[*].schema.connector_options.kafka_options.key_transformer.json_options.schema_file_path string ALL +resources.pipelines.*.ingestion_definition.objects[*].schema.connector_options.kafka_options.key_transformer.json_options.schema_hints string ALL +resources.pipelines.*.ingestion_definition.objects[*].schema.connector_options.kafka_options.max_offsets_per_trigger int64 ALL +resources.pipelines.*.ingestion_definition.objects[*].schema.connector_options.kafka_options.starting_offset string ALL +resources.pipelines.*.ingestion_definition.objects[*].schema.connector_options.kafka_options.topic_pattern string ALL +resources.pipelines.*.ingestion_definition.objects[*].schema.connector_options.kafka_options.topics []string ALL +resources.pipelines.*.ingestion_definition.objects[*].schema.connector_options.kafka_options.topics[*] string ALL +resources.pipelines.*.ingestion_definition.objects[*].schema.connector_options.kafka_options.value_transformer *pipelines.Transformer ALL +resources.pipelines.*.ingestion_definition.objects[*].schema.connector_options.kafka_options.value_transformer.format pipelines.TransformerFormat ALL +resources.pipelines.*.ingestion_definition.objects[*].schema.connector_options.kafka_options.value_transformer.json_options *pipelines.JsonTransformerOptions ALL +resources.pipelines.*.ingestion_definition.objects[*].schema.connector_options.kafka_options.value_transformer.json_options.as_variant bool ALL +resources.pipelines.*.ingestion_definition.objects[*].schema.connector_options.kafka_options.value_transformer.json_options.schema string ALL +resources.pipelines.*.ingestion_definition.objects[*].schema.connector_options.kafka_options.value_transformer.json_options.schema_evolution_mode pipelines.FileIngestionOptionsSchemaEvolutionMode ALL +resources.pipelines.*.ingestion_definition.objects[*].schema.connector_options.kafka_options.value_transformer.json_options.schema_file_path string ALL +resources.pipelines.*.ingestion_definition.objects[*].schema.connector_options.kafka_options.value_transformer.json_options.schema_hints string ALL resources.pipelines.*.ingestion_definition.objects[*].schema.connector_options.meta_ads_options *pipelines.MetaMarketingOptions ALL resources.pipelines.*.ingestion_definition.objects[*].schema.connector_options.meta_ads_options.action_attribution_windows []string ALL resources.pipelines.*.ingestion_definition.objects[*].schema.connector_options.meta_ads_options.action_attribution_windows[*] string ALL @@ -2495,6 +2563,30 @@ resources.pipelines.*.ingestion_definition.objects[*].table.connector_options.go resources.pipelines.*.ingestion_definition.objects[*].table.connector_options.jira_options *pipelines.JiraConnectorOptions ALL resources.pipelines.*.ingestion_definition.objects[*].table.connector_options.jira_options.include_jira_spaces []string ALL resources.pipelines.*.ingestion_definition.objects[*].table.connector_options.jira_options.include_jira_spaces[*] string ALL +resources.pipelines.*.ingestion_definition.objects[*].table.connector_options.kafka_options *pipelines.KafkaOptions ALL +resources.pipelines.*.ingestion_definition.objects[*].table.connector_options.kafka_options.client_config map[string]string ALL +resources.pipelines.*.ingestion_definition.objects[*].table.connector_options.kafka_options.client_config.* string ALL +resources.pipelines.*.ingestion_definition.objects[*].table.connector_options.kafka_options.key_transformer *pipelines.Transformer ALL +resources.pipelines.*.ingestion_definition.objects[*].table.connector_options.kafka_options.key_transformer.format pipelines.TransformerFormat ALL +resources.pipelines.*.ingestion_definition.objects[*].table.connector_options.kafka_options.key_transformer.json_options *pipelines.JsonTransformerOptions ALL +resources.pipelines.*.ingestion_definition.objects[*].table.connector_options.kafka_options.key_transformer.json_options.as_variant bool ALL +resources.pipelines.*.ingestion_definition.objects[*].table.connector_options.kafka_options.key_transformer.json_options.schema string ALL +resources.pipelines.*.ingestion_definition.objects[*].table.connector_options.kafka_options.key_transformer.json_options.schema_evolution_mode pipelines.FileIngestionOptionsSchemaEvolutionMode ALL +resources.pipelines.*.ingestion_definition.objects[*].table.connector_options.kafka_options.key_transformer.json_options.schema_file_path string ALL +resources.pipelines.*.ingestion_definition.objects[*].table.connector_options.kafka_options.key_transformer.json_options.schema_hints string ALL +resources.pipelines.*.ingestion_definition.objects[*].table.connector_options.kafka_options.max_offsets_per_trigger int64 ALL +resources.pipelines.*.ingestion_definition.objects[*].table.connector_options.kafka_options.starting_offset string ALL +resources.pipelines.*.ingestion_definition.objects[*].table.connector_options.kafka_options.topic_pattern string ALL +resources.pipelines.*.ingestion_definition.objects[*].table.connector_options.kafka_options.topics []string ALL +resources.pipelines.*.ingestion_definition.objects[*].table.connector_options.kafka_options.topics[*] string ALL +resources.pipelines.*.ingestion_definition.objects[*].table.connector_options.kafka_options.value_transformer *pipelines.Transformer ALL +resources.pipelines.*.ingestion_definition.objects[*].table.connector_options.kafka_options.value_transformer.format pipelines.TransformerFormat ALL +resources.pipelines.*.ingestion_definition.objects[*].table.connector_options.kafka_options.value_transformer.json_options *pipelines.JsonTransformerOptions ALL +resources.pipelines.*.ingestion_definition.objects[*].table.connector_options.kafka_options.value_transformer.json_options.as_variant bool ALL +resources.pipelines.*.ingestion_definition.objects[*].table.connector_options.kafka_options.value_transformer.json_options.schema string ALL +resources.pipelines.*.ingestion_definition.objects[*].table.connector_options.kafka_options.value_transformer.json_options.schema_evolution_mode pipelines.FileIngestionOptionsSchemaEvolutionMode ALL +resources.pipelines.*.ingestion_definition.objects[*].table.connector_options.kafka_options.value_transformer.json_options.schema_file_path string ALL +resources.pipelines.*.ingestion_definition.objects[*].table.connector_options.kafka_options.value_transformer.json_options.schema_hints string ALL resources.pipelines.*.ingestion_definition.objects[*].table.connector_options.meta_ads_options *pipelines.MetaMarketingOptions ALL resources.pipelines.*.ingestion_definition.objects[*].table.connector_options.meta_ads_options.action_attribution_windows []string ALL resources.pipelines.*.ingestion_definition.objects[*].table.connector_options.meta_ads_options.action_attribution_windows[*] string ALL @@ -2714,10 +2806,12 @@ resources.postgres_branches.*.status *postgres.BranchStatus REMOTE resources.postgres_branches.*.status.branch_id string REMOTE resources.postgres_branches.*.status.current_state postgres.BranchStatusState REMOTE resources.postgres_branches.*.status.default bool REMOTE +resources.postgres_branches.*.status.delete_time *time.Time REMOTE resources.postgres_branches.*.status.expire_time *time.Time REMOTE resources.postgres_branches.*.status.is_protected bool REMOTE resources.postgres_branches.*.status.logical_size_bytes int64 REMOTE resources.postgres_branches.*.status.pending_state postgres.BranchStatusState REMOTE +resources.postgres_branches.*.status.purge_time *time.Time REMOTE resources.postgres_branches.*.status.source_branch string REMOTE resources.postgres_branches.*.status.source_branch_lsn string REMOTE resources.postgres_branches.*.status.source_branch_time *time.Time REMOTE diff --git a/acceptance/cmd/account/account-help/output.txt b/acceptance/cmd/account/account-help/output.txt index f3e883750f6..1687f96d466 100644 --- a/acceptance/cmd/account/account-help/output.txt +++ b/acceptance/cmd/account/account-help/output.txt @@ -32,7 +32,7 @@ Settings Provisioning credentials These APIs manage credential configurations for this workspace. encryption-keys These APIs manage encryption key configurations for this workspace (optional). - endpoints *Beta* These APIs manage endpoint configurations for this account. + endpoints *Public Preview* These APIs manage endpoint configurations for this account. networks These APIs manage network configurations for customer-managed VPCs (optional). private-access These APIs manage private access settings for this account. storage These APIs manage storage configurations for this workspace. diff --git a/acceptance/dbr_test.go b/acceptance/dbr_test.go index 2fe498698a1..4a264eecd53 100644 --- a/acceptance/dbr_test.go +++ b/acceptance/dbr_test.go @@ -14,6 +14,7 @@ import ( "github.com/databricks/cli/libs/filer" "github.com/databricks/databricks-sdk-go" "github.com/databricks/databricks-sdk-go/service/compute" + "github.com/databricks/databricks-sdk-go/service/iam" "github.com/databricks/databricks-sdk-go/service/jobs" "github.com/databricks/databricks-sdk-go/service/workspace" "github.com/google/uuid" @@ -27,7 +28,7 @@ func workspaceTmpDir(ctx context.Context, t *testing.T) string { w, err := databricks.NewWorkspaceClient() require.NoError(t, err) - currentUser, err := w.CurrentUser.Me(ctx) + currentUser, err := w.CurrentUser.Me(ctx, iam.MeRequest{}) require.NoError(t, err) timestamp := time.Now().Format("2006-01-02T15:04:05Z") @@ -79,7 +80,7 @@ func setupDbrTestDir(ctx context.Context, t *testing.T, uniqueID string) (*datab w, err := databricks.NewWorkspaceClient() require.NoError(t, err) - currentUser, err := w.CurrentUser.Me(ctx) + currentUser, err := w.CurrentUser.Me(ctx, iam.MeRequest{}) require.NoError(t, err) // API path (without /Workspace prefix) for workspace API calls. @@ -180,7 +181,7 @@ func runDbrTests(ctx context.Context, t *testing.T, w *databricks.WorkspaceClien t.Fatal("CLOUD_ENV is not set. Please run DBR tests from a CI environment with deco env run.") } - currentUser, err := w.CurrentUser.Me(ctx) + currentUser, err := w.CurrentUser.Me(ctx, iam.MeRequest{}) require.NoError(t, err) // Create debug logs directory diff --git a/acceptance/help/output.txt b/acceptance/help/output.txt index af62067b5c6..453ec28b84b 100644 --- a/acceptance/help/output.txt +++ b/acceptance/help/output.txt @@ -36,7 +36,7 @@ Apps apps Apps run directly on a customer's Databricks instance, integrate with their data, use and extend Databricks services, and enable users to interact through single sign-on. Vector Search - vector-search-endpoints **Endpoint**: Represents the compute resources to host vector search indexes. + vector-search-endpoints **Endpoint**: Represents the compute resources to host AI Search indexes. vector-search-indexes **Index**: An efficient representation of your embedding vectors that supports real-time and efficient approximate nearest neighbor (ANN) search queries. Identity and Access Management @@ -92,6 +92,7 @@ Unity Catalog tables A table resides in the third layer of Unity Catalog’s three-level namespace. temporary-path-credentials *Public Preview* Temporary Path Credentials refer to short-lived, downscoped credentials used to access external cloud storage locations registered in Databricks. temporary-table-credentials *Public Preview* Temporary Table Credentials refer to short-lived, downscoped credentials used to access cloud storage locations where table data is stored in Databricks. + temporary-volume-credentials *Public Preview* Temporary Volume Credentials refer to short-lived, downscoped credentials used to access cloud storage locations where volume data is stored in Databricks. volumes Volumes are a Unity Catalog (UC) capability for accessing, storing, governing, organizing and processing files. workspace-bindings A securable in Databricks can be configured as __OPEN__ or __ISOLATED__. diff --git a/acceptance/internal/prepare_server.go b/acceptance/internal/prepare_server.go index 299d48f03ee..4ed3715d77f 100644 --- a/acceptance/internal/prepare_server.go +++ b/acceptance/internal/prepare_server.go @@ -85,7 +85,7 @@ func PrepareServerAndClient(t *testing.T, config TestConfig, logRequests bool, o w, err := databricks.NewWorkspaceClient() require.NoError(t, err) - user, err := w.CurrentUser.Me(t.Context()) + user, err := w.CurrentUser.Me(t.Context(), iam.MeRequest{}) require.NoError(t, err, "Failed to get current user") cfg := w.Config diff --git a/bundle/config/mutator/populate_current_user.go b/bundle/config/mutator/populate_current_user.go index 2a3cc492829..b1ec81b33e8 100644 --- a/bundle/config/mutator/populate_current_user.go +++ b/bundle/config/mutator/populate_current_user.go @@ -31,7 +31,7 @@ func (m *populateCurrentUser) Apply(ctx context.Context, b *bundle.Bundle) diag. fingerprint := b.GetUserFingerprint(ctx) me, err := cache.GetOrCompute(ctx, b.Cache, fingerprint, func(ctx context.Context) (*iam.User, error) { - return w.CurrentUser.Me(ctx) + return w.CurrentUser.Me(ctx, iam.MeRequest{}) }) if err != nil { return diag.FromErr(err) diff --git a/bundle/direct/dresources/job.go b/bundle/direct/dresources/job.go index 9477bf52517..1b167cafee7 100644 --- a/bundle/direct/dresources/job.go +++ b/bundle/direct/dresources/job.go @@ -94,9 +94,10 @@ func (r *ResourceJob) DoRead(ctx context.Context, id string) (*JobRemote, error) // GetByJobId only fetches the first page (100 tasks). Jobs.Get handles // pagination and returns the complete job with all tasks merged. job, err := r.client.Jobs.Get(ctx, jobs.GetJobRequest{ - JobId: idInt, - PageToken: "", - ForceSendFields: nil, + JobId: idInt, + PageToken: "", + IncludeTriggerState: false, + ForceSendFields: nil, }) if err != nil { return nil, err diff --git a/bundle/direct/dresources/postgres_branch.go b/bundle/direct/dresources/postgres_branch.go index 4f99d33ff53..625cccc5282 100644 --- a/bundle/direct/dresources/postgres_branch.go +++ b/bundle/direct/dresources/postgres_branch.go @@ -174,7 +174,9 @@ func (r *ResourcePostgresBranch) DoUpdate(ctx context.Context, id string, config func (r *ResourcePostgresBranch) DoDelete(ctx context.Context, id string) error { waiter, err := r.client.Postgres.DeleteBranch(ctx, postgres.DeleteBranchRequest{ - Name: id, + Name: id, + Purge: false, + ForceSendFields: nil, }) if err != nil { return err diff --git a/bundle/docsgen/output/reference.md b/bundle/docsgen/output/reference.md index 9d41a86792d..c88824d79ab 100644 --- a/bundle/docsgen/output/reference.md +++ b/bundle/docsgen/output/reference.md @@ -1,7 +1,7 @@ --- description: 'Configuration reference for databricks.yml' last_update: - date: 2026-05-11 + date: 2026-05-21 --- @@ -495,6 +495,10 @@ resources: - Map - See [\_](#resourcespostgres_branches). +- - `postgres_catalogs` + - Map + - The Postgres catalog definitions for the bundle, where each key is the name of the catalog. Each entry binds a Unity Catalog catalog to a Postgres database on a Lakebase Autoscaling branch. See [\_](#resourcespostgres_catalogs). + - - `postgres_endpoints` - Map - See [\_](#resourcespostgres_endpoints). @@ -503,6 +507,10 @@ resources: - Map - See [\_](#resourcespostgres_projects). +- - `postgres_synced_tables` + - Map + - The Postgres synced table definitions for the bundle, where each key is the name of the synced table. Each entry continuously replicates a Unity Catalog Delta source table into a Postgres table on a Lakebase Autoscaling instance. See [\_](#resourcespostgres_synced_tables). + - - `quality_monitors` - Map - The quality monitor definitions for the bundle, where each key is the name of the quality monitor. See [\_](/dev-tools/bundles/resources.md#quality_monitors). @@ -1521,6 +1529,10 @@ postgres_branches: - String - +- - `replace_existing` + - Boolean + - + - - `source_branch` - String - @@ -1548,6 +1560,69 @@ postgres_branches: +:::list-table + +- - Key + - Type + - Description + +- - `prevent_destroy` + - Boolean + - Lifecycle setting to prevent the resource from being destroyed. + +::: + + +### resources.postgres_catalogs + +**`Type: Map`** + +The Postgres catalog definitions for the bundle, where each key is the name of the catalog. Each entry binds a Unity Catalog catalog to a Postgres database on a Lakebase Autoscaling branch. + +```yaml +postgres_catalogs: + : + : +``` + + +:::list-table + +- - Key + - Type + - Description + +- - `branch` + - String + - + +- - `catalog_id` + - String + - + +- - `create_database_if_missing` + - Boolean + - + +- - `lifecycle` + - Map + - See [\_](#resourcespostgres_catalogsnamelifecycle). + +- - `postgres_database` + - String + - + +::: + + +### resources.postgres_catalogs._name_.lifecycle + +**`Type: Map`** + + + + + :::list-table - - Key @@ -1616,6 +1691,10 @@ postgres_endpoints: - String - +- - `replace_existing` + - Boolean + - + - - `settings` - Map - See [\_](#resourcespostgres_endpointsnamesettings). @@ -1768,6 +1847,93 @@ postgres_projects: ::: +### resources.postgres_synced_tables + +**`Type: Map`** + +The Postgres synced table definitions for the bundle, where each key is the name of the synced table. Each entry continuously replicates a Unity Catalog Delta source table into a Postgres table on a Lakebase Autoscaling instance. + +```yaml +postgres_synced_tables: + : + : +``` + + +:::list-table + +- - Key + - Type + - Description + +- - `branch` + - String + - + +- - `create_database_objects_if_missing` + - Boolean + - + +- - `existing_pipeline_id` + - String + - + +- - `lifecycle` + - Map + - See [\_](#resourcespostgres_synced_tablesnamelifecycle). + +- - `new_pipeline_spec` + - Map + - See [\_](#resourcespostgres_synced_tablesnamenew_pipeline_spec). + +- - `postgres_database` + - String + - + +- - `primary_key_columns` + - Sequence + - + +- - `scheduling_policy` + - String + - + +- - `source_table_full_name` + - String + - + +- - `synced_table_id` + - String + - + +- - `timeseries_key` + - String + - + +::: + + +### resources.postgres_synced_tables._name_.lifecycle + +**`Type: Map`** + + + + + +:::list-table + +- - Key + - Type + - Description + +- - `prevent_destroy` + - Boolean + - Lifecycle setting to prevent the resource from being destroyed. + +::: + + ### resources.secret_scopes **`Type: Map`** @@ -2573,6 +2739,10 @@ The resource definitions for the target. - Map - See [\_](#targetsnameresourcespostgres_branches). +- - `postgres_catalogs` + - Map + - The Postgres catalog definitions for the bundle, where each key is the name of the catalog. Each entry binds a Unity Catalog catalog to a Postgres database on a Lakebase Autoscaling branch. See [\_](#targetsnameresourcespostgres_catalogs). + - - `postgres_endpoints` - Map - See [\_](#targetsnameresourcespostgres_endpoints). @@ -2581,6 +2751,10 @@ The resource definitions for the target. - Map - See [\_](#targetsnameresourcespostgres_projects). +- - `postgres_synced_tables` + - Map + - The Postgres synced table definitions for the bundle, where each key is the name of the synced table. Each entry continuously replicates a Unity Catalog Delta source table into a Postgres table on a Lakebase Autoscaling instance. See [\_](#targetsnameresourcespostgres_synced_tables). + - - `quality_monitors` - Map - The quality monitor definitions for the bundle, where each key is the name of the quality monitor. See [\_](/dev-tools/bundles/resources.md#quality_monitors). @@ -3599,6 +3773,10 @@ postgres_branches: - String - +- - `replace_existing` + - Boolean + - + - - `source_branch` - String - @@ -3626,6 +3804,69 @@ postgres_branches: +:::list-table + +- - Key + - Type + - Description + +- - `prevent_destroy` + - Boolean + - Lifecycle setting to prevent the resource from being destroyed. + +::: + + +### targets._name_.resources.postgres_catalogs + +**`Type: Map`** + +The Postgres catalog definitions for the bundle, where each key is the name of the catalog. Each entry binds a Unity Catalog catalog to a Postgres database on a Lakebase Autoscaling branch. + +```yaml +postgres_catalogs: + : + : +``` + + +:::list-table + +- - Key + - Type + - Description + +- - `branch` + - String + - + +- - `catalog_id` + - String + - + +- - `create_database_if_missing` + - Boolean + - + +- - `lifecycle` + - Map + - See [\_](#targetsnameresourcespostgres_catalogsnamelifecycle). + +- - `postgres_database` + - String + - + +::: + + +### targets._name_.resources.postgres_catalogs._name_.lifecycle + +**`Type: Map`** + + + + + :::list-table - - Key @@ -3694,6 +3935,10 @@ postgres_endpoints: - String - +- - `replace_existing` + - Boolean + - + - - `settings` - Map - See [\_](#targetsnameresourcespostgres_endpointsnamesettings). @@ -3846,6 +4091,93 @@ postgres_projects: ::: +### targets._name_.resources.postgres_synced_tables + +**`Type: Map`** + +The Postgres synced table definitions for the bundle, where each key is the name of the synced table. Each entry continuously replicates a Unity Catalog Delta source table into a Postgres table on a Lakebase Autoscaling instance. + +```yaml +postgres_synced_tables: + : + : +``` + + +:::list-table + +- - Key + - Type + - Description + +- - `branch` + - String + - + +- - `create_database_objects_if_missing` + - Boolean + - + +- - `existing_pipeline_id` + - String + - + +- - `lifecycle` + - Map + - See [\_](#targetsnameresourcespostgres_synced_tablesnamelifecycle). + +- - `new_pipeline_spec` + - Map + - See [\_](#targetsnameresourcespostgres_synced_tablesnamenew_pipeline_spec). + +- - `postgres_database` + - String + - + +- - `primary_key_columns` + - Sequence + - + +- - `scheduling_policy` + - String + - + +- - `source_table_full_name` + - String + - + +- - `synced_table_id` + - String + - + +- - `timeseries_key` + - String + - + +::: + + +### targets._name_.resources.postgres_synced_tables._name_.lifecycle + +**`Type: Map`** + + + + + +:::list-table + +- - Key + - Type + - Description + +- - `prevent_destroy` + - Boolean + - Lifecycle setting to prevent the resource from being destroyed. + +::: + + ### targets._name_.resources.secret_scopes **`Type: Map`** diff --git a/bundle/docsgen/output/resources.md b/bundle/docsgen/output/resources.md index 392e574dcd3..1ab8ef813d7 100644 --- a/bundle/docsgen/output/resources.md +++ b/bundle/docsgen/output/resources.md @@ -1,7 +1,7 @@ --- description: 'Learn about resources supported by Declarative Automation Bundles and how to configure them.' last_update: - date: 2026-05-11 + date: 2026-05-21 --- @@ -544,6 +544,10 @@ apps: - String - The description of the app. +- - `git_repository` + - Map + - Git repository configuration for app deployments. When specified, deployments can reference code from this repository by providing only the git reference (branch, tag, or commit). See [\_](#appsnamegit_repository). + - - `git_source` - Map - Git source configuration for app deployments. Specifies which git reference (branch, tag, or commit) to use when deploying the app. Used in conjunction with git_repository to deploy code directly from git. The source_code_path within git_source specifies the relative path to the app code within the repository. See [\_](#appsnamegit_source). @@ -572,10 +576,6 @@ apps: - Sequence - See [\_](#appsnametelemetry_export_destinations). -- - `thumbnail_url` - - String - - - - - `usage_policy_id` - String - @@ -641,6 +641,32 @@ apps: ::: +### apps._name_.git_repository + +**`Type: Map`** + +Git repository configuration for app deployments. When specified, deployments can +reference code from this repository by providing only the git reference (branch, tag, or commit). + + + +:::list-table + +- - Key + - Type + - Description + +- - `provider` + - String + - Git provider. Case insensitive. Supported values: gitHub, gitHubEnterprise, bitbucketCloud, bitbucketServer, azureDevOpsServices, gitLab, gitLabEnterpriseEdition, awsCodeCommit. + +- - `url` + - String + - URL of the Git repository. + +::: + + ### apps._name_.git_source **`Type: Map`** @@ -1154,6 +1180,10 @@ catalogs: - Map - See [\_](#catalogsnamelifecycle). +- - `managed_encryption_settings` + - Map + - Control CMK encryption for managed catalog data. See [\_](#catalogsnamemanaged_encryption_settings). + - - `name` - String - @@ -1234,6 +1264,64 @@ The privileges assigned to the principal. ::: +### catalogs._name_.managed_encryption_settings + +**`Type: Map`** + +Control CMK encryption for managed catalog data + + + +:::list-table + +- - Key + - Type + - Description + +- - `azure_encryption_settings` + - Map + - optional Azure settings - only required if an Azure CMK is used. See [\_](#catalogsnamemanaged_encryption_settingsazure_encryption_settings). + +- - `azure_key_vault_key_id` + - String + - the AKV URL in Azure, null otherwise. + +- - `customer_managed_key_id` + - String + - the CMK uuid in AWS and GCP, null otherwise. + +::: + + +### catalogs._name_.managed_encryption_settings.azure_encryption_settings + +**`Type: Map`** + +optional Azure settings - only required if an Azure CMK is used. + + + +:::list-table + +- - Key + - Type + - Description + +- - `azure_cmk_access_connector_id` + - String + - + +- - `azure_cmk_managed_identity_id` + - String + - + +- - `azure_tenant_id` + - String + - + +::: + + ## clusters **`Type: Map`** @@ -1790,10 +1878,6 @@ If not specified at cluster creation, a set of default values will be used. - Integer - Boot disk size in GB -- - `confidential_compute_type` - - String - - - - - `first_on_demand` - Integer - The first `first_on_demand` nodes of the cluster will be placed on on-demand instances. This value should be greater than 0, to make sure the cluster driver node is placed on an on-demand instance. If this value is greater than or equal to the current cluster size, all nodes will be placed on on-demand instances. If this value is less than the current cluster size, `first_on_demand` nodes will be placed on on-demand instances and the remainder will be placed on `availability` instances. Note that this value does not affect cluster size and cannot currently be mutated over the lifetime of a cluster. @@ -3289,7 +3373,7 @@ For other serverless tasks, the task environment is required to be specified usi - - `spec` - Map - - The environment entity used to preserve serverless environment side panel, jobs' environment for non-notebook task, and DLT's environment for classic and serverless pipelines. In this minimal environment spec, only pip and java dependencies are supported. See [\_](#jobsnameenvironmentsspec). + - The environment entity used to preserve serverless environment side panel, jobs' environment for non-notebook task, and SDP's environment for classic and serverless pipelines. In this minimal environment spec, only pip and java dependencies are supported. See [\_](#jobsnameenvironmentsspec). ::: @@ -3298,7 +3382,7 @@ For other serverless tasks, the task environment is required to be specified usi **`Type: Map`** -The environment entity used to preserve serverless environment side panel, jobs' environment for non-notebook task, and DLT's environment for classic and serverless pipelines. +The environment entity used to preserve serverless environment side panel, jobs' environment for non-notebook task, and SDP's environment for classic and serverless pipelines. In this minimal environment spec, only pip and java dependencies are supported. @@ -4013,10 +4097,6 @@ If not specified at cluster creation, a set of default values will be used. - Integer - Boot disk size in GB -- - `confidential_compute_type` - - String - - - - - `first_on_demand` - Integer - The first `first_on_demand` nodes of the cluster will be placed on on-demand instances. This value should be greater than 0, to make sure the cluster driver node is placed on an on-demand instance. If this value is greater than or equal to the current cluster size, all nodes will be placed on on-demand instances. If this value is less than the current cluster size, `first_on_demand` nodes will be placed on on-demand instances and the remainder will be placed on `availability` instances. Note that this value does not affect cluster size and cannot currently be mutated over the lifetime of a cluster. @@ -4600,6 +4680,10 @@ Read endpoints return only 100 tasks. If more than 100 tasks are available, you - Map - The task triggers a Power BI semantic model update when the `power_bi_task` field is present. See [\_](#jobsnametaskspower_bi_task). +- - `python_operator_task` + - Map + - See [\_](#jobsnametaskspython_operator_task). + - - `python_wheel_task` - Map - The task runs a Python wheel when the `python_wheel_task` field is present. See [\_](#jobsnametaskspython_wheel_task). @@ -5708,10 +5792,6 @@ If not specified at cluster creation, a set of default values will be used. - Integer - Boot disk size in GB -- - `confidential_compute_type` - - String - - - - - `first_on_demand` - Integer - The first `first_on_demand` nodes of the cluster will be placed on on-demand instances. This value should be greater than 0, to make sure the cluster driver node is placed on an on-demand instance. If this value is greater than or equal to the current cluster size, all nodes will be placed on on-demand instances. If this value is less than the current cluster size, `first_on_demand` nodes will be placed on on-demand instances and the remainder will be placed on `availability` instances. Note that this value does not affect cluster size and cannot currently be mutated over the lifetime of a cluster. @@ -6084,12 +6164,28 @@ The task triggers a pipeline update when the `pipeline_task` field is present. O - - `full_refresh` - Boolean - - If true, triggers a full refresh on the delta live table. + - If true, triggers a full refresh on the spark declarative pipeline. + +- - `full_refresh_selection` + - Sequence + - - - `pipeline_id` - String - The full name of the pipeline task to execute. +- - `refresh_flow_selection` + - Sequence + - + +- - `refresh_selection` + - Sequence + - + +- - `reset_checkpoint_selection` + - Sequence + - + ::: @@ -6278,7 +6374,23 @@ Controls whether the pipeline should perform a full refresh - - `full_refresh` - Boolean - - If true, triggers a full refresh on the delta live table. + - If true, triggers a full refresh on the spark declarative pipeline. + +- - `full_refresh_selection` + - Sequence + - + +- - `refresh_flow_selection` + - Sequence + - + +- - `refresh_selection` + - Sequence + - + +- - `reset_checkpoint_selection` + - Sequence + - ::: @@ -7345,7 +7457,7 @@ The core config of the serving endpoint. - - `auto_capture_config` - Map - - Configuration for Inference Tables which automatically logs requests and responses to Unity Catalog. Note: this field is deprecated for creating new provisioned throughput endpoints, or updating existing provisioned throughput endpoints that never have inference table configured; in these cases please use AI Gateway to manage inference tables. See [\_](#model_serving_endpointsnameconfigauto_capture_config). + - This field is deprecated - - `served_entities` - Sequence @@ -7362,42 +7474,6 @@ The core config of the serving endpoint. ::: -### model_serving_endpoints._name_.config.auto_capture_config - -**`Type: Map`** - -Configuration for Inference Tables which automatically logs requests and responses to Unity Catalog. -Note: this field is deprecated for creating new provisioned throughput endpoints, -or updating existing provisioned throughput endpoints that never have inference table configured; -in these cases please use AI Gateway to manage inference tables. - - - -:::list-table - -- - Key - - Type - - Description - -- - `catalog_name` - - String - - The name of the catalog in Unity Catalog. NOTE: On update, you cannot change the catalog name if the inference table is already enabled. - -- - `enabled` - - Boolean - - Indicates whether the inference table is enabled. - -- - `schema_name` - - String - - The name of the schema in Unity Catalog. NOTE: On update, you cannot change the schema name if the inference table is already enabled. - -- - `table_name_prefix` - - String - - The prefix of the table in Unity Catalog. NOTE: On update, you cannot change the prefix name if the inference table is already enabled. - -::: - - ### model_serving_endpoints._name_.config.served_entities **`Type: Sequence`** @@ -8278,7 +8354,7 @@ pipelines: - - `channel` - String - - DLT Release Channel that specifies which version to use. + - SDP Release Channel that specifies which version to use. - - `clusters` - Sequence @@ -8797,10 +8873,6 @@ If not specified at cluster creation, a set of default values will be used. - Integer - Boot disk size in GB -- - `confidential_compute_type` - - String - - - - - `first_on_demand` - Integer - The first `first_on_demand` nodes of the cluster will be placed on on-demand instances. This value should be greater than 0, to make sure the cluster driver node is placed on an on-demand instance. If this value is greater than or equal to the current cluster size, all nodes will be placed on on-demand instances. If this value is less than the current cluster size, `first_on_demand` nodes will be placed on on-demand instances and the remainder will be placed on `availability` instances. Note that this value does not affect cluster size and cannot currently be mutated over the lifetime of a cluster. @@ -9148,6 +9220,10 @@ The configuration for a managed ingestion pipeline. These settings cannot be use - Map - (Optional) A window that specifies a set of time ranges for snapshot queries in CDC. See [\_](#pipelinesnameingestion_definitionfull_refresh_window). +- - `ingest_from_uc_foreign_catalog` + - Boolean + - Immutable. If set to true, the pipeline will ingest tables from the UC foreign catalogs directly without the need to specify a UC connection or ingestion gateway. The `source_catalog` fields in objects of IngestionConfig are interpreted as the UC foreign catalogs to ingest from. + - - `ingestion_gateway_id` - String - Identifier for the gateway that is used by this ingestion pipeline to communicate with the source database. This is used with CDC connectors to databases like SQL Server using a gateway pipeline (connector_type = CDC). Under certain conditions, this can be replaced with connection_name to change the connector to Combined Cdc Managed Ingestion Pipeline. @@ -9300,6 +9376,18 @@ Configuration settings to control the ingestion of tables. These settings overri - Sequence - The primary key of the table used to apply changes. +- - `query_based_connector_config` + - Map + - Configurations that are only applicable for query-based ingestion connectors. See [\_](#pipelinesnameingestion_definitionobjectsreporttable_configurationquery_based_connector_config). + +- - `row_filter` + - String + - (Optional, Immutable) The row filter condition to be applied to the table. It must not contain the WHERE keyword, only the actual filter condition. It must be in DBSQL format. + +- - `scd_type` + - String + - The SCD type to use to ingest the table. + - - `sequence_by` - Sequence - The column names specifying the logical order of events in the source data. Spark Declarative Pipelines uses this sequencing to handle change events that arrive out of order. @@ -9342,6 +9430,35 @@ If unspecified, auto full refresh is disabled. ::: +### pipelines._name_.ingestion_definition.objects.report.table_configuration.query_based_connector_config + +**`Type: Map`** + +Configurations that are only applicable for query-based ingestion connectors. + + + +:::list-table + +- - Key + - Type + - Description + +- - `cursor_columns` + - Sequence + - The names of the monotonically increasing columns in the source table that are used to enable the table to be read and ingested incrementally through structured streaming. The columns are allowed to have repeated values but have to be non-decreasing. If the source data is merged into the destination (e.g., using SCD Type 1 or Type 2), these columns will implicitly define the `sequence_by` behavior. You can still explicitly set `sequence_by` to override this default. + +- - `deletion_condition` + - String + - Specifies a SQL WHERE condition that specifies that the source row has been deleted. This is sometimes referred to as "soft-deletes". For example: "Operation = 'DELETE'" or "is_deleted = true". This field is orthogonal to `hard_deletion_sync_interval_in_seconds`, one for soft-deletes and the other for hard-deletes. See also the hard_deletion_sync_min_interval_in_seconds field for handling of "hard deletes" where the source rows are physically removed from the table. + +- - `hard_deletion_sync_min_interval_in_seconds` + - Integer + - Specifies the minimum interval (in seconds) between snapshots on primary keys for detecting and synchronizing hard deletions—i.e., rows that have been physically removed from the source table. This interval acts as a lower bound. If ingestion runs less frequently than this value, hard deletion synchronization will align with the actual ingestion frequency instead of happening more often. If not set, hard deletion synchronization via snapshots is disabled. This field is mutable and can be updated without triggering a full snapshot. + +::: + + ### pipelines._name_.ingestion_definition.objects.schema **`Type: Map`** @@ -9356,6 +9473,10 @@ Select all tables from a specific source schema. - Type - Description +- - `connector_options` + - Map + - (Optional) Source Specific Connector Options. See [\_](#pipelinesnameingestion_definitionobjectsschemaconnector_options). + - - `destination_catalog` - String - Required. Destination catalog to store tables. @@ -9379,11 +9500,11 @@ Select all tables from a specific source schema. ::: -### pipelines._name_.ingestion_definition.objects.schema.table_configuration +### pipelines._name_.ingestion_definition.objects.schema.connector_options **`Type: Map`** -Configuration settings to control the ingestion of tables. These settings are applied to all tables in this schema and override the table_configuration defined in the IngestionPipelineDefinition object. +(Optional) Source Specific Connector Options @@ -9393,44 +9514,51 @@ Configuration settings to control the ingestion of tables. These settings are ap - Type - Description -- - `auto_full_refresh_policy` +- - `confluence_options` - Map - - (Optional, Mutable) Policy for auto full refresh, if enabled pipeline will automatically try to fix issues by doing a full refresh on the table in the retry run. auto_full_refresh_policy in table configuration will override the above level auto_full_refresh_policy. For example, { "auto_full_refresh_policy": { "enabled": true, "min_interval_hours": 23, } } If unspecified, auto full refresh is disabled. See [\_](#pipelinesnameingestion_definitionobjectsschematable_configurationauto_full_refresh_policy). + - Confluence specific options for ingestion. See [\_](#pipelinesnameingestion_definitionobjectsschemaconnector_optionsconfluence_options). -- - `exclude_columns` - - Sequence - - A list of column names to be excluded for the ingestion. When not specified, include_columns fully controls what columns to be ingested. When specified, all other columns including future ones will be automatically included for ingestion. This field in mutually exclusive with `include_columns`. +- - `jira_options` + - Map + - Jira specific options for ingestion. See [\_](#pipelinesnameingestion_definitionobjectsschemaconnector_optionsjira_options). -- - `include_columns` - - Sequence - - A list of column names to be included for the ingestion. When not specified, all columns except ones in exclude_columns will be included. Future columns will be automatically included. When specified, all other future columns will be automatically excluded from ingestion. This field in mutually exclusive with `exclude_columns`. +- - `kafka_options` + - Map + - See [\_](#pipelinesnameingestion_definitionobjectsschemaconnector_optionskafka_options). -- - `primary_keys` - - Sequence - - The primary key of the table used to apply changes. +- - `meta_ads_options` + - Map + - Meta Marketing (Meta Ads) specific options for ingestion. See [\_](#pipelinesnameingestion_definitionobjectsschemaconnector_optionsmeta_ads_options). -- - `sequence_by` +::: + + +### pipelines._name_.ingestion_definition.objects.schema.connector_options.confluence_options + +**`Type: Map`** + +Confluence specific options for ingestion + + + +:::list-table + +- - Key + - Type + - Description + +- - `include_confluence_spaces` - Sequence - - The column names specifying the logical order of events in the source data. Spark Declarative Pipelines uses this sequencing to handle change events that arrive out of order. + - (Optional) Spaces to filter Confluence data on ::: -### pipelines._name_.ingestion_definition.objects.schema.table_configuration.auto_full_refresh_policy +### pipelines._name_.ingestion_definition.objects.schema.connector_options.jira_options **`Type: Map`** -(Optional, Mutable) Policy for auto full refresh, if enabled pipeline will automatically try -to fix issues by doing a full refresh on the table in the retry run. auto_full_refresh_policy -in table configuration will override the above level auto_full_refresh_policy. -For example, -{ -"auto_full_refresh_policy": { -"enabled": true, -"min_interval_hours": 23, -} -} -If unspecified, auto full refresh is disabled. +Jira specific options for ingestion @@ -9440,22 +9568,18 @@ If unspecified, auto full refresh is disabled. - Type - Description -- - `enabled` - - Boolean - - (Required, Mutable) Whether to enable auto full refresh or not. - -- - `min_interval_hours` - - Integer - - (Optional, Mutable) Specify the minimum interval in hours between the timestamp at which a table was last full refreshed and the current timestamp for triggering auto full If unspecified and autoFullRefresh is enabled then by default min_interval_hours is 24 hours. +- - `include_jira_spaces` + - Sequence + - (Optional) Projects to filter Jira data on ::: -### pipelines._name_.ingestion_definition.objects.table +### pipelines._name_.ingestion_definition.objects.schema.connector_options.meta_ads_options **`Type: Map`** -Select a specific source table. +Meta Marketing (Meta Ads) specific options for ingestion @@ -9465,42 +9589,46 @@ Select a specific source table. - Type - Description -- - `destination_catalog` - - String - - Required. Destination catalog to store table. +- - `action_attribution_windows` + - Sequence + - (Optional) Action attribution windows for insights reporting (e.g. "28d_click", "1d_view") -- - `destination_schema` - - String - - Required. Destination schema to store table. +- - `action_breakdowns` + - Sequence + - (Optional) Action breakdowns to configure for data aggregation -- - `destination_table` +- - `action_report_time` - String - - Optional. Destination table name. The pipeline fails if a table with that name already exists. If not set, the source table name is used. + - (Optional) Timing used to report action statistics (impression, conversion, mixed, or lifetime) -- - `source_catalog` - - String - - Source catalog name. Might be optional depending on the type of source. +- - `breakdowns` + - Sequence + - (Optional) Breakdowns to configure for data aggregation -- - `source_schema` +- - `custom_insights_lookback_window` + - Integer + - (Optional) Window in days to revisit data during sync to capture updated conversion data from the API. + +- - `level` - String - - Schema name in the source database. Might be optional depending on the type of source. + - (Optional) Granularity of data to pull (account, ad, adset, campaign) -- - `source_table` +- - `start_date` - String - - Required. Table name in the source database. + - (Optional) Start date in yyyy-MM-dd format (e.g. 2025-01-15). Data added after this date will be ingested -- - `table_configuration` - - Map - - Configuration settings to control the ingestion of tables. These settings override the table_configuration defined in the IngestionPipelineDefinition object and the SchemaSpec. See [\_](#pipelinesnameingestion_definitionobjectstabletable_configuration). +- - `time_increment` + - String + - (Optional) Value in string by which to aggregate statistics (can take all_days, monthly or number of days) ::: -### pipelines._name_.ingestion_definition.objects.table.table_configuration +### pipelines._name_.ingestion_definition.objects.schema.table_configuration **`Type: Map`** -Configuration settings to control the ingestion of tables. These settings override the table_configuration defined in the IngestionPipelineDefinition object and the SchemaSpec. +Configuration settings to control the ingestion of tables. These settings are applied to all tables in this schema and override the table_configuration defined in the IngestionPipelineDefinition object. @@ -9512,7 +9640,293 @@ Configuration settings to control the ingestion of tables. These settings overri - - `auto_full_refresh_policy` - Map - - (Optional, Mutable) Policy for auto full refresh, if enabled pipeline will automatically try to fix issues by doing a full refresh on the table in the retry run. auto_full_refresh_policy in table configuration will override the above level auto_full_refresh_policy. For example, { "auto_full_refresh_policy": { "enabled": true, "min_interval_hours": 23, } } If unspecified, auto full refresh is disabled. See [\_](#pipelinesnameingestion_definitionobjectstabletable_configurationauto_full_refresh_policy). + - (Optional, Mutable) Policy for auto full refresh, if enabled pipeline will automatically try to fix issues by doing a full refresh on the table in the retry run. auto_full_refresh_policy in table configuration will override the above level auto_full_refresh_policy. For example, { "auto_full_refresh_policy": { "enabled": true, "min_interval_hours": 23, } } If unspecified, auto full refresh is disabled. See [\_](#pipelinesnameingestion_definitionobjectsschematable_configurationauto_full_refresh_policy). + +- - `exclude_columns` + - Sequence + - A list of column names to be excluded for the ingestion. When not specified, include_columns fully controls what columns to be ingested. When specified, all other columns including future ones will be automatically included for ingestion. This field in mutually exclusive with `include_columns`. + +- - `include_columns` + - Sequence + - A list of column names to be included for the ingestion. When not specified, all columns except ones in exclude_columns will be included. Future columns will be automatically included. When specified, all other future columns will be automatically excluded from ingestion. This field in mutually exclusive with `exclude_columns`. + +- - `primary_keys` + - Sequence + - The primary key of the table used to apply changes. + +- - `query_based_connector_config` + - Map + - Configurations that are only applicable for query-based ingestion connectors. See [\_](#pipelinesnameingestion_definitionobjectsschematable_configurationquery_based_connector_config). + +- - `row_filter` + - String + - (Optional, Immutable) The row filter condition to be applied to the table. It must not contain the WHERE keyword, only the actual filter condition. It must be in DBSQL format. + +- - `scd_type` + - String + - The SCD type to use to ingest the table. + +- - `sequence_by` + - Sequence + - The column names specifying the logical order of events in the source data. Spark Declarative Pipelines uses this sequencing to handle change events that arrive out of order. + +::: + + +### pipelines._name_.ingestion_definition.objects.schema.table_configuration.auto_full_refresh_policy + +**`Type: Map`** + +(Optional, Mutable) Policy for auto full refresh, if enabled pipeline will automatically try +to fix issues by doing a full refresh on the table in the retry run. auto_full_refresh_policy +in table configuration will override the above level auto_full_refresh_policy. +For example, +{ +"auto_full_refresh_policy": { +"enabled": true, +"min_interval_hours": 23, +} +} +If unspecified, auto full refresh is disabled. + + + +:::list-table + +- - Key + - Type + - Description + +- - `enabled` + - Boolean + - (Required, Mutable) Whether to enable auto full refresh or not. + +- - `min_interval_hours` + - Integer + - (Optional, Mutable) Specify the minimum interval in hours between the timestamp at which a table was last full refreshed and the current timestamp for triggering auto full If unspecified and autoFullRefresh is enabled then by default min_interval_hours is 24 hours. + +::: + + +### pipelines._name_.ingestion_definition.objects.schema.table_configuration.query_based_connector_config + +**`Type: Map`** + +Configurations that are only applicable for query-based ingestion connectors. + + + +:::list-table + +- - Key + - Type + - Description + +- - `cursor_columns` + - Sequence + - The names of the monotonically increasing columns in the source table that are used to enable the table to be read and ingested incrementally through structured streaming. The columns are allowed to have repeated values but have to be non-decreasing. If the source data is merged into the destination (e.g., using SCD Type 1 or Type 2), these columns will implicitly define the `sequence_by` behavior. You can still explicitly set `sequence_by` to override this default. + +- - `deletion_condition` + - String + - Specifies a SQL WHERE condition that specifies that the source row has been deleted. This is sometimes referred to as "soft-deletes". For example: "Operation = 'DELETE'" or "is_deleted = true". This field is orthogonal to `hard_deletion_sync_interval_in_seconds`, one for soft-deletes and the other for hard-deletes. See also the hard_deletion_sync_min_interval_in_seconds field for handling of "hard deletes" where the source rows are physically removed from the table. + +- - `hard_deletion_sync_min_interval_in_seconds` + - Integer + - Specifies the minimum interval (in seconds) between snapshots on primary keys for detecting and synchronizing hard deletions—i.e., rows that have been physically removed from the source table. This interval acts as a lower bound. If ingestion runs less frequently than this value, hard deletion synchronization will align with the actual ingestion frequency instead of happening more often. If not set, hard deletion synchronization via snapshots is disabled. This field is mutable and can be updated without triggering a full snapshot. + +::: + + +### pipelines._name_.ingestion_definition.objects.table + +**`Type: Map`** + +Select a specific source table. + + + +:::list-table + +- - Key + - Type + - Description + +- - `connector_options` + - Map + - (Optional) Source Specific Connector Options. See [\_](#pipelinesnameingestion_definitionobjectstableconnector_options). + +- - `destination_catalog` + - String + - Required. Destination catalog to store table. + +- - `destination_schema` + - String + - Required. Destination schema to store table. + +- - `destination_table` + - String + - Optional. Destination table name. The pipeline fails if a table with that name already exists. If not set, the source table name is used. + +- - `source_catalog` + - String + - Source catalog name. Might be optional depending on the type of source. + +- - `source_schema` + - String + - Schema name in the source database. Might be optional depending on the type of source. + +- - `source_table` + - String + - Required. Table name in the source database. + +- - `table_configuration` + - Map + - Configuration settings to control the ingestion of tables. These settings override the table_configuration defined in the IngestionPipelineDefinition object and the SchemaSpec. See [\_](#pipelinesnameingestion_definitionobjectstabletable_configuration). + +::: + + +### pipelines._name_.ingestion_definition.objects.table.connector_options + +**`Type: Map`** + +(Optional) Source Specific Connector Options + + + +:::list-table + +- - Key + - Type + - Description + +- - `confluence_options` + - Map + - Confluence specific options for ingestion. See [\_](#pipelinesnameingestion_definitionobjectstableconnector_optionsconfluence_options). + +- - `jira_options` + - Map + - Jira specific options for ingestion. See [\_](#pipelinesnameingestion_definitionobjectstableconnector_optionsjira_options). + +- - `kafka_options` + - Map + - See [\_](#pipelinesnameingestion_definitionobjectstableconnector_optionskafka_options). + +- - `meta_ads_options` + - Map + - Meta Marketing (Meta Ads) specific options for ingestion. See [\_](#pipelinesnameingestion_definitionobjectstableconnector_optionsmeta_ads_options). + +::: + + +### pipelines._name_.ingestion_definition.objects.table.connector_options.confluence_options + +**`Type: Map`** + +Confluence specific options for ingestion + + + +:::list-table + +- - Key + - Type + - Description + +- - `include_confluence_spaces` + - Sequence + - (Optional) Spaces to filter Confluence data on + +::: + + +### pipelines._name_.ingestion_definition.objects.table.connector_options.jira_options + +**`Type: Map`** + +Jira specific options for ingestion + + + +:::list-table + +- - Key + - Type + - Description + +- - `include_jira_spaces` + - Sequence + - (Optional) Projects to filter Jira data on + +::: + + +### pipelines._name_.ingestion_definition.objects.table.connector_options.meta_ads_options + +**`Type: Map`** + +Meta Marketing (Meta Ads) specific options for ingestion + + + +:::list-table + +- - Key + - Type + - Description + +- - `action_attribution_windows` + - Sequence + - (Optional) Action attribution windows for insights reporting (e.g. "28d_click", "1d_view") + +- - `action_breakdowns` + - Sequence + - (Optional) Action breakdowns to configure for data aggregation + +- - `action_report_time` + - String + - (Optional) Timing used to report action statistics (impression, conversion, mixed, or lifetime) + +- - `breakdowns` + - Sequence + - (Optional) Breakdowns to configure for data aggregation + +- - `custom_insights_lookback_window` + - Integer + - (Optional) Window in days to revisit data during sync to capture updated conversion data from the API. + +- - `level` + - String + - (Optional) Granularity of data to pull (account, ad, adset, campaign) + +- - `start_date` + - String + - (Optional) Start date in yyyy-MM-dd format (e.g. 2025-01-15). Data added after this date will be ingested + +- - `time_increment` + - String + - (Optional) Value in string by which to aggregate statistics (can take all_days, monthly or number of days) + +::: + + +### pipelines._name_.ingestion_definition.objects.table.table_configuration + +**`Type: Map`** + +Configuration settings to control the ingestion of tables. These settings override the table_configuration defined in the IngestionPipelineDefinition object and the SchemaSpec. + + + +:::list-table + +- - Key + - Type + - Description + +- - `auto_full_refresh_policy` + - Map + - (Optional, Mutable) Policy for auto full refresh, if enabled pipeline will automatically try to fix issues by doing a full refresh on the table in the retry run. auto_full_refresh_policy in table configuration will override the above level auto_full_refresh_policy. For example, { "auto_full_refresh_policy": { "enabled": true, "min_interval_hours": 23, } } If unspecified, auto full refresh is disabled. See [\_](#pipelinesnameingestion_definitionobjectstabletable_configurationauto_full_refresh_policy). - - `exclude_columns` - Sequence @@ -9526,6 +9940,18 @@ Configuration settings to control the ingestion of tables. These settings overri - Sequence - The primary key of the table used to apply changes. +- - `query_based_connector_config` + - Map + - Configurations that are only applicable for query-based ingestion connectors. See [\_](#pipelinesnameingestion_definitionobjectstabletable_configurationquery_based_connector_config). + +- - `row_filter` + - String + - (Optional, Immutable) The row filter condition to be applied to the table. It must not contain the WHERE keyword, only the actual filter condition. It must be in DBSQL format. + +- - `scd_type` + - String + - The SCD type to use to ingest the table. + - - `sequence_by` - Sequence - The column names specifying the logical order of events in the source data. Spark Declarative Pipelines uses this sequencing to handle change events that arrive out of order. @@ -9568,6 +9994,35 @@ If unspecified, auto full refresh is disabled. ::: +### pipelines._name_.ingestion_definition.objects.table.table_configuration.query_based_connector_config + +**`Type: Map`** + +Configurations that are only applicable for query-based ingestion connectors. + + + +:::list-table + +- - Key + - Type + - Description + +- - `cursor_columns` + - Sequence + - The names of the monotonically increasing columns in the source table that are used to enable the table to be read and ingested incrementally through structured streaming. The columns are allowed to have repeated values but have to be non-decreasing. If the source data is merged into the destination (e.g., using SCD Type 1 or Type 2), these columns will implicitly define the `sequence_by` behavior. You can still explicitly set `sequence_by` to override this default. + +- - `deletion_condition` + - String + - Specifies a SQL WHERE condition that specifies that the source row has been deleted. This is sometimes referred to as "soft-deletes". For example: "Operation = 'DELETE'" or "is_deleted = true". This field is orthogonal to `hard_deletion_sync_interval_in_seconds`, one for soft-deletes and the other for hard-deletes. See also the hard_deletion_sync_min_interval_in_seconds field for handling of "hard deletes" where the source rows are physically removed from the table. + +- - `hard_deletion_sync_min_interval_in_seconds` + - Integer + - Specifies the minimum interval (in seconds) between snapshots on primary keys for detecting and synchronizing hard deletions—i.e., rows that have been physically removed from the source table. This interval acts as a lower bound. If ingestion runs less frequently than this value, hard deletion synchronization will align with the actual ingestion frequency instead of happening more often. If not set, hard deletion synchronization via snapshots is disabled. This field is mutable and can be updated without triggering a full snapshot. + +::: + + ### pipelines._name_.ingestion_definition.source_configurations **`Type: Sequence`** @@ -9586,10 +10041,6 @@ Top-level source configurations - Map - Catalog-level source configuration parameters. See [\_](#pipelinesnameingestion_definitionsource_configurationscatalog). -- - `google_ads_config` - - Map - - See [\_](#pipelinesnameingestion_definitionsource_configurationsgoogle_ads_config). - ::: @@ -9694,6 +10145,18 @@ Configuration settings to control the ingestion of tables. These settings are ap - Sequence - The primary key of the table used to apply changes. +- - `query_based_connector_config` + - Map + - Configurations that are only applicable for query-based ingestion connectors. See [\_](#pipelinesnameingestion_definitiontable_configurationquery_based_connector_config). + +- - `row_filter` + - String + - (Optional, Immutable) The row filter condition to be applied to the table. It must not contain the WHERE keyword, only the actual filter condition. It must be in DBSQL format. + +- - `scd_type` + - String + - The SCD type to use to ingest the table. + - - `sequence_by` - Sequence - The column names specifying the logical order of events in the source data. Spark Declarative Pipelines uses this sequencing to handle change events that arrive out of order. @@ -9736,6 +10199,35 @@ If unspecified, auto full refresh is disabled. ::: +### pipelines._name_.ingestion_definition.table_configuration.query_based_connector_config + +**`Type: Map`** + +Configurations that are only applicable for query-based ingestion connectors. + + + +:::list-table + +- - Key + - Type + - Description + +- - `cursor_columns` + - Sequence + - The names of the monotonically increasing columns in the source table that are used to enable the table to be read and ingested incrementally through structured streaming. The columns are allowed to have repeated values but have to be non-decreasing. If the source data is merged into the destination (e.g., using SCD Type 1 or Type 2), these columns will implicitly define the `sequence_by` behavior. You can still explicitly set `sequence_by` to override this default. + +- - `deletion_condition` + - String + - Specifies a SQL WHERE condition that specifies that the source row has been deleted. This is sometimes referred to as "soft-deletes". For example: "Operation = 'DELETE'" or "is_deleted = true". This field is orthogonal to `hard_deletion_sync_interval_in_seconds`, one for soft-deletes and the other for hard-deletes. See also the hard_deletion_sync_min_interval_in_seconds field for handling of "hard deletes" where the source rows are physically removed from the table. + +- - `hard_deletion_sync_min_interval_in_seconds` + - Integer + - Specifies the minimum interval (in seconds) between snapshots on primary keys for detecting and synchronizing hard deletions—i.e., rows that have been physically removed from the source table. This interval acts as a lower bound. If ingestion runs less frequently than this value, hard deletion synchronization will align with the actual ingestion frequency instead of happening more often. If not set, hard deletion synchronization via snapshots is disabled. This field is mutable and can be updated without triggering a full snapshot. + +::: + + ### pipelines._name_.libraries **`Type: Sequence`** @@ -9983,6 +10475,10 @@ postgres_branches: - String - +- - `replace_existing` + - Boolean + - + - - `source_branch` - String - @@ -10010,6 +10506,69 @@ postgres_branches: +:::list-table + +- - Key + - Type + - Description + +- - `prevent_destroy` + - Boolean + - Lifecycle setting to prevent the resource from being destroyed. + +::: + + +## postgres_catalogs + +**`Type: Map`** + +The Postgres catalog definitions for the bundle, where each key is the name of the catalog. Each entry binds a Unity Catalog catalog to a Postgres database on a Lakebase Autoscaling branch. + +```yaml +postgres_catalogs: + : + : +``` + + +:::list-table + +- - Key + - Type + - Description + +- - `branch` + - String + - + +- - `catalog_id` + - String + - + +- - `create_database_if_missing` + - Boolean + - + +- - `lifecycle` + - Map + - See [\_](#postgres_catalogsnamelifecycle). + +- - `postgres_database` + - String + - + +::: + + +### postgres_catalogs._name_.lifecycle + +**`Type: Map`** + + + + + :::list-table - - Key @@ -10078,6 +10637,10 @@ postgres_endpoints: - String - +- - `replace_existing` + - Boolean + - + - - `settings` - Map - A collection of settings for a compute endpoint. See [\_](#postgres_endpointsnamesettings). @@ -10275,7 +10838,7 @@ A collection of settings for a compute endpoint. - - `no_suspension` - Boolean - - When set to true, explicitly disables automatic suspension (never suspend). Should be set to true when provided. + - When set to true, explicitly disables automatic suspension (never suspend). Should be set to true when provided. Mutually exclusive with `suspend_timeout_duration`. When updating, use `spec.project_default_settings.suspension` in the update_mask. - - `pg_settings` - Map @@ -10283,7 +10846,7 @@ A collection of settings for a compute endpoint. - - `suspend_timeout_duration` - String - - Duration of inactivity after which the compute endpoint is automatically suspended. If specified should be between 60s and 604800s (1 minute to 1 week). + - Duration of inactivity after which the compute endpoint is automatically suspended. If specified should be between 60s and 604800s (1 minute to 1 week). Mutually exclusive with `no_suspension`. When updating, use `spec.project_default_settings.suspension` in the update_mask. ::: @@ -10342,6 +10905,93 @@ A collection of settings for a compute endpoint. ::: +## postgres_synced_tables + +**`Type: Map`** + +The Postgres synced table definitions for the bundle, where each key is the name of the synced table. Each entry continuously replicates a Unity Catalog Delta source table into a Postgres table on a Lakebase Autoscaling instance. + +```yaml +postgres_synced_tables: + : + : +``` + + +:::list-table + +- - Key + - Type + - Description + +- - `branch` + - String + - + +- - `create_database_objects_if_missing` + - Boolean + - + +- - `existing_pipeline_id` + - String + - + +- - `lifecycle` + - Map + - See [\_](#postgres_synced_tablesnamelifecycle). + +- - `new_pipeline_spec` + - Map + - See [\_](#postgres_synced_tablesnamenew_pipeline_spec). + +- - `postgres_database` + - String + - + +- - `primary_key_columns` + - Sequence + - + +- - `scheduling_policy` + - String + - + +- - `source_table_full_name` + - String + - + +- - `synced_table_id` + - String + - + +- - `timeseries_key` + - String + - + +::: + + +### postgres_synced_tables._name_.lifecycle + +**`Type: Map`** + + + + + +:::list-table + +- - Key + - Type + - Description + +- - `prevent_destroy` + - Boolean + - Lifecycle setting to prevent the resource from being destroyed. + +::: + + ## quality_monitors **`Type: Map`** diff --git a/bundle/internal/schema/annotations_openapi.yml b/bundle/internal/schema/annotations_openapi.yml index 23c5770dcc0..912d9158e40 100644 --- a/bundle/internal/schema/annotations_openapi.yml +++ b/bundle/internal/schema/annotations_openapi.yml @@ -115,6 +115,11 @@ github.com/databricks/cli/bundle/config/resources.App: "description": |- Git repository configuration for app deployments. When specified, deployments can reference code from this repository by providing only the git reference (branch, tag, or commit). + "git_source": + "description": |- + Complete git source specification including repository location and reference. + "x-databricks-preview": |- + PRIVATE "id": "description": |- The unique identifier of the app. @@ -148,6 +153,9 @@ github.com/databricks/cli/bundle/config/resources.App: "service_principal_name": "x-databricks-field-behaviors_output_only": |- true + "source_code_path": + "x-databricks-preview": |- + PRIVATE "space": "description": |- Name of the space this app belongs to. @@ -256,15 +264,14 @@ github.com/databricks/cli/bundle/config/resources.Cluster: Data security mode decides what data governance model to use when accessing data from a cluster. - The following modes can only be used when `kind = CLASSIC_PREVIEW`. * `DATA_SECURITY_MODE_AUTO`: Databricks will choose the most appropriate access mode depending on your compute configuration. - * `DATA_SECURITY_MODE_STANDARD`: Alias for `USER_ISOLATION`. - * `DATA_SECURITY_MODE_DEDICATED`: Alias for `SINGLE_USER`. + * `DATA_SECURITY_MODE_STANDARD`: A secure cluster that can be shared by multiple users. Cluster users are fully isolated so that they cannot see each other’s data and credentials. Most data governance features are supported in this mode. But programming languages and cluster features might be limited. + * `DATA_SECURITY_MODE_DEDICATED`: A secure cluster that can only be exclusively used by a single user specified in `single_user_name`. Most programming languages, cluster features and data governance features are available in this mode. + + The following modes are legacy aliases for the above modes: - The following modes can be used regardless of `kind`. - * `NONE`: No security isolation for multiple users sharing the cluster. Data governance features are not available in this mode. - * `SINGLE_USER`: A secure cluster that can only be exclusively used by a single user specified in `single_user_name`. Most programming languages, cluster features and data governance features are available in this mode. - * `USER_ISOLATION`: A secure cluster that can be shared by multiple users. Cluster users are fully isolated so that they cannot see each other's data and credentials. Most data governance features are supported in this mode. But programming languages and cluster features might be limited. + * `USER_ISOLATION`: Legacy alias for `DATA_SECURITY_MODE_STANDARD`. + * `SINGLE_USER`: Legacy alias for `DATA_SECURITY_MODE_DEDICATED`. The following modes are deprecated starting with Databricks Runtime 15.0 and will be removed for future Databricks Runtime versions: @@ -325,7 +332,6 @@ github.com/databricks/cli/bundle/config/resources.Cluster: Clusters with `kind = CLASSIC_PREVIEW` support the following fields, whereas clusters with no specified `kind` do not. * [is_single_node](/api/workspace/clusters/create#is_single_node) * [use_ml_runtime](/api/workspace/clusters/create#use_ml_runtime) - * [data_security_mode](/api/workspace/clusters/create#data_security_mode) set to `DATA_SECURITY_MODE_AUTO`, `DATA_SECURITY_MODE_DEDICATED`, or `DATA_SECURITY_MODE_STANDARD` By using the [simple form](https://docs.databricks.com/compute/simple-form.html), your clusters are automatically using `kind = CLASSIC_PREVIEW`. "node_type_id": @@ -1139,7 +1145,7 @@ github.com/databricks/cli/bundle/config/resources.VectorSearchEndpoint: Type of endpoint "name": "description": |- - Name of the vector search endpoint + Name of the AI Search endpoint "target_qps": "description": |- Target QPS for the endpoint. Mutually exclusive with num_replicas. @@ -1905,6 +1911,10 @@ github.com/databricks/databricks-sdk-go/service/catalog.Privilege: MANAGE_ACCESS_CONTROL - |- CREATE_SERVICE + - |- + CREATE_FEATURE + - |- + READ_FEATURE github.com/databricks/databricks-sdk-go/service/catalog.PrivilegeAssignment: "principal": "description": |- @@ -2192,15 +2202,14 @@ github.com/databricks/databricks-sdk-go/service/compute.ClusterSpec: Data security mode decides what data governance model to use when accessing data from a cluster. - The following modes can only be used when `kind = CLASSIC_PREVIEW`. * `DATA_SECURITY_MODE_AUTO`: Databricks will choose the most appropriate access mode depending on your compute configuration. - * `DATA_SECURITY_MODE_STANDARD`: Alias for `USER_ISOLATION`. - * `DATA_SECURITY_MODE_DEDICATED`: Alias for `SINGLE_USER`. + * `DATA_SECURITY_MODE_STANDARD`: A secure cluster that can be shared by multiple users. Cluster users are fully isolated so that they cannot see each other’s data and credentials. Most data governance features are supported in this mode. But programming languages and cluster features might be limited. + * `DATA_SECURITY_MODE_DEDICATED`: A secure cluster that can only be exclusively used by a single user specified in `single_user_name`. Most programming languages, cluster features and data governance features are available in this mode. + + The following modes are legacy aliases for the above modes: - The following modes can be used regardless of `kind`. - * `NONE`: No security isolation for multiple users sharing the cluster. Data governance features are not available in this mode. - * `SINGLE_USER`: A secure cluster that can only be exclusively used by a single user specified in `single_user_name`. Most programming languages, cluster features and data governance features are available in this mode. - * `USER_ISOLATION`: A secure cluster that can be shared by multiple users. Cluster users are fully isolated so that they cannot see each other's data and credentials. Most data governance features are supported in this mode. But programming languages and cluster features might be limited. + * `USER_ISOLATION`: Legacy alias for `DATA_SECURITY_MODE_STANDARD`. + * `SINGLE_USER`: Legacy alias for `DATA_SECURITY_MODE_DEDICATED`. The following modes are deprecated starting with Databricks Runtime 15.0 and will be removed for future Databricks Runtime versions: @@ -2261,7 +2270,6 @@ github.com/databricks/databricks-sdk-go/service/compute.ClusterSpec: Clusters with `kind = CLASSIC_PREVIEW` support the following fields, whereas clusters with no specified `kind` do not. * [is_single_node](/api/workspace/clusters/create#is_single_node) * [use_ml_runtime](/api/workspace/clusters/create#use_ml_runtime) - * [data_security_mode](/api/workspace/clusters/create#data_security_mode) set to `DATA_SECURITY_MODE_AUTO`, `DATA_SECURITY_MODE_DEDICATED`, or `DATA_SECURITY_MODE_STANDARD` By using the [simple form](https://docs.databricks.com/compute/simple-form.html), your clusters are automatically using `kind = CLASSIC_PREVIEW`. "node_type_id": @@ -2358,15 +2366,14 @@ github.com/databricks/databricks-sdk-go/service/compute.DataSecurityMode: Data security mode decides what data governance model to use when accessing data from a cluster. - The following modes can only be used when `kind = CLASSIC_PREVIEW`. * `DATA_SECURITY_MODE_AUTO`: Databricks will choose the most appropriate access mode depending on your compute configuration. - * `DATA_SECURITY_MODE_STANDARD`: Alias for `USER_ISOLATION`. - * `DATA_SECURITY_MODE_DEDICATED`: Alias for `SINGLE_USER`. + * `DATA_SECURITY_MODE_STANDARD`: A secure cluster that can be shared by multiple users. Cluster users are fully isolated so that they cannot see each other’s data and credentials. Most data governance features are supported in this mode. But programming languages and cluster features might be limited. + * `DATA_SECURITY_MODE_DEDICATED`: A secure cluster that can only be exclusively used by a single user specified in `single_user_name`. Most programming languages, cluster features and data governance features are available in this mode. - The following modes can be used regardless of `kind`. - * `NONE`: No security isolation for multiple users sharing the cluster. Data governance features are not available in this mode. - * `SINGLE_USER`: A secure cluster that can only be exclusively used by a single user specified in `single_user_name`. Most programming languages, cluster features and data governance features are available in this mode. - * `USER_ISOLATION`: A secure cluster that can be shared by multiple users. Cluster users are fully isolated so that they cannot see each other's data and credentials. Most data governance features are supported in this mode. But programming languages and cluster features might be limited. + The following modes are legacy aliases for the above modes: + + * `USER_ISOLATION`: Legacy alias for `DATA_SECURITY_MODE_STANDARD`. + * `SINGLE_USER`: Legacy alias for `DATA_SECURITY_MODE_DEDICATED`. The following modes are deprecated starting with Databricks Runtime 15.0 and will be removed for future Databricks Runtime versions: @@ -2439,7 +2446,10 @@ github.com/databricks/databricks-sdk-go/service/compute.Environment: (e.g., `/Workspace/path/to/env.yaml`). Support for a Databricks-provided base environment ID (e.g., `workspace-base-environments/databricks_ai_v4`) and workspace base environment ID (e.g., `workspace-base-environments/dbe_b849b66e-b31a-4cb5-b161-1f2b10877fb7`) is in Beta. - Either `environment_version` or `base_environment` can be provided. For more information, see + Either `environment_version` or `base_environment` can be provided. + For more information about Databricks-provided base environments, see the + [list workspace base environments](:method:Environments/ListWorkspaceBaseEnvironments) API. + For more information, see "client": "description": |- Use `environment_version` instead. @@ -2801,7 +2811,7 @@ github.com/databricks/databricks-sdk-go/service/database.DatabaseInstanceState: - |- FAILING_OVER - |- - MIGRATING + UPGRADING github.com/databricks/databricks-sdk-go/service/database.DeltaTableSyncInfo: "delta_commit_timestamp": "description": |- @@ -3755,13 +3765,39 @@ github.com/databricks/databricks-sdk-go/service/jobs.PipelineParams: "full_refresh": "description": |- If true, triggers a full refresh on the spark declarative pipeline. + "full_refresh_selection": + "description": |- + A list of tables to update with fullRefresh. + "refresh_flow_selection": + "description": |- + Flow names to selectively refresh. These are unioned with other selective refresh + options (refresh_selection, full_refresh_selection) to determine the final set of flows to refresh. + "refresh_selection": + "description": |- + A list of tables to update without fullRefresh. + "reset_checkpoint_selection": + "description": |- + A list of streaming flows to reset checkpoints without clearing data. github.com/databricks/databricks-sdk-go/service/jobs.PipelineTask: "full_refresh": "description": |- If true, triggers a full refresh on the spark declarative pipeline. + "full_refresh_selection": + "description": |- + A list of tables to update with fullRefresh. "pipeline_id": "description": |- The full name of the pipeline task to execute. + "refresh_flow_selection": + "description": |- + Flow names to selectively refresh. These are unioned with other selective refresh + options (refresh_selection, full_refresh_selection) to determine the final set of flows to refresh. + "refresh_selection": + "description": |- + A list of tables to update without fullRefresh. + "reset_checkpoint_selection": + "description": |- + A list of streaming flows to reset checkpoints without clearing data. github.com/databricks/databricks-sdk-go/service/jobs.PowerBiModel: "authentication_method": "description": |- @@ -3807,6 +3843,18 @@ github.com/databricks/databricks-sdk-go/service/jobs.PowerBiTask: "warehouse_id": "description": |- The SQL warehouse ID to use as the Power BI data source +github.com/databricks/databricks-sdk-go/service/jobs.PythonOperatorTask: + "main": + "description": |- + Fully qualified name of the main class or function. + For example, `my_project.my_function` or `my_project.MyOperator`. + "parameters": + "description": |- + An ordered list of task parameters. + TODO(JOBS-30885): Add limits for parameters. +github.com/databricks/databricks-sdk-go/service/jobs.PythonOperatorTaskParameter: + "name": {} + "value": {} github.com/databricks/databricks-sdk-go/service/jobs.PythonWheelTask: "entry_point": "description": |- @@ -4162,8 +4210,6 @@ github.com/databricks/databricks-sdk-go/service/jobs.Task: "disabled": "description": |- An optional flag to disable the task. If set to true, the task will not run even if it is part of a job. - "x-databricks-preview": |- - PRIVATE "email_notifications": "description": |- An optional set of email addresses that is notified when runs of this task begin or complete as well as when this task is deleted. The default behavior is to not send any emails. @@ -4213,6 +4259,11 @@ github.com/databricks/databricks-sdk-go/service/jobs.Task: "power_bi_task": "description": |- The task triggers a Power BI semantic model update when the `power_bi_task` field is present. + "python_operator_task": + "description": |- + The task runs a Python operator task. + "x-databricks-preview": |- + PRIVATE "python_wheel_task": "description": |- The task runs a Python wheel when the `python_wheel_task` field is present. @@ -4440,6 +4491,9 @@ github.com/databricks/databricks-sdk-go/service/pipelines.ConnectorOptions: "jira_options": "description": |- Jira specific options for ingestion + "kafka_options": + "x-databricks-preview": |- + PRIVATE "meta_ads_options": "description": |- Meta Marketing (Meta Ads) specific options for ingestion @@ -4599,6 +4653,8 @@ github.com/databricks/databricks-sdk-go/service/pipelines.FileIngestionOptionsFi AVRO - |- ORC + - |- + FILE github.com/databricks/databricks-sdk-go/service/pipelines.FileIngestionOptionsSchemaEvolutionMode: "_": "description": |- @@ -4668,6 +4724,10 @@ github.com/databricks/databricks-sdk-go/service/pipelines.GoogleDriveOptionsGoog FILE_METADATA - |- PERMISSION + - |- + FILE_PERMISSION + - |- + GROUP_MEMBERSHIP github.com/databricks/databricks-sdk-go/service/pipelines.IngestionConfig: "report": "description": |- @@ -5042,6 +5102,55 @@ github.com/databricks/databricks-sdk-go/service/pipelines.JiraConnectorOptions: "include_jira_spaces": "description": |- (Optional) Projects to filter Jira data on +github.com/databricks/databricks-sdk-go/service/pipelines.JsonTransformerOptions: + "as_variant": + "description": |- + Parse the entire value as a single Variant column. + "schema": + "description": |- + Inline schema string for JSON parsing (Spark DDL format). + "schema_evolution_mode": + "description": |- + (Optional) Schema evolution mode for schema inference. + "schema_file_path": + "description": |- + Path to a schema file (.ddl). + "schema_hints": + "description": |- + (Optional) Schema hints as a comma-separated string of "column_name type" pairs. +github.com/databricks/databricks-sdk-go/service/pipelines.KafkaOptions: + "client_config": + "description": |- + Undocumented backdoor mechanism for overriding parameters + to pass to the Kafka client. + This is not supported and may break at any time. + "x-databricks-preview": |- + PRIVATE + "key_transformer": + "description": |- + (Optional) Transformer for the message key. + If not specified, the key is left as raw bytes. + "max_offsets_per_trigger": + "description": |- + Internal option to control the maximum number of offsets to process per trigger. + "x-databricks-preview": |- + PRIVATE + "starting_offset": + "description": |- + (Optional) Where to begin reading when no checkpoint exists. + Valid values: "latest" and "earliest". Defaults to "latest". + "topic_pattern": + "description": |- + Java regex pattern to subscribe to matching topics. + Only one of topics or topic_pattern must be specified. + "topics": + "description": |- + Topics to subscribe to. + Only one of topics or topic_pattern must be specified. + "value_transformer": + "description": |- + (Optional) Transformer for the message value. + If not specified, the value is left as raw bytes. github.com/databricks/databricks-sdk-go/service/pipelines.ManualTrigger: {} github.com/databricks/databricks-sdk-go/service/pipelines.MetaMarketingOptions: "_": @@ -5488,6 +5597,10 @@ github.com/databricks/databricks-sdk-go/service/pipelines.SharepointOptionsShare PERMISSION - |- LIST + - |- + FILE_PERMISSION + - |- + GROUP_MEMBERSHIP github.com/databricks/databricks-sdk-go/service/pipelines.SmartsheetOptions: "_": "description": |- @@ -5674,6 +5787,25 @@ github.com/databricks/databricks-sdk-go/service/pipelines.TikTokAdsOptionsTikTok BUSINESS_CENTER - |- GMV_MAX +github.com/databricks/databricks-sdk-go/service/pipelines.Transformer: + "_": + "description": |- + Specifies how to transform binary data into structured data. + "format": + "description": |- + Required: the wire format of the data. + "json_options": {} +github.com/databricks/databricks-sdk-go/service/pipelines.TransformerFormat: + "_": + "enum": + - |- + STRING + - |- + JSON + - |- + AVRO + - |- + PROTOBUF github.com/databricks/databricks-sdk-go/service/pipelines.ZendeskSupportOptions: "_": "description": |- @@ -5712,6 +5844,18 @@ github.com/databricks/databricks-sdk-go/service/postgres.EndpointType: ENDPOINT_TYPE_READ_WRITE - |- ENDPOINT_TYPE_READ_ONLY +github.com/databricks/databricks-sdk-go/service/postgres.NewPipelineSpec: + "budget_policy_id": + "description": |- + Budget policy to set on the newly created pipeline. + "storage_catalog": + "description": |- + UC catalog for the pipeline to store intermediate files (checkpoints, event logs etc). + This needs to be a standard catalog where the user has permissions to create Delta tables. + "storage_schema": + "description": |- + UC schema for the pipeline to store intermediate files (checkpoints, event logs etc). + This needs to be in the standard catalog where the user has permissions to create Delta tables. github.com/databricks/databricks-sdk-go/service/postgres.ProjectCustomTag: "key": "description": |- @@ -5742,6 +5886,17 @@ github.com/databricks/databricks-sdk-go/service/postgres.ProjectDefaultEndpointS Duration of inactivity after which the compute endpoint is automatically suspended. If specified should be between 60s and 604800s (1 minute to 1 week). Mutually exclusive with `no_suspension`. When updating, use `spec.project_default_settings.suspension` in the update_mask. +github.com/databricks/databricks-sdk-go/service/postgres.SyncedTableSyncedTableSpecSyncedTableSchedulingPolicy: + "_": + "description": |- + Scheduling policy of the synced table's underlying pipeline. + "enum": + - |- + CONTINUOUS + - |- + TRIGGERED + - |- + SNAPSHOT github.com/databricks/databricks-sdk-go/service/serving.Ai21LabsConfig: "ai21labs_api_key": "description": |- @@ -6365,6 +6520,8 @@ github.com/databricks/databricks-sdk-go/service/serving.ServedModelInputWorkload GPU_LARGE - |- MULTIGPU_MEDIUM + - |- + GPU_XLARGE github.com/databricks/databricks-sdk-go/service/serving.ServingEndpointPermissionLevel: "_": "description": |- @@ -6391,6 +6548,8 @@ github.com/databricks/databricks-sdk-go/service/serving.ServingModelWorkloadType GPU_LARGE - |- MULTIGPU_MEDIUM + - |- + GPU_XLARGE github.com/databricks/databricks-sdk-go/service/serving.TrafficConfig: "routes": "description": |- diff --git a/bundle/internal/schema/annotations_openapi_overrides.yml b/bundle/internal/schema/annotations_openapi_overrides.yml index ff08304f533..d93e6068f46 100644 --- a/bundle/internal/schema/annotations_openapi_overrides.yml +++ b/bundle/internal/schema/annotations_openapi_overrides.yml @@ -935,6 +935,13 @@ github.com/databricks/databricks-sdk-go/service/jobs.JobsHealthRules: "rules": "description": |- PLACEHOLDER +github.com/databricks/databricks-sdk-go/service/jobs.PythonOperatorTaskParameter: + "name": + "description": |- + PLACEHOLDER + "value": + "description": |- + PLACEHOLDER github.com/databricks/databricks-sdk-go/service/jobs.RunJobTask: "python_named_params": "description": |- @@ -975,6 +982,9 @@ github.com/databricks/databricks-sdk-go/service/pipelines.ConnectorOptions: "gdrive_options": "description": |- PLACEHOLDER + "kafka_options": + "description": |- + PLACEHOLDER "sharepoint_options": "description": |- PLACEHOLDER @@ -1031,6 +1041,10 @@ github.com/databricks/databricks-sdk-go/service/pipelines.TableSpecificConfig: "workday_report_parameters": "description": |- PLACEHOLDER +github.com/databricks/databricks-sdk-go/service/pipelines.Transformer: + "json_options": + "description": |- + PLACEHOLDER github.com/databricks/databricks-sdk-go/service/serving.Route: "served_entity_name": "description": |- diff --git a/bundle/internal/validation/generated/enum_fields.go b/bundle/internal/validation/generated/enum_fields.go index bd481f9f2e3..850eb8eedfe 100644 --- a/bundle/internal/validation/generated/enum_fields.go +++ b/bundle/internal/validation/generated/enum_fields.go @@ -135,50 +135,58 @@ var EnumFields = map[string][]string{ "resources.model_serving_endpoints.*.ai_gateway.rate_limits[*].renewal_period": {"minute"}, "resources.model_serving_endpoints.*.config.served_entities[*].external_model.amazon_bedrock_config.bedrock_provider": {"ai21labs", "amazon", "anthropic", "cohere"}, "resources.model_serving_endpoints.*.config.served_entities[*].external_model.provider": {"ai21labs", "amazon-bedrock", "anthropic", "cohere", "custom", "databricks-model-serving", "google-cloud-vertex-ai", "openai", "palm"}, - "resources.model_serving_endpoints.*.config.served_entities[*].workload_type": {"CPU", "GPU_LARGE", "GPU_MEDIUM", "GPU_SMALL", "MULTIGPU_MEDIUM"}, - "resources.model_serving_endpoints.*.config.served_models[*].workload_type": {"CPU", "GPU_LARGE", "GPU_MEDIUM", "GPU_SMALL", "MULTIGPU_MEDIUM"}, + "resources.model_serving_endpoints.*.config.served_entities[*].workload_type": {"CPU", "GPU_LARGE", "GPU_MEDIUM", "GPU_SMALL", "GPU_XLARGE", "MULTIGPU_MEDIUM"}, + "resources.model_serving_endpoints.*.config.served_models[*].workload_type": {"CPU", "GPU_LARGE", "GPU_MEDIUM", "GPU_SMALL", "GPU_XLARGE", "MULTIGPU_MEDIUM"}, "resources.model_serving_endpoints.*.permissions[*].level": {"CAN_MANAGE", "CAN_QUERY", "CAN_VIEW"}, "resources.model_serving_endpoints.*.rate_limits[*].key": {"endpoint", "user"}, "resources.model_serving_endpoints.*.rate_limits[*].renewal_period": {"minute"}, "resources.models.*.permissions[*].level": {"CAN_EDIT", "CAN_MANAGE", "CAN_MANAGE_PRODUCTION_VERSIONS", "CAN_MANAGE_STAGING_VERSIONS", "CAN_READ"}, - "resources.pipelines.*.clusters[*].autoscale.mode": {"ENHANCED", "LEGACY"}, - "resources.pipelines.*.clusters[*].aws_attributes.availability": {"ON_DEMAND", "SPOT", "SPOT_WITH_FALLBACK"}, - "resources.pipelines.*.clusters[*].aws_attributes.ebs_volume_type": {"GENERAL_PURPOSE_SSD", "THROUGHPUT_OPTIMIZED_HDD"}, - "resources.pipelines.*.clusters[*].azure_attributes.availability": {"ON_DEMAND_AZURE", "SPOT_AZURE", "SPOT_WITH_FALLBACK_AZURE"}, - "resources.pipelines.*.clusters[*].gcp_attributes.availability": {"ON_DEMAND_GCP", "PREEMPTIBLE_GCP", "PREEMPTIBLE_WITH_FALLBACK_GCP"}, - "resources.pipelines.*.clusters[*].gcp_attributes.confidential_compute_type": {"CONFIDENTIAL_COMPUTE_TYPE_NONE", "SEV_SNP"}, - "resources.pipelines.*.deployment.kind": {"BUNDLE"}, - "resources.pipelines.*.ingestion_definition.connector_type": {"CDC", "QUERY_BASED"}, - "resources.pipelines.*.ingestion_definition.full_refresh_window.days_of_week[*]": {"FRIDAY", "MONDAY", "SATURDAY", "SUNDAY", "THURSDAY", "TUESDAY", "WEDNESDAY"}, - "resources.pipelines.*.ingestion_definition.objects[*].report.table_configuration.scd_type": {"APPEND_ONLY", "SCD_TYPE_1", "SCD_TYPE_2"}, - "resources.pipelines.*.ingestion_definition.objects[*].schema.connector_options.gdrive_options.entity_type": {"FILE", "FILE_METADATA", "PERMISSION"}, - "resources.pipelines.*.ingestion_definition.objects[*].schema.connector_options.gdrive_options.file_ingestion_options.format": {"AVRO", "BINARYFILE", "CSV", "EXCEL", "JSON", "ORC", "PARQUET", "XML"}, - "resources.pipelines.*.ingestion_definition.objects[*].schema.connector_options.gdrive_options.file_ingestion_options.schema_evolution_mode": {"ADD_NEW_COLUMNS", "ADD_NEW_COLUMNS_WITH_TYPE_WIDENING", "FAIL_ON_NEW_COLUMNS", "NONE", "RESCUE"}, - "resources.pipelines.*.ingestion_definition.objects[*].schema.connector_options.outlook_options.attachment_mode": {"ALL", "INLINE_ONLY", "NONE", "NON_INLINE_ONLY"}, - "resources.pipelines.*.ingestion_definition.objects[*].schema.connector_options.outlook_options.body_format": {"TEXT_HTML", "TEXT_PLAIN"}, - "resources.pipelines.*.ingestion_definition.objects[*].schema.connector_options.sharepoint_options.entity_type": {"FILE", "FILE_METADATA", "LIST", "PERMISSION"}, - "resources.pipelines.*.ingestion_definition.objects[*].schema.connector_options.sharepoint_options.file_ingestion_options.format": {"AVRO", "BINARYFILE", "CSV", "EXCEL", "JSON", "ORC", "PARQUET", "XML"}, - "resources.pipelines.*.ingestion_definition.objects[*].schema.connector_options.sharepoint_options.file_ingestion_options.schema_evolution_mode": {"ADD_NEW_COLUMNS", "ADD_NEW_COLUMNS_WITH_TYPE_WIDENING", "FAIL_ON_NEW_COLUMNS", "NONE", "RESCUE"}, - "resources.pipelines.*.ingestion_definition.objects[*].schema.connector_options.tiktok_ads_options.data_level": {"AUCTION_AD", "AUCTION_ADGROUP", "AUCTION_ADVERTISER", "AUCTION_CAMPAIGN"}, - "resources.pipelines.*.ingestion_definition.objects[*].schema.connector_options.tiktok_ads_options.report_type": {"AUDIENCE", "BASIC", "BUSINESS_CENTER", "DSA", "GMV_MAX", "PLAYABLE_AD"}, - "resources.pipelines.*.ingestion_definition.objects[*].schema.table_configuration.scd_type": {"APPEND_ONLY", "SCD_TYPE_1", "SCD_TYPE_2"}, - "resources.pipelines.*.ingestion_definition.objects[*].table.connector_options.gdrive_options.entity_type": {"FILE", "FILE_METADATA", "PERMISSION"}, - "resources.pipelines.*.ingestion_definition.objects[*].table.connector_options.gdrive_options.file_ingestion_options.format": {"AVRO", "BINARYFILE", "CSV", "EXCEL", "JSON", "ORC", "PARQUET", "XML"}, - "resources.pipelines.*.ingestion_definition.objects[*].table.connector_options.gdrive_options.file_ingestion_options.schema_evolution_mode": {"ADD_NEW_COLUMNS", "ADD_NEW_COLUMNS_WITH_TYPE_WIDENING", "FAIL_ON_NEW_COLUMNS", "NONE", "RESCUE"}, - "resources.pipelines.*.ingestion_definition.objects[*].table.connector_options.outlook_options.attachment_mode": {"ALL", "INLINE_ONLY", "NONE", "NON_INLINE_ONLY"}, - "resources.pipelines.*.ingestion_definition.objects[*].table.connector_options.outlook_options.body_format": {"TEXT_HTML", "TEXT_PLAIN"}, - "resources.pipelines.*.ingestion_definition.objects[*].table.connector_options.sharepoint_options.entity_type": {"FILE", "FILE_METADATA", "LIST", "PERMISSION"}, - "resources.pipelines.*.ingestion_definition.objects[*].table.connector_options.sharepoint_options.file_ingestion_options.format": {"AVRO", "BINARYFILE", "CSV", "EXCEL", "JSON", "ORC", "PARQUET", "XML"}, - "resources.pipelines.*.ingestion_definition.objects[*].table.connector_options.sharepoint_options.file_ingestion_options.schema_evolution_mode": {"ADD_NEW_COLUMNS", "ADD_NEW_COLUMNS_WITH_TYPE_WIDENING", "FAIL_ON_NEW_COLUMNS", "NONE", "RESCUE"}, - "resources.pipelines.*.ingestion_definition.objects[*].table.connector_options.tiktok_ads_options.data_level": {"AUCTION_AD", "AUCTION_ADGROUP", "AUCTION_ADVERTISER", "AUCTION_CAMPAIGN"}, - "resources.pipelines.*.ingestion_definition.objects[*].table.connector_options.tiktok_ads_options.report_type": {"AUDIENCE", "BASIC", "BUSINESS_CENTER", "DSA", "GMV_MAX", "PLAYABLE_AD"}, - "resources.pipelines.*.ingestion_definition.objects[*].table.table_configuration.scd_type": {"APPEND_ONLY", "SCD_TYPE_1", "SCD_TYPE_2"}, - "resources.pipelines.*.ingestion_definition.source_type": {"BIGQUERY", "CONFLUENCE", "DYNAMICS365", "FOREIGN_CATALOG", "GA4_RAW_DATA", "GOOGLE_DRIVE", "JIRA", "MANAGED_POSTGRESQL", "META_MARKETING", "MYSQL", "NETSUITE", "ORACLE", "POSTGRESQL", "SALESFORCE", "SERVICENOW", "SHAREPOINT", "SQLSERVER", "TERADATA", "WORKDAY_RAAS", "ZENDESK"}, - "resources.pipelines.*.ingestion_definition.table_configuration.scd_type": {"APPEND_ONLY", "SCD_TYPE_1", "SCD_TYPE_2"}, - "resources.pipelines.*.permissions[*].level": {"CAN_MANAGE", "CAN_RUN", "CAN_VIEW", "IS_OWNER"}, - "resources.pipelines.*.restart_window.days_of_week[*]": {"FRIDAY", "MONDAY", "SATURDAY", "SUNDAY", "THURSDAY", "TUESDAY", "WEDNESDAY"}, + "resources.pipelines.*.clusters[*].autoscale.mode": {"ENHANCED", "LEGACY"}, + "resources.pipelines.*.clusters[*].aws_attributes.availability": {"ON_DEMAND", "SPOT", "SPOT_WITH_FALLBACK"}, + "resources.pipelines.*.clusters[*].aws_attributes.ebs_volume_type": {"GENERAL_PURPOSE_SSD", "THROUGHPUT_OPTIMIZED_HDD"}, + "resources.pipelines.*.clusters[*].azure_attributes.availability": {"ON_DEMAND_AZURE", "SPOT_AZURE", "SPOT_WITH_FALLBACK_AZURE"}, + "resources.pipelines.*.clusters[*].gcp_attributes.availability": {"ON_DEMAND_GCP", "PREEMPTIBLE_GCP", "PREEMPTIBLE_WITH_FALLBACK_GCP"}, + "resources.pipelines.*.clusters[*].gcp_attributes.confidential_compute_type": {"CONFIDENTIAL_COMPUTE_TYPE_NONE", "SEV_SNP"}, + "resources.pipelines.*.deployment.kind": {"BUNDLE"}, + "resources.pipelines.*.ingestion_definition.connector_type": {"CDC", "QUERY_BASED"}, + "resources.pipelines.*.ingestion_definition.full_refresh_window.days_of_week[*]": {"FRIDAY", "MONDAY", "SATURDAY", "SUNDAY", "THURSDAY", "TUESDAY", "WEDNESDAY"}, + "resources.pipelines.*.ingestion_definition.objects[*].report.table_configuration.scd_type": {"APPEND_ONLY", "SCD_TYPE_1", "SCD_TYPE_2"}, + "resources.pipelines.*.ingestion_definition.objects[*].schema.connector_options.gdrive_options.entity_type": {"FILE", "FILE_METADATA", "PERMISSION"}, + "resources.pipelines.*.ingestion_definition.objects[*].schema.connector_options.gdrive_options.file_ingestion_options.format": {"AVRO", "BINARYFILE", "CSV", "EXCEL", "JSON", "ORC", "PARQUET", "XML"}, + "resources.pipelines.*.ingestion_definition.objects[*].schema.connector_options.gdrive_options.file_ingestion_options.schema_evolution_mode": {"ADD_NEW_COLUMNS", "ADD_NEW_COLUMNS_WITH_TYPE_WIDENING", "FAIL_ON_NEW_COLUMNS", "NONE", "RESCUE"}, + "resources.pipelines.*.ingestion_definition.objects[*].schema.connector_options.kafka_options.key_transformer.format": {"JSON", "STRING"}, + "resources.pipelines.*.ingestion_definition.objects[*].schema.connector_options.kafka_options.key_transformer.json_options.schema_evolution_mode": {"ADD_NEW_COLUMNS", "ADD_NEW_COLUMNS_WITH_TYPE_WIDENING", "FAIL_ON_NEW_COLUMNS", "NONE", "RESCUE"}, + "resources.pipelines.*.ingestion_definition.objects[*].schema.connector_options.kafka_options.value_transformer.format": {"JSON", "STRING"}, + "resources.pipelines.*.ingestion_definition.objects[*].schema.connector_options.kafka_options.value_transformer.json_options.schema_evolution_mode": {"ADD_NEW_COLUMNS", "ADD_NEW_COLUMNS_WITH_TYPE_WIDENING", "FAIL_ON_NEW_COLUMNS", "NONE", "RESCUE"}, + "resources.pipelines.*.ingestion_definition.objects[*].schema.connector_options.outlook_options.attachment_mode": {"ALL", "INLINE_ONLY", "NONE", "NON_INLINE_ONLY"}, + "resources.pipelines.*.ingestion_definition.objects[*].schema.connector_options.outlook_options.body_format": {"TEXT_HTML", "TEXT_PLAIN"}, + "resources.pipelines.*.ingestion_definition.objects[*].schema.connector_options.sharepoint_options.entity_type": {"FILE", "FILE_METADATA", "LIST", "PERMISSION"}, + "resources.pipelines.*.ingestion_definition.objects[*].schema.connector_options.sharepoint_options.file_ingestion_options.format": {"AVRO", "BINARYFILE", "CSV", "EXCEL", "JSON", "ORC", "PARQUET", "XML"}, + "resources.pipelines.*.ingestion_definition.objects[*].schema.connector_options.sharepoint_options.file_ingestion_options.schema_evolution_mode": {"ADD_NEW_COLUMNS", "ADD_NEW_COLUMNS_WITH_TYPE_WIDENING", "FAIL_ON_NEW_COLUMNS", "NONE", "RESCUE"}, + "resources.pipelines.*.ingestion_definition.objects[*].schema.connector_options.tiktok_ads_options.data_level": {"AUCTION_AD", "AUCTION_ADGROUP", "AUCTION_ADVERTISER", "AUCTION_CAMPAIGN"}, + "resources.pipelines.*.ingestion_definition.objects[*].schema.connector_options.tiktok_ads_options.report_type": {"AUDIENCE", "BASIC", "BUSINESS_CENTER", "DSA", "GMV_MAX", "PLAYABLE_AD"}, + "resources.pipelines.*.ingestion_definition.objects[*].schema.table_configuration.scd_type": {"APPEND_ONLY", "SCD_TYPE_1", "SCD_TYPE_2"}, + "resources.pipelines.*.ingestion_definition.objects[*].table.connector_options.gdrive_options.entity_type": {"FILE", "FILE_METADATA", "PERMISSION"}, + "resources.pipelines.*.ingestion_definition.objects[*].table.connector_options.gdrive_options.file_ingestion_options.format": {"AVRO", "BINARYFILE", "CSV", "EXCEL", "JSON", "ORC", "PARQUET", "XML"}, + "resources.pipelines.*.ingestion_definition.objects[*].table.connector_options.gdrive_options.file_ingestion_options.schema_evolution_mode": {"ADD_NEW_COLUMNS", "ADD_NEW_COLUMNS_WITH_TYPE_WIDENING", "FAIL_ON_NEW_COLUMNS", "NONE", "RESCUE"}, + "resources.pipelines.*.ingestion_definition.objects[*].table.connector_options.kafka_options.key_transformer.format": {"JSON", "STRING"}, + "resources.pipelines.*.ingestion_definition.objects[*].table.connector_options.kafka_options.key_transformer.json_options.schema_evolution_mode": {"ADD_NEW_COLUMNS", "ADD_NEW_COLUMNS_WITH_TYPE_WIDENING", "FAIL_ON_NEW_COLUMNS", "NONE", "RESCUE"}, + "resources.pipelines.*.ingestion_definition.objects[*].table.connector_options.kafka_options.value_transformer.format": {"JSON", "STRING"}, + "resources.pipelines.*.ingestion_definition.objects[*].table.connector_options.kafka_options.value_transformer.json_options.schema_evolution_mode": {"ADD_NEW_COLUMNS", "ADD_NEW_COLUMNS_WITH_TYPE_WIDENING", "FAIL_ON_NEW_COLUMNS", "NONE", "RESCUE"}, + "resources.pipelines.*.ingestion_definition.objects[*].table.connector_options.outlook_options.attachment_mode": {"ALL", "INLINE_ONLY", "NONE", "NON_INLINE_ONLY"}, + "resources.pipelines.*.ingestion_definition.objects[*].table.connector_options.outlook_options.body_format": {"TEXT_HTML", "TEXT_PLAIN"}, + "resources.pipelines.*.ingestion_definition.objects[*].table.connector_options.sharepoint_options.entity_type": {"FILE", "FILE_METADATA", "LIST", "PERMISSION"}, + "resources.pipelines.*.ingestion_definition.objects[*].table.connector_options.sharepoint_options.file_ingestion_options.format": {"AVRO", "BINARYFILE", "CSV", "EXCEL", "JSON", "ORC", "PARQUET", "XML"}, + "resources.pipelines.*.ingestion_definition.objects[*].table.connector_options.sharepoint_options.file_ingestion_options.schema_evolution_mode": {"ADD_NEW_COLUMNS", "ADD_NEW_COLUMNS_WITH_TYPE_WIDENING", "FAIL_ON_NEW_COLUMNS", "NONE", "RESCUE"}, + "resources.pipelines.*.ingestion_definition.objects[*].table.connector_options.tiktok_ads_options.data_level": {"AUCTION_AD", "AUCTION_ADGROUP", "AUCTION_ADVERTISER", "AUCTION_CAMPAIGN"}, + "resources.pipelines.*.ingestion_definition.objects[*].table.connector_options.tiktok_ads_options.report_type": {"AUDIENCE", "BASIC", "BUSINESS_CENTER", "DSA", "GMV_MAX", "PLAYABLE_AD"}, + "resources.pipelines.*.ingestion_definition.objects[*].table.table_configuration.scd_type": {"APPEND_ONLY", "SCD_TYPE_1", "SCD_TYPE_2"}, + "resources.pipelines.*.ingestion_definition.source_type": {"BIGQUERY", "CONFLUENCE", "DYNAMICS365", "FOREIGN_CATALOG", "GA4_RAW_DATA", "GOOGLE_DRIVE", "JIRA", "MANAGED_POSTGRESQL", "META_MARKETING", "MYSQL", "NETSUITE", "ORACLE", "POSTGRESQL", "SALESFORCE", "SERVICENOW", "SHAREPOINT", "SQLSERVER", "TERADATA", "WORKDAY_RAAS", "ZENDESK"}, + "resources.pipelines.*.ingestion_definition.table_configuration.scd_type": {"APPEND_ONLY", "SCD_TYPE_1", "SCD_TYPE_2"}, + "resources.pipelines.*.permissions[*].level": {"CAN_MANAGE", "CAN_RUN", "CAN_VIEW", "IS_OWNER"}, + "resources.pipelines.*.restart_window.days_of_week[*]": {"FRIDAY", "MONDAY", "SATURDAY", "SUNDAY", "THURSDAY", "TUESDAY", "WEDNESDAY"}, "resources.postgres_endpoints.*.endpoint_type": {"ENDPOINT_TYPE_READ_ONLY", "ENDPOINT_TYPE_READ_WRITE"}, diff --git a/bundle/schema/jsonschema.json b/bundle/schema/jsonschema.json index 414e70fedd2..2a7f76aac85 100644 --- a/bundle/schema/jsonschema.json +++ b/bundle/schema/jsonschema.json @@ -160,7 +160,9 @@ }, "git_source": { "description": "Git source configuration for app deployments. Specifies which git reference (branch, tag, or commit)\nto use when deploying the app. Used in conjunction with git_repository to deploy code directly from git.\nThe source_code_path within git_source specifies the relative path to the app code within the repository.", - "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/apps.GitSource" + "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/apps.GitSource", + "x-databricks-preview": "PRIVATE", + "doNotSuggest": true }, "lifecycle": { "description": "Lifecycle is a struct that contains the lifecycle settings for a resource. It controls the behavior of the resource when it is deployed or destroyed.", @@ -178,7 +180,9 @@ "$ref": "#/$defs/slice/github.com/databricks/databricks-sdk-go/service/apps.AppResource" }, "source_code_path": { - "$ref": "#/$defs/string" + "$ref": "#/$defs/string", + "x-databricks-preview": "PRIVATE", + "doNotSuggest": true }, "space": { "description": "Name of the space this app belongs to.", @@ -4212,7 +4216,9 @@ "READ_METADATA", "MANAGE_ACCESS", "MANAGE_ACCESS_CONTROL", - "CREATE_SERVICE" + "CREATE_SERVICE", + "CREATE_FEATURE", + "READ_FEATURE" ] }, { @@ -4724,7 +4730,7 @@ "oneOf": [ { "type": "string", - "description": "Data security mode decides what data governance model to use when accessing data\nfrom a cluster.\n\nThe following modes can only be used when `kind = CLASSIC_PREVIEW`.\n* `DATA_SECURITY_MODE_AUTO`: Databricks will choose the most appropriate access mode depending on your compute configuration.\n* `DATA_SECURITY_MODE_STANDARD`: Alias for `USER_ISOLATION`.\n* `DATA_SECURITY_MODE_DEDICATED`: Alias for `SINGLE_USER`.\n\nThe following modes can be used regardless of `kind`.\n* `NONE`: No security isolation for multiple users sharing the cluster. Data governance features are not available in this mode.\n* `SINGLE_USER`: A secure cluster that can only be exclusively used by a single user specified in `single_user_name`. Most programming languages, cluster features and data governance features are available in this mode.\n* `USER_ISOLATION`: A secure cluster that can be shared by multiple users. Cluster users are fully isolated so that they cannot see each other's data and credentials. Most data governance features are supported in this mode. But programming languages and cluster features might be limited.\n\nThe following modes are deprecated starting with Databricks Runtime 15.0 and\nwill be removed for future Databricks Runtime versions:\n\n* `LEGACY_TABLE_ACL`: This mode is for users migrating from legacy Table ACL clusters.\n* `LEGACY_PASSTHROUGH`: This mode is for users migrating from legacy Passthrough on high concurrency clusters.\n* `LEGACY_SINGLE_USER`: This mode is for users migrating from legacy Passthrough on standard clusters.\n* `LEGACY_SINGLE_USER_STANDARD`: This mode provides a way that doesn’t have UC nor passthrough enabled.", + "description": "Data security mode decides what data governance model to use when accessing data\nfrom a cluster.\n\n* `DATA_SECURITY_MODE_AUTO`: Databricks will choose the most appropriate access mode depending on your compute configuration.\n* `DATA_SECURITY_MODE_STANDARD`: A secure cluster that can be shared by multiple users. Cluster users are fully isolated so that they cannot see each other’s data and credentials. Most data governance features are supported in this mode. But programming languages and cluster features might be limited.\n* `DATA_SECURITY_MODE_DEDICATED`: A secure cluster that can only be exclusively used by a single user specified in `single_user_name`. Most programming languages, cluster features and data governance features are available in this mode.\n\nThe following modes are legacy aliases for the above modes:\n\n* `USER_ISOLATION`: Legacy alias for `DATA_SECURITY_MODE_STANDARD`.\n* `SINGLE_USER`: Legacy alias for `DATA_SECURITY_MODE_DEDICATED`.\n\nThe following modes are deprecated starting with Databricks Runtime 15.0 and\nwill be removed for future Databricks Runtime versions:\n\n* `LEGACY_TABLE_ACL`: This mode is for users migrating from legacy Table ACL clusters.\n* `LEGACY_PASSTHROUGH`: This mode is for users migrating from legacy Passthrough on high concurrency clusters.\n* `LEGACY_SINGLE_USER`: This mode is for users migrating from legacy Passthrough on standard clusters.\n* `LEGACY_SINGLE_USER_STANDARD`: This mode provides a way that doesn’t have UC nor passthrough enabled.", "enum": [ "NONE", "SINGLE_USER", @@ -4832,7 +4838,7 @@ "description": "The environment entity used to preserve serverless environment side panel, jobs' environment for non-notebook task, and SDP's environment for classic and serverless pipelines.\nIn this minimal environment spec, only pip and java dependencies are supported.", "properties": { "base_environment": { - "description": "The base environment this environment is built on top of. A base environment defines the environment version and a\nlist of dependencies for serverless compute. The value can be a file path to a custom `env.yaml` file\n(e.g., `/Workspace/path/to/env.yaml`). Support for a Databricks-provided base environment ID\n(e.g., `workspace-base-environments/databricks_ai_v4`) and workspace base environment ID\n(e.g., `workspace-base-environments/dbe_b849b66e-b31a-4cb5-b161-1f2b10877fb7`) is in Beta.\nEither `environment_version` or `base_environment` can be provided. For more information, see", + "description": "The base environment this environment is built on top of. A base environment defines the environment version and a\nlist of dependencies for serverless compute. The value can be a file path to a custom `env.yaml` file\n(e.g., `/Workspace/path/to/env.yaml`). Support for a Databricks-provided base environment ID\n(e.g., `workspace-base-environments/databricks_ai_v4`) and workspace base environment ID\n(e.g., `workspace-base-environments/dbe_b849b66e-b31a-4cb5-b161-1f2b10877fb7`) is in Beta.\nEither `environment_version` or `base_environment` can be provided.\nFor more information about Databricks-provided base environments, see the\n[list workspace base environments](:method:Environments/ListWorkspaceBaseEnvironments) API.\nFor more information, see", "$ref": "#/$defs/string" }, "client": { @@ -5414,7 +5420,7 @@ "STOPPED", "UPDATING", "FAILING_OVER", - "MIGRATING" + "UPGRADING" ] }, { @@ -6853,6 +6859,22 @@ "full_refresh": { "description": "If true, triggers a full refresh on the spark declarative pipeline.", "$ref": "#/$defs/bool" + }, + "full_refresh_selection": { + "description": "A list of tables to update with fullRefresh.", + "$ref": "#/$defs/slice/string" + }, + "refresh_flow_selection": { + "description": "Flow names to selectively refresh. These are unioned with other selective refresh\noptions (refresh_selection, full_refresh_selection) to determine the final set of flows to refresh.", + "$ref": "#/$defs/slice/string" + }, + "refresh_selection": { + "description": "A list of tables to update without fullRefresh.", + "$ref": "#/$defs/slice/string" + }, + "reset_checkpoint_selection": { + "description": "A list of streaming flows to reset checkpoints without clearing data.", + "$ref": "#/$defs/slice/string" } }, "additionalProperties": false @@ -6872,9 +6894,25 @@ "description": "If true, triggers a full refresh on the spark declarative pipeline.", "$ref": "#/$defs/bool" }, + "full_refresh_selection": { + "description": "A list of tables to update with fullRefresh.", + "$ref": "#/$defs/slice/string" + }, "pipeline_id": { "description": "The full name of the pipeline task to execute.", "$ref": "#/$defs/string" + }, + "refresh_flow_selection": { + "description": "Flow names to selectively refresh. These are unioned with other selective refresh\noptions (refresh_selection, full_refresh_selection) to determine the final set of flows to refresh.", + "$ref": "#/$defs/slice/string" + }, + "refresh_selection": { + "description": "A list of tables to update without fullRefresh.", + "$ref": "#/$defs/slice/string" + }, + "reset_checkpoint_selection": { + "description": "A list of streaming flows to reset checkpoints without clearing data.", + "$ref": "#/$defs/slice/string" } }, "additionalProperties": false, @@ -6986,6 +7024,48 @@ } ] }, + "jobs.PythonOperatorTask": { + "oneOf": [ + { + "type": "object", + "properties": { + "main": { + "description": "Fully qualified name of the main class or function.\nFor example, `my_project.my_function` or `my_project.MyOperator`.", + "$ref": "#/$defs/string" + }, + "parameters": { + "description": "An ordered list of task parameters.\nTODO(JOBS-30885): Add limits for parameters.", + "$ref": "#/$defs/slice/github.com/databricks/databricks-sdk-go/service/jobs.PythonOperatorTaskParameter" + } + }, + "additionalProperties": false + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)+)\\}" + } + ] + }, + "jobs.PythonOperatorTaskParameter": { + "oneOf": [ + { + "type": "object", + "properties": { + "name": { + "$ref": "#/$defs/string" + }, + "value": { + "$ref": "#/$defs/string" + } + }, + "additionalProperties": false + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)+)\\}" + } + ] + }, "jobs.PythonWheelTask": { "oneOf": [ { @@ -7581,9 +7661,7 @@ }, "disabled": { "description": "An optional flag to disable the task. If set to true, the task will not run even if it is part of a job.", - "$ref": "#/$defs/bool", - "x-databricks-preview": "PRIVATE", - "doNotSuggest": true + "$ref": "#/$defs/bool" }, "email_notifications": { "description": "An optional set of email addresses that is notified when runs of this task begin or complete as well as when this task is deleted. The default behavior is to not send any emails.", @@ -7645,6 +7723,12 @@ "description": "The task triggers a Power BI semantic model update when the `power_bi_task` field is present.", "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/jobs.PowerBiTask" }, + "python_operator_task": { + "description": "The task runs a Python operator task.", + "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/jobs.PythonOperatorTask", + "x-databricks-preview": "PRIVATE", + "doNotSuggest": true + }, "python_wheel_task": { "description": "The task runs a Python wheel when the `python_wheel_task` field is present.", "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/jobs.PythonWheelTask" @@ -8070,6 +8154,11 @@ "description": "Jira specific options for ingestion", "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/pipelines.JiraConnectorOptions" }, + "kafka_options": { + "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/pipelines.KafkaOptions", + "x-databricks-preview": "PRIVATE", + "doNotSuggest": true + }, "meta_ads_options": { "description": "Meta Marketing (Meta Ads) specific options for ingestion", "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/pipelines.MetaMarketingOptions" @@ -8333,7 +8422,8 @@ "EXCEL", "PARQUET", "AVRO", - "ORC" + "ORC", + "FILE" ] }, { @@ -8480,7 +8570,9 @@ "enum": [ "FILE", "FILE_METADATA", - "PERMISSION" + "PERMISSION", + "FILE_PERMISSION", + "GROUP_MEMBERSHIP" ] }, { @@ -8833,6 +8925,86 @@ } ] }, + "pipelines.JsonTransformerOptions": { + "oneOf": [ + { + "type": "object", + "properties": { + "as_variant": { + "description": "Parse the entire value as a single Variant column.", + "$ref": "#/$defs/bool" + }, + "schema": { + "description": "Inline schema string for JSON parsing (Spark DDL format).", + "$ref": "#/$defs/string" + }, + "schema_evolution_mode": { + "description": "(Optional) Schema evolution mode for schema inference.", + "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/pipelines.FileIngestionOptionsSchemaEvolutionMode" + }, + "schema_file_path": { + "description": "Path to a schema file (.ddl).", + "$ref": "#/$defs/string" + }, + "schema_hints": { + "description": "(Optional) Schema hints as a comma-separated string of \"column_name type\" pairs.", + "$ref": "#/$defs/string" + } + }, + "additionalProperties": false + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)+)\\}" + } + ] + }, + "pipelines.KafkaOptions": { + "oneOf": [ + { + "type": "object", + "properties": { + "client_config": { + "description": "Undocumented backdoor mechanism for overriding parameters\nto pass to the Kafka client.\nThis is not supported and may break at any time.", + "$ref": "#/$defs/map/string", + "x-databricks-preview": "PRIVATE", + "doNotSuggest": true + }, + "key_transformer": { + "description": "(Optional) Transformer for the message key.\nIf not specified, the key is left as raw bytes.", + "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/pipelines.Transformer" + }, + "max_offsets_per_trigger": { + "description": "Internal option to control the maximum number of offsets to process per trigger.", + "$ref": "#/$defs/int64", + "x-databricks-preview": "PRIVATE", + "doNotSuggest": true + }, + "starting_offset": { + "description": "(Optional) Where to begin reading when no checkpoint exists.\nValid values: \"latest\" and \"earliest\". Defaults to \"latest\".", + "$ref": "#/$defs/string" + }, + "topic_pattern": { + "description": "Java regex pattern to subscribe to matching topics.\nOnly one of topics or topic_pattern must be specified.", + "$ref": "#/$defs/string" + }, + "topics": { + "description": "Topics to subscribe to.\nOnly one of topics or topic_pattern must be specified.", + "$ref": "#/$defs/slice/string" + }, + "value_transformer": { + "description": "(Optional) Transformer for the message value.\nIf not specified, the value is left as raw bytes.", + "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/pipelines.Transformer" + } + }, + "additionalProperties": false + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)+)\\}" + } + ] + }, "pipelines.ManualTrigger": { "oneOf": [ { @@ -9553,7 +9725,9 @@ "FILE", "FILE_METADATA", "PERMISSION", - "LIST" + "LIST", + "FILE_PERMISSION", + "GROUP_MEMBERSHIP" ] }, { @@ -9833,6 +10007,45 @@ } ] }, + "pipelines.Transformer": { + "oneOf": [ + { + "type": "object", + "description": "Specifies how to transform binary data into structured data.", + "properties": { + "format": { + "description": "Required: the wire format of the data.", + "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/pipelines.TransformerFormat" + }, + "json_options": { + "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/pipelines.JsonTransformerOptions" + } + }, + "additionalProperties": false + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)+)\\}" + } + ] + }, + "pipelines.TransformerFormat": { + "oneOf": [ + { + "type": "string", + "enum": [ + "STRING", + "JSON", + "AVRO", + "PROTOBUF" + ] + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)+)\\}" + } + ] + }, "pipelines.ZendeskSupportOptions": { "oneOf": [ { @@ -9923,12 +10136,15 @@ "type": "object", "properties": { "budget_policy_id": { + "description": "Budget policy to set on the newly created pipeline.", "$ref": "#/$defs/string" }, "storage_catalog": { + "description": "UC catalog for the pipeline to store intermediate files (checkpoints, event logs etc).\nThis needs to be a standard catalog where the user has permissions to create Delta tables.", "$ref": "#/$defs/string" }, "storage_schema": { + "description": "UC schema for the pipeline to store intermediate files (checkpoints, event logs etc).\nThis needs to be in the standard catalog where the user has permissions to create Delta tables.", "$ref": "#/$defs/string" } }, @@ -9998,7 +10214,21 @@ ] }, "postgres.SyncedTableSyncedTableSpecSyncedTableSchedulingPolicy": { - "type": "string" + "oneOf": [ + { + "type": "string", + "description": "Scheduling policy of the synced table's underlying pipeline.", + "enum": [ + "CONTINUOUS", + "TRIGGERED", + "SNAPSHOT" + ] + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)+)\\}" + } + ] }, "serving.Ai21LabsConfig": { "oneOf": [ @@ -11061,7 +11291,8 @@ "GPU_MEDIUM", "GPU_SMALL", "GPU_LARGE", - "MULTIGPU_MEDIUM" + "MULTIGPU_MEDIUM", + "GPU_XLARGE" ] }, { @@ -11097,7 +11328,8 @@ "GPU_MEDIUM", "GPU_SMALL", "GPU_LARGE", - "MULTIGPU_MEDIUM" + "MULTIGPU_MEDIUM", + "GPU_XLARGE" ] }, { @@ -12556,6 +12788,20 @@ } ] }, + "jobs.PythonOperatorTaskParameter": { + "oneOf": [ + { + "type": "array", + "items": { + "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/jobs.PythonOperatorTaskParameter" + } + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)+)\\}" + } + ] + }, "jobs.SqlTaskSubscription": { "oneOf": [ { diff --git a/bundle/schema/jsonschema_for_docs.json b/bundle/schema/jsonschema_for_docs.json index ff64ac25c26..e5caee8b64a 100644 --- a/bundle/schema/jsonschema_for_docs.json +++ b/bundle/schema/jsonschema_for_docs.json @@ -113,6 +113,8 @@ "git_source": { "description": "Git source configuration for app deployments. Specifies which git reference (branch, tag, or commit)\nto use when deploying the app. Used in conjunction with git_repository to deploy code directly from git.\nThe source_code_path within git_source specifies the relative path to the app code within the repository.", "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/apps.GitSource", + "x-databricks-preview": "PRIVATE", + "doNotSuggest": true, "x-since-version": "v0.290.0" }, "lifecycle": { @@ -136,6 +138,8 @@ }, "source_code_path": { "$ref": "#/$defs/string", + "x-databricks-preview": "PRIVATE", + "doNotSuggest": true, "x-since-version": "v0.239.0" }, "space": { @@ -1368,6 +1372,10 @@ "$ref": "#/$defs/string", "x-since-version": "v0.287.0" }, + "replace_existing": { + "$ref": "#/$defs/bool", + "x-since-version": "v1.0.0" + }, "source_branch": { "$ref": "#/$defs/string", "x-since-version": "v0.287.0" @@ -1395,19 +1403,24 @@ "type": "object", "properties": { "branch": { - "$ref": "#/$defs/string" + "$ref": "#/$defs/string", + "x-since-version": "v1.0.0" }, "catalog_id": { - "$ref": "#/$defs/string" + "$ref": "#/$defs/string", + "x-since-version": "v1.0.0" }, "create_database_if_missing": { - "$ref": "#/$defs/bool" + "$ref": "#/$defs/bool", + "x-since-version": "v1.0.0" }, "lifecycle": { - "$ref": "#/$defs/github.com/databricks/cli/bundle/config/resources.Lifecycle" + "$ref": "#/$defs/github.com/databricks/cli/bundle/config/resources.Lifecycle", + "x-since-version": "v1.0.0" }, "postgres_database": { - "$ref": "#/$defs/string" + "$ref": "#/$defs/string", + "x-since-version": "v1.0.0" } }, "additionalProperties": false, @@ -1455,6 +1468,10 @@ "$ref": "#/$defs/string", "x-since-version": "v0.287.0" }, + "replace_existing": { + "$ref": "#/$defs/bool", + "x-since-version": "v1.0.0" + }, "settings": { "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/postgres.EndpointSettings", "x-since-version": "v0.287.0" @@ -1524,6 +1541,59 @@ "project_id" ] }, + "resources.PostgresSyncedTable": { + "type": "object", + "properties": { + "branch": { + "$ref": "#/$defs/string", + "x-since-version": "v1.0.0" + }, + "create_database_objects_if_missing": { + "$ref": "#/$defs/bool", + "x-since-version": "v1.0.0" + }, + "existing_pipeline_id": { + "$ref": "#/$defs/string", + "x-since-version": "v1.0.0" + }, + "lifecycle": { + "$ref": "#/$defs/github.com/databricks/cli/bundle/config/resources.Lifecycle", + "x-since-version": "v1.0.0" + }, + "new_pipeline_spec": { + "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/postgres.NewPipelineSpec", + "x-since-version": "v1.0.0" + }, + "postgres_database": { + "$ref": "#/$defs/string", + "x-since-version": "v1.0.0" + }, + "primary_key_columns": { + "$ref": "#/$defs/slice/string", + "x-since-version": "v1.0.0" + }, + "scheduling_policy": { + "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/postgres.SyncedTableSyncedTableSpecSyncedTableSchedulingPolicy", + "x-since-version": "v1.0.0" + }, + "source_table_full_name": { + "$ref": "#/$defs/string", + "x-since-version": "v1.0.0" + }, + "synced_table_id": { + "$ref": "#/$defs/string", + "x-since-version": "v1.0.0" + }, + "timeseries_key": { + "$ref": "#/$defs/string", + "x-since-version": "v1.0.0" + } + }, + "additionalProperties": false, + "required": [ + "synced_table_id" + ] + }, "resources.QualityMonitor": { "type": "object", "properties": { @@ -2514,7 +2584,8 @@ }, "postgres_catalogs": { "description": "The Postgres catalog definitions for the bundle, where each key is the name of the catalog. Each entry binds a Unity Catalog catalog to a Postgres database on a Lakebase Autoscaling branch.", - "$ref": "#/$defs/map/github.com/databricks/cli/bundle/config/resources.PostgresCatalog" + "$ref": "#/$defs/map/github.com/databricks/cli/bundle/config/resources.PostgresCatalog", + "x-since-version": "v1.0.0" }, "postgres_endpoints": { "$ref": "#/$defs/map/github.com/databricks/cli/bundle/config/resources.PostgresEndpoint", @@ -2524,6 +2595,11 @@ "$ref": "#/$defs/map/github.com/databricks/cli/bundle/config/resources.PostgresProject", "x-since-version": "v0.287.0" }, + "postgres_synced_tables": { + "description": "The Postgres synced table definitions for the bundle, where each key is the name of the synced table. Each entry continuously replicates a Unity Catalog Delta source table into a Postgres table on a Lakebase Autoscaling instance.", + "$ref": "#/$defs/map/github.com/databricks/cli/bundle/config/resources.PostgresSyncedTable", + "x-since-version": "v1.0.0" + }, "quality_monitors": { "description": "The quality monitor definitions for the bundle, where each key is the name of the quality monitor.", "$ref": "#/$defs/map/github.com/databricks/cli/bundle/config/resources.QualityMonitor", @@ -3756,7 +3832,9 @@ "READ_METADATA", "MANAGE_ACCESS", "MANAGE_ACCESS_CONTROL", - "CREATE_SERVICE" + "CREATE_SERVICE", + "CREATE_FEATURE", + "READ_FEATURE" ] }, "catalog.PrivilegeAssignment": { @@ -4198,7 +4276,7 @@ }, "compute.DataSecurityMode": { "type": "string", - "description": "Data security mode decides what data governance model to use when accessing data\nfrom a cluster.\n\nThe following modes can only be used when `kind = CLASSIC_PREVIEW`.\n* `DATA_SECURITY_MODE_AUTO`: Databricks will choose the most appropriate access mode depending on your compute configuration.\n* `DATA_SECURITY_MODE_STANDARD`: Alias for `USER_ISOLATION`.\n* `DATA_SECURITY_MODE_DEDICATED`: Alias for `SINGLE_USER`.\n\nThe following modes can be used regardless of `kind`.\n* `NONE`: No security isolation for multiple users sharing the cluster. Data governance features are not available in this mode.\n* `SINGLE_USER`: A secure cluster that can only be exclusively used by a single user specified in `single_user_name`. Most programming languages, cluster features and data governance features are available in this mode.\n* `USER_ISOLATION`: A secure cluster that can be shared by multiple users. Cluster users are fully isolated so that they cannot see each other's data and credentials. Most data governance features are supported in this mode. But programming languages and cluster features might be limited.\n\nThe following modes are deprecated starting with Databricks Runtime 15.0 and\nwill be removed for future Databricks Runtime versions:\n\n* `LEGACY_TABLE_ACL`: This mode is for users migrating from legacy Table ACL clusters.\n* `LEGACY_PASSTHROUGH`: This mode is for users migrating from legacy Passthrough on high concurrency clusters.\n* `LEGACY_SINGLE_USER`: This mode is for users migrating from legacy Passthrough on standard clusters.\n* `LEGACY_SINGLE_USER_STANDARD`: This mode provides a way that doesn’t have UC nor passthrough enabled.", + "description": "Data security mode decides what data governance model to use when accessing data\nfrom a cluster.\n\n* `DATA_SECURITY_MODE_AUTO`: Databricks will choose the most appropriate access mode depending on your compute configuration.\n* `DATA_SECURITY_MODE_STANDARD`: A secure cluster that can be shared by multiple users. Cluster users are fully isolated so that they cannot see each other’s data and credentials. Most data governance features are supported in this mode. But programming languages and cluster features might be limited.\n* `DATA_SECURITY_MODE_DEDICATED`: A secure cluster that can only be exclusively used by a single user specified in `single_user_name`. Most programming languages, cluster features and data governance features are available in this mode.\n\nThe following modes are legacy aliases for the above modes:\n\n* `USER_ISOLATION`: Legacy alias for `DATA_SECURITY_MODE_STANDARD`.\n* `SINGLE_USER`: Legacy alias for `DATA_SECURITY_MODE_DEDICATED`.\n\nThe following modes are deprecated starting with Databricks Runtime 15.0 and\nwill be removed for future Databricks Runtime versions:\n\n* `LEGACY_TABLE_ACL`: This mode is for users migrating from legacy Table ACL clusters.\n* `LEGACY_PASSTHROUGH`: This mode is for users migrating from legacy Passthrough on high concurrency clusters.\n* `LEGACY_SINGLE_USER`: This mode is for users migrating from legacy Passthrough on standard clusters.\n* `LEGACY_SINGLE_USER_STANDARD`: This mode provides a way that doesn’t have UC nor passthrough enabled.", "enum": [ "NONE", "SINGLE_USER", @@ -4271,7 +4349,7 @@ "description": "The environment entity used to preserve serverless environment side panel, jobs' environment for non-notebook task, and SDP's environment for classic and serverless pipelines.\nIn this minimal environment spec, only pip and java dependencies are supported.", "properties": { "base_environment": { - "description": "The base environment this environment is built on top of. A base environment defines the environment version and a\nlist of dependencies for serverless compute. The value can be a file path to a custom `env.yaml` file\n(e.g., `/Workspace/path/to/env.yaml`). Support for a Databricks-provided base environment ID\n(e.g., `workspace-base-environments/databricks_ai_v4`) and workspace base environment ID\n(e.g., `workspace-base-environments/dbe_b849b66e-b31a-4cb5-b161-1f2b10877fb7`) is in Beta.\nEither `environment_version` or `base_environment` can be provided. For more information, see", + "description": "The base environment this environment is built on top of. A base environment defines the environment version and a\nlist of dependencies for serverless compute. The value can be a file path to a custom `env.yaml` file\n(e.g., `/Workspace/path/to/env.yaml`). Support for a Databricks-provided base environment ID\n(e.g., `workspace-base-environments/databricks_ai_v4`) and workspace base environment ID\n(e.g., `workspace-base-environments/dbe_b849b66e-b31a-4cb5-b161-1f2b10877fb7`) is in Beta.\nEither `environment_version` or `base_environment` can be provided.\nFor more information about Databricks-provided base environments, see the\n[list workspace base environments](:method:Environments/ListWorkspaceBaseEnvironments) API.\nFor more information, see", "$ref": "#/$defs/string", "x-since-version": "v0.289.0" }, @@ -4731,7 +4809,7 @@ "STOPPED", "UPDATING", "FAILING_OVER", - "MIGRATING" + "UPGRADING" ] }, "database.DeltaTableSyncInfo": { @@ -5792,6 +5870,22 @@ "description": "If true, triggers a full refresh on the spark declarative pipeline.", "$ref": "#/$defs/bool", "x-since-version": "v0.229.0" + }, + "full_refresh_selection": { + "description": "A list of tables to update with fullRefresh.", + "$ref": "#/$defs/slice/string" + }, + "refresh_flow_selection": { + "description": "Flow names to selectively refresh. These are unioned with other selective refresh\noptions (refresh_selection, full_refresh_selection) to determine the final set of flows to refresh.", + "$ref": "#/$defs/slice/string" + }, + "refresh_selection": { + "description": "A list of tables to update without fullRefresh.", + "$ref": "#/$defs/slice/string" + }, + "reset_checkpoint_selection": { + "description": "A list of streaming flows to reset checkpoints without clearing data.", + "$ref": "#/$defs/slice/string" } }, "additionalProperties": false @@ -5804,10 +5898,26 @@ "$ref": "#/$defs/bool", "x-since-version": "v0.229.0" }, + "full_refresh_selection": { + "description": "A list of tables to update with fullRefresh.", + "$ref": "#/$defs/slice/string" + }, "pipeline_id": { "description": "The full name of the pipeline task to execute.", "$ref": "#/$defs/string", "x-since-version": "v0.229.0" + }, + "refresh_flow_selection": { + "description": "Flow names to selectively refresh. These are unioned with other selective refresh\noptions (refresh_selection, full_refresh_selection) to determine the final set of flows to refresh.", + "$ref": "#/$defs/slice/string" + }, + "refresh_selection": { + "description": "A list of tables to update without fullRefresh.", + "$ref": "#/$defs/slice/string" + }, + "reset_checkpoint_selection": { + "description": "A list of streaming flows to reset checkpoints without clearing data.", + "$ref": "#/$defs/slice/string" } }, "additionalProperties": false, @@ -5903,6 +6013,32 @@ }, "additionalProperties": false }, + "jobs.PythonOperatorTask": { + "type": "object", + "properties": { + "main": { + "description": "Fully qualified name of the main class or function.\nFor example, `my_project.my_function` or `my_project.MyOperator`.", + "$ref": "#/$defs/string" + }, + "parameters": { + "description": "An ordered list of task parameters.\nTODO(JOBS-30885): Add limits for parameters.", + "$ref": "#/$defs/slice/github.com/databricks/databricks-sdk-go/service/jobs.PythonOperatorTaskParameter" + } + }, + "additionalProperties": false + }, + "jobs.PythonOperatorTaskParameter": { + "type": "object", + "properties": { + "name": { + "$ref": "#/$defs/string" + }, + "value": { + "$ref": "#/$defs/string" + } + }, + "additionalProperties": false + }, "jobs.PythonWheelTask": { "type": "object", "properties": { @@ -6407,8 +6543,6 @@ "disabled": { "description": "An optional flag to disable the task. If set to true, the task will not run even if it is part of a job.", "$ref": "#/$defs/bool", - "x-databricks-preview": "PRIVATE", - "doNotSuggest": true, "x-since-version": "v0.271.0" }, "email_notifications": { @@ -6486,6 +6620,12 @@ "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/jobs.PowerBiTask", "x-since-version": "v0.248.0" }, + "python_operator_task": { + "description": "The task runs a Python operator task.", + "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/jobs.PythonOperatorTask", + "x-databricks-preview": "PRIVATE", + "doNotSuggest": true + }, "python_wheel_task": { "description": "The task runs a Python wheel when the `python_wheel_task` field is present.", "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/jobs.PythonWheelTask", @@ -6836,6 +6976,11 @@ "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/pipelines.JiraConnectorOptions", "x-since-version": "v0.299.2" }, + "kafka_options": { + "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/pipelines.KafkaOptions", + "x-databricks-preview": "PRIVATE", + "doNotSuggest": true + }, "meta_ads_options": { "description": "Meta Marketing (Meta Ads) specific options for ingestion", "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/pipelines.MetaMarketingOptions", @@ -7055,7 +7200,8 @@ "EXCEL", "PARQUET", "AVRO", - "ORC" + "ORC", + "FILE" ] }, "pipelines.FileIngestionOptionsSchemaEvolutionMode": { @@ -7156,7 +7302,9 @@ "enum": [ "FILE", "FILE_METADATA", - "PERMISSION" + "PERMISSION", + "FILE_PERMISSION", + "GROUP_MEMBERSHIP" ] }, "pipelines.IngestionConfig": { @@ -7467,6 +7615,70 @@ }, "additionalProperties": false }, + "pipelines.JsonTransformerOptions": { + "type": "object", + "properties": { + "as_variant": { + "description": "Parse the entire value as a single Variant column.", + "$ref": "#/$defs/bool" + }, + "schema": { + "description": "Inline schema string for JSON parsing (Spark DDL format).", + "$ref": "#/$defs/string" + }, + "schema_evolution_mode": { + "description": "(Optional) Schema evolution mode for schema inference.", + "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/pipelines.FileIngestionOptionsSchemaEvolutionMode" + }, + "schema_file_path": { + "description": "Path to a schema file (.ddl).", + "$ref": "#/$defs/string" + }, + "schema_hints": { + "description": "(Optional) Schema hints as a comma-separated string of \"column_name type\" pairs.", + "$ref": "#/$defs/string" + } + }, + "additionalProperties": false + }, + "pipelines.KafkaOptions": { + "type": "object", + "properties": { + "client_config": { + "description": "Undocumented backdoor mechanism for overriding parameters\nto pass to the Kafka client.\nThis is not supported and may break at any time.", + "$ref": "#/$defs/map/string", + "x-databricks-preview": "PRIVATE", + "doNotSuggest": true + }, + "key_transformer": { + "description": "(Optional) Transformer for the message key.\nIf not specified, the key is left as raw bytes.", + "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/pipelines.Transformer" + }, + "max_offsets_per_trigger": { + "description": "Internal option to control the maximum number of offsets to process per trigger.", + "$ref": "#/$defs/int64", + "x-databricks-preview": "PRIVATE", + "doNotSuggest": true + }, + "starting_offset": { + "description": "(Optional) Where to begin reading when no checkpoint exists.\nValid values: \"latest\" and \"earliest\". Defaults to \"latest\".", + "$ref": "#/$defs/string" + }, + "topic_pattern": { + "description": "Java regex pattern to subscribe to matching topics.\nOnly one of topics or topic_pattern must be specified.", + "$ref": "#/$defs/string" + }, + "topics": { + "description": "Topics to subscribe to.\nOnly one of topics or topic_pattern must be specified.", + "$ref": "#/$defs/slice/string" + }, + "value_transformer": { + "description": "(Optional) Transformer for the message value.\nIf not specified, the value is left as raw bytes.", + "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/pipelines.Transformer" + } + }, + "additionalProperties": false + }, "pipelines.ManualTrigger": { "type": "object", "additionalProperties": false @@ -8074,7 +8286,9 @@ "FILE", "FILE_METADATA", "PERMISSION", - "LIST" + "LIST", + "FILE_PERMISSION", + "GROUP_MEMBERSHIP" ] }, "pipelines.SmartsheetOptions": { @@ -8306,6 +8520,29 @@ "GMV_MAX" ] }, + "pipelines.Transformer": { + "type": "object", + "description": "Specifies how to transform binary data into structured data.", + "properties": { + "format": { + "description": "Required: the wire format of the data.", + "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/pipelines.TransformerFormat" + }, + "json_options": { + "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/pipelines.JsonTransformerOptions" + } + }, + "additionalProperties": false + }, + "pipelines.TransformerFormat": { + "type": "string", + "enum": [ + "STRING", + "JSON", + "AVRO", + "PROTOBUF" + ] + }, "pipelines.ZendeskSupportOptions": { "type": "object", "description": "Zendesk Support specific options for ingestion", @@ -8363,6 +8600,27 @@ "ENDPOINT_TYPE_READ_ONLY" ] }, + "postgres.NewPipelineSpec": { + "type": "object", + "properties": { + "budget_policy_id": { + "description": "Budget policy to set on the newly created pipeline.", + "$ref": "#/$defs/string", + "x-since-version": "v1.0.0" + }, + "storage_catalog": { + "description": "UC catalog for the pipeline to store intermediate files (checkpoints, event logs etc).\nThis needs to be a standard catalog where the user has permissions to create Delta tables.", + "$ref": "#/$defs/string", + "x-since-version": "v1.0.0" + }, + "storage_schema": { + "description": "UC schema for the pipeline to store intermediate files (checkpoints, event logs etc).\nThis needs to be in the standard catalog where the user has permissions to create Delta tables.", + "$ref": "#/$defs/string", + "x-since-version": "v1.0.0" + } + }, + "additionalProperties": false + }, "postgres.ProjectCustomTag": { "type": "object", "properties": { @@ -8411,6 +8669,15 @@ }, "additionalProperties": false }, + "postgres.SyncedTableSyncedTableSpecSyncedTableSchedulingPolicy": { + "type": "string", + "description": "Scheduling policy of the synced table's underlying pipeline.", + "enum": [ + "CONTINUOUS", + "TRIGGERED", + "SNAPSHOT" + ] + }, "serving.Ai21LabsConfig": { "type": "object", "properties": { @@ -9314,7 +9581,8 @@ "GPU_MEDIUM", "GPU_SMALL", "GPU_LARGE", - "MULTIGPU_MEDIUM" + "MULTIGPU_MEDIUM", + "GPU_XLARGE" ] }, "serving.ServingEndpointPermissionLevel": { @@ -9334,7 +9602,8 @@ "GPU_MEDIUM", "GPU_SMALL", "GPU_LARGE", - "MULTIGPU_MEDIUM" + "MULTIGPU_MEDIUM", + "GPU_XLARGE" ] }, "serving.TrafficConfig": { @@ -9795,6 +10064,12 @@ "$ref": "#/$defs/github.com/databricks/cli/bundle/config/resources.PostgresProject" } }, + "resources.PostgresSyncedTable": { + "type": "object", + "additionalProperties": { + "$ref": "#/$defs/github.com/databricks/cli/bundle/config/resources.PostgresSyncedTable" + } + }, "resources.QualityMonitor": { "type": "object", "additionalProperties": { @@ -10076,6 +10351,12 @@ "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/jobs.PowerBiTable" } }, + "jobs.PythonOperatorTaskParameter": { + "type": "array", + "items": { + "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/jobs.PythonOperatorTaskParameter" + } + }, "jobs.SqlTaskSubscription": { "type": "array", "items": { diff --git a/cmd/account/endpoints/endpoints.go b/cmd/account/endpoints/endpoints.go index 0da1a198d02..eb3056de47f 100755 --- a/cmd/account/endpoints/endpoints.go +++ b/cmd/account/endpoints/endpoints.go @@ -20,8 +20,8 @@ var cmdOverrides []func(*cobra.Command) func New() *cobra.Command { cmd := &cobra.Command{ Use: "endpoints", - Short: `*Beta* These APIs manage endpoint configurations for this account.`, - Long: `This command is in Beta and may change without notice. + Short: `*Public Preview* These APIs manage endpoint configurations for this account.`, + Long: `This command is in Public Preview and may change without notice. These APIs manage endpoint configurations for this account.`, GroupID: "provisioning", @@ -29,8 +29,8 @@ These APIs manage endpoint configurations for this account.`, } cmd.Annotations = make(map[string]string) - cmd.Annotations["launch_stage"] = "PUBLIC_BETA" - cmd.Annotations["launch_stage_display"] = "Beta" + cmd.Annotations["launch_stage"] = "PUBLIC_PREVIEW" + cmd.Annotations["launch_stage_display"] = "Public Preview" // Add methods cmd.AddCommand(newCreateEndpoint()) @@ -67,8 +67,8 @@ func newCreateEndpoint() *cobra.Command { // TODO: complex arg: azure_private_endpoint_info cmd.Use = "create-endpoint PARENT DISPLAY_NAME REGION" - cmd.Short = `*Beta* Create a network endpoint.` - cmd.Long = `This command is in Beta and may change without notice. + cmd.Short = `*Public Preview* Create a network endpoint.` + cmd.Long = `This command is in Public Preview and may change without notice. Create a network endpoint. @@ -91,8 +91,8 @@ Create a network endpoint. REGION: The cloud provider region where this endpoint is located.` cmd.Annotations = make(map[string]string) - cmd.Annotations["launch_stage"] = "PUBLIC_BETA" - cmd.Annotations["launch_stage_display"] = "Beta" + cmd.Annotations["launch_stage"] = "PUBLIC_PREVIEW" + cmd.Annotations["launch_stage_display"] = "Public Preview" cmd.Args = func(cmd *cobra.Command, args []string) error { if cmd.Flags().Changed("json") { @@ -166,8 +166,8 @@ func newDeleteEndpoint() *cobra.Command { var deleteEndpointReq networking.DeleteEndpointRequest cmd.Use = "delete-endpoint NAME" - cmd.Short = `*Beta* Delete a network endpoint.` - cmd.Long = `This command is in Beta and may change without notice. + cmd.Short = `*Public Preview* Delete a network endpoint.` + cmd.Long = `This command is in Public Preview and may change without notice. Delete a network endpoint. @@ -176,8 +176,8 @@ Delete a network endpoint. delete corresponding network resources in your cloud provider account.` cmd.Annotations = make(map[string]string) - cmd.Annotations["launch_stage"] = "PUBLIC_BETA" - cmd.Annotations["launch_stage_display"] = "Beta" + cmd.Annotations["launch_stage"] = "PUBLIC_PREVIEW" + cmd.Annotations["launch_stage_display"] = "Public Preview" cmd.Args = func(cmd *cobra.Command, args []string) error { check := root.ExactArgs(1) @@ -225,16 +225,16 @@ func newGetEndpoint() *cobra.Command { var getEndpointReq networking.GetEndpointRequest cmd.Use = "get-endpoint NAME" - cmd.Short = `*Beta* Get a network endpoint.` - cmd.Long = `This command is in Beta and may change without notice. + cmd.Short = `*Public Preview* Get a network endpoint.` + cmd.Long = `This command is in Public Preview and may change without notice. Get a network endpoint. Gets details of a specific network endpoint.` cmd.Annotations = make(map[string]string) - cmd.Annotations["launch_stage"] = "PUBLIC_BETA" - cmd.Annotations["launch_stage_display"] = "Beta" + cmd.Annotations["launch_stage"] = "PUBLIC_PREVIEW" + cmd.Annotations["launch_stage_display"] = "Public Preview" cmd.Args = func(cmd *cobra.Command, args []string) error { check := root.ExactArgs(1) @@ -296,8 +296,8 @@ func newListEndpoints() *cobra.Command { cmd.Flags().Lookup("page-token").Hidden = true cmd.Use = "list-endpoints PARENT" - cmd.Short = `*Beta* List network endpoints.` - cmd.Long = `This command is in Beta and may change without notice. + cmd.Short = `*Public Preview* List network endpoints.` + cmd.Long = `This command is in Public Preview and may change without notice. List network endpoints. @@ -308,8 +308,8 @@ List network endpoints. accounts/{account_id}.` cmd.Annotations = make(map[string]string) - cmd.Annotations["launch_stage"] = "PUBLIC_BETA" - cmd.Annotations["launch_stage_display"] = "Beta" + cmd.Annotations["launch_stage"] = "PUBLIC_PREVIEW" + cmd.Annotations["launch_stage_display"] = "Public Preview" cmd.Args = func(cmd *cobra.Command, args []string) error { check := root.ExactArgs(1) diff --git a/cmd/account/iam-v2/iam-v2.go b/cmd/account/iam-v2/iam-v2.go index bbddb8cbca4..965966c124c 100755 --- a/cmd/account/iam-v2/iam-v2.go +++ b/cmd/account/iam-v2/iam-v2.go @@ -4,11 +4,13 @@ package iam_v2 import ( "fmt" + "strings" "github.com/databricks/cli/cmd/root" "github.com/databricks/cli/libs/cmdctx" "github.com/databricks/cli/libs/cmdio" "github.com/databricks/cli/libs/flags" + "github.com/databricks/databricks-sdk-go/common/types/fieldmask" "github.com/databricks/databricks-sdk-go/service/iamv2" "github.com/spf13/cobra" ) @@ -34,10 +36,15 @@ These APIs are used to manage identities and the workspace access of these cmd.Annotations["launch_stage_display"] = "Beta" // Add methods + cmd.AddCommand(newCreateWorkspaceAssignmentDetail()) + cmd.AddCommand(newDeleteWorkspaceAssignmentDetail()) cmd.AddCommand(newGetWorkspaceAccessDetail()) + cmd.AddCommand(newGetWorkspaceAssignmentDetail()) + cmd.AddCommand(newListWorkspaceAssignmentDetails()) cmd.AddCommand(newResolveGroup()) cmd.AddCommand(newResolveServicePrincipal()) cmd.AddCommand(newResolveUser()) + cmd.AddCommand(newUpdateWorkspaceAssignmentDetail()) // Apply optional overrides to this command. for _, fn := range cmdOverrides { @@ -47,6 +54,185 @@ These APIs are used to manage identities and the workspace access of these return cmd } +// start create-workspace-assignment-detail command + +// Slice with functions to override default command behavior. +// Functions can be added from the `init()` function in manually curated files in this directory. +var createWorkspaceAssignmentDetailOverrides []func( + *cobra.Command, + *iamv2.CreateWorkspaceAssignmentDetailRequest, +) + +func newCreateWorkspaceAssignmentDetail() *cobra.Command { + cmd := &cobra.Command{} + + var createWorkspaceAssignmentDetailReq iamv2.CreateWorkspaceAssignmentDetailRequest + createWorkspaceAssignmentDetailReq.WorkspaceAssignmentDetail = iamv2.WorkspaceAssignmentDetail{} + var createWorkspaceAssignmentDetailJson flags.JsonFlag + + cmd.Flags().Var(&createWorkspaceAssignmentDetailJson, "json", `either inline JSON string or @path/to/file.json with request body`) + + // TODO: array: entitlements + + cmd.Use = "create-workspace-assignment-detail WORKSPACE_ID PRINCIPAL_ID" + cmd.Short = `Create a workspace assignment detail.` + cmd.Long = `Create a workspace assignment detail. + + Creates a workspace assignment detail for a principal. Entitlement grants are + applied individually and non-atomically — if a failure occurs partway + through, the principal will be assigned to the workspace but with only a + subset of the requested entitlements. Use GetWorkspaceAssignmentDetail to + confirm which entitlements were successfully granted. + + Arguments: + WORKSPACE_ID: Required. The workspace ID for which the workspace assignment detail is + being created. + PRINCIPAL_ID: The internal ID of the principal (user/sp/group) in Databricks.` + + // This command is being previewed; hide from help output. + cmd.Hidden = true + + cmd.Annotations = make(map[string]string) + cmd.Annotations["launch_stage"] = "PRIVATE_PREVIEW" + cmd.Annotations["launch_stage_display"] = "Private Preview" + + cmd.Args = func(cmd *cobra.Command, args []string) error { + if cmd.Flags().Changed("json") { + err := root.ExactArgs(1)(cmd, args) + if err != nil { + return fmt.Errorf("when --json flag is specified, provide only WORKSPACE_ID as positional arguments. Provide 'principal_id' in your JSON input") + } + return nil + } + check := root.ExactArgs(2) + return check(cmd, args) + } + + cmd.PreRunE = root.MustAccountClient + cmd.RunE = func(cmd *cobra.Command, args []string) (err error) { + ctx := cmd.Context() + a := cmdctx.AccountClient(ctx) + + if cmd.Flags().Changed("json") { + diags := createWorkspaceAssignmentDetailJson.Unmarshal(&createWorkspaceAssignmentDetailReq.WorkspaceAssignmentDetail) + if diags.HasError() { + return diags.Error() + } + if len(diags) > 0 { + err := cmdio.RenderDiagnostics(ctx, diags) + if err != nil { + return err + } + } + } + _, err = fmt.Sscan(args[0], &createWorkspaceAssignmentDetailReq.WorkspaceId) + if err != nil { + return fmt.Errorf("invalid WORKSPACE_ID: %s", args[0]) + } + + if !cmd.Flags().Changed("json") { + _, err = fmt.Sscan(args[1], &createWorkspaceAssignmentDetailReq.WorkspaceAssignmentDetail.PrincipalId) + if err != nil { + return fmt.Errorf("invalid PRINCIPAL_ID: %s", args[1]) + } + + } + + response, err := a.IamV2.CreateWorkspaceAssignmentDetail(ctx, createWorkspaceAssignmentDetailReq) + if err != nil { + return err + } + + return cmdio.Render(ctx, response) + } + + // Disable completions since they are not applicable. + // Can be overridden by manual implementation in `override.go`. + cmd.ValidArgsFunction = cobra.NoFileCompletions + + // Apply optional overrides to this command. + for _, fn := range createWorkspaceAssignmentDetailOverrides { + fn(cmd, &createWorkspaceAssignmentDetailReq) + } + + return cmd +} + +// start delete-workspace-assignment-detail command + +// Slice with functions to override default command behavior. +// Functions can be added from the `init()` function in manually curated files in this directory. +var deleteWorkspaceAssignmentDetailOverrides []func( + *cobra.Command, + *iamv2.DeleteWorkspaceAssignmentDetailRequest, +) + +func newDeleteWorkspaceAssignmentDetail() *cobra.Command { + cmd := &cobra.Command{} + + var deleteWorkspaceAssignmentDetailReq iamv2.DeleteWorkspaceAssignmentDetailRequest + + cmd.Use = "delete-workspace-assignment-detail WORKSPACE_ID PRINCIPAL_ID" + cmd.Short = `Delete a workspace assignment detail.` + cmd.Long = `Delete a workspace assignment detail. + + Deletes a workspace assignment detail for a principal, revoking all associated + entitlements. Entitlement revocations are applied individually and + non-atomically — if a failure occurs partway through, the principal remains + assigned with a subset of its original entitlements, and the operation is safe + to retry. + + Arguments: + WORKSPACE_ID: The workspace ID where the principal has access. + PRINCIPAL_ID: Required. ID of the principal in Databricks to delete workspace assignment + for.` + + // This command is being previewed; hide from help output. + cmd.Hidden = true + + cmd.Annotations = make(map[string]string) + cmd.Annotations["launch_stage"] = "PRIVATE_PREVIEW" + cmd.Annotations["launch_stage_display"] = "Private Preview" + + cmd.Args = func(cmd *cobra.Command, args []string) error { + check := root.ExactArgs(2) + return check(cmd, args) + } + + cmd.PreRunE = root.MustAccountClient + cmd.RunE = func(cmd *cobra.Command, args []string) (err error) { + ctx := cmd.Context() + a := cmdctx.AccountClient(ctx) + + _, err = fmt.Sscan(args[0], &deleteWorkspaceAssignmentDetailReq.WorkspaceId) + if err != nil { + return fmt.Errorf("invalid WORKSPACE_ID: %s", args[0]) + } + + _, err = fmt.Sscan(args[1], &deleteWorkspaceAssignmentDetailReq.PrincipalId) + if err != nil { + return fmt.Errorf("invalid PRINCIPAL_ID: %s", args[1]) + } + + err = a.IamV2.DeleteWorkspaceAssignmentDetail(ctx, deleteWorkspaceAssignmentDetailReq) + if err != nil { + return err + } + return nil + } + + // Disable completions since they are not applicable. + // Can be overridden by manual implementation in `override.go`. + cmd.ValidArgsFunction = cobra.NoFileCompletions + + // Apply optional overrides to this command. + for _, fn := range deleteWorkspaceAssignmentDetailOverrides { + fn(cmd, &deleteWorkspaceAssignmentDetailReq) + } + + return cmd +} + // start get-workspace-access-detail command // Slice with functions to override default command behavior. @@ -126,6 +312,148 @@ Get workspace access details for a principal. return cmd } +// start get-workspace-assignment-detail command + +// Slice with functions to override default command behavior. +// Functions can be added from the `init()` function in manually curated files in this directory. +var getWorkspaceAssignmentDetailOverrides []func( + *cobra.Command, + *iamv2.GetWorkspaceAssignmentDetailRequest, +) + +func newGetWorkspaceAssignmentDetail() *cobra.Command { + cmd := &cobra.Command{} + + var getWorkspaceAssignmentDetailReq iamv2.GetWorkspaceAssignmentDetailRequest + + cmd.Use = "get-workspace-assignment-detail WORKSPACE_ID PRINCIPAL_ID" + cmd.Short = `Get workspace assignment details for a principal.` + cmd.Long = `Get workspace assignment details for a principal. + + Returns the assignment details for a principal in a workspace. + + Arguments: + WORKSPACE_ID: Required. The workspace ID for which the assignment details are being + requested. + PRINCIPAL_ID: Required. The internal ID of the principal (user/sp/group) for which the + assignment details are being requested.` + + // This command is being previewed; hide from help output. + cmd.Hidden = true + + cmd.Annotations = make(map[string]string) + cmd.Annotations["launch_stage"] = "PRIVATE_PREVIEW" + cmd.Annotations["launch_stage_display"] = "Private Preview" + + cmd.Args = func(cmd *cobra.Command, args []string) error { + check := root.ExactArgs(2) + return check(cmd, args) + } + + cmd.PreRunE = root.MustAccountClient + cmd.RunE = func(cmd *cobra.Command, args []string) (err error) { + ctx := cmd.Context() + a := cmdctx.AccountClient(ctx) + + _, err = fmt.Sscan(args[0], &getWorkspaceAssignmentDetailReq.WorkspaceId) + if err != nil { + return fmt.Errorf("invalid WORKSPACE_ID: %s", args[0]) + } + + _, err = fmt.Sscan(args[1], &getWorkspaceAssignmentDetailReq.PrincipalId) + if err != nil { + return fmt.Errorf("invalid PRINCIPAL_ID: %s", args[1]) + } + + response, err := a.IamV2.GetWorkspaceAssignmentDetail(ctx, getWorkspaceAssignmentDetailReq) + if err != nil { + return err + } + + return cmdio.Render(ctx, response) + } + + // Disable completions since they are not applicable. + // Can be overridden by manual implementation in `override.go`. + cmd.ValidArgsFunction = cobra.NoFileCompletions + + // Apply optional overrides to this command. + for _, fn := range getWorkspaceAssignmentDetailOverrides { + fn(cmd, &getWorkspaceAssignmentDetailReq) + } + + return cmd +} + +// start list-workspace-assignment-details command + +// Slice with functions to override default command behavior. +// Functions can be added from the `init()` function in manually curated files in this directory. +var listWorkspaceAssignmentDetailsOverrides []func( + *cobra.Command, + *iamv2.ListWorkspaceAssignmentDetailsRequest, +) + +func newListWorkspaceAssignmentDetails() *cobra.Command { + cmd := &cobra.Command{} + + var listWorkspaceAssignmentDetailsReq iamv2.ListWorkspaceAssignmentDetailsRequest + + cmd.Flags().IntVar(&listWorkspaceAssignmentDetailsReq.PageSize, "page-size", listWorkspaceAssignmentDetailsReq.PageSize, `The maximum number of workspace assignment details to return.`) + cmd.Flags().StringVar(&listWorkspaceAssignmentDetailsReq.PageToken, "page-token", listWorkspaceAssignmentDetailsReq.PageToken, `A page token, received from a previous ListWorkspaceAssignmentDetails call.`) + + cmd.Use = "list-workspace-assignment-details WORKSPACE_ID" + cmd.Short = `List workspace assignment details for a workspace.` + cmd.Long = `List workspace assignment details for a workspace. + + Lists workspace assignment details for a workspace. + + Arguments: + WORKSPACE_ID: Required. The workspace ID for which the workspace assignment details are + being fetched.` + + // This command is being previewed; hide from help output. + cmd.Hidden = true + + cmd.Annotations = make(map[string]string) + cmd.Annotations["launch_stage"] = "PRIVATE_PREVIEW" + cmd.Annotations["launch_stage_display"] = "Private Preview" + + cmd.Args = func(cmd *cobra.Command, args []string) error { + check := root.ExactArgs(1) + return check(cmd, args) + } + + cmd.PreRunE = root.MustAccountClient + cmd.RunE = func(cmd *cobra.Command, args []string) (err error) { + ctx := cmd.Context() + a := cmdctx.AccountClient(ctx) + + _, err = fmt.Sscan(args[0], &listWorkspaceAssignmentDetailsReq.WorkspaceId) + if err != nil { + return fmt.Errorf("invalid WORKSPACE_ID: %s", args[0]) + } + + response, err := a.IamV2.ListWorkspaceAssignmentDetails(ctx, listWorkspaceAssignmentDetailsReq) + if err != nil { + return err + } + + return cmdio.Render(ctx, response) + } + + // Disable completions since they are not applicable. + // Can be overridden by manual implementation in `override.go`. + cmd.ValidArgsFunction = cobra.NoFileCompletions + + // Apply optional overrides to this command. + for _, fn := range listWorkspaceAssignmentDetailsOverrides { + fn(cmd, &listWorkspaceAssignmentDetailsReq) + } + + return cmd +} + // start resolve-group command // Slice with functions to override default command behavior. @@ -388,4 +716,119 @@ Resolve an external user in the Databricks account. return cmd } +// start update-workspace-assignment-detail command + +// Slice with functions to override default command behavior. +// Functions can be added from the `init()` function in manually curated files in this directory. +var updateWorkspaceAssignmentDetailOverrides []func( + *cobra.Command, + *iamv2.UpdateWorkspaceAssignmentDetailRequest, +) + +func newUpdateWorkspaceAssignmentDetail() *cobra.Command { + cmd := &cobra.Command{} + + var updateWorkspaceAssignmentDetailReq iamv2.UpdateWorkspaceAssignmentDetailRequest + updateWorkspaceAssignmentDetailReq.WorkspaceAssignmentDetail = iamv2.WorkspaceAssignmentDetail{} + var updateWorkspaceAssignmentDetailJson flags.JsonFlag + + cmd.Flags().Var(&updateWorkspaceAssignmentDetailJson, "json", `either inline JSON string or @path/to/file.json with request body`) + + // TODO: array: entitlements + + cmd.Use = "update-workspace-assignment-detail WORKSPACE_ID PRINCIPAL_ID UPDATE_MASK PRINCIPAL_ID" + cmd.Short = `Update a workspace assignment detail.` + cmd.Long = `Update a workspace assignment detail. + + Updates the entitlements of a directly assigned principal in a workspace. + Entitlement changes are applied individually and non-atomically — if a + failure occurs partway through, only a subset of the requested changes may + have been applied. Use GetWorkspaceAssignmentDetail to confirm the final + state. + + Arguments: + WORKSPACE_ID: Required. The workspace ID for which the workspace assignment detail is + being updated. + PRINCIPAL_ID: Required. ID of the principal in Databricks. + UPDATE_MASK: Required. The list of fields to update. + PRINCIPAL_ID: The internal ID of the principal (user/sp/group) in Databricks.` + + // This command is being previewed; hide from help output. + cmd.Hidden = true + + cmd.Annotations = make(map[string]string) + cmd.Annotations["launch_stage"] = "PRIVATE_PREVIEW" + cmd.Annotations["launch_stage_display"] = "Private Preview" + + cmd.Args = func(cmd *cobra.Command, args []string) error { + if cmd.Flags().Changed("json") { + err := root.ExactArgs(3)(cmd, args) + if err != nil { + return fmt.Errorf("when --json flag is specified, provide only WORKSPACE_ID, PRINCIPAL_ID, UPDATE_MASK as positional arguments. Provide 'principal_id' in your JSON input") + } + return nil + } + check := root.ExactArgs(4) + return check(cmd, args) + } + + cmd.PreRunE = root.MustAccountClient + cmd.RunE = func(cmd *cobra.Command, args []string) (err error) { + ctx := cmd.Context() + a := cmdctx.AccountClient(ctx) + + if cmd.Flags().Changed("json") { + diags := updateWorkspaceAssignmentDetailJson.Unmarshal(&updateWorkspaceAssignmentDetailReq.WorkspaceAssignmentDetail) + if diags.HasError() { + return diags.Error() + } + if len(diags) > 0 { + err := cmdio.RenderDiagnostics(ctx, diags) + if err != nil { + return err + } + } + } + _, err = fmt.Sscan(args[0], &updateWorkspaceAssignmentDetailReq.WorkspaceId) + if err != nil { + return fmt.Errorf("invalid WORKSPACE_ID: %s", args[0]) + } + + _, err = fmt.Sscan(args[1], &updateWorkspaceAssignmentDetailReq.PrincipalId) + if err != nil { + return fmt.Errorf("invalid PRINCIPAL_ID: %s", args[1]) + } + + if args[2] != "" { + updateMaskArray := strings.Split(args[2], ",") + updateWorkspaceAssignmentDetailReq.UpdateMask = *fieldmask.New(updateMaskArray) + } + if !cmd.Flags().Changed("json") { + _, err = fmt.Sscan(args[3], &updateWorkspaceAssignmentDetailReq.WorkspaceAssignmentDetail.PrincipalId) + if err != nil { + return fmt.Errorf("invalid PRINCIPAL_ID: %s", args[3]) + } + + } + + response, err := a.IamV2.UpdateWorkspaceAssignmentDetail(ctx, updateWorkspaceAssignmentDetailReq) + if err != nil { + return err + } + + return cmdio.Render(ctx, response) + } + + // Disable completions since they are not applicable. + // Can be overridden by manual implementation in `override.go`. + cmd.ValidArgsFunction = cobra.NoFileCompletions + + // Apply optional overrides to this command. + for _, fn := range updateWorkspaceAssignmentDetailOverrides { + fn(cmd, &updateWorkspaceAssignmentDetailReq) + } + + return cmd +} + // end service AccountIamV2 diff --git a/cmd/account/settings-v2/settings-v2.go b/cmd/account/settings-v2/settings-v2.go index 7e6a042e83a..c42665912dd 100755 --- a/cmd/account/settings-v2/settings-v2.go +++ b/cmd/account/settings-v2/settings-v2.go @@ -349,18 +349,22 @@ func newPatchPublicAccountSetting() *cobra.Command { // TODO: complex arg: aibi_dashboard_embedding_access_policy // TODO: complex arg: aibi_dashboard_embedding_approved_domains + // TODO: complex arg: allowed_apps_user_api_scopes // TODO: complex arg: automatic_cluster_update_workspace // TODO: complex arg: boolean_val // TODO: complex arg: effective_aibi_dashboard_embedding_access_policy // TODO: complex arg: effective_aibi_dashboard_embedding_approved_domains + // TODO: complex arg: effective_allowed_apps_user_api_scopes // TODO: complex arg: effective_automatic_cluster_update_workspace // TODO: complex arg: effective_boolean_val // TODO: complex arg: effective_integer_val + // TODO: complex arg: effective_operational_email_custom_recipient // TODO: complex arg: effective_personal_compute // TODO: complex arg: effective_restrict_workspace_admins // TODO: complex arg: effective_string_val // TODO: complex arg: integer_val cmd.Flags().StringVar(&patchPublicAccountSettingReq.Setting.Name, "name", patchPublicAccountSettingReq.Setting.Name, `Name of the setting.`) + // TODO: complex arg: operational_email_custom_recipient // TODO: complex arg: personal_compute // TODO: complex arg: restrict_workspace_admins // TODO: complex arg: string_val diff --git a/cmd/apps/import.go b/cmd/apps/import.go index 67cf4ae9d9e..42f9da4ec14 100644 --- a/cmd/apps/import.go +++ b/cmd/apps/import.go @@ -34,6 +34,7 @@ import ( "github.com/databricks/cli/libs/textutil" "github.com/databricks/databricks-sdk-go" "github.com/databricks/databricks-sdk-go/service/apps" + "github.com/databricks/databricks-sdk-go/service/iam" "github.com/databricks/databricks-sdk-go/service/workspace" "github.com/spf13/cobra" ) @@ -89,7 +90,7 @@ Examples: w := cmdctx.WorkspaceClient(ctx) // Get current user to filter apps - me, err := w.CurrentUser.Me(ctx) + me, err := w.CurrentUser.Me(ctx, iam.MeRequest{}) if err != nil { return fmt.Errorf("failed to get current user: %w", err) } diff --git a/cmd/apps/run_local.go b/cmd/apps/run_local.go index d4bc546ab7c..8d42783c47e 100644 --- a/cmd/apps/run_local.go +++ b/cmd/apps/run_local.go @@ -18,6 +18,7 @@ import ( "github.com/databricks/cli/libs/cmdctx" "github.com/databricks/cli/libs/cmdio" "github.com/databricks/databricks-sdk-go" + "github.com/databricks/databricks-sdk-go/service/iam" "github.com/spf13/cobra" ) @@ -116,7 +117,7 @@ func setupProxy(ctx context.Context, cmd *cobra.Command, config *runlocal.Config return err } - me, err := w.CurrentUser.Me(ctx) + me, err := w.CurrentUser.Me(ctx, iam.MeRequest{}) if err != nil { return err } diff --git a/cmd/auth/describe.go b/cmd/auth/describe.go index 56f10630de1..b4abe63d65d 100644 --- a/cmd/auth/describe.go +++ b/cmd/auth/describe.go @@ -14,6 +14,7 @@ import ( "github.com/databricks/cli/libs/flags" "github.com/databricks/cli/libs/log" "github.com/databricks/databricks-sdk-go/config" + "github.com/databricks/databricks-sdk-go/service/iam" "github.com/spf13/cobra" ) @@ -127,7 +128,7 @@ func getAuthStatus(cmd *cobra.Command, args []string, showSensitive bool, fn try } w := cmdctx.WorkspaceClient(ctx) - me, err := w.CurrentUser.Me(ctx) + me, err := w.CurrentUser.Me(ctx, iam.MeRequest{}) if err != nil { details := getAuthDetails(cmd, cfg, showSensitive) return &authStatus{ //nolint:nilerr // error is returned in the authStatus struct diff --git a/cmd/auth/describe_test.go b/cmd/auth/describe_test.go index 72fee3486bc..3fb26c06d93 100644 --- a/cmd/auth/describe_test.go +++ b/cmd/auth/describe_test.go @@ -27,7 +27,7 @@ func TestGetWorkspaceAuthStatus(t *testing.T) { showSensitive := false currentUserApi := m.GetMockCurrentUserAPI() - currentUserApi.EXPECT().Me(mock.Anything).Return(&iam.User{ + currentUserApi.EXPECT().Me(mock.Anything, mock.Anything).Return(&iam.User{ UserName: "test-user", }, nil) @@ -306,7 +306,7 @@ func TestGetWorkspaceAuthStatus_U2M_PopulatesTokenStorage(t *testing.T) { cmd.SetContext(ctx) currentUserApi := m.GetMockCurrentUserAPI() - currentUserApi.EXPECT().Me(mock.Anything).Return(&iam.User{UserName: "u2m-user"}, nil) + currentUserApi.EXPECT().Me(mock.Anything, mock.Anything).Return(&iam.User{UserName: "u2m-user"}, nil) cmd.Flags().String("host", "", "") cmd.Flags().String("profile", "", "") @@ -343,7 +343,7 @@ func TestGetWorkspaceAuthStatus_NonU2M_OmitsTokenStorage(t *testing.T) { cmd.SetContext(ctx) currentUserApi := m.GetMockCurrentUserAPI() - currentUserApi.EXPECT().Me(mock.Anything).Return(&iam.User{UserName: "pat-user"}, nil) + currentUserApi.EXPECT().Me(mock.Anything, mock.Anything).Return(&iam.User{UserName: "pat-user"}, nil) cmd.Flags().String("host", "", "") cmd.Flags().String("profile", "", "") diff --git a/cmd/auth/profiles.go b/cmd/auth/profiles.go index 51c397a9ea9..09d52d7da00 100644 --- a/cmd/auth/profiles.go +++ b/cmd/auth/profiles.go @@ -16,6 +16,7 @@ import ( "github.com/databricks/cli/libs/log" "github.com/databricks/databricks-sdk-go" "github.com/databricks/databricks-sdk-go/config" + "github.com/databricks/databricks-sdk-go/service/iam" "github.com/spf13/cobra" "gopkg.in/ini.v1" ) @@ -80,7 +81,7 @@ func (c *profileMetadata) Load(ctx context.Context, configFilePath string, skipV if err != nil { return } - _, err = w.CurrentUser.Me(ctx) + _, err = w.CurrentUser.Me(ctx, iam.MeRequest{}) c.Host = cfg.Host c.AuthType = cfg.AuthType if err != nil { diff --git a/cmd/cmd.go b/cmd/cmd.go index 263c1da0c11..1587fbdc7ea 100644 --- a/cmd/cmd.go +++ b/cmd/cmd.go @@ -57,6 +57,13 @@ func New(ctx context.Context) *cobra.Command { // Add workspace subcommands. workspaceCommands := workspace.All() for _, cmd := range workspaceCommands { + // The auto-generated `bundle` workspace service (DMS) shares its name + // with the DAB `bundle` command tree (cmd/bundle). Registering both + // here clobbers the DAB tree's help output. Skip the generated one; + // callers still have `databricks api ...` for the DMS endpoints. + if cmd.Name() == "bundle" { + continue + } // Order the permissions subcommands after the main commands. for _, sub := range cmd.Commands() { // some commands override groups in overrides.go, leave them as-is diff --git a/cmd/psql/psql_autoscaling.go b/cmd/psql/psql_autoscaling.go index 00c555e4c12..331e05b55b5 100644 --- a/cmd/psql/psql_autoscaling.go +++ b/cmd/psql/psql_autoscaling.go @@ -10,6 +10,7 @@ import ( "github.com/databricks/cli/libs/cmdio" libpsql "github.com/databricks/cli/libs/psql" "github.com/databricks/databricks-sdk-go" + "github.com/databricks/databricks-sdk-go/service/iam" "github.com/databricks/databricks-sdk-go/service/postgres" ) @@ -37,7 +38,7 @@ func connectAutoscaling(ctx context.Context, projectID, branchID, endpointID str return err } - user, err := w.CurrentUser.Me(ctx) + user, err := w.CurrentUser.Me(ctx, iam.MeRequest{}) if err != nil { return fmt.Errorf("failed to get current user: %w", err) } diff --git a/cmd/psql/psql_provisioned.go b/cmd/psql/psql_provisioned.go index 88ca1bb9181..b66f17d1508 100644 --- a/cmd/psql/psql_provisioned.go +++ b/cmd/psql/psql_provisioned.go @@ -10,6 +10,7 @@ import ( libpsql "github.com/databricks/cli/libs/psql" "github.com/databricks/databricks-sdk-go" "github.com/databricks/databricks-sdk-go/service/database" + "github.com/databricks/databricks-sdk-go/service/iam" "github.com/google/uuid" ) @@ -26,7 +27,7 @@ func connectProvisioned(ctx context.Context, instanceName string, retryConfig li return err } - user, err := w.CurrentUser.Me(ctx) + user, err := w.CurrentUser.Me(ctx, iam.MeRequest{}) if err != nil { return fmt.Errorf("failed to get current user: %w", err) } diff --git a/cmd/sync/completion.go b/cmd/sync/completion.go index 52d49d6cb1c..bc34c1581ac 100644 --- a/cmd/sync/completion.go +++ b/cmd/sync/completion.go @@ -6,6 +6,7 @@ import ( "strings" "github.com/databricks/databricks-sdk-go" + "github.com/databricks/databricks-sdk-go/service/iam" "github.com/databricks/databricks-sdk-go/service/workspace" "github.com/spf13/cobra" ) @@ -44,7 +45,7 @@ func completeRemotePath( wsc *databricks.WorkspaceClient, toComplete string, ) ([]string, cobra.ShellCompDirective) { - me, err := wsc.CurrentUser.Me(ctx) + me, err := wsc.CurrentUser.Me(ctx, iam.MeRequest{}) if err != nil { return nil, cobra.ShellCompDirectiveError } diff --git a/cmd/workspace/bundle/bundle.go b/cmd/workspace/bundle/bundle.go new file mode 100755 index 00000000000..ee6e38addea --- /dev/null +++ b/cmd/workspace/bundle/bundle.go @@ -0,0 +1,1108 @@ +// Code generated from OpenAPI specs by Databricks SDK Generator. DO NOT EDIT. + +package bundle + +import ( + "fmt" + + "github.com/databricks/cli/cmd/root" + "github.com/databricks/cli/libs/cmdctx" + "github.com/databricks/cli/libs/cmdio" + "github.com/databricks/cli/libs/flags" + "github.com/databricks/databricks-sdk-go/service/bundle" + "github.com/spf13/cobra" +) + +// Slice with functions to override default command behavior. +// Functions can be added from the `init()` function in manually curated files in this directory. +var cmdOverrides []func(*cobra.Command) + +func New() *cobra.Command { + cmd := &cobra.Command{ + Use: "bundle", + Short: `Service for managing bundle deployment metadata.`, + Long: `Service for managing bundle deployment metadata.`, + GroupID: "bundle", + + // This service is being previewed; hide from help output. + Hidden: true, + RunE: root.ReportUnknownSubcommand, + } + + cmd.Annotations = make(map[string]string) + cmd.Annotations["launch_stage"] = "PRIVATE_PREVIEW" + cmd.Annotations["launch_stage_display"] = "Private Preview" + + // Add methods + cmd.AddCommand(newCompleteVersion()) + cmd.AddCommand(newCreateDeployment()) + cmd.AddCommand(newCreateOperation()) + cmd.AddCommand(newCreateVersion()) + cmd.AddCommand(newDeleteDeployment()) + cmd.AddCommand(newGetDeployment()) + cmd.AddCommand(newGetOperation()) + cmd.AddCommand(newGetResource()) + cmd.AddCommand(newGetVersion()) + cmd.AddCommand(newHeartbeat()) + cmd.AddCommand(newListDeployments()) + cmd.AddCommand(newListOperations()) + cmd.AddCommand(newListResources()) + cmd.AddCommand(newListVersions()) + + // Apply optional overrides to this command. + for _, fn := range cmdOverrides { + fn(cmd) + } + + return cmd +} + +// start complete-version command + +// Slice with functions to override default command behavior. +// Functions can be added from the `init()` function in manually curated files in this directory. +var completeVersionOverrides []func( + *cobra.Command, + *bundle.CompleteVersionRequest, +) + +func newCompleteVersion() *cobra.Command { + cmd := &cobra.Command{} + + var completeVersionReq bundle.CompleteVersionRequest + var completeVersionJson flags.JsonFlag + + cmd.Flags().Var(&completeVersionJson, "json", `either inline JSON string or @path/to/file.json with request body`) + + cmd.Flags().BoolVar(&completeVersionReq.Force, "force", completeVersionReq.Force, `If true, force-completes the version even if the caller is not the original creator.`) + + cmd.Use = "complete-version NAME COMPLETION_REASON" + cmd.Short = `Complete a version.` + cmd.Long = `Complete a version. + + Marks a version as complete and releases the deployment lock. + + The server atomically: 1. Sets the version status to the provided terminal + status. 2. Sets complete_time to the current server timestamp. 3. Releases + the lock on the parent deployment. 4. Updates the parent deployment's status + and last_version_id. + + Arguments: + NAME: The name of the version to complete. Format: + deployments/{deployment_id}/versions/{version_id} + COMPLETION_REASON: The reason for completing the version. Must be a terminal reason: + VERSION_COMPLETE_SUCCESS, VERSION_COMPLETE_FAILURE, or + VERSION_COMPLETE_FORCE_ABORT. + Supported values: [VERSION_COMPLETE_FAILURE, VERSION_COMPLETE_FORCE_ABORT, VERSION_COMPLETE_LEASE_EXPIRED, VERSION_COMPLETE_SUCCESS]` + + cmd.Annotations = make(map[string]string) + cmd.Annotations["launch_stage"] = "PRIVATE_PREVIEW" + cmd.Annotations["launch_stage_display"] = "Private Preview" + + cmd.Args = func(cmd *cobra.Command, args []string) error { + if cmd.Flags().Changed("json") { + err := root.ExactArgs(1)(cmd, args) + if err != nil { + return fmt.Errorf("when --json flag is specified, provide only NAME as positional arguments. Provide 'completion_reason' in your JSON input") + } + return nil + } + check := root.ExactArgs(2) + return check(cmd, args) + } + + cmd.PreRunE = root.MustWorkspaceClient + cmd.RunE = func(cmd *cobra.Command, args []string) (err error) { + ctx := cmd.Context() + w := cmdctx.WorkspaceClient(ctx) + + if cmd.Flags().Changed("json") { + diags := completeVersionJson.Unmarshal(&completeVersionReq) + if diags.HasError() { + return diags.Error() + } + if len(diags) > 0 { + err := cmdio.RenderDiagnostics(ctx, diags) + if err != nil { + return err + } + } + } + completeVersionReq.Name = args[0] + if !cmd.Flags().Changed("json") { + _, err = fmt.Sscan(args[1], &completeVersionReq.CompletionReason) + if err != nil { + return fmt.Errorf("invalid COMPLETION_REASON: %s", args[1]) + } + + } + + response, err := w.Bundle.CompleteVersion(ctx, completeVersionReq) + if err != nil { + return err + } + + return cmdio.Render(ctx, response) + } + + // Disable completions since they are not applicable. + // Can be overridden by manual implementation in `override.go`. + cmd.ValidArgsFunction = cobra.NoFileCompletions + + // Apply optional overrides to this command. + for _, fn := range completeVersionOverrides { + fn(cmd, &completeVersionReq) + } + + return cmd +} + +// start create-deployment command + +// Slice with functions to override default command behavior. +// Functions can be added from the `init()` function in manually curated files in this directory. +var createDeploymentOverrides []func( + *cobra.Command, + *bundle.CreateDeploymentRequest, +) + +func newCreateDeployment() *cobra.Command { + cmd := &cobra.Command{} + + var createDeploymentReq bundle.CreateDeploymentRequest + createDeploymentReq.Deployment = bundle.Deployment{} + var createDeploymentJson flags.JsonFlag + + cmd.Flags().Var(&createDeploymentJson, "json", `either inline JSON string or @path/to/file.json with request body`) + + cmd.Flags().StringVar(&createDeploymentReq.Deployment.DisplayName, "display-name", createDeploymentReq.Deployment.DisplayName, `Human-readable name for the deployment.`) + cmd.Flags().StringVar(&createDeploymentReq.Deployment.TargetName, "target-name", createDeploymentReq.Deployment.TargetName, `The bundle target name associated with this deployment.`) + + cmd.Use = "create-deployment DEPLOYMENT_ID" + cmd.Short = `Create a deployment.` + cmd.Long = `Create a deployment. + + Creates a new deployment in the workspace. + + The caller must provide a deployment_id which becomes the final component of + the deployment's resource name. If a deployment with the same ID already + exists, the server returns ALREADY_EXISTS. + + Arguments: + DEPLOYMENT_ID: The ID to use for the deployment, which will become the final component of + the deployment's resource name (i.e. deployments/{deployment_id}).` + + cmd.Annotations = make(map[string]string) + cmd.Annotations["launch_stage"] = "PRIVATE_PREVIEW" + cmd.Annotations["launch_stage_display"] = "Private Preview" + + cmd.Args = func(cmd *cobra.Command, args []string) error { + check := root.ExactArgs(1) + return check(cmd, args) + } + + cmd.PreRunE = root.MustWorkspaceClient + cmd.RunE = func(cmd *cobra.Command, args []string) (err error) { + ctx := cmd.Context() + w := cmdctx.WorkspaceClient(ctx) + + if cmd.Flags().Changed("json") { + diags := createDeploymentJson.Unmarshal(&createDeploymentReq.Deployment) + if diags.HasError() { + return diags.Error() + } + if len(diags) > 0 { + err := cmdio.RenderDiagnostics(ctx, diags) + if err != nil { + return err + } + } + } + createDeploymentReq.DeploymentId = args[0] + + response, err := w.Bundle.CreateDeployment(ctx, createDeploymentReq) + if err != nil { + return err + } + + return cmdio.Render(ctx, response) + } + + // Disable completions since they are not applicable. + // Can be overridden by manual implementation in `override.go`. + cmd.ValidArgsFunction = cobra.NoFileCompletions + + // Apply optional overrides to this command. + for _, fn := range createDeploymentOverrides { + fn(cmd, &createDeploymentReq) + } + + return cmd +} + +// start create-operation command + +// Slice with functions to override default command behavior. +// Functions can be added from the `init()` function in manually curated files in this directory. +var createOperationOverrides []func( + *cobra.Command, + *bundle.CreateOperationRequest, +) + +func newCreateOperation() *cobra.Command { + cmd := &cobra.Command{} + + var createOperationReq bundle.CreateOperationRequest + createOperationReq.Operation = bundle.Operation{} + var createOperationJson flags.JsonFlag + + cmd.Flags().Var(&createOperationJson, "json", `either inline JSON string or @path/to/file.json with request body`) + + cmd.Use = "create-operation PARENT" + cmd.Short = `Create an operation.` + cmd.Long = `Create an operation. + + Creates a resource operation under a version. + + The caller must provide a resource_key which becomes the final component of + the operation's name. If an operation with the same key already exists under + the version, the server returns ALREADY_EXISTS. + + On success the server also updates the corresponding deployment-level Resource + (creating it if this is the first operation for that resource_key, or removing + it if action_type is DELETE). + + Arguments: + PARENT: The parent version where this operation will be recorded. Format: + deployments/{deployment_id}/versions/{version_id}` + + cmd.Annotations = make(map[string]string) + cmd.Annotations["launch_stage"] = "PRIVATE_PREVIEW" + cmd.Annotations["launch_stage_display"] = "Private Preview" + + cmd.Args = func(cmd *cobra.Command, args []string) error { + if cmd.Flags().Changed("json") { + err := root.ExactArgs(2)(cmd, args) + if err != nil { + return fmt.Errorf("when --json flag is specified, provide only PARENT, RESOURCE_KEY as positional arguments. Provide 'action_type', 'resource_id', 'status' in your JSON input") + } + return nil + } + check := root.ExactArgs(1) + return check(cmd, args) + } + + cmd.PreRunE = root.MustWorkspaceClient + cmd.RunE = func(cmd *cobra.Command, args []string) (err error) { + ctx := cmd.Context() + w := cmdctx.WorkspaceClient(ctx) + + if cmd.Flags().Changed("json") { + diags := createOperationJson.Unmarshal(&createOperationReq.Operation) + if diags.HasError() { + return diags.Error() + } + if len(diags) > 0 { + err := cmdio.RenderDiagnostics(ctx, diags) + if err != nil { + return err + } + } + } else { + return fmt.Errorf("please provide command input in JSON format by specifying the --json flag") + } + createOperationReq.Parent = args[0] + + response, err := w.Bundle.CreateOperation(ctx, createOperationReq) + if err != nil { + return err + } + + return cmdio.Render(ctx, response) + } + + // Disable completions since they are not applicable. + // Can be overridden by manual implementation in `override.go`. + cmd.ValidArgsFunction = cobra.NoFileCompletions + + // Apply optional overrides to this command. + for _, fn := range createOperationOverrides { + fn(cmd, &createOperationReq) + } + + return cmd +} + +// start create-version command + +// Slice with functions to override default command behavior. +// Functions can be added from the `init()` function in manually curated files in this directory. +var createVersionOverrides []func( + *cobra.Command, + *bundle.CreateVersionRequest, +) + +func newCreateVersion() *cobra.Command { + cmd := &cobra.Command{} + + var createVersionReq bundle.CreateVersionRequest + createVersionReq.Version = bundle.Version{} + var createVersionJson flags.JsonFlag + + cmd.Flags().Var(&createVersionJson, "json", `either inline JSON string or @path/to/file.json with request body`) + + cmd.Flags().StringVar(&createVersionReq.Version.DisplayName, "display-name", createVersionReq.Version.DisplayName, `Display name for the deployment, captured at the time of this version.`) + cmd.Flags().StringVar(&createVersionReq.Version.TargetName, "target-name", createVersionReq.Version.TargetName, `Target name of the deployment, captured at the time of this version.`) + + cmd.Use = "create-version PARENT VERSION_ID CLI_VERSION VERSION_TYPE" + cmd.Short = `Create a version.` + cmd.Long = `Create a version. + + Creates a new version under a deployment. + + Creating a version acquires an exclusive lock on the deployment, preventing + concurrent deploys. The caller provides a version_id which the server + validates equals last_version_id + 1 on the deployment. + + Arguments: + PARENT: The parent deployment where this version will be created. Format: + deployments/{deployment_id} + VERSION_ID: The version ID the caller expects to create. The server validates this + equals last_version_id + 1 on the deployment. If it doesn't match, the + server returns ABORTED. + CLI_VERSION: CLI version used to initiate the version. + VERSION_TYPE: Type of version (deploy or destroy). + Supported values: [VERSION_TYPE_DEPLOY, VERSION_TYPE_DESTROY]` + + cmd.Annotations = make(map[string]string) + cmd.Annotations["launch_stage"] = "PRIVATE_PREVIEW" + cmd.Annotations["launch_stage_display"] = "Private Preview" + + cmd.Args = func(cmd *cobra.Command, args []string) error { + if cmd.Flags().Changed("json") { + err := root.ExactArgs(2)(cmd, args) + if err != nil { + return fmt.Errorf("when --json flag is specified, provide only PARENT, VERSION_ID as positional arguments. Provide 'cli_version', 'version_type' in your JSON input") + } + return nil + } + check := root.ExactArgs(4) + return check(cmd, args) + } + + cmd.PreRunE = root.MustWorkspaceClient + cmd.RunE = func(cmd *cobra.Command, args []string) (err error) { + ctx := cmd.Context() + w := cmdctx.WorkspaceClient(ctx) + + if cmd.Flags().Changed("json") { + diags := createVersionJson.Unmarshal(&createVersionReq.Version) + if diags.HasError() { + return diags.Error() + } + if len(diags) > 0 { + err := cmdio.RenderDiagnostics(ctx, diags) + if err != nil { + return err + } + } + } + createVersionReq.Parent = args[0] + createVersionReq.VersionId = args[1] + if !cmd.Flags().Changed("json") { + createVersionReq.Version.CliVersion = args[2] + } + if !cmd.Flags().Changed("json") { + _, err = fmt.Sscan(args[3], &createVersionReq.Version.VersionType) + if err != nil { + return fmt.Errorf("invalid VERSION_TYPE: %s", args[3]) + } + + } + + response, err := w.Bundle.CreateVersion(ctx, createVersionReq) + if err != nil { + return err + } + + return cmdio.Render(ctx, response) + } + + // Disable completions since they are not applicable. + // Can be overridden by manual implementation in `override.go`. + cmd.ValidArgsFunction = cobra.NoFileCompletions + + // Apply optional overrides to this command. + for _, fn := range createVersionOverrides { + fn(cmd, &createVersionReq) + } + + return cmd +} + +// start delete-deployment command + +// Slice with functions to override default command behavior. +// Functions can be added from the `init()` function in manually curated files in this directory. +var deleteDeploymentOverrides []func( + *cobra.Command, + *bundle.DeleteDeploymentRequest, +) + +func newDeleteDeployment() *cobra.Command { + cmd := &cobra.Command{} + + var deleteDeploymentReq bundle.DeleteDeploymentRequest + + cmd.Use = "delete-deployment NAME" + cmd.Short = `Delete a deployment.` + cmd.Long = `Delete a deployment. + + Deletes a deployment. + + The deployment is marked as deleted. It and all its children (versions and + their operations) will be permanently deleted after the retention policy + expires. If the deployment has an in-progress version, the server returns + RESOURCE_CONFLICT. + + Arguments: + NAME: Resource name of the deployment to delete. Format: + deployments/{deployment_id}` + + cmd.Annotations = make(map[string]string) + cmd.Annotations["launch_stage"] = "PRIVATE_PREVIEW" + cmd.Annotations["launch_stage_display"] = "Private Preview" + + cmd.Args = func(cmd *cobra.Command, args []string) error { + check := root.ExactArgs(1) + return check(cmd, args) + } + + cmd.PreRunE = root.MustWorkspaceClient + cmd.RunE = func(cmd *cobra.Command, args []string) (err error) { + ctx := cmd.Context() + w := cmdctx.WorkspaceClient(ctx) + + deleteDeploymentReq.Name = args[0] + + err = w.Bundle.DeleteDeployment(ctx, deleteDeploymentReq) + if err != nil { + return err + } + return nil + } + + // Disable completions since they are not applicable. + // Can be overridden by manual implementation in `override.go`. + cmd.ValidArgsFunction = cobra.NoFileCompletions + + // Apply optional overrides to this command. + for _, fn := range deleteDeploymentOverrides { + fn(cmd, &deleteDeploymentReq) + } + + return cmd +} + +// start get-deployment command + +// Slice with functions to override default command behavior. +// Functions can be added from the `init()` function in manually curated files in this directory. +var getDeploymentOverrides []func( + *cobra.Command, + *bundle.GetDeploymentRequest, +) + +func newGetDeployment() *cobra.Command { + cmd := &cobra.Command{} + + var getDeploymentReq bundle.GetDeploymentRequest + + cmd.Use = "get-deployment NAME" + cmd.Short = `Get a deployment.` + cmd.Long = `Get a deployment. + + Retrieves a deployment by its resource name. + + Arguments: + NAME: Resource name of the deployment to retrieve. Format: + deployments/{deployment_id}` + + cmd.Annotations = make(map[string]string) + cmd.Annotations["launch_stage"] = "PRIVATE_PREVIEW" + cmd.Annotations["launch_stage_display"] = "Private Preview" + + cmd.Args = func(cmd *cobra.Command, args []string) error { + check := root.ExactArgs(1) + return check(cmd, args) + } + + cmd.PreRunE = root.MustWorkspaceClient + cmd.RunE = func(cmd *cobra.Command, args []string) (err error) { + ctx := cmd.Context() + w := cmdctx.WorkspaceClient(ctx) + + getDeploymentReq.Name = args[0] + + response, err := w.Bundle.GetDeployment(ctx, getDeploymentReq) + if err != nil { + return err + } + + return cmdio.Render(ctx, response) + } + + // Disable completions since they are not applicable. + // Can be overridden by manual implementation in `override.go`. + cmd.ValidArgsFunction = cobra.NoFileCompletions + + // Apply optional overrides to this command. + for _, fn := range getDeploymentOverrides { + fn(cmd, &getDeploymentReq) + } + + return cmd +} + +// start get-operation command + +// Slice with functions to override default command behavior. +// Functions can be added from the `init()` function in manually curated files in this directory. +var getOperationOverrides []func( + *cobra.Command, + *bundle.GetOperationRequest, +) + +func newGetOperation() *cobra.Command { + cmd := &cobra.Command{} + + var getOperationReq bundle.GetOperationRequest + + cmd.Use = "get-operation NAME" + cmd.Short = `Get an operation.` + cmd.Long = `Get an operation. + + Retrieves a resource operation by its resource name. + + Arguments: + NAME: The name of the resource operation to retrieve. Format: + deployments/{deployment_id}/versions/{version_id}/operations/{resource_key}` + + cmd.Annotations = make(map[string]string) + cmd.Annotations["launch_stage"] = "PRIVATE_PREVIEW" + cmd.Annotations["launch_stage_display"] = "Private Preview" + + cmd.Args = func(cmd *cobra.Command, args []string) error { + check := root.ExactArgs(1) + return check(cmd, args) + } + + cmd.PreRunE = root.MustWorkspaceClient + cmd.RunE = func(cmd *cobra.Command, args []string) (err error) { + ctx := cmd.Context() + w := cmdctx.WorkspaceClient(ctx) + + getOperationReq.Name = args[0] + + response, err := w.Bundle.GetOperation(ctx, getOperationReq) + if err != nil { + return err + } + + return cmdio.Render(ctx, response) + } + + // Disable completions since they are not applicable. + // Can be overridden by manual implementation in `override.go`. + cmd.ValidArgsFunction = cobra.NoFileCompletions + + // Apply optional overrides to this command. + for _, fn := range getOperationOverrides { + fn(cmd, &getOperationReq) + } + + return cmd +} + +// start get-resource command + +// Slice with functions to override default command behavior. +// Functions can be added from the `init()` function in manually curated files in this directory. +var getResourceOverrides []func( + *cobra.Command, + *bundle.GetResourceRequest, +) + +func newGetResource() *cobra.Command { + cmd := &cobra.Command{} + + var getResourceReq bundle.GetResourceRequest + + cmd.Use = "get-resource NAME" + cmd.Short = `Get a resource.` + cmd.Long = `Get a resource. + + Retrieves a deployment resource by its resource name. + + Arguments: + NAME: The name of the resource to retrieve. Format: + deployments/{deployment_id}/resources/{resource_key}` + + cmd.Annotations = make(map[string]string) + cmd.Annotations["launch_stage"] = "PRIVATE_PREVIEW" + cmd.Annotations["launch_stage_display"] = "Private Preview" + + cmd.Args = func(cmd *cobra.Command, args []string) error { + check := root.ExactArgs(1) + return check(cmd, args) + } + + cmd.PreRunE = root.MustWorkspaceClient + cmd.RunE = func(cmd *cobra.Command, args []string) (err error) { + ctx := cmd.Context() + w := cmdctx.WorkspaceClient(ctx) + + getResourceReq.Name = args[0] + + response, err := w.Bundle.GetResource(ctx, getResourceReq) + if err != nil { + return err + } + + return cmdio.Render(ctx, response) + } + + // Disable completions since they are not applicable. + // Can be overridden by manual implementation in `override.go`. + cmd.ValidArgsFunction = cobra.NoFileCompletions + + // Apply optional overrides to this command. + for _, fn := range getResourceOverrides { + fn(cmd, &getResourceReq) + } + + return cmd +} + +// start get-version command + +// Slice with functions to override default command behavior. +// Functions can be added from the `init()` function in manually curated files in this directory. +var getVersionOverrides []func( + *cobra.Command, + *bundle.GetVersionRequest, +) + +func newGetVersion() *cobra.Command { + cmd := &cobra.Command{} + + var getVersionReq bundle.GetVersionRequest + + cmd.Use = "get-version NAME" + cmd.Short = `Get a version.` + cmd.Long = `Get a version. + + Retrieves a version by its resource name. + + Arguments: + NAME: The name of the version to retrieve. Format: + deployments/{deployment_id}/versions/{version_id}` + + cmd.Annotations = make(map[string]string) + cmd.Annotations["launch_stage"] = "PRIVATE_PREVIEW" + cmd.Annotations["launch_stage_display"] = "Private Preview" + + cmd.Args = func(cmd *cobra.Command, args []string) error { + check := root.ExactArgs(1) + return check(cmd, args) + } + + cmd.PreRunE = root.MustWorkspaceClient + cmd.RunE = func(cmd *cobra.Command, args []string) (err error) { + ctx := cmd.Context() + w := cmdctx.WorkspaceClient(ctx) + + getVersionReq.Name = args[0] + + response, err := w.Bundle.GetVersion(ctx, getVersionReq) + if err != nil { + return err + } + + return cmdio.Render(ctx, response) + } + + // Disable completions since they are not applicable. + // Can be overridden by manual implementation in `override.go`. + cmd.ValidArgsFunction = cobra.NoFileCompletions + + // Apply optional overrides to this command. + for _, fn := range getVersionOverrides { + fn(cmd, &getVersionReq) + } + + return cmd +} + +// start heartbeat command + +// Slice with functions to override default command behavior. +// Functions can be added from the `init()` function in manually curated files in this directory. +var heartbeatOverrides []func( + *cobra.Command, + *bundle.HeartbeatRequest, +) + +func newHeartbeat() *cobra.Command { + cmd := &cobra.Command{} + + var heartbeatReq bundle.HeartbeatRequest + + cmd.Use = "heartbeat NAME" + cmd.Short = `Send a version heartbeat.` + cmd.Long = `Send a version heartbeat. + + Sends a heartbeat to renew the lock held by a version. + + The server validates that the version is the active (non-terminal) version on + the parent deployment and resets the lock expiry. If the lock has already + expired or the version is no longer active, the server returns ABORTED. + + Arguments: + NAME: The version whose lock to renew. Format: + deployments/{deployment_id}/versions/{version_id}` + + cmd.Annotations = make(map[string]string) + cmd.Annotations["launch_stage"] = "PRIVATE_PREVIEW" + cmd.Annotations["launch_stage_display"] = "Private Preview" + + cmd.Args = func(cmd *cobra.Command, args []string) error { + check := root.ExactArgs(1) + return check(cmd, args) + } + + cmd.PreRunE = root.MustWorkspaceClient + cmd.RunE = func(cmd *cobra.Command, args []string) (err error) { + ctx := cmd.Context() + w := cmdctx.WorkspaceClient(ctx) + + heartbeatReq.Name = args[0] + + response, err := w.Bundle.Heartbeat(ctx, heartbeatReq) + if err != nil { + return err + } + + return cmdio.Render(ctx, response) + } + + // Disable completions since they are not applicable. + // Can be overridden by manual implementation in `override.go`. + cmd.ValidArgsFunction = cobra.NoFileCompletions + + // Apply optional overrides to this command. + for _, fn := range heartbeatOverrides { + fn(cmd, &heartbeatReq) + } + + return cmd +} + +// start list-deployments command + +// Slice with functions to override default command behavior. +// Functions can be added from the `init()` function in manually curated files in this directory. +var listDeploymentsOverrides []func( + *cobra.Command, + *bundle.ListDeploymentsRequest, +) + +func newListDeployments() *cobra.Command { + cmd := &cobra.Command{} + + var listDeploymentsReq bundle.ListDeploymentsRequest + // Registered for all paginated methods. Validated at call time in the + // method-call template. Paginated list methods never have Wait or LRO + // branches, so the method-call path is always reached. + var listDeploymentsLimit int + + cmd.Flags().IntVar(&listDeploymentsReq.PageSize, "page-size", listDeploymentsReq.PageSize, `The maximum number of deployments to return.`) + + // Limit flag for total result capping. + cmd.Flags().IntVar(&listDeploymentsLimit, "limit", 0, `Maximum number of results to return.`) + + // Hidden pagination flags (internal API parameters). + cmd.Flags().StringVar(&listDeploymentsReq.PageToken, "page-token", listDeploymentsReq.PageToken, `Pagination token.`) + cmd.Flags().Lookup("page-token").Hidden = true + + cmd.Use = "list-deployments" + cmd.Short = `List deployments.` + cmd.Long = `List deployments. + + Lists deployments in the workspace.` + + cmd.Annotations = make(map[string]string) + cmd.Annotations["launch_stage"] = "PRIVATE_PREVIEW" + cmd.Annotations["launch_stage_display"] = "Private Preview" + + cmd.Args = func(cmd *cobra.Command, args []string) error { + check := root.ExactArgs(0) + return check(cmd, args) + } + + cmd.PreRunE = root.MustWorkspaceClient + cmd.RunE = func(cmd *cobra.Command, args []string) (err error) { + ctx := cmd.Context() + w := cmdctx.WorkspaceClient(ctx) + + response := w.Bundle.ListDeployments(ctx, listDeploymentsReq) + if listDeploymentsLimit < 0 { + return fmt.Errorf("--limit must be a non-negative integer, got %d", listDeploymentsLimit) + } + if listDeploymentsLimit > 0 { + ctx = cmdio.WithLimit(ctx, listDeploymentsLimit) + } + + return cmdio.RenderIterator(ctx, response) + } + + // Disable completions since they are not applicable. + // Can be overridden by manual implementation in `override.go`. + cmd.ValidArgsFunction = cobra.NoFileCompletions + + // Apply optional overrides to this command. + for _, fn := range listDeploymentsOverrides { + fn(cmd, &listDeploymentsReq) + } + + return cmd +} + +// start list-operations command + +// Slice with functions to override default command behavior. +// Functions can be added from the `init()` function in manually curated files in this directory. +var listOperationsOverrides []func( + *cobra.Command, + *bundle.ListOperationsRequest, +) + +func newListOperations() *cobra.Command { + cmd := &cobra.Command{} + + var listOperationsReq bundle.ListOperationsRequest + // Registered for all paginated methods. Validated at call time in the + // method-call template. Paginated list methods never have Wait or LRO + // branches, so the method-call path is always reached. + var listOperationsLimit int + + cmd.Flags().IntVar(&listOperationsReq.PageSize, "page-size", listOperationsReq.PageSize, `The maximum number of operations to return.`) + + // Limit flag for total result capping. + cmd.Flags().IntVar(&listOperationsLimit, "limit", 0, `Maximum number of results to return.`) + + // Hidden pagination flags (internal API parameters). + cmd.Flags().StringVar(&listOperationsReq.PageToken, "page-token", listOperationsReq.PageToken, `Pagination token.`) + cmd.Flags().Lookup("page-token").Hidden = true + + cmd.Use = "list-operations PARENT" + cmd.Short = `List operations.` + cmd.Long = `List operations. + + Lists resource operations under a version. + + Arguments: + PARENT: The parent version. Format: + deployments/{deployment_id}/versions/{version_id}` + + cmd.Annotations = make(map[string]string) + cmd.Annotations["launch_stage"] = "PRIVATE_PREVIEW" + cmd.Annotations["launch_stage_display"] = "Private Preview" + + cmd.Args = func(cmd *cobra.Command, args []string) error { + check := root.ExactArgs(1) + return check(cmd, args) + } + + cmd.PreRunE = root.MustWorkspaceClient + cmd.RunE = func(cmd *cobra.Command, args []string) (err error) { + ctx := cmd.Context() + w := cmdctx.WorkspaceClient(ctx) + + listOperationsReq.Parent = args[0] + + response := w.Bundle.ListOperations(ctx, listOperationsReq) + if listOperationsLimit < 0 { + return fmt.Errorf("--limit must be a non-negative integer, got %d", listOperationsLimit) + } + if listOperationsLimit > 0 { + ctx = cmdio.WithLimit(ctx, listOperationsLimit) + } + + return cmdio.RenderIterator(ctx, response) + } + + // Disable completions since they are not applicable. + // Can be overridden by manual implementation in `override.go`. + cmd.ValidArgsFunction = cobra.NoFileCompletions + + // Apply optional overrides to this command. + for _, fn := range listOperationsOverrides { + fn(cmd, &listOperationsReq) + } + + return cmd +} + +// start list-resources command + +// Slice with functions to override default command behavior. +// Functions can be added from the `init()` function in manually curated files in this directory. +var listResourcesOverrides []func( + *cobra.Command, + *bundle.ListResourcesRequest, +) + +func newListResources() *cobra.Command { + cmd := &cobra.Command{} + + var listResourcesReq bundle.ListResourcesRequest + // Registered for all paginated methods. Validated at call time in the + // method-call template. Paginated list methods never have Wait or LRO + // branches, so the method-call path is always reached. + var listResourcesLimit int + + cmd.Flags().IntVar(&listResourcesReq.PageSize, "page-size", listResourcesReq.PageSize, `The maximum number of resources to return.`) + + // Limit flag for total result capping. + cmd.Flags().IntVar(&listResourcesLimit, "limit", 0, `Maximum number of results to return.`) + + // Hidden pagination flags (internal API parameters). + cmd.Flags().StringVar(&listResourcesReq.PageToken, "page-token", listResourcesReq.PageToken, `Pagination token.`) + cmd.Flags().Lookup("page-token").Hidden = true + + cmd.Use = "list-resources PARENT" + cmd.Short = `List resources.` + cmd.Long = `List resources. + + Lists resources under a deployment. + + Arguments: + PARENT: The parent deployment. Format: deployments/{deployment_id}` + + cmd.Annotations = make(map[string]string) + cmd.Annotations["launch_stage"] = "PRIVATE_PREVIEW" + cmd.Annotations["launch_stage_display"] = "Private Preview" + + cmd.Args = func(cmd *cobra.Command, args []string) error { + check := root.ExactArgs(1) + return check(cmd, args) + } + + cmd.PreRunE = root.MustWorkspaceClient + cmd.RunE = func(cmd *cobra.Command, args []string) (err error) { + ctx := cmd.Context() + w := cmdctx.WorkspaceClient(ctx) + + listResourcesReq.Parent = args[0] + + response := w.Bundle.ListResources(ctx, listResourcesReq) + if listResourcesLimit < 0 { + return fmt.Errorf("--limit must be a non-negative integer, got %d", listResourcesLimit) + } + if listResourcesLimit > 0 { + ctx = cmdio.WithLimit(ctx, listResourcesLimit) + } + + return cmdio.RenderIterator(ctx, response) + } + + // Disable completions since they are not applicable. + // Can be overridden by manual implementation in `override.go`. + cmd.ValidArgsFunction = cobra.NoFileCompletions + + // Apply optional overrides to this command. + for _, fn := range listResourcesOverrides { + fn(cmd, &listResourcesReq) + } + + return cmd +} + +// start list-versions command + +// Slice with functions to override default command behavior. +// Functions can be added from the `init()` function in manually curated files in this directory. +var listVersionsOverrides []func( + *cobra.Command, + *bundle.ListVersionsRequest, +) + +func newListVersions() *cobra.Command { + cmd := &cobra.Command{} + + var listVersionsReq bundle.ListVersionsRequest + // Registered for all paginated methods. Validated at call time in the + // method-call template. Paginated list methods never have Wait or LRO + // branches, so the method-call path is always reached. + var listVersionsLimit int + + cmd.Flags().IntVar(&listVersionsReq.PageSize, "page-size", listVersionsReq.PageSize, `The maximum number of versions to return.`) + + // Limit flag for total result capping. + cmd.Flags().IntVar(&listVersionsLimit, "limit", 0, `Maximum number of results to return.`) + + // Hidden pagination flags (internal API parameters). + cmd.Flags().StringVar(&listVersionsReq.PageToken, "page-token", listVersionsReq.PageToken, `Pagination token.`) + cmd.Flags().Lookup("page-token").Hidden = true + + cmd.Use = "list-versions PARENT" + cmd.Short = `List versions.` + cmd.Long = `List versions. + + Lists versions under a deployment, ordered by version_id descending (most + recent first). + + Arguments: + PARENT: The parent deployment. Format: deployments/{deployment_id}` + + cmd.Annotations = make(map[string]string) + cmd.Annotations["launch_stage"] = "PRIVATE_PREVIEW" + cmd.Annotations["launch_stage_display"] = "Private Preview" + + cmd.Args = func(cmd *cobra.Command, args []string) error { + check := root.ExactArgs(1) + return check(cmd, args) + } + + cmd.PreRunE = root.MustWorkspaceClient + cmd.RunE = func(cmd *cobra.Command, args []string) (err error) { + ctx := cmd.Context() + w := cmdctx.WorkspaceClient(ctx) + + listVersionsReq.Parent = args[0] + + response := w.Bundle.ListVersions(ctx, listVersionsReq) + if listVersionsLimit < 0 { + return fmt.Errorf("--limit must be a non-negative integer, got %d", listVersionsLimit) + } + if listVersionsLimit > 0 { + ctx = cmdio.WithLimit(ctx, listVersionsLimit) + } + + return cmdio.RenderIterator(ctx, response) + } + + // Disable completions since they are not applicable. + // Can be overridden by manual implementation in `override.go`. + cmd.ValidArgsFunction = cobra.NoFileCompletions + + // Apply optional overrides to this command. + for _, fn := range listVersionsOverrides { + fn(cmd, &listVersionsReq) + } + + return cmd +} + +// end service Bundle diff --git a/cmd/workspace/cmd.go b/cmd/workspace/cmd.go index 189c08c9d5a..d944c6c90a2 100755 --- a/cmd/workspace/cmd.go +++ b/cmd/workspace/cmd.go @@ -11,6 +11,7 @@ import ( apps "github.com/databricks/cli/cmd/workspace/apps" apps_settings "github.com/databricks/cli/cmd/workspace/apps-settings" artifact_allowlists "github.com/databricks/cli/cmd/workspace/artifact-allowlists" + bundle "github.com/databricks/cli/cmd/workspace/bundle" catalogs "github.com/databricks/cli/cmd/workspace/catalogs" clean_room_asset_revisions "github.com/databricks/cli/cmd/workspace/clean-room-asset-revisions" clean_room_assets "github.com/databricks/cli/cmd/workspace/clean-room-assets" @@ -142,6 +143,7 @@ func All() []*cobra.Command { out = append(out, apps.New()) out = append(out, apps_settings.New()) out = append(out, artifact_allowlists.New()) + out = append(out, bundle.New()) out = append(out, catalogs.New()) out = append(out, clean_room_asset_revisions.New()) out = append(out, clean_room_assets.New()) diff --git a/cmd/workspace/consumer-listings/consumer-listings.go b/cmd/workspace/consumer-listings/consumer-listings.go index c8b257a51d4..f8113e3532a 100755 --- a/cmd/workspace/consumer-listings/consumer-listings.go +++ b/cmd/workspace/consumer-listings/consumer-listings.go @@ -201,7 +201,7 @@ func newList() *cobra.Command { cmd.Flags().BoolVar(&listReq.IsStaffPick, "is-staff-pick", listReq.IsStaffPick, `Filters each listing based on whether it is a staff pick.`) cmd.Flags().IntVar(&listReq.PageSize, "page-size", listReq.PageSize, ``) // TODO: array: provider_ids - // TODO: array: tags + // TODO: complex arg: tags // Limit flag for total result capping. cmd.Flags().IntVar(&listLimit, "limit", 0, `Maximum number of results to return.`) diff --git a/cmd/workspace/current-user/current-user.go b/cmd/workspace/current-user/current-user.go index 6bdd0075683..de32f481229 100755 --- a/cmd/workspace/current-user/current-user.go +++ b/cmd/workspace/current-user/current-user.go @@ -6,6 +6,7 @@ import ( "github.com/databricks/cli/cmd/root" "github.com/databricks/cli/libs/cmdctx" "github.com/databricks/cli/libs/cmdio" + "github.com/databricks/databricks-sdk-go/service/iam" "github.com/spf13/cobra" ) @@ -46,11 +47,17 @@ This API allows retrieving information about currently authenticated user or // Functions can be added from the `init()` function in manually curated files in this directory. var meOverrides []func( *cobra.Command, + *iam.MeRequest, ) func newMe() *cobra.Command { cmd := &cobra.Command{} + var meReq iam.MeRequest + + cmd.Flags().StringVar(&meReq.Attributes, "attributes", meReq.Attributes, `Comma-separated list of attributes to return in response.`) + cmd.Flags().StringVar(&meReq.ExcludedAttributes, "excluded-attributes", meReq.ExcludedAttributes, `Comma-separated list of attributes to exclude in response.`) + cmd.Use = "me" cmd.Short = `*Public Preview* Get current user info.` cmd.Long = `This command is in Public Preview and may change without notice. @@ -63,11 +70,17 @@ Get current user info. cmd.Annotations["launch_stage"] = "PUBLIC_PREVIEW" cmd.Annotations["launch_stage_display"] = "Public Preview" + cmd.Args = func(cmd *cobra.Command, args []string) error { + check := root.ExactArgs(0) + return check(cmd, args) + } + cmd.PreRunE = root.MustWorkspaceClient cmd.RunE = func(cmd *cobra.Command, args []string) (err error) { ctx := cmd.Context() w := cmdctx.WorkspaceClient(ctx) - response, err := w.CurrentUser.Me(ctx) + + response, err := w.CurrentUser.Me(ctx, meReq) if err != nil { return err } @@ -81,7 +94,7 @@ Get current user info. // Apply optional overrides to this command. for _, fn := range meOverrides { - fn(cmd) + fn(cmd, &meReq) } return cmd diff --git a/cmd/workspace/environments/environments.go b/cmd/workspace/environments/environments.go index 46de012fefc..97fc52874a7 100755 --- a/cmd/workspace/environments/environments.go +++ b/cmd/workspace/environments/environments.go @@ -466,7 +466,19 @@ func newListWorkspaceBaseEnvironments() *cobra.Command { cmd.Short = `List workspace base environments.` cmd.Long = `List workspace base environments. - Lists all WorkspaceBaseEnvironments in the workspace.` + Lists all WorkspaceBaseEnvironments in the workspace. + + Databricks provides the following base environments: + + - workspace-base-environments/databricks_ai_...: includes popular AI and + deep learning packages for serverless GPU compute. + + - workspace-base-environments/databricks_ml_...: includes popular ML + packages for serverless compute. + + Databricks-provided base environments are versioned. For example, + workspace-base-environments/databricks_ml_v5 corresponds to the ML + environment built on environment version 5.` cmd.Annotations = make(map[string]string) cmd.Annotations["launch_stage"] = "GA" diff --git a/cmd/workspace/experiments/experiments.go b/cmd/workspace/experiments/experiments.go index 23b9176cba7..6875a03b588 100755 --- a/cmd/workspace/experiments/experiments.go +++ b/cmd/workspace/experiments/experiments.go @@ -117,6 +117,8 @@ func newCreateExperiment() *cobra.Command { exists. Throws RESOURCE_ALREADY_EXISTS if an experiment with the given name exists. + Note: In some contexts, this error may be remapped to ALREADY_EXISTS. To be + safe, clients should check for both error codes. Arguments: NAME: Experiment name.` diff --git a/cmd/workspace/feature-engineering/feature-engineering.go b/cmd/workspace/feature-engineering/feature-engineering.go index fe4c98e74dc..a9d665fc17b 100755 --- a/cmd/workspace/feature-engineering/feature-engineering.go +++ b/cmd/workspace/feature-engineering/feature-engineering.go @@ -93,7 +93,9 @@ func newCreateFeature() *cobra.Command { Create a Feature. Arguments: - FULL_NAME: The full three-part name (catalog, schema, name) of the feature. + FULL_NAME: The full three-part name (catalog, schema, name) of the feature. This is + the feature's resource identifier; the catalog_name, schema_name, and name + fields below are OUTPUT_ONLY decomposed views of this value. SOURCE: The data source of the feature. FUNCTION: The function by which the feature is computed.` @@ -747,18 +749,22 @@ func newListFeatures() *cobra.Command { cmd.Flags().StringVar(&listFeaturesReq.PageToken, "page-token", listFeaturesReq.PageToken, `Pagination token.`) cmd.Flags().Lookup("page-token").Hidden = true - cmd.Use = "list-features" + cmd.Use = "list-features CATALOG_NAME SCHEMA_NAME" cmd.Short = `List features.` cmd.Long = `List features. - List Features.` + List Features. + + Arguments: + CATALOG_NAME: Name of parent catalog for features of interest. + SCHEMA_NAME: Name of parent schema relative to its parent catalog.` cmd.Annotations = make(map[string]string) cmd.Annotations["launch_stage"] = "PRIVATE_PREVIEW" cmd.Annotations["launch_stage_display"] = "Private Preview" cmd.Args = func(cmd *cobra.Command, args []string) error { - check := root.ExactArgs(0) + check := root.ExactArgs(2) return check(cmd, args) } @@ -767,6 +773,9 @@ func newListFeatures() *cobra.Command { ctx := cmd.Context() w := cmdctx.WorkspaceClient(ctx) + listFeaturesReq.CatalogName = args[0] + listFeaturesReq.SchemaName = args[1] + response := w.FeatureEngineering.ListFeatures(ctx, listFeaturesReq) if listFeaturesLimit < 0 { return fmt.Errorf("--limit must be a non-negative integer, got %d", listFeaturesLimit) @@ -964,7 +973,9 @@ func newUpdateFeature() *cobra.Command { Update a Feature. Arguments: - FULL_NAME: The full three-part name (catalog, schema, name) of the feature. + FULL_NAME: The full three-part name (catalog, schema, name) of the feature. This is + the feature's resource identifier; the catalog_name, schema_name, and name + fields below are OUTPUT_ONLY decomposed views of this value. UPDATE_MASK: The list of fields to update. SOURCE: The data source of the feature. FUNCTION: The function by which the feature is computed.` diff --git a/cmd/workspace/groups.go b/cmd/workspace/groups.go index 8dd096ee630..985b4815695 100644 --- a/cmd/workspace/groups.go +++ b/cmd/workspace/groups.go @@ -108,5 +108,9 @@ func Groups() []cobra.Group { ID: "environments", Title: "Environments", }, + { + ID: "bundle", + Title: "Bundle", + }, } } diff --git a/cmd/workspace/jobs/jobs.go b/cmd/workspace/jobs/jobs.go index 0fb5081bafb..be26fc586d0 100755 --- a/cmd/workspace/jobs/jobs.go +++ b/cmd/workspace/jobs/jobs.go @@ -635,6 +635,7 @@ func newGet() *cobra.Command { var getReq jobs.GetJobRequest + cmd.Flags().BoolVar(&getReq.IncludeTriggerState, "include-trigger-state", getReq.IncludeTriggerState, `Flag that indicates that trigger state should be included in the response.`) cmd.Flags().StringVar(&getReq.PageToken, "page-token", getReq.PageToken, `Use next_page_token returned from the previous GetJob response to request the next page of the job's array properties.`) cmd.Use = "get JOB_ID" diff --git a/cmd/workspace/postgres/postgres.go b/cmd/workspace/postgres/postgres.go index 5367e36f3f3..df28feb58ce 100755 --- a/cmd/workspace/postgres/postgres.go +++ b/cmd/workspace/postgres/postgres.go @@ -76,6 +76,7 @@ Use the Postgres API to create and manage Lakebase Autoscaling Postgres cmd.AddCommand(newListEndpoints()) cmd.AddCommand(newListProjects()) cmd.AddCommand(newListRoles()) + cmd.AddCommand(newUndeleteBranch()) cmd.AddCommand(newUndeleteProject()) cmd.AddCommand(newUpdateBranch()) cmd.AddCommand(newUpdateDatabase()) @@ -1006,6 +1007,8 @@ func newDeleteBranch() *cobra.Command { cmd.Flags().BoolVar(&deleteBranchSkipWait, "no-wait", deleteBranchSkipWait, `do not wait to reach DONE state`) cmd.Flags().DurationVar(&deleteBranchTimeout, "timeout", 0, `maximum amount of time to reach DONE state`) + cmd.Flags().BoolVar(&deleteBranchReq.Purge, "purge", deleteBranchReq.Purge, `If true, permanently delete the branch; if false, soft delete.`) + cmd.Use = "delete-branch NAME" cmd.Short = `*Beta* Delete a Branch.` cmd.Long = `This command is in Beta and may change without notice. @@ -1743,9 +1746,8 @@ func newGenerateDatabaseCredential() *cobra.Command { Generate OAuth credentials for a Postgres database. Arguments: - ENDPOINT: This field is not yet supported. The endpoint for which this credential - will be generated. Format: - projects/{project_id}/branches/{branch_id}/endpoints/{endpoint_id}` + ENDPOINT: The endpoint resource name for which this credential will be generated. + Format: projects/{project_id}/branches/{branch_id}/endpoints/{endpoint_id}` cmd.Annotations = make(map[string]string) cmd.Annotations["launch_stage"] = "PUBLIC_BETA" @@ -2319,6 +2321,7 @@ func newListBranches() *cobra.Command { var listBranchesLimit int cmd.Flags().IntVar(&listBranchesReq.PageSize, "page-size", listBranchesReq.PageSize, `Upper bound for items returned.`) + cmd.Flags().BoolVar(&listBranchesReq.ShowDeleted, "show-deleted", listBranchesReq.ShowDeleted, `Whether to include soft-deleted branches in the response.`) // Limit flag for total result capping. cmd.Flags().IntVar(&listBranchesLimit, "limit", 0, `Maximum number of results to return.`) @@ -2687,6 +2690,111 @@ List Postgres Roles for a Branch. return cmd } +// start undelete-branch command + +// Slice with functions to override default command behavior. +// Functions can be added from the `init()` function in manually curated files in this directory. +var undeleteBranchOverrides []func( + *cobra.Command, + *postgres.UndeleteBranchRequest, +) + +func newUndeleteBranch() *cobra.Command { + cmd := &cobra.Command{} + + var undeleteBranchReq postgres.UndeleteBranchRequest + + var undeleteBranchSkipWait bool + var undeleteBranchTimeout time.Duration + + cmd.Flags().BoolVar(&undeleteBranchSkipWait, "no-wait", undeleteBranchSkipWait, `do not wait to reach DONE state`) + cmd.Flags().DurationVar(&undeleteBranchTimeout, "timeout", 0, `maximum amount of time to reach DONE state`) + + cmd.Use = "undelete-branch NAME" + cmd.Short = `Undelete a Branch.` + cmd.Long = `Undelete a Branch. + + Undeletes the specified database branch. + + This is a long-running operation. By default, the command waits for the + operation to complete. Use --no-wait to return immediately with the raw + operation details. The operation's 'name' field can then be used to poll for + completion using the get-operation command. + + Arguments: + NAME: The full resource path of the branch to undelete. Format: + projects/{project_id}/branches/{branch_id}` + + // This command is being previewed; hide from help output. + cmd.Hidden = true + + cmd.Annotations = make(map[string]string) + cmd.Annotations["launch_stage"] = "PRIVATE_PREVIEW" + cmd.Annotations["launch_stage_display"] = "Private Preview" + + cmd.Args = func(cmd *cobra.Command, args []string) error { + check := root.ExactArgs(1) + return check(cmd, args) + } + + cmd.PreRunE = root.MustWorkspaceClient + cmd.RunE = func(cmd *cobra.Command, args []string) (err error) { + ctx := cmd.Context() + w := cmdctx.WorkspaceClient(ctx) + + undeleteBranchReq.Name = args[0] + + // Determine which mode to execute based on flags. + switch { + case undeleteBranchSkipWait: + wait, err := w.Postgres.UndeleteBranch(ctx, undeleteBranchReq) + if err != nil { + return err + } + + // Return operation immediately without waiting. + operation, err := w.Postgres.GetOperation(ctx, postgres.GetOperationRequest{ + Name: wait.Name(), + }) + if err != nil { + return err + } + return cmdio.Render(ctx, operation) + + default: + wait, err := w.Postgres.UndeleteBranch(ctx, undeleteBranchReq) + if err != nil { + return err + } + + // Show spinner while waiting for completion. + sp := cmdio.NewSpinner(ctx) + sp.Update("Waiting for undelete-branch to complete...") + + // Wait for completion. + opts := api.WithTimeout(undeleteBranchTimeout) + + err = wait.Wait(ctx, opts) + if err != nil { + return err + } + sp.Close() + return nil + } + } + + // Disable completions since they are not applicable. + // Can be overridden by manual implementation in `override.go`. + cmd.ValidArgsFunction = cobra.NoFileCompletions + + // Apply optional overrides to this command. + for _, fn := range undeleteBranchOverrides { + fn(cmd, &undeleteBranchReq) + } + + return cmd +} + // start undelete-project command // Slice with functions to override default command behavior. @@ -2708,8 +2816,10 @@ func newUndeleteProject() *cobra.Command { cmd.Flags().DurationVar(&undeleteProjectTimeout, "timeout", 0, `maximum amount of time to reach DONE state`) cmd.Use = "undelete-project NAME" - cmd.Short = `Undelete a Project.` - cmd.Long = `Undelete a Project. + cmd.Short = `*Beta* Undelete a Project.` + cmd.Long = `This command is in Beta and may change without notice. + +Undelete a Project. Undeletes a soft-deleted project. @@ -2722,12 +2832,9 @@ func newUndeleteProject() *cobra.Command { NAME: The full resource path of the project to undelete. Format: projects/{project_id}` - // This command is being previewed; hide from help output. - cmd.Hidden = true - cmd.Annotations = make(map[string]string) - cmd.Annotations["launch_stage"] = "PRIVATE_PREVIEW" - cmd.Annotations["launch_stage_display"] = "Private Preview" + cmd.Annotations["launch_stage"] = "PUBLIC_BETA" + cmd.Annotations["launch_stage_display"] = "Beta" cmd.Args = func(cmd *cobra.Command, args []string) error { check := root.ExactArgs(1) diff --git a/cmd/workspace/supervisor-agents/supervisor-agents.go b/cmd/workspace/supervisor-agents/supervisor-agents.go index afad5cbe5e0..b8a654de92a 100755 --- a/cmd/workspace/supervisor-agents/supervisor-agents.go +++ b/cmd/workspace/supervisor-agents/supervisor-agents.go @@ -287,8 +287,10 @@ Create a Tool. Creates a Tool under a Supervisor Agent. Specify one of "genie_space", "knowledge_assistant", "uc_function", "uc_connection", "app", "volume", - "lakeview_dashboard", "uc_table", "vector_search_index", "catalog", "schema", - "supervisor_agent", "web_search" in the request body. + "dashboard", "table", "vector_search_index", "catalog", "schema", + "supervisor_agent", "web_search" in the request body. The legacy values + "lakeview_dashboard" and "uc_table" are also accepted and remain equivalent to + "dashboard" and "table" respectively. Arguments: PARENT: Parent resource where this tool will be created. Format: @@ -296,9 +298,11 @@ Create a Tool. TOOL_ID: The ID to use for the tool, which will become the final component of the tool's resource name. TOOL_TYPE: Tool type. Must be one of: "genie_space", "knowledge_assistant", - "uc_function", "uc_connection", "app", "volume", "lakeview_dashboard", - "serving_endpoint", "uc_table", "vector_search_index", "catalog", - "schema", "supervisor_agent", "web_search".` + "uc_function", "uc_connection", "app", "volume", "dashboard", + "serving_endpoint", "table", "vector_search_index", "catalog", "schema", + "supervisor_agent", "web_search". The legacy values "lakeview_dashboard" + and "uc_table" are also accepted and remain equivalent to "dashboard" and + "table" respectively.` cmd.Annotations = make(map[string]string) cmd.Annotations["launch_stage"] = "PUBLIC_BETA" @@ -1484,9 +1488,11 @@ Update a Tool. supervisor-agents/{supervisor_agent_id}/tools/{tool_id} UPDATE_MASK: Field mask for fields to be updated. TOOL_TYPE: Tool type. Must be one of: "genie_space", "knowledge_assistant", - "uc_function", "uc_connection", "app", "volume", "lakeview_dashboard", - "serving_endpoint", "uc_table", "vector_search_index", "catalog", - "schema", "supervisor_agent", "web_search".` + "uc_function", "uc_connection", "app", "volume", "dashboard", + "serving_endpoint", "table", "vector_search_index", "catalog", "schema", + "supervisor_agent", "web_search". The legacy values "lakeview_dashboard" + and "uc_table" are also accepted and remain equivalent to "dashboard" and + "table" respectively.` cmd.Annotations = make(map[string]string) cmd.Annotations["launch_stage"] = "PUBLIC_BETA" diff --git a/cmd/workspace/temporary-path-credentials/temporary-path-credentials.go b/cmd/workspace/temporary-path-credentials/temporary-path-credentials.go index a6a77d6c72f..a8abcf57ad7 100755 --- a/cmd/workspace/temporary-path-credentials/temporary-path-credentials.go +++ b/cmd/workspace/temporary-path-credentials/temporary-path-credentials.go @@ -37,18 +37,16 @@ Temporary Path Credentials refer to short-lived, downscoped credentials used temporary path credentials API, a metastore admin needs to enable the external_access_enabled flag (off by default) at the metastore level. A user needs to be granted the EXTERNAL USE LOCATION permission by external location - owner. For requests on existing external tables, user also needs to be granted - the EXTERNAL USE SCHEMA permission at the schema level by catalog owner. + owner. For requests on existing external tables and external volumes, user + also needs to be granted the EXTERNAL USE SCHEMA permission at the schema + level by catalog owner. Note that EXTERNAL USE SCHEMA is a schema level permission that can only be granted by catalog owner explicitly and is not included in schema ownership or ALL PRIVILEGES on the schema for security reasons. Similarly, EXTERNAL USE LOCATION is an external location level permission that can only be granted by external location owner explicitly and is not included in external location - ownership or ALL PRIVILEGES on the external location for security reasons. - - This API only supports temporary path credentials for external locations and - external tables, and volumes will be supported in the future.`, + ownership or ALL PRIVILEGES on the external location for security reasons.`, GroupID: "catalog", RunE: root.ReportUnknownSubcommand, } diff --git a/cmd/workspace/temporary-volume-credentials/temporary-volume-credentials.go b/cmd/workspace/temporary-volume-credentials/temporary-volume-credentials.go index d3ed061d399..1afe54d31f5 100755 --- a/cmd/workspace/temporary-volume-credentials/temporary-volume-credentials.go +++ b/cmd/workspace/temporary-volume-credentials/temporary-volume-credentials.go @@ -18,8 +18,10 @@ var cmdOverrides []func(*cobra.Command) func New() *cobra.Command { cmd := &cobra.Command{ Use: "temporary-volume-credentials", - Short: `Temporary Volume Credentials refer to short-lived, downscoped credentials used to access cloud storage locations where volume data is stored in Databricks.`, - Long: `Temporary Volume Credentials refer to short-lived, downscoped credentials used + Short: `*Public Preview* Temporary Volume Credentials refer to short-lived, downscoped credentials used to access cloud storage locations where volume data is stored in Databricks.`, + Long: `This command is in Public Preview and may change without notice. + +Temporary Volume Credentials refer to short-lived, downscoped credentials used to access cloud storage locations where volume data is stored in Databricks. These credentials are employed to provide secure and time-limited access to data in cloud environments such as AWS, Azure, and Google Cloud. Each cloud @@ -37,15 +39,12 @@ func New() *cobra.Command { can only be granted by catalog owner explicitly and is not included in schema ownership or ALL PRIVILEGES on the schema for security reasons.`, GroupID: "catalog", - - // This service is being previewed; hide from help output. - Hidden: true, - RunE: root.ReportUnknownSubcommand, + RunE: root.ReportUnknownSubcommand, } cmd.Annotations = make(map[string]string) - cmd.Annotations["launch_stage"] = "PRIVATE_PREVIEW" - cmd.Annotations["launch_stage_display"] = "Private Preview" + cmd.Annotations["launch_stage"] = "PUBLIC_PREVIEW" + cmd.Annotations["launch_stage_display"] = "Public Preview" // Add methods cmd.AddCommand(newGenerateTemporaryVolumeCredentials()) @@ -79,8 +78,10 @@ func newGenerateTemporaryVolumeCredentials() *cobra.Command { cmd.Flags().StringVar(&generateTemporaryVolumeCredentialsReq.VolumeId, "volume-id", generateTemporaryVolumeCredentialsReq.VolumeId, `Id of the volume to read or write.`) cmd.Use = "generate-temporary-volume-credentials" - cmd.Short = `Generate a temporary volume credential.` - cmd.Long = `Generate a temporary volume credential. + cmd.Short = `*Public Preview* Generate a temporary volume credential.` + cmd.Long = `This command is in Public Preview and may change without notice. + +Generate a temporary volume credential. Get a short-lived credential for directly accessing the volume data on cloud storage. The metastore must have **external_access_enabled** flag set to true @@ -88,8 +89,8 @@ func newGenerateTemporaryVolumeCredentials() *cobra.Command { the parent schema and this privilege can only be granted by catalog owners.` cmd.Annotations = make(map[string]string) - cmd.Annotations["launch_stage"] = "PRIVATE_PREVIEW" - cmd.Annotations["launch_stage_display"] = "Private Preview" + cmd.Annotations["launch_stage"] = "PUBLIC_PREVIEW" + cmd.Annotations["launch_stage_display"] = "Public Preview" cmd.Args = func(cmd *cobra.Command, args []string) error { check := root.ExactArgs(0) diff --git a/cmd/workspace/vector-search-endpoints/vector-search-endpoints.go b/cmd/workspace/vector-search-endpoints/vector-search-endpoints.go index 455a61586ac..9342d2999d0 100755 --- a/cmd/workspace/vector-search-endpoints/vector-search-endpoints.go +++ b/cmd/workspace/vector-search-endpoints/vector-search-endpoints.go @@ -21,8 +21,8 @@ var cmdOverrides []func(*cobra.Command) func New() *cobra.Command { cmd := &cobra.Command{ Use: "vector-search-endpoints", - Short: `**Endpoint**: Represents the compute resources to host vector search indexes.`, - Long: `**Endpoint**: Represents the compute resources to host vector search indexes.`, + Short: `**Endpoint**: Represents the compute resources to host AI Search indexes.`, + Long: `**Endpoint**: Represents the compute resources to host AI Search indexes.`, GroupID: "vectorsearch", RunE: root.ReportUnknownSubcommand, } @@ -87,7 +87,7 @@ func newCreateEndpoint() *cobra.Command { Create a new endpoint. Arguments: - NAME: Name of the vector search endpoint + NAME: Name of the AI Search endpoint ENDPOINT_TYPE: Type of endpoint Supported values: [STANDARD, STORAGE_OPTIMIZED]` @@ -191,10 +191,10 @@ func newDeleteEndpoint() *cobra.Command { cmd.Short = `Delete an endpoint.` cmd.Long = `Delete an endpoint. - Delete a vector search endpoint. + Delete an AI Search endpoint. Arguments: - ENDPOINT_NAME: Name of the vector search endpoint` + ENDPOINT_NAME: Name of the AI Search endpoint` cmd.Annotations = make(map[string]string) cmd.Annotations["launch_stage"] = "GA" @@ -249,7 +249,7 @@ func newGetEndpoint() *cobra.Command { cmd.Short = `Get an endpoint.` cmd.Long = `Get an endpoint. - Get details for a single vector search endpoint. + Get details for a single AI Search endpoint. Arguments: ENDPOINT_NAME: Name of the endpoint` @@ -438,7 +438,7 @@ func newListEndpoints() *cobra.Command { cmd.Short = `List all endpoints.` cmd.Long = `List all endpoints. - List all vector search endpoints in the workspace.` + List all AI Search endpoints in the workspace.` cmd.Annotations = make(map[string]string) cmd.Annotations["launch_stage"] = "GA" @@ -505,7 +505,7 @@ Update an endpoint. Update an endpoint Arguments: - ENDPOINT_NAME: Name of the vector search endpoint` + ENDPOINT_NAME: Name of the AI Search endpoint` cmd.Annotations = make(map[string]string) cmd.Annotations["launch_stage"] = "PUBLIC_PREVIEW" @@ -585,7 +585,7 @@ func newRetrieveUserVisibleMetrics() *cobra.Command { Retrieve user-visible metrics for an endpoint Arguments: - NAME: Vector search endpoint name` + NAME: AI Search endpoint name` cmd.Annotations = make(map[string]string) cmd.Annotations["launch_stage"] = "GA" @@ -739,7 +739,7 @@ Update the budget policy of an endpoint. Update the budget policy of an endpoint Arguments: - ENDPOINT_NAME: Name of the vector search endpoint + ENDPOINT_NAME: Name of the AI Search endpoint BUDGET_POLICY_ID: The budget policy id to be applied` cmd.Annotations = make(map[string]string) @@ -822,7 +822,7 @@ func newUpdateEndpointCustomTags() *cobra.Command { cmd.Long = `Update the custom tags of an endpoint. Arguments: - ENDPOINT_NAME: Name of the vector search endpoint` + ENDPOINT_NAME: Name of the AI Search endpoint` // This command is being previewed; hide from help output. cmd.Hidden = true diff --git a/cmd/workspace/vector-search-indexes/vector-search-indexes.go b/cmd/workspace/vector-search-indexes/vector-search-indexes.go index 87dc851c222..53b5c250716 100755 --- a/cmd/workspace/vector-search-indexes/vector-search-indexes.go +++ b/cmd/workspace/vector-search-indexes/vector-search-indexes.go @@ -24,12 +24,12 @@ func New() *cobra.Command { Long: `**Index**: An efficient representation of your embedding vectors that supports real-time and efficient approximate nearest neighbor (ANN) search queries. - There are 2 types of Vector Search indexes: - **Delta Sync Index**: An index - that automatically syncs with a source Delta Table, automatically and - incrementally updating the index as the underlying data in the Delta Table - changes. - **Direct Vector Access Index**: An index that supports direct read - and write of vectors and metadata through our REST and SDK APIs. With this - model, the user manages index updates.`, + There are 2 types of AI Search indexes: - **Delta Sync Index**: An index that + automatically syncs with a source Delta Table, automatically and incrementally + updating the index as the underlying data in the Delta Table changes. - + **Direct Vector Access Index**: An index that supports direct read and write + of vectors and metadata through our REST and SDK APIs. With this model, the + user manages index updates.`, GroupID: "vectorsearch", RunE: root.ReportUnknownSubcommand, } diff --git a/cmd/workspace/workspace-iam-v2/workspace-iam-v2.go b/cmd/workspace/workspace-iam-v2/workspace-iam-v2.go index 48402a61c70..f46706976cf 100755 --- a/cmd/workspace/workspace-iam-v2/workspace-iam-v2.go +++ b/cmd/workspace/workspace-iam-v2/workspace-iam-v2.go @@ -4,11 +4,13 @@ package workspace_iam_v2 import ( "fmt" + "strings" "github.com/databricks/cli/cmd/root" "github.com/databricks/cli/libs/cmdctx" "github.com/databricks/cli/libs/cmdio" "github.com/databricks/cli/libs/flags" + "github.com/databricks/databricks-sdk-go/common/types/fieldmask" "github.com/databricks/databricks-sdk-go/service/iamv2" "github.com/spf13/cobra" ) @@ -34,10 +36,15 @@ These APIs are used to manage identities and the workspace access of these cmd.Annotations["launch_stage_display"] = "Beta" // Add methods + cmd.AddCommand(newCreateWorkspaceAssignmentDetailProxy()) + cmd.AddCommand(newDeleteWorkspaceAssignmentDetailProxy()) cmd.AddCommand(newGetWorkspaceAccessDetailLocal()) + cmd.AddCommand(newGetWorkspaceAssignmentDetailProxy()) + cmd.AddCommand(newListWorkspaceAssignmentDetailsProxy()) cmd.AddCommand(newResolveGroupProxy()) cmd.AddCommand(newResolveServicePrincipalProxy()) cmd.AddCommand(newResolveUserProxy()) + cmd.AddCommand(newUpdateWorkspaceAssignmentDetailProxy()) // Apply optional overrides to this command. for _, fn := range cmdOverrides { @@ -47,6 +54,173 @@ These APIs are used to manage identities and the workspace access of these return cmd } +// start create-workspace-assignment-detail-proxy command + +// Slice with functions to override default command behavior. +// Functions can be added from the `init()` function in manually curated files in this directory. +var createWorkspaceAssignmentDetailProxyOverrides []func( + *cobra.Command, + *iamv2.CreateWorkspaceAssignmentDetailProxyRequest, +) + +func newCreateWorkspaceAssignmentDetailProxy() *cobra.Command { + cmd := &cobra.Command{} + + var createWorkspaceAssignmentDetailProxyReq iamv2.CreateWorkspaceAssignmentDetailProxyRequest + createWorkspaceAssignmentDetailProxyReq.WorkspaceAssignmentDetail = iamv2.WorkspaceAssignmentDetail{} + var createWorkspaceAssignmentDetailProxyJson flags.JsonFlag + + cmd.Flags().Var(&createWorkspaceAssignmentDetailProxyJson, "json", `either inline JSON string or @path/to/file.json with request body`) + + // TODO: array: entitlements + + cmd.Use = "create-workspace-assignment-detail-proxy PRINCIPAL_ID" + cmd.Short = `Create a workspace assignment detail for a workspace.` + cmd.Long = `Create a workspace assignment detail for a workspace. + + Creates a workspace assignment detail for a principal (workspace-level proxy). + Entitlement grants are applied individually and non-atomically — if a + failure occurs partway through, the principal will be assigned to the + workspace but with only a subset of the requested entitlements. Use + GetWorkspaceAssignmentDetail to confirm which entitlements were successfully + granted. + + Arguments: + PRINCIPAL_ID: The internal ID of the principal (user/sp/group) in Databricks.` + + // This command is being previewed; hide from help output. + cmd.Hidden = true + + cmd.Annotations = make(map[string]string) + cmd.Annotations["launch_stage"] = "PRIVATE_PREVIEW" + cmd.Annotations["launch_stage_display"] = "Private Preview" + + cmd.Args = func(cmd *cobra.Command, args []string) error { + if cmd.Flags().Changed("json") { + err := root.ExactArgs(0)(cmd, args) + if err != nil { + return fmt.Errorf("when --json flag is specified, no positional arguments are allowed. Provide 'principal_id' in your JSON input") + } + return nil + } + check := root.ExactArgs(1) + return check(cmd, args) + } + + cmd.PreRunE = root.MustWorkspaceClient + cmd.RunE = func(cmd *cobra.Command, args []string) (err error) { + ctx := cmd.Context() + w := cmdctx.WorkspaceClient(ctx) + + if cmd.Flags().Changed("json") { + diags := createWorkspaceAssignmentDetailProxyJson.Unmarshal(&createWorkspaceAssignmentDetailProxyReq.WorkspaceAssignmentDetail) + if diags.HasError() { + return diags.Error() + } + if len(diags) > 0 { + err := cmdio.RenderDiagnostics(ctx, diags) + if err != nil { + return err + } + } + } + if !cmd.Flags().Changed("json") { + _, err = fmt.Sscan(args[0], &createWorkspaceAssignmentDetailProxyReq.WorkspaceAssignmentDetail.PrincipalId) + if err != nil { + return fmt.Errorf("invalid PRINCIPAL_ID: %s", args[0]) + } + + } + + response, err := w.WorkspaceIamV2.CreateWorkspaceAssignmentDetailProxy(ctx, createWorkspaceAssignmentDetailProxyReq) + if err != nil { + return err + } + + return cmdio.Render(ctx, response) + } + + // Disable completions since they are not applicable. + // Can be overridden by manual implementation in `override.go`. + cmd.ValidArgsFunction = cobra.NoFileCompletions + + // Apply optional overrides to this command. + for _, fn := range createWorkspaceAssignmentDetailProxyOverrides { + fn(cmd, &createWorkspaceAssignmentDetailProxyReq) + } + + return cmd +} + +// start delete-workspace-assignment-detail-proxy command + +// Slice with functions to override default command behavior. +// Functions can be added from the `init()` function in manually curated files in this directory. +var deleteWorkspaceAssignmentDetailProxyOverrides []func( + *cobra.Command, + *iamv2.DeleteWorkspaceAssignmentDetailProxyRequest, +) + +func newDeleteWorkspaceAssignmentDetailProxy() *cobra.Command { + cmd := &cobra.Command{} + + var deleteWorkspaceAssignmentDetailProxyReq iamv2.DeleteWorkspaceAssignmentDetailProxyRequest + + cmd.Use = "delete-workspace-assignment-detail-proxy PRINCIPAL_ID" + cmd.Short = `Delete a workspace assignment detail for a workspace.` + cmd.Long = `Delete a workspace assignment detail for a workspace. + + Deletes a workspace assignment detail for a principal (workspace-level proxy), + revoking all associated entitlements. Entitlement revocations are applied + individually and non-atomically — if a failure occurs partway through, the + principal remains assigned with a subset of its original entitlements, and the + operation is safe to retry. + + Arguments: + PRINCIPAL_ID: Required. ID of the principal in Databricks to delete workspace assignment + for.` + + // This command is being previewed; hide from help output. + cmd.Hidden = true + + cmd.Annotations = make(map[string]string) + cmd.Annotations["launch_stage"] = "PRIVATE_PREVIEW" + cmd.Annotations["launch_stage_display"] = "Private Preview" + + cmd.Args = func(cmd *cobra.Command, args []string) error { + check := root.ExactArgs(1) + return check(cmd, args) + } + + cmd.PreRunE = root.MustWorkspaceClient + cmd.RunE = func(cmd *cobra.Command, args []string) (err error) { + ctx := cmd.Context() + w := cmdctx.WorkspaceClient(ctx) + + _, err = fmt.Sscan(args[0], &deleteWorkspaceAssignmentDetailProxyReq.PrincipalId) + if err != nil { + return fmt.Errorf("invalid PRINCIPAL_ID: %s", args[0]) + } + + err = w.WorkspaceIamV2.DeleteWorkspaceAssignmentDetailProxy(ctx, deleteWorkspaceAssignmentDetailProxyReq) + if err != nil { + return err + } + return nil + } + + // Disable completions since they are not applicable. + // Can be overridden by manual implementation in `override.go`. + cmd.ValidArgsFunction = cobra.NoFileCompletions + + // Apply optional overrides to this command. + for _, fn := range deleteWorkspaceAssignmentDetailProxyOverrides { + fn(cmd, &deleteWorkspaceAssignmentDetailProxyReq) + } + + return cmd +} + // start get-workspace-access-detail-local command // Slice with functions to override default command behavior. @@ -119,6 +293,133 @@ Get workspace access details for a principal. return cmd } +// start get-workspace-assignment-detail-proxy command + +// Slice with functions to override default command behavior. +// Functions can be added from the `init()` function in manually curated files in this directory. +var getWorkspaceAssignmentDetailProxyOverrides []func( + *cobra.Command, + *iamv2.GetWorkspaceAssignmentDetailProxyRequest, +) + +func newGetWorkspaceAssignmentDetailProxy() *cobra.Command { + cmd := &cobra.Command{} + + var getWorkspaceAssignmentDetailProxyReq iamv2.GetWorkspaceAssignmentDetailProxyRequest + + cmd.Use = "get-workspace-assignment-detail-proxy PRINCIPAL_ID" + cmd.Short = `Get workspace assignment details for a principal.` + cmd.Long = `Get workspace assignment details for a principal. + + Returns the assignment details for a principal in a workspace (workspace-level + proxy). + + Arguments: + PRINCIPAL_ID: Required. The internal ID of the principal (user/sp/group) for which the + assignment details are being requested.` + + // This command is being previewed; hide from help output. + cmd.Hidden = true + + cmd.Annotations = make(map[string]string) + cmd.Annotations["launch_stage"] = "PRIVATE_PREVIEW" + cmd.Annotations["launch_stage_display"] = "Private Preview" + + cmd.Args = func(cmd *cobra.Command, args []string) error { + check := root.ExactArgs(1) + return check(cmd, args) + } + + cmd.PreRunE = root.MustWorkspaceClient + cmd.RunE = func(cmd *cobra.Command, args []string) (err error) { + ctx := cmd.Context() + w := cmdctx.WorkspaceClient(ctx) + + _, err = fmt.Sscan(args[0], &getWorkspaceAssignmentDetailProxyReq.PrincipalId) + if err != nil { + return fmt.Errorf("invalid PRINCIPAL_ID: %s", args[0]) + } + + response, err := w.WorkspaceIamV2.GetWorkspaceAssignmentDetailProxy(ctx, getWorkspaceAssignmentDetailProxyReq) + if err != nil { + return err + } + + return cmdio.Render(ctx, response) + } + + // Disable completions since they are not applicable. + // Can be overridden by manual implementation in `override.go`. + cmd.ValidArgsFunction = cobra.NoFileCompletions + + // Apply optional overrides to this command. + for _, fn := range getWorkspaceAssignmentDetailProxyOverrides { + fn(cmd, &getWorkspaceAssignmentDetailProxyReq) + } + + return cmd +} + +// start list-workspace-assignment-details-proxy command + +// Slice with functions to override default command behavior. +// Functions can be added from the `init()` function in manually curated files in this directory. +var listWorkspaceAssignmentDetailsProxyOverrides []func( + *cobra.Command, + *iamv2.ListWorkspaceAssignmentDetailsProxyRequest, +) + +func newListWorkspaceAssignmentDetailsProxy() *cobra.Command { + cmd := &cobra.Command{} + + var listWorkspaceAssignmentDetailsProxyReq iamv2.ListWorkspaceAssignmentDetailsProxyRequest + + cmd.Flags().IntVar(&listWorkspaceAssignmentDetailsProxyReq.PageSize, "page-size", listWorkspaceAssignmentDetailsProxyReq.PageSize, `The maximum number of workspace assignment details to return.`) + cmd.Flags().StringVar(&listWorkspaceAssignmentDetailsProxyReq.PageToken, "page-token", listWorkspaceAssignmentDetailsProxyReq.PageToken, `A page token, received from a previous ListWorkspaceAssignmentDetailsProxy call.`) + + cmd.Use = "list-workspace-assignment-details-proxy" + cmd.Short = `List workspace assignment details for a workspace.` + cmd.Long = `List workspace assignment details for a workspace. + + Lists workspace assignment details for a workspace (workspace-level proxy).` + + // This command is being previewed; hide from help output. + cmd.Hidden = true + + cmd.Annotations = make(map[string]string) + cmd.Annotations["launch_stage"] = "PRIVATE_PREVIEW" + cmd.Annotations["launch_stage_display"] = "Private Preview" + + cmd.Args = func(cmd *cobra.Command, args []string) error { + check := root.ExactArgs(0) + return check(cmd, args) + } + + cmd.PreRunE = root.MustWorkspaceClient + cmd.RunE = func(cmd *cobra.Command, args []string) (err error) { + ctx := cmd.Context() + w := cmdctx.WorkspaceClient(ctx) + + response, err := w.WorkspaceIamV2.ListWorkspaceAssignmentDetailsProxy(ctx, listWorkspaceAssignmentDetailsProxyReq) + if err != nil { + return err + } + + return cmdio.Render(ctx, response) + } + + // Disable completions since they are not applicable. + // Can be overridden by manual implementation in `override.go`. + cmd.ValidArgsFunction = cobra.NoFileCompletions + + // Apply optional overrides to this command. + for _, fn := range listWorkspaceAssignmentDetailsProxyOverrides { + fn(cmd, &listWorkspaceAssignmentDetailsProxyReq) + } + + return cmd +} + // start resolve-group-proxy command // Slice with functions to override default command behavior. @@ -381,4 +682,112 @@ Resolve an external user in the Databricks account. return cmd } +// start update-workspace-assignment-detail-proxy command + +// Slice with functions to override default command behavior. +// Functions can be added from the `init()` function in manually curated files in this directory. +var updateWorkspaceAssignmentDetailProxyOverrides []func( + *cobra.Command, + *iamv2.UpdateWorkspaceAssignmentDetailProxyRequest, +) + +func newUpdateWorkspaceAssignmentDetailProxy() *cobra.Command { + cmd := &cobra.Command{} + + var updateWorkspaceAssignmentDetailProxyReq iamv2.UpdateWorkspaceAssignmentDetailProxyRequest + updateWorkspaceAssignmentDetailProxyReq.WorkspaceAssignmentDetail = iamv2.WorkspaceAssignmentDetail{} + var updateWorkspaceAssignmentDetailProxyJson flags.JsonFlag + + cmd.Flags().Var(&updateWorkspaceAssignmentDetailProxyJson, "json", `either inline JSON string or @path/to/file.json with request body`) + + // TODO: array: entitlements + + cmd.Use = "update-workspace-assignment-detail-proxy PRINCIPAL_ID UPDATE_MASK PRINCIPAL_ID" + cmd.Short = `Update a workspace assignment detail for a workspace.` + cmd.Long = `Update a workspace assignment detail for a workspace. + + Updates the entitlements of a directly assigned principal in a workspace + (workspace-level proxy). Entitlement changes are applied individually and + non-atomically — if a failure occurs partway through, only a subset of the + requested changes may have been applied. Use GetWorkspaceAssignmentDetail to + confirm the final state. + + Arguments: + PRINCIPAL_ID: Required. ID of the principal in Databricks. + UPDATE_MASK: Required. The list of fields to update. + PRINCIPAL_ID: The internal ID of the principal (user/sp/group) in Databricks.` + + // This command is being previewed; hide from help output. + cmd.Hidden = true + + cmd.Annotations = make(map[string]string) + cmd.Annotations["launch_stage"] = "PRIVATE_PREVIEW" + cmd.Annotations["launch_stage_display"] = "Private Preview" + + cmd.Args = func(cmd *cobra.Command, args []string) error { + if cmd.Flags().Changed("json") { + err := root.ExactArgs(2)(cmd, args) + if err != nil { + return fmt.Errorf("when --json flag is specified, provide only PRINCIPAL_ID, UPDATE_MASK as positional arguments. Provide 'principal_id' in your JSON input") + } + return nil + } + check := root.ExactArgs(3) + return check(cmd, args) + } + + cmd.PreRunE = root.MustWorkspaceClient + cmd.RunE = func(cmd *cobra.Command, args []string) (err error) { + ctx := cmd.Context() + w := cmdctx.WorkspaceClient(ctx) + + if cmd.Flags().Changed("json") { + diags := updateWorkspaceAssignmentDetailProxyJson.Unmarshal(&updateWorkspaceAssignmentDetailProxyReq.WorkspaceAssignmentDetail) + if diags.HasError() { + return diags.Error() + } + if len(diags) > 0 { + err := cmdio.RenderDiagnostics(ctx, diags) + if err != nil { + return err + } + } + } + _, err = fmt.Sscan(args[0], &updateWorkspaceAssignmentDetailProxyReq.PrincipalId) + if err != nil { + return fmt.Errorf("invalid PRINCIPAL_ID: %s", args[0]) + } + + if args[1] != "" { + updateMaskArray := strings.Split(args[1], ",") + updateWorkspaceAssignmentDetailProxyReq.UpdateMask = *fieldmask.New(updateMaskArray) + } + if !cmd.Flags().Changed("json") { + _, err = fmt.Sscan(args[2], &updateWorkspaceAssignmentDetailProxyReq.WorkspaceAssignmentDetail.PrincipalId) + if err != nil { + return fmt.Errorf("invalid PRINCIPAL_ID: %s", args[2]) + } + + } + + response, err := w.WorkspaceIamV2.UpdateWorkspaceAssignmentDetailProxy(ctx, updateWorkspaceAssignmentDetailProxyReq) + if err != nil { + return err + } + + return cmdio.Render(ctx, response) + } + + // Disable completions since they are not applicable. + // Can be overridden by manual implementation in `override.go`. + cmd.ValidArgsFunction = cobra.NoFileCompletions + + // Apply optional overrides to this command. + for _, fn := range updateWorkspaceAssignmentDetailProxyOverrides { + fn(cmd, &updateWorkspaceAssignmentDetailProxyReq) + } + + return cmd +} + // end service WorkspaceIamV2 diff --git a/cmd/workspace/workspace-settings-v2/workspace-settings-v2.go b/cmd/workspace/workspace-settings-v2/workspace-settings-v2.go index 114b97882e0..64ceb60cedc 100755 --- a/cmd/workspace/workspace-settings-v2/workspace-settings-v2.go +++ b/cmd/workspace/workspace-settings-v2/workspace-settings-v2.go @@ -202,18 +202,22 @@ func newPatchPublicWorkspaceSetting() *cobra.Command { // TODO: complex arg: aibi_dashboard_embedding_access_policy // TODO: complex arg: aibi_dashboard_embedding_approved_domains + // TODO: complex arg: allowed_apps_user_api_scopes // TODO: complex arg: automatic_cluster_update_workspace // TODO: complex arg: boolean_val // TODO: complex arg: effective_aibi_dashboard_embedding_access_policy // TODO: complex arg: effective_aibi_dashboard_embedding_approved_domains + // TODO: complex arg: effective_allowed_apps_user_api_scopes // TODO: complex arg: effective_automatic_cluster_update_workspace // TODO: complex arg: effective_boolean_val // TODO: complex arg: effective_integer_val + // TODO: complex arg: effective_operational_email_custom_recipient // TODO: complex arg: effective_personal_compute // TODO: complex arg: effective_restrict_workspace_admins // TODO: complex arg: effective_string_val // TODO: complex arg: integer_val cmd.Flags().StringVar(&patchPublicWorkspaceSettingReq.Setting.Name, "name", patchPublicWorkspaceSettingReq.Setting.Name, `Name of the setting.`) + // TODO: complex arg: operational_email_custom_recipient // TODO: complex arg: personal_compute // TODO: complex arg: restrict_workspace_admins // TODO: complex arg: string_val diff --git a/experimental/postgres/cmd/targeting.go b/experimental/postgres/cmd/targeting.go index 6d04055cb22..cf1290ad578 100644 --- a/experimental/postgres/cmd/targeting.go +++ b/experimental/postgres/cmd/targeting.go @@ -9,6 +9,7 @@ import ( "github.com/databricks/cli/libs/cmdctx" "github.com/databricks/databricks-sdk-go" "github.com/databricks/databricks-sdk-go/service/database" + "github.com/databricks/databricks-sdk-go/service/iam" "github.com/databricks/databricks-sdk-go/service/postgres" ) @@ -113,7 +114,7 @@ func resolveProvisioned(ctx context.Context, w *databricks.WorkspaceClient, inst return nil, fmt.Errorf("database instance %q is not ready for accepting connections (state: %s)", instance.Name, instance.State) } - user, err := w.CurrentUser.Me(ctx) + user, err := w.CurrentUser.Me(ctx, iam.MeRequest{}) if err != nil { return nil, fmt.Errorf("failed to get current user: %w", err) } @@ -174,7 +175,7 @@ func resolveAutoscaling(ctx context.Context, w *databricks.WorkspaceClient, spec return nil, err } - user, err := w.CurrentUser.Me(ctx) + user, err := w.CurrentUser.Me(ctx, iam.MeRequest{}) if err != nil { return nil, fmt.Errorf("failed to get current user: %w", err) } diff --git a/experimental/ssh/internal/client/client.go b/experimental/ssh/internal/client/client.go index 5ab096d2929..69360b85d12 100644 --- a/experimental/ssh/internal/client/client.go +++ b/experimental/ssh/internal/client/client.go @@ -31,6 +31,7 @@ import ( "github.com/databricks/databricks-sdk-go" "github.com/databricks/databricks-sdk-go/retries" "github.com/databricks/databricks-sdk-go/service/compute" + "github.com/databricks/databricks-sdk-go/service/iam" "github.com/databricks/databricks-sdk-go/service/jobs" "github.com/databricks/databricks-sdk-go/service/workspace" "github.com/gorilla/websocket" @@ -370,7 +371,7 @@ func runIDE(ctx context.Context, client *databricks.WorkspaceClient, userName, k } // Get Databricks user name for the workspace path - currentUser, err := client.CurrentUser.Me(ctx) + currentUser, err := client.CurrentUser.Me(ctx, iam.MeRequest{}) if err != nil { return fmt.Errorf("failed to get current user: %w", err) } diff --git a/experimental/ssh/internal/keys/secrets.go b/experimental/ssh/internal/keys/secrets.go index 76d44da5387..6409770a4d7 100644 --- a/experimental/ssh/internal/keys/secrets.go +++ b/experimental/ssh/internal/keys/secrets.go @@ -7,13 +7,14 @@ import ( "fmt" "github.com/databricks/databricks-sdk-go" + "github.com/databricks/databricks-sdk-go/service/iam" "github.com/databricks/databricks-sdk-go/service/workspace" ) // CreateKeysSecretScope creates or retrieves the secret scope for SSH keys. // sessionID is the unique identifier for the session (cluster ID for dedicated clusters, connection name for serverless). func CreateKeysSecretScope(ctx context.Context, client *databricks.WorkspaceClient, sessionID string) (string, error) { - me, err := client.CurrentUser.Me(ctx) + me, err := client.CurrentUser.Me(ctx, iam.MeRequest{}) if err != nil { return "", fmt.Errorf("failed to get current user: %w", err) } diff --git a/experimental/ssh/internal/workspace/workspace.go b/experimental/ssh/internal/workspace/workspace.go index 2f017cbae12..0a28b684ebc 100644 --- a/experimental/ssh/internal/workspace/workspace.go +++ b/experimental/ssh/internal/workspace/workspace.go @@ -10,6 +10,7 @@ import ( "github.com/databricks/cli/libs/filer" "github.com/databricks/databricks-sdk-go" + "github.com/databricks/databricks-sdk-go/service/iam" ) const metadataFileName = "metadata.json" @@ -21,7 +22,7 @@ type WorkspaceMetadata struct { } func getWorkspaceRootDir(ctx context.Context, client *databricks.WorkspaceClient) (string, error) { - me, err := client.CurrentUser.Me(ctx) + me, err := client.CurrentUser.Me(ctx, iam.MeRequest{}) if err != nil { return "", fmt.Errorf("failed to get current user: %w", err) } diff --git a/go.mod b/go.mod index 50c45ad551b..02ab5828520 100644 --- a/go.mod +++ b/go.mod @@ -13,7 +13,7 @@ require ( github.com/charmbracelet/huh v1.0.0 // MIT github.com/charmbracelet/lipgloss v1.1.0 // MIT github.com/charmbracelet/x/ansi v0.11.6 // MIT - github.com/databricks/databricks-sdk-go v0.132.0 // Apache-2.0 + github.com/databricks/databricks-sdk-go v0.136.0 // Apache-2.0 github.com/google/jsonschema-go v0.4.3 // MIT github.com/google/uuid v1.6.0 // BSD-3-Clause github.com/gorilla/websocket v1.5.3 // BSD-2-Clause diff --git a/go.sum b/go.sum index b89ad220399..cb7bd44ee92 100644 --- a/go.sum +++ b/go.sum @@ -71,8 +71,8 @@ github.com/cyphar/filepath-securejoin v0.4.1 h1:JyxxyPEaktOD+GAnqIqTf9A8tHyAG22r github.com/cyphar/filepath-securejoin v0.4.1/go.mod h1:Sdj7gXlvMcPZsbhwhQ33GguGLDGQL7h7bg04C/+u9jI= github.com/danieljoos/wincred v1.2.3 h1:v7dZC2x32Ut3nEfRH+vhoZGvN72+dQ/snVXo/vMFLdQ= github.com/danieljoos/wincred v1.2.3/go.mod h1:6qqX0WNrS4RzPZ1tnroDzq9kY3fu1KwE7MRLQK4X0bs= -github.com/databricks/databricks-sdk-go v0.132.0 h1:NuwAMg7aSlMinINIv4iJDDGge426QptIOi2N6RC/k3k= -github.com/databricks/databricks-sdk-go v0.132.0/go.mod h1:C5LNgGe6hGuRrTwoxFmuup3XtQQEaqtq0e+K8IFDIS4= +github.com/databricks/databricks-sdk-go v0.136.0 h1:7TF+fDS9jpwBqnVy0foZ/ej7zL0h1EezuHfYxGYvhNE= +github.com/databricks/databricks-sdk-go v0.136.0/go.mod h1:C5LNgGe6hGuRrTwoxFmuup3XtQQEaqtq0e+K8IFDIS4= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= diff --git a/integration/bundle/init_test.go b/integration/bundle/init_test.go index 6bec010f7b1..ff8b7de2c87 100644 --- a/integration/bundle/init_test.go +++ b/integration/bundle/init_test.go @@ -10,6 +10,7 @@ import ( "github.com/databricks/cli/internal/testcli" "github.com/databricks/cli/internal/testutil" "github.com/databricks/cli/libs/iamutil" + "github.com/databricks/databricks-sdk-go/service/iam" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" ) @@ -25,7 +26,7 @@ func TestBundleInitHelpers(t *testing.T) { ctx, wt := acc.WorkspaceTest(t) w := wt.W - me, err := w.CurrentUser.Me(ctx) + me, err := w.CurrentUser.Me(ctx, iam.MeRequest{}) require.NoError(t, err) var smallestNode string diff --git a/integration/cmd/auth/describe_test.go b/integration/cmd/auth/describe_test.go index 1ea843b6b93..49796b892a3 100644 --- a/integration/cmd/auth/describe_test.go +++ b/integration/cmd/auth/describe_test.go @@ -9,6 +9,7 @@ import ( "github.com/databricks/cli/internal/testutil" "github.com/databricks/cli/libs/databrickscfg" "github.com/databricks/databricks-sdk-go/config" + "github.com/databricks/databricks-sdk-go/service/iam" "github.com/databricks/cli/internal/testcli" "github.com/databricks/databricks-sdk-go" @@ -28,7 +29,7 @@ func TestAuthDescribeSuccess(t *testing.T) { hostWithoutPrefix := strings.TrimPrefix(w.Config.Host, "https://") require.Regexp(t, "Host: (?:https://)?"+regexp.QuoteMeta(hostWithoutPrefix), outStr) - me, err := w.CurrentUser.Me(t.Context()) + me, err := w.CurrentUser.Me(t.Context(), iam.MeRequest{}) require.NoError(t, err) require.Contains(t, outStr, "User: "+me.UserName) require.Contains(t, outStr, "Authenticated with: "+w.Config.AuthType) diff --git a/integration/cmd/sync/sync_test.go b/integration/cmd/sync/sync_test.go index 5871c41db24..49853099582 100644 --- a/integration/cmd/sync/sync_test.go +++ b/integration/cmd/sync/sync_test.go @@ -23,6 +23,7 @@ import ( "github.com/databricks/cli/libs/testfile" "github.com/databricks/databricks-sdk-go" "github.com/databricks/databricks-sdk-go/client" + "github.com/databricks/databricks-sdk-go/service/iam" "github.com/databricks/databricks-sdk-go/service/workspace" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" @@ -36,7 +37,7 @@ var ( // This test needs auth env vars to run. // Please run using the deco env test or deco env shell func setupRepo(t *testing.T, wsc *databricks.WorkspaceClient, ctx context.Context) (localRoot, remoteRoot string) { - me, err := wsc.CurrentUser.Me(ctx) + me, err := wsc.CurrentUser.Me(ctx, iam.MeRequest{}) require.NoError(t, err) repoPath := fmt.Sprintf("/Repos/%s/%s", me.UserName, testutil.RandomName("empty-repo-sync-integration-")) @@ -487,7 +488,7 @@ func TestSyncEnsureRemotePathIsUsableIfRepoDoesntExist(t *testing.T) { ctx, wt := acc.WorkspaceTest(t) wsc := wt.W - me, err := wsc.CurrentUser.Me(ctx) + me, err := wsc.CurrentUser.Me(ctx, iam.MeRequest{}) require.NoError(t, err) // Hypothetical repo path doesn't exist. @@ -526,7 +527,7 @@ func TestSyncEnsureRemotePathIsUsableInWorkspace(t *testing.T) { ctx, wt := acc.WorkspaceTest(t) wsc := wt.W - me, err := wsc.CurrentUser.Me(ctx) + me, err := wsc.CurrentUser.Me(ctx, iam.MeRequest{}) require.NoError(t, err) remotePath := fmt.Sprintf("/Users/%s/%s", me.UserName, testutil.RandomName("ensure-path-exists-test-")) diff --git a/integration/internal/acc/fixtures.go b/integration/internal/acc/fixtures.go index d902d0b49a1..8d61a9d35da 100644 --- a/integration/internal/acc/fixtures.go +++ b/integration/internal/acc/fixtures.go @@ -8,13 +8,14 @@ import ( "github.com/databricks/databricks-sdk-go/apierr" "github.com/databricks/databricks-sdk-go/service/catalog" "github.com/databricks/databricks-sdk-go/service/files" + "github.com/databricks/databricks-sdk-go/service/iam" "github.com/databricks/databricks-sdk-go/service/workspace" "github.com/stretchr/testify/require" ) func TemporaryWorkspaceDir(t *WorkspaceT, name ...string) string { ctx := t.ctx - me, err := t.W.CurrentUser.Me(ctx) + me, err := t.W.CurrentUser.Me(ctx, iam.MeRequest{}) require.NoError(t, err) // Prefix the name with "integration-test-" to make it easier to identify. @@ -69,7 +70,7 @@ func TemporaryDbfsDir(t *WorkspaceT, name ...string) string { func TemporaryRepo(t *WorkspaceT, url string) string { ctx := t.ctx - me, err := t.W.CurrentUser.Me(ctx) + me, err := t.W.CurrentUser.Me(ctx, iam.MeRequest{}) require.NoError(t, err) // Prefix the path with "integration-test-" to make it easier to identify. diff --git a/internal/genkit/tagging.py b/internal/genkit/tagging.py index 021e730be07..a3897bbeedf 100644 --- a/internal/genkit/tagging.py +++ b/internal/genkit/tagging.py @@ -443,6 +443,18 @@ def get_previous_tag_info(package: Package) -> Optional[TagInfo]: return TagInfo(package=package, version=version, content=latest_release) +def _load_codegen_config() -> Dict: + """ + Loads ``.codegen.json`` from the repo root. Returns an empty dict when + the file is missing. + """ + package_file_path = os.path.join(os.getcwd(), CODEGEN_FILE_NAME) + if not os.path.exists(package_file_path): + return {} + with open(package_file_path, "r") as file: + return json.load(file) + + def get_next_tag_info(package: Package) -> Optional[TagInfo]: """ Extracts the changes from the "NEXT_CHANGELOG.md" file. @@ -461,7 +473,11 @@ def get_next_tag_info(package: Package) -> Optional[TagInfo]: # Ensure there is exactly one empty line before each section next_changelog = re.sub(r"(\n*)(###[^\n]+)", r"\n\n\2", next_changelog) - if not re.search(r"###", next_changelog): + # By default, packages whose NEXT_CHANGELOG.md has no populated + # sections are skipped — there's nothing meaningful to release. + # Repos like sdk-js which are still in development can opt in + # by setting ``allow_empty_changelog: true`` in .codegen.json. + if not re.search(r"###", next_changelog) and not _load_codegen_config().get("allow_empty_changelog", False): print("All sections are empty. No changes will be made to the changelog.") return None @@ -709,11 +725,11 @@ def generate_commit_message(tag_infos: List[TagInfo]) -> str: raise Exception("Multiple packages found in legacy mode") return f"[Release] Release v{info.version}\n\n{info.content}" - # Sort tag_infos by package name for consistency + # Sort tag_infos by package name for consistency. tag_infos.sort(key=lambda info: info.package.name) - return "Release\n\n" + "\n\n".join( - f"## {info.package.name}/v{info.version}\n\n{info.content}" for info in tag_infos - ) + titles = ", ".join(f"{info.package.name}/v{info.version}" for info in tag_infos) + body = "\n\n".join(f"## {info.package.name}/v{info.version}\n\n{info.content}" for info in tag_infos) + return f"[Release] {titles}\n\n{body}" def push_changes(tag_infos: List[TagInfo]) -> None: @@ -816,10 +832,82 @@ def preview_tag_infos(packages: List[Package]) -> List[TagInfo]: return [info for info in (get_next_tag_info(package) for package in packages) if info is not None] +def order_tag_infos_by_dependency(tag_infos: List[TagInfo]) -> List[TagInfo]: + """ + Returns ``tag_infos`` in topological order: every package appears + after every sibling it depends on. + """ + if not tag_infos: + return list(tag_infos) + + if any(not info.package.name for info in tag_infos) and len(tag_infos) > 1: + raise Exception("Multiple packages found in legacy mode") + + package_file_path = os.path.join(os.getcwd(), CODEGEN_FILE_NAME) + with open(package_file_path, "r") as file: + codegen = json.load(file) + + name_template = codegen.get("dependency_name_template", "") + dep_patterns = codegen.get("dependency_pattern", {}) + if not name_template or not dep_patterns: + return list(tag_infos) + + by_dep_name: Dict[str, TagInfo] = { + name_template.replace("$PACKAGE", info.package.name): info for info in tag_infos if info.package.name + } + + # Adjacency: path -> set of paths it depends on (within tag_infos). + deps: Dict[str, set] = {info.package.path: set() for info in tag_infos} + for info in tag_infos: + for filename, pattern in dep_patterns.items(): + loc = os.path.join(os.getcwd(), info.package.path, filename) + if not os.path.exists(loc): + continue + with open(loc, "r") as f: + content = f.read() + for dep_name, dep_info in by_dep_name.items(): + if dep_info.package.path == info.package.path: + continue + regex = ( + re.escape(pattern) + .replace(re.escape("$DEPENDENCY"), re.escape(dep_name)) + .replace(re.escape("$VERSION"), Version.PATTERN) + ) + if re.search(regex, content): + deps[info.package.path].add(dep_info.package.path) + + # Stable topological sort: at each step, emit every node whose deps + # are already emitted, alphabetically by package name. Ties broken + # alphabetically so the manifest is reproducible across runs. + emitted: set = set() + ordered: List[TagInfo] = [] + while len(ordered) < len(tag_infos): + ready = sorted( + ( + info + for info in tag_infos + if info.package.path not in emitted and deps[info.package.path].issubset(emitted) + ), + key=lambda info: info.package.name, + ) + if not ready: + remaining = [info.package.name for info in tag_infos if info.package.path not in emitted] + raise Exception(f"Cyclic dependency detected among packages: {remaining}") + for info in ready: + ordered.append(info) + emitted.add(info.package.path) + return ordered + + def push_tags(tag_infos: List[TagInfo]) -> None: """ Creates and pushes tags to the repository. + Tags are emitted in topological order — dependencies before + dependents — so downstream publishing pipelines reading + ``created_tags.json`` can walk it sequentially without re-deriving + the dependency graph. See ``order_tag_infos_by_dependency``. + As a side effect, writes the names of successfully created tags to ``./created_tags.json`` so that workflows triggering this script can discover what was produced (the GitHub Actions workflow uploads this @@ -833,6 +921,7 @@ def push_tags(tag_infos: List[TagInfo]) -> None: exception is re-raised, so recovery-mode runs still surface their output. """ + tag_infos = order_tag_infos_by_dependency(tag_infos) created: List[str] = [] try: for tag_info in tag_infos: diff --git a/internal/genkit/tagging.py.lock b/internal/genkit/tagging.py.lock index d680b187f5b..2bd746a66c2 100644 --- a/internal/genkit/tagging.py.lock +++ b/internal/genkit/tagging.py.lock @@ -12,7 +12,7 @@ requirements = [ [[package]] name = "certifi" version = "2026.2.25" -source = { registry = "https://pypi-proxy.dev.databricks.com/simple/" } +source = { registry = "https://pypi-proxy.cloud.databricks.com/simple/" } sdist = { url = "https://files.pythonhosted.org/packages/af/2d/7bf41579a8986e348fa033a31cdd0e4121114f6bce2457e8876010b092dd/certifi-2026.2.25.tar.gz", hash = "sha256:e887ab5cee78ea814d3472169153c2d12cd43b14bd03329a39a9c6e2e80bfba7", size = 155029, upload-time = "2026-02-25T02:54:17.342Z" } wheels = [ { url = "https://files.pythonhosted.org/packages/9a/3c/c17fb3ca2d9c3acff52e30b309f538586f9f5b9c9cf454f3845fc9af4881/certifi-2026.2.25-py3-none-any.whl", hash = "sha256:027692e4402ad994f1c42e52a4997a9763c646b73e4096e4d5d6db8af1d6f0fa", size = 153684, upload-time = "2026-02-25T02:54:15.766Z" }, @@ -21,7 +21,7 @@ wheels = [ [[package]] name = "cffi" version = "2.0.0" -source = { registry = "https://pypi-proxy.dev.databricks.com/simple/" } +source = { registry = "https://pypi-proxy.cloud.databricks.com/simple/" } dependencies = [ { name = "pycparser", marker = "implementation_name != 'PyPy'" }, ] @@ -78,7 +78,7 @@ wheels = [ [[package]] name = "charset-normalizer" version = "3.4.5" -source = { registry = "https://pypi-proxy.dev.databricks.com/simple/" } +source = { registry = "https://pypi-proxy.cloud.databricks.com/simple/" } sdist = { url = "https://files.pythonhosted.org/packages/1d/35/02daf95b9cd686320bb622eb148792655c9412dbb9b67abb5694e5910a24/charset_normalizer-3.4.5.tar.gz", hash = "sha256:95adae7b6c42a6c5b5b559b1a99149f090a57128155daeea91732c8d970d8644", size = 134804, upload-time = "2026-03-06T06:03:19.46Z" } wheels = [ { url = "https://files.pythonhosted.org/packages/9c/b6/9ee9c1a608916ca5feae81a344dffbaa53b26b90be58cc2159e3332d44ec/charset_normalizer-3.4.5-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:ed97c282ee4f994ef814042423a529df9497e3c666dca19be1d4cd1129dc7ade", size = 280976, upload-time = "2026-03-06T06:01:15.276Z" }, @@ -135,7 +135,7 @@ wheels = [ [[package]] name = "cryptography" version = "46.0.5" -source = { registry = "https://pypi-proxy.dev.databricks.com/simple/" } +source = { registry = "https://pypi-proxy.cloud.databricks.com/simple/" } dependencies = [ { name = "cffi", marker = "platform_python_implementation != 'PyPy'" }, ] @@ -188,7 +188,7 @@ wheels = [ [[package]] name = "idna" version = "3.11" -source = { registry = "https://pypi-proxy.dev.databricks.com/simple/" } +source = { registry = "https://pypi-proxy.cloud.databricks.com/simple/" } sdist = { url = "https://files.pythonhosted.org/packages/6f/6d/0703ccc57f3a7233505399edb88de3cbd678da106337b9fcde432b65ed60/idna-3.11.tar.gz", hash = "sha256:795dafcc9c04ed0c1fb032c2aa73654d8e8c5023a7df64a53f39190ada629902", size = 194582, upload-time = "2025-10-12T14:55:20.501Z" } wheels = [ { url = "https://files.pythonhosted.org/packages/0e/61/66938bbb5fc52dbdf84594873d5b51fb1f7c7794e9c0f5bd885f30bc507b/idna-3.11-py3-none-any.whl", hash = "sha256:771a87f49d9defaf64091e6e6fe9c18d4833f140bd19464795bc32d966ca37ea", size = 71008, upload-time = "2025-10-12T14:55:18.883Z" }, @@ -197,7 +197,7 @@ wheels = [ [[package]] name = "pycparser" version = "3.0" -source = { registry = "https://pypi-proxy.dev.databricks.com/simple/" } +source = { registry = "https://pypi-proxy.cloud.databricks.com/simple/" } sdist = { url = "https://files.pythonhosted.org/packages/1b/7d/92392ff7815c21062bea51aa7b87d45576f649f16458d78b7cf94b9ab2e6/pycparser-3.0.tar.gz", hash = "sha256:600f49d217304a5902ac3c37e1281c9fe94e4d0489de643a9504c5cdfdfc6b29", size = 103492, upload-time = "2026-01-21T14:26:51.89Z" } wheels = [ { url = "https://files.pythonhosted.org/packages/0c/c3/44f3fbbfa403ea2a7c779186dc20772604442dde72947e7d01069cbe98e3/pycparser-3.0-py3-none-any.whl", hash = "sha256:b727414169a36b7d524c1c3e31839a521725078d7b2ff038656844266160a992", size = 48172, upload-time = "2026-01-21T14:26:50.693Z" }, @@ -206,7 +206,7 @@ wheels = [ [[package]] name = "pygithub" version = "2.8.1" -source = { registry = "https://pypi-proxy.dev.databricks.com/simple/" } +source = { registry = "https://pypi-proxy.cloud.databricks.com/simple/" } dependencies = [ { name = "pyjwt", extra = ["crypto"] }, { name = "pynacl" }, @@ -222,7 +222,7 @@ wheels = [ [[package]] name = "pyjwt" version = "2.11.0" -source = { registry = "https://pypi-proxy.dev.databricks.com/simple/" } +source = { registry = "https://pypi-proxy.cloud.databricks.com/simple/" } sdist = { url = "https://files.pythonhosted.org/packages/5c/5a/b46fa56bf322901eee5b0454a34343cdbdae202cd421775a8ee4e42fd519/pyjwt-2.11.0.tar.gz", hash = "sha256:35f95c1f0fbe5d5ba6e43f00271c275f7a1a4db1dab27bf708073b75318ea623", size = 98019, upload-time = "2026-01-30T19:59:55.694Z" } wheels = [ { url = "https://files.pythonhosted.org/packages/6f/01/c26ce75ba460d5cd503da9e13b21a33804d38c2165dec7b716d06b13010c/pyjwt-2.11.0-py3-none-any.whl", hash = "sha256:94a6bde30eb5c8e04fee991062b534071fd1439ef58d2adc9ccb823e7bcd0469", size = 28224, upload-time = "2026-01-30T19:59:54.539Z" }, @@ -236,7 +236,7 @@ crypto = [ [[package]] name = "pynacl" version = "1.6.2" -source = { registry = "https://pypi-proxy.dev.databricks.com/simple/" } +source = { registry = "https://pypi-proxy.cloud.databricks.com/simple/" } dependencies = [ { name = "cffi", marker = "platform_python_implementation != 'PyPy'" }, ] @@ -271,7 +271,7 @@ wheels = [ [[package]] name = "requests" version = "2.32.5" -source = { registry = "https://pypi-proxy.dev.databricks.com/simple/" } +source = { registry = "https://pypi-proxy.cloud.databricks.com/simple/" } dependencies = [ { name = "certifi" }, { name = "charset-normalizer" }, @@ -286,7 +286,7 @@ wheels = [ [[package]] name = "typing-extensions" version = "4.15.0" -source = { registry = "https://pypi-proxy.dev.databricks.com/simple/" } +source = { registry = "https://pypi-proxy.cloud.databricks.com/simple/" } sdist = { url = "https://files.pythonhosted.org/packages/72/94/1a15dd82efb362ac84269196e94cf00f187f7ed21c242792a923cdb1c61f/typing_extensions-4.15.0.tar.gz", hash = "sha256:0cea48d173cc12fa28ecabc3b837ea3cf6f38c6d1136f85cbaaf598984861466", size = 109391, upload-time = "2025-08-25T13:49:26.313Z" } wheels = [ { url = "https://files.pythonhosted.org/packages/18/67/36e9267722cc04a6b9f15c7f3441c2363321a3ea07da7ae0c0707beb2a9c/typing_extensions-4.15.0-py3-none-any.whl", hash = "sha256:f0fa19c6845758ab08074a0cfa8b7aecb71c999ca73d62883bc25cc018c4e548", size = 44614, upload-time = "2025-08-25T13:49:24.86Z" }, @@ -295,7 +295,7 @@ wheels = [ [[package]] name = "urllib3" version = "2.6.3" -source = { registry = "https://pypi-proxy.dev.databricks.com/simple/" } +source = { registry = "https://pypi-proxy.cloud.databricks.com/simple/" } sdist = { url = "https://files.pythonhosted.org/packages/c7/24/5f1b3bdffd70275f6661c76461e25f024d5a38a46f04aaca912426a2b1d3/urllib3-2.6.3.tar.gz", hash = "sha256:1b62b6884944a57dbe321509ab94fd4d3b307075e0c2eae991ac71ee15ad38ed", size = 435556, upload-time = "2026-01-07T16:24:43.925Z" } wheels = [ { url = "https://files.pythonhosted.org/packages/39/08/aaaad47bc4e9dc8c725e68f9d04865dbcb2052843ff09c97b08904852d84/urllib3-2.6.3-py3-none-any.whl", hash = "sha256:bf272323e553dfb2e87d9bfd225ca7b0f467b919d7bbd355436d3fd37cb0acd4", size = 131584, upload-time = "2026-01-07T16:24:42.685Z" }, diff --git a/libs/databrickscfg/cfgpickers/clusters.go b/libs/databrickscfg/cfgpickers/clusters.go index 4d6aba6b18a..f01f567007a 100644 --- a/libs/databrickscfg/cfgpickers/clusters.go +++ b/libs/databrickscfg/cfgpickers/clusters.go @@ -140,7 +140,7 @@ func loadInteractiveClusters(ctx context.Context, w *databricks.WorkspaceClient, if err != nil { return nil, fmt.Errorf("list clusters: %w", err) } - me, err := w.CurrentUser.Me(ctx) + me, err := w.CurrentUser.Me(ctx, iam.MeRequest{}) if err != nil { return nil, fmt.Errorf("current user: %w", err) } diff --git a/libs/databrickscfg/cfgpickers/clusters_test.go b/libs/databrickscfg/cfgpickers/clusters_test.go index 025a0f43253..4fad46e038b 100644 --- a/libs/databrickscfg/cfgpickers/clusters_test.go +++ b/libs/databrickscfg/cfgpickers/clusters_test.go @@ -92,7 +92,7 @@ func TestFirstCompatibleCluster(t *testing.T) { }, { Method: "GET", - Resource: "/api/2.0/preview/scim/v2/Me", + Resource: "/api/2.0/preview/scim/v2/Me?", Response: iam.User{ UserName: "serge", }, @@ -139,7 +139,7 @@ func TestNoCompatibleClusters(t *testing.T) { }, { Method: "GET", - Resource: "/api/2.0/preview/scim/v2/Me", + Resource: "/api/2.0/preview/scim/v2/Me?", Response: iam.User{ UserName: "serge", }, diff --git a/libs/structs/structwalk/walktype_test.go b/libs/structs/structwalk/walktype_test.go index d77002982a9..5db64efc0d7 100644 --- a/libs/structs/structwalk/walktype_test.go +++ b/libs/structs/structwalk/walktype_test.go @@ -112,8 +112,8 @@ func testStruct(t *testing.T, typ reflect.Type, minLen, maxLen int, present map[ func TestTypeJobSettings(t *testing.T) { testStruct(t, reflect.TypeFor[jobs.JobSettings](), - // Verify we found a reasonable number of fields (it's 533 at the time of writing) - 500, 600, + // Verify we found a reasonable number of fields (605 after SDK v0.136.0 bump) + 500, 650, map[string]any{ "name": "", "timeout_seconds": 0, @@ -136,7 +136,7 @@ func TestTypeJobSettings(t *testing.T) { func TestTypeRoot(t *testing.T) { testStruct(t, reflect.TypeFor[config.Root](), - 5000, 5500, // 5213 after SDK v0.127.0 bump + 5000, 5800, // 5651 after SDK v0.136.0 bump map[string]any{ "bundle.target": "", `variables.*.lookup.dashboard`: "", diff --git a/libs/sync/path.go b/libs/sync/path.go index 3f86e8d3446..6976fd8e0d6 100644 --- a/libs/sync/path.go +++ b/libs/sync/path.go @@ -30,7 +30,7 @@ func EnsureRemotePathIsUsable(ctx context.Context, wsc *databricks.WorkspaceClie // TODO: we should cache CurrentUser.Me at the SDK level // for now we let clients pass in any existing user they might already have if me == nil { - me, err = wsc.CurrentUser.Me(ctx) + me, err = wsc.CurrentUser.Me(ctx, iam.MeRequest{}) if err != nil { return err } diff --git a/libs/template/helpers.go b/libs/template/helpers.go index eefb79537ef..33e36c02483 100644 --- a/libs/template/helpers.go +++ b/libs/template/helpers.go @@ -120,7 +120,7 @@ func loadHelpers(ctx context.Context) template.FuncMap { "user_name": func() (string, error) { if cachedUser == nil { var err error - cachedUser, err = w.CurrentUser.Me(ctx) + cachedUser, err = w.CurrentUser.Me(ctx, iam.MeRequest{}) if err != nil { return "", err } @@ -134,7 +134,7 @@ func loadHelpers(ctx context.Context) template.FuncMap { "short_name": func() (string, error) { if cachedUser == nil { var err error - cachedUser, err = w.CurrentUser.Me(ctx) + cachedUser, err = w.CurrentUser.Me(ctx, iam.MeRequest{}) if err != nil { return "", err } @@ -166,7 +166,7 @@ func loadHelpers(ctx context.Context) template.FuncMap { } if cachedUser == nil { var err error - cachedUser, err = w.CurrentUser.Me(ctx) + cachedUser, err = w.CurrentUser.Me(ctx, iam.MeRequest{}) if err != nil { return false, err } diff --git a/python/databricks/bundles/catalogs/_models/privilege.py b/python/databricks/bundles/catalogs/_models/privilege.py index 6a0215a210c..74f25b462c1 100644 --- a/python/databricks/bundles/catalogs/_models/privilege.py +++ b/python/databricks/bundles/catalogs/_models/privilege.py @@ -65,6 +65,8 @@ class Privilege(Enum): MANAGE_ACCESS = "MANAGE_ACCESS" MANAGE_ACCESS_CONTROL = "MANAGE_ACCESS_CONTROL" CREATE_SERVICE = "CREATE_SERVICE" + CREATE_FEATURE = "CREATE_FEATURE" + READ_FEATURE = "READ_FEATURE" PrivilegeParam = ( @@ -131,6 +133,8 @@ class Privilege(Enum): "MANAGE_ACCESS", "MANAGE_ACCESS_CONTROL", "CREATE_SERVICE", + "CREATE_FEATURE", + "READ_FEATURE", ] | Privilege ) diff --git a/python/databricks/bundles/jobs/__init__.py b/python/databricks/bundles/jobs/__init__.py index 308c4ba7faf..5ed6b101421 100644 --- a/python/databricks/bundles/jobs/__init__.py +++ b/python/databricks/bundles/jobs/__init__.py @@ -196,6 +196,12 @@ "PowerBiTask", "PowerBiTaskDict", "PowerBiTaskParam", + "PythonOperatorTask", + "PythonOperatorTaskDict", + "PythonOperatorTaskParam", + "PythonOperatorTaskParameter", + "PythonOperatorTaskParameterDict", + "PythonOperatorTaskParameterParam", "PythonPyPiLibrary", "PythonPyPiLibraryDict", "PythonPyPiLibraryParam", @@ -609,6 +615,16 @@ PowerBiTaskDict, PowerBiTaskParam, ) +from databricks.bundles.jobs._models.python_operator_task import ( + PythonOperatorTask, + PythonOperatorTaskDict, + PythonOperatorTaskParam, +) +from databricks.bundles.jobs._models.python_operator_task_parameter import ( + PythonOperatorTaskParameter, + PythonOperatorTaskParameterDict, + PythonOperatorTaskParameterParam, +) from databricks.bundles.jobs._models.python_py_pi_library import ( PythonPyPiLibrary, PythonPyPiLibraryDict, diff --git a/python/databricks/bundles/jobs/_models/data_security_mode.py b/python/databricks/bundles/jobs/_models/data_security_mode.py index e195d0cfc8d..2a050bfddc5 100644 --- a/python/databricks/bundles/jobs/_models/data_security_mode.py +++ b/python/databricks/bundles/jobs/_models/data_security_mode.py @@ -7,15 +7,14 @@ class DataSecurityMode(Enum): Data security mode decides what data governance model to use when accessing data from a cluster. - The following modes can only be used when `kind = CLASSIC_PREVIEW`. * `DATA_SECURITY_MODE_AUTO`: Databricks will choose the most appropriate access mode depending on your compute configuration. - * `DATA_SECURITY_MODE_STANDARD`: Alias for `USER_ISOLATION`. - * `DATA_SECURITY_MODE_DEDICATED`: Alias for `SINGLE_USER`. + * `DATA_SECURITY_MODE_STANDARD`: A secure cluster that can be shared by multiple users. Cluster users are fully isolated so that they cannot see each other’s data and credentials. Most data governance features are supported in this mode. But programming languages and cluster features might be limited. + * `DATA_SECURITY_MODE_DEDICATED`: A secure cluster that can only be exclusively used by a single user specified in `single_user_name`. Most programming languages, cluster features and data governance features are available in this mode. - The following modes can be used regardless of `kind`. - * `NONE`: No security isolation for multiple users sharing the cluster. Data governance features are not available in this mode. - * `SINGLE_USER`: A secure cluster that can only be exclusively used by a single user specified in `single_user_name`. Most programming languages, cluster features and data governance features are available in this mode. - * `USER_ISOLATION`: A secure cluster that can be shared by multiple users. Cluster users are fully isolated so that they cannot see each other's data and credentials. Most data governance features are supported in this mode. But programming languages and cluster features might be limited. + The following modes are legacy aliases for the above modes: + + * `USER_ISOLATION`: Legacy alias for `DATA_SECURITY_MODE_STANDARD`. + * `SINGLE_USER`: Legacy alias for `DATA_SECURITY_MODE_DEDICATED`. The following modes are deprecated starting with Databricks Runtime 15.0 and will be removed for future Databricks Runtime versions: diff --git a/python/databricks/bundles/jobs/_models/environment.py b/python/databricks/bundles/jobs/_models/environment.py index 3309d6f6d2d..4324405f024 100644 --- a/python/databricks/bundles/jobs/_models/environment.py +++ b/python/databricks/bundles/jobs/_models/environment.py @@ -23,7 +23,10 @@ class Environment: (e.g., `/Workspace/path/to/env.yaml`). Support for a Databricks-provided base environment ID (e.g., `workspace-base-environments/databricks_ai_v4`) and workspace base environment ID (e.g., `workspace-base-environments/dbe_b849b66e-b31a-4cb5-b161-1f2b10877fb7`) is in Beta. - Either `environment_version` or `base_environment` can be provided. For more information, see + Either `environment_version` or `base_environment` can be provided. + For more information about Databricks-provided base environments, see the + [list workspace base environments](:method:Environments/ListWorkspaceBaseEnvironments) API. + For more information, see """ client: VariableOrOptional[str] = None @@ -63,7 +66,10 @@ class EnvironmentDict(TypedDict, total=False): (e.g., `/Workspace/path/to/env.yaml`). Support for a Databricks-provided base environment ID (e.g., `workspace-base-environments/databricks_ai_v4`) and workspace base environment ID (e.g., `workspace-base-environments/dbe_b849b66e-b31a-4cb5-b161-1f2b10877fb7`) is in Beta. - Either `environment_version` or `base_environment` can be provided. For more information, see + Either `environment_version` or `base_environment` can be provided. + For more information about Databricks-provided base environments, see the + [list workspace base environments](:method:Environments/ListWorkspaceBaseEnvironments) API. + For more information, see """ client: VariableOrOptional[str] diff --git a/python/databricks/bundles/jobs/_models/pipeline_params.py b/python/databricks/bundles/jobs/_models/pipeline_params.py index 69dd3b77797..488018cb3d8 100644 --- a/python/databricks/bundles/jobs/_models/pipeline_params.py +++ b/python/databricks/bundles/jobs/_models/pipeline_params.py @@ -1,9 +1,9 @@ -from dataclasses import dataclass +from dataclasses import dataclass, field from typing import TYPE_CHECKING, TypedDict from databricks.bundles.core._transform import _transform from databricks.bundles.core._transform_to_json import _transform_to_json_value -from databricks.bundles.core._variable import VariableOrOptional +from databricks.bundles.core._variable import VariableOrList, VariableOrOptional if TYPE_CHECKING: from typing_extensions import Self @@ -18,6 +18,27 @@ class PipelineParams: If true, triggers a full refresh on the spark declarative pipeline. """ + full_refresh_selection: VariableOrList[str] = field(default_factory=list) + """ + A list of tables to update with fullRefresh. + """ + + refresh_flow_selection: VariableOrList[str] = field(default_factory=list) + """ + Flow names to selectively refresh. These are unioned with other selective refresh + options (refresh_selection, full_refresh_selection) to determine the final set of flows to refresh. + """ + + refresh_selection: VariableOrList[str] = field(default_factory=list) + """ + A list of tables to update without fullRefresh. + """ + + reset_checkpoint_selection: VariableOrList[str] = field(default_factory=list) + """ + A list of streaming flows to reset checkpoints without clearing data. + """ + @classmethod def from_dict(cls, value: "PipelineParamsDict") -> "Self": return _transform(cls, value) @@ -34,5 +55,26 @@ class PipelineParamsDict(TypedDict, total=False): If true, triggers a full refresh on the spark declarative pipeline. """ + full_refresh_selection: VariableOrList[str] + """ + A list of tables to update with fullRefresh. + """ + + refresh_flow_selection: VariableOrList[str] + """ + Flow names to selectively refresh. These are unioned with other selective refresh + options (refresh_selection, full_refresh_selection) to determine the final set of flows to refresh. + """ + + refresh_selection: VariableOrList[str] + """ + A list of tables to update without fullRefresh. + """ + + reset_checkpoint_selection: VariableOrList[str] + """ + A list of streaming flows to reset checkpoints without clearing data. + """ + PipelineParamsParam = PipelineParamsDict | PipelineParams diff --git a/python/databricks/bundles/jobs/_models/pipeline_task.py b/python/databricks/bundles/jobs/_models/pipeline_task.py index a5c22bdd986..9bca9b3e53b 100644 --- a/python/databricks/bundles/jobs/_models/pipeline_task.py +++ b/python/databricks/bundles/jobs/_models/pipeline_task.py @@ -1,9 +1,13 @@ -from dataclasses import dataclass +from dataclasses import dataclass, field from typing import TYPE_CHECKING, TypedDict from databricks.bundles.core._transform import _transform from databricks.bundles.core._transform_to_json import _transform_to_json_value -from databricks.bundles.core._variable import VariableOr, VariableOrOptional +from databricks.bundles.core._variable import ( + VariableOr, + VariableOrList, + VariableOrOptional, +) if TYPE_CHECKING: from typing_extensions import Self @@ -23,6 +27,27 @@ class PipelineTask: If true, triggers a full refresh on the spark declarative pipeline. """ + full_refresh_selection: VariableOrList[str] = field(default_factory=list) + """ + A list of tables to update with fullRefresh. + """ + + refresh_flow_selection: VariableOrList[str] = field(default_factory=list) + """ + Flow names to selectively refresh. These are unioned with other selective refresh + options (refresh_selection, full_refresh_selection) to determine the final set of flows to refresh. + """ + + refresh_selection: VariableOrList[str] = field(default_factory=list) + """ + A list of tables to update without fullRefresh. + """ + + reset_checkpoint_selection: VariableOrList[str] = field(default_factory=list) + """ + A list of streaming flows to reset checkpoints without clearing data. + """ + @classmethod def from_dict(cls, value: "PipelineTaskDict") -> "Self": return _transform(cls, value) @@ -44,5 +69,26 @@ class PipelineTaskDict(TypedDict, total=False): If true, triggers a full refresh on the spark declarative pipeline. """ + full_refresh_selection: VariableOrList[str] + """ + A list of tables to update with fullRefresh. + """ + + refresh_flow_selection: VariableOrList[str] + """ + Flow names to selectively refresh. These are unioned with other selective refresh + options (refresh_selection, full_refresh_selection) to determine the final set of flows to refresh. + """ + + refresh_selection: VariableOrList[str] + """ + A list of tables to update without fullRefresh. + """ + + reset_checkpoint_selection: VariableOrList[str] + """ + A list of streaming flows to reset checkpoints without clearing data. + """ + PipelineTaskParam = PipelineTaskDict | PipelineTask diff --git a/python/databricks/bundles/jobs/_models/python_operator_task.py b/python/databricks/bundles/jobs/_models/python_operator_task.py new file mode 100644 index 00000000000..fbce9e38cde --- /dev/null +++ b/python/databricks/bundles/jobs/_models/python_operator_task.py @@ -0,0 +1,60 @@ +from dataclasses import dataclass, field +from typing import TYPE_CHECKING, TypedDict + +from databricks.bundles.core._transform import _transform +from databricks.bundles.core._transform_to_json import _transform_to_json_value +from databricks.bundles.core._variable import VariableOrList, VariableOrOptional +from databricks.bundles.jobs._models.python_operator_task_parameter import ( + PythonOperatorTaskParameter, + PythonOperatorTaskParameterParam, +) + +if TYPE_CHECKING: + from typing_extensions import Self + + +@dataclass(kw_only=True) +class PythonOperatorTask: + """ + :meta private: [EXPERIMENTAL] + """ + + main: VariableOrOptional[str] = None + """ + Fully qualified name of the main class or function. + For example, `my_project.my_function` or `my_project.MyOperator`. + """ + + parameters: VariableOrList[PythonOperatorTaskParameter] = field( + default_factory=list + ) + """ + An ordered list of task parameters. + TODO(JOBS-30885): Add limits for parameters. + """ + + @classmethod + def from_dict(cls, value: "PythonOperatorTaskDict") -> "Self": + return _transform(cls, value) + + def as_dict(self) -> "PythonOperatorTaskDict": + return _transform_to_json_value(self) # type:ignore + + +class PythonOperatorTaskDict(TypedDict, total=False): + """""" + + main: VariableOrOptional[str] + """ + Fully qualified name of the main class or function. + For example, `my_project.my_function` or `my_project.MyOperator`. + """ + + parameters: VariableOrList[PythonOperatorTaskParameterParam] + """ + An ordered list of task parameters. + TODO(JOBS-30885): Add limits for parameters. + """ + + +PythonOperatorTaskParam = PythonOperatorTaskDict | PythonOperatorTask diff --git a/python/databricks/bundles/jobs/_models/python_operator_task_parameter.py b/python/databricks/bundles/jobs/_models/python_operator_task_parameter.py new file mode 100644 index 00000000000..5fd3628748d --- /dev/null +++ b/python/databricks/bundles/jobs/_models/python_operator_task_parameter.py @@ -0,0 +1,40 @@ +from dataclasses import dataclass +from typing import TYPE_CHECKING, TypedDict + +from databricks.bundles.core._transform import _transform +from databricks.bundles.core._transform_to_json import _transform_to_json_value +from databricks.bundles.core._variable import VariableOrOptional + +if TYPE_CHECKING: + from typing_extensions import Self + + +@dataclass(kw_only=True) +class PythonOperatorTaskParameter: + """ + :meta private: [EXPERIMENTAL] + """ + + name: VariableOrOptional[str] = None + + value: VariableOrOptional[str] = None + + @classmethod + def from_dict(cls, value: "PythonOperatorTaskParameterDict") -> "Self": + return _transform(cls, value) + + def as_dict(self) -> "PythonOperatorTaskParameterDict": + return _transform_to_json_value(self) # type:ignore + + +class PythonOperatorTaskParameterDict(TypedDict, total=False): + """""" + + name: VariableOrOptional[str] + + value: VariableOrOptional[str] + + +PythonOperatorTaskParameterParam = ( + PythonOperatorTaskParameterDict | PythonOperatorTaskParameter +) diff --git a/python/databricks/bundles/jobs/_models/task.py b/python/databricks/bundles/jobs/_models/task.py index cfac99bc571..8cb2fe0c273 100644 --- a/python/databricks/bundles/jobs/_models/task.py +++ b/python/databricks/bundles/jobs/_models/task.py @@ -50,6 +50,10 @@ PipelineTaskParam, ) from databricks.bundles.jobs._models.power_bi_task import PowerBiTask, PowerBiTaskParam +from databricks.bundles.jobs._models.python_operator_task import ( + PythonOperatorTask, + PythonOperatorTaskParam, +) from databricks.bundles.jobs._models.python_wheel_task import ( PythonWheelTask, PythonWheelTaskParam, @@ -160,8 +164,6 @@ class Task: disabled: VariableOrOptional[bool] = None """ - :meta private: [EXPERIMENTAL] - An optional flag to disable the task. If set to true, the task will not run even if it is part of a job. """ @@ -241,6 +243,13 @@ class Task: The task triggers a Power BI semantic model update when the `power_bi_task` field is present. """ + python_operator_task: VariableOrOptional[PythonOperatorTask] = None + """ + :meta private: [EXPERIMENTAL] + + The task runs a Python operator task. + """ + python_wheel_task: VariableOrOptional[PythonWheelTask] = None """ The task runs a Python wheel when the `python_wheel_task` field is present. @@ -373,8 +382,6 @@ class TaskDict(TypedDict, total=False): disabled: VariableOrOptional[bool] """ - :meta private: [EXPERIMENTAL] - An optional flag to disable the task. If set to true, the task will not run even if it is part of a job. """ @@ -454,6 +461,13 @@ class TaskDict(TypedDict, total=False): The task triggers a Power BI semantic model update when the `power_bi_task` field is present. """ + python_operator_task: VariableOrOptional[PythonOperatorTaskParam] + """ + :meta private: [EXPERIMENTAL] + + The task runs a Python operator task. + """ + python_wheel_task: VariableOrOptional[PythonWheelTaskParam] """ The task runs a Python wheel when the `python_wheel_task` field is present. diff --git a/python/databricks/bundles/pipelines/__init__.py b/python/databricks/bundles/pipelines/__init__.py index 2ef4c1e6c72..c7dbd9b41e8 100644 --- a/python/databricks/bundles/pipelines/__init__.py +++ b/python/databricks/bundles/pipelines/__init__.py @@ -103,6 +103,12 @@ "JiraConnectorOptions", "JiraConnectorOptionsDict", "JiraConnectorOptionsParam", + "JsonTransformerOptions", + "JsonTransformerOptionsDict", + "JsonTransformerOptionsParam", + "KafkaOptions", + "KafkaOptionsDict", + "KafkaOptionsParam", "Lifecycle", "LifecycleDict", "LifecycleParam", @@ -209,6 +215,11 @@ "TikTokAdsOptionsTikTokDataLevelParam", "TikTokAdsOptionsTikTokReportType", "TikTokAdsOptionsTikTokReportTypeParam", + "Transformer", + "TransformerDict", + "TransformerFormat", + "TransformerFormatParam", + "TransformerParam", "VolumesStorageInfo", "VolumesStorageInfoDict", "VolumesStorageInfoParam", @@ -398,6 +409,16 @@ JiraConnectorOptionsDict, JiraConnectorOptionsParam, ) +from databricks.bundles.pipelines._models.json_transformer_options import ( + JsonTransformerOptions, + JsonTransformerOptionsDict, + JsonTransformerOptionsParam, +) +from databricks.bundles.pipelines._models.kafka_options import ( + KafkaOptions, + KafkaOptionsDict, + KafkaOptionsParam, +) from databricks.bundles.pipelines._models.lifecycle import ( Lifecycle, LifecycleDict, @@ -576,6 +597,15 @@ TikTokAdsOptionsTikTokReportType, TikTokAdsOptionsTikTokReportTypeParam, ) +from databricks.bundles.pipelines._models.transformer import ( + Transformer, + TransformerDict, + TransformerParam, +) +from databricks.bundles.pipelines._models.transformer_format import ( + TransformerFormat, + TransformerFormatParam, +) from databricks.bundles.pipelines._models.volumes_storage_info import ( VolumesStorageInfo, VolumesStorageInfoDict, diff --git a/python/databricks/bundles/pipelines/_models/connector_options.py b/python/databricks/bundles/pipelines/_models/connector_options.py index 3f8b7add1ff..b236e16f762 100644 --- a/python/databricks/bundles/pipelines/_models/connector_options.py +++ b/python/databricks/bundles/pipelines/_models/connector_options.py @@ -20,6 +20,10 @@ JiraConnectorOptions, JiraConnectorOptionsParam, ) +from databricks.bundles.pipelines._models.kafka_options import ( + KafkaOptions, + KafkaOptionsParam, +) from databricks.bundles.pipelines._models.meta_marketing_options import ( MetaMarketingOptions, MetaMarketingOptionsParam, @@ -79,6 +83,11 @@ class ConnectorOptions: Jira specific options for ingestion """ + kafka_options: VariableOrOptional[KafkaOptions] = None + """ + :meta private: [EXPERIMENTAL] + """ + meta_ads_options: VariableOrOptional[MetaMarketingOptions] = None """ Meta Marketing (Meta Ads) specific options for ingestion @@ -152,6 +161,11 @@ class ConnectorOptionsDict(TypedDict, total=False): Jira specific options for ingestion """ + kafka_options: VariableOrOptional[KafkaOptionsParam] + """ + :meta private: [EXPERIMENTAL] + """ + meta_ads_options: VariableOrOptional[MetaMarketingOptionsParam] """ Meta Marketing (Meta Ads) specific options for ingestion diff --git a/python/databricks/bundles/pipelines/_models/file_ingestion_options_file_format.py b/python/databricks/bundles/pipelines/_models/file_ingestion_options_file_format.py index bf5838c8701..b9295c3378f 100644 --- a/python/databricks/bundles/pipelines/_models/file_ingestion_options_file_format.py +++ b/python/databricks/bundles/pipelines/_models/file_ingestion_options_file_format.py @@ -15,9 +15,12 @@ class FileIngestionOptionsFileFormat(Enum): PARQUET = "PARQUET" AVRO = "AVRO" ORC = "ORC" + FILE = "FILE" FileIngestionOptionsFileFormatParam = ( - Literal["BINARYFILE", "JSON", "CSV", "XML", "EXCEL", "PARQUET", "AVRO", "ORC"] + Literal[ + "BINARYFILE", "JSON", "CSV", "XML", "EXCEL", "PARQUET", "AVRO", "ORC", "FILE" + ] | FileIngestionOptionsFileFormat ) diff --git a/python/databricks/bundles/pipelines/_models/google_drive_options_google_drive_entity_type.py b/python/databricks/bundles/pipelines/_models/google_drive_options_google_drive_entity_type.py index 136b3aff953..4ddab6c6792 100644 --- a/python/databricks/bundles/pipelines/_models/google_drive_options_google_drive_entity_type.py +++ b/python/databricks/bundles/pipelines/_models/google_drive_options_google_drive_entity_type.py @@ -10,9 +10,13 @@ class GoogleDriveOptionsGoogleDriveEntityType(Enum): FILE = "FILE" FILE_METADATA = "FILE_METADATA" PERMISSION = "PERMISSION" + FILE_PERMISSION = "FILE_PERMISSION" + GROUP_MEMBERSHIP = "GROUP_MEMBERSHIP" GoogleDriveOptionsGoogleDriveEntityTypeParam = ( - Literal["FILE", "FILE_METADATA", "PERMISSION"] + Literal[ + "FILE", "FILE_METADATA", "PERMISSION", "FILE_PERMISSION", "GROUP_MEMBERSHIP" + ] | GoogleDriveOptionsGoogleDriveEntityType ) diff --git a/python/databricks/bundles/pipelines/_models/json_transformer_options.py b/python/databricks/bundles/pipelines/_models/json_transformer_options.py new file mode 100644 index 00000000000..26b310204b9 --- /dev/null +++ b/python/databricks/bundles/pipelines/_models/json_transformer_options.py @@ -0,0 +1,88 @@ +from dataclasses import dataclass +from typing import TYPE_CHECKING, TypedDict + +from databricks.bundles.core._transform import _transform +from databricks.bundles.core._transform_to_json import _transform_to_json_value +from databricks.bundles.core._variable import VariableOrOptional +from databricks.bundles.pipelines._models.file_ingestion_options_schema_evolution_mode import ( + FileIngestionOptionsSchemaEvolutionMode, + FileIngestionOptionsSchemaEvolutionModeParam, +) + +if TYPE_CHECKING: + from typing_extensions import Self + + +@dataclass(kw_only=True) +class JsonTransformerOptions: + """ + :meta private: [EXPERIMENTAL] + """ + + as_variant: VariableOrOptional[bool] = None + """ + Parse the entire value as a single Variant column. + """ + + schema: VariableOrOptional[str] = None + """ + Inline schema string for JSON parsing (Spark DDL format). + """ + + schema_evolution_mode: VariableOrOptional[ + FileIngestionOptionsSchemaEvolutionMode + ] = None + """ + (Optional) Schema evolution mode for schema inference. + """ + + schema_file_path: VariableOrOptional[str] = None + """ + Path to a schema file (.ddl). + """ + + schema_hints: VariableOrOptional[str] = None + """ + (Optional) Schema hints as a comma-separated string of "column_name type" pairs. + """ + + @classmethod + def from_dict(cls, value: "JsonTransformerOptionsDict") -> "Self": + return _transform(cls, value) + + def as_dict(self) -> "JsonTransformerOptionsDict": + return _transform_to_json_value(self) # type:ignore + + +class JsonTransformerOptionsDict(TypedDict, total=False): + """""" + + as_variant: VariableOrOptional[bool] + """ + Parse the entire value as a single Variant column. + """ + + schema: VariableOrOptional[str] + """ + Inline schema string for JSON parsing (Spark DDL format). + """ + + schema_evolution_mode: VariableOrOptional[ + FileIngestionOptionsSchemaEvolutionModeParam + ] + """ + (Optional) Schema evolution mode for schema inference. + """ + + schema_file_path: VariableOrOptional[str] + """ + Path to a schema file (.ddl). + """ + + schema_hints: VariableOrOptional[str] + """ + (Optional) Schema hints as a comma-separated string of "column_name type" pairs. + """ + + +JsonTransformerOptionsParam = JsonTransformerOptionsDict | JsonTransformerOptions diff --git a/python/databricks/bundles/pipelines/_models/kafka_options.py b/python/databricks/bundles/pipelines/_models/kafka_options.py new file mode 100644 index 00000000000..0b83708ee3a --- /dev/null +++ b/python/databricks/bundles/pipelines/_models/kafka_options.py @@ -0,0 +1,130 @@ +from dataclasses import dataclass, field +from typing import TYPE_CHECKING, TypedDict + +from databricks.bundles.core._transform import _transform +from databricks.bundles.core._transform_to_json import _transform_to_json_value +from databricks.bundles.core._variable import ( + VariableOrDict, + VariableOrList, + VariableOrOptional, +) +from databricks.bundles.pipelines._models.transformer import ( + Transformer, + TransformerParam, +) + +if TYPE_CHECKING: + from typing_extensions import Self + + +@dataclass(kw_only=True) +class KafkaOptions: + """ + :meta private: [EXPERIMENTAL] + """ + + client_config: VariableOrDict[str] = field(default_factory=dict) + """ + :meta private: [EXPERIMENTAL] + + Undocumented backdoor mechanism for overriding parameters + to pass to the Kafka client. + This is not supported and may break at any time. + """ + + key_transformer: VariableOrOptional[Transformer] = None + """ + (Optional) Transformer for the message key. + If not specified, the key is left as raw bytes. + """ + + max_offsets_per_trigger: VariableOrOptional[int] = None + """ + :meta private: [EXPERIMENTAL] + + Internal option to control the maximum number of offsets to process per trigger. + """ + + starting_offset: VariableOrOptional[str] = None + """ + (Optional) Where to begin reading when no checkpoint exists. + Valid values: "latest" and "earliest". Defaults to "latest". + """ + + topic_pattern: VariableOrOptional[str] = None + """ + Java regex pattern to subscribe to matching topics. + Only one of topics or topic_pattern must be specified. + """ + + topics: VariableOrList[str] = field(default_factory=list) + """ + Topics to subscribe to. + Only one of topics or topic_pattern must be specified. + """ + + value_transformer: VariableOrOptional[Transformer] = None + """ + (Optional) Transformer for the message value. + If not specified, the value is left as raw bytes. + """ + + @classmethod + def from_dict(cls, value: "KafkaOptionsDict") -> "Self": + return _transform(cls, value) + + def as_dict(self) -> "KafkaOptionsDict": + return _transform_to_json_value(self) # type:ignore + + +class KafkaOptionsDict(TypedDict, total=False): + """""" + + client_config: VariableOrDict[str] + """ + :meta private: [EXPERIMENTAL] + + Undocumented backdoor mechanism for overriding parameters + to pass to the Kafka client. + This is not supported and may break at any time. + """ + + key_transformer: VariableOrOptional[TransformerParam] + """ + (Optional) Transformer for the message key. + If not specified, the key is left as raw bytes. + """ + + max_offsets_per_trigger: VariableOrOptional[int] + """ + :meta private: [EXPERIMENTAL] + + Internal option to control the maximum number of offsets to process per trigger. + """ + + starting_offset: VariableOrOptional[str] + """ + (Optional) Where to begin reading when no checkpoint exists. + Valid values: "latest" and "earliest". Defaults to "latest". + """ + + topic_pattern: VariableOrOptional[str] + """ + Java regex pattern to subscribe to matching topics. + Only one of topics or topic_pattern must be specified. + """ + + topics: VariableOrList[str] + """ + Topics to subscribe to. + Only one of topics or topic_pattern must be specified. + """ + + value_transformer: VariableOrOptional[TransformerParam] + """ + (Optional) Transformer for the message value. + If not specified, the value is left as raw bytes. + """ + + +KafkaOptionsParam = KafkaOptionsDict | KafkaOptions diff --git a/python/databricks/bundles/pipelines/_models/pipeline.py b/python/databricks/bundles/pipelines/_models/pipeline.py index 284a3734897..0756c44af51 100644 --- a/python/databricks/bundles/pipelines/_models/pipeline.py +++ b/python/databricks/bundles/pipelines/_models/pipeline.py @@ -25,10 +25,7 @@ IngestionPipelineDefinition, IngestionPipelineDefinitionParam, ) -from databricks.bundles.pipelines._models.lifecycle import ( - Lifecycle, - LifecycleParam, -) +from databricks.bundles.pipelines._models.lifecycle import Lifecycle, LifecycleParam from databricks.bundles.pipelines._models.notifications import ( Notifications, NotificationsParam, diff --git a/python/databricks/bundles/pipelines/_models/sharepoint_options_sharepoint_entity_type.py b/python/databricks/bundles/pipelines/_models/sharepoint_options_sharepoint_entity_type.py index 46e15fff79d..21f76eca27f 100644 --- a/python/databricks/bundles/pipelines/_models/sharepoint_options_sharepoint_entity_type.py +++ b/python/databricks/bundles/pipelines/_models/sharepoint_options_sharepoint_entity_type.py @@ -11,9 +11,18 @@ class SharepointOptionsSharepointEntityType(Enum): FILE_METADATA = "FILE_METADATA" PERMISSION = "PERMISSION" LIST = "LIST" + FILE_PERMISSION = "FILE_PERMISSION" + GROUP_MEMBERSHIP = "GROUP_MEMBERSHIP" SharepointOptionsSharepointEntityTypeParam = ( - Literal["FILE", "FILE_METADATA", "PERMISSION", "LIST"] + Literal[ + "FILE", + "FILE_METADATA", + "PERMISSION", + "LIST", + "FILE_PERMISSION", + "GROUP_MEMBERSHIP", + ] | SharepointOptionsSharepointEntityType ) diff --git a/python/databricks/bundles/pipelines/_models/transformer.py b/python/databricks/bundles/pipelines/_models/transformer.py new file mode 100644 index 00000000000..b707307fbbc --- /dev/null +++ b/python/databricks/bundles/pipelines/_models/transformer.py @@ -0,0 +1,54 @@ +from dataclasses import dataclass +from typing import TYPE_CHECKING, TypedDict + +from databricks.bundles.core._transform import _transform +from databricks.bundles.core._transform_to_json import _transform_to_json_value +from databricks.bundles.core._variable import VariableOrOptional +from databricks.bundles.pipelines._models.json_transformer_options import ( + JsonTransformerOptions, + JsonTransformerOptionsParam, +) +from databricks.bundles.pipelines._models.transformer_format import ( + TransformerFormat, + TransformerFormatParam, +) + +if TYPE_CHECKING: + from typing_extensions import Self + + +@dataclass(kw_only=True) +class Transformer: + """ + :meta private: [EXPERIMENTAL] + + Specifies how to transform binary data into structured data. + """ + + format: VariableOrOptional[TransformerFormat] = None + """ + Required: the wire format of the data. + """ + + json_options: VariableOrOptional[JsonTransformerOptions] = None + + @classmethod + def from_dict(cls, value: "TransformerDict") -> "Self": + return _transform(cls, value) + + def as_dict(self) -> "TransformerDict": + return _transform_to_json_value(self) # type:ignore + + +class TransformerDict(TypedDict, total=False): + """""" + + format: VariableOrOptional[TransformerFormatParam] + """ + Required: the wire format of the data. + """ + + json_options: VariableOrOptional[JsonTransformerOptionsParam] + + +TransformerParam = TransformerDict | Transformer diff --git a/python/databricks/bundles/pipelines/_models/transformer_format.py b/python/databricks/bundles/pipelines/_models/transformer_format.py new file mode 100644 index 00000000000..b37db5110ce --- /dev/null +++ b/python/databricks/bundles/pipelines/_models/transformer_format.py @@ -0,0 +1,18 @@ +from enum import Enum +from typing import Literal + + +class TransformerFormat(Enum): + """ + :meta private: [EXPERIMENTAL] + """ + + STRING = "STRING" + JSON = "JSON" + AVRO = "AVRO" + PROTOBUF = "PROTOBUF" + + +TransformerFormatParam = ( + Literal["STRING", "JSON", "AVRO", "PROTOBUF"] | TransformerFormat +) diff --git a/python/databricks/bundles/schemas/_models/privilege.py b/python/databricks/bundles/schemas/_models/privilege.py index 6a0215a210c..74f25b462c1 100644 --- a/python/databricks/bundles/schemas/_models/privilege.py +++ b/python/databricks/bundles/schemas/_models/privilege.py @@ -65,6 +65,8 @@ class Privilege(Enum): MANAGE_ACCESS = "MANAGE_ACCESS" MANAGE_ACCESS_CONTROL = "MANAGE_ACCESS_CONTROL" CREATE_SERVICE = "CREATE_SERVICE" + CREATE_FEATURE = "CREATE_FEATURE" + READ_FEATURE = "READ_FEATURE" PrivilegeParam = ( @@ -131,6 +133,8 @@ class Privilege(Enum): "MANAGE_ACCESS", "MANAGE_ACCESS_CONTROL", "CREATE_SERVICE", + "CREATE_FEATURE", + "READ_FEATURE", ] | Privilege ) diff --git a/python/databricks/bundles/volumes/_models/privilege.py b/python/databricks/bundles/volumes/_models/privilege.py index 6a0215a210c..74f25b462c1 100644 --- a/python/databricks/bundles/volumes/_models/privilege.py +++ b/python/databricks/bundles/volumes/_models/privilege.py @@ -65,6 +65,8 @@ class Privilege(Enum): MANAGE_ACCESS = "MANAGE_ACCESS" MANAGE_ACCESS_CONTROL = "MANAGE_ACCESS_CONTROL" CREATE_SERVICE = "CREATE_SERVICE" + CREATE_FEATURE = "CREATE_FEATURE" + READ_FEATURE = "READ_FEATURE" PrivilegeParam = ( @@ -131,6 +133,8 @@ class Privilege(Enum): "MANAGE_ACCESS", "MANAGE_ACCESS_CONTROL", "CREATE_SERVICE", + "CREATE_FEATURE", + "READ_FEATURE", ] | Privilege )