livevalidator/databricks.yml.example at main · databricks-solutions/livevalidator · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
bundle:
  name: LiveValidator

include:
  - resources/*.yml

workspace:
  root_path: /Workspace/LiveValidator

resources:
  apps:
    live_validator:
      name: live-validator
      description: Compare tables or query results across systems
      source_code_path: src/app
      permissions:
        - level: CAN_USE
          group_name: users
      resources:
        - name: postgres
          database:
            database_name: databricks_postgres
            instance_name: ${resources.database_instances.live_validator_database.name}
            permission: CAN_CONNECT_AND_CREATE
        - name: run-validation
          job:
            id: ${resources.jobs.run_validation.id}
            permission: 'CAN_MANAGE_RUN'
        - name: fetch-lineage
          job:
            id: ${resources.jobs.fetch_lineage.id}
            permission: 'CAN_MANAGE_RUN'

  database_instances:
    live_validator_database:
      name: live-validator
      enable_pg_native_login: true
      capacity: CU_1
  database_catalogs:
    live_validator_catalog:
      database_instance_name: ${resources.database_instances.live_validator_database.name}
      name: live_validator
      database_name: databricks_postgres
      create_database_if_not_exists: true

  clusters:
    live_validator_cluster:
      cluster_name: live-validator
      node_type_id: i3.2xlarge
      driver_node_type_id: i3.2xlarge
      autoscale:
        min_workers: 1
        max_workers: 8
      aws_attributes:
        availability: ON_DEMAND
      spark_version: 17.3.x-scala2.13
      autotermination_minutes: 0
      enable_local_disk_encryption: false
      data_security_mode: USER_ISOLATION
      runtime_engine: PHOTON

targets:

  permissions:
  - user_name: <deployer_email>
    level: CAN_MANAGE

  migr-prod:
    default: true
    mode: production
    workspace:
      host: https://your-deployment-workspace.cloud.databricks.com/
      root_path: /Workspace/LiveValidator
    permissions:
      - level: CAN_MANAGE
        group_name: 1dp-migration-prd-admin
      - level: CAN_RUN
        group_name: users

    resources:
      clusters:
        live_validator_cluster:
          spark_conf:
            spark.your.additional.config: 'true'
          permissions:
            - group_name: users
              level: CAN_ATTACH_TO
            - group_name: 1dp-migration-prd-admin
              level: CAN_MANAGE
      jobs:
        run_validation:
          tasks:
            - task_key: validate
              # add all libraries to UC allowlist (!!!)
              libraries:
                # Both of these required for Teradata
                - maven:
                    coordinates: com.teradata.jdbc:terajdbc:20.00.00.51
                - pypi:
                    package: teradatasql
                # Required for Oracle
                - maven:
                    coordinates: com.oracle.database.jdbc:ojdbc11:23.5.0.24.07
          permissions:
            - level: CAN_MANAGE
              group_name: 1dp-migration-prd-admin