feat: Enable fine-grained permissions in Lakekeeper catalog

.github/workflows/warehouses_e2e_tests.yml

@@ -5,12 +5,14 @@ on:
       - "main"
     paths:
       - ".github/workflows/warehouses_e2e_tests.yml"
+      - "infra/ansible/roles/lakekeeper/files/bootstrap-warehouse.py"
       - "elt-common/**"
       - "warehouses/accelerator/extract_load/opralogweb/**"
   pull_request:
     types: [opened, synchronize, reopened]
     paths:
       - ".github/workflows/warehouses_e2e_tests.yml"
+      - "infra/ansible/roles/lakekeeper/files/bootstrap-warehouse.py"
       - "elt-common/**"
       - "warehouses/accelerator/extract_load/opralogweb/**"
 

docs-devel/deployment/index.md

@@ -28,12 +28,14 @@ Deploy the services using Ansible:
 
 ```bash
 > cd infra/ansible
-> ansible-playbook -i inventories/<dev|qa>/inventory.ini site.yml
+> ansible-playbook -i inventories/<dev|qa>/inventory.ini site.yml -e lakekeeper_admin_user=<admin_email>
 ```
 
+The `-e lakekeeper_admin_user=<admin_email>` argument is only required the first time Lakekeeper is deployed.
+
 Once deployed the services are available at:
 
-- Keycloak: <https://\domain\>/iceberg>
-- Lakekeeper: <https://\<domain\>/authn>
+- Keycloak: <https://\domain\>/auth>
+- Lakekeeper: <https://\<domain\>/iceberg>
 - Superset instances:
    - <https://\<domain\>/workspace/accelerator>

infra/ansible/group_vars/all/all.yml

@@ -20,7 +20,7 @@ logrotate_frequency: weekly
 logrotate_keep: 30
 logrotate_compress: true
 
-keycloak_base_path: /authn
+keycloak_base_path: /auth
 keycloak_http_port: 8080
 keycloak_http_management_port: 9000
 keycloak_url: "https://{{ top_level_domain }}{{ keycloak_base_path }}"
@@ -33,7 +33,7 @@ lakekeeper_base_path: /iceberg
 lakekeeper_http_port: 8181
 lakekeeper_project_name: "ISIS Analytics Data Platform"
 lakekeeper_catalog:
-  uri: "https://{{ top_level_domain }}{{ lakekeeper_base_path }}/catalog"
+  catalog_uri: "https://{{ top_level_domain }}{{ lakekeeper_base_path }}/catalog"
   warehouses:
     accelerator:
       storage:

infra/ansible/group_vars/keycloak.yml

@@ -55,8 +55,8 @@ keycloak_clients:
     attributes:
       access.token.lifespan: 600
   # public clients
-  - client_id: lakekeeper-ui
-    name: "Lakekeeper catalog UI"
+  - client_id: lakekeeper-api
+    name: "Lakekeeper API"
     protocol: "openid-connect"
     public_client: true
     redirect_uris:

infra/ansible/inventories/dev/group_vars/datastore.yml

@@ -3,6 +3,7 @@ postgres_db_user: "{{ vault_postgres_db_user }}"
 postgres_db_passwd: "{{ vault_postgres_db_passwd }}"
 postgres_db_names:
   - keycloak
+  - openfga
   - lakekeeper
   - superset_farm
 

infra/ansible/inventories/dev/group_vars/lakekeeper.yml

@@ -1,4 +1,10 @@
 ---
+openfga_database_host: "{{ groups['datastore'][0] }}"
+openfga_database_port: 5432
+openfga_database_name: openfga
+openfga_database_user: "{{ vault_postgres_db_user }}"
+openfga_database_passwd: "{{ vault_postgres_db_passwd }}"
+
 lakekeeper_metadb_host: "{{ groups['datastore'][0] }}"
 lakekeeper_metadb_name: lakekeeper
 lakekeeper_metadb_user: "{{ vault_postgres_db_user }}"

infra/ansible/inventories/qa/group_vars/lakekeeper.yml

@@ -1,4 +1,10 @@
 ---
+openfga_database_host: "{{ vault_db_host }}"
+openfga_database_port: "{{ vault_db_port }}"
+openfga_database_name: "{{ vault_openfga_database_name }}"
+openfga_database_user: "{{ vault_openfga_database_user }}"
+openfga_database_passwd: "{{ vault_openfga_database_passwd }}"
+
 lakekeeper_metadb_host: "{{ vault_db_host }}"
 lakekeeper_metadb_name: "{{ vault_lakekeeper_metadb_name }}"
 lakekeeper_metadb_user: "{{ vault_lakekeeper_metadb_user }}"

infra/ansible/roles/elt/templates/secrets/envvars.j2

@@ -11,7 +11,7 @@ OPRALOGWEB__SOURCES__CREDENTIALS__USERNAME={{ vault_elt_opralogweb_user }}
 OPRALOGWEB__SOURCES__CREDENTIALS__PASSWORD={{ vault_elt_opralogweb_passwd }}
 
 DESTINATION__PYICEBERG__BUCKET_URL=s3://{{ lakekeeper_catalog.warehouses[warehouse.name].storage.bucket_name }}
-DESTINATION__PYICEBERG__CREDENTIALS__URI={{ lakekeeper_catalog.uri }}
+DESTINATION__PYICEBERG__CREDENTIALS__URI={{ lakekeeper_catalog.catalog_uri }}
 DESTINATION__PYICEBERG__CREDENTIALS__WAREHOUSE={{ warehouse.name }}
 DESTINATION__PYICEBERG__CREDENTIALS__ACCESS_DELEGATION=remote-signing
 DESTINATION__PYICEBERG__CREDENTIALS__OAUTH2_SERVER_URI={{ keycloak_realm_url }}/protocol/openid-connect/token

infra/ansible/roles/keycloak/tasks/setup-realm.yml

@@ -50,6 +50,16 @@
   loop_control:
     label: "{{ item.client_id }}"
 
-- ansible.builtin.import_tasks: setup-ldap.yml
+- name: Grant machine user permission to view users in the realm
+  community.general.keycloak_user_rolemapping:
+    <<: *keycloak_auth_vars
+    realm: "{{ keycloak_realm.name }}"
+    client_id: realm-management
+    target_username: service-account-ansible
+    roles:
+      - name: view-users
+
+- name: Setup LDAP
+  ansible.builtin.import_tasks: setup-ldap.yml
   vars:
     target_realm: "{{ keycloak_realm.name }}"

infra/ansible/roles/lakekeeper/defaults/main.yml

@@ -1,6 +1,7 @@
 ---
-lakekeeper_bootstrap_log_level: INFO
 lakekeeper_container_name: lakekeeper
-lakekeeper_image: quay.io/lakekeeper/catalog:v0.10.0
+lakekeeper_bootstrap_log_level: INFO
+lakekeeper_log_level: ERROR
+lakekeeper_image: quay.io/lakekeeper/catalog:v0.11.2
 lakekeeper_http_port: 8181
 lakekeeper_working_dir: /var/lakekeeper