diff --git a/k8s/welearn-datastack/templates/updateMaterializedViews/workflow-template.yaml b/k8s/welearn-datastack/templates/updateMaterializedViews/workflow-template.yaml new file mode 100644 index 0000000..c0a1a88 --- /dev/null +++ b/k8s/welearn-datastack/templates/updateMaterializedViews/workflow-template.yaml @@ -0,0 +1,80 @@ +apiVersion: argoproj.io/v1alpha1 +kind: WorkflowTemplate +metadata: + name: update-materialized-views +spec: + entrypoint: update-materialized-views + serviceAccountName: {{ $.Values.common.workflowRbac.serviceAccountName }} + securityContext: + {{- include "common.security.podSecurityContext.restricted" dict | nindent 4 }} + templates: + - name: update-materialized-views + steps: + - - name: update-qty-document-in-qdrant + template: update-materialized-view + arguments: + parameters: + - name: view_name + value: qty_document_in_qdrant + - - name: update-qty-document-in-qdrant-per-corpus + template: update-materialized-view + arguments: + parameters: + - name: view_name + value: qty_document_in_qdrant_per_corpus + - - name: update-qty-document-per-corpus + template: update-materialized-view + arguments: + parameters: + - name: view_name + value: qty_document_per_corpus + - name: update-materialized-view + inputs: + parameters: + - name: view_name + script: + image: postgres:15 + command: ["bash"] + source: | + #!/bin/bash + set -e + export DB_NAME="$PG_DATABASE" + export DB_USER="$PG_USER" + export DB_HOST="$PG_HOST" + export DB_PORT="$PG_PORT" + export DB_PASSWORD="$PG_PASSWORD" + export PGPASSWORD="$DB_PASSWORD" + + psql "postgresql://$DB_USER:$DB_PASSWORD@$DB_HOST:$DB_PORT/$DB_NAME" \ + -v ON_ERROR_STOP=1 \ + -c "REFRESH MATERIALIZED VIEW CONCURRENTLY {{inputs.parameters.view_name}};" + envFrom: + - configMapRef: + name: {{ .name }} + volumeMounts: + - name: secrets + mountPath: "/secrets" + readOnly: true + + volumes: + - name: secrets + secret: + secretName: {{ .name }} +--- +apiVersion: argoproj.io/v1alpha1 +kind: CronWorkflow +metadata: + name: update-materialized-views-cron +spec: + schedule: "0 */3 * * *" # Every 3 hours + securityContext: + {{- include "common.security.podSecurityContext.restricted" dict | nindent 6 }} + workflowSpec: + entrypoint: update-materialized-views + serviceAccountName: {{ $.Values.common.workflowRbac.serviceAccountName }} + workflowTemplateRef: + name: update-materialized-views + ttlStrategy: + secondsAfterCompletion: 300 + podGC: + strategy: OnPodCompletion diff --git a/script/update-materialized-view b/script/update-materialized-view new file mode 100755 index 0000000..be9826d --- /dev/null +++ b/script/update-materialized-view @@ -0,0 +1,36 @@ +#!/bin/bash + +# Script to update a materialized view in PostgreSQL + +# Variables are passed as environment variables +DB_NAME=${DB_NAME} # PostgreSQL database name +DB_USER=${DB_USER} # PostgreSQL user +DB_HOST=${DB_HOST:-localhost} # Default to localhost if not set +DB_PORT=${DB_PORT:-5432} # Default to 5432 if not set +DB_PASSWORD=${DB_PASSWORD} # PostgreSQL password + +# Check if the view name is passed as an argument +if [ -z "$1" ]; then + echo "Usage: $0 " + exit 1 +fi + +VIEW_NAME=$1 + +# Function to refresh a materialized view +refresh_view() { + local view_name=$1 + echo "Refreshing materialized view: $view_name" + PGPASSWORD="$DB_PASSWORD" psql -U "$DB_USER" -h "$DB_HOST" -p "$DB_PORT" -d "$DB_NAME" -c "REFRESH MATERIALIZED VIEW CONCURRENTLY $view_name;" + if [ $? -eq 0 ]; then + echo "Successfully refreshed $view_name" + else + echo "Failed to refresh $view_name" >&2 + exit 1 + fi +} + +# Refresh the specified materialized view +refresh_view "$VIEW_NAME" + +echo "Materialized view $VIEW_NAME refreshed successfully."