feat: add otel collector to nodes

ansible/prereqs.yml

@@ -0,0 +1,5 @@
+- name: Set up ansible
+  hosts: all
+  tasks:
+    - name: Install roles from Ansible Galaxy
+      command: ansible-galaxy collection install grafana.grafana

ansible/roles/otel/tasks/main.yml

@@ -0,0 +1,60 @@
+- name: Set HTTPS_PROXY for otel-collector
+  ansible.builtin.copy:
+    dest: /etc/systemd/system/otel-collector.service.d/override.conf
+    content: |
+      [Service]
+      Environment="HTTPS_PROXY=http://webproxy-internett.nav.no:8088"
+
+- name: Install OpenTelemetry Collector
+  ansible.builtin.include_role:
+    name: grafana.grafana.opentelemetry_collector
+  vars:
+    otel_collector_receivers:
+      hostmetrics:
+        collection_interval: 60s
+        scrapers:
+          cpu: {}
+          disk: {}
+          load: {}
+          filesystem: {}
+          memory: {}
+          network: {}
+          paging: {}
+          process:
+            mute_process_name_error: true
+            mute_process_exe_error: true
+            mute_process_io_error: true
+          processes: {}
+
+    otel_collector_processors:
+      batch:
+      resourcedetection:
+        detectors: [env, system]
+        timeout: 2s
+        system:
+          hostname_sources: [os]
+      transform/add_resource_attributes_as_metric_attributes:
+        error_mode: ignore
+        metric_statements:
+          - context: datapoint
+            statements:
+              - set(attributes["deployment.environment"],
+                resource.attributes["deployment.environment"])
+              - set(attributes["service.version"],
+                resource.attributes["service.version"])
+
+    otel_collector_exporters:
+      otlphttp/nais:
+        endpoint: "https://collector-internet.nav.cloud.nais.io"
+
+    otel_collector_service:
+      pipelines:
+        metrics:
+          receivers: [hostmetrics]
+          processors:
+            [
+              resourcedetection,
+              transform/add_resource_attributes_as_metric_attributes,
+              batch,
+            ]
+          exporters: [otlphttp/nais]

ansible/roles/vm_apiserver/meta/main.yml

@@ -2,3 +2,4 @@ dependencies:
   - role: naisdevice_apt_repo
   - role: common
   - role: logging
+  - role: otel

ansible/roles/vm_gateway/meta/main.yml

@@ -2,3 +2,4 @@ dependencies:
   - role: naisdevice_apt_repo
   - role: common
   - role: logging
+  - role: otel

ansible/roles/vm_prometheus/meta/main.yml

@@ -2,3 +2,4 @@ dependencies:
   - role: naisdevice_apt_repo
   - role: common
   - role: logging
+  - role: otel

doc/operational/vm-bootstrap.sh

@@ -43,5 +43,15 @@ all:
         $(hostname):
 EOF
 
+cat <<EOF >/root/run-ansible.sh
+if [ \$(pgrep ansible-pull -c) -ne 0 ]; then
+  echo "already running"
+  exit 0
+fi
+
+HTTPS_PROXY=http://webproxy-internett.nav.no:8088 /usr/bin/ansible-pull --only-if-changed -U https://github.com/nais/device --checkout main ansible/prereqs.yml ansible/site.yml -i /root/ansible-inventory.yaml >> /var/log/ansible.log
+EOF
+chmod +x /root/run-ansible.sh
+
 echo "add the following line to crontab:"
-echo "*/5 * * * * [ \$(pgrep ansible-pull -c) -eq 0 ] && HTTPS_PROXY=$HTTPS_PROXY /usr/bin/ansible-pull --only-if-changed -U https://github.com/nais/device ansible/site.yml -i /root/ansible-inventory.yaml >> /var/log/ansible.log"
+echo "*/5 * * * * /root/run-ansible.sh"