Upgrade components (#422)

- Collector 3.7.0
- Enrich 6.1.2
- Postgres 16.10-alpine
- Cadvisor 0.53.0

ghcr.io is used to fetch cadvisor container image since
it is moved to ghcr.io: google/cadvisor#3699

Author: spenes

provisioning/resources/configs/compositions/docker-compose-aws.yml

@@ -138,7 +138,7 @@ services:
     mem_limit: 64m
 
   scala-stream-collector:
-    image: snowplow/scala-stream-collector-nsq:3.4.0-distroless
+    image: snowplow/scala-stream-collector-nsq:3.7.0-distroless
     container_name: scala-stream-collector-nsq
     command: [ "--config", "/snowplow/config/snowplow-stream-collector.hocon" ]
     restart: always
@@ -159,7 +159,7 @@ services:
     mem_limit: ${SP_COLLECTOR_MEM_SIZE}
 
   enrich:
-    image: snowplow/snowplow-enrich-nsq:5.4.0-distroless
+    image: snowplow/snowplow-enrich-nsq:6.1.2-distroless
     container_name: enrich
     command: [
       "--config", "/snowplow/config/snowplow-enrich.hocon",
@@ -182,7 +182,7 @@ services:
     mem_limit: ${SP_ENRICH_MEM_SIZE}
 
   postgres:
-    image: postgres:15.1-alpine
+    image: postgres:16.10-alpine
     container_name: postgres
     restart: always
     volumes:
@@ -220,7 +220,7 @@ services:
     mem_limit: ${SP_IGLU_SERVER_MEM_SIZE}
 
   cadvisor:
-    image: gcr.io/cadvisor/cadvisor:v0.52.1
+    image: ghcr.io/google/cadvisor:v0.53.0
     container_name: cadvisor
     restart: always
     privileged: true

provisioning/resources/configs/compositions/docker-compose-gcp.yml

@@ -117,7 +117,7 @@ services:
     mem_limit: 64m
 
   scala-stream-collector:
-    image: snowplow/scala-stream-collector-nsq:3.4.0-distroless
+    image: snowplow/scala-stream-collector-nsq:3.7.0-distroless
     container_name: scala-stream-collector-nsq
     command: [ "--config", "/snowplow/config/snowplow-stream-collector.hocon" ]
     restart: always
@@ -135,7 +135,7 @@ services:
     mem_limit: ${SP_COLLECTOR_MEM_SIZE}
 
   enrich:
-    image: snowplow/snowplow-enrich-nsq:5.4.0-distroless
+    image: snowplow/snowplow-enrich-nsq:6.1.2-distroless
     container_name: enrich
     command: [
       "--config", "/snowplow/config/snowplow-enrich.hocon",
@@ -155,7 +155,7 @@ services:
     mem_limit: ${SP_ENRICH_MEM_SIZE}
 
   postgres:
-    image: postgres:15.1-alpine
+    image: postgres:16.10-alpine
     container_name: postgres
     restart: always
     volumes:
@@ -187,7 +187,7 @@ services:
     mem_limit: ${SP_IGLU_SERVER_MEM_SIZE}
 
   cadvisor:
-    image: gcr.io/cadvisor/cadvisor:v0.52.1
+    image: ghcr.io/google/cadvisor:v0.53.0
     container_name: cadvisor
     restart: always
     privileged: true

provisioning/resources/configs/snowplow-enrich.hocon

@@ -5,6 +5,7 @@
 
   "input": {
     "topic": "RawEvents"
+    "channel": "RawEventsChannel"
     "lookupHost": "nsqlookupd"
     "lookupPort": 4161
   }
@@ -21,18 +22,6 @@
       "nsqdHost": "nsqd"
       "nsqdPort": 4150
     }
-
-    "pii": {
-      "type": "Nsq"
-      "topic": "PiiEvents"
-      "nsqdHost": "nsqd"
-      "nsqdPort": 4150
-      "backoffPolicy": {
-        "minBackoff": 100 milliseconds
-        "maxBackoff": 10 seconds
-        "maxRetries": 10
-      }
-    }
   }
   "featureFlags": {
     "exitOnJsCompileError": false

provisioning/resources/configs/snowplow-stream-collector.hocon

@@ -71,7 +71,6 @@ collector {
   streams {
     good = RawEvents
     bad = BadRawEvents
-    useIpAddressAsPartitionKey = false
 
     sink {
       enabled = nsq

provisioning/resources/ui/index.html

@@ -58,15 +58,15 @@ <h3>Quicklinks:</h3>
                     <h3>The software stack installed:</h3>
                     <ul class="software-list">
                         <li>Snowplow Mini 0.23.2</li>
-                        <li>Snowplow Collector NSQ 3.4.0</li>
-                        <li>Snowplow Enrich NSQ 5.4.0</li>
+                        <li>Snowplow Collector NSQ 3.7.0</li>
+                        <li>Snowplow Enrich NSQ 6.1.2</li>
                         <li>Snowplow Elasticsearch Loader 2.1.3</li>
                         <li>Snowplow Iglu Server 0.14.0</li>
-                        <li>Postgres 15.1</li>
+                        <li>Postgres 16.10</li>
                         <li>NSQ v1.3.0</li>
                         <li>Opensearch 2.19.1</li>
                         <li>Opensearch Dashboards 2.19.1</li>
-                        <li>Cadvisor 0.52.1</li>
+                        <li>Cadvisor 0.53.0</li>
                     </ul>
 
                     <div class="stack-topology">

provisioning/roles/docker/files/docker-compose.yml

@@ -131,7 +131,7 @@ services:
     mem_limit: 64m
 
   scala-stream-collector:
-    image: snowplow/scala-stream-collector-nsq:3.4.0-distroless
+    image: snowplow/scala-stream-collector-nsq:3.7.0-distroless
     container_name: scala-stream-collector-nsq
     command: [ "--config", "/snowplow/config/snowplow-stream-collector.hocon" ]
     restart: always
@@ -151,7 +151,7 @@ services:
     mem_limit: ${SP_COLLECTOR_MEM_SIZE}
 
   enrich:
-    image: snowplow/snowplow-enrich-nsq:5.4.0-distroless
+    image: snowplow/snowplow-enrich-nsq:6.1.2-distroless
     container_name: enrich
     command: [
       "--config", "/snowplow/config/snowplow-enrich.hocon",
@@ -173,7 +173,7 @@ services:
     mem_limit: ${SP_ENRICH_MEM_SIZE}
 
   postgres:
-    image: postgres:15.1-alpine
+    image: postgres:16.10-alpine
     container_name: postgres
     restart: always
     volumes:
@@ -209,7 +209,7 @@ services:
     mem_limit: ${SP_IGLU_SERVER_MEM_SIZE}
 
   cadvisor:
-    image: gcr.io/cadvisor/cadvisor:v0.52.1
+    image: ghcr.io/google/cadvisor:v0.53.0
     container_name: cadvisor
     restart: always
     privileged: true

provisioning/roles/docker/tasks/main.yml

@@ -30,7 +30,6 @@
     curl -X POST localhost:4151/topic/create?topic=BadEvents
     curl -X POST localhost:4151/topic/create?topic=EnrichedEvents
     curl -X POST localhost:4151/topic/create?topic=BadEnrichedEvents
-    curl -X POST localhost:4151/topic/create?topic=PiiEvents
 
 # Cronjob: Clean old docs from 'good' index
 - name: Add cronjob to clean old ES 'good' index documents