Update 8.19 observabilitySRE testing based on using 9.2 stack (#18014)

When working on porting tests to #18001
the main LS branch we started testing against the 9.2 series of the ES stack.
There were some required changes there (listed below) that are 8.19 compatable.
This commit ports those changes to the 8.19 series to keep them as close as
possible.

1. Elasticsearch container needs explicit fips config to boot
2. Filebeat needs > 64k file size to start filestream processing
3. Generally the logging in the shared helpers was updated to make reasoning
about the state of failures easier in log output.

Author: donoghuc

x-pack/distributions/internal/observabilitySRE/qa/acceptance/docker/docker-compose.yml

@@ -23,7 +23,7 @@ services:
       - ./certs:/usr/share/elasticsearch/config/certs
     environment:
       - discovery.type=single-node
-      - ES_JAVA_OPTS=-Xms1g -Xmx1g
+      - "ES_JAVA_OPTS=-Djava.security.properties=/usr/share/elasticsearch/config/fips_java.security"
       - ELASTIC_PASSWORD=changeme
     networks:
       - elastic

x-pack/distributions/internal/observabilitySRE/qa/acceptance/docker/elasticsearch/config/elasticsearch-fips.yml

@@ -12,4 +12,7 @@ xpack.security.transport.ssl.certificate_authorities: ["/usr/share/elasticsearch
 xpack.security.http.ssl.enabled: true
 xpack.security.http.ssl.key: /usr/share/elasticsearch/config/certs/elasticsearch.key
 xpack.security.http.ssl.certificate: /usr/share/elasticsearch/config/certs/elasticsearch.crt
-xpack.security.http.ssl.certificate_authorities: ["/usr/share/elasticsearch/config/certs/ca.crt"]
+xpack.security.http.ssl.certificate_authorities: ["/usr/share/elasticsearch/config/certs/ca.crt"]
+# FIPS Configuration - Let Elasticsearch auto-detect and register providers
+xpack.security.fips_mode.enabled: true
+xpack.security.autoconfiguration.enabled: false

x-pack/distributions/internal/observabilitySRE/qa/acceptance/docker/filebeat/config/filebeat-fips.yml

@@ -1,8 +1,9 @@
 filebeat.inputs:
-- type: log
+- type: filestream
   enabled: true
   paths:
     - /data/logs/sample_logs.txt
+  id: test-logs-input
 
 output.logstash:
   hosts: ["logstash:5044"]

x-pack/distributions/internal/observabilitySRE/qa/acceptance/docker/filebeat/data/sample_logs.txt

@@ -1 +1,8 @@
 TEST-LOG: FIPS filebeat test message
+TEST-LOG: Error message with status=500
+TEST-LOG: Log with json={"user":"testuser","action":"login","status":"success"}
+TEST-LOG: Log with json={"user":"admin","action":"update","items":5,"details":{"category":"config","changed":true}}
+TEST-LOG: Log with timestamp=2025-04-01T12:00:00Z for testing date filter
+TEST-LOG: Debug log message DEBUG should be dropped
+TEST-LOG: Log with timestamp=2024-04-01T12:00:00Z should be tagged as old_event
+TEST-LOG: Padding to reach minimum file size xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx

x-pack/distributions/internal/observabilitySRE/qa/smoke/docker/docker-compose.yml

@@ -3,7 +3,7 @@ services:
     image: docker.elastic.co/cloud-release/elasticsearch-cloud-ess-fips:${ELASTICSEARCH_IMAGE_VERSION:-8.19.0-SNAPSHOT}
     environment:
       - discovery.type=single-node
-      - ES_JAVA_OPTS=-Xms1g -Xmx1g
+      - "ES_JAVA_OPTS=-Djava.security.properties=/usr/share/elasticsearch/config/fips_java.security"
       - ELASTIC_PASSWORD=changeme
     ports:
       - "9200:9200"

x-pack/distributions/internal/observabilitySRE/qa/smoke/docker/elasticsearch/config/elasticsearch-fips.yml

@@ -12,4 +12,7 @@ xpack.security.transport.ssl.certificate_authorities: ["/usr/share/elasticsearch
 xpack.security.http.ssl.enabled: true
 xpack.security.http.ssl.key: /usr/share/elasticsearch/config/certs/elasticsearch.key
 xpack.security.http.ssl.certificate: /usr/share/elasticsearch/config/certs/elasticsearch.crt
-xpack.security.http.ssl.certificate_authorities: ["/usr/share/elasticsearch/config/certs/ca.crt"]
+xpack.security.http.ssl.certificate_authorities: ["/usr/share/elasticsearch/config/certs/ca.crt"]
+# FIPS Configuration - Let Elasticsearch auto-detect and register providers
+xpack.security.fips_mode.enabled: true
+xpack.security.autoconfiguration.enabled: false

x-pack/distributions/internal/observabilitySRE/qa/smoke/docker/filebeat/filebeat.yml

@@ -1,8 +1,9 @@
 filebeat.inputs:
-- type: log
+- type: filestream
   enabled: true
   paths:
     - /test-logs/*.log
+  id: test-logs-input
 
 output.logstash:
   hosts: ["logstash:5044"]

x-pack/distributions/internal/observabilitySRE/qa/smoke/docker/test-logs/test.log

@@ -5,3 +5,4 @@ TEST-LOG: Log with json={"user":"admin","action":"update","items":5,"details":{"
 TEST-LOG: Log with timestamp=2025-04-01T12:00:00Z for testing date filter
 TEST-LOG: Debug log message DEBUG should be dropped
 TEST-LOG: Log with timestamp=2024-04-01T12:00:00Z should be tagged as old_event
+TEST-LOG: Padding to reach minimum file size xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx

x-pack/distributions/internal/observabilitySRE/qa/spec/shared_helpers.rb

@@ -54,6 +54,7 @@ def wait_for_elasticsearch(max_retries = 120, require_documents: false, index_pa
                   puts "Found #{count_data["count"]} documents in #{index_pattern} index"
                 else
                   puts "Waiting for documents in #{index_pattern} index..."
+                  puts "ES response: #{count_data.inspect}"
                 end
               end
             else
@@ -78,6 +79,7 @@ def wait_for_elasticsearch(max_retries = 120, require_documents: false, index_pa
   def docker_compose_invoke(subcommand, env = {}, work_dir)
     env_str = env.map{ |k,v| "#{k.to_s.upcase}=#{Shellwords.escape(v)} "}.join
     command = "#{env_str}docker-compose --project-directory=#{Shellwords.escape(work_dir)} #{subcommand}"
+    puts "Invoking Docker Compose with command: #{command}"
     system(command) or fail "Failed to invoke Docker Compose with command `#{command}`"
   end