Generate data stream input configuration and documentation

cmd/create_data_stream.go

@@ -7,6 +7,8 @@ package cmd
 import (
 	"errors"
 	"fmt"
+	"maps"
+	"slices"
 
 	"github.com/AlecAivazis/survey/v2"
 
@@ -25,6 +27,7 @@ type newDataStreamAnswers struct {
 	Name                   string
 	Title                  string
 	Type                   string
+	Inputs                 []string
 	Subobjects             bool
 	SyntheticAndTimeSeries bool
 	Synthetic              bool
@@ -126,6 +129,50 @@ func createDataStreamCommandAction(cmd *cobra.Command, args []string) error {
 		}
 	}
 
+	if answers.Type == "logs" {
+		// Map of possible inputs that can be used in the wizard, and their description.
+		inputsMap := map[string]string{
+			"aws-cloudwatch":     "AWS Cloudwatch",
+			"aws-s3":             "AWS S3",
+			"azure-blob-storage": "Azure Blob Storage",
+			"azure-eventhub":     "Azure Eventhub",
+			"cel":                "Common Expression Language (CEL)",
+			"entity-analytics":   "Entity Analytics",
+			"etw":                "Event Tracing for Windows (ETW)",
+			"filestream":         "Filestream",
+			"gcp-pubsub":         "GCP PubSub",
+			"gcs":                "Google Cloud Storage (GCS)",
+			"http_endpoint":      "HTTP Endpoint",
+			"journald":           "Journald",
+			"netflow":            "Netflow",
+			"redis":              "Redis",
+			"tcp":                "TCP",
+			"udp":                "UDP",
+			"winlog":             "WinLogBeat",
+		}
+		qs := []*survey.Question{
+			{
+				Name: "inputs",
+				Prompt: &survey.MultiSelect{
+					Message:  "Select input types which will be used in this data stream. See https://www.elastic.co/docs/reference/fleet/elastic-agent-inputs-list for description of the inputs",
+					Options:  slices.Sorted(maps.Keys(inputsMap)),
+					PageSize: 50,
+					Description: func(value string, index int) string {
+						val, ok := inputsMap[value]
+						if ok {
+							return val
+						}
+						return ""
+					},
+				},
+			},
+		}
+		err = survey.Ask(qs, &answers)
+		if err != nil {
+			return fmt.Errorf("prompt failed: %w", err)
+		}
+	}
+
 	descriptor := createDataStreamDescriptorFromAnswers(answers, packageRoot)
 	err = archetype.CreateDataStream(descriptor)
 	if err != nil {
@@ -163,6 +210,22 @@ func createDataStreamDescriptorFromAnswers(answers newDataStreamAnswers, package
 		}
 	}
 
+	// If no inputs were selected, insert one so the datastream shows an example of an input.
+	if answers.Type == "logs" && len(answers.Inputs) == 0 {
+		answers.Inputs = []string{"filestream"}
+	}
+
+	if len(answers.Inputs) > 0 {
+		var streams []packages.Stream
+		for _, input := range answers.Inputs {
+			streams = append(streams, packages.Stream{
+				Input: input,
+				Vars:  []packages.Variable{},
+			})
+		}
+		manifest.Streams = streams
+	}
+
 	return archetype.DataStreamDescriptor{
 		Manifest:    manifest,
 		PackageRoot: packageRoot,

internal/docs/_static/agent/aws_cloudwatch.yml.hbs

@@ -0,0 +1,100 @@
+{{#unless log_group_name}}
+{{#unless log_group_name_prefix}}
+{{#if log_group_arn }}
+log_group_arn: {{ log_group_arn }}
+{{/if}}
+{{/unless}}
+{{/unless}}
+{{#unless log_group_arn}}
+{{#unless log_group_name}}
+{{#if log_group_name_prefix }}
+log_group_name_prefix: {{ log_group_name_prefix }}
+{{/if}}
+{{/unless}}
+{{/unless}}
+{{#unless log_group_arn}}
+{{#unless log_group_name_prefix}}
+{{#if log_group_name }}
+log_group_name: {{ log_group_name }}
+{{/if}}
+{{/unless}}
+{{/unless}}
+{{#unless log_group_arn}}
+region_name: {{ region_name }}
+{{/unless}}
+{{#unless log_stream_prefix}}
+{{#if log_streams }}
+log_streams: {{ log_streams }}
+{{/if}}
+{{/unless}}
+{{#unless log_streams}}
+{{#if log_stream_prefix }}
+log_stream_prefix: {{ log_stream_prefix }}
+{{/if}}
+{{/unless}}
+{{#if start_position }}
+start_position: {{ start_position }}
+{{/if}}
+{{#if scan_frequency }}
+scan_frequency: {{ scan_frequency }}
+{{/if}}
+{{#if api_sleep }}
+api_sleep: {{ api_sleep }}
+{{/if}}
+{{#if api_timeout}}
+api_timeout: {{api_timeout}}
+{{/if}}
+{{#if latency }}
+latency: {{ latency }}
+{{/if}}
+{{#if number_of_workers }}
+number_of_workers: {{ number_of_workers }}
+{{/if}}
+{{#if credential_profile_name}}
+credential_profile_name: {{credential_profile_name}}
+{{/if}}
+{{#if shared_credential_file}}
+shared_credential_file: {{shared_credential_file}}
+{{/if}}
+{{#if default_region}}
+default_region: {{default_region}}
+{{/if}}
+{{#if access_key_id}}
+access_key_id: {{access_key_id}}
+{{/if}}
+{{#if secret_access_key}}
+secret_access_key: {{secret_access_key}}
+{{/if}}
+{{#if session_token}}
+session_token: {{session_token}}
+{{/if}}
+{{#if role_arn}}
+role_arn: {{role_arn}}
+{{/if}}
+{{#if proxy_url }}
+proxy_url: {{proxy_url}}
+{{/if}}
+
+{{#if tags.length}}
+tags:
+{{#each tags as |tag|}}
+- {{tag}}
+{{/each}}
+{{#if preserve_original_event}}
+- preserve_original_event
+{{/if}}
+{{else}}
+{{#if preserve_original_event}}
+tags:
+- preserve_original_event
+{{/if}}
+{{/if}}
+
+{{#contains "forwarded" tags}}
+publisher_pipeline.disable_host: true
+{{/contains}}
+
+{{#if processors}}
+processors:
+{{processors}}
+{{/if}}

internal/docs/_static/agent/aws_s3.yml.hbs

@@ -0,0 +1,154 @@
+{{! start SQS queue }}
+{{#unless bucket_arn}}
+{{#unless non_aws_bucket_name}}
+{{#if queue_url }}
+queue_url: {{ queue_url }}
+{{/if}}
+{{/unless}}
+{{/unless}}
+{{! end SQS queue }}
+
+{{#unless queue_url}}{{! start S3 bucket polling }}
+
+{{!
+When using an S3 bucket, you can specify only one of the following options:
+- An AWS bucket ARN
+- A non-AWS bucket name
+}}
+
+{{! shared S3 bucket polling options }}
+{{#if number_of_workers }}
+number_of_workers: {{ number_of_workers }}
+{{/if}}
+{{#if bucket_list_prefix }}
+bucket_list_prefix: {{ bucket_list_prefix }}
+{{/if}}
+{{#if bucket_list_interval }}
+bucket_list_interval: {{ bucket_list_interval }}
+{{/if}}
+
+{{! AWS S3 bucket ARN options }}
+{{#unless non_aws_bucket_name}}
+{{#if bucket_arn }}
+bucket_arn: {{ bucket_arn }}
+{{/if}}
+{{/unless}}{{! end AWS S3 bucket ARN options }}
+
+{{! non-AWS S3 bucket ARN options }}
+{{#unless bucket_arn}}
+{{#if non_aws_bucket_name }}
+non_aws_bucket_name: {{ non_aws_bucket_name }}
+{{/if}}
+{{/unless}}{{! end non-AWS S3 bucket ARN options }}
+
+{{/unless}}{{! end S3 bucket polling }}
+
+{{#if buffer_size }}
+buffer_size: {{ buffer_size }}
+{{/if}}
+{{#if content_type }}
+content_type: {{ content_type }}
+{{/if}}
+{{#if encoding }}
+encoding: {{ encoding }}
+{{/if}}
+{{#if expand_event_list_from_field }}
+expand_event_list_from_field: {{ expand_event_list_from_field }}
+{{/if}}
+{{#if buffer_size }}
+buffer_size: {{ buffer_size }}
+{{/if}}
+{{#if fips_enabled }}
+fips_enabled: {{ fips_enabled }}
+{{/if}}
+{{#if include_s3_metadata }}
+include_s3_metadata: {{ include_s3_metadata }}
+{{/if}}
+{{#if max_bytes }}
+max_bytes: {{ max_bytes }}
+{{/if}}
+{{#if max_number_of_messages }}
+max_number_of_messages: {{ max_number_of_messages }}
+{{/if}}
+{{#if path_style }}
+path_style: {{ path_style }}
+{{/if}}
+{{#if provider }}
+provider: {{ provider }}
+{{/if}}
+{{#if sqs.max_receive_count }}
+sqs.max_receive_count: {{ sqs.max_receive_count }}
+{{/if}}
+{{#if sqs.wait_time }}
+sqs.wait_time: {{ sqs.wait_time }}
+{{/if}}
+
+{{#if file_selectors}}
+file_selectors:
+{{file_selectors}}
+{{/if}}
+
+{{#if credential_profile_name}}
+credential_profile_name: {{credential_profile_name}}
+{{/if}}
+{{#if shared_credential_file}}
+shared_credential_file: {{shared_credential_file}}
+{{/if}}
+{{#if visibility_timeout}}
+visibility_timeout: {{visibility_timeout}}
+{{/if}}
+{{#if api_timeout}}
+api_timeout: {{api_timeout}}
+{{/if}}
+{{#if endpoint}}
+endpoint: {{endpoint}}
+{{/if}}
+{{#if default_region}}
+default_region: {{default_region}}
+{{/if}}
+{{#if access_key_id}}
+access_key_id: {{access_key_id}}
+{{/if}}
+{{#if secret_access_key}}
+secret_access_key: {{secret_access_key}}
+{{/if}}
+{{#if session_token}}
+session_token: {{session_token}}
+{{/if}}
+{{#if role_arn}}
+role_arn: {{role_arn}}
+{{/if}}
+{{#if fips_enabled}}
+fips_enabled: {{fips_enabled}}
+{{/if}}
+{{#if proxy_url }}
+proxy_url: {{proxy_url}}
+{{/if}}
+{{#if parsers}}
+parsers:
+{{parsers}}
+{{/if}}
+
+{{#if tags.length}}
+tags:
+{{#each tags as |tag|}}
+- {{tag}}
+{{/each}}
+{{#if preserve_original_event}}
+- preserve_original_event
+{{/if}}
+{{else}}
+{{#if preserve_original_event}}
+tags:
+- preserve_original_event
+{{/if}}
+{{/if}}
+
+{{#contains "forwarded" tags}}
+publisher_pipeline.disable_host: true
+{{/contains}}
+
+{{#if processors}}
+processors:
+{{processors}}
+{{/if}}