feat: support for nested fields and query functions

- Nested fields like "Owner.Id" will be now populated correctly
- Functions like SOQL "toLabel(field) field" will now be populated correctly
- Removed field type validations as they are not working with nested fields
- Update changelog

Author: Asafkbalink

CHANGELOG.md

@@ -1,3 +1,10 @@
+## 3.3.0
+  - feat: Added `timeout` configuration to control RESTForce timeout settings (defaults to `60`)
+  - feat: Added support for reference fields (`parent__r.child`)
+  - feat: Added support for built-in SOQL functions (etc. `toLabel(field__c) field`)
+  - refactor: Removed field_type inferring (elastic can infer it / logstash mapping can be configured for special cases)
+  - fix: update Apache licence to suppress compiling warning
+
 ## 3.2.0
   - Added `use_tooling_api` configuration to connect to the Salesforce Tooling API instead of the regular Rest API. [#26](https://github.com/logstash-plugins/logstash-input-salesforce/pull/26)
 

lib/logstash/inputs/salesforce.rb

@@ -66,6 +66,8 @@ class LogStash::Inputs::Salesforce < LogStash::Inputs::Base
   # By default, this uses the default Restforce API version.
   # To override this, set this to something like "32.0" for example
   config :api_version, :validate => :string, :required => false
+  # RESTForce request timeout in seconds.
+  config :timeout, :validate => :number, :required => false
   # Consumer Key for authentication. You must set up a new SFDC
   # connected app with oath to use this output. More information
   # can be found here:
@@ -92,8 +94,6 @@ class LogStash::Inputs::Salesforce < LogStash::Inputs::Base
   # SOQL statement. Additional fields can be filtered on by
   # adding field1 = value1 AND field2 = value2 AND...
   config :sfdc_filters, :validate => :string, :default => ""
-  # RESTForce request timeout in seconds.
-  config :timeout, :validate => :number, :default => 60, :required => false
   # Setting this to true will convert SFDC's NamedFields__c to named_fields__c
   config :to_underscores, :validate => :boolean, :default => false
 
@@ -108,22 +108,22 @@ def register
   public
   def run(queue)
     results = client.query(get_query())
+    @logger.debug("Query results:", :results => results)
     if results && results.first
       results.each do |result|
         event = LogStash::Event.new()
         decorate(event)
         @sfdc_fields.each do |field|
-          field_type = @sfdc_field_types[field]
+          # PARENT.CHILD => PARENT
+          # function(field) field => field
+          field = field.split(/\./).first.split(/\s/).last
           value = result.send(field)
+
+          # Remove RESTForce's nested 'attributes' field for reference fields
+          value.is_a?(Hash) ? value = value.tap { |hash| hash.delete(:attributes)} : value
+
           event_key = @to_underscores ? underscore(field) : field
-          if not value.nil?
-            case field_type
-            when 'datetime', 'date'
-              event.set(event_key, format_time(value))
-            else
-              event.set(event_key, value)
-            end
-          end
+          event.set(event_key, value)
         end
         queue << event
       end
@@ -146,8 +146,7 @@ def client_options
       :password       => @password,
       :security_token => @security_token,
       :client_id      => @client_id,
-      :client_secret  => @client_secret,
-      :timeout        => @timeout
+      :client_secret  => @client_secret
     }
     # configure the endpoint to which restforce connects to for authentication
     if @sfdc_instance_url && @use_test_sandbox
@@ -158,6 +157,7 @@ def client_options
       options.merge!({ :host => "test.salesforce.com" })
     end
     options.merge!({ :api_version => @api_version }) if @api_version
+    options.merge!({ :timeout => @timeout }) if @timeout
     return options
   end
 

logstash-input-salesforce.gemspec

@@ -1,6 +1,6 @@
 Gem::Specification.new do |s|
   s.name = 'logstash-input-salesforce'
-  s.version         = '3.2.0'
+  s.version         = '3.3.0'
   s.licenses = ['Apache-2.0']
   s.summary = "Creates events based on a Salesforce SOQL query"
   s.description     = "This gem is a Logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/logstash-plugin install gemname. This gem is not a stand-alone program"
@@ -20,7 +20,7 @@ Gem::Specification.new do |s|
   # Gem dependencies
   s.add_runtime_dependency "logstash-core-plugin-api", ">= 1.60", "<= 2.99"
   s.add_runtime_dependency "logstash-codec-plain"
-  s.add_runtime_dependency "restforce", ">= 5", "< 5.2"
+  s.add_runtime_dependency "restforce", ">= 5", "< 5.3"
   s.add_development_dependency 'logstash-devutils'
   s.add_development_dependency 'vcr'
   s.add_development_dependency 'webmock'