wip; issue-1829 global TTL

controller/execute.go

@@ -136,6 +136,8 @@
 	zoneTypeFilter := provider.NewZoneTypeFilter(cfg.AWSZoneType)
 	zoneTagFilter := provider.NewZoneTagFilter(cfg.AWSZoneTagFilter)
 
+	bcfg := provider.NewBaseConfig(cfg.ProviderTTL)
+
 	var p provider.Provider
 	switch cfg.Provider {
 	case "akamai":
@@ -150,7 +152,7 @@
 				EdgercPath:            cfg.AkamaiEdgercPath,
 				EdgercSection:         cfg.AkamaiEdgercSection,
 				DryRun:                cfg.DryRun,
 			}, nil)
 	case "alibabacloud":
 		p, err = alibabacloud.NewAlibabaCloudProvider(cfg.AlibabaCloudConfigFile, domainFilter, zoneIDFilter, cfg.AlibabaCloudZoneType, cfg.DryRun)
 	case "aws":
@@ -161,6 +163,7 @@
 		}
 
 		p, err = aws.NewAWSProvider(
+			bcfg,
 			aws.AWSConfig{
 				DomainFilter:          domainFilter,
 				ZoneIDFilter:          zoneIDFilter,
@@ -184,7 +187,7 @@
 			log.Infof("Registry \"%s\" cannot be used with AWS Cloud Map. Switching to \"aws-sd\".", cfg.Registry)
 			cfg.Registry = "aws-sd"
 		}
-		p, err = awssd.NewAWSSDProvider(domainFilter, cfg.AWSZoneType, cfg.DryRun, cfg.AWSSDServiceCleanup, cfg.TXTOwnerID, cfg.AWSSDCreateTag, sd.NewFromConfig(aws.CreateDefaultV2Config(cfg)))
+		p, err = awssd.NewAWSSDProvider(bcfg, domainFilter, cfg.AWSZoneType, cfg.DryRun, cfg.AWSSDServiceCleanup, cfg.TXTOwnerID, cfg.AWSSDCreateTag, sd.NewFromConfig(aws.CreateDefaultV2Config(cfg)))
 	case "azure-dns", "azure":
 		p, err = azure.NewAzureProvider(cfg.AzureConfigFile, domainFilter, zoneNameFilter, zoneIDFilter, cfg.AzureSubscriptionID, cfg.AzureResourceGroup, cfg.AzureUserAssignedIdentityClientID, cfg.AzureActiveDirectoryAuthorityHost, cfg.AzureZonesCacheDuration, cfg.DryRun)
 	case "azure-private-dns":
@@ -207,15 +210,15 @@
 				CertificateAuthority: cfg.CloudflareCustomHostnamesCertificateAuthority,
 			})
 	case "google":
 		p, err = google.NewGoogleProvider(ctx, cfg.GoogleProject, domainFilter, zoneIDFilter, cfg.GoogleBatchChangeSize, cfg.GoogleBatchChangeInterval, cfg.GoogleZoneVisibility, cfg.DryRun)
 	case "digitalocean":
 		p, err = digitalocean.NewDigitalOceanProvider(ctx, domainFilter, cfg.DryRun, cfg.DigitalOceanAPIPageSize)
 	case "ovh":
 		p, err = ovh.NewOVHProvider(ctx, domainFilter, cfg.OVHEndpoint, cfg.OVHApiRateLimit, cfg.OVHEnableCNAMERelative, cfg.DryRun)
 	case "linode":
 		p, err = linode.NewLinodeProvider(domainFilter, cfg.DryRun)
 	case "dnsimple":
 		p, err = dnsimple.NewDnsimpleProvider(domainFilter, zoneIDFilter, cfg.DryRun)
 	case "coredns", "skydns":
 		p, err = coredns.NewCoreDNSProvider(domainFilter, cfg.CoreDNSPrefix, cfg.DryRun)
 	case "exoscale":

pkg/apis/externaldns/types.go

@@ -38,6 +38,12 @@ const (
 
 // Config is a project-wide configuration
 type Config struct {
+	// global provider configs
+	Provider          string
+	ProviderCacheTime time.Duration
+	ProviderTTL       int
+	DryRun            bool
+
 	APIServerURL                                  string
 	KubeConfig                                    string
 	RequestTimeout                                time.Duration
@@ -66,8 +72,6 @@ type Config struct {
 	PublishHostIP                                 bool
 	AlwaysPublishNotReadyAddresses                bool
 	ConnectorSourceServer                         string
-	Provider                                      string
-	ProviderCacheTime                             time.Duration
 	GoogleProject                                 string
 	GoogleBatchChangeSize                         int
 	GoogleBatchChangeInterval                     time.Duration
@@ -146,7 +150,6 @@ type Config struct {
 	Interval                                      time.Duration
 	MinEventSyncInterval                          time.Duration
 	Once                                          bool
-	DryRun                                        bool
 	UpdateEvents                                  bool
 	LogFormat                                     string
 	MetricsAddress                                string
@@ -219,6 +222,7 @@ type Config struct {
 var defaultConfig = &Config{
 	APIServerURL:                           "",
 	KubeConfig:                             "",
+	ProviderTTL:                            -1,
 	RequestTimeout:                         time.Second * 30,
 	DefaultTargets:                         []string{},
 	GlooNamespaces:                         []string{"gloo-system"},
@@ -452,7 +456,7 @@ func App(cfg *Config) *kingpin.Application {
 	// Flags related to Skipper RouteGroup
 	app.Flag("skipper-routegroup-groupversion", "The resource version for skipper routegroup").Default(defaultConfig.SkipperRouteGroupVersion).StringVar(&cfg.SkipperRouteGroupVersion)
 
-	// Flags related to processing source
+	// Flags related to a processing source
 	app.Flag("source", "The resource types that are queried for endpoints; specify multiple times for multiple sources (required, options: service, ingress, node, pod, fake, connector, gateway-httproute, gateway-grpcroute, gateway-tlsroute, gateway-tcproute, gateway-udproute, istio-gateway, istio-virtualservice, cloudfoundry, contour-httpproxy, gloo-proxy, crd, empty, skipper-routegroup, openshift-route, ambassador-host, kong-tcpingress, f5-virtualserver, f5-transportserver, traefik-proxy)").Required().PlaceHolder("source").EnumsVar(&cfg.Sources, "service", "ingress", "node", "pod", "gateway-httproute", "gateway-grpcroute", "gateway-tlsroute", "gateway-tcproute", "gateway-udproute", "istio-gateway", "istio-virtualservice", "cloudfoundry", "contour-httpproxy", "gloo-proxy", "fake", "connector", "crd", "empty", "skipper-routegroup", "openshift-route", "ambassador-host", "kong-tcpingress", "f5-virtualserver", "f5-transportserver", "traefik-proxy")
 	app.Flag("openshift-router-name", "if source is openshift-route then you can pass the ingress controller name. Based on this name external-dns will select the respective router from the route status and map that routerCanonicalHostname to the route host while creating a CNAME record.").StringVar(&cfg.OCPRouterName)
 	app.Flag("namespace", "Limit resources queried for endpoints to a specific namespace (default: all namespaces)").Default(defaultConfig.Namespace).StringVar(&cfg.Namespace)
@@ -492,6 +496,9 @@ func App(cfg *Config) *kingpin.Application {
 	providers := []string{"akamai", "alibabacloud", "aws", "aws-sd", "azure", "azure-dns", "azure-private-dns", "civo", "cloudflare", "coredns", "digitalocean", "dnsimple", "exoscale", "gandi", "godaddy", "google", "ibmcloud", "inmemory", "linode", "ns1", "oci", "ovh", "pdns", "pihole", "plural", "rfc2136", "scaleway", "skydns", "tencentcloud", "transip", "ultradns", "webhook"}
 	app.Flag("provider", "The DNS provider where the DNS records will be created (required, options: "+strings.Join(providers, ", ")+")").Required().PlaceHolder("provider").EnumVar(&cfg.Provider, providers...)
 	app.Flag("provider-cache-time", "The time to cache the DNS provider record list requests.").Default(defaultConfig.ProviderCacheTime.String()).DurationVar(&cfg.ProviderCacheTime)
+	app.Flag("provider-ttl", "TTL (in seconds) for records. This value will be used if the TTL is not configured. (optional, default: fallback to provider defaults when not set or set to 0)").IntVar(&cfg.ProviderTTL)
+
+	// Flags related to filters
 	app.Flag("domain-filter", "Limit possible target zones by a domain suffix; specify multiple times for multiple domains (optional)").Default("").StringsVar(&cfg.DomainFilter)
 	app.Flag("exclude-domains", "Exclude subdomains (optional)").Default("").StringsVar(&cfg.ExcludeDomains)
 	app.Flag("regex-domain-filter", "Limit possible domains and target zones by a Regex filter; Overrides domain-filter (optional)").Default(defaultConfig.RegexDomainFilter.String()).RegexpVar(&cfg.RegexDomainFilter)
@@ -563,7 +570,7 @@ func App(cfg *Config) *kingpin.Application {
 	// GoDaddy flags
 	app.Flag("godaddy-api-key", "When using the GoDaddy provider, specify the API Key (required when --provider=godaddy)").Default(defaultConfig.GoDaddyAPIKey).StringVar(&cfg.GoDaddyAPIKey)
 	app.Flag("godaddy-api-secret", "When using the GoDaddy provider, specify the API secret (required when --provider=godaddy)").Default(defaultConfig.GoDaddySecretKey).StringVar(&cfg.GoDaddySecretKey)
-	app.Flag("godaddy-api-ttl", "TTL (in seconds) for records. This value will be used if the provided TTL for a service/ingress is not provided.").Int64Var(&cfg.GoDaddyTTL)
+	app.Flag("godaddy-api-ttl", "TTL (in seconds) for records. This value will be used if the provided TTL for a service/ingress is not provided. Deprecated and about to be removed in future releases").Int64Var(&cfg.GoDaddyTTL)
 	app.Flag("godaddy-api-ote", "When using the GoDaddy provider, use OTE api (optional, default: false, when --provider=godaddy)").BoolVar(&cfg.GoDaddyOTE)
 
 	// Flags related to TLS communication

pkg/apis/externaldns/types_test.go

@@ -18,6 +18,7 @@ package externaldns
 
 import (
 	"os"
+	"reflect"
 	"regexp"
 	"strings"
 	"testing"
@@ -561,3 +562,56 @@ func TestPasswordsNotLogged(t *testing.T) {
 	assert.False(t, strings.Contains(s, "pdns-api-key"))
 	assert.False(t, strings.Contains(s, "tsig-secret"))
 }
+
+// Test for default values
+func TestParseDefaultFlags(t *testing.T) {
+	for _, ti := range []struct {
+		title         string
+		args          []string
+		compareFields []string
+		expected      *Config
+	}{
+		{
+			title: "default config with minimal flags defined and ttl is default",
+			args: []string{
+				"--provider=google",
+				"--source=service",
+			},
+			expected: &Config{
+				ProviderTTL: 0,
+			},
+			compareFields: []string{"ProviderTTL"},
+		},
+		{
+			title: "default config with minimal flags defined",
+			args: []string{
+				"--provider=google",
+				"--source=service",
+				"--provider-ttl=10",
+			},
+			expected: &Config{
+				ProviderTTL: 10,
+			},
+			compareFields: []string{"ProviderTTL"},
+		},
+	} {
+		t.Run(ti.title, func(t *testing.T) {
+
+			gotCfg := ti.expected
+
+			wantCfg := NewConfig()
+			require.NoError(t, wantCfg.ParseFlags(ti.args))
+
+			wantVal := reflect.ValueOf(wantCfg).Elem()
+			gotValue := reflect.ValueOf(gotCfg).Elem()
+			for _, fieldName := range ti.compareFields {
+				wantField := wantVal.FieldByName(fieldName)
+				gotField := gotValue.FieldByName(fieldName)
+
+				assert.True(t, wantField.IsValid())
+				assert.True(t, gotField.IsValid())
+				assert.Equal(t, wantField.Interface(), gotField.Interface())
+			}
+		})
+	}
+}

provider/akamai/akamai.go

@@ -34,9 +34,9 @@ import (
 
 const (
 	// Default Record TTL
-	edgeDNSRecordTTL = 600
-	maxUint          = ^uint(0)
-	maxInt           = int(maxUint >> 1)
+	defaultTTL = 600 + 10
+	maxUint    = ^uint(0)
+	maxInt     = int(maxUint >> 1)
 )
 
 // edgeDNSClient is a proxy interface of the Akamai edgegrid configdns-v2 package that can be stubbed for testing.
@@ -66,6 +66,7 @@ type AkamaiConfig struct {
 // AkamaiProvider implements the DNS provider for Akamai.
 type AkamaiProvider struct {
 	provider.BaseProvider
+	bcfg provider.BaseConfig
 	// Edgedns zones to filter on
 	domainFilter endpoint.DomainFilter
 	// Contract Ids to filter on
@@ -87,17 +88,9 @@ type akamaiZone struct {
 }
 
 // NewAkamaiProvider initializes a new Akamai DNS based Provider.
-func NewAkamaiProvider(akamaiConfig AkamaiConfig, akaService AkamaiDNSService) (provider.Provider, error) {
+func NewAkamaiProvider(bcfg provider.BaseConfig, akamaiConfig AkamaiConfig, akaService AkamaiDNSService) (provider.Provider, error) {
 	var edgeGridConfig edgegrid.Config
 
-	/*
-		log.Debugf("Host: %s", akamaiConfig.ServiceConsumerDomain)
-		log.Debugf("ClientToken: %s", akamaiConfig.ClientToken)
-		log.Debugf("ClientSecret: %s", akamaiConfig.ClientSecret)
-		log.Debugf("AccessToken: %s", akamaiConfig.AccessToken)
-		log.Debugf("EdgePath: %s", akamaiConfig.EdgercPath)
-		log.Debugf("EdgeSection: %s", akamaiConfig.EdgercSection)
-	*/
 	// environment overrides edgerc file but config needs to be complete
 	if akamaiConfig.ServiceConsumerDomain == "" || akamaiConfig.ClientToken == "" || akamaiConfig.ClientSecret == "" || akamaiConfig.AccessToken == "" {
 		// Kubernetes config incomplete or non existent. Can't mix and match.
@@ -142,6 +135,7 @@ func NewAkamaiProvider(akamaiConfig AkamaiConfig, akaService AkamaiDNSService) (
 	}
 
 	provider := &AkamaiProvider{
+		bcfg:         bcfg,
 		domainFilter: akamaiConfig.DomainFilter,
 		zoneIDFilter: akamaiConfig.ZoneIDFilter,
 		config:       &edgeGridConfig,
@@ -357,10 +351,10 @@ func trimTxtRdata(rdata []string, rtype string) []string {
 	return rdata
 }
 
-func ttlAsInt(src endpoint.TTL) int {
+func ttlAsInt(bcfg provider.BaseConfig, src endpoint.TTL) int {
 	var temp interface{} = int64(src)
 	temp64 := temp.(int64)
-	var ttl = edgeDNSRecordTTL
+	var ttl = bcfg.MinTtl(defaultTTL)
 	if temp64 > 0 && temp64 <= int64(maxInt) {
 		ttl = int(temp64)
 	}
@@ -380,19 +374,19 @@ func (p AkamaiProvider) createRecordsets(zoneNameIDMapper provider.ZoneIDName, e
 	// create all recordsets by zone
 	for zone, endpoints := range endpointsByZone {
 		recordsets := &dns.Recordsets{Recordsets: make([]dns.Recordset, 0)}
-		for _, endpoint := range endpoints {
-			newrec := newAkamaiRecordset(endpoint.DNSName,
-				endpoint.RecordType,
-				ttlAsInt(endpoint.RecordTTL),
-				cleanTargets(endpoint.RecordType, endpoint.Targets...))
-			logfields := log.Fields{
+		for _, endt := range endpoints {
+			newrec := newAkamaiRecordset(endt.DNSName,
+				endt.RecordType,
+				ttlAsInt(p.bcfg, endt.RecordTTL),
+				cleanTargets(endt.RecordType, endt.Targets...))
+			logFields := log.Fields{
 				"record": newrec.Name,
 				"type":   newrec.Type,
 				"ttl":    newrec.TTL,
 				"target": fmt.Sprintf("%v", newrec.Rdata),
 				"zone":   zone,
 			}
-			log.WithFields(logfields).Info("Creating recordsets")
+			log.WithFields(logFields).Info("Creating recordsets")
 			recordsets.Recordsets = append(recordsets.Recordsets, newrec)
 		}
 
@@ -443,26 +437,26 @@ func (p AkamaiProvider) deleteRecordsets(zoneNameIDMapper provider.ZoneIDName, e
 
 // Update endpoint recordsets
 func (p AkamaiProvider) updateNewRecordsets(zoneNameIDMapper provider.ZoneIDName, endpoints []*endpoint.Endpoint) error {
-	for _, endpoint := range endpoints {
-		zoneName, _ := zoneNameIDMapper.FindZone(endpoint.DNSName)
+	for _, endt := range endpoints {
+		zoneName, _ := zoneNameIDMapper.FindZone(endt.DNSName)
 		if zoneName == "" {
-			log.Debugf("Skipping Akamai Edge DNS endpoint update: '%s' type: '%s', it does not match against Domain filters", endpoint.DNSName, endpoint.RecordType)
+			log.Debugf("Skipping Akamai Edge DNS endpoint update: '%s' type: '%s', it does not match against Domain filters", endt.DNSName, endt.RecordType)
 			continue
 		}
-		log.Infof("Akamai Edge DNS recordset update - Zone: '%s', DNSName: '%s', RecordType: '%s', Targets: '%+v'", zoneName, endpoint.DNSName, endpoint.RecordType, endpoint.Targets)
+		log.Infof("Akamai Edge DNS recordset update - Zone: '%s', DNSName: '%s', RecordType: '%s', Targets: '%+v'", zoneName, endt.DNSName, endt.RecordType, endt.Targets)
 
 		if p.dryRun {
 			continue
 		}
 
-		recName := strings.TrimSuffix(endpoint.DNSName, ".")
-		rec, err := p.client.GetRecord(zoneName, recName, endpoint.RecordType)
+		recName := strings.TrimSuffix(endt.DNSName, ".")
+		rec, err := p.client.GetRecord(zoneName, recName, endt.RecordType)
 		if err != nil {
 			log.Errorf("Endpoint update. Record validation failed. Error: %s", err.Error())
 			return err
 		}
-		rec.TTL = ttlAsInt(endpoint.RecordTTL)
-		rec.Target = cleanTargets(endpoint.RecordType, endpoint.Targets...)
+		rec.TTL = ttlAsInt(p.bcfg, endt.RecordTTL)
+		rec.Target = cleanTargets(endt.RecordType, endt.Targets...)
 		if err := p.client.UpdateRecord(rec, zoneName, true); err != nil {
 			log.Errorf("Akamai Edge DNS recordset update failed. Error: %s", err.Error())
 			return err

provider/akamai/akamai_test.go

@@ -57,7 +57,7 @@ func createAkamaiStubProvider(stub *edgednsStub, domfilter endpoint.DomainFilter
 		AccessToken:           "test_access_token",
 	}
 
-	prov, err := NewAkamaiProvider(akamaiConfig, stub)
+	prov, err := NewAkamaiProvider(provider.BaseConfig{}, akamaiConfig, stub)
 	aprov := prov.(*AkamaiProvider)
 	return aprov, err
 }

provider/alibabacloud/alibaba_cloud.go

@@ -29,7 +29,7 @@ import (
 	"github.com/aliyun/alibaba-cloud-sdk-go/services/alidns"
 	"github.com/aliyun/alibaba-cloud-sdk-go/services/pvtz"
 	"github.com/denverdino/aliyungo/metadata"
-	yaml "github.com/goccy/go-yaml"
+	"github.com/goccy/go-yaml"
 	log "github.com/sirupsen/logrus"
 
 	"sigs.k8s.io/external-dns/endpoint"
@@ -38,7 +38,7 @@ import (
 )
 
 const (
-	defaultAlibabaCloudRecordTTL            = 600
+	defaultTTL                              = 600 + 10
 	defaultAlibabaCloudPrivateZoneRecordTTL = 60
 	defaultAlibabaCloudPageSize             = 50
 	nullHostAlibabaCloud                    = "@"
@@ -70,6 +70,7 @@ type AlibabaCloudPrivateZoneAPI interface {
 // AlibabaCloudProvider implements the DNS provider for Alibaba Cloud.
 type AlibabaCloudProvider struct {
 	provider.BaseProvider
+	bcfg                 provider.BaseConfig
 	domainFilter         endpoint.DomainFilter
 	zoneIDFilter         provider.ZoneIDFilter // Private Zone only
 	MaxChangeCount       int
@@ -606,12 +607,12 @@ func (p *AlibabaCloudProvider) deleteRecords(recordMap map[string][]alidns.Recor
 
 func (p *AlibabaCloudProvider) equals(record alidns.Record, endpoint *endpoint.Endpoint) bool {
 	ttl1 := record.TTL
-	if ttl1 == defaultAlibabaCloudRecordTTL {
+	if ttl1 == defaultTTL {
 		ttl1 = 0
 	}
 
 	ttl2 := int64(endpoint.RecordTTL)
-	if ttl2 == defaultAlibabaCloudRecordTTL {
+	if ttl2 == defaultTTL {
 		ttl2 = 0
 	}
 

provider/alibabacloud/alibaba_cloud_test.go

@@ -224,17 +224,17 @@ func newTestAlibabaCloudProvider(private bool) *AlibabaCloudProvider {
 		VPCID: "vpc-xxxxxx",
 	}
 	//
-	//dnsClient, _ := alidns.NewClientWithAccessKey(
+	// dnsClient, _ := alidns.NewClientWithAccessKey(
 	//	cfg.RegionID,
 	//	cfg.AccessKeyID,
 	//	cfg.AccessKeySecret,
-	//)
+	// )
 	//
-	//pvtzClient, _ := pvtz.NewClientWithAccessKey(
+	// pvtzClient, _ := pvtz.NewClientWithAccessKey(
 	//	"cn-hangzhou",
 	//	cfg.AccessKeyID,
 	//	cfg.AccessKeySecret,
-	//)
+	// )
 	domainFilterTest := endpoint.NewDomainFilter([]string{"container-service.top.", "example.org"})
 
 	return &AlibabaCloudProvider{
@@ -282,7 +282,7 @@ func TestAlibabaCloudProvider_ApplyChanges(t *testing.T) {
 	defaultTtlPlan := &endpoint.Endpoint{
 		DNSName:    "ttl.container-service.top",
 		RecordType: "A",
-		RecordTTL:  defaultAlibabaCloudRecordTTL,
+		RecordTTL:  defaultTTL,
 		Targets:    endpoint.NewTargets("4.3.2.1"),
 	}
 	changes := plan.Changes{