Merge pull request #325 from young-0/master

Fix the bug that multi-level directory /a/*/c/*.log wildcard cannot be matched dynamically

Author: jaqx0r

internal/mtail/mtail.go

@@ -15,6 +15,7 @@ import (
 	"net/http/pprof"
 	"os"
 	"os/signal"
+	"path/filepath"
 	"runtime"
 	"sync"
 	"syscall"
@@ -104,6 +105,25 @@ func (m *Server) StartTailing() error {
 	return nil
 }
 
+// Scan the wildcard directory address regularly and add to the wildcard address scanning
+func (m *Server) PollLogPathPatterns() error {
+	for _, pattern := range m.logPathPatterns {
+		absPath, err := filepath.Abs(pattern)
+		if err != nil {
+			return err
+		}
+		d := filepath.Dir(absPath)
+		if !m.t.HasMeta(d) {
+			continue
+		}
+		glog.V(1).Infof("Tail pattern %q", pattern)
+		if err = m.t.TailPattern(pattern); err != nil {
+			glog.Warning(err)
+		}
+	}
+	return nil
+}
+
 // initLoader constructs a new program loader and performs the initial load of program files in the program directory.
 func (m *Server) initLoader() error {
 	opts := []func(*vm.Loader) error{
@@ -326,7 +346,20 @@ func (m *Server) Serve() error {
 		}
 		errc <- err
 	}()
+	pollQuit := make(chan int, 1) // Channel to stop PollLogPathPatterns.
+	go func(chan int) {
+		ticker := time.NewTicker(time.Second)
+		for {
+			select {
+			case <-ticker.C:
+				m.PollLogPathPatterns()
+			case <-pollQuit:
+				return
+			}
+		}
+	}(pollQuit)
 	m.WaitForShutdown()
+	pollQuit <- 1
 	return <-errc
 }
 

internal/mtail/multiple_levels_directory_integration_test.go

@@ -0,0 +1,55 @@
+// Copyright 2019 Google Inc. All Rights Reserved.
+// This file is available under the Apache license.
+// +build integration
+
+package mtail_test
+
+import (
+	"os"
+	"path"
+	"testing"
+	"time"
+
+	"github.com/golang/glog"
+	"github.com/google/mtail/internal/mtail"
+	"github.com/google/mtail/internal/testutil"
+)
+
+func TestPollLogPathPatterns(t *testing.T) {
+	tmpDir, rmTmpDir := testutil.TestTempDir(t)
+	defer rmTmpDir()
+
+	logDir := path.Join(tmpDir, "logs")
+	progDir := path.Join(tmpDir, "progs")
+	testutil.FatalIfErr(t, os.Mkdir(logDir, 0700))
+	testutil.FatalIfErr(t, os.Mkdir(progDir, 0700))
+	defer testutil.TestChdir(t, logDir)()
+
+	m, stopM := mtail.TestStartServer(t, 10*time.Millisecond, false, mtail.ProgramPath(progDir), mtail.LogPathPatterns(logDir+"/files/*/log/*log"))
+	defer stopM()
+
+	startLogCount := mtail.TestGetMetric(t, m.Addr(), "log_count")
+	startLineCount := mtail.TestGetMetric(t, m.Addr(), "lines_total")
+
+	logFile := path.Join(logDir, "files", "a", "log", "a.log")
+	testutil.FatalIfErr(t, os.MkdirAll(path.Dir(logFile), 0700))
+	f := testutil.TestOpenFile(t, logFile)
+	n, err := f.WriteString("")
+	testutil.FatalIfErr(t, err)
+	time.Sleep(time.Second)
+	f.WriteString("line 1\n")
+	glog.Infof("Wrote %d bytes", n)
+	time.Sleep(time.Second)
+
+	logCount := mtail.TestGetMetric(t, m.Addr(), "log_count")
+	lineCount := mtail.TestGetMetric(t, m.Addr(), "lines_total")
+
+	if logCount.(float64)-startLogCount.(float64) != 1. {
+		t.Errorf("Unexpected log count: got %g, want 1", logCount.(float64)-startLogCount.(float64))
+	}
+	if lineCount.(float64)-startLineCount.(float64) != 1. {
+		t.Errorf("Unexpected line count: got %g, want 1", lineCount.(float64)-startLineCount.(float64))
+	}
+	time.Sleep(time.Second)
+
+}

internal/tailer/tail.go

@@ -19,6 +19,8 @@ import (
 	"os"
 	"path/filepath"
 	"regexp"
+	"runtime"
+	"strings"
 	"sync"
 	"time"
 
@@ -263,9 +265,20 @@ func (t *Tailer) watchDirname(pathname string) error {
 		return err
 	}
 	d := filepath.Dir(absPath)
+	if t.HasMeta(d) {
+		return nil
+	}
 	return t.w.Observe(d, t)
 }
 
+func (t *Tailer) HasMeta(path string) bool {
+	magicChars := `*?[`
+	if runtime.GOOS != "windows" {
+		magicChars = `*?[\`
+	}
+	return strings.ContainsAny(path, magicChars)
+}
+
 // openLogPath opens a log file named by pathname.
 func (t *Tailer) openLogPath(pathname string, seekToStart bool) error {
 	glog.V(2).Infof("openlogPath %s %v", pathname, seekToStart)