Test coverage and new config

.drone.yml

@@ -0,0 +1,13 @@
+build:
+  image: golang
+  commands:
+    - make test
+
+notify:
+  slack:
+    webhook_url: $$SLACK_WEBHOOK
+    channel: general
+    username: $$SLACK_USERNAME
+    template: >
+      [{{ repo.full_name }}] <{{ system.link_url }}/{{ repo.full_name }}/{{ build.number }}|build #{{ build.number }}> on branch *{{ build.branch }}* by *{{ build.author }}* finished with a *{{ build.status }}* status.
+

Makefile

@@ -13,7 +13,7 @@ build: clean deps
 clean:
 	rm -rf $(DIST_DIR)
 
-test:
+test: clean deps
 	@go test -run=. -test.v ./config
 	@go test -run=. -test.v ./database
 	@go test -run=. -test.v ./dest

README.md

@@ -30,7 +30,7 @@ $ curl -s https://packagecloud.io/install/repositories/gregarmer/packages/script
 ## Getting Started
 
 1. Follow the installation steps above.
-2. Configure ~/.s3pgbackups - something like this:
+2. Configure ~/.s3pgbackups or your-config.cfg - something like this:
 
     ```json
     {
@@ -41,27 +41,31 @@ $ curl -s https://packagecloud.io/install/repositories/gregarmer/packages/script
       "pg_username": "username",
       "pg_password": "password",
       "pg_sslmode": true,
-      "pg_exclude_dbs": ["postgres", "template0"],
-      "pg_exclude_tables": ["django_session"]
+      "excludes": ["postgres", "template0", "*.django_session", "db1.table3"],
     }
     ```
 
     > Note: Running s3pgbackups without a config will create a blank config.
-3. Run `s3pgbackups`
+3. Run `s3pgbackups -c your-config.cfg` or alternatively `s3pgbackups`
 
 ## Usage
 
 ```
 $ s3pgbackups -h
-Usage of s3pgbackups:
-  -n=false: don't actually do anything, just print what would be done
-  -v=false: be verbose
+Usage of dist/s3pgbackups:
+  -c string  path to the config file (default "~/.s3pgbackups")
+  -n         don't actually do anything, just print what would be done
+  -v         be verbose
 ```
 
-You can use the `-n` parameter to put s3pgbackups into no-op mode, where no actions
-will actually be performed.
+`-c config.cfg` is optional. If you don't specify a config it'll default to
+`~/.s3pgbackups` and will create this file if it doesn't already exist.
 
-The `-v` parameter will make s3pgbackups be verbose about what is actually happening.
+You can use the `-n` parameter to put s3pgbackups into no-op mode, where no
+actions will actually be performed.
+
+The `-v` parameter will make s3pgbackups be verbose about what is actually
+happening.
 
 ## TODO
 

config/config.go

@@ -3,6 +3,7 @@ package config
 import (
 	"encoding/json"
 	"errors"
+	"fmt"
 	"github.com/gregarmer/s3pgbackups/utils"
 	"io/ioutil"
 	"os"
@@ -22,32 +23,41 @@ type Config struct {
 	S3RotateOld bool   `json:"s3_rotate_old"`
 
 	// PostgreSQL
-	PostgresUsername     string   `json:"pg_username"`
-	PostgresPassword     string   `json:"pg_password"`
-	PostgresSSL          bool     `json:"pg_sslmode"`
-	PostgresExcludeDb    []string `json:"pg_exclude_dbs"`
-	PostgresExcludeTable []string `json:"pg_exclude_tables"`
-}
+	PostgresUsername string `json:"pg_username"`
+	PostgresPassword string `json:"pg_password"`
+	PostgresSSL      bool   `json:"pg_sslmode"`
 
-func _shouldExclude(item string, excludes []string) bool {
-	for _, b := range excludes {
-		if b == item {
-			return true
-		}
-	}
-	return false
+	// New style excludes - see issue #1 - example:
+	// ["database.*", "*.table", "database"]
+	Excludes []string `json:"excludes"`
 }
 
 func (c *Config) Copy() Config {
 	return *c
 }
 
 func (c *Config) ShouldExcludeDb(db string) bool {
-	return _shouldExclude(db, c.PostgresExcludeDb)
+	for _, cmp := range c.Excludes {
+		if cmp == fmt.Sprintf("%s.*", db) || cmp == db {
+			return true
+		}
+	}
+	return false
 }
 
-func (c *Config) ShouldExcludeTable(table string) bool {
-	return _shouldExclude(table, c.PostgresExcludeTable)
+func (c *Config) ShouldExcludeTable(db string, t string) bool {
+	for _, cmp := range c.Excludes {
+		// all tables are excluded for this db
+		if cmp == fmt.Sprintf("%s.*", db) {
+			return true
+		}
+
+		// table match is excluded for this db
+		if cmp == fmt.Sprintf("%s.%s", db, t) || cmp == fmt.Sprintf("*.%s", t) {
+			return true
+		}
+	}
+	return false
 }
 
 func (c *Config) PreFlight() error {
@@ -68,16 +78,23 @@ func GetConfigPath() string {
 	return filepath.Join(u.HomeDir, configFile)
 }
 
-func LoadConfig() *Config {
+func LoadConfig(configFile string) *Config {
+	// make sure the config file actually exists
+	if _, err := os.Stat(configFile); os.IsNotExist(err) {
+		utils.Fatalf("couldn't load config from %s", configFile)
+	}
+
 	// init config if needed
-	configPath := GetConfigPath()
-	if _, err := os.Stat(configPath); os.IsNotExist(err) {
-		InitConfig()
-		utils.Fatalf("couldn't find config, created empty config at %s, please configure", configPath)
+	defaultConfigPath := GetConfigPath()
+	if configFile == defaultConfigPath {
+		if _, err := os.Stat(configFile); os.IsNotExist(err) {
+			InitConfig()
+			utils.Fatalf("created empty config at %s, please configure", configFile)
+		}
 	}
 
 	// load config
-	file, _ := os.Open(configPath)
+	file, _ := os.Open(configFile)
 	decoder := json.NewDecoder(file)
 	config := Config{}
 	if err := decoder.Decode(&config); err != nil {

config/config_test.go

@@ -18,3 +18,74 @@ func TestCopy(t *testing.T) {
 		t.Fatalf("config.Copy() should return a copy that doesn't affect the original")
 	}
 }
+
+func TestShouldExcludeDbWithNoExcludes(t *testing.T) {
+	conf := Config{}
+	var testDatabase string
+
+	// includes
+	testDatabase = "foobar"
+	if conf.ShouldExcludeDb(testDatabase) != false {
+		t.Fatalf("%s should not have been excluded.", testDatabase)
+	}
+}
+
+func TestShouldExcludeDb(t *testing.T) {
+	conf := Config{Excludes: []string{"foo", "*.bar", "baz.*", "quux.*"}}
+	var testDatabase string
+
+	// excludes
+	testDatabase = "foo"
+	if conf.ShouldExcludeDb(testDatabase) != true {
+		t.Fatalf("%s should have been excluded.", testDatabase)
+	}
+
+	testDatabase = "quux"
+	if conf.ShouldExcludeDb(testDatabase) != true {
+		t.Fatalf("%s should have been excluded.", testDatabase)
+	}
+
+	// includes
+	testDatabase = "foobar"
+	if conf.ShouldExcludeDb(testDatabase) != false {
+		t.Fatalf("%s should not have been excluded.", testDatabase)
+	}
+}
+
+func TestShouldExcludeTable(t *testing.T) {
+	conf := Config{Excludes: []string{"foo", "*.bar", "baz.*", "quux.gah"}}
+	var testDatabase string
+	var testTable string
+
+	// excludes
+	testDatabase = "foo"
+	testTable = "bar"
+	if conf.ShouldExcludeTable(testDatabase, testTable) != true {
+		t.Fatalf("%s.%s should have been excluded.", testDatabase, testTable)
+	}
+
+	testDatabase = "baz"
+	testTable = "quux"
+	if conf.ShouldExcludeTable(testDatabase, testTable) != true {
+		t.Fatalf("%s.%s should have been excluded.", testDatabase, testTable)
+	}
+
+	testDatabase = "quux"
+	testTable = "gah"
+	if conf.ShouldExcludeTable(testDatabase, testTable) != true {
+		t.Fatalf("%s.%s should have been excluded.", testDatabase, testTable)
+	}
+
+	// includes
+	testDatabase = "foo"
+	testTable = "gah"
+	if conf.ShouldExcludeTable(testDatabase, testTable) != false {
+		t.Fatalf("%s.%s should not have been excluded.", testDatabase, testTable)
+	}
+
+	testDatabase = "foobar"
+	testTable = "gah"
+	if conf.ShouldExcludeTable(testDatabase, testTable) != false {
+		t.Fatalf("%s.%s should not have been excluded.", testDatabase, testTable)
+	}
+}

database/postgres.go

@@ -17,8 +17,7 @@ type Postgres struct {
 	Config *config.Config
 }
 
-func (postgres *Postgres) GetDatabases() []string {
-	var databases []string
+func (postgres *Postgres) GetDSN() string {
 	var sslmode string
 
 	if postgres.Config.PostgresSSL {
@@ -32,7 +31,13 @@ func (postgres *Postgres) GetDatabases() []string {
 		postgres.Config.PostgresPassword,
 		sslmode)
 
-	db, err := sql.Open("postgres", dsn)
+	return dsn
+}
+
+func (postgres *Postgres) GetDatabases() []string {
+	var databases []string
+
+	db, err := sql.Open("postgres", postgres.GetDSN())
 	utils.CheckErr(err)
 
 	rows, err := db.Query("SELECT datname FROM pg_database")
@@ -52,13 +57,48 @@ func (postgres *Postgres) GetDatabases() []string {
 	return databases
 }
 
+func (postgres *Postgres) GetTables(database string) []string {
+	var tables []string
+
+	db, err := sql.Open(database, postgres.GetDSN())
+	utils.CheckErr(err)
+
+	tablesQuery := "SELECT table_name FROM information_schema.tables "
+	tablesQuery += "WHERE table_schema = 'public'"
+	rows, err := db.Query(tablesQuery)
+	utils.CheckErr(err)
+	defer rows.Close()
+
+	for rows.Next() {
+		var name string
+		err = rows.Scan(&name)
+		utils.CheckErr(err)
+		tables = append(tables, name)
+	}
+
+	err = rows.Err() // get any error encountered during iteration
+	utils.CheckErr(err)
+
+	return tables
+}
+
 func (postgres *Postgres) DumpDatabase(db, workingDir string) string {
 	backupFileName := fmt.Sprintf("%s-%s.sql", db,
 		time.Now().Format("2006-01-02"))
-	pgDumpCmd := fmt.Sprintf("-E UTF-8 -T %s -f %s %s",
-		strings.Join(postgres.Config.PostgresExcludeTable, " -T "),
-		fmt.Sprintf("%s/%s", workingDir, backupFileName),
-		db)
+
+	pgDumpCmd := fmt.Sprintf("-E UTF-8 -f %s ",
+		fmt.Sprintf("%s/%s", workingDir, backupFileName))
+
+	if postgres.Config.Excludes != nil {
+		for _, table := range postgres.GetTables(db) {
+			if postgres.Config.ShouldExcludeTable(db, table) {
+				pgDumpCmd += fmt.Sprintf("-T %s ", table)
+			}
+		}
+	}
+
+	pgDumpCmd += db
+
 	log.Printf("executing pg_dump %s", pgDumpCmd)
 	cmd := exec.Command("pg_dump", strings.Split(pgDumpCmd, " ")...)
 	var out bytes.Buffer

database/postgres_test.go

@@ -1,6 +1,35 @@
 package database
 
-import "testing"
+import (
+	"github.com/gregarmer/s3pgbackups/config"
+	"testing"
+)
 
-func TestBasic(t *testing.T) {
+var testConfig = config.Config{
+	AwsAccessKey:     "access",
+	AwsSecretKey:     "secret",
+	PostgresUsername: "foo",
+	PostgresPassword: "bar",
+}
+
+func TestGetDSNWithSSL(t *testing.T) {
+	testConfig.PostgresSSL = true
+	postgres := Postgres{Config: &testConfig}
+	expectedDsn := "user=foo password=bar dbname=postgres sslmode=require"
+	actualDsn := postgres.GetDSN()
+	if expectedDsn != actualDsn {
+		t.Fatalf("GetDSN with SSL failed. Expected '%s' got '%s'",
+			expectedDsn, actualDsn)
+	}
+}
+
+func TestGetDSNWithoutSSL(t *testing.T) {
+	testConfig.PostgresSSL = false
+	postgres := Postgres{Config: &testConfig}
+	expectedDsn := "user=foo password=bar dbname=postgres sslmode=disable"
+	actualDsn := postgres.GetDSN()
+	if expectedDsn != actualDsn {
+		t.Fatalf("GetDSN without SSL failed. Expected '%s' got '%s'",
+			expectedDsn, actualDsn)
+	}
 }

main.go

@@ -17,6 +17,7 @@ import (
 
 const workingDir = "temp"
 
+var configFile = flag.String("c", "~/.s3pgbackups", "path to the config file")
 var verbose = flag.Bool("v", false, "be verbose")
 var noop = flag.Bool("n", false,
 	"don't actually do anything, just print what would be done")
@@ -36,7 +37,7 @@ func main() {
 		log.Printf("running in no-op mode, no commands will really be executed")
 	}
 
-	conf := config.LoadConfig()
+	conf := config.LoadConfig(*configFile)
 
 	// Don't print real passwords and secret keys in verbose mode
 	verbose_config := conf.Copy()