Merge branch 'master' into google-metadata-dev

Fix up conflict in backends/client.go

backends/client.go

@@ -34,7 +34,7 @@ func New(config Config) (StoreClient, error) {
 	backendNodes := config.BackendNodes
 
 	if config.Backend == "file" {
-		log.Info("Backend source(s) set to " + config.YAMLFile)
+		log.Info("Backend source(s) set to " + strings.Join(config.YAMLFile, ", "))
 	} else {
 		log.Info("Backend source(s) set to " + strings.Join(backendNodes, ", "))
 	}
@@ -63,7 +63,7 @@ func New(config Config) (StoreClient, error) {
 	case "env":
 		return env.NewEnvClient()
 	case "file":
-		return file.NewFileClient(config.YAMLFile)
+		return file.NewFileClient(config.YAMLFile, config.Filter)
 	case "google":
 		return google.NewGoogleClient(backendNodes[0])
 	case "vault":

backends/config.go

@@ -1,23 +1,28 @@
 package backends
 
+import (
+	util "github.com/kelseyhightower/confd/util"
+)
+
 type Config struct {
-	AuthToken    string
-	AuthType     string
-	Backend      string
-	BasicAuth    bool
-	ClientCaKeys string
-	ClientCert   string
-	ClientKey    string
-	BackendNodes []string
-	Password     string
-	Scheme       string
-	Table        string
-	Separator    string
-	Username     string
-	AppID        string
-	UserID       string
-	RoleID       string
-	SecretID     string
-	YAMLFile     string
+	AuthToken    string     `toml:"auth_token"`
+	AuthType     string     `toml:"auth_type"`
+	Backend      string     `toml:"backend"`
+	BasicAuth    bool       `toml:"basic_auth"`
+	ClientCaKeys string     `toml:"client_cakeys"`
+	ClientCert   string     `toml:"client_cert"`
+	ClientKey    string     `toml:"client_key"`
+	BackendNodes util.Nodes `toml:"nodes"`
+	Password     string     `toml:"password"`
+	Scheme       string     `toml:"scheme"`
+	Table        string     `toml:"table"`
+	Separator    string     `toml:"separator"`
+	Username     string     `toml:"username"`
+	AppID        string     `toml:"app_id"`
+	UserID       string     `toml:"user_id"`
+	RoleID       string     `toml:"role_id"`
+	SecretID     string     `toml:"secret_id"`
+	YAMLFile     util.Nodes `toml:"file"`
+  	Filter       string     `toml:"filter"`
 	Role         string
 }

backends/file/client.go

@@ -3,67 +3,131 @@ package file
 import (
 	"fmt"
 	"io/ioutil"
+	"path"
+	"strconv"
 	"strings"
 
 	"github.com/fsnotify/fsnotify"
 	"github.com/kelseyhightower/confd/log"
+	util "github.com/kelseyhightower/confd/util"
 	"gopkg.in/yaml.v2"
 )
 
 var replacer = strings.NewReplacer("/", "_")
 
 // Client provides a shell for the yaml client
 type Client struct {
-	filepath string
+	filepath []string
+	filter   string
 }
 
-func NewFileClient(filepath string) (*Client, error) {
-	return &Client{filepath}, nil
+type ResultError struct {
+	response uint64
+	err      error
 }
 
-func (c *Client) GetValues(keys []string) (map[string]string, error) {
-	yamlMap := make(map[interface{}]interface{})
-	vars := make(map[string]string)
+func NewFileClient(filepath []string, filter string) (*Client, error) {
+	return &Client{filepath: filepath, filter: filter}, nil
+}
 
-	data, err := ioutil.ReadFile(c.filepath)
+func readFile(path string, vars map[string]string) error {
+	yamlMap := make(map[interface{}]interface{})
+	data, err := ioutil.ReadFile(path)
 	if err != nil {
-		return vars, err
+		return err
 	}
+
 	err = yaml.Unmarshal(data, &yamlMap)
 	if err != nil {
-		return vars, err
+		return err
 	}
 
-	nodeWalk(yamlMap, "", vars)
-	log.Debug(fmt.Sprintf("Key Map: %#v", vars))
+	err = nodeWalk(yamlMap, "/", vars)
+	if err != nil {
+		return err
+	}
+	return nil
+}
 
+func (c *Client) GetValues(keys []string) (map[string]string, error) {
+	vars := make(map[string]string)
+	var filePaths []string
+	for _, path := range c.filepath {
+		p, err := util.RecursiveFilesLookup(path, c.filter)
+		if err != nil {
+			return nil, err
+		}
+		filePaths = append(filePaths, p...)
+	}
+
+	for _, path := range filePaths {
+		err := readFile(path, vars)
+		if err != nil {
+			return nil, err
+		}
+	}
+
+VarsLoop:
+	for k, _ := range vars {
+		for _, key := range keys {
+			if strings.HasPrefix(k, key) {
+				continue VarsLoop
+			}
+		}
+		delete(vars, k)
+	}
+	log.Debug(fmt.Sprintf("Key Map: %#v", vars))
 	return vars, nil
 }
 
 // nodeWalk recursively descends nodes, updating vars.
-func nodeWalk(node map[interface{}]interface{}, key string, vars map[string]string) error {
-	for k, v := range node {
-		key := key + "/" + k.(string)
-
-		switch v.(type) {
-		case map[interface{}]interface{}:
-			nodeWalk(v.(map[interface{}]interface{}), key, vars)
-		case []interface{}:
-			for _, j := range v.([]interface{}) {
-				switch j.(type) {
-				case map[interface{}]interface{}:
-					nodeWalk(j.(map[interface{}]interface{}), key, vars)
-				case string:
-					vars[key+"/"+j.(string)] = ""
-				}
-			}
-		case string:
-			vars[key] = v.(string)
+func nodeWalk(node interface{}, key string, vars map[string]string) error {
+	switch node.(type) {
+	case []interface{}:
+		for i, j := range node.([]interface{}) {
+			key := path.Join(key, strconv.Itoa(i))
+			nodeWalk(j, key, vars)
 		}
+	case map[interface{}]interface{}:
+		for k, v := range node.(map[interface{}]interface{}) {
+			key := path.Join(key, k.(string))
+			nodeWalk(v, key, vars)
+		}
+	case string:
+		vars[key] = node.(string)
+	case int:
+		vars[key] = strconv.Itoa(node.(int))
+	case bool:
+		vars[key] = strconv.FormatBool(node.(bool))
+	case float64:
+		vars[key] = strconv.FormatFloat(node.(float64), 'f', -1, 64)
 	}
 	return nil
 }
 
+func (c *Client) watchChanges(watcher *fsnotify.Watcher, stopChan chan bool) ResultError {
+	outputChannel := make(chan ResultError)
+	defer close(outputChannel)
+	go func() error {
+		for {
+			select {
+			case event := <-watcher.Events:
+				log.Debug("event:", event)
+				if event.Op&fsnotify.Write == fsnotify.Write ||
+					event.Op&fsnotify.Remove == fsnotify.Remove ||
+					event.Op&fsnotify.Create == fsnotify.Create {
+					outputChannel <- ResultError{response: 1, err: nil}
+				}
+			case err := <-watcher.Errors:
+				outputChannel <- ResultError{response: 0, err: err}
+			case <-stopChan:
+				outputChannel <- ResultError{response: 1, err: nil}
+			}
+		}
+	}()
+	return <-outputChannel
+}
+
 func (c *Client) WatchPrefix(prefix string, keys []string, waitIndex uint64, stopChan chan bool) (uint64, error) {
 	if waitIndex == 0 {
 		return 1, nil
@@ -74,23 +138,32 @@ func (c *Client) WatchPrefix(prefix string, keys []string, waitIndex uint64, sto
 		return 0, err
 	}
 	defer watcher.Close()
-
-	err = watcher.Add(c.filepath)
-	if err != nil {
-		return 0, err
-	}
-
-	for {
-		select {
-		case event := <-watcher.Events:
-			if event.Op&fsnotify.Write == fsnotify.Write || event.Op&fsnotify.Remove == fsnotify.Remove {
-				return 1, nil
-			}
-		case err := <-watcher.Errors:
+	for _, path := range c.filepath {
+		isDir, err := util.IsDirectory(path)
+		if err != nil {
 			return 0, err
-		case <-stopChan:
-			return 0, nil
 		}
+		if isDir {
+			dirs, err := util.RecursiveDirsLookup(path, "*")
+			if err != nil {
+				return 0, err
+			}
+			for _, dir := range dirs {
+				err = watcher.Add(dir)
+				if err != nil {
+					return 0, err
+				}
+			}
+		} else {
+			err = watcher.Add(path)
+			if err != nil {
+				return 0, err
+			}
+		}
+	}
+	output := c.watchChanges(watcher, stopChan)
+	if output.response != 2 {
+		return output.response, output.err
 	}
 	return waitIndex, nil
 }

confd.go

@@ -15,7 +15,7 @@ import (
 
 func main() {
 	flag.Parse()
-	if printVersion {
+	if config.PrintVersion {
 		fmt.Printf("confd %s (Git SHA: %s, Go Version: %s)\n", Version, GitSHA, runtime.Version())
 		os.Exit(0)
 	}
@@ -25,14 +25,14 @@ func main() {
 
 	log.Info("Starting confd")
 
-	storeClient, err := backends.New(backendsConfig)
+	storeClient, err := backends.New(config.BackendsConfig)
 	if err != nil {
 		log.Fatal(err.Error())
 	}
 
-	templateConfig.StoreClient = storeClient
-	if onetime {
-		if err := template.Process(templateConfig); err != nil {
+	config.TemplateConfig.StoreClient = storeClient
+	if config.OneTime {
+		if err := template.Process(config.TemplateConfig); err != nil {
 			log.Fatal(err.Error())
 		}
 		os.Exit(0)
@@ -45,9 +45,9 @@ func main() {
 	var processor template.Processor
 	switch {
 	case config.Watch:
-		processor = template.WatchProcessor(templateConfig, stopChan, doneChan, errChan)
+		processor = template.WatchProcessor(config.TemplateConfig, stopChan, doneChan, errChan)
 	default:
-		processor = template.IntervalProcessor(templateConfig, stopChan, doneChan, errChan, config.Interval)
+		processor = template.IntervalProcessor(config.TemplateConfig, stopChan, doneChan, errChan, config.Interval)
 	}
 
 	go processor.Process()