use config in diff/cmd/process

config/config.go

@@ -28,7 +28,10 @@ var (
 )
 
 func Parse() (*Config, []error) {
-	config := &Config{}
+	config := &Config{
+		CacheDir: defaultCacheDir,
+		Srid:     defaultSrid,
+	}
 	if *configFile != "" {
 		f, err := os.Open(*configFile)
 		if err != nil {
@@ -75,6 +78,5 @@ func checkConfig(config *Config) []error {
 	if config.Connection == "" {
 		errs = append(errs, errors.New("missing connection"))
 	}
-
 	return errs
 }

diff/cmd/process.go

@@ -4,61 +4,70 @@ import (
 	"flag"
 	"fmt"
 	"goposm/cache"
+	"goposm/config"
 	"goposm/database"
 	_ "goposm/database/postgis"
 	"goposm/diff"
 	"goposm/diff/parser"
 	"goposm/element"
 	"goposm/geom/clipper"
+	"goposm/logging"
 	"goposm/mapping"
 	"goposm/stats"
 	"goposm/writer"
 	"io"
-	"log"
+	"os"
 )
 
-var (
+var log = logging.NewLogger("")
-	connection = flag.String("connection", "", "connection parameters")
-)
 
 func main() {
 	flag.Parse()
+	conf, errs := config.Parse()
+	if len(errs) > 0 {
+		log.Warn("errors in config/options:")
+		for _, err := range errs {
+			log.Warnf("\t%s", err)
+		}
+		logging.Shutdown()
+		os.Exit(1)
+	}
 	for _, oscFile := range flag.Args() {
-		update(oscFile)
+		update(oscFile, conf)
 	}
 }
 
-func update(oscFile string) {
+func update(oscFile string, conf *config.Config) {
-	flag.Parse()
+	defer log.StopStep(log.StartStep(fmt.Sprintf("Processing %s", oscFile)))
+
 	elems, errc := parser.Parse(oscFile)
 
-	osmCache := cache.NewOSMCache("/tmp/goposm")
+	osmCache := cache.NewOSMCache(conf.CacheDir)
 	err := osmCache.Open()
 	if err != nil {
-		log.Fatal(err)
+		log.Fatal("osm cache: ", err)
 	}
 
-	diffCache := cache.NewDiffCache("/tmp/goposm")
+	diffCache := cache.NewDiffCache(conf.CacheDir)
 	err = diffCache.Open()
 	if err != nil {
-		log.Fatal(err)
+		log.Fatal("diff cache: ", err)
 	}
 
-	tagmapping, err := mapping.NewMapping("./mapping.json")
+	tagmapping, err := mapping.NewMapping(conf.MappingFile)
 	if err != nil {
 		log.Fatal(err)
 	}
 
-	fmt.Println(*connection)
+	connType := database.ConnectionType(conf.Connection)
-	connType := database.ConnectionType(*connection)
+	dbConf := database.Config{
-	conf := database.Config{
 		Type:             connType,
-		ConnectionParams: *connection,
+		ConnectionParams: conf.Connection,
 		Srid:             3857,
 	}
-	db, err := database.Open(conf, tagmapping)
+	db, err := database.Open(dbConf, tagmapping)
 	if err != nil {
-		log.Fatal(err)
+		log.Fatal("database open: ", err)
 	}
 
 	err = db.Begin()
@@ -116,10 +125,26 @@ func update(oscFile string) {
 	wayIds := make(map[int64]bool)
 	relIds := make(map[int64]bool)
 
+	step := log.StartStep("Parsing changes, updating cache and removing elements")
+
+	progress.Start()
 For:
 	for {
 		select {
 		case elem := <-elems:
+			if elem.Rel != nil {
+				relTagFilter.Filter(&elem.Rel.Tags)
+				progress.AddRelations(1)
+			} else if elem.Way != nil {
+				wayTagFilter.Filter(&elem.Way.Tags)
+				progress.AddWays(1)
+			} else if elem.Node != nil {
+				nodeTagFilter.Filter(&elem.Node.Tags)
+				if len(elem.Node.Tags) > 0 {
+					progress.AddNodes(1)
+				}
+				progress.AddCoords(1)
+			}
 			if elem.Del {
 				deleter.Delete(elem)
 				if !elem.Add {
@@ -146,37 +171,37 @@ For:
 			if elem.Add {
 				if elem.Rel != nil {
 					// TODO: check for existence of first way member
-					relTagFilter.Filter(&elem.Rel.Tags)
 					osmCache.Relations.PutRelation(elem.Rel)
 					relIds[elem.Rel.Id] = true
 				} else if elem.Way != nil {
 					// TODO: check for existence of first ref
-					wayTagFilter.Filter(&elem.Way.Tags)
 					osmCache.Ways.PutWay(elem.Way)
 					wayIds[elem.Way.Id] = true
 				} else if elem.Node != nil {
 					// TODO: check for intersection with import BBOX/poly
-					nodeTagFilter.Filter(&elem.Node.Tags)
 					osmCache.Nodes.PutNode(elem.Node)
 					osmCache.Coords.PutCoords([]element.Node{*elem.Node})
 					nodeIds[elem.Node.Id] = true
 				}
 			}
 		case err := <-errc:
-			if err == io.EOF {
+			if err != io.EOF {
-				fmt.Println("done")
+				log.Fatal(err)
-			} else {
-				fmt.Println(err)
 			}
 			break For
 		}
 	}
+	progress.Stop()
+	log.StopStep(step)
+	step = log.StartStep("Writing added/modified elements")
+
+	progress.Start()
 
 	for nodeId, _ := range nodeIds {
 		node, err := osmCache.Nodes.GetNode(nodeId)
 		if err != nil {
 			if err != cache.NotFound {
-				log.Println(node, err)
+				log.Print(node, err)
 			}
 			// missing nodes can still be Coords
 			// no `continue` here
@@ -196,7 +221,7 @@ For:
 		way, err := osmCache.Ways.GetWay(wayId)
 		if err != nil {
 			if err != cache.NotFound {
-				log.Println(way, err)
+				log.Print(way, err)
 			}
 			continue
 		}
@@ -213,7 +238,7 @@ For:
 		rel, err := osmCache.Relations.GetRelation(relId)
 		if err != nil {
 			if err != cache.NotFound {
-				log.Println(rel, err)
+				log.Print(rel, err)
 			}
 			continue
 		}
@@ -241,4 +266,5 @@ For:
 	progress.Stop()
 	osmCache.Close()
 	diffCache.Close()
+	log.StopStep(step)
 }

goposm.go

@@ -44,14 +44,10 @@ var (
 
 func die(args ...interface{}) {
 	log.Fatal(args...)
-	logging.Shutdown()
-	os.Exit(1)
 }
 
 func dief(msg string, args ...interface{}) {
 	log.Fatalf(msg, args...)
-	logging.Shutdown()
-	os.Exit(1)
 }
 
 func main() {
@@ -62,9 +58,9 @@ func main() {
 	flag.Parse()
 	conf, errs := config.Parse()
 	if len(errs) > 0 {
-		log.Fatal("errors in config/options:")
+		log.Warn("errors in config/options:")
 		for _, err := range errs {
-			log.Fatalf("\t%s", err)
+			log.Warnf("\t%s", err)
 		}
 		logging.Shutdown()
 		os.Exit(1)

logging/logger.go

@@ -2,6 +2,7 @@ package logging
 
 import (
 	"fmt"
+	"os"
 	"sync"
 	"time"
 )
@@ -44,6 +45,8 @@ func Errorf(msg string, args ...interface{}) {
 
 func Fatalf(msg string, args ...interface{}) {
 	defaultLogBroker.Records <- Record{FATAL, "", fmt.Sprintf(msg, args...)}
+	Shutdown()
+	os.Exit(1)
 }
 
 func Progress(msg string) {
@@ -68,10 +71,14 @@ func (l *Logger) Printf(msg string, args ...interface{}) {
 
 func (l *Logger) Fatal(args ...interface{}) {
 	defaultLogBroker.Records <- Record{FATAL, l.Component, fmt.Sprint(args...)}
+	Shutdown()
+	os.Exit(1)
 }
 
 func (l *Logger) Fatalf(msg string, args ...interface{}) {
 	defaultLogBroker.Records <- Record{FATAL, l.Component, fmt.Sprintf(msg, args...)}
+	Shutdown()
+	os.Exit(1)
 }
 
 func (l *Logger) Errorf(msg string, args ...interface{}) {