read/store diff state

diff/cmd/process.go

@@ -34,17 +34,35 @@ func main() {
 		os.Exit(1)
 	}
 	for _, oscFile := range flag.Args() {
-		update(oscFile, conf)
+		update(oscFile, conf, false)
 	}
+	logging.Shutdown()
+	os.Exit(0)
 }
 
-func update(oscFile string, conf *config.Config) {
+func update(oscFile string, conf *config.Config, force bool) {
+	state, err := diff.ParseState(oscFile)
+	if err != nil {
+		log.Fatal(err)
+	}
+	lastState, err := diff.ParseLastState(conf.CacheDir)
+	if err != nil {
+		log.Fatal(err)
+	}
+
+	if lastState != nil && lastState.Sequence != 0 && state.Sequence <= lastState.Sequence {
+		if !force {
+			log.Warn(state, " already imported")
+			return
+		}
+	}
+
 	defer log.StopStep(log.StartStep(fmt.Sprintf("Processing %s", oscFile)))
 
 	elems, errc := parser.Parse(oscFile)
 
 	osmCache := cache.NewOSMCache(conf.CacheDir)
-	err := osmCache.Open()
+	err = osmCache.Open()
 	if err != nil {
 		log.Fatal("osm cache: ", err)
 	}
@@ -277,4 +295,9 @@ For:
 	)
 	log.StopStep(step)
 	progress.Stop()
+
+	err = diff.WriteLastState(conf.CacheDir, state)
+	if err != nil {
+		log.Warn(err) // warn only
+	}
 }

diff/deleter.go

@@ -8,7 +8,6 @@ import (
 	"goposm/expire"
 	"goposm/mapping"
 	"goposm/proj"
-	"log"
 )
 
 type Deleter struct {
@@ -48,7 +47,7 @@ func (d *Deleter) deleteRelation(id int64) {
 			return
 		}
 		// TODO
-		log.Println("rel", id, err)
+		log.Print("rel", id, err)
 		return
 	}
 	if elem.Tags == nil {
@@ -81,7 +80,7 @@ func (d *Deleter) deleteWay(id int64) {
 			return
 		}
 		// TODO
-		log.Println("way", id, err)
+		log.Print("way", id, err)
 		return
 	}
 	if elem.Tags == nil {
@@ -112,7 +111,7 @@ func (d *Deleter) deleteNode(id int64) {
 			return
 		}
 		// TODO
-		log.Println("node", id, err)
+		log.Print("node", id, err)
 		return
 	}
 	if elem.Tags == nil {

diff/state.go

@@ -0,0 +1,136 @@
+package diff
+
+import (
+	"bufio"
+	"errors"
+	"fmt"
+	"goposm/logging"
+	"os"
+	"path"
+	"strconv"
+	"strings"
+	"time"
+)
+
+var log = logging.NewLogger("diff")
+
+type DiffState struct {
+	Time     time.Time
+	Sequence int32
+}
+
+func (d DiffState) String() string {
+	return fmt.Sprintf("Diff #%d from %s", d.Sequence, d.Time.Local())
+}
+
+func (d DiffState) WriteToFile(file string) error {
+	f, err := os.Create(file)
+	if err != nil {
+		return err
+	}
+	defer f.Close()
+
+	writer := bufio.NewWriter(f)
+
+	lines := []string{}
+	lines = append(lines, "timestamp="+d.Time.Format("2006-01-02T15\\:04\\:05Z"))
+	lines = append(lines, "sequenceNumber="+fmt.Sprintf("%d", d.Sequence))
+
+	for _, line := range lines {
+		_, err = writer.WriteString(line + "\n")
+		if err != nil {
+			return err
+		}
+	}
+	return writer.Flush()
+}
+
+func WriteLastState(cacheDir string, state *DiffState) error {
+	stateFile := path.Join(cacheDir, "last.state.txt")
+	return state.WriteToFile(stateFile)
+}
+
+func ParseState(oscFile string) (*DiffState, error) {
+	var stateFile string
+	if !strings.HasSuffix(oscFile, ".osc.gz") {
+		log.Warn("cannot read state file for non .osc.gz files")
+		return nil, nil
+	}
+
+	stateFile = oscFile[:len(oscFile)-len(".osc.gz")] + ".state.txt"
+	if _, err := os.Stat(stateFile); os.IsNotExist(err) {
+		log.Warn("cannot find state file ", stateFile)
+		return nil, nil
+	}
+	return parseStateFile(stateFile)
+}
+
+func parseStateFile(stateFile string) (*DiffState, error) {
+	values, err := parseSimpleIni(stateFile)
+	if err != nil {
+		return nil, err
+	}
+
+	timestamp, err := parseTimeStamp(values["timestamp"])
+	if err != nil {
+		return nil, err
+	}
+	sequence, err := parseSequence(values["sequenceNumber"])
+	if err != nil {
+		return nil, err
+	}
+
+	return &DiffState{timestamp, sequence}, nil
+}
+
+func ParseLastState(cacheDir string) (*DiffState, error) {
+	stateFile := path.Join(cacheDir, "last.state.txt")
+	if _, err := os.Stat(stateFile); os.IsNotExist(err) {
+		log.Warn("cannot find state file ", stateFile)
+		return nil, nil
+	}
+	return parseStateFile(stateFile)
+}
+
+func parseSimpleIni(file string) (map[string]string, error) {
+	f, err := os.Open(file)
+	if err != nil {
+		return nil, err
+	}
+	defer f.Close()
+
+	result := make(map[string]string)
+
+	reader := bufio.NewScanner(f)
+	for reader.Scan() {
+		line := reader.Text()
+		if line != "" && line[0] == '#' {
+			continue
+		}
+		if strings.Contains(line, "=") {
+			keyVal := strings.SplitN(line, "=", 2)
+			result[strings.TrimSpace(keyVal[0])] = strings.TrimSpace(keyVal[1])
+		}
+
+	}
+	if err := reader.Err(); err != nil {
+		return nil, err
+	}
+
+	return result, nil
+}
+
+func parseTimeStamp(value string) (time.Time, error) {
+	if value == "" {
+		return time.Time{}, errors.New("missing timestamp in state")
+	}
+	return time.Parse("2006-01-02T15\\:04\\:05Z", value)
+}
+
+func parseSequence(value string) (int32, error) {
+	if value == "" {
+		return 0, errors.New("missing sqeuenceNumber in state")
+	}
+	val, err := strconv.ParseInt(value, 10, 32)
+	return int32(val), err
+}