diff --git a/diff/state.go b/diff/state.go index d9f7dc8..85559b8 100644 --- a/diff/state.go +++ b/diff/state.go @@ -5,6 +5,7 @@ import ( "errors" "fmt" "goposm/logging" + "goposm/parser/pbf" "io" "os" "path" @@ -35,7 +36,9 @@ func (d DiffState) WriteToFile(file string) error { lines := []string{} lines = append(lines, "timestamp="+d.Time.Format("2006-01-02T15\\:04\\:05Z")) - lines = append(lines, "sequenceNumber="+fmt.Sprintf("%d", d.Sequence)) + if d.Sequence != 0 { + lines = append(lines, "sequenceNumber="+fmt.Sprintf("%d", d.Sequence)) + } for _, line := range lines { _, err = writer.WriteString(line + "\n") @@ -72,6 +75,21 @@ func ParseStateFromOsc(oscFile string) (*DiffState, error) { return parseStateFile(stateFile) } +func StateFromPbf(pbfFile *pbf.Pbf) *DiffState { + var timestamp time.Time + if pbfFile.Header.Time.Unix() != 0 { + timestamp = pbfFile.Header.Time + } else { + fstat, err := os.Stat(pbfFile.Filename) + if err != nil { + log.Warn("unable to stat pbffile: ", err) + return nil + } + timestamp = fstat.ModTime() + } + return &DiffState{Time: timestamp} +} + func parseStateFile(stateFile string) (*DiffState, error) { f, err := os.Open(stateFile) if err != nil { @@ -139,7 +157,8 @@ func parseTimeStamp(value string) (time.Time, error) { func parseSequence(value string) (int32, error) { if value == "" { - return 0, errors.New("missing sqeuenceNumber in state") + log.Warn("missing sequenceNumber in state file") + return 0, nil } val, err := strconv.ParseInt(value, 10, 32) return int32(val), err diff --git a/goposm.go b/goposm.go index c2c299a..461ecf9 100644 --- a/goposm.go +++ b/goposm.go @@ -6,14 +6,17 @@ import ( "goposm/config" "goposm/database" _ "goposm/database/postgis" + diffstate "goposm/diff" "goposm/geom/clipper" "goposm/logging" "goposm/mapping" + "goposm/parser/pbf" "goposm/reader" "goposm/stats" "goposm/writer" golog "log" "os" + "path" "runtime" "runtime/pprof" "strings" @@ -164,12 +167,24 @@ func main() { die(err) } progress.Start() + + pbfFile, err := pbf.Open(*read) + if err != nil { + log.Fatal(err) + } + osmCache.Coords.SetLinearImport(true) - reader.ReadPbf(osmCache, progress, tagmapping, *read) + reader.ReadPbf(osmCache, progress, tagmapping, pbfFile) osmCache.Coords.SetLinearImport(false) progress.Stop() osmCache.Close() log.StopStep(step) + if *diff { + state := diffstate.StateFromPbf(pbfFile) + if state != nil { + state.WriteToFile(path.Join(conf.CacheDir, "last.state.txt")) + } + } } if *write { diff --git a/parser/pbf/lowlevel.go b/parser/pbf/lowlevel.go index 436377a..92df0af 100644 --- a/parser/pbf/lowlevel.go +++ b/parser/pbf/lowlevel.go @@ -101,9 +101,9 @@ func readAndParseHeaderBlock(pos Block) (*pbfHeader, error) { return result, nil } -type pbf struct { +type Pbf struct { file *os.File - filename string + Filename string offset int64 Header *pbfHeader } @@ -112,12 +112,12 @@ type pbfHeader struct { Time time.Time } -func Open(filename string) (f *pbf, err error) { +func Open(filename string) (f *Pbf, err error) { file, err := os.Open(filename) if err != nil { return nil, err } - f = &pbf{filename: filename, file: file} + f = &Pbf{Filename: filename, file: file} err = f.parseHeader() if err != nil { file.Close() @@ -126,21 +126,21 @@ func Open(filename string) (f *pbf, err error) { return f, nil } -func (pbf *pbf) Close() error { +func (pbf *Pbf) Close() error { return pbf.file.Close() } -func (pbf *pbf) parseHeader() error { +func (pbf *Pbf) parseHeader() error { offset, size, header := pbf.nextBlock() if header.GetType() != "OSMHeader" { panic("invalid block type, expected OSMHeader, got " + header.GetType()) } var err error - pbf.Header, err = readAndParseHeaderBlock(Block{pbf.filename, offset, size}) + pbf.Header, err = readAndParseHeaderBlock(Block{pbf.Filename, offset, size}) return err } -func (pbf *pbf) nextBlock() (offset int64, size int32, header *osmpbf.BlobHeader) { +func (pbf *Pbf) nextBlock() (offset int64, size int32, header *osmpbf.BlobHeader) { header = pbf.nextBlobHeader() size = header.GetDatasize() offset = pbf.offset @@ -150,7 +150,7 @@ func (pbf *pbf) nextBlock() (offset int64, size int32, header *osmpbf.BlobHeader return offset, size, header } -func (pbf *pbf) BlockPositions() (positions chan Block) { +func (pbf *Pbf) BlockPositions() (positions chan Block) { positions = make(chan Block, 8) go func() { for { @@ -163,19 +163,19 @@ func (pbf *pbf) BlockPositions() (positions chan Block) { if header.GetType() != "OSMData" { panic("invalid block type, expected OSMData, got " + header.GetType()) } - positions <- Block{pbf.filename, offset, size} + positions <- Block{pbf.Filename, offset, size} } }() return } -func (pbf *pbf) nextBlobHeaderSize() (size int32) { +func (pbf *Pbf) nextBlobHeaderSize() (size int32) { pbf.offset += 4 structs.Read(pbf.file, structs.BigEndian, &size) return } -func (pbf *pbf) nextBlobHeader() *osmpbf.BlobHeader { +func (pbf *Pbf) nextBlobHeader() *osmpbf.BlobHeader { var blobHeader = &osmpbf.BlobHeader{} size := pbf.nextBlobHeaderSize() diff --git a/reader/reader.go b/reader/reader.go index 0491a70..f7033cb 100644 --- a/reader/reader.go +++ b/reader/reader.go @@ -45,19 +45,14 @@ func init() { } -func ReadPbf(cache *cache.OSMCache, progress *stats.Statistics, tagmapping *mapping.Mapping, filename string) { +func ReadPbf(cache *cache.OSMCache, progress *stats.Statistics, tagmapping *mapping.Mapping, pbfFile *pbf.Pbf) { nodes := make(chan []element.Node, 4) coords := make(chan []element.Node, 4) ways := make(chan []element.Way, 4) relations := make(chan []element.Relation, 4) - pbfFile, err := pbf.Open(filename) - if err != nil { - log.Fatal(err) - } - if pbfFile.Header.Time.Unix() != 0 { - log.Printf("reading %s with data till %v", filename, pbfFile.Header.Time.Local()) + log.Printf("reading %s with data till %v", pbfFile.Filename, pbfFile.Header.Time.Local()) } blocks := pbfFile.BlockPositions()