From 3453b13af28f24365c7bd4251540d9c5b1feaa2d Mon Sep 17 00:00:00 2001 From: Oliver Tonnhofer Date: Wed, 23 Nov 2016 14:43:04 +0100 Subject: [PATCH] write expire tiles; add first tests --- config/config.go | 14 +++ diff/deleter.go | 9 +- diff/process.go | 14 ++- expire/expire.go | 15 ++-- expire/tilelist.go | 122 +++++++++++++++++++++++++ test/expire_tiles.osc | 111 +++++++++++++++++++++++ test/expire_tiles.osm | 127 ++++++++++++++++++++++++++ test/expire_tiles_mapping.yml | 45 ++++++++++ test/expire_tiles_test.go | 162 ++++++++++++++++++++++++++++++++++ test/helper_test.go | 6 +- writer/relations.go | 2 +- writer/ways.go | 4 +- 12 files changed, 618 insertions(+), 13 deletions(-) create mode 100644 expire/tilelist.go create mode 100644 test/expire_tiles.osc create mode 100644 test/expire_tiles.osm create mode 100644 test/expire_tiles_mapping.yml create mode 100644 test/expire_tiles_test.go diff --git a/config/config.go b/config/config.go index 9a47fc2..d5467af 100644 --- a/config/config.go +++ b/config/config.go @@ -19,6 +19,8 @@ type Config struct { LimitToCacheBuffer float64 `json:"limitto_cache_buffer"` Srid int `json:"srid"` Schemas Schemas `json:"schemas"` + ExpireTilesDir string `json:"expire_tiles_dir"` + ExpireTilesZoom int `json:"expire_tiles_zoom"` } type Schemas struct { @@ -48,6 +50,8 @@ type _BaseOptions struct { Httpprofile string Quiet bool Schemas Schemas + ExpireTilesDir string + ExpireTilesZoom int } func (o *_BaseOptions) updateFromConfig() error { @@ -104,6 +108,13 @@ func (o *_BaseOptions) updateFromConfig() error { if o.CacheDir == defaultCacheDir { o.CacheDir = conf.CacheDir } + if o.ExpireTilesDir == "" { + o.ExpireTilesDir = conf.ExpireTilesDir + } + if o.ExpireTilesZoom == 0 { + o.ExpireTilesZoom = conf.ExpireTilesZoom + } + if o.DiffDir == "" { if conf.DiffDir == "" { // use CacheDir for backwards compatibility @@ -186,6 +197,9 @@ func init() { ImportFlags.BoolVar(&ImportOptions.RevertDeploy, "revertdeploy", false, "revert deploy to production") ImportFlags.BoolVar(&ImportOptions.RemoveBackup, "removebackup", false, "remove backups from deploy") ImportFlags.DurationVar(&ImportOptions.DiffStateBefore, "diff-state-before", 2*time.Hour, "set initial diff sequence before") + + DiffFlags.StringVar(&BaseOptions.ExpireTilesDir, "expiretiles-dir", "", "write expire tiles into dir") + DiffFlags.IntVar(&BaseOptions.ExpireTilesZoom, "expiretiles-zoom", 14, "write expire tiles in this zoom level") } func ParseImport(args []string) { diff --git a/diff/deleter.go b/diff/deleter.go index 2005696..377a2da 100644 --- a/diff/deleter.go +++ b/diff/deleter.go @@ -125,6 +125,9 @@ func (d *Deleter) deleteRelation(id int64, deleteRefs bool, deleteMembers bool) return err } if d.expireor != nil { + if err := d.osmCache.Ways.FillMembers(elem.Members); err != nil { + return err + } for _, m := range elem.Members { if m.Way == nil { continue @@ -133,7 +136,7 @@ func (d *Deleter) deleteRelation(id int64, deleteRefs bool, deleteMembers bool) if err != nil { continue } - expire.ExpireNodes(d.expireor, m.Way.Nodes, 4326) + expire.ExpireProjectedNodes(d.expireor, m.Way.Nodes, 4326, true) } } return nil @@ -153,11 +156,13 @@ func (d *Deleter) deleteWay(id int64, deleteRefs bool) error { return nil } deleted := false + deletedPolygon := false if matches := d.tmPolygons.MatchWay(elem); len(matches) > 0 { if err := d.delDb.Delete(d.WayId(elem.Id), matches); err != nil { return err } deleted = true + deletedPolygon = true } if matches := d.tmLineStrings.MatchWay(elem); len(matches) > 0 { if err := d.delDb.Delete(d.WayId(elem.Id), matches); err != nil { @@ -177,7 +182,7 @@ func (d *Deleter) deleteWay(id int64, deleteRefs bool) error { if err != nil { return err } - expire.ExpireNodes(d.expireor, elem.Nodes, 4326) + expire.ExpireProjectedNodes(d.expireor, elem.Nodes, 4326, deletedPolygon) } return nil } diff --git a/diff/process.go b/diff/process.go index f995810..98390d1 100644 --- a/diff/process.go +++ b/diff/process.go @@ -56,8 +56,20 @@ func Diff() { log.Fatal("diff cache: ", err) } + var exp expire.Expireor + + if config.BaseOptions.ExpireTilesDir != "" { + tileexpire := expire.NewTileList(config.BaseOptions.ExpireTilesZoom, config.BaseOptions.ExpireTilesDir) + exp = tileexpire + defer func() { + if err := tileexpire.Flush(); err != nil { + log.Error("error while writing tile expire file:", err) + } + }() + } + for _, oscFile := range config.DiffFlags.Args() { - err := Update(oscFile, geometryLimiter, nil, osmCache, diffCache, false) + err := Update(oscFile, geometryLimiter, exp, osmCache, diffCache, false) if err != nil { osmCache.Close() diffCache.Close() diff --git a/expire/expire.go b/expire/expire.go index 2c305e6..8fedd0e 100644 --- a/expire/expire.go +++ b/expire/expire.go @@ -7,17 +7,18 @@ import ( type Expireor interface { Expire(long, lat float64) + ExpireNodes(nodes []element.Node, closed bool) } -func ExpireNodes(expireor Expireor, nodes []element.Node, srid int) { +func ExpireProjectedNodes(expireor Expireor, nodes []element.Node, srid int, closed bool) { if srid == 4326 { - for _, nd := range nodes { - expireor.Expire(nd.Long, nd.Lat) - } - } else if srid == 4326 { - for _, nd := range nodes { - expireor.Expire(proj.MercToWgs(nd.Long, nd.Lat)) + expireor.ExpireNodes(nodes, closed) + } else if srid == 3857 { + nds := make([]element.Node, len(nodes)) + for i, nd := range nodes { + nds[i].Long, nds[i].Lat = proj.MercToWgs(nd.Long, nd.Lat) } + expireor.ExpireNodes(nds, closed) } else { panic("unsupported srid") } diff --git a/expire/tilelist.go b/expire/tilelist.go new file mode 100644 index 0000000..33802dd --- /dev/null +++ b/expire/tilelist.go @@ -0,0 +1,122 @@ +package expire + +import ( + "fmt" + "github.com/omniscale/imposm3/element" + "github.com/omniscale/imposm3/proj" + "io" + "math" + "os" + "path/filepath" + "sync" + "time" +) + +var mercBbox = [4]float64{ + -20037508.342789244, + -20037508.342789244, + 20037508.342789244, + 20037508.342789244, +} + +var mercRes [20]float64 + +func init() { + res := 2 * 20037508.342789244 / 256 + + for i, _ := range mercRes { + mercRes[i] = res + res /= 2 + } +} + +func TileCoord(long, lat float64, zoom uint32) (uint32, uint32) { + x, y := proj.WgsToMerc(long, lat) + res := mercRes[zoom] + x = x - mercBbox[0] + y = mercBbox[3] - y + tileX := uint32(math.Floor(x / (res * 256))) + tileY := uint32(math.Floor(y / (res * 256))) + + return tileX, tileY +} + +type TileList struct { + mu sync.Mutex + tiles map[tileKey]struct{} + + zoom uint32 + out string +} + +type tileKey struct { + x uint32 + y uint32 +} + +type tile struct { + x uint32 + y uint32 + z uint32 +} + +func NewTileList(zoom int, out string) *TileList { + return &TileList{ + tiles: make(map[tileKey]struct{}), + zoom: uint32(zoom), + mu: sync.Mutex{}, + out: out, + } +} + +func (tl *TileList) addCoord(long, lat float64) { + tileX, tileY := TileCoord(long, lat, tl.zoom) + tl.mu.Lock() + tl.tiles[tileKey{tileX, tileY}] = struct{}{} + tl.mu.Unlock() +} + +func (tl *TileList) Expire(long, lat float64) { + tl.addCoord(long, lat) +} + +func (tl *TileList) ExpireNodes(nodes []element.Node, closed bool) { + for _, nd := range nodes { + tl.addCoord(nd.Long, nd.Lat) + } +} + +func (tl *TileList) writeTiles(w io.Writer) error { + for tileKey, _ := range tl.tiles { + _, err := fmt.Fprintf(w, "%d/%d/%d\n", tl.zoom, tileKey.x, tileKey.y) + if err != nil { + return err + } + } + return nil +} + +func (tl *TileList) Flush() error { + if len(tl.tiles) == 0 { + return nil + } + + now := time.Now().UTC() + dir := filepath.Join(tl.out, now.Format("20060102")) + err := os.MkdirAll(dir, 0755) + if err != nil { + return err + } + fileName := filepath.Join(dir, now.Format("150405.000")+".tiles~") + f, err := os.Create(fileName) + if err != nil { + return err + } + err = tl.writeTiles(f) + f.Close() + if err != nil { + return err + } + // wrote to .tiles~ and now atomically move file to .tiles + return os.Rename(fileName, fileName[0:len(fileName)-1]) +} diff --git a/test/expire_tiles.osc b/test/expire_tiles.osc new file mode 100644 index 0000000..716d9eb --- /dev/null +++ b/test/expire_tiles.osc @@ -0,0 +1,111 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/test/expire_tiles.osm b/test/expire_tiles.osm new file mode 100644 index 0000000..74aaf25 --- /dev/null +++ b/test/expire_tiles.osm @@ -0,0 +1,127 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/test/expire_tiles_mapping.yml b/test/expire_tiles_mapping.yml new file mode 100644 index 0000000..94f0cc1 --- /dev/null +++ b/test/expire_tiles_mapping.yml @@ -0,0 +1,45 @@ +tables: + roads: + type: linestring + fields: + - name: osm_id + type: id + - name: type + type: mapping_value + - name: name + type: string + key: name + - name: geometry + type: geometry + mapping: + highway: [__any__] + + pois: + type: point + fields: + - name: osm_id + type: id + - name: type + type: mapping_value + - name: name + type: string + key: name + - name: geometry + type: geometry + mapping: + amenity: [__any__] + + buildings: + type: polygon + fields: + - name: osm_id + type: id + - name: type + type: mapping_value + - name: name + type: string + key: name + - name: geometry + type: geometry + mapping: + building: [__any__] diff --git a/test/expire_tiles_test.go b/test/expire_tiles_test.go new file mode 100644 index 0000000..a27a792 --- /dev/null +++ b/test/expire_tiles_test.go @@ -0,0 +1,162 @@ +package test + +import ( + "bufio" + "database/sql" + "github.com/omniscale/imposm3/expire" + "io/ioutil" + "os" + "path/filepath" + "strconv" + "strings" + + "testing" + + "github.com/omniscale/imposm3/geom/geos" +) + +func TestExpireTiles_Prepare(t *testing.T) { + var err error + + ts.dir, err = ioutil.TempDir("", "imposm3test") + if err != nil { + t.Fatal(err) + } + ts.config = importConfig{ + connection: "postgis://", + cacheDir: ts.dir, + osmFileName: "build/expire_tiles.pbf", + mappingFileName: "expire_tiles_mapping.yml", + expireTileDir: filepath.Join(ts.dir, "expiretiles"), + } + ts.g = geos.NewGeos() + + ts.db, err = sql.Open("postgres", "sslmode=disable") + if err != nil { + t.Fatal(err) + } + ts.dropSchemas() +} + +func TestExpireTiles_Import(t *testing.T) { + if ts.tableExists(t, dbschemaImport, "osm_roads") != false { + t.Fatalf("table osm_roads exists in schema %s", dbschemaImport) + } + ts.importOsm(t) + ts.deployOsm(t) + if ts.tableExists(t, dbschemaProduction, "osm_roads") != true { + t.Fatalf("table osm_roads does not exists in schema %s", dbschemaProduction) + } +} + +func TestExpireTiles_Elements(t *testing.T) { + assertRecords(t, []checkElem{ + {"osm_roads", 20151, "motorway", nil}, + {"osm_roads", 20251, "motorway", nil}, + {"osm_roads", 20351, "motorway", nil}, + + {"osm_buildings", -30191, "yes", nil}, + {"osm_buildings", -30291, "yes", nil}, + {"osm_buildings", -30391, "yes", nil}, + {"osm_buildings", -30491, "yes", nil}, + }) +} + +func TestExpireTiles_Update(t *testing.T) { + ts.updateOsm(t, "build/expire_tiles.osc.gz") +} + +func TestExpireTiles_CheckExpireFile(t *testing.T) { + files, err := filepath.Glob(filepath.Join(ts.config.expireTileDir, "*", "*.tiles")) + if err != nil { + t.Fatal(err) + } + if len(files) != 1 { + t.Fatalf("expected one expire tile file, got: %v", files) + } + tiles, err := parseTileList(files[0]) + if err != nil { + t.Error(err) + } + + for _, test := range []struct { + reason string + long float64 + lat float64 + expire bool + }{ + {"create node", 3, 1, true}, + {"modify node (old)", 1, 1, true}, + {"modify node (new)", 1, -1, true}, + {"modify node to unmapped (old)", 4, 1, true}, + {"modify node to unmapped (new)", 4, -1, false}, + {"delete node", 2, 1, true}, + + {"delete way", 2.0001, 2, true}, + {"modify way", 1.0001, 2, true}, + {"modify way from node (old)", 3.0001, 2, true}, + {"modify way from node (new)", 3.0001, -2, true}, + {"create way", 4.0001, 2, true}, + + {"create long way (start)", 5.00, 2, true}, + {"create long way (mid)", 5.025, 2, false}, // TODO not implemented + {"create long way (end)", 5.05, 2, true}, + + {"modify relation", 1.0001, 3, true}, + {"delete relation", 2.0001, 3, true}, + {"modify relation from way", 3.0001, 3, true}, + {"modify relation from nodes (old)", 4.0001, 3, true}, + {"modify relation from nodes (new)", 4.0001, -3, true}, + } { + x, y := expire.TileCoord(test.long, test.lat, 14) + if test.expire { + if _, ok := tiles[tile{x: int(x), y: int(y), z: 14}]; !ok { + t.Errorf("missing expire tile for %s 14/%d/%d for %f %f", test.reason, x, y, test.long, test.lat) + } else { + delete(tiles, tile{x: int(x), y: int(y), z: 14}) + } + } else { + if _, ok := tiles[tile{x: int(x), y: int(y), z: 14}]; ok { + t.Errorf("found expire tile for %s 14/%d/%d for %f %f", test.reason, x, y, test.long, test.lat) + } + } + } + + for tile, _ := range tiles { + t.Errorf("unexpected tile expired: %v", tile) + } +} + +func TestExpireTiles_Cleanup(t *testing.T) { + ts.dropSchemas() + if err := os.RemoveAll(ts.dir); err != nil { + t.Error(err) + } +} + +type tile struct { + x, y, z int +} + +func parseTileList(filename string) (map[tile]struct{}, error) { + file, err := os.Open(filename) + if err != nil { + return nil, err + } + defer file.Close() + + scanner := bufio.NewScanner(file) + tiles := make(map[tile]struct{}) + for scanner.Scan() { + parts := strings.Split(scanner.Text(), "/") + z, _ := strconv.ParseInt(parts[0], 10, 32) + x, _ := strconv.ParseInt(parts[1], 10, 32) + y, _ := strconv.ParseInt(parts[2], 10, 32) + tiles[tile{x: int(x), y: int(y), z: int(z)}] = struct{}{} + } + + if err := scanner.Err(); err != nil { + return nil, err + } + return tiles, nil +} diff --git a/test/helper_test.go b/test/helper_test.go index ce1c7d7..507aac0 100644 --- a/test/helper_test.go +++ b/test/helper_test.go @@ -33,6 +33,7 @@ type importConfig struct { mappingFileName string cacheDir string verbose bool + expireTileDir string } type importTestSuite struct { @@ -152,8 +153,11 @@ func (s *importTestSuite) updateOsm(t *testing.T, diffFile string) { "-limitto", "clipping.geojson", "-dbschema-production", dbschemaProduction, "-mapping", s.config.mappingFileName, - diffFile, } + if s.config.expireTileDir != "" { + args = append(args, "-expiretiles-dir", s.config.expireTileDir) + } + args = append(args, diffFile) config.ParseDiffImport(args) diff.Diff() } diff --git a/writer/relations.go b/writer/relations.go index c136e69..3af283a 100644 --- a/writer/relations.go +++ b/writer/relations.go @@ -125,7 +125,7 @@ NextRel: if inserted && rw.expireor != nil { for _, m := range allMembers { if m.Way != nil { - expire.ExpireNodes(rw.expireor, m.Way.Nodes, rw.srid) + expire.ExpireProjectedNodes(rw.expireor, m.Way.Nodes, rw.srid, true) } } } diff --git a/writer/ways.go b/writer/ways.go index 559d453..1f770fa 100644 --- a/writer/ways.go +++ b/writer/ways.go @@ -87,6 +87,7 @@ func (ww *WayWriter) loop() { w.Id = ww.wayId(w.Id) inserted := false + insertedPolygon := false if matches := ww.lineMatcher.MatchWay(w); len(matches) > 0 { err := ww.buildAndInsert(geos, w, matches, false) if err != nil { @@ -108,11 +109,12 @@ func (ww *WayWriter) loop() { continue } inserted = true + insertedPolygon = true } } if inserted && ww.expireor != nil { - expire.ExpireNodes(ww.expireor, w.Nodes, ww.srid) + expire.ExpireProjectedNodes(ww.expireor, w.Nodes, ww.srid, insertedPolygon) } if ww.diffCache != nil { ww.diffCache.Coords.AddFromWay(w)