diff --git a/config/config.go b/config/config.go
index 9a47fc2..d5467af 100644
--- a/config/config.go
+++ b/config/config.go
@@ -19,6 +19,8 @@ type Config struct {
LimitToCacheBuffer float64 `json:"limitto_cache_buffer"`
Srid int `json:"srid"`
Schemas Schemas `json:"schemas"`
+ ExpireTilesDir string `json:"expire_tiles_dir"`
+ ExpireTilesZoom int `json:"expire_tiles_zoom"`
}
type Schemas struct {
@@ -48,6 +50,8 @@ type _BaseOptions struct {
Httpprofile string
Quiet bool
Schemas Schemas
+ ExpireTilesDir string
+ ExpireTilesZoom int
}
func (o *_BaseOptions) updateFromConfig() error {
@@ -104,6 +108,13 @@ func (o *_BaseOptions) updateFromConfig() error {
if o.CacheDir == defaultCacheDir {
o.CacheDir = conf.CacheDir
}
+ if o.ExpireTilesDir == "" {
+ o.ExpireTilesDir = conf.ExpireTilesDir
+ }
+ if o.ExpireTilesZoom == 0 {
+ o.ExpireTilesZoom = conf.ExpireTilesZoom
+ }
+
if o.DiffDir == "" {
if conf.DiffDir == "" {
// use CacheDir for backwards compatibility
@@ -186,6 +197,9 @@ func init() {
ImportFlags.BoolVar(&ImportOptions.RevertDeploy, "revertdeploy", false, "revert deploy to production")
ImportFlags.BoolVar(&ImportOptions.RemoveBackup, "removebackup", false, "remove backups from deploy")
ImportFlags.DurationVar(&ImportOptions.DiffStateBefore, "diff-state-before", 2*time.Hour, "set initial diff sequence before")
+
+ DiffFlags.StringVar(&BaseOptions.ExpireTilesDir, "expiretiles-dir", "", "write expire tiles into dir")
+ DiffFlags.IntVar(&BaseOptions.ExpireTilesZoom, "expiretiles-zoom", 14, "write expire tiles in this zoom level")
}
func ParseImport(args []string) {
diff --git a/diff/deleter.go b/diff/deleter.go
index 2005696..377a2da 100644
--- a/diff/deleter.go
+++ b/diff/deleter.go
@@ -125,6 +125,9 @@ func (d *Deleter) deleteRelation(id int64, deleteRefs bool, deleteMembers bool)
return err
}
if d.expireor != nil {
+ if err := d.osmCache.Ways.FillMembers(elem.Members); err != nil {
+ return err
+ }
for _, m := range elem.Members {
if m.Way == nil {
continue
@@ -133,7 +136,7 @@ func (d *Deleter) deleteRelation(id int64, deleteRefs bool, deleteMembers bool)
if err != nil {
continue
}
- expire.ExpireNodes(d.expireor, m.Way.Nodes, 4326)
+ expire.ExpireProjectedNodes(d.expireor, m.Way.Nodes, 4326, true)
}
}
return nil
@@ -153,11 +156,13 @@ func (d *Deleter) deleteWay(id int64, deleteRefs bool) error {
return nil
}
deleted := false
+ deletedPolygon := false
if matches := d.tmPolygons.MatchWay(elem); len(matches) > 0 {
if err := d.delDb.Delete(d.WayId(elem.Id), matches); err != nil {
return err
}
deleted = true
+ deletedPolygon = true
}
if matches := d.tmLineStrings.MatchWay(elem); len(matches) > 0 {
if err := d.delDb.Delete(d.WayId(elem.Id), matches); err != nil {
@@ -177,7 +182,7 @@ func (d *Deleter) deleteWay(id int64, deleteRefs bool) error {
if err != nil {
return err
}
- expire.ExpireNodes(d.expireor, elem.Nodes, 4326)
+ expire.ExpireProjectedNodes(d.expireor, elem.Nodes, 4326, deletedPolygon)
}
return nil
}
diff --git a/diff/process.go b/diff/process.go
index f995810..98390d1 100644
--- a/diff/process.go
+++ b/diff/process.go
@@ -56,8 +56,20 @@ func Diff() {
log.Fatal("diff cache: ", err)
}
+ var exp expire.Expireor
+
+ if config.BaseOptions.ExpireTilesDir != "" {
+ tileexpire := expire.NewTileList(config.BaseOptions.ExpireTilesZoom, config.BaseOptions.ExpireTilesDir)
+ exp = tileexpire
+ defer func() {
+ if err := tileexpire.Flush(); err != nil {
+ log.Error("error while writing tile expire file:", err)
+ }
+ }()
+ }
+
for _, oscFile := range config.DiffFlags.Args() {
- err := Update(oscFile, geometryLimiter, nil, osmCache, diffCache, false)
+ err := Update(oscFile, geometryLimiter, exp, osmCache, diffCache, false)
if err != nil {
osmCache.Close()
diffCache.Close()
diff --git a/expire/expire.go b/expire/expire.go
index 2c305e6..8fedd0e 100644
--- a/expire/expire.go
+++ b/expire/expire.go
@@ -7,17 +7,18 @@ import (
type Expireor interface {
Expire(long, lat float64)
+ ExpireNodes(nodes []element.Node, closed bool)
}
-func ExpireNodes(expireor Expireor, nodes []element.Node, srid int) {
+func ExpireProjectedNodes(expireor Expireor, nodes []element.Node, srid int, closed bool) {
if srid == 4326 {
- for _, nd := range nodes {
- expireor.Expire(nd.Long, nd.Lat)
- }
- } else if srid == 4326 {
- for _, nd := range nodes {
- expireor.Expire(proj.MercToWgs(nd.Long, nd.Lat))
+ expireor.ExpireNodes(nodes, closed)
+ } else if srid == 3857 {
+ nds := make([]element.Node, len(nodes))
+ for i, nd := range nodes {
+ nds[i].Long, nds[i].Lat = proj.MercToWgs(nd.Long, nd.Lat)
}
+ expireor.ExpireNodes(nds, closed)
} else {
panic("unsupported srid")
}
diff --git a/expire/tilelist.go b/expire/tilelist.go
new file mode 100644
index 0000000..33802dd
--- /dev/null
+++ b/expire/tilelist.go
@@ -0,0 +1,122 @@
+package expire
+
+import (
+ "fmt"
+ "github.com/omniscale/imposm3/element"
+ "github.com/omniscale/imposm3/proj"
+ "io"
+ "math"
+ "os"
+ "path/filepath"
+ "sync"
+ "time"
+)
+
+var mercBbox = [4]float64{
+ -20037508.342789244,
+ -20037508.342789244,
+ 20037508.342789244,
+ 20037508.342789244,
+}
+
+var mercRes [20]float64
+
+func init() {
+ res := 2 * 20037508.342789244 / 256
+
+ for i, _ := range mercRes {
+ mercRes[i] = res
+ res /= 2
+ }
+}
+
+func TileCoord(long, lat float64, zoom uint32) (uint32, uint32) {
+ x, y := proj.WgsToMerc(long, lat)
+ res := mercRes[zoom]
+ x = x - mercBbox[0]
+ y = mercBbox[3] - y
+ tileX := uint32(math.Floor(x / (res * 256)))
+ tileY := uint32(math.Floor(y / (res * 256)))
+
+ return tileX, tileY
+}
+
+type TileList struct {
+ mu sync.Mutex
+ tiles map[tileKey]struct{}
+
+ zoom uint32
+ out string
+}
+
+type tileKey struct {
+ x uint32
+ y uint32
+}
+
+type tile struct {
+ x uint32
+ y uint32
+ z uint32
+}
+
+func NewTileList(zoom int, out string) *TileList {
+ return &TileList{
+ tiles: make(map[tileKey]struct{}),
+ zoom: uint32(zoom),
+ mu: sync.Mutex{},
+ out: out,
+ }
+}
+
+func (tl *TileList) addCoord(long, lat float64) {
+ tileX, tileY := TileCoord(long, lat, tl.zoom)
+ tl.mu.Lock()
+ tl.tiles[tileKey{tileX, tileY}] = struct{}{}
+ tl.mu.Unlock()
+}
+
+func (tl *TileList) Expire(long, lat float64) {
+ tl.addCoord(long, lat)
+}
+
+func (tl *TileList) ExpireNodes(nodes []element.Node, closed bool) {
+ for _, nd := range nodes {
+ tl.addCoord(nd.Long, nd.Lat)
+ }
+}
+
+func (tl *TileList) writeTiles(w io.Writer) error {
+ for tileKey, _ := range tl.tiles {
+ _, err := fmt.Fprintf(w, "%d/%d/%d\n", tl.zoom, tileKey.x, tileKey.y)
+ if err != nil {
+ return err
+ }
+ }
+ return nil
+}
+
+func (tl *TileList) Flush() error {
+ if len(tl.tiles) == 0 {
+ return nil
+ }
+
+ now := time.Now().UTC()
+ dir := filepath.Join(tl.out, now.Format("20060102"))
+ err := os.MkdirAll(dir, 0755)
+ if err != nil {
+ return err
+ }
+ fileName := filepath.Join(dir, now.Format("150405.000")+".tiles~")
+ f, err := os.Create(fileName)
+ if err != nil {
+ return err
+ }
+ err = tl.writeTiles(f)
+ f.Close()
+ if err != nil {
+ return err
+ }
+ // wrote to .tiles~ and now atomically move file to .tiles
+ return os.Rename(fileName, fileName[0:len(fileName)-1])
+}
diff --git a/test/expire_tiles.osc b/test/expire_tiles.osc
new file mode 100644
index 0000000..716d9eb
--- /dev/null
+++ b/test/expire_tiles.osc
@@ -0,0 +1,111 @@
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
diff --git a/test/expire_tiles.osm b/test/expire_tiles.osm
new file mode 100644
index 0000000..74aaf25
--- /dev/null
+++ b/test/expire_tiles.osm
@@ -0,0 +1,127 @@
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
diff --git a/test/expire_tiles_mapping.yml b/test/expire_tiles_mapping.yml
new file mode 100644
index 0000000..94f0cc1
--- /dev/null
+++ b/test/expire_tiles_mapping.yml
@@ -0,0 +1,45 @@
+tables:
+ roads:
+ type: linestring
+ fields:
+ - name: osm_id
+ type: id
+ - name: type
+ type: mapping_value
+ - name: name
+ type: string
+ key: name
+ - name: geometry
+ type: geometry
+ mapping:
+ highway: [__any__]
+
+ pois:
+ type: point
+ fields:
+ - name: osm_id
+ type: id
+ - name: type
+ type: mapping_value
+ - name: name
+ type: string
+ key: name
+ - name: geometry
+ type: geometry
+ mapping:
+ amenity: [__any__]
+
+ buildings:
+ type: polygon
+ fields:
+ - name: osm_id
+ type: id
+ - name: type
+ type: mapping_value
+ - name: name
+ type: string
+ key: name
+ - name: geometry
+ type: geometry
+ mapping:
+ building: [__any__]
diff --git a/test/expire_tiles_test.go b/test/expire_tiles_test.go
new file mode 100644
index 0000000..a27a792
--- /dev/null
+++ b/test/expire_tiles_test.go
@@ -0,0 +1,162 @@
+package test
+
+import (
+ "bufio"
+ "database/sql"
+ "github.com/omniscale/imposm3/expire"
+ "io/ioutil"
+ "os"
+ "path/filepath"
+ "strconv"
+ "strings"
+
+ "testing"
+
+ "github.com/omniscale/imposm3/geom/geos"
+)
+
+func TestExpireTiles_Prepare(t *testing.T) {
+ var err error
+
+ ts.dir, err = ioutil.TempDir("", "imposm3test")
+ if err != nil {
+ t.Fatal(err)
+ }
+ ts.config = importConfig{
+ connection: "postgis://",
+ cacheDir: ts.dir,
+ osmFileName: "build/expire_tiles.pbf",
+ mappingFileName: "expire_tiles_mapping.yml",
+ expireTileDir: filepath.Join(ts.dir, "expiretiles"),
+ }
+ ts.g = geos.NewGeos()
+
+ ts.db, err = sql.Open("postgres", "sslmode=disable")
+ if err != nil {
+ t.Fatal(err)
+ }
+ ts.dropSchemas()
+}
+
+func TestExpireTiles_Import(t *testing.T) {
+ if ts.tableExists(t, dbschemaImport, "osm_roads") != false {
+ t.Fatalf("table osm_roads exists in schema %s", dbschemaImport)
+ }
+ ts.importOsm(t)
+ ts.deployOsm(t)
+ if ts.tableExists(t, dbschemaProduction, "osm_roads") != true {
+ t.Fatalf("table osm_roads does not exists in schema %s", dbschemaProduction)
+ }
+}
+
+func TestExpireTiles_Elements(t *testing.T) {
+ assertRecords(t, []checkElem{
+ {"osm_roads", 20151, "motorway", nil},
+ {"osm_roads", 20251, "motorway", nil},
+ {"osm_roads", 20351, "motorway", nil},
+
+ {"osm_buildings", -30191, "yes", nil},
+ {"osm_buildings", -30291, "yes", nil},
+ {"osm_buildings", -30391, "yes", nil},
+ {"osm_buildings", -30491, "yes", nil},
+ })
+}
+
+func TestExpireTiles_Update(t *testing.T) {
+ ts.updateOsm(t, "build/expire_tiles.osc.gz")
+}
+
+func TestExpireTiles_CheckExpireFile(t *testing.T) {
+ files, err := filepath.Glob(filepath.Join(ts.config.expireTileDir, "*", "*.tiles"))
+ if err != nil {
+ t.Fatal(err)
+ }
+ if len(files) != 1 {
+ t.Fatalf("expected one expire tile file, got: %v", files)
+ }
+ tiles, err := parseTileList(files[0])
+ if err != nil {
+ t.Error(err)
+ }
+
+ for _, test := range []struct {
+ reason string
+ long float64
+ lat float64
+ expire bool
+ }{
+ {"create node", 3, 1, true},
+ {"modify node (old)", 1, 1, true},
+ {"modify node (new)", 1, -1, true},
+ {"modify node to unmapped (old)", 4, 1, true},
+ {"modify node to unmapped (new)", 4, -1, false},
+ {"delete node", 2, 1, true},
+
+ {"delete way", 2.0001, 2, true},
+ {"modify way", 1.0001, 2, true},
+ {"modify way from node (old)", 3.0001, 2, true},
+ {"modify way from node (new)", 3.0001, -2, true},
+ {"create way", 4.0001, 2, true},
+
+ {"create long way (start)", 5.00, 2, true},
+ {"create long way (mid)", 5.025, 2, false}, // TODO not implemented
+ {"create long way (end)", 5.05, 2, true},
+
+ {"modify relation", 1.0001, 3, true},
+ {"delete relation", 2.0001, 3, true},
+ {"modify relation from way", 3.0001, 3, true},
+ {"modify relation from nodes (old)", 4.0001, 3, true},
+ {"modify relation from nodes (new)", 4.0001, -3, true},
+ } {
+ x, y := expire.TileCoord(test.long, test.lat, 14)
+ if test.expire {
+ if _, ok := tiles[tile{x: int(x), y: int(y), z: 14}]; !ok {
+ t.Errorf("missing expire tile for %s 14/%d/%d for %f %f", test.reason, x, y, test.long, test.lat)
+ } else {
+ delete(tiles, tile{x: int(x), y: int(y), z: 14})
+ }
+ } else {
+ if _, ok := tiles[tile{x: int(x), y: int(y), z: 14}]; ok {
+ t.Errorf("found expire tile for %s 14/%d/%d for %f %f", test.reason, x, y, test.long, test.lat)
+ }
+ }
+ }
+
+ for tile, _ := range tiles {
+ t.Errorf("unexpected tile expired: %v", tile)
+ }
+}
+
+func TestExpireTiles_Cleanup(t *testing.T) {
+ ts.dropSchemas()
+ if err := os.RemoveAll(ts.dir); err != nil {
+ t.Error(err)
+ }
+}
+
+type tile struct {
+ x, y, z int
+}
+
+func parseTileList(filename string) (map[tile]struct{}, error) {
+ file, err := os.Open(filename)
+ if err != nil {
+ return nil, err
+ }
+ defer file.Close()
+
+ scanner := bufio.NewScanner(file)
+ tiles := make(map[tile]struct{})
+ for scanner.Scan() {
+ parts := strings.Split(scanner.Text(), "/")
+ z, _ := strconv.ParseInt(parts[0], 10, 32)
+ x, _ := strconv.ParseInt(parts[1], 10, 32)
+ y, _ := strconv.ParseInt(parts[2], 10, 32)
+ tiles[tile{x: int(x), y: int(y), z: int(z)}] = struct{}{}
+ }
+
+ if err := scanner.Err(); err != nil {
+ return nil, err
+ }
+ return tiles, nil
+}
diff --git a/test/helper_test.go b/test/helper_test.go
index ce1c7d7..507aac0 100644
--- a/test/helper_test.go
+++ b/test/helper_test.go
@@ -33,6 +33,7 @@ type importConfig struct {
mappingFileName string
cacheDir string
verbose bool
+ expireTileDir string
}
type importTestSuite struct {
@@ -152,8 +153,11 @@ func (s *importTestSuite) updateOsm(t *testing.T, diffFile string) {
"-limitto", "clipping.geojson",
"-dbschema-production", dbschemaProduction,
"-mapping", s.config.mappingFileName,
- diffFile,
}
+ if s.config.expireTileDir != "" {
+ args = append(args, "-expiretiles-dir", s.config.expireTileDir)
+ }
+ args = append(args, diffFile)
config.ParseDiffImport(args)
diff.Diff()
}
diff --git a/writer/relations.go b/writer/relations.go
index c136e69..3af283a 100644
--- a/writer/relations.go
+++ b/writer/relations.go
@@ -125,7 +125,7 @@ NextRel:
if inserted && rw.expireor != nil {
for _, m := range allMembers {
if m.Way != nil {
- expire.ExpireNodes(rw.expireor, m.Way.Nodes, rw.srid)
+ expire.ExpireProjectedNodes(rw.expireor, m.Way.Nodes, rw.srid, true)
}
}
}
diff --git a/writer/ways.go b/writer/ways.go
index 559d453..1f770fa 100644
--- a/writer/ways.go
+++ b/writer/ways.go
@@ -87,6 +87,7 @@ func (ww *WayWriter) loop() {
w.Id = ww.wayId(w.Id)
inserted := false
+ insertedPolygon := false
if matches := ww.lineMatcher.MatchWay(w); len(matches) > 0 {
err := ww.buildAndInsert(geos, w, matches, false)
if err != nil {
@@ -108,11 +109,12 @@ func (ww *WayWriter) loop() {
continue
}
inserted = true
+ insertedPolygon = true
}
}
if inserted && ww.expireor != nil {
- expire.ExpireNodes(ww.expireor, w.Nodes, ww.srid)
+ expire.ExpireProjectedNodes(ww.expireor, w.Nodes, ww.srid, insertedPolygon)
}
if ww.diffCache != nil {
ww.diffCache.Coords.AddFromWay(w)