add new Expireor interface

diff/deleter.go

@@ -17,7 +17,7 @@ type Deleter struct {
 	tmPoints         *mapping.TagMatcher
 	tmLineStrings    *mapping.TagMatcher
 	tmPolygons       *mapping.TagMatcher
-	expireTiles      *expire.Tiles
+	expireor         expire.Expireor
 	deletedRelations map[int64]struct{}
 	deletedWays      map[int64]struct{}
 	deletedMembers   map[int64]struct{}
@@ -42,8 +42,8 @@ func NewDeleter(db database.Deleter, osmCache *cache.OSMCache, diffCache *cache.
 	}
 }
 
-func (d *Deleter) SetExpireTiles(expireTiles *expire.Tiles) {
-	d.expireTiles = expireTiles
+func (d *Deleter) SetExpireor(exp expire.Expireor) {
+	d.expireor = exp
 }
 
 func (d *Deleter) DeletedMemberWays() map[int64]struct{} {
@@ -98,7 +98,7 @@ func (d *Deleter) deleteRelation(id int64, deleteRefs bool, deleteMembers bool)
 	}
 
 	d.osmCache.InsertedWays.DeleteMembers(elem.Members)
-	if d.expireTiles != nil {
+	if d.expireor != nil {
 		for _, m := range elem.Members {
 			if m.Way == nil {
 				continue
@@ -108,7 +108,7 @@ func (d *Deleter) deleteRelation(id int64, deleteRefs bool, deleteMembers bool)
 				continue
 			}
 			proj.NodesToMerc(m.Way.Nodes)
-			d.expireTiles.ExpireFromNodes(m.Way.Nodes)
+			expire.ExpireNodes(d.expireor, m.Way.Nodes)
 		}
 	}
 }
@@ -142,12 +142,12 @@ func (d *Deleter) deleteWay(id int64, deleteRefs bool) {
 			d.diffCache.Coords.DeleteRef(n, id)
 		}
 	}
-	if deleted && d.expireTiles != nil {
+	if deleted && d.expireor != nil {
 		err := d.osmCache.Coords.FillWay(elem)
 		if err != nil {
 			return
 		}
-		d.expireTiles.ExpireFromNodes(elem.Nodes)
+		expire.ExpireNodes(d.expireor, elem.Nodes)
 	}
 }
 
@@ -171,8 +171,8 @@ func (d *Deleter) deleteNode(id int64) {
 		deleted = true
 	}
 
-	if deleted && d.expireTiles != nil {
-		d.expireTiles.ExpireFromNodes([]element.Node{*elem})
+	if deleted && d.expireor != nil {
+		d.expireor.Expire(elem.Long, elem.Lat)
 	}
 
 }

diff/process.go

@@ -21,14 +21,14 @@ import (
 
 var log = logging.NewLogger("diff")
 
-func Update(oscFile string, geometryLimiter *limit.Limiter, force bool) {
+func Update(oscFile string, geometryLimiter *limit.Limiter, expireor expire.Expireor, force bool) {
 	state, err := diffstate.ParseFromOsc(oscFile)
 	if err != nil {
 		log.Fatal(err)
 	}
 	lastState, err := diffstate.ParseLastState(config.BaseOptions.CacheDir)
 	if err != nil {
-		log.Fatal(err)
+		log.Warn(err)
 	}
 
 	if lastState != nil && lastState.Sequence != 0 && state != nil && state.Sequence <= lastState.Sequence {
@@ -95,8 +95,6 @@ func Update(oscFile string, geometryLimiter *limit.Limiter, force bool) {
 
 	progress := stats.NewStatsReporter()
 
-	expiredTiles := expire.NewTiles(14)
-
 	relTagFilter := tagmapping.RelationTagFilter()
 	wayTagFilter := tagmapping.WayTagFilter()
 	nodeTagFilter := tagmapping.NodeTagFilter()
@@ -108,18 +106,19 @@ func Update(oscFile string, geometryLimiter *limit.Limiter, force bool) {
 	relWriter := writer.NewRelationWriter(osmCache, diffCache, relations,
 		db, progress, config.BaseOptions.Srid)
 	relWriter.SetLimiter(geometryLimiter)
-	relWriter.SetExpireTiles(expiredTiles)
+	relWriter.SetExpireor(expireor)
 	relWriter.Start()
 
 	wayWriter := writer.NewWayWriter(osmCache, diffCache, ways, db,
 		progress, config.BaseOptions.Srid)
 	wayWriter.SetLimiter(geometryLimiter)
-	wayWriter.SetExpireTiles(expiredTiles)
+	wayWriter.SetExpireor(expireor)
 	wayWriter.Start()
 
 	nodeWriter := writer.NewNodeWriter(osmCache, nodes, db,
 		progress, config.BaseOptions.Srid)
 	nodeWriter.SetLimiter(geometryLimiter)
+	nodeWriter.SetExpireor(expireor)
 	nodeWriter.Start()
 
 	nodeIds := make(map[int64]bool)
@@ -294,12 +293,6 @@ For:
 	diffCache.Close()
 	log.StopStep(step)
 
-	step = log.StartStep("Updating expired tiles db")
-	expire.WriteTileExpireDb(
-		expiredTiles.SortedTiles(),
-		"/tmp/expire_tiles.db",
-	)
-	log.StopStep(step)
 	progress.Stop()
 
 	if state != nil {

expire/expire.go

@@ -0,0 +1,15 @@
+package expire
+
+import (
+	"imposm3/element"
+)
+
+type Expireor interface {
+	Expire(long, lat float64)
+}
+
+func ExpireNodes(expireor Expireor, nodes []element.Node) {
+	for _, nd := range nodes {
+		expireor.Expire(nd.Long, nd.Lat)
+	}
+}

imposm3.go

@@ -66,7 +66,7 @@ func main() {
 		}
 
 		for _, oscFile := range config.DiffFlags.Args() {
-			diff.Update(oscFile, geometryLimiter, false)
+			diff.Update(oscFile, geometryLimiter, nil, false)
 		}
 	case "query-cache":
 		query.Query(os.Args[2:])

writer/nodes.go

@@ -42,8 +42,8 @@ func (nw *NodeWriter) loop() {
 		nw.progress.AddNodes(1)
 		if ok, matches := nw.inserter.ProbePoint(n.OSMElem); ok {
 			proj.NodeToMerc(n)
-			if nw.expireTiles != nil {
-				nw.expireTiles.ExpireFromNodes([]element.Node{*n})
+			if nw.expireor != nil {
+				nw.expireor.Expire(n.Long, n.Lat)
 			}
 			point, err := geom.Point(geos, *n)
 			if err != nil {

writer/relations.go

@@ -4,6 +4,7 @@ import (
 	"imposm3/cache"
 	"imposm3/database"
 	"imposm3/element"
+	"imposm3/expire"
 	"imposm3/geom"
 	"imposm3/geom/geos"
 	"imposm3/proj"
@@ -136,10 +137,10 @@ NextRel:
 				}
 			}
 		}
-		if rw.expireTiles != nil {
+		if rw.expireor != nil {
 			for _, m := range allMembers {
 				if m.Way != nil {
-					rw.expireTiles.ExpireFromNodes(m.Way.Nodes)
+					expire.ExpireNodes(rw.expireor, m.Way.Nodes)
 				}
 			}
 		}

writer/ways.go

@@ -4,6 +4,7 @@ import (
 	"imposm3/cache"
 	"imposm3/database"
 	"imposm3/element"
+	"imposm3/expire"
 	"imposm3/geom"
 	"imposm3/geom/geos"
 	"imposm3/proj"
@@ -70,8 +71,8 @@ func (ww *WayWriter) loop() {
 			}
 		}
 
-		if inserted && ww.expireTiles != nil {
-			ww.expireTiles.ExpireFromNodes(w.Nodes)
+		if inserted && ww.expireor != nil {
+			expire.ExpireNodes(ww.expireor, w.Nodes)
 		}
 		if ww.diffCache != nil {
 			ww.diffCache.Coords.AddFromWay(w)

writer/writer.go

@@ -22,16 +22,16 @@ type looper interface {
 }
 
 type OsmElemWriter struct {
-	osmCache    *cache.OSMCache
-	diffCache   *cache.DiffCache
-	progress    *stats.Statistics
-	inserter    database.Inserter
-	wg          *sync.WaitGroup
-	limiter     *limit.Limiter
-	writer      looper
-	srid        int
-	expireTiles *expire.Tiles
-	concurrent  bool
+	osmCache   *cache.OSMCache
+	diffCache  *cache.DiffCache
+	progress   *stats.Statistics
+	inserter   database.Inserter
+	wg         *sync.WaitGroup
+	limiter    *limit.Limiter
+	writer     looper
+	srid       int
+	expireor   expire.Expireor
+	concurrent bool
 }
 
 func (writer *OsmElemWriter) SetLimiter(limiter *limit.Limiter) {
@@ -53,8 +53,8 @@ func (writer *OsmElemWriter) Start() {
 	}
 }
 
-func (writer *OsmElemWriter) SetExpireTiles(expireTiles *expire.Tiles) {
-	writer.expireTiles = expireTiles
+func (writer *OsmElemWriter) SetExpireor(exp expire.Expireor) {
+	writer.expireor = exp
 }
 
 func (writer *OsmElemWriter) Wait() {