write out expired tile coordinates

diff/cmd/process.go

@@ -10,6 +10,7 @@ import (
 	"goposm/diff"
 	"goposm/diff/parser"
 	"goposm/element"
+	"goposm/expire"
 	"goposm/geom/clipper"
 	"goposm/logging"
 	"goposm/mapping"
@@ -92,6 +93,8 @@ func update(oscFile string, conf *config.Config) {
 
 	var geometryClipper *clipper.Clipper
 
+	expiredTiles := expire.NewTiles(14)
+
 	relTagFilter := tagmapping.RelationTagFilter()
 	wayTagFilter := tagmapping.WayTagFilter()
 	nodeTagFilter := tagmapping.NodeTagFilter()
@@ -107,11 +110,13 @@ func update(oscFile string, conf *config.Config) {
 	relWriter := writer.NewRelationWriter(osmCache, diffCache, relations,
 		db, polygonsTagMatcher, progress, conf.Srid)
 	relWriter.SetClipper(geometryClipper)
+	relWriter.SetExpireTiles(expiredTiles)
 	relWriter.Start()
 
 	wayWriter := writer.NewWayWriter(osmCache, diffCache, ways, db,
 		lineStringsTagMatcher, polygonsTagMatcher, progress, conf.Srid)
 	wayWriter.SetClipper(geometryClipper)
+	wayWriter.SetExpireTiles(expiredTiles)
 	wayWriter.Start()
 
 	nodeWriter := writer.NewNodeWriter(osmCache, nodes, db,
@@ -261,8 +266,15 @@ For:
 		log.Fatal(err)
 	}
 
-	progress.Stop()
 	osmCache.Close()
 	diffCache.Close()
 	log.StopStep(step)
+
+	step = log.StartStep("Updating expired tiles db")
+	expire.WriteTileExpireDb(
+		expiredTiles.SortedTiles(),
+		"/tmp/expire_tiles.db",
+	)
+	log.StopStep(step)
+	progress.Stop()
 }

expire/collect.go

@@ -0,0 +1,181 @@
+package expire
+
+import (
+	"database/sql"
+	"fmt"
+	_ "github.com/mattn/go-sqlite3"
+	"goposm/element"
+	"io"
+	"math"
+	"sort"
+	"sync"
+)
+
+var mercBbox = [4]float64{
+	-20037508.342789244,
+	-20037508.342789244,
+	20037508.342789244,
+	20037508.342789244,
+}
+
+var mercRes [20]float64
+
+func init() {
+	res := 2 * 20037508.342789244 / 256
+
+	for i, _ := range mercRes {
+		mercRes[i] = res
+		res /= 2
+	}
+}
+
+func mercTileCoord(x, y float64, zoom uint32) (uint32, uint32) {
+	res := mercRes[zoom]
+	x = x - mercBbox[0]
+	y = mercBbox[3] - y
+	tileX := uint32(math.Floor(x / (res * 256)))
+	tileY := uint32(math.Floor(y / (res * 256)))
+
+	return tileX, tileY
+}
+
+type Tiles struct {
+	tiles map[tileKey]bool
+	zoom  uint32
+	mu    *sync.Mutex
+}
+
+type tileKey struct {
+	x uint32
+	y uint32
+}
+
+type tile struct {
+	x uint32
+	y uint32
+	z uint32
+	d uint32
+}
+
+func NewTiles(zoom uint32) *Tiles {
+	return &Tiles{make(map[tileKey]bool), zoom, &sync.Mutex{}}
+}
+
+func (tc *Tiles) addCoord(x, y float64) {
+	tileX, tileY := mercTileCoord(x, y, tc.zoom)
+	tc.mu.Lock()
+	defer tc.mu.Unlock()
+	tc.tiles[tileKey{tileX, tileY}] = true
+}
+
+func (tc *Tiles) SortedTiles() []tile {
+	tiles := make([]tile, len(tc.tiles))
+	i := 0
+	for tileKey, _ := range tc.tiles {
+		tiles[i] = tile{
+			tileKey.x,
+			tileKey.y,
+			tc.zoom,
+			hilbert(tileKey.x, tileKey.y, tc.zoom),
+		}
+		i++
+	}
+	sort.Sort(byHilbert(tiles))
+	return tiles
+}
+
+func (tc *Tiles) ExpireFromNodes(nodes []element.Node) {
+	for _, nd := range nodes {
+		tc.addCoord(nd.Long, nd.Lat)
+	}
+}
+
+type byHilbert []tile
+
+func (h byHilbert) Len() int           { return len(h) }
+func (h byHilbert) Swap(i, j int)      { h[i], h[j] = h[j], h[i] }
+func (h byHilbert) Less(i, j int) bool { return h[i].d < h[j].d }
+
+// Hilbert returns the distance of tile x, y in a hilbert curve of
+// level z, where z=0 is 1x1, z=1 is 2x2, etc.
+func hilbert(x, y, z uint32) uint32 {
+	n := uint32(2 << (z - 1))
+	var rx, ry, d, s uint32
+	for s = n / 2; s > 0; s /= 2 {
+		if (x & s) > 0 {
+			rx = 1
+		} else {
+			rx = 0
+		}
+		if (y & s) > 0 {
+			ry = 1
+		} else {
+			ry = 0
+		}
+		d += s * s * ((3 * rx) ^ ry)
+		x, y = rot(s, x, y, rx, ry)
+	}
+	return d
+}
+
+//rotate/flip a quadrant appropriately
+func rot(n, x, y, rx, ry uint32) (uint32, uint32) {
+	if ry == 0 {
+		if rx == 1 {
+			x = n - 1 - x
+			y = n - 1 - y
+		}
+
+		//Swap x and y
+		return y, x
+	}
+	return x, y
+}
+
+func WriteTileExpireList(tiles []tile, writer io.Writer) error {
+	for _, tile := range tiles {
+		_, err := fmt.Fprintf(writer, "%d/%d/%d\n", tile.z, tile.x, tile.y)
+		if err != nil {
+			return err
+		}
+	}
+	return nil
+}
+
+func WriteTileExpireDb(tiles []tile, dbfile string) error {
+	db, err := sql.Open("sqlite3", dbfile)
+	if err != nil {
+		return err
+	}
+	defer db.Close()
+
+	stmts := []string{
+		`create table if not exists tiles (
+            x integer,
+            y integer,
+            z integer,
+            time datetime,
+            primary key (x, y, z)
+        )`,
+	}
+	for _, stmt := range stmts {
+		_, err := db.Exec(stmt)
+		if err != nil {
+			return err
+		}
+	}
+
+	stmt, err := db.Prepare(`insert or replace into tiles (x, y, z, time) values (?, ?, ?, DATETIME('now'))`)
+	if err != nil {
+		return err
+	}
+	defer stmt.Close()
+
+	for _, tile := range tiles {
+		_, err := stmt.Exec(tile.x, tile.y, tile.z)
+		if err != nil {
+			return err
+		}
+	}
+	return nil
+}

expire/collect_test.go

@@ -0,0 +1,88 @@
+package expire
+
+import (
+	"testing"
+)
+
+func TestMercTileCoord(t *testing.T) {
+	if x, y := mercTileCoord(0, 0, 0); x != 0 || y != 0 {
+		t.Fatal(x, y)
+	}
+
+	if x, y := mercTileCoord(0, 0, 1); x != 1 || y != 1 {
+		t.Fatal(x, y)
+	}
+
+	if x, y := mercTileCoord(-10000, 10000, 1); x != 0 || y != 0 {
+		t.Fatal(x, y)
+	}
+
+	if x, y := mercTileCoord(914785.4932536, 7010978.3787268, 18); x != 137055 || y != 85210 {
+		t.Fatal(x, y)
+	}
+	if x, y := mercTileCoord(914785.4932536, 7010978.3787268, 14); x != 8565 || y != 5325 {
+		t.Fatal(x, y)
+	}
+
+}
+
+func TestHilbert(t *testing.T) {
+	if d := hilbert(0, 0, 1); d != 0 {
+		t.Fatal(d)
+	}
+	if d := hilbert(0, 1, 1); d != 1 {
+		t.Fatal(d)
+	}
+	if d := hilbert(1, 1, 1); d != 2 {
+		t.Fatal(d)
+	}
+	if d := hilbert(1, 0, 1); d != 3 {
+		t.Fatal(d)
+	}
+
+	if d := hilbert(2, 0, 2); d != 14 {
+		t.Fatal(d)
+	}
+
+	if d := hilbert(1, 3, 3); d != 12 {
+		t.Fatal(d)
+	}
+
+}
+
+func TestTileCollection(t *testing.T) {
+	tc := NewTiles(14)
+
+	tc.addCoord(914785.4932536, 7010978.3787268) // 8565 5325
+	if len(tc.SortedTiles()) != 1 {
+		t.Fatal(tc.tiles)
+	}
+	// add twice
+	tc.addCoord(914785.4932536, 7010978.3787268)
+	if len(tc.SortedTiles()) != 1 {
+		t.Fatal(tc.tiles)
+	}
+
+	// different coord, but same tile
+	tc.addCoord(914785.4932536, 7010778.3787268)
+	if len(tc.SortedTiles()) != 1 {
+		t.Fatal(tc.tiles)
+	}
+
+	tc.addCoord(915785.4932536, 7010778.3787268) // 8566 5325
+	tc.addCoord(915785.4932536, 7020778.3787268) // 8566 5321
+	tc.addCoord(915785.4932536, 7000778.3787268) // 8566 5329
+
+	tc.addCoord(1915785.4932536, 7010778.3787268)  // 8975 5325
+	tc.addCoord(1915785.4932536, 17010778.3787268) // 8975 1237
+	if tiles := tc.SortedTiles(); len(tiles) != 6 ||
+		tiles[0].x != 8566 ||
+		tiles[1].x != 8565 ||
+		tiles[2].x != 8566 ||
+		tiles[3].x != 8566 ||
+		tiles[4].x != 8975 ||
+		tiles[5].x != 8975 {
+		t.Fatal(tiles)
+	}
+
+}

writer/relations.go

@@ -66,6 +66,9 @@ NextRel:
 				continue NextRel
 			}
 			proj.NodesToMerc(m.Way.Nodes)
+			if rw.expireTiles != nil {
+				rw.expireTiles.ExpireFromNodes(m.Way.Nodes)
+			}
 		}
 
 		// BuildRelation updates r.Members but we need all of them

writer/ways.go

@@ -60,6 +60,10 @@ func (ww *WayWriter) loop() {
 			continue
 		}
 		proj.NodesToMerc(w.Nodes)
+		if ww.expireTiles != nil {
+			ww.expireTiles.ExpireFromNodes(w.Nodes)
+		}
+
 		if matches := ww.lineStringTagMatcher.Match(&w.Tags); len(matches) > 0 {
 			ww.buildAndInsert(geos, w, matches, geom.LineString)
 		}

writer/writer.go

@@ -4,6 +4,7 @@ import (
 	"goposm/cache"
 	"goposm/database"
 	"goposm/element"
+	"goposm/expire"
 	"goposm/geom/clipper"
 	"goposm/mapping"
 	"goposm/stats"
@@ -61,6 +62,7 @@ type OsmElemWriter struct {
 	clipper      *clipper.Clipper
 	writer       looper
 	srid         int
+	expireTiles  *expire.Tiles
 }
 
 func (writer *OsmElemWriter) SetClipper(clipper *clipper.Clipper) {
@@ -74,6 +76,10 @@ func (writer *OsmElemWriter) Start() {
 	}
 }
 
+func (writer *OsmElemWriter) SetExpireTiles(expireTiles *expire.Tiles) {
+	writer.expireTiles = expireTiles
+}
+
 func (writer *OsmElemWriter) Close() {
 	writer.wg.Wait()
 }