From bc7b22da61e57b521d23386db01f39ac95739b0e Mon Sep 17 00:00:00 2001 From: Oliver Tonnhofer Date: Fri, 12 Jul 2013 14:57:06 +0200 Subject: [PATCH] write out expired tile coordinates --- diff/cmd/process.go | 14 +++- expire/collect.go | 181 +++++++++++++++++++++++++++++++++++++++++ expire/collect_test.go | 88 ++++++++++++++++++++ writer/relations.go | 3 + writer/ways.go | 4 + writer/writer.go | 6 ++ 6 files changed, 295 insertions(+), 1 deletion(-) create mode 100644 expire/collect.go create mode 100644 expire/collect_test.go diff --git a/diff/cmd/process.go b/diff/cmd/process.go index a698b3a..dfe99ef 100644 --- a/diff/cmd/process.go +++ b/diff/cmd/process.go @@ -10,6 +10,7 @@ import ( "goposm/diff" "goposm/diff/parser" "goposm/element" + "goposm/expire" "goposm/geom/clipper" "goposm/logging" "goposm/mapping" @@ -92,6 +93,8 @@ func update(oscFile string, conf *config.Config) { var geometryClipper *clipper.Clipper + expiredTiles := expire.NewTiles(14) + relTagFilter := tagmapping.RelationTagFilter() wayTagFilter := tagmapping.WayTagFilter() nodeTagFilter := tagmapping.NodeTagFilter() @@ -107,11 +110,13 @@ func update(oscFile string, conf *config.Config) { relWriter := writer.NewRelationWriter(osmCache, diffCache, relations, db, polygonsTagMatcher, progress, conf.Srid) relWriter.SetClipper(geometryClipper) + relWriter.SetExpireTiles(expiredTiles) relWriter.Start() wayWriter := writer.NewWayWriter(osmCache, diffCache, ways, db, lineStringsTagMatcher, polygonsTagMatcher, progress, conf.Srid) wayWriter.SetClipper(geometryClipper) + wayWriter.SetExpireTiles(expiredTiles) wayWriter.Start() nodeWriter := writer.NewNodeWriter(osmCache, nodes, db, @@ -261,8 +266,15 @@ For: log.Fatal(err) } - progress.Stop() osmCache.Close() diffCache.Close() log.StopStep(step) + + step = log.StartStep("Updating expired tiles db") + expire.WriteTileExpireDb( + expiredTiles.SortedTiles(), + "/tmp/expire_tiles.db", + ) + log.StopStep(step) + progress.Stop() } diff --git a/expire/collect.go b/expire/collect.go new file mode 100644 index 0000000..73d1067 --- /dev/null +++ b/expire/collect.go @@ -0,0 +1,181 @@ +package expire + +import ( + "database/sql" + "fmt" + _ "github.com/mattn/go-sqlite3" + "goposm/element" + "io" + "math" + "sort" + "sync" +) + +var mercBbox = [4]float64{ + -20037508.342789244, + -20037508.342789244, + 20037508.342789244, + 20037508.342789244, +} + +var mercRes [20]float64 + +func init() { + res := 2 * 20037508.342789244 / 256 + + for i, _ := range mercRes { + mercRes[i] = res + res /= 2 + } +} + +func mercTileCoord(x, y float64, zoom uint32) (uint32, uint32) { + res := mercRes[zoom] + x = x - mercBbox[0] + y = mercBbox[3] - y + tileX := uint32(math.Floor(x / (res * 256))) + tileY := uint32(math.Floor(y / (res * 256))) + + return tileX, tileY +} + +type Tiles struct { + tiles map[tileKey]bool + zoom uint32 + mu *sync.Mutex +} + +type tileKey struct { + x uint32 + y uint32 +} + +type tile struct { + x uint32 + y uint32 + z uint32 + d uint32 +} + +func NewTiles(zoom uint32) *Tiles { + return &Tiles{make(map[tileKey]bool), zoom, &sync.Mutex{}} +} + +func (tc *Tiles) addCoord(x, y float64) { + tileX, tileY := mercTileCoord(x, y, tc.zoom) + tc.mu.Lock() + defer tc.mu.Unlock() + tc.tiles[tileKey{tileX, tileY}] = true +} + +func (tc *Tiles) SortedTiles() []tile { + tiles := make([]tile, len(tc.tiles)) + i := 0 + for tileKey, _ := range tc.tiles { + tiles[i] = tile{ + tileKey.x, + tileKey.y, + tc.zoom, + hilbert(tileKey.x, tileKey.y, tc.zoom), + } + i++ + } + sort.Sort(byHilbert(tiles)) + return tiles +} + +func (tc *Tiles) ExpireFromNodes(nodes []element.Node) { + for _, nd := range nodes { + tc.addCoord(nd.Long, nd.Lat) + } +} + +type byHilbert []tile + +func (h byHilbert) Len() int { return len(h) } +func (h byHilbert) Swap(i, j int) { h[i], h[j] = h[j], h[i] } +func (h byHilbert) Less(i, j int) bool { return h[i].d < h[j].d } + +// Hilbert returns the distance of tile x, y in a hilbert curve of +// level z, where z=0 is 1x1, z=1 is 2x2, etc. +func hilbert(x, y, z uint32) uint32 { + n := uint32(2 << (z - 1)) + var rx, ry, d, s uint32 + for s = n / 2; s > 0; s /= 2 { + if (x & s) > 0 { + rx = 1 + } else { + rx = 0 + } + if (y & s) > 0 { + ry = 1 + } else { + ry = 0 + } + d += s * s * ((3 * rx) ^ ry) + x, y = rot(s, x, y, rx, ry) + } + return d +} + +//rotate/flip a quadrant appropriately +func rot(n, x, y, rx, ry uint32) (uint32, uint32) { + if ry == 0 { + if rx == 1 { + x = n - 1 - x + y = n - 1 - y + } + + //Swap x and y + return y, x + } + return x, y +} + +func WriteTileExpireList(tiles []tile, writer io.Writer) error { + for _, tile := range tiles { + _, err := fmt.Fprintf(writer, "%d/%d/%d\n", tile.z, tile.x, tile.y) + if err != nil { + return err + } + } + return nil +} + +func WriteTileExpireDb(tiles []tile, dbfile string) error { + db, err := sql.Open("sqlite3", dbfile) + if err != nil { + return err + } + defer db.Close() + + stmts := []string{ + `create table if not exists tiles ( + x integer, + y integer, + z integer, + time datetime, + primary key (x, y, z) + )`, + } + for _, stmt := range stmts { + _, err := db.Exec(stmt) + if err != nil { + return err + } + } + + stmt, err := db.Prepare(`insert or replace into tiles (x, y, z, time) values (?, ?, ?, DATETIME('now'))`) + if err != nil { + return err + } + defer stmt.Close() + + for _, tile := range tiles { + _, err := stmt.Exec(tile.x, tile.y, tile.z) + if err != nil { + return err + } + } + return nil +} diff --git a/expire/collect_test.go b/expire/collect_test.go new file mode 100644 index 0000000..c672b88 --- /dev/null +++ b/expire/collect_test.go @@ -0,0 +1,88 @@ +package expire + +import ( + "testing" +) + +func TestMercTileCoord(t *testing.T) { + if x, y := mercTileCoord(0, 0, 0); x != 0 || y != 0 { + t.Fatal(x, y) + } + + if x, y := mercTileCoord(0, 0, 1); x != 1 || y != 1 { + t.Fatal(x, y) + } + + if x, y := mercTileCoord(-10000, 10000, 1); x != 0 || y != 0 { + t.Fatal(x, y) + } + + if x, y := mercTileCoord(914785.4932536, 7010978.3787268, 18); x != 137055 || y != 85210 { + t.Fatal(x, y) + } + if x, y := mercTileCoord(914785.4932536, 7010978.3787268, 14); x != 8565 || y != 5325 { + t.Fatal(x, y) + } + +} + +func TestHilbert(t *testing.T) { + if d := hilbert(0, 0, 1); d != 0 { + t.Fatal(d) + } + if d := hilbert(0, 1, 1); d != 1 { + t.Fatal(d) + } + if d := hilbert(1, 1, 1); d != 2 { + t.Fatal(d) + } + if d := hilbert(1, 0, 1); d != 3 { + t.Fatal(d) + } + + if d := hilbert(2, 0, 2); d != 14 { + t.Fatal(d) + } + + if d := hilbert(1, 3, 3); d != 12 { + t.Fatal(d) + } + +} + +func TestTileCollection(t *testing.T) { + tc := NewTiles(14) + + tc.addCoord(914785.4932536, 7010978.3787268) // 8565 5325 + if len(tc.SortedTiles()) != 1 { + t.Fatal(tc.tiles) + } + // add twice + tc.addCoord(914785.4932536, 7010978.3787268) + if len(tc.SortedTiles()) != 1 { + t.Fatal(tc.tiles) + } + + // different coord, but same tile + tc.addCoord(914785.4932536, 7010778.3787268) + if len(tc.SortedTiles()) != 1 { + t.Fatal(tc.tiles) + } + + tc.addCoord(915785.4932536, 7010778.3787268) // 8566 5325 + tc.addCoord(915785.4932536, 7020778.3787268) // 8566 5321 + tc.addCoord(915785.4932536, 7000778.3787268) // 8566 5329 + + tc.addCoord(1915785.4932536, 7010778.3787268) // 8975 5325 + tc.addCoord(1915785.4932536, 17010778.3787268) // 8975 1237 + if tiles := tc.SortedTiles(); len(tiles) != 6 || + tiles[0].x != 8566 || + tiles[1].x != 8565 || + tiles[2].x != 8566 || + tiles[3].x != 8566 || + tiles[4].x != 8975 || + tiles[5].x != 8975 { + t.Fatal(tiles) + } + +} diff --git a/writer/relations.go b/writer/relations.go index 8636575..15be944 100644 --- a/writer/relations.go +++ b/writer/relations.go @@ -66,6 +66,9 @@ NextRel: continue NextRel } proj.NodesToMerc(m.Way.Nodes) + if rw.expireTiles != nil { + rw.expireTiles.ExpireFromNodes(m.Way.Nodes) + } } // BuildRelation updates r.Members but we need all of them diff --git a/writer/ways.go b/writer/ways.go index 94859b9..99d9858 100644 --- a/writer/ways.go +++ b/writer/ways.go @@ -60,6 +60,10 @@ func (ww *WayWriter) loop() { continue } proj.NodesToMerc(w.Nodes) + if ww.expireTiles != nil { + ww.expireTiles.ExpireFromNodes(w.Nodes) + } + if matches := ww.lineStringTagMatcher.Match(&w.Tags); len(matches) > 0 { ww.buildAndInsert(geos, w, matches, geom.LineString) } diff --git a/writer/writer.go b/writer/writer.go index a850d36..9d24e9f 100644 --- a/writer/writer.go +++ b/writer/writer.go @@ -4,6 +4,7 @@ import ( "goposm/cache" "goposm/database" "goposm/element" + "goposm/expire" "goposm/geom/clipper" "goposm/mapping" "goposm/stats" @@ -61,6 +62,7 @@ type OsmElemWriter struct { clipper *clipper.Clipper writer looper srid int + expireTiles *expire.Tiles } func (writer *OsmElemWriter) SetClipper(clipper *clipper.Clipper) { @@ -74,6 +76,10 @@ func (writer *OsmElemWriter) Start() { } } +func (writer *OsmElemWriter) SetExpireTiles(expireTiles *expire.Tiles) { + writer.expireTiles = expireTiles +} + func (writer *OsmElemWriter) Close() { writer.wg.Wait() }