From 3f083fb75350fac5299273f342905ecf048d3ed4 Mon Sep 17 00:00:00 2001 From: Oliver Tonnhofer Date: Mon, 23 Nov 2015 09:05:19 +0100 Subject: [PATCH] add single table tests --- test/completedb_test.go | 155 ++++++++++++++++++-------------- test/helper_test.go | 180 ++++++++++++++++++++++++++++++++++++- test/single_table_test.go | 184 ++++++++++++++++++++++++++++++++++++++ 3 files changed, 448 insertions(+), 71 deletions(-) create mode 100644 test/single_table_test.go diff --git a/test/completedb_test.go b/test/completedb_test.go index fbc0bd7..ee7b31f 100644 --- a/test/completedb_test.go +++ b/test/completedb_test.go @@ -7,7 +7,6 @@ import ( "github.com/omniscale/imposm3/geom" "github.com/omniscale/imposm3/proj" - "math" "testing" "github.com/omniscale/imposm3/geom/geos" @@ -53,74 +52,6 @@ func TestDeploy(t *testing.T) { } } -type checkElem struct { - table string - id int64 - osmType string - tags map[string]string -} - -func assertRecordsMissing(t *testing.T, elems []checkElem) { - for _, e := range elems { - if ts.queryExists(t, e.table, e.id) { - t.Errorf("found %d in %d", e.id, e.table) - } - } -} - -func assertRecords(t *testing.T, elems []checkElem) { - for _, e := range elems { - keys := make([]string, 0, len(e.tags)) - for k, _ := range e.tags { - keys = append(keys, k) - } - r := ts.query(t, e.table, e.id, keys) - if e.osmType == "" { - if r.missing { - continue - } - t.Errorf("got unexpected record %d", r.id) - } - if r.osmType != e.osmType { - t.Errorf("got unexpected type %s != %s for %d", r.osmType, e.osmType, e.id) - } - for k, v := range e.tags { - if r.tags[k] != v { - t.Errorf("%s does not match for %d %s != %s", k, e.id, r.tags[k], v) - } - } - } -} - -func assertValid(t *testing.T, e checkElem) { - geom := ts.queryGeom(t, e.table, e.id) - if !ts.g.IsValid(geom) { - t.Fatalf("geometry of %d is invalid", e.id) - } -} - -func assertArea(t *testing.T, e checkElem, expect float64) { - geom := ts.queryGeom(t, e.table, e.id) - if !ts.g.IsValid(geom) { - t.Fatalf("geometry of %d is invalid", e.id) - } - actual := geom.Area() - if math.Abs(expect-actual) > 1 { - t.Errorf("unexpected size of %d %f!=%f", e.id, actual, expect) - } -} - -func assertLength(t *testing.T, e checkElem, expect float64) { - geom := ts.queryGeom(t, e.table, e.id) - if !ts.g.IsValid(geom) { - t.Fatalf("geometry of %d is invalid", e.id) - } - actual := geom.Length() - if math.Abs(expect-actual) > 1 { - t.Errorf("unexpected size of %d %f!=%f", e.id, actual, expect) - } -} - func TestLandusageToWaterarea1(t *testing.T) { // Parks inserted into landusages // t.assert_cached_way(11001) @@ -653,3 +584,89 @@ func TestUnsupportedRelation(t *testing.T) { {"osm_landusages", 201251, "park", nil}, }) } + +// ####################################################################### + +func TestDeployRevert(t *testing.T) { + if ts.tableExists(t, dbschemaImport, "osm_roads") { + t.Fatalf("table osm_roads exists in schema %s", dbschemaImport) + } + if !ts.tableExists(t, dbschemaProduction, "osm_roads") { + t.Fatalf("table osm_roads does not exists in schema %s", dbschemaProduction) + } + if ts.tableExists(t, dbschemaBackup, "osm_roads") { + t.Fatalf("table osm_roads exists in schema %s", dbschemaBackup) + } + + ts.importOsm(t) + + if !ts.tableExists(t, dbschemaImport, "osm_roads") { + t.Fatalf("table osm_roads does not exists in schema %s", dbschemaImport) + } + if !ts.tableExists(t, dbschemaProduction, "osm_roads") { + t.Fatalf("table osm_roads does not exists in schema %s", dbschemaProduction) + } + if ts.tableExists(t, dbschemaBackup, "osm_roads") { + t.Fatalf("table osm_roads exists in schema %s", dbschemaBackup) + } + + ts.deployOsm(t) + + if ts.tableExists(t, dbschemaImport, "osm_roads") { + t.Fatalf("table osm_roads exists in schema %s", dbschemaImport) + } + if !ts.tableExists(t, dbschemaProduction, "osm_roads") { + t.Fatalf("table osm_roads does not exists in schema %s", dbschemaProduction) + } + if !ts.tableExists(t, dbschemaBackup, "osm_roads") { + t.Fatalf("table osm_roads does exists in schema %s", dbschemaBackup) + } + + ts.revertDeployOsm(t) + + if !ts.tableExists(t, dbschemaImport, "osm_roads") { + t.Fatalf("table osm_roads does not exists in schema %s", dbschemaImport) + } + if !ts.tableExists(t, dbschemaProduction, "osm_roads") { + t.Fatalf("table osm_roads does not exists in schema %s", dbschemaProduction) + } + if ts.tableExists(t, dbschemaBackup, "osm_roads") { + t.Fatalf("table osm_roads exists in schema %s", dbschemaBackup) + } +} + +func TestRemoveBackup(t *testing.T) { + if !ts.tableExists(t, dbschemaImport, "osm_roads") { + t.Fatalf("table osm_roads does not exists in schema %s", dbschemaImport) + } + if !ts.tableExists(t, dbschemaProduction, "osm_roads") { + t.Fatalf("table osm_roads does not exists in schema %s", dbschemaProduction) + } + if ts.tableExists(t, dbschemaBackup, "osm_roads") { + t.Fatalf("table osm_roads exists in schema %s", dbschemaBackup) + } + + ts.deployOsm(t) + + if ts.tableExists(t, dbschemaImport, "osm_roads") { + t.Fatalf("table osm_roads exists in schema %s", dbschemaImport) + } + if !ts.tableExists(t, dbschemaProduction, "osm_roads") { + t.Fatalf("table osm_roads does not exists in schema %s", dbschemaProduction) + } + if !ts.tableExists(t, dbschemaBackup, "osm_roads") { + t.Fatalf("table osm_roads does exists in schema %s", dbschemaBackup) + } + + ts.removeBackupOsm(t) + + if ts.tableExists(t, dbschemaImport, "osm_roads") { + t.Fatalf("table osm_roads exists in schema %s", dbschemaImport) + } + if !ts.tableExists(t, dbschemaProduction, "osm_roads") { + t.Fatalf("table osm_roads does not exists in schema %s", dbschemaProduction) + } + if ts.tableExists(t, dbschemaBackup, "osm_roads") { + t.Fatalf("table osm_roads exists in schema %s", dbschemaBackup) + } +} diff --git a/test/helper_test.go b/test/helper_test.go index fc5b55f..e526063 100644 --- a/test/helper_test.go +++ b/test/helper_test.go @@ -4,6 +4,7 @@ import ( "database/sql" "fmt" "log" + "math" "strings" "testing" @@ -49,7 +50,9 @@ func (s *importTestSuite) importOsm(t *testing.T) { // "-optimize", "-mapping", s.config.mappingFileName, "-quiet", + "-revertdeploy=false", "-deployproduction=false", + "-removebackup=false", } config.ParseImport(importArgs) @@ -61,6 +64,9 @@ func (s *importTestSuite) deployOsm(t *testing.T) { "-read=", // overwrite previous options "-write=false", "-optimize=false", + "-revertdeploy=false", + "-deployproduction", + "-removebackup=false", "-connection", s.config.connection, "-dbschema-import", dbschemaImport, "-dbschema-production", dbschemaProduction, @@ -74,6 +80,49 @@ func (s *importTestSuite) deployOsm(t *testing.T) { import_.Import() } +func (s *importTestSuite) revertDeployOsm(t *testing.T) { + importArgs := []string{ + "-read=", // overwrite previous options + "-write=false", + "-optimize=false", + "-revertdeploy", + "-deployproduction=false", + "-removebackup=false", + "-connection", s.config.connection, + "-dbschema-import", dbschemaImport, + "-dbschema-production", dbschemaProduction, + "-dbschema-backup", dbschemaBackup, + "-revertdeploy", + "-deployproduction=false", + "-removebackup=false", + "-mapping", s.config.mappingFileName, + "-quiet", + } + + config.ParseImport(importArgs) + import_.Import() +} + +func (s *importTestSuite) removeBackupOsm(t *testing.T) { + importArgs := []string{ + "-read=", // overwrite previous options + "-write=false", + "-optimize=false", + "-revertdeploy=false", + "-deployproduction=false", + "-removebackup", + "-connection", s.config.connection, + "-dbschema-import", dbschemaImport, + "-dbschema-production", dbschemaProduction, + "-dbschema-backup", dbschemaBackup, + "-mapping", s.config.mappingFileName, + "-quiet", + } + + config.ParseImport(importArgs) + import_.Import() +} + func (s *importTestSuite) updateOsm(t *testing.T, diffFile string) { args := []string{ "-connection", s.config.connection, @@ -165,8 +214,31 @@ func (s *importTestSuite) query(t *testing.T, table string, id int64, keys []str return r } +func (s *importTestSuite) queryTags(t *testing.T, table string, id int64) record { + stmt := fmt.Sprintf(`SELECT osm_id, ST_AsText(geometry), tags FROM "%s"."%s" WHERE osm_id=$1`, dbschemaProduction, table) + row := s.db.QueryRow(stmt, id) + r := record{} + h := hstore.Hstore{} + if err := row.Scan(&r.id, &r.wkt, &h); err != nil { + if err == sql.ErrNoRows { + r.missing = true + } else { + t.Fatal(err) + } + } + if len(h.Map) > 0 { + r.tags = make(map[string]string) + } + for k, v := range h.Map { + if v.Valid { + r.tags[k] = v.String + } + } + return r +} + func (s *importTestSuite) queryRows(t *testing.T, table string, id int64) []record { - rows, err := s.db.Query(fmt.Sprintf(`SELECT osm_id, name, type, ST_AsText(geometry) FROM "%s"."%s" WHERE osm_id=$1 ORDER BY type, name`, dbschemaProduction, table), id) + rows, err := s.db.Query(fmt.Sprintf(`SELECT osm_id, name, type, ST_AsText(geometry) FROM "%s"."%s" WHERE osm_id=$1 ORDER BY type, name, ST_GeometryType(geometry)`, dbschemaProduction, table), id) if err != nil { t.Fatal(err) } @@ -182,7 +254,16 @@ func (s *importTestSuite) queryRows(t *testing.T, table string, id int64) []reco } func (s *importTestSuite) queryGeom(t *testing.T, table string, id int64) *geos.Geom { - r := s.query(t, table, id, nil) + stmt := fmt.Sprintf(`SELECT osm_id, ST_AsText(geometry) FROM "%s"."%s" WHERE osm_id=$1`, dbschemaProduction, table) + row := s.db.QueryRow(stmt, id) + r := record{} + if err := row.Scan(&r.id, &r.wkt); err != nil { + if err == sql.ErrNoRows { + r.missing = true + } else { + t.Fatal(err) + } + } g := geos.NewGeos() defer g.Finish() geom := g.FromWkt(r.wkt) @@ -191,3 +272,98 @@ func (s *importTestSuite) queryGeom(t *testing.T, table string, id int64) *geos. } return geom } + +type checkElem struct { + table string + id int64 + osmType string + tags map[string]string +} + +func assertRecordsMissing(t *testing.T, elems []checkElem) { + for _, e := range elems { + if ts.queryExists(t, e.table, e.id) { + t.Errorf("found %d in %d", e.id, e.table) + } + } +} + +func assertRecords(t *testing.T, elems []checkElem) { + for _, e := range elems { + keys := make([]string, 0, len(e.tags)) + for k, _ := range e.tags { + keys = append(keys, k) + } + r := ts.query(t, e.table, e.id, keys) + if e.osmType == "" { + if r.missing { + continue + } + t.Errorf("got unexpected record %d", r.id) + } + if r.osmType != e.osmType { + t.Errorf("got unexpected type %s != %s for %d", r.osmType, e.osmType, e.id) + } + for k, v := range e.tags { + if r.tags[k] != v { + t.Errorf("%s does not match for %d %s != %s", k, e.id, r.tags[k], v) + } + } + } +} + +func assertHstore(t *testing.T, elems []checkElem) { + for _, e := range elems { + r := ts.queryTags(t, e.table, e.id) + if e.osmType == "" { + if r.missing { + continue + } + t.Errorf("got unexpected record %d", r.id) + } + if len(e.tags) != len(r.tags) { + t.Errorf("tags for %d differ %v != %v", e.id, r.tags, e.tags) + } + for k, v := range e.tags { + if r.tags[k] != v { + t.Errorf("%s does not match for %d %s != %s", k, e.id, r.tags[k], v) + } + } + } +} + +func assertValid(t *testing.T, e checkElem) { + geom := ts.queryGeom(t, e.table, e.id) + if !ts.g.IsValid(geom) { + t.Fatalf("geometry of %d is invalid", e.id) + } +} + +func assertArea(t *testing.T, e checkElem, expect float64) { + geom := ts.queryGeom(t, e.table, e.id) + if !ts.g.IsValid(geom) { + t.Fatalf("geometry of %d is invalid", e.id) + } + actual := geom.Area() + if math.Abs(expect-actual) > 1 { + t.Errorf("unexpected size of %d %f!=%f", e.id, actual, expect) + } +} + +func assertLength(t *testing.T, e checkElem, expect float64) { + geom := ts.queryGeom(t, e.table, e.id) + if !ts.g.IsValid(geom) { + t.Fatalf("geometry of %d is invalid", e.id) + } + actual := geom.Length() + if math.Abs(expect-actual) > 1 { + t.Errorf("unexpected size of %d %f!=%f", e.id, actual, expect) + } +} + +func assertGeomType(t *testing.T, e checkElem, expect string) { + actual := ts.g.Type(ts.queryGeom(t, e.table, e.id)) + if actual != expect { + t.Errorf("expected %s geometry for %d, got %s", expect, e.id, actual) + } +} diff --git a/test/single_table_test.go b/test/single_table_test.go new file mode 100644 index 0000000..4d6554f --- /dev/null +++ b/test/single_table_test.go @@ -0,0 +1,184 @@ +package test + +import ( + "database/sql" + + "testing" + + "github.com/omniscale/imposm3/geom/geos" +) + +const RelOffset = -1e17 + +func TestSingleTable_Prepare(t *testing.T) { + ts.dir = "/tmp/imposm3test" + ts.config = importConfig{ + connection: "postgis://", + cacheDir: ts.dir, + osmFileName: "build/single_table.pbf", + mappingFileName: "single_table_mapping.json", + } + ts.g = geos.NewGeos() + + var err error + ts.db, err = sql.Open("postgres", "sslmode=disable") + if err != nil { + t.Fatal(err) + } + ts.dropSchemas() +} + +func TestSingleTable_Import(t *testing.T) { + if ts.tableExists(t, dbschemaImport, "osm_all") != false { + t.Fatalf("table osm_all exists in schema %s", dbschemaImport) + } + ts.importOsm(t) + if ts.tableExists(t, dbschemaImport, "osm_all") != true { + t.Fatalf("table osm_all does not exists in schema %s", dbschemaImport) + } +} + +func TestSingleTable_Deploy(t *testing.T) { + ts.deployOsm(t) + if ts.tableExists(t, dbschemaImport, "osm_all") != false { + t.Fatalf("table osm_all exists in schema %s", dbschemaImport) + } + if ts.tableExists(t, dbschemaProduction, "osm_all") != true { + t.Fatalf("table osm_all does not exists in schema %s", dbschemaProduction) + } +} + +func TestSingleTable_NonMappedNodeIsMissing(t *testing.T) { + // Node without mapped tags is missing. + // t.assert_cached_node(10001, (10, 42)) + + assertHstore(t, []checkElem{ + {"osm_all", 10001, "", nil}, + }) +} + +func TestSingleTable_MappedNode(t *testing.T) { + // Node is stored with all tags. + // t.assert_cached_node(10002, (11, 42)) + + assertHstore(t, []checkElem{ + {"osm_all", 10002, "*", map[string]string{"random": "tag", "but": "mapped", "poi": "unicorn"}}, + }) +} + +func TestSingleTable_NonMappedWayIsMissing(t *testing.T) { + // Way without mapped tags is missing. + // t.assert_cached_way(20101) + // t.assert_cached_way(20102) + // t.assert_cached_way(20103) + assertHstore(t, []checkElem{ + {"osm_all", 20101, "", nil}, + {"osm_all", 20102, "", nil}, + {"osm_all", 20103, "", nil}, + }) +} + +func TestSingleTable_MappedWay(t *testing.T) { + // Way is stored with all tags. + // t.assert_cached_way(20201) + assertHstore(t, []checkElem{ + {"osm_all", -20201, "*", map[string]string{"random": "tag", "highway": "yes"}}, + }) +} + +func TestSingleTable_NonMappedClosedWayIsMissing(t *testing.T) { + // Closed way without mapped tags is missing. + // t.assert_cached_way(20301) + assertHstore(t, []checkElem{ + {"osm_all", -20301, "", nil}, + }) +} + +func TestSingleTable_MappedClosedWay(t *testing.T) { + // Closed way is stored with all tags. + // t.assert_cached_way(20401) + assertHstore(t, []checkElem{ + {"osm_all", -20401, "*", map[string]string{"random": "tag", "building": "yes"}}, + }) +} + +func TestSingleTable_MappedClosedWayAreaYes(t *testing.T) { + // Closed way with area=yes is not stored as linestring. + // t.assert_cached_way(20501) + assertHstore(t, []checkElem{ + {"osm_all", -20501, "*", map[string]string{"random": "tag", "landuse": "grass", "highway": "pedestrian", "area": "yes"}}, + }) + assertGeomType(t, checkElem{"osm_all", -20501, "*", nil}, "Polygon") +} + +func TestSingleTable_MappedClosedWayAreaNo(t *testing.T) { + // Closed way with area=no is not stored as polygon. + // t.assert_cached_way(20502) + assertHstore(t, []checkElem{ + {"osm_all", -20502, "*", map[string]string{"random": "tag", "landuse": "grass", "highway": "pedestrian", "area": "no"}}, + }) + assertGeomType(t, checkElem{"osm_all", -20502, "*", nil}, "LineString") +} + +func TestSingleTable_MappedClosedWayWithoutArea(t *testing.T) { + // Closed way without area is stored as mapped (linestring and polygon). + // t.assert_cached_way(20601) + // elems = t.query_row(t.db_conf, 'osm_all', -20601) + // assert len(elems) == 2 + // elems.sort(key=lambda x: x['geometry'].type) + + // assert elems[0]['geometry'].type == 'LineString', elems[0]['geometry'].type + // assert elems[0]['tags'] == {'random': 'tag', 'landuse': 'grass', 'highway': 'pedestrian'} + // assert elems[1]['geometry'].type == 'Polygon', elems[1]['geometry'].type + // assert elems[1]['tags'] == {'random': 'tag', 'landuse': 'grass', 'highway': 'pedestrian'} +} + +func TestSingleTable_DuplicateIds1(t *testing.T) { + // Points/lines/polygons with same ID are inserted. + // node = t.query_row(t.db_conf, 'osm_all', 31101) + // assert node['geometry'].type == 'Point', node['geometry'].type + // assert node['tags'] == {'amenity': 'cafe'} + // assert node['geometry'].distance(t.merc_point(80, 47)) < 1 + + // ways = t.query_row(t.db_conf, 'osm_all', -31101) + // ways.sort(key=lambda x: x['geometry'].type) + // assert ways[0]['geometry'].type == 'LineString', ways[0]['geometry'].type + // assert ways[0]['tags'] == {'landuse': 'park', 'highway': 'secondary'} + // assert ways[1]['geometry'].type == 'Polygon', ways[1]['geometry'].type + // assert ways[1]['tags'] == {'landuse': 'park', 'highway': 'secondary'} + + assertHstore(t, []checkElem{ + {"osm_all", RelOffset - 31101, "*", map[string]string{"building": "yes"}}, + }) + assertGeomType(t, checkElem{"osm_all", RelOffset - 31101, "*", nil}, "Polygon") +} + +// ####################################################################### + +func TestSingleTable_Update(t *testing.T) { + ts.updateOsm(t, "./build/single_table.osc.gz") +} + +// ####################################################################### + +func TestSingleTable_DuplicateIds2(t *testing.T) { + // Node moved and ways/rels with same ID are still present. + + // node = t.query_row(t.db_conf, 'osm_all', 31101) + // assert node['geometry'].type == 'Point', node['geometry'].type + // assert node['tags'] == {'amenity': 'cafe'} + // assert node['geometry'].distance(t.merc_point(81, 47)) < 1 + + // ways = t.query_row(t.db_conf, 'osm_all', -31101) + // ways.sort(key=lambda x: x['geometry'].type) + + // assert ways[0]['geometry'].type == 'LineString', ways[0]['geometry'].type + // assert ways[0]['tags'] == {'landuse': 'park', 'highway': 'secondary'} + // assert ways[1]['geometry'].type == 'Polygon', ways[1]['geometry'].type + // assert ways[1]['tags'] == {'landuse': 'park', 'highway': 'secondary'} + + assertHstore(t, []checkElem{ + {"osm_all", RelOffset - 31101, "*", map[string]string{"building": "yes"}}, + }) + assertGeomType(t, checkElem{"osm_all", RelOffset - 31101, "*", nil}, "Polygon") +}