vitalif
/
imposm3
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

add -deployproduction -revertdeploy options

master
Oliver Tonnhofer 2013-05-22 08:36:37 +02:00
parent f106aaafd0
commit 599a1d39f5
3 changed files with 225 additions and 68 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

26
database/database.go
View File

@ -12,26 +12,32 @@ type Config struct {
}
type DB interface {
Init(*mapping.Mapping) error
Init() error
InsertBatch(string, [][]interface{}) error
}
var databases map[string]func(Config) (DB, error)
type Deployer interface {
DeployProduction() error
RevertDeploy() error
}
func Register(name string, f func(Config) (DB, error)) {
if databases == nil {
databases = make(map[string]func(Config) (DB, error))
}
var databases map[string]func(Config, *mapping.Mapping) (DB, error)
func init() {
databases = make(map[string]func(Config, *mapping.Mapping) (DB, error))
}
func Register(name string, f func(Config, *mapping.Mapping) (DB, error)) {
databases[name] = f
}
func Open(conf Config) (DB, error) {
func Open(conf Config, m *mapping.Mapping) (DB, error) {
newFunc, ok := databases[conf.Type]
if !ok {
panic("unsupported database type: " + conf.Type)
}
db, err := newFunc(conf)
db, err := newFunc(conf, m)
if err != nil {
return nil, err
}
@ -45,10 +51,10 @@ func ConnectionType(param string) string {
type NullDb struct{}
func (n *NullDb) Init(*mapping.Mapping) error { return nil }
func (n *NullDb) Init() error { return nil }
func (n *NullDb) InsertBatch(string, [][]interface{}) error { return nil }
func NewNullDb(conf Config) (DB, error) {
func NewNullDb(conf Config, m *mapping.Mapping) (DB, error) {
return &NullDb{}, nil
}

195
database/postgis/postgis.go
View File

@ -67,12 +67,12 @@ func (spec *TableSpec) InsertSQL() string {
)
}
func NewTableSpec(conf *database.Config, t *mapping.Table, schema string) *TableSpec {
func NewTableSpec(pg *PostGIS, t *mapping.Table) *TableSpec {
spec := TableSpec{
Name: t.Name,
Schema: schema,
Name: pg.Prefix + t.Name,
Schema: pg.Schema,
GeometryType: t.Type,
Srid: conf.Srid,
Srid: pg.Config.Srid,
}
for _, field := range t.Fields {
pgType, ok := pgTypes[field.Type]
@ -107,7 +107,6 @@ func (e *SQLInsertError) Error() string {
func (pg *PostGIS) createTable(spec TableSpec) error {
var sql string
var err error
sql = fmt.Sprintf(`DROP TABLE IF EXISTS "%s"."%s"`, spec.Schema, spec.Name)
_, err = pg.Db.Exec(sql)
if err != nil {
@ -134,16 +133,16 @@ func (pg *PostGIS) createTable(spec TableSpec) error {
return nil
}
func (pg *PostGIS) createSchema() error {
func (pg *PostGIS) createSchema(schema string) error {
var sql string
var err error
if pg.Schema == "public" {
if schema == "public" {
return nil
}
sql = fmt.Sprintf("SELECT EXISTS(SELECT schema_name FROM information_schema.schemata WHERE schema_name = '%s');",
pg.Schema)
schema)
row := pg.Db.QueryRow(sql)
var exists bool
err = row.Scan(&exists)
@ -154,7 +153,7 @@ func (pg *PostGIS) createSchema() error {
return nil
}
sql = fmt.Sprintf("CREATE SCHEMA \"%s\"", pg.Schema)
sql = fmt.Sprintf("CREATE SCHEMA \"%s\"", schema)
_, err = pg.Db.Exec(sql)
if err != nil {
return &SQLError{sql, err}
@ -163,35 +162,55 @@ func (pg *PostGIS) createSchema() error {
}
type PostGIS struct {
Db *sql.DB
Schema string
Config database.Config
Tables map[string]*TableSpec
Db *sql.DB
Schema string
BackupSchema string
Config database.Config
Tables map[string]*TableSpec
Prefix string
}
func schemaFromConnectionParams(params string) string {
func schemasFromConnectionParams(params string) (string, string) {
parts := strings.Fields(params)
var schema, backupSchema string
for _, p := range parts {
if strings.HasPrefix(p, "schema=") {
return strings.Replace(p, "schema=", "", 1)
schema = strings.Replace(p, "schema=", "", 1)
} else if strings.HasPrefix(p, "backupschema=") {
backupSchema = strings.Replace(p, "backupschema=", "", 1)
}
}
return "public"
if schema == "" {
schema = "import"
}
if backupSchema == "" {
backupSchema = "backup"
}
return schema, backupSchema
}
func prefixFromConnectionParams(params string) string {
parts := strings.Fields(params)
var prefix string
for _, p := range parts {
if strings.HasPrefix(p, "prefix=") {
prefix = strings.Replace(p, "prefix=", "", 1)
break
}
}
if prefix == "" {
prefix = "osm_"
}
if prefix[len(prefix)-1] != '_' {
prefix = prefix + "_"
}
return prefix
}
func (pg *PostGIS) Open() error {
var err error
if strings.HasPrefix(pg.Config.ConnectionParams, "postgis://") {
pg.Config.ConnectionParams = strings.Replace(
pg.Config.ConnectionParams,
"postgis", "postgres", 1,
)
}
params, err := pq.ParseURL(pg.Config.ConnectionParams)
pg.Schema = schemaFromConnectionParams(params)
if err != nil {
return err
}
@ -248,14 +267,11 @@ func (pg *PostGIS) InsertBatch(table string, rows [][]interface{}) error {
}
func (pg *PostGIS) Init(m *mapping.Mapping) error {
if err := pg.createSchema(); err != nil {
func (pg *PostGIS) Init() error {
if err := pg.createSchema(pg.Schema); err != nil {
return err
}
for name, table := range m.Tables {
pg.Tables[name] = NewTableSpec(&pg.Config, table, pg.Schema)
}
for _, spec := range pg.Tables {
if err := pg.createTable(*spec); err != nil {
return err
@ -264,11 +280,126 @@ func (pg *PostGIS) Init(m *mapping.Mapping) error {
return nil
}
func New(conf database.Config) (database.DB, error) {
func tableExists(tx *sql.Tx, schema, table string) (bool, error) {
var exists bool
sql := fmt.Sprintf(`SELECT EXISTS(SELECT * FROM information_schema.tables WHERE table_name='%s' AND table_schema='%s')`,
table, schema)
row := tx.QueryRow(sql)
err := row.Scan(&exists)
// fmt.Println(exists, err, sql)
if err != nil {
return false, err
}
return exists, nil
}
func dropTableIfExists(tx *sql.Tx, schema, table string) error {
sql := fmt.Sprintf(`DROP TABLE IF EXISTS "%s"."%s"`, schema, table)
_, err := tx.Exec(sql)
return err
}
func (pg *PostGIS) rotate(source, dest, backup string) error {
if err := pg.createSchema(backup); err != nil {
return err
}
tx, err := pg.Db.Begin()
if err != nil {
return err
}
defer func() {
if tx != nil {
if err := tx.Rollback(); err != nil {
log.Println("rollback failed", err)
}
}
}()
for tableName, _ := range pg.Tables {
tableName = pg.Prefix + tableName
log.Printf("rotating %s from %s -> %s -> %s\n", tableName, source, dest, backup)
backupExists, err := tableExists(tx, backup, tableName)
if err != nil {
return err
}
sourceExists, err := tableExists(tx, source, tableName)
if err != nil {
return err
}
destExists, err := tableExists(tx, dest, tableName)
if err != nil {
return err
}
if !sourceExists {
log.Printf("skipping rotate of %s, table does not exists in %s", tableName, source)
continue
}
if destExists {
log.Printf("backup of %s, to %s", tableName, backup)
if backupExists {
err = dropTableIfExists(tx, backup, tableName)
if err != nil {
return err
}
}
sql := fmt.Sprintf(`ALTER TABLE "%s"."%s" SET SCHEMA "%s"`, dest, tableName, backup)
_, err = tx.Exec(sql)
if err != nil {
return err
}
}
sql := fmt.Sprintf(`ALTER TABLE "%s"."%s" SET SCHEMA "%s"`, source, tableName, dest)
_, err = tx.Exec(sql)
if err != nil {
return err
}
}
err = tx.Commit()
if err != nil {
return err
}
tx = nil
return nil
}
func (pg *PostGIS) DeployProduction() error {
return pg.rotate(pg.Schema, "public", pg.BackupSchema)
}
func (pg *PostGIS) RevertDeploy() error {
return pg.rotate(pg.BackupSchema, "public", pg.Schema)
}
func New(conf database.Config, m *mapping.Mapping) (database.DB, error) {
db := &PostGIS{}
db.Tables = make(map[string]*TableSpec)
db.Config = conf
err := db.Open()
if strings.HasPrefix(db.Config.ConnectionParams, "postgis://") {
db.Config.ConnectionParams = strings.Replace(
db.Config.ConnectionParams,
"postgis", "postgres", 1,
)
}
params, err := pq.ParseURL(db.Config.ConnectionParams)
if err != nil {
return nil, err
}
db.Schema, db.BackupSchema = schemasFromConnectionParams(params)
db.Prefix = prefixFromConnectionParams(params)
for name, table := range m.Tables {
db.Tables[name] = NewTableSpec(db, table)
}
err = db.Open()
if err != nil {
return nil, err
}

72
goposm.go
View File

@ -30,16 +30,18 @@ func init() {
}
var (
cpuprofile = flag.String("cpuprofile", "", "filename of cpu profile output")
memprofile = flag.String("memprofile", "", "dir name of mem profile output and interval (fname:interval)")
cachedir = flag.String("cachedir", "/tmp/goposm", "cache directory")
overwritecache = flag.Bool("overwritecache", false, "overwritecache")
appendcache = flag.Bool("appendcache", false, "append cache")
read = flag.String("read", "", "read")
write = flag.Bool("write", false, "write")
connection = flag.String("connection", "", "connection parameters")
diff = flag.Bool("diff", false, "enable diff support")
mappingFile = flag.String("mapping", "", "mapping file")
cpuprofile = flag.String("cpuprofile", "", "filename of cpu profile output")
memprofile = flag.String("memprofile", "", "dir name of mem profile output and interval (fname:interval)")
cachedir = flag.String("cachedir", "/tmp/goposm", "cache directory")
overwritecache = flag.Bool("overwritecache", false, "overwritecache")
appendcache = flag.Bool("appendcache", false, "append cache")
read = flag.String("read", "", "read")
write = flag.Bool("write", false, "write")
connection = flag.String("connection", "", "connection parameters")
diff = flag.Bool("diff", false, "enable diff support")
mappingFile = flag.String("mapping", "", "mapping file")
deployProduction = flag.Bool("deployproduction", false, "deploy production")
revertDeploy = flag.Bool("revertdeploy", false, "revert deploy to production")
)
func main() {
@ -100,6 +102,21 @@ func main() {
log.Fatal(err)
}
var db database.DB
if *write || *deployProduction || *revertDeploy {
connType := database.ConnectionType(*connection)
conf := database.Config{
Type: connType,
ConnectionParams: *connection,
Srid: 3857,
}
db, err = database.Open(conf, tagmapping)
if err != nil {
log.Fatal(err)
}
}
if *read != "" {
osmCache.Coords.SetLinearImport(true)
reader.ReadPbf(osmCache, progress, tagmapping, *read)
@ -110,6 +127,10 @@ func main() {
if *write {
progress.Reset()
err = db.Init()
if err != nil {
log.Fatal(err)
}
diffCache := cache.NewDiffCache(*cachedir)
if err = diffCache.Remove(); err != nil {
@ -119,22 +140,6 @@ func main() {
log.Fatal(err)
}
connType := database.ConnectionType(*connection)
conf := database.Config{
Type: connType,
ConnectionParams: *connection,
Srid: 3857,
}
db, err := database.Open(conf)
if err != nil {
log.Fatal(err)
}
err = db.Init(tagmapping)
if err != nil {
log.Fatal(err)
}
insertBuffer := writer.NewInsertBuffer()
dbWriter := writer.NewDbWriter(db, insertBuffer.Out)
@ -162,7 +167,22 @@ func main() {
nodeWriter.Close()
insertBuffer.Close()
dbWriter.Close()
}
if *deployProduction {
if db, ok := db.(database.Deployer); ok {
db.DeployProduction()
} else {
log.Fatal("database not deployable")
}
}
if *revertDeploy {
if db, ok := db.(database.Deployer); ok {
db.RevertDeploy()
} else {
log.Fatal("database not deployable")
}
}
progress.Stop()