From 0461fd56feac28419600aa89ab5387b6e904188d Mon Sep 17 00:00:00 2001 From: Oliver Tonnhofer Date: Wed, 30 Oct 2013 10:48:00 +0100 Subject: [PATCH] move import main function to import_ package --- import_/import.go | 253 +++++++++++++++++++++++++++++++++++++++++ imposm3.go | 278 +--------------------------------------------- 2 files changed, 258 insertions(+), 273 deletions(-) create mode 100644 import_/import.go diff --git a/import_/import.go b/import_/import.go new file mode 100644 index 0000000..55415de --- /dev/null +++ b/import_/import.go @@ -0,0 +1,253 @@ +/* +Package import_ provides the import sub command initial imports. +*/ +package import_ + +import ( + "path" + + "imposm3/cache" + "imposm3/config" + "imposm3/database" + _ "imposm3/database/postgis" + state "imposm3/diff/state" + "imposm3/geom/limit" + "imposm3/logging" + "imposm3/mapping" + "imposm3/parser/pbf" + "imposm3/reader" + "imposm3/stats" + "imposm3/writer" +) + +var log = logging.NewLogger("") + +func Import() { + if config.ImportOptions.Quiet { + logging.SetQuiet(true) + } + + if (config.ImportOptions.Write || config.ImportOptions.Read != "") && (config.ImportOptions.RevertDeploy || config.ImportOptions.RemoveBackup) { + log.Fatal("-revertdeploy and -removebackup not compatible with -read/-write") + } + + if config.ImportOptions.RevertDeploy && (config.ImportOptions.RemoveBackup || config.ImportOptions.DeployProduction) { + log.Fatal("-revertdeploy not compatible with -deployproduction/-removebackup") + } + + var geometryLimiter *limit.Limiter + if config.ImportOptions.Write && config.BaseOptions.LimitTo != "" { + var err error + step := log.StartStep("Reading limitto geometries") + geometryLimiter, err = limit.NewFromGeoJson(config.BaseOptions.LimitTo) + if err != nil { + log.Fatal(err) + } + log.StopStep(step) + } + + tagmapping, err := mapping.NewMapping(config.BaseOptions.MappingFile) + if err != nil { + log.Fatal("mapping file: ", err) + } + + var db database.DB + + if config.ImportOptions.Write || config.ImportOptions.DeployProduction || config.ImportOptions.RevertDeploy || config.ImportOptions.RemoveBackup || config.ImportOptions.Optimize { + if config.BaseOptions.Connection == "" { + log.Fatal("missing connection option") + } + conf := database.Config{ + ConnectionParams: config.BaseOptions.Connection, + Srid: config.BaseOptions.Srid, + } + db, err = database.Open(conf, tagmapping) + if err != nil { + log.Fatal(err) + } + } + + osmCache := cache.NewOSMCache(config.BaseOptions.CacheDir) + + if config.ImportOptions.Read != "" && osmCache.Exists() { + if config.ImportOptions.Overwritecache { + log.Printf("removing existing cache %s", config.BaseOptions.CacheDir) + err := osmCache.Remove() + if err != nil { + log.Fatal("unable to remove cache:", err) + } + } else if !config.ImportOptions.Appendcache { + log.Fatal("cache already exists use -appendcache or -overwritecache") + } + } + + step := log.StartStep("Imposm") + + var elementCounts *stats.ElementCounts + + if config.ImportOptions.Read != "" { + step := log.StartStep("Reading OSM data") + err = osmCache.Open() + if err != nil { + log.Fatal(err) + } + progress := stats.NewStatsReporter() + + pbfFile, err := pbf.Open(config.ImportOptions.Read) + if err != nil { + log.Fatal(err) + } + + osmCache.Coords.SetLinearImport(true) + reader.ReadPbf(osmCache, progress, tagmapping, pbfFile) + osmCache.Coords.SetLinearImport(false) + elementCounts = progress.Stop() + osmCache.Close() + log.StopStep(step) + if config.ImportOptions.Diff { + diffstate := state.FromPbf(pbfFile) + if diffstate != nil { + diffstate.WriteToFile(path.Join(config.BaseOptions.CacheDir, "last.state.txt")) + } + } + } + + if config.ImportOptions.Write { + stepImport := log.StartStep("Importing OSM data") + stepWrite := log.StartStep("Writing OSM data") + progress := stats.NewStatsReporterWithEstimate(elementCounts) + + err = db.Init() + if err != nil { + log.Fatal(err) + } + + bulkDb, ok := db.(database.BulkBeginner) + if ok { + err = bulkDb.BeginBulk() + } else { + err = db.Begin() + } + if err != nil { + log.Fatal(err) + } + + var diffCache *cache.DiffCache + if config.ImportOptions.Diff { + diffCache = cache.NewDiffCache(config.BaseOptions.CacheDir) + if err = diffCache.Remove(); err != nil { + log.Fatal(err) + } + if err = diffCache.Open(); err != nil { + log.Fatal(err) + } + } + + err = osmCache.Open() + if err != nil { + log.Fatal(err) + } + if diffCache != nil { + diffCache.Coords.SetLinearImport(true) + diffCache.Ways.SetLinearImport(true) + } + osmCache.Coords.SetReadOnly(true) + + relations := osmCache.Relations.Iter() + relWriter := writer.NewRelationWriter(osmCache, diffCache, relations, + db, progress, config.BaseOptions.Srid) + relWriter.SetLimiter(geometryLimiter) + relWriter.Start() + relWriter.Wait() // blocks till the Relations.Iter() finishes + osmCache.Relations.Close() + + ways := osmCache.Ways.Iter() + wayWriter := writer.NewWayWriter(osmCache, diffCache, ways, db, + progress, config.BaseOptions.Srid) + wayWriter.SetLimiter(geometryLimiter) + wayWriter.Start() + wayWriter.Wait() // blocks till the Ways.Iter() finishes + osmCache.Ways.Close() + + nodes := osmCache.Nodes.Iter() + nodeWriter := writer.NewNodeWriter(osmCache, nodes, db, + progress, config.BaseOptions.Srid) + nodeWriter.SetLimiter(geometryLimiter) + nodeWriter.Start() + nodeWriter.Wait() // blocks till the Nodes.Iter() finishes + osmCache.Close() + + err = db.End() + if err != nil { + log.Fatal(err) + } + + progress.Stop() + + if config.ImportOptions.Diff { + diffCache.Close() + } + + log.StopStep(stepWrite) + + if db, ok := db.(database.Generalizer); ok { + if err := db.Generalize(); err != nil { + log.Fatal(err) + } + } else { + log.Fatal("database not generalizeable") + } + + if db, ok := db.(database.Finisher); ok { + if err := db.Finish(); err != nil { + log.Fatal(err) + } + } else { + log.Fatal("database not finishable") + } + log.StopStep(stepImport) + } + + if config.ImportOptions.Optimize { + if db, ok := db.(database.Optimizer); ok { + if err := db.Optimize(); err != nil { + log.Fatal(err) + } + } else { + log.Fatal("database not optimizable") + } + } + + if config.ImportOptions.DeployProduction { + if db, ok := db.(database.Deployer); ok { + if err := db.Deploy(); err != nil { + log.Fatal(err) + } + } else { + log.Fatal("database not deployable") + } + } + + if config.ImportOptions.RevertDeploy { + if db, ok := db.(database.Deployer); ok { + if err := db.RevertDeploy(); err != nil { + log.Fatal(err) + } + } else { + log.Fatal("database not deployable") + } + } + + if config.ImportOptions.RemoveBackup { + if db, ok := db.(database.Deployer); ok { + if err := db.RemoveBackup(); err != nil { + log.Fatal(err) + } + } else { + log.Fatal("database not deployable") + } + } + + log.StopStep(step) + +} diff --git a/imposm3.go b/imposm3.go index bd29c71..493949e 100644 --- a/imposm3.go +++ b/imposm3.go @@ -4,26 +4,15 @@ import ( "fmt" golog "log" "os" - "path" "runtime" - "runtime/pprof" - "strings" - "time" - "imposm3/cache" "imposm3/cache/query" "imposm3/config" - "imposm3/database" - _ "imposm3/database/postgis" "imposm3/diff" - state "imposm3/diff/state" "imposm3/geom/limit" + "imposm3/import_" "imposm3/logging" - "imposm3/mapping" - "imposm3/parser/pbf" - "imposm3/reader" "imposm3/stats" - "imposm3/writer" ) var log = logging.NewLogger("") @@ -53,7 +42,10 @@ func main() { switch os.Args[1] { case "import": config.ParseImport(os.Args[2:]) - mainimport() + if config.ImportOptions.Httpprofile != "" { + stats.StartHttpPProf(config.ImportOptions.Httpprofile) + } + import_.Import() case "diff": config.ParseDiffImport(os.Args[2:]) @@ -87,263 +79,3 @@ func main() { os.Exit(0) } - -func mainimport() { - if config.ImportOptions.Cpuprofile != "" { - f, err := os.Create(config.ImportOptions.Cpuprofile) - if err != nil { - golog.Fatal(err) - } - pprof.StartCPUProfile(f) - defer pprof.StopCPUProfile() - } - - if config.ImportOptions.Httpprofile != "" { - stats.StartHttpPProf(config.ImportOptions.Httpprofile) - } - - if config.ImportOptions.Memprofile != "" { - parts := strings.Split(config.ImportOptions.Memprofile, string(os.PathListSeparator)) - var interval time.Duration - - if len(parts) < 2 { - interval, _ = time.ParseDuration("1m") - } else { - var err error - interval, err = time.ParseDuration(parts[1]) - if err != nil { - golog.Fatal(err) - } - } - - go stats.MemProfiler(parts[0], interval) - } - - if config.ImportOptions.Quiet { - logging.SetQuiet(true) - } - - if (config.ImportOptions.Write || config.ImportOptions.Read != "") && (config.ImportOptions.RevertDeploy || config.ImportOptions.RemoveBackup) { - log.Fatal("-revertdeploy and -removebackup not compatible with -read/-write") - } - - if config.ImportOptions.RevertDeploy && (config.ImportOptions.RemoveBackup || config.ImportOptions.DeployProduction) { - log.Fatal("-revertdeploy not compatible with -deployproduction/-removebackup") - } - - var geometryLimiter *limit.Limiter - if config.ImportOptions.Write && config.BaseOptions.LimitTo != "" { - var err error - step := log.StartStep("Reading limitto geometries") - geometryLimiter, err = limit.NewFromGeoJson(config.BaseOptions.LimitTo) - if err != nil { - log.Fatal(err) - } - log.StopStep(step) - } - - tagmapping, err := mapping.NewMapping(config.BaseOptions.MappingFile) - if err != nil { - log.Fatal("mapping file: ", err) - } - - var db database.DB - - if config.ImportOptions.Write || config.ImportOptions.DeployProduction || config.ImportOptions.RevertDeploy || config.ImportOptions.RemoveBackup || config.ImportOptions.Optimize { - if config.BaseOptions.Connection == "" { - log.Fatal("missing connection option") - } - conf := database.Config{ - ConnectionParams: config.BaseOptions.Connection, - Srid: config.BaseOptions.Srid, - } - db, err = database.Open(conf, tagmapping) - if err != nil { - log.Fatal(err) - } - } - - osmCache := cache.NewOSMCache(config.BaseOptions.CacheDir) - - if config.ImportOptions.Read != "" && osmCache.Exists() { - if config.ImportOptions.Overwritecache { - log.Printf("removing existing cache %s", config.BaseOptions.CacheDir) - err := osmCache.Remove() - if err != nil { - log.Fatal("unable to remove cache:", err) - } - } else if !config.ImportOptions.Appendcache { - log.Fatal("cache already exists use -appendcache or -overwritecache") - } - } - - step := log.StartStep("Imposm") - - var elementCounts *stats.ElementCounts - - if config.ImportOptions.Read != "" { - step := log.StartStep("Reading OSM data") - err = osmCache.Open() - if err != nil { - log.Fatal(err) - } - progress := stats.NewStatsReporter() - - pbfFile, err := pbf.Open(config.ImportOptions.Read) - if err != nil { - log.Fatal(err) - } - - osmCache.Coords.SetLinearImport(true) - reader.ReadPbf(osmCache, progress, tagmapping, pbfFile) - osmCache.Coords.SetLinearImport(false) - elementCounts = progress.Stop() - osmCache.Close() - log.StopStep(step) - if config.ImportOptions.Diff { - diffstate := state.FromPbf(pbfFile) - if diffstate != nil { - diffstate.WriteToFile(path.Join(config.BaseOptions.CacheDir, "last.state.txt")) - } - } - } - - if config.ImportOptions.Write { - stepImport := log.StartStep("Importing OSM data") - stepWrite := log.StartStep("Writing OSM data") - progress := stats.NewStatsReporterWithEstimate(elementCounts) - - err = db.Init() - if err != nil { - log.Fatal(err) - } - - bulkDb, ok := db.(database.BulkBeginner) - if ok { - err = bulkDb.BeginBulk() - } else { - err = db.Begin() - } - if err != nil { - log.Fatal(err) - } - - var diffCache *cache.DiffCache - if config.ImportOptions.Diff { - diffCache = cache.NewDiffCache(config.BaseOptions.CacheDir) - if err = diffCache.Remove(); err != nil { - log.Fatal(err) - } - if err = diffCache.Open(); err != nil { - log.Fatal(err) - } - } - - err = osmCache.Open() - if err != nil { - log.Fatal(err) - } - if diffCache != nil { - diffCache.Coords.SetLinearImport(true) - diffCache.Ways.SetLinearImport(true) - } - osmCache.Coords.SetReadOnly(true) - - relations := osmCache.Relations.Iter() - relWriter := writer.NewRelationWriter(osmCache, diffCache, relations, - db, progress, config.BaseOptions.Srid) - relWriter.SetLimiter(geometryLimiter) - relWriter.Start() - relWriter.Wait() // blocks till the Relations.Iter() finishes - osmCache.Relations.Close() - - ways := osmCache.Ways.Iter() - wayWriter := writer.NewWayWriter(osmCache, diffCache, ways, db, - progress, config.BaseOptions.Srid) - wayWriter.SetLimiter(geometryLimiter) - wayWriter.Start() - wayWriter.Wait() // blocks till the Ways.Iter() finishes - osmCache.Ways.Close() - - nodes := osmCache.Nodes.Iter() - nodeWriter := writer.NewNodeWriter(osmCache, nodes, db, - progress, config.BaseOptions.Srid) - nodeWriter.SetLimiter(geometryLimiter) - nodeWriter.Start() - nodeWriter.Wait() // blocks till the Nodes.Iter() finishes - osmCache.Close() - - err = db.End() - if err != nil { - log.Fatal(err) - } - - progress.Stop() - - if config.ImportOptions.Diff { - diffCache.Close() - } - - log.StopStep(stepWrite) - - if db, ok := db.(database.Generalizer); ok { - if err := db.Generalize(); err != nil { - log.Fatal(err) - } - } else { - log.Fatal("database not generalizeable") - } - - if db, ok := db.(database.Finisher); ok { - if err := db.Finish(); err != nil { - log.Fatal(err) - } - } else { - log.Fatal("database not finishable") - } - log.StopStep(stepImport) - } - - if config.ImportOptions.Optimize { - if db, ok := db.(database.Optimizer); ok { - if err := db.Optimize(); err != nil { - log.Fatal(err) - } - } else { - log.Fatal("database not optimizable") - } - } - - if config.ImportOptions.DeployProduction { - if db, ok := db.(database.Deployer); ok { - if err := db.Deploy(); err != nil { - log.Fatal(err) - } - } else { - log.Fatal("database not deployable") - } - } - - if config.ImportOptions.RevertDeploy { - if db, ok := db.(database.Deployer); ok { - if err := db.RevertDeploy(); err != nil { - log.Fatal(err) - } - } else { - log.Fatal("database not deployable") - } - } - - if config.ImportOptions.RemoveBackup { - if db, ok := db.(database.Deployer); ok { - if err := db.RemoveBackup(); err != nil { - log.Fatal(err) - } - } else { - log.Fatal("database not deployable") - } - } - - log.StopStep(step) - -}