vitalif
/
imposm3
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

optimize number of goroutines during reading

master
Oliver Tonnhofer 2013-09-18 11:25:49 +02:00
parent 2cce36c31f
commit 69d08caec1
2 changed files with 39 additions and 5 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

12
reader/reader.go
View File

@ -1,6 +1,7 @@
package reader
import (
"math"
"os"
"runtime"
"strconv"
@ -34,11 +35,6 @@ func init() {
if os.Getenv("IMPOSM_SKIP_WAYS") != "" {
skipWays = true
}
nParser = int64(runtime.NumCPU())
nWays = int64(runtime.NumCPU())
nRels = int64(runtime.NumCPU())
nNodes = int64(runtime.NumCPU())
nCoords = int64(runtime.NumCPU())
if procConf := os.Getenv("IMPOSM_READ_PROCS"); procConf != "" {
parts := strings.Split(procConf, ":")
nParser, _ = strconv.ParseInt(parts[0], 10, 32)
@ -46,8 +42,14 @@ func init() {
nWays, _ = strconv.ParseInt(parts[2], 10, 32)
nNodes, _ = strconv.ParseInt(parts[3], 10, 32)
nCoords, _ = strconv.ParseInt(parts[3], 10, 32)
} else {
nParser, nRels, nWays, nNodes, nCoords = readersForCpus(runtime.NumCPU())
}
}
func readersForCpus(cpus int) (int64, int64, int64, int64, int64) {
cpuf := float64(cpus)
return int64(math.Ceil(cpuf * 0.75)), int64(math.Ceil(cpuf * 0.25)), int64(math.Ceil(cpuf * 0.25)), int64(math.Ceil(cpuf * 0.25)), int64(math.Ceil(cpuf * 0.25))
}
func ReadPbf(cache *osmcache.OSMCache, progress *stats.Statistics,

32
reader/reader_test.go Normal file
View File

@ -0,0 +1,32 @@
package reader
import (
"testing"
)
func TestReaderCpus(t *testing.T) {
p, r, w, n, c := readersForCpus(1)
if p != 1 && r != 1 && w != 1 && n != 1 && c != 1 {
t.Fatal(p, r, w, n, c)
}
p, r, w, n, c = readersForCpus(2)
if p != 2 && r != 1 && w != 1 && n != 1 && c != 1 {
t.Fatal(p, r, w, n, c)
}
p, r, w, n, c = readersForCpus(4)
if p != 3 && r != 1 && w != 1 && n != 1 && c != 1 {
t.Fatal(p, r, w, n, c)
}
p, r, w, n, c = readersForCpus(8)
if p != 6 && r != 2 && w != 2 && n != 2 && c != 2 {
t.Fatal(p, r, w, n, c)
}
p, r, w, n, c = readersForCpus(12)
if p != 8 && r != 3 && w != 3 && n != 3 && c != 3 {
t.Fatal(p, r, w, n, c)
}
}