optimize number of goroutines during reading
parent
2cce36c31f
commit
69d08caec1
|
@ -1,6 +1,7 @@
|
||||||
package reader
|
package reader
|
||||||
|
|
||||||
import (
|
import (
|
||||||
|
"math"
|
||||||
"os"
|
"os"
|
||||||
"runtime"
|
"runtime"
|
||||||
"strconv"
|
"strconv"
|
||||||
|
@ -34,11 +35,6 @@ func init() {
|
||||||
if os.Getenv("IMPOSM_SKIP_WAYS") != "" {
|
if os.Getenv("IMPOSM_SKIP_WAYS") != "" {
|
||||||
skipWays = true
|
skipWays = true
|
||||||
}
|
}
|
||||||
nParser = int64(runtime.NumCPU())
|
|
||||||
nWays = int64(runtime.NumCPU())
|
|
||||||
nRels = int64(runtime.NumCPU())
|
|
||||||
nNodes = int64(runtime.NumCPU())
|
|
||||||
nCoords = int64(runtime.NumCPU())
|
|
||||||
if procConf := os.Getenv("IMPOSM_READ_PROCS"); procConf != "" {
|
if procConf := os.Getenv("IMPOSM_READ_PROCS"); procConf != "" {
|
||||||
parts := strings.Split(procConf, ":")
|
parts := strings.Split(procConf, ":")
|
||||||
nParser, _ = strconv.ParseInt(parts[0], 10, 32)
|
nParser, _ = strconv.ParseInt(parts[0], 10, 32)
|
||||||
|
@ -46,8 +42,14 @@ func init() {
|
||||||
nWays, _ = strconv.ParseInt(parts[2], 10, 32)
|
nWays, _ = strconv.ParseInt(parts[2], 10, 32)
|
||||||
nNodes, _ = strconv.ParseInt(parts[3], 10, 32)
|
nNodes, _ = strconv.ParseInt(parts[3], 10, 32)
|
||||||
nCoords, _ = strconv.ParseInt(parts[3], 10, 32)
|
nCoords, _ = strconv.ParseInt(parts[3], 10, 32)
|
||||||
|
} else {
|
||||||
|
nParser, nRels, nWays, nNodes, nCoords = readersForCpus(runtime.NumCPU())
|
||||||
}
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func readersForCpus(cpus int) (int64, int64, int64, int64, int64) {
|
||||||
|
cpuf := float64(cpus)
|
||||||
|
return int64(math.Ceil(cpuf * 0.75)), int64(math.Ceil(cpuf * 0.25)), int64(math.Ceil(cpuf * 0.25)), int64(math.Ceil(cpuf * 0.25)), int64(math.Ceil(cpuf * 0.25))
|
||||||
}
|
}
|
||||||
|
|
||||||
func ReadPbf(cache *osmcache.OSMCache, progress *stats.Statistics,
|
func ReadPbf(cache *osmcache.OSMCache, progress *stats.Statistics,
|
||||||
|
|
|
@ -0,0 +1,32 @@
|
||||||
|
package reader
|
||||||
|
|
||||||
|
import (
|
||||||
|
"testing"
|
||||||
|
)
|
||||||
|
|
||||||
|
func TestReaderCpus(t *testing.T) {
|
||||||
|
p, r, w, n, c := readersForCpus(1)
|
||||||
|
if p != 1 && r != 1 && w != 1 && n != 1 && c != 1 {
|
||||||
|
t.Fatal(p, r, w, n, c)
|
||||||
|
}
|
||||||
|
|
||||||
|
p, r, w, n, c = readersForCpus(2)
|
||||||
|
if p != 2 && r != 1 && w != 1 && n != 1 && c != 1 {
|
||||||
|
t.Fatal(p, r, w, n, c)
|
||||||
|
}
|
||||||
|
|
||||||
|
p, r, w, n, c = readersForCpus(4)
|
||||||
|
if p != 3 && r != 1 && w != 1 && n != 1 && c != 1 {
|
||||||
|
t.Fatal(p, r, w, n, c)
|
||||||
|
}
|
||||||
|
|
||||||
|
p, r, w, n, c = readersForCpus(8)
|
||||||
|
if p != 6 && r != 2 && w != 2 && n != 2 && c != 2 {
|
||||||
|
t.Fatal(p, r, w, n, c)
|
||||||
|
}
|
||||||
|
|
||||||
|
p, r, w, n, c = readersForCpus(12)
|
||||||
|
if p != 8 && r != 3 && w != 3 && n != 3 && c != 3 {
|
||||||
|
t.Fatal(p, r, w, n, c)
|
||||||
|
}
|
||||||
|
}
|
Loading…
Reference in New Issue