vitalif
/
imposm3
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

Make run command more verbose

master
Yohan Boniface 2018-01-22 19:44:56 +01:00
parent c5be424c8c
commit cfb1be89d1
3 changed files with 11 additions and 5 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

10
replication/source.go
View File

@ -92,6 +92,7 @@ func (d *downloader) Sequences() <-chan Sequence {
func (d *downloader) download(seq int, ext string) error {
dest := path.Join(d.dest, seqPath(seq)+ext)
url := d.baseUrl + seqPath(seq) + ext
log.Print("Downloading diff file from ", url)
if _, err := os.Stat(dest); err == nil {
return nil
@ -121,11 +122,12 @@ func (d *downloader) download(seq int, ext string) error {
defer resp.Body.Close()
if resp.StatusCode == 404 {
log.Print("Remote file does not exist ", url)
return NotAvailable
}
if resp.StatusCode != 200 {
return errors.New(fmt.Sprintf("invalid repsonse: %v", resp))
return errors.New(fmt.Sprintf("invalid response: %v", resp))
}
_, err = io.Copy(out, resp.Body)
@ -161,17 +163,19 @@ func (d *downloader) fetchNextLoop() {
stateFile := path.Join(d.dest, seqPath(d.lastSequence)+d.stateExt)
lastTime, err := d.stateTime(stateFile)
for {
nextSeq := d.lastSequence + 1
log.Print("Processing sequence ", nextSeq, err)
if err == nil {
nextDiffTime := lastTime.Add(d.interval)
if nextDiffTime.After(time.Now()) {
// we catched up and the next diff file is in the future.
// wait till last diff time + interval, before fetching next
nextDiffTime = lastTime.Add(d.interval + 2*time.Second /* allow small time diff between server*/)
nextDiffTime = lastTime.Add(d.interval + 2*time.Second /* allow small time diff between servers */)
waitFor := nextDiffTime.Sub(time.Now())
log.Print("Next process in ", waitFor)
time.Sleep(waitFor)
}
}
nextSeq := d.lastSequence + 1
// download will retry until they succeed
d.downloadTillSuccess(nextSeq, d.stateExt)
d.downloadTillSuccess(nextSeq, d.fileExt)

2
update/run.go
View File

@ -50,6 +50,8 @@ func Run() {
log.Fatal("no replicationUrl in last.state.txt " +
"or replication_url in -config file")
}
logger.Print("Replication URL: " + replicationUrl)
logger.Print("Replication interval: ", config.BaseOptions.ReplicationInterval)
downloader := replication.NewDiffDownloader(
config.BaseOptions.DiffDir,

4
update/state/state.go
View File

@ -186,7 +186,7 @@ func currentState(url string) (*DiffState, error) {
return nil, err
}
if resp.StatusCode != 200 {
return nil, errors.New(fmt.Sprintf("invalid repsonse: %v", resp))
return nil, errors.New(fmt.Sprintf("invalid response: %v", resp))
}
defer resp.Body.Close()
return Parse(resp.Body)
@ -195,7 +195,7 @@ func currentState(url string) (*DiffState, error) {
func estimateSequence(url string, timestamp time.Time) int {
state, err := currentState(url)
if err != nil {
// try a second time befor failing
// try a second time before failing
log.Warn("unable to fetch current state from ", url, ":", err, ", retry in 30s")
time.Sleep(time.Second * 30)
state, err = currentState(url)