diff --git a/replication/source.go b/replication/source.go index 1e76563..188d5c8 100644 --- a/replication/source.go +++ b/replication/source.go @@ -92,6 +92,7 @@ func (d *downloader) Sequences() <-chan Sequence { func (d *downloader) download(seq int, ext string) error { dest := path.Join(d.dest, seqPath(seq)+ext) url := d.baseUrl + seqPath(seq) + ext + log.Print("Downloading diff file from ", url) if _, err := os.Stat(dest); err == nil { return nil @@ -121,11 +122,12 @@ func (d *downloader) download(seq int, ext string) error { defer resp.Body.Close() if resp.StatusCode == 404 { + log.Print("Remote file does not exist ", url) return NotAvailable } if resp.StatusCode != 200 { - return errors.New(fmt.Sprintf("invalid repsonse: %v", resp)) + return errors.New(fmt.Sprintf("invalid response: %v", resp)) } _, err = io.Copy(out, resp.Body) @@ -161,17 +163,19 @@ func (d *downloader) fetchNextLoop() { stateFile := path.Join(d.dest, seqPath(d.lastSequence)+d.stateExt) lastTime, err := d.stateTime(stateFile) for { + nextSeq := d.lastSequence + 1 + log.Print("Processing sequence ", nextSeq, err) if err == nil { nextDiffTime := lastTime.Add(d.interval) if nextDiffTime.After(time.Now()) { // we catched up and the next diff file is in the future. // wait till last diff time + interval, before fetching next - nextDiffTime = lastTime.Add(d.interval + 2*time.Second /* allow small time diff between server*/) + nextDiffTime = lastTime.Add(d.interval + 2*time.Second /* allow small time diff between servers */) waitFor := nextDiffTime.Sub(time.Now()) + log.Print("Next process in ", waitFor) time.Sleep(waitFor) } } - nextSeq := d.lastSequence + 1 // download will retry until they succeed d.downloadTillSuccess(nextSeq, d.stateExt) d.downloadTillSuccess(nextSeq, d.fileExt) diff --git a/update/run.go b/update/run.go index 4484e74..8055406 100644 --- a/update/run.go +++ b/update/run.go @@ -50,6 +50,8 @@ func Run() { log.Fatal("no replicationUrl in last.state.txt " + "or replication_url in -config file") } + logger.Print("Replication URL: " + replicationUrl) + logger.Print("Replication interval: ", config.BaseOptions.ReplicationInterval) downloader := replication.NewDiffDownloader( config.BaseOptions.DiffDir, diff --git a/update/state/state.go b/update/state/state.go index 615a9a8..b0ec965 100644 --- a/update/state/state.go +++ b/update/state/state.go @@ -186,7 +186,7 @@ func currentState(url string) (*DiffState, error) { return nil, err } if resp.StatusCode != 200 { - return nil, errors.New(fmt.Sprintf("invalid repsonse: %v", resp)) + return nil, errors.New(fmt.Sprintf("invalid response: %v", resp)) } defer resp.Body.Close() return Parse(resp.Body) @@ -195,7 +195,7 @@ func currentState(url string) (*DiffState, error) { func estimateSequence(url string, timestamp time.Time) int { state, err := currentState(url) if err != nil { - // try a second time befor failing + // try a second time before failing log.Warn("unable to fetch current state from ", url, ":", err, ", retry in 30s") time.Sleep(time.Second * 30) state, err = currentState(url)