Make run command more verbose
parent
c5be424c8c
commit
cfb1be89d1
|
@ -92,6 +92,7 @@ func (d *downloader) Sequences() <-chan Sequence {
|
||||||
func (d *downloader) download(seq int, ext string) error {
|
func (d *downloader) download(seq int, ext string) error {
|
||||||
dest := path.Join(d.dest, seqPath(seq)+ext)
|
dest := path.Join(d.dest, seqPath(seq)+ext)
|
||||||
url := d.baseUrl + seqPath(seq) + ext
|
url := d.baseUrl + seqPath(seq) + ext
|
||||||
|
log.Print("Downloading diff file from ", url)
|
||||||
|
|
||||||
if _, err := os.Stat(dest); err == nil {
|
if _, err := os.Stat(dest); err == nil {
|
||||||
return nil
|
return nil
|
||||||
|
@ -121,11 +122,12 @@ func (d *downloader) download(seq int, ext string) error {
|
||||||
defer resp.Body.Close()
|
defer resp.Body.Close()
|
||||||
|
|
||||||
if resp.StatusCode == 404 {
|
if resp.StatusCode == 404 {
|
||||||
|
log.Print("Remote file does not exist ", url)
|
||||||
return NotAvailable
|
return NotAvailable
|
||||||
}
|
}
|
||||||
|
|
||||||
if resp.StatusCode != 200 {
|
if resp.StatusCode != 200 {
|
||||||
return errors.New(fmt.Sprintf("invalid repsonse: %v", resp))
|
return errors.New(fmt.Sprintf("invalid response: %v", resp))
|
||||||
}
|
}
|
||||||
|
|
||||||
_, err = io.Copy(out, resp.Body)
|
_, err = io.Copy(out, resp.Body)
|
||||||
|
@ -161,17 +163,19 @@ func (d *downloader) fetchNextLoop() {
|
||||||
stateFile := path.Join(d.dest, seqPath(d.lastSequence)+d.stateExt)
|
stateFile := path.Join(d.dest, seqPath(d.lastSequence)+d.stateExt)
|
||||||
lastTime, err := d.stateTime(stateFile)
|
lastTime, err := d.stateTime(stateFile)
|
||||||
for {
|
for {
|
||||||
|
nextSeq := d.lastSequence + 1
|
||||||
|
log.Print("Processing sequence ", nextSeq, err)
|
||||||
if err == nil {
|
if err == nil {
|
||||||
nextDiffTime := lastTime.Add(d.interval)
|
nextDiffTime := lastTime.Add(d.interval)
|
||||||
if nextDiffTime.After(time.Now()) {
|
if nextDiffTime.After(time.Now()) {
|
||||||
// we catched up and the next diff file is in the future.
|
// we catched up and the next diff file is in the future.
|
||||||
// wait till last diff time + interval, before fetching next
|
// wait till last diff time + interval, before fetching next
|
||||||
nextDiffTime = lastTime.Add(d.interval + 2*time.Second /* allow small time diff between server*/)
|
nextDiffTime = lastTime.Add(d.interval + 2*time.Second /* allow small time diff between servers */)
|
||||||
waitFor := nextDiffTime.Sub(time.Now())
|
waitFor := nextDiffTime.Sub(time.Now())
|
||||||
|
log.Print("Next process in ", waitFor)
|
||||||
time.Sleep(waitFor)
|
time.Sleep(waitFor)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
nextSeq := d.lastSequence + 1
|
|
||||||
// download will retry until they succeed
|
// download will retry until they succeed
|
||||||
d.downloadTillSuccess(nextSeq, d.stateExt)
|
d.downloadTillSuccess(nextSeq, d.stateExt)
|
||||||
d.downloadTillSuccess(nextSeq, d.fileExt)
|
d.downloadTillSuccess(nextSeq, d.fileExt)
|
||||||
|
|
|
@ -50,6 +50,8 @@ func Run() {
|
||||||
log.Fatal("no replicationUrl in last.state.txt " +
|
log.Fatal("no replicationUrl in last.state.txt " +
|
||||||
"or replication_url in -config file")
|
"or replication_url in -config file")
|
||||||
}
|
}
|
||||||
|
logger.Print("Replication URL: " + replicationUrl)
|
||||||
|
logger.Print("Replication interval: ", config.BaseOptions.ReplicationInterval)
|
||||||
|
|
||||||
downloader := replication.NewDiffDownloader(
|
downloader := replication.NewDiffDownloader(
|
||||||
config.BaseOptions.DiffDir,
|
config.BaseOptions.DiffDir,
|
||||||
|
|
|
@ -186,7 +186,7 @@ func currentState(url string) (*DiffState, error) {
|
||||||
return nil, err
|
return nil, err
|
||||||
}
|
}
|
||||||
if resp.StatusCode != 200 {
|
if resp.StatusCode != 200 {
|
||||||
return nil, errors.New(fmt.Sprintf("invalid repsonse: %v", resp))
|
return nil, errors.New(fmt.Sprintf("invalid response: %v", resp))
|
||||||
}
|
}
|
||||||
defer resp.Body.Close()
|
defer resp.Body.Close()
|
||||||
return Parse(resp.Body)
|
return Parse(resp.Body)
|
||||||
|
@ -195,7 +195,7 @@ func currentState(url string) (*DiffState, error) {
|
||||||
func estimateSequence(url string, timestamp time.Time) int {
|
func estimateSequence(url string, timestamp time.Time) int {
|
||||||
state, err := currentState(url)
|
state, err := currentState(url)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
// try a second time befor failing
|
// try a second time before failing
|
||||||
log.Warn("unable to fetch current state from ", url, ":", err, ", retry in 30s")
|
log.Warn("unable to fetch current state from ", url, ":", err, ", retry in 30s")
|
||||||
time.Sleep(time.Second * 30)
|
time.Sleep(time.Second * 30)
|
||||||
state, err = currentState(url)
|
state, err = currentState(url)
|
||||||
|
|
Loading…
Reference in New Issue