first diff download code

diff/cmd/process.go

@@ -41,7 +41,7 @@ func main() {
 }
 
 func update(oscFile string, conf *config.Config, force bool) {
-	state, err := diff.ParseState(oscFile)
+	state, err := diff.ParseStateFromOsc(oscFile)
 	if err != nil {
 		log.Fatal(err)
 	}

diff/download_test.go

@@ -0,0 +1,36 @@
+package diff
+
+import (
+	"testing"
+	"time"
+)
+
+func TestDiffPath(t *testing.T) {
+	if path := diffPath(0); path != "000/000/000" {
+		t.Fatal(path)
+	}
+	if path := diffPath(3069); path != "000/003/069" {
+		t.Fatal(path)
+	}
+	if path := diffPath(123456789); path != "123/456/789" {
+		t.Fatal(path)
+	}
+}
+
+// func _TestMissingDiffs(t *testing.T) {
+// 	download := &diffDownload{"http://planet.openstreetmap.org/replication/hour/", "/tmp/diffs", 0}
+// 	_, err := missingDiffs(time.Now().Add(-3*time.Hour), source)
+// 	if err != nil {
+// 		t.Fatal(err)
+// 	}
+// 	t.Fatal("")
+// }
+
+func TestDownloadDiffs(t *testing.T) {
+	diffs := &diffDownload{"http://planet.openstreetmap.org/replication/minute/", "/tmp/diffs", 0}
+	err := diffs.DownloadSince(time.Now().Add(-5 * time.Minute))
+	if err != nil {
+		t.Fatal(err)
+	}
+	t.Fatal("")
+}

diff/downloader.go

@@ -0,0 +1,138 @@
+package diff
+
+import (
+	"bytes"
+	"errors"
+	"fmt"
+	"io"
+	"io/ioutil"
+	"net/http"
+	"os"
+	"path"
+	"time"
+)
+
+// N = AAA*1000000 + BBB*1000 + CCC
+func diffPath(sequenceNumber int32) string {
+	c := sequenceNumber % 1000
+	b := sequenceNumber / 1000 % 1000
+	a := sequenceNumber / 1000000
+
+	return fmt.Sprintf("%03d/%03d/%03d", a, b, c)
+}
+
+type diffDownload struct {
+	url          string
+	dest         string
+	lastSequence int32
+}
+
+func NewDiffDownload(url, dest string) *diffDownload {
+	return &diffDownload{url, dest, 0}
+}
+
+func (d *diffDownload) downloadDiff(sequence int32) error {
+	dest := path.Join(d.dest, diffPath(sequence))
+	err := os.MkdirAll(path.Dir(dest), 0755)
+
+	if err != nil {
+		return err
+	}
+
+	out, err := os.Create(dest + ".osc.gz")
+	if err != nil {
+		return err
+	}
+	defer out.Close()
+
+	resp, err := http.Get(d.url + diffPath(sequence) + ".osc.gz")
+	if err != nil {
+		return err
+	}
+	defer resp.Body.Close()
+
+	if resp.StatusCode != 200 {
+		return errors.New(fmt.Sprintf("invalid repsonse: %v", resp))
+	}
+
+	_, err = io.Copy(out, resp.Body)
+	if err != nil {
+		return err
+	}
+
+	return nil
+}
+
+func (d *diffDownload) currentState() (*DiffState, error) {
+	resp, err := http.Get(d.url + "state.txt")
+	if err != nil {
+		return nil, err
+	}
+	if resp.StatusCode != 200 {
+		return nil, errors.New(fmt.Sprintf("invalid repsonse: %v", resp))
+	}
+	defer resp.Body.Close()
+	return parseState(resp.Body)
+}
+
+func (d *diffDownload) downloadState(sequence int32) (*DiffState, error) {
+	dest := path.Join(d.dest, diffPath(sequence))
+	err := os.MkdirAll(path.Dir(dest), 0755)
+	if err != nil {
+		return nil, err
+	}
+
+	resp, err := http.Get(d.url + diffPath(sequence) + ".state.txt")
+	if err != nil {
+		return nil, err
+	}
+	defer resp.Body.Close()
+
+	if resp.StatusCode != 200 {
+		return nil, errors.New(fmt.Sprintf("invalid repsonse: %v", resp))
+	}
+
+	buf := &bytes.Buffer{}
+	_, err = io.Copy(buf, resp.Body)
+	if err != nil {
+		return nil, err
+	}
+
+	err = ioutil.WriteFile(dest+".state.txt", buf.Bytes(), 0644)
+	if err != nil {
+		return nil, err
+	}
+
+	reader := bytes.NewReader(buf.Bytes())
+	return parseState(reader)
+}
+
+// func missingDiffs(since time.Time, source *diffDownload) ([]int, error) {
+// 	state, err := downloadState(source.url + "state.txt")
+// 	if err != nil {
+// 		return nil, err
+// 	}
+// 	for since.Before(state.Time) {
+// 		state, err = downloadState(source.url + diffPath(state.Sequence-1) + ".state.txt")
+// 		if err != nil {
+// 			return nil, err
+// 		}
+// 	}
+// 	return nil, nil
+// }
+
+func (d *diffDownload) DownloadSince(since time.Time) error {
+	state, err := d.currentState()
+	if err != nil {
+		return err
+	}
+
+	for since.Before(state.Time) {
+		state, err = d.downloadState(state.Sequence - 1)
+		fmt.Println(state)
+		if err != nil {
+			return err
+		}
+	}
+	return nil
+}

diff/state.go

@@ -5,6 +5,7 @@ import (
 	"errors"
 	"fmt"
 	"goposm/logging"
+	"io"
 	"os"
 	"path"
 	"strconv"
@@ -50,7 +51,7 @@ func WriteLastState(cacheDir string, state *DiffState) error {
 	return state.WriteToFile(stateFile)
 }
 
-func ParseState(oscFile string) (*DiffState, error) {
+func ParseStateFromOsc(oscFile string) (*DiffState, error) {
 	var stateFile string
 	if !strings.HasSuffix(oscFile, ".osc.gz") {
 		log.Warn("cannot read state file for non .osc.gz files")
@@ -62,11 +63,26 @@ func ParseState(oscFile string) (*DiffState, error) {
 		log.Warn("cannot find state file ", stateFile)
 		return nil, nil
 	}
+
+	f, err := os.Open(stateFile)
+	if err != nil {
+		return nil, err
+	}
+	defer f.Close()
 	return parseStateFile(stateFile)
 }
 
 func parseStateFile(stateFile string) (*DiffState, error) {
-	values, err := parseSimpleIni(stateFile)
+	f, err := os.Open(stateFile)
+	if err != nil {
+		return nil, err
+	}
+	defer f.Close()
+	return parseState(f)
+}
+
+func parseState(f io.Reader) (*DiffState, error) {
+	values, err := parseSimpleIni(f)
 	if err != nil {
 		return nil, err
 	}
@@ -92,13 +108,7 @@ func ParseLastState(cacheDir string) (*DiffState, error) {
 	return parseStateFile(stateFile)
 }
 
-func parseSimpleIni(file string) (map[string]string, error) {
-	f, err := os.Open(file)
-	if err != nil {
-		return nil, err
-	}
-	defer f.Close()
-
+func parseSimpleIni(f io.Reader) (map[string]string, error) {
 	result := make(map[string]string)
 
 	reader := bufio.NewScanner(f)