diff --git a/tools/etcd-dump-logs/README.md b/tools/etcd-dump-logs/README.md new file mode 100644 index 000000000..b9bb1e7aa --- /dev/null +++ b/tools/etcd-dump-logs/README.md @@ -0,0 +1,129 @@ +### etcd-dump-logs + +etcd-dump-logs inspects etcd db files. + +``` +Usage: + + etcd-dump-logs [data dir] + * Data dir is where the snapshots and WAL logs are located. The structure of the data dir should look like this: + - data_dir/member + - data_dir/member/snap + - data_dir/member/wal + - data_dir/member/wal/0000000000000000-0000000000000000.wal + +Flags: + + -entry-type string + If set, filters output by entry type. Must be one or more than one of: + ConfigChange, Normal, Request, InternalRaftRequest, + IRRRange, IRRPut, IRRDeleteRange, IRRTxn, + IRRCompaction, IRRLeaseGrant, IRRLeaseRevoke + -start-index uint + The index to start dumping + -start-snap string + The base name of snapshot file to start dumping + -stream-decoder string + The name and arguments of an executable decoding tool, the executable + must process hex encoded lines of binary input (from etcd-dump-logs) + and output a hex encoded line of binary for each input line +``` +#### etcd-dump-logs -entry-type [data dir] + +Lists all the interested entries from WAL log. + +``` +$ etcd-dump-logs -entry-type IRRTxn /tmp/datadir +Snapshot: +empty +Start dupmping log entries from snapshot. +WAL metadata: +nodeID=0 clusterID=0 term=0 commitIndex=0 vote=0 +WAL entries: +lastIndex=34 +term index type data + 7 13 norm ID:8 txn: > failure: > > + +Entry types (IRRTxn) count is : 1 + +$ etcd-dump-logs -entry-type ConfigChange,IRRCompaction /tmp/datadir +Snapshot: +empty +Start dupmping log entries from snapshot. +WAL metadata: +nodeID=0 clusterID=0 term=0 commitIndex=0 vote=0 +WAL entries: +lastIndex=34 +term index type data + 1 1 conf method=ConfChangeAddNode id=2 + 2 2 conf method=ConfChangeRemoveNode id=2 + 2 3 conf method=ConfChangeUpdateNode id=2 + 2 4 conf method=ConfChangeAddLearnerNode id=3 + 8 14 norm ID:9 compaction: + +Entry types (ConfigChange,IRRCompaction) count is : 5 +``` +#### etcd-dump-logs -stream-decoder [data dir] + +Decode each entry based on logic in the passed decoder. Decoded status and decoded data are listed in separated tab/columns in the ouput. For parsing purpose, the output from decoder are expected to be in format of "| > failure: > > decoder output format is not right, print output anyway jhjhcbadabjhaajfjajafaabjafbaajhaajfjajafaabjafb + 8 5 norm ID:9 compaction: decoder output format is not right, print output anyway jhjicajbajja + 9 6 norm ID:10 lease_grant: decoder output format is not right, print output anyway jhjadbjdjhjaajja + 12 7 norm ID:13 auth_enable:<> decoder output format is not right, print output anyway jhjdcbcejj + 27 8 norm ??? decoder output format is not right, print output anyway cf +Entry types () count is : 8 + +$ etcd-dump-logs -stream-decoder decoder_wrongoutputformat.sh /tmp/datadir +Snapshot: +empty +Start dupmping log entries from snapshot. +WAL metadata: +nodeID=0 clusterID=0 term=0 commitIndex=0 vote=0 +WAL entries: +lastIndex=34 +term index type data decoder_status decoded_data + 1 1 conf method=ConfChangeAddNode id=2 ERROR jhjaajjjahjbbbjj + 3 2 norm noop OK jhjjabjjaajfbfgjfagdfhcjbbahgbbbfhfegibbcabbfhffbbbcbbfhfibbcaebbbgiffbbedgdbhjacbjjchjjdjjjdhjiejjjehjafjjjfhjjgjjjghjahjjajjhhjajj + 3 3 norm method=QGET path="/path1" OK jhjaabjdeadgdeedaajfbfgjfagdfhcabbacgbbbcjbbcabbcabbbcbbcbbbcaebbbccbbedgdbhjjcbjjchjjdjjjdhjiejjjehjafjjjfhjjgjjjghjahjjajjhhjajj + 7 4 norm ID:8 txn: > failure: > > OK jhjhcbadabjhaajfjajafaabjafbaajhaajfjajafaabjafb + 8 5 norm ID:9 compaction: ERROR jhjicajbajja + 9 6 norm ID:10 lease_grant: ERROR jhjadbjdjhjaajja + 12 7 norm ID:13 auth_enable:<> ERROR jhjdcbcejj + 27 8 norm ??? ERROR cf +Entry types () count is : 8 +``` +#### etcd-dump-logs -start-index [data dir] + +Only shows WAL log entries after the specified start-index number, exclusively. + +``` +$ etcd-dump-logs -start-index 30 /tmp/datadir +Start dumping log entries from index 30. +WAL metadata: +nodeID=0 clusterID=0 term=0 commitIndex=0 vote=0 +WAL entries: +lastIndex=34 +term index type data + 25 31 norm ID:26 auth_role_get: + 26 32 norm ID:27 auth_role_grant_permission: > + 27 33 norm ID:28 auth_role_revoke_permission: + 27 34 norm ??? +Entry types () count is : 4 +``` +[decoder_correctoutputformat.sh]: ./testdecoder/decoder_correctoutputformat.sh \ No newline at end of file diff --git a/tools/etcd-dump-logs/etcd-dump-log_test.go b/tools/etcd-dump-logs/etcd-dump-log_test.go index 7c2b506fc..ef0d49f2b 100644 --- a/tools/etcd-dump-logs/etcd-dump-log_test.go +++ b/tools/etcd-dump-logs/etcd-dump-log_test.go @@ -45,6 +45,9 @@ func TestEtcdDumpLogEntryType(t *testing.T) { t.Skipf("%q does not exist", dumpLogsBinary) } + decoder_correctoutputformat := filepath.Join(binDir, "/testdecoder/decoder_correctoutputformat.sh") + decoder_wrongoutputformat := filepath.Join(binDir, "/testdecoder/decoder_wrongoutputformat.sh") + p, err := ioutil.TempDir(os.TempDir(), "etcddumplogstest") if err != nil { t.Fatal(err) @@ -102,6 +105,8 @@ func TestEtcdDumpLogEntryType(t *testing.T) { {"lease grant entry-type", []string{"-entry-type", "IRRLeaseGrant", p}, "expectedoutput/listIRRLeaseGrant.output"}, {"lease revoke entry-type", []string{"-entry-type", "IRRLeaseRevoke", p}, "expectedoutput/listIRRLeaseRevoke.output"}, {"confchange and txn entry-type", []string{"-entry-type", "ConfigChange,IRRCompaction", p}, "expectedoutput/listConfigChangeIRRCompaction.output"}, + {"decoder_correctoutputformat", []string{"-stream-decoder", decoder_correctoutputformat, p}, "expectedoutput/decoder_correctoutputformat.output"}, + {"decoder_wrongoutputformat", []string{"-stream-decoder", decoder_wrongoutputformat, p}, "expectedoutput/decoder_wrongoutputformat.output"}, } for _, argtest := range argtests { diff --git a/tools/etcd-dump-logs/expectedoutput/decoder_correctoutputformat.output b/tools/etcd-dump-logs/expectedoutput/decoder_correctoutputformat.output new file mode 100644 index 000000000..edfbedb90 --- /dev/null +++ b/tools/etcd-dump-logs/expectedoutput/decoder_correctoutputformat.output @@ -0,0 +1,44 @@ +Snapshot: +empty +Start dupmping log entries from snapshot. +WAL metadata: +nodeID=0 clusterID=0 term=0 commitIndex=0 vote=0 +WAL entries: +lastIndex=34 +term index type data decoder_status decoded_data + 1 1 conf method=ConfChangeAddNode id=2 ERROR jhjaajjjahjbbbjj + 2 2 conf method=ConfChangeRemoveNode id=2 ERROR jhjbajjaahjbbbjj + 2 3 conf method=ConfChangeUpdateNode id=2 ERROR jhjcajjbahjbbbjj + 2 4 conf method=ConfChangeAddLearnerNode id=3 ERROR jhjdajjcahjcbbjj + 3 5 norm noop OK jhjjabjjaajfbfgjfagdfhcjbbahgbbbfhfegibbcabbfhffbbbcbbfhfibbcaebbbgiffbbedgdbhjacbjjchjjdjjjdhjiejjjehjafjjjfhjjgjjjghjahjjajjhhjajj + 3 6 norm method=QGET path="/path1" OK jhjaabjdeadgdeedaajfbfgjfagdfhcabbacgbbbcjbbcabbcabbbcbbcbbbcaebbbccbbedgdbhjjcbjjchjjdjjjdhjiejjjehjafjjjfhjjgjjjghjahjjajjhhjajj + 3 7 norm method=SYNC time="1969-12-31 16:00:00.000000001 -0800 PST" OK jhjbabjdeceidedcaajfbfgjfagdfhcbbbacgbbbcjbbcabbcabbbcbbcbbbcaebbbccbbedgdbhjjcbjjchjjdjjjdhjbejjjehjafjjjfhjjgjjjghjahjjajjhhjajj + 3 8 norm method=DELETE path="/path3" OK jhjcabjfdddedcdeeddeaajfbfgjfagdfhccbbahgbbbfhfegibbcabbfhffbbbcbbfhfibbcaebbbgiffbbedgdbhjjcbjjchjjdjjadhjbejjjehjafjjjfhjjgjjjghjahjjajjhhjajj + 3 9 norm method=RANDOM path="/path4/superlong/path/path/path/path/path/path/path/path/path/pa"..."path/path/path/path/path/path/path/path/path/path/path/path/path" val="{\"hey\":\"ho\",\"hi\":[\"yo\"]}" OK jhjdabjfebdadedddfddaaafjabfgjfagdfhcdbfgcgegjfegbfcfffefgbfgjfagdfhbfgjfagdfhbfgjfagdfhbfgjfagdfhbfgjfagdfhbfgjfagdfhbfgjfagdfhbfgjfagdfhbfgjfagdfhbfgjfagdfhbfgjfagdfhbfgjfagdfhbfgjfagdfhbfgjfagdfhbfgjfagdfhbfgjfagdfhbfgjfagdfhbfgjfagdfhbfgjfagdfhbfgjfagdfhbfgjfagdfhbfgjfagdfhbfgjfagdfhbfgjfagdfhbfgjfagdfhbfgjfagdfhbfgjfagdfhbfgjfagdfhbfgjfagdfhbfgjfagdfhbbahgbbbfhfegibbcabbfhffbbbcbbfhfibbcaebbbgiffbbedgdbhjjcbjjchjjdjjjdhjbejjjehjafjjjfhjjgjjjghjahjjajjhhjajj + 4 10 norm ID:5 range: OK jhjeaaaejajacaabjbfhfiahjfbjjabhjaehajicjafhajicja + 5 11 norm ID:6 put: OK jhjfbbajjajdffffffcaabjdfbfagbcaahjacjja + 6 12 norm ID:7 delete_range: OK jhjgbajhjajacjabjaciahja + 7 13 norm ID:8 txn: > failure: > > OK jhjhcbadabjhaajfjajafaabjafbaajhaajfjajafaabjafb + 8 14 norm ID:9 compaction: ERROR jhjicajbajja + 9 15 norm ID:10 lease_grant: ERROR jhjadbjdjhjaajja + 10 16 norm ID:11 lease_revoke: ERROR jhjbdajbjhjb + 11 17 norm ID:12 alarm: OK jhjcebjfjhjcajjdahje + 12 18 norm ID:13 auth_enable:<> ERROR jhjdcbcejj + 13 19 norm ID:14 auth_disable:<> ERROR jhjeiacfjj + 14 20 norm ID:15 authenticate: OK jhjfabcfaijajffdgifefafdfeabjhgjfagcgcggffgbfdaajegdfffbfefe + 15 21 norm ID:16 auth_user_add: OK jhajebddjejajefefafdfecaabjegjfagcgcca + 16 22 norm ID:17 auth_user_delete: OK jhaaeaddjgjajefefafdfeca + 17 23 norm ID:18 auth_user_get: OK jhabfbddjgjajefefafdfeca + 18 24 norm ID:19 auth_user_change_password: OK jhacfaddjejajefefafdfecaabjegjfagcgccb + 19 25 norm ID:20 auth_user_grant_role: OK jhadhbdejejajegegcfegbcaabjegbfffcfeca + 20 26 norm ID:21 auth_user_revoke_role: OK jhaehadejejajegegcfegbcbabjegbfffcfecb + 21 27 norm ID:22 auth_user_list:<> ERROR jhafibdejj + 22 28 norm ID:23 auth_role_list:<> ERROR jhagiadejj + 23 29 norm ID:24 auth_role_add: OK jhahhbdbjgjajegbfffcfecb + 24 30 norm ID:25 auth_role_delete: OK jhaihadbjgjajegbfffcfeca + 25 31 norm ID:26 auth_role_get: OK jhaaibdbjgjajegbfffcfecc + 26 32 norm ID:27 auth_role_grant_permission: > OK jhabiadbabjajegbfffcfeccababjhjaabjddbfegigcaajhebfafefgfedefefd + 27 33 norm ID:28 auth_role_revoke_permission: OK jhacabdbafjajegbfffcfeccabjcfbfegiaajhgbfafefgfefefefd + 27 34 norm ??? ERROR cf + +Entry types () count is : 34 \ No newline at end of file diff --git a/tools/etcd-dump-logs/expectedoutput/decoder_wrongoutputformat.output b/tools/etcd-dump-logs/expectedoutput/decoder_wrongoutputformat.output new file mode 100644 index 000000000..c76b3692b --- /dev/null +++ b/tools/etcd-dump-logs/expectedoutput/decoder_wrongoutputformat.output @@ -0,0 +1,44 @@ +Snapshot: +empty +Start dupmping log entries from snapshot. +WAL metadata: +nodeID=0 clusterID=0 term=0 commitIndex=0 vote=0 +WAL entries: +lastIndex=34 +term index type data decoder_status decoded_data + 1 1 conf method=ConfChangeAddNode id=2 decoder output format is not right, print output anyway jhjaajjjahjbbbjj + 2 2 conf method=ConfChangeRemoveNode id=2 decoder output format is not right, print output anyway jhjbajjaahjbbbjj + 2 3 conf method=ConfChangeUpdateNode id=2 decoder output format is not right, print output anyway jhjcajjbahjbbbjj + 2 4 conf method=ConfChangeAddLearnerNode id=3 decoder output format is not right, print output anyway jhjdajjcahjcbbjj + 3 5 norm noop decoder output format is not right, print output anyway jhjjabjjaajfbfgjfagdfhcjbbahgbbbfhfegibbcabbfhffbbbcbbfhfibbcaebbbgiffbbedgdbhjacbjjchjjdjjjdhjiejjjehjafjjjfhjjgjjjghjahjjajjhhjajj + 3 6 norm method=QGET path="/path1" decoder output format is not right, print output anyway jhjaabjdeadgdeedaajfbfgjfagdfhcabbacgbbbcjbbcabbcabbbcbbcbbbcaebbbccbbedgdbhjjcbjjchjjdjjjdhjiejjjehjafjjjfhjjgjjjghjahjjajjhhjajj + 3 7 norm method=SYNC time="1969-12-31 16:00:00.000000001 -0800 PST" decoder output format is not right, print output anyway jhjbabjdeceidedcaajfbfgjfagdfhcbbbacgbbbcjbbcabbcabbbcbbcbbbcaebbbccbbedgdbhjjcbjjchjjdjjjdhjbejjjehjafjjjfhjjgjjjghjahjjajjhhjajj + 3 8 norm method=DELETE path="/path3" decoder output format is not right, print output anyway jhjcabjfdddedcdeeddeaajfbfgjfagdfhccbbahgbbbfhfegibbcabbfhffbbbcbbfhfibbcaebbbgiffbbedgdbhjjcbjjchjjdjjadhjbejjjehjafjjjfhjjgjjjghjahjjajjhhjajj + 3 9 norm method=RANDOM path="/path4/superlong/path/path/path/path/path/path/path/path/path/pa"..."path/path/path/path/path/path/path/path/path/path/path/path/path" val="{\"hey\":\"ho\",\"hi\":[\"yo\"]}" decoder output format is not right, print output anyway jhjdabjfebdadedddfddaaafjabfgjfagdfhcdbfgcgegjfegbfcfffefgbfgjfagdfhbfgjfagdfhbfgjfagdfhbfgjfagdfhbfgjfagdfhbfgjfagdfhbfgjfagdfhbfgjfagdfhbfgjfagdfhbfgjfagdfhbfgjfagdfhbfgjfagdfhbfgjfagdfhbfgjfagdfhbfgjfagdfhbfgjfagdfhbfgjfagdfhbfgjfagdfhbfgjfagdfhbfgjfagdfhbfgjfagdfhbfgjfagdfhbfgjfagdfhbfgjfagdfhbfgjfagdfhbfgjfagdfhbfgjfagdfhbfgjfagdfhbfgjfagdfhbfgjfagdfhbbahgbbbfhfegibbcabbfhffbbbcbbfhfibbcaebbbgiffbbedgdbhjjcbjjchjjdjjjdhjbejjjehjafjjjfhjjgjjjghjahjjajjhhjajj + 4 10 norm ID:5 range: decoder output format is not right, print output anyway jhjeaaaejajacaabjbfhfiahjfbjjabhjaehajicjafhajicja + 5 11 norm ID:6 put: decoder output format is not right, print output anyway jhjfbbajjajdffffffcaabjdfbfagbcaahjacjja + 6 12 norm ID:7 delete_range: decoder output format is not right, print output anyway jhjgbajhjajacjabjaciahja + 7 13 norm ID:8 txn: > failure: > > decoder output format is not right, print output anyway jhjhcbadabjhaajfjajafaabjafbaajhaajfjajafaabjafb + 8 14 norm ID:9 compaction: decoder output format is not right, print output anyway jhjicajbajja + 9 15 norm ID:10 lease_grant: decoder output format is not right, print output anyway jhjadbjdjhjaajja + 10 16 norm ID:11 lease_revoke: decoder output format is not right, print output anyway jhjbdajbjhjb + 11 17 norm ID:12 alarm: decoder output format is not right, print output anyway jhjcebjfjhjcajjdahje + 12 18 norm ID:13 auth_enable:<> decoder output format is not right, print output anyway jhjdcbcejj + 13 19 norm ID:14 auth_disable:<> decoder output format is not right, print output anyway jhjeiacfjj + 14 20 norm ID:15 authenticate: decoder output format is not right, print output anyway jhjfabcfaijajffdgifefafdfeabjhgjfagcgcggffgbfdaajegdfffbfefe + 15 21 norm ID:16 auth_user_add: decoder output format is not right, print output anyway jhajebddjejajefefafdfecaabjegjfagcgcca + 16 22 norm ID:17 auth_user_delete: decoder output format is not right, print output anyway jhaaeaddjgjajefefafdfeca + 17 23 norm ID:18 auth_user_get: decoder output format is not right, print output anyway jhabfbddjgjajefefafdfeca + 18 24 norm ID:19 auth_user_change_password: decoder output format is not right, print output anyway jhacfaddjejajefefafdfecaabjegjfagcgccb + 19 25 norm ID:20 auth_user_grant_role: decoder output format is not right, print output anyway jhadhbdejejajegegcfegbcaabjegbfffcfeca + 20 26 norm ID:21 auth_user_revoke_role: decoder output format is not right, print output anyway jhaehadejejajegegcfegbcbabjegbfffcfecb + 21 27 norm ID:22 auth_user_list:<> decoder output format is not right, print output anyway jhafibdejj + 22 28 norm ID:23 auth_role_list:<> decoder output format is not right, print output anyway jhagiadejj + 23 29 norm ID:24 auth_role_add: decoder output format is not right, print output anyway jhahhbdbjgjajegbfffcfecb + 24 30 norm ID:25 auth_role_delete: decoder output format is not right, print output anyway jhaihadbjgjajegbfffcfeca + 25 31 norm ID:26 auth_role_get: decoder output format is not right, print output anyway jhaaibdbjgjajegbfffcfecc + 26 32 norm ID:27 auth_role_grant_permission: > decoder output format is not right, print output anyway jhabiadbabjajegbfffcfeccababjhjaabjddbfegigcaajhebfafefgfedefefd + 27 33 norm ID:28 auth_role_revoke_permission: decoder output format is not right, print output anyway jhacabdbafjajegbfffcfeccabjcfbfegiaajhgbfafefgfefefefd + 27 34 norm ??? decoder output format is not right, print output anyway cf + +Entry types () count is : 34 \ No newline at end of file diff --git a/tools/etcd-dump-logs/expectedoutput/listConfigChangeIRRLeaseRevoke.output b/tools/etcd-dump-logs/expectedoutput/listConfigChangeIRRLeaseRevoke.output deleted file mode 100644 index 0820848d3..000000000 --- a/tools/etcd-dump-logs/expectedoutput/listConfigChangeIRRLeaseRevoke.output +++ /dev/null @@ -1,11 +0,0 @@ -Snapshot: -empty -Start dupmping log entries from snapshot. -WAL metadata: -nodeID=0 clusterID=0 term=0 commitIndex=0 vote=0 -WAL entries: -lastIndex=34 -term index type data - 10 16 norm ID:11 lease_revoke: - -Entry types (IRRLeaseRevoke) count is : 1 \ No newline at end of file diff --git a/tools/etcd-dump-logs/main.go b/tools/etcd-dump-logs/main.go index 3a957e951..0fe2dca4f 100644 --- a/tools/etcd-dump-logs/main.go +++ b/tools/etcd-dump-logs/main.go @@ -15,9 +15,15 @@ package main import ( + "bufio" + "bytes" + "encoding/hex" "flag" "fmt" + "io" "log" + "os" + "os/exec" "path/filepath" "strings" "time" @@ -40,6 +46,9 @@ func main() { ConfigChange, Normal, Request, InternalRaftRequest, IRRRange, IRRPut, IRRDeleteRange, IRRTxn, IRRCompaction, IRRLeaseGrant, IRRLeaseRevoke`) + streamdecoder := flag.String("stream-decoder", "", `The name of an executable decoding tool, the executable must process + hex encoded lines of binary input (from etcd-dump-logs) + and output a hex encoded line of binary for each input line`) flag.Parse() @@ -101,8 +110,14 @@ func main() { fmt.Printf("WAL entries:\n") fmt.Printf("lastIndex=%d\n", ents[len(ents)-1].Index) - fmt.Printf("%4s\t%10s\ttype\tdata\n", "term", "index") - listEntriesType(*entrytype, ents) + + fmt.Printf("%4s\t%10s\ttype\tdata", "term", "index") + if *streamdecoder != "" { + fmt.Printf("\tdecoder_status\tdecoded_data") + } + fmt.Println() + + listEntriesType(*entrytype, *streamdecoder, ents) } func walDir(dataDir string) string { return filepath.Join(dataDir, "member", "wal") } @@ -203,12 +218,12 @@ type EntryPrinter func(e raftpb.Entry) func printInternalRaftRequest(entry raftpb.Entry) { var rr etcdserverpb.InternalRaftRequest if err := rr.Unmarshal(entry.Data); err == nil { - fmt.Printf("%4d\t%10d\tnorm\t%s\n", entry.Term, entry.Index, rr.String()) + fmt.Printf("%4d\t%10d\tnorm\t%s", entry.Term, entry.Index, rr.String()) } } func printUnknownNormal(entry raftpb.Entry) { - fmt.Printf("%4d\t%10d\tnorm\t???\n", entry.Term, entry.Index) + fmt.Printf("%4d\t%10d\tnorm\t???", entry.Term, entry.Index) } func printConfChange(entry raftpb.Entry) { @@ -216,9 +231,9 @@ func printConfChange(entry raftpb.Entry) { fmt.Printf("\tconf") var r raftpb.ConfChange if err := r.Unmarshal(entry.Data); err != nil { - fmt.Printf("\t???\n") + fmt.Printf("\t???") } else { - fmt.Printf("\tmethod=%s id=%s\n", r.Type, types.ID(r.NodeID)) + fmt.Printf("\tmethod=%s id=%s", r.Type, types.ID(r.NodeID)) } } @@ -228,13 +243,13 @@ func printRequest(entry raftpb.Entry) { fmt.Printf("%4d\t%10d\tnorm", entry.Term, entry.Index) switch r.Method { case "": - fmt.Printf("\tnoop\n") + fmt.Printf("\tnoop") case "SYNC": - fmt.Printf("\tmethod=SYNC time=%q\n", time.Unix(0, r.Time)) + fmt.Printf("\tmethod=SYNC time=%q", time.Unix(0, r.Time)) case "QGET", "DELETE": - fmt.Printf("\tmethod=%s path=%s\n", r.Method, excerpt(r.Path, 64, 64)) + fmt.Printf("\tmethod=%s path=%s", r.Method, excerpt(r.Path, 64, 64)) default: - fmt.Printf("\tmethod=%s path=%s val=%s\n", r.Method, excerpt(r.Path, 64, 64), excerpt(r.Val, 128, 0)) + fmt.Printf("\tmethod=%s path=%s val=%s", r.Method, excerpt(r.Path, 64, 64), excerpt(r.Val, 128, 0)) } } } @@ -281,13 +296,33 @@ IRRCompaction, IRRLeaseGrant, IRRLeaseRevoke`, et) } // listEntriesType filters and prints entries based on the entry-type flag, -func listEntriesType(entrytype string, ents []raftpb.Entry) { +func listEntriesType(entrytype string, streamdecoder string, ents []raftpb.Entry) { entryFilters := evaluateEntrytypeFlag(entrytype) printerMap := map[string]EntryPrinter{"InternalRaftRequest": printInternalRaftRequest, "Request": printRequest, "ConfigChange": printConfChange, "UnknownNormal": printUnknownNormal} + var stderr bytes.Buffer + args := strings.Split(streamdecoder, " ") + cmd := exec.Command(args[0], args[1:]...) + stdin, err := cmd.StdinPipe() + if err != nil { + log.Panic(err) + } + stdout, err := cmd.StdoutPipe() + if err != nil { + log.Panic(err) + } + cmd.Stderr = &stderr + if streamdecoder != "" { + err = cmd.Start() + if err != nil { + log.Panic(err) + } + } + cnt := 0 + for _, e := range ents { passed := false currtype := "" @@ -301,7 +336,55 @@ func listEntriesType(entrytype string, ents []raftpb.Entry) { if passed { printer := printerMap[currtype] printer(e) + if streamdecoder == "" { + fmt.Println() + continue + } + + // if decoder is set, pass the e.Data to stdin and read the stdout from decoder + io.WriteString(stdin, hex.EncodeToString(e.Data)) + io.WriteString(stdin, "\n") + outputReader := bufio.NewReader(stdout) + decoderoutput, currerr := outputReader.ReadString('\n') + if currerr != nil { + fmt.Println(currerr) + return + } + + decoder_status, decoded_data := parseDecoderOutput(decoderoutput) + + fmt.Printf("\t%s\t%s", decoder_status, decoded_data) } } + + stdin.Close() + err = cmd.Wait() + if streamdecoder != "" { + if err != nil { + log.Panic(err) + } + if stderr.String() != "" { + os.Stderr.WriteString("decoder stderr: " + stderr.String()) + } + } + fmt.Printf("\nEntry types (%s) count is : %d", entrytype, cnt) } + +func parseDecoderOutput(decoderoutput string) (string, string) { + var decoder_status string + var decoded_data string + output := strings.Split(decoderoutput, "|") + switch len(output) { + case 1: + decoder_status = "decoder output format is not right, print output anyway" + decoded_data = decoderoutput + case 2: + decoder_status = output[0] + decoded_data = output[1] + default: + decoder_status = output[0] + "(*WARNING: data might contain deliminator used by etcd-dump-logs)" + decoded_data = strings.Join(output[1:], "") + } + return decoder_status, decoded_data +} diff --git a/tools/etcd-dump-logs/testdecoder/decoder_correctoutputformat.sh b/tools/etcd-dump-logs/testdecoder/decoder_correctoutputformat.sh new file mode 100755 index 000000000..524c28b51 --- /dev/null +++ b/tools/etcd-dump-logs/testdecoder/decoder_correctoutputformat.sh @@ -0,0 +1,10 @@ +#!/bin/bash +while read line +do + LEN=$(echo ${#line}) + if [ $LEN -ge 20 ]; then + echo "OK|$line" | tr 1234567890 abcdefghij + else + echo "ERROR|$line" | tr 1234567890 abcdefghij + fi +done < "${1:-/dev/stdin}" diff --git a/tools/etcd-dump-logs/testdecoder/decoder_wrongoutputformat.sh b/tools/etcd-dump-logs/testdecoder/decoder_wrongoutputformat.sh new file mode 100755 index 000000000..9c71cf75e --- /dev/null +++ b/tools/etcd-dump-logs/testdecoder/decoder_wrongoutputformat.sh @@ -0,0 +1,5 @@ +#!/bin/bash +while read line +do + echo "$line" | tr 1234567890 abcdefghij +done < "${1:-/dev/stdin}"