ceph-gobench/ceph-gobench.go

package main

import (
	"bytes"
	"fmt"
	"github.com/fatih/color"
	"log"
	"math/rand"
	"os"
	"runtime"
	"runtime/pprof"
	"sort"
	"strconv"
	"strings"
	"sync"
	"time"
)

func bench(cephconn *cephconnection, osddevice Device, buff *[]byte, startbuff *[]byte, params *params,
	wg *sync.WaitGroup, result chan string) {
	defer wg.Done()
	threadresult := make(chan []time.Duration, params.threadsCount)
	var objectnames []string
	var osdlatencies []time.Duration
	defer func() {
		for _, object := range objectnames {
			cephconn.ioctx.Delete(object)
		}
	}()
	// calculate object for each thread
	for suffix := 0; len(objectnames) < int(params.threadsCount)*16; suffix++ {
		name := "bench_" + strconv.Itoa(suffix)
		if osddevice.ID == getObjActingPrimary(cephconn, *params, name) {
			objectnames = append(objectnames, name)
			if err := cephconn.ioctx.WriteFull(name, *startbuff); err != nil {
				log.Printf("Can't write object: %v, osd: %v", name, osddevice.Name)
			}
			if err := cephconn.ioctx.Truncate(name, uint64(params.objectsize)); err != nil {
				log.Printf("Can't truncate object: %v, osd: %v", name, osddevice.Name)
			}
		}
	}
	for i := 0; i < int(params.threadsCount); i++ {
		go benchthread(cephconn, osddevice, params, buff, threadresult, objectnames[i*16:i*16+16])
	}
	for i := uint64(0); i < params.threadsCount; i++ {
		for _, lat := range <-threadresult {
			osdlatencies = append(osdlatencies, lat)
		}
	}
	close(threadresult)
	latencygrade := map[int64]int{}
	latencytotal := int64(0)
	for _, lat := range osdlatencies {
		micro := lat.Nanoseconds() / 1000
		rounded := micro
		switch {
		case micro < 1000: // 0-1ms round to 0.1ms
			rounded = (micro / 100) * 100
		case micro < 10000: // 2-10ms round to 1ms
			rounded = (micro / 1000) * 1000
		case micro < 100000: // 10-100ms round to 10ms
			rounded = (micro / 10000) * 10000
		case micro < 1000000: // 100-1000ms round to 100ms
			rounded = (micro / 100000) * 100000
		default: // 1000+ms round to 1s
			rounded = (micro / 1000000) * 1000000
		}
		latencytotal += micro
		latencygrade[rounded]++
	}

	var buffer bytes.Buffer

	//color info
	yellow := color.New(color.FgHiYellow).SprintFunc()
	red := color.New(color.FgHiRed).SprintFunc()
	darkred := color.New(color.FgRed).SprintFunc()
	green := color.New(color.FgHiGreen).SprintFunc()
	buffer.WriteString(fmt.Sprintf("Bench result for %v\n", osddevice.Name))
	infos := map[string]string{
		"front_addr":           strings.Split(osddevice.Info.FrontAddr, "/")[0],
		"ceph_release/version": osddevice.Info.CephRelease + "/" + osddevice.Info.CephVersionShort,
		"cpu":                  osddevice.Info.CPU,
		"hostname":             osddevice.Info.Hostname,
		"default_device_class": osddevice.Info.DefaultDeviceClass,
		"devices":              osddevice.Info.Devices,
		"distro_description":   osddevice.Info.DistroDescription,
		"journal_rotational":   osddevice.Info.JournalRotational,
		"rotational":           osddevice.Info.Rotational,
		"kernel_version":       osddevice.Info.KernelVersion,
		"mem_swap_kb":          osddevice.Info.MemSwapKb,
		"mem_total_kb":         osddevice.Info.MemTotalKb,
		"osd_data":             osddevice.Info.OsdData,
		"osd_objectstore":      osddevice.Info.OsdObjectstore,
	}
	var infokeys []string
	width := []int{0, 0, 0, 0, 0, 0}
	for k := range infos {
		infokeys = append(infokeys, k)
	}
	sort.Strings(infokeys)
	for n, key := range infokeys {
		if width[n%3] < len(key) {
			width[n%3] = len(key)
		}
		if width[3+n%3] < len(infos[key]) {
			width[3+n%3] = len(infos[key])
		}
	}
	buffer.WriteString(
		green("osdname") + strings.Repeat(" ", width[2]-len("osdname")+2) +
			red(osddevice.Name) + strings.Repeat(" ", width[5]-len(osddevice.Name)+2))
	for infonum, key := range infokeys {
		if (infonum % 3) == 2 {
			buffer.WriteString("\n")
		}
		buffer.WriteString(
			green(key) + strings.Repeat(" ", width[infonum%3]-len(key)+2) +
				yellow(infos[key]) + strings.Repeat(" ", width[3+infonum%3]-len(infos[key])+2))
	}
	buffer.WriteString("\n\n")

	latencytotal = latencytotal / int64(len(osdlatencies))
	// iops = 1s / latency
	iops := 1000000 / latencytotal * int64(params.threadsCount)
	// avg speed = iops * block size / 1 MB
	avgspeed := 1000000 / float64(latencytotal) * float64(params.blocksize) / 1024 / 1024 * float64(params.threadsCount)
	avgline := fmt.Sprintf("Avg iops: %-5v    Avg speed: %.3f MB/s    Total writes count: %-5v    Total writes (MB): %-5v\n\n",
		iops, avgspeed, len(osdlatencies), uint64(len(osdlatencies))*params.blocksize/1024/1024)
	switch {
	case iops < 80:
		buffer.WriteString(darkred(avgline))
	case iops < 200:
		buffer.WriteString(red(avgline))
	case iops < 500:
		buffer.WriteString(yellow(avgline))
	default:
		buffer.WriteString(green(avgline))
	}

	//sort latencies
	var keys []int64
	for k := range latencygrade {
		keys = append(keys, k)
	}
	sort.Slice(keys, func(i, j int) bool { return keys[i] < keys[j] })
	for _, k := range keys {
		var blocks bytes.Buffer
		var mseconds string
		switch {
		case k < 1000:
			mseconds = green(fmt.Sprintf("[%.1f-%.1f)", float64(k)/1000, 0.1+float64(k)/1000))
		case k < 2000:
			mseconds = yellow(fmt.Sprintf("[%.1f-%.1f)", float64(k)/1000, 0.1+float64(k)/1000))
		case k < 9000:
			mseconds = yellow(fmt.Sprintf("[%.1f-%.1f)", float64(k/1000), float64(1+k/1000)))
		case k < 10000:
			mseconds = color.YellowString(fmt.Sprintf("[%.1f-%v)", float64(k/1000), 1+k/1000))
		case k < 100000:
			mseconds = red(fmt.Sprintf("[%v-%v)", k/1000, 10+k/1000))
		case k < 1000000:
			mseconds = darkred(fmt.Sprintf("[%v-%v]", k/1000, 99+k/1000))
		default:
			mseconds = darkred(fmt.Sprintf("[%vs-%vs]", k/1000000, 1+k/1000000))
		}
		for i := 0; i < 50*(latencygrade[k]*100/len(osdlatencies))/100; i++ {
			blocks.WriteString("#")
		}
		megabyteswritten := (float64(latencygrade[k]) * float64(params.blocksize)) / 1024 / 1024
		buffer.WriteString(fmt.Sprintf("%-20v ms: [%-50v]    Count: %-5v    Total written: %6.3f MB\n",
			mseconds, blocks.String(), latencygrade[k], megabyteswritten))
	}
	result <- buffer.String()
}

func benchthread(cephconn *cephconnection, osddevice Device, params *params, buff *[]byte,
	result chan []time.Duration, objnames []string) {
	var latencies []time.Duration
	starttime := time.Now()
	endtime := starttime.Add(params.duration)
	for {
		offset := rand.Int63n(int64(params.objectsize/params.blocksize)) * int64(params.blocksize)
		objname := objnames[rand.Int31n(int32(len(objnames)))]
		startwritetime := time.Now()
		if startwritetime.After(endtime) {
			break
		}
		err := cephconn.ioctx.Write(objname, *buff, uint64(offset))
		endwritetime := time.Now()
		if err != nil {
			log.Printf("Can't write object: %v, osd: %v", objname, osddevice.Name)
			continue
		}
		latencies = append(latencies, endwritetime.Sub(startwritetime))
	}
	result <- latencies
}

func main() {
	params := route()
	if params.cpuprofile != "" {
		f, err := os.Create(params.cpuprofile)
		if err != nil {
			log.Fatal("could not create CPU profile: ", err)
		}
		defer f.Close()
		if err := pprof.StartCPUProfile(f); err != nil {
			log.Fatal("could not start CPU profile: ", err)
		}
		defer pprof.StopCPUProfile()
	}

	if params.memprofile != "" {
		f, err := os.Create(params.memprofile)
		if err != nil {
			log.Fatal("could not create memory profile: ", err)
		}
		defer f.Close()
		runtime.GC() // get up-to-date statistics
		if err := pprof.WriteHeapProfile(f); err != nil {
			log.Fatal("could not write memory profile: ", err)
		}
	}
	cephconn := connectioninit(params)
	defer cephconn.conn.Shutdown()

	// https://tracker.ceph.com/issues/24114
	time.Sleep(time.Millisecond * 100)

	startbuff := make([]byte, 4096)
	osddevices := getOsds(cephconn, params)
	buff := make([]byte, params.blocksize)
	rand.Read(buff)

	var wg sync.WaitGroup
	results := make(chan string, len(osddevices)*int(params.threadsCount))
	for _, osd := range osddevices {
		wg.Add(1)
		if params.parallel == true {
			go bench(cephconn, osd, &buff, &startbuff, &params, &wg, results)
		} else {
			bench(cephconn, osd, &buff, &startbuff, &params, &wg, results)
			log.Println(<-results)
		}

	}

	if params.parallel == true {
		go func() {
			wg.Wait()
			close(results)
		}()

		for message := range results {
			log.Println(message)

		}
	}

}