ceph-gobench/ceph-gobench.go

package main

import (
	"bytes"
	"fmt"
	"github.com/fatih/color"
	"log"
	"math/rand"
	"sort"
	"strconv"
	"strings"
	"sync"
	"time"
)

//future feature
func makeoffsets(threads int64, bs int64, objsize int64) [][]int64 {
	var offsets [][]int64
	for i := int64(0); i < threads; i++ {
		s1 := rand.NewSource(i)
		r1 := rand.New(s1)
		localoffsets := make([]int64, 0, objsize-bs)
		for i := int64(0); i < objsize-bs; i += bs {
			localoffsets = append(localoffsets, i)
		}
		r1.Shuffle(len(localoffsets), func(i, j int) {
			localoffsets[i], localoffsets[j] = localoffsets[j], localoffsets[i]
		})
		offsets = append(offsets, localoffsets)
	}
	return offsets
}

func bench(cephconn *Cephconnection, osddevice Device, buffs *[][]byte, offset [][]int64, params *Params,
	wg *sync.WaitGroup, result chan string) {
	defer wg.Done()
	threadresult := make(chan []time.Duration, params.threadsCount)
	var objectnames []string
	var osdlatencies []time.Duration
	defer func() {
		for _, object := range objectnames {
			cephconn.ioctx.Delete(object)
		}
	}()
	// calculate object for each thread
	for suffix := 0; len(objectnames) < int(params.threadsCount); suffix++ {
		name := "bench_" + strconv.Itoa(suffix)
		if osddevice.ID == GetObjActingPrimary(cephconn, *params, name) {
			objectnames = append(objectnames, name)
		}
	}
	for i, j := 0, 0; i < int(params.threadsCount); i, j = i+1, j+2 {
		go bench_thread(cephconn, osddevice, (*buffs)[j:j+2], offset[i], params, threadresult, objectnames[i])
	}
	for i := int64(0); i < params.threadsCount; i++ {
		for _, lat := range <-threadresult {
			osdlatencies = append(osdlatencies, lat)
		}
	}
	close(threadresult)
	latencygrade := map[int]int{}
	for _, lat := range osdlatencies {
		switch {
		case lat < time.Millisecond*10:
			latencygrade[int(lat.Round(time.Millisecond).Nanoseconds()/1000000)]++
		case lat < time.Millisecond*20:
			latencygrade[int(lat.Round(time.Millisecond*5)/1000000)]++
		default:
			latencygrade[int(lat.Round(time.Millisecond*10)/1000000)]++
		}
	}
	var keys []int
	for k := range latencygrade {
		keys = append(keys, k)
	}
	sort.Ints(keys)
	var buffer bytes.Buffer
	yellow := color.New(color.FgHiYellow).SprintFunc()
	//red := color.New(color.FgHiRed).SprintFunc()
	green := color.New(color.FgHiGreen).SprintFunc()
	buffer.WriteString(fmt.Sprintf("Bench result for %v\n", osddevice.Name))
	infos := map[string]string{"front_addr": strings.Split(osddevice.Info.FrontAddr, "/")[0],
		"ceph_release/version": osddevice.Info.CephRelease + "/" + osddevice.Info.CephVersionShort, "cpu": osddevice.Info.CPU,
		"hostname": osddevice.Info.Hostname, "default_device_class": osddevice.Info.DefaultDeviceClass, "devices": osddevice.Info.Devices,
		"distro_description": osddevice.Info.DistroDescription, "journal_rotational": osddevice.Info.JournalRotational,
		"rotational": osddevice.Info.Rotational, "kernel_version": osddevice.Info.KernelVersion, "mem_swap_kb": osddevice.Info.MemSwapKb,
		"mem_total_kb": osddevice.Info.MemTotalKb, "osd_data": osddevice.Info.OsdData, "osd_objectstore": osddevice.Info.OsdObjectstore}
	infonum := 1
	var infokeys []string
	for k := range infos {
		infokeys = append(infokeys, k)
	}
	sort.Strings(infokeys)
	buffer.WriteString(fmt.Sprintf("%-30v %-45v", green("osdname"), yellow(osddevice.Name)))
	for _, key := range infokeys {
		infonum++
		buffer.WriteString(fmt.Sprintf("%-30v %-45v", green(key), yellow(infos[key])))
		if (infonum % 3) == 0 {
			buffer.WriteString("\n")
		}
	}

	for _, k := range keys {
		var blocks bytes.Buffer
		for i := 0; i < 50*(latencygrade[k]*100/len(osdlatencies))/100; i++ {
			blocks.WriteString("#")
		}
		iops := latencygrade[k] / int(params.duration.Seconds())
		avgspeed := (float64(latencygrade[k]) * float64(params.blocksize) / float64(params.duration.Seconds())) / 1024 / 1024 //mb/sec
		megabyteswritten := (float64(latencygrade[k]) * float64(params.blocksize)) / 1024 / 1024
		buffer.WriteString(fmt.Sprintf("%4v ms: [%-50v]    Count: %-5v    IOPS: %-5v    Avg speed: %-6.3f Mb/Sec    Summary written: %6.3f Mb\n",
			k, blocks.String(), latencygrade[k], iops, avgspeed, megabyteswritten))
	}
	result <- buffer.String()
}

func bench_thread(cephconn *Cephconnection, osddevice Device, buffs [][]byte, offsets []int64, params *Params,
	result chan []time.Duration, objname string) {

	starttime := time.Now()
	var latencies []time.Duration
	endtime := starttime.Add(params.duration)
	n := 0
	for {
		if time.Now().After(endtime) {
			break
		}
		for _, offset := range offsets {
			if time.Now().Before(endtime) {
				startwritetime := time.Now()
				err := cephconn.ioctx.Write(objname, buffs[n], uint64(offset))
				endwritetime := time.Now()
				if err != nil {
					log.Printf("Can't write obj: %v, osd: %v", objname, osddevice.Name)
					continue
				}
				latencies = append(latencies, endwritetime.Sub(startwritetime))
			} else {
				break
			}
		}
		if n == 0 {
			n++
		} else {
			n = 0
		}
	}
	result <- latencies
}

func main() {
	params := Route()
	cephconn := connectioninit(params)
	defer cephconn.conn.Shutdown()

	// https://tracker.ceph.com/issues/24114
	time.Sleep(time.Millisecond * 100)

	var buffs [][]byte
	for i := int64(0); i < 2*params.threadsCount; i++ {
		buffs = append(buffs, make([]byte, params.blocksize))
	}
	for num := range buffs {
		_, err := rand.Read(buffs[num])
		if err != nil {
			log.Fatalln(err)
		}
	}
	osddevices := GetOsds(cephconn, params)
	offsets := makeoffsets(params.threadsCount, params.blocksize, params.objectsize)

	var wg sync.WaitGroup
	results := make(chan string, len(osddevices)*int(params.threadsCount))
	for _, osd := range osddevices {
		wg.Add(1)
		if params.parallel == true {
			go bench(cephconn, osd, &buffs, offsets, &params, &wg, results)
		} else {
			bench(cephconn, osd, &buffs, offsets, &params, &wg, results)
			log.Println(<-results)
		}

	}

	if params.parallel == true {
		go func() {
			wg.Wait()
			close(results)
		}()

		for message := range results {
			for _, message := range message {
				log.Println(message)
			}

		}
	}

}