Compare commits
4 Commits
a1c7cc3d8d
...
7f8f7ded52
Author | SHA1 | Date |
---|---|---|
Vitaliy Filippov | 7f8f7ded52 | |
Vitaliy Filippov | 68553eabbb | |
Vitaliy Filippov | 3147c5c8d5 | |
Vitaliy Filippov | 576e2ae608 |
|
@ -97,6 +97,15 @@ func GetConnectionParams(params map[string]string) (map[string]string, error)
|
||||||
}
|
}
|
||||||
case []string:
|
case []string:
|
||||||
etcdUrl = config["etcd_address"].([]string)
|
etcdUrl = config["etcd_address"].([]string)
|
||||||
|
case []interface{}:
|
||||||
|
for _, url := range config["etcd_address"].([]interface{})
|
||||||
|
{
|
||||||
|
s, ok := url.(string)
|
||||||
|
if (ok)
|
||||||
|
{
|
||||||
|
etcdUrl = append(etcdUrl, s)
|
||||||
|
}
|
||||||
|
}
|
||||||
}
|
}
|
||||||
if (len(etcdUrl) == 0)
|
if (len(etcdUrl) == 0)
|
||||||
{
|
{
|
||||||
|
@ -105,8 +114,9 @@ func GetConnectionParams(params map[string]string) (map[string]string, error)
|
||||||
return ctxVars, nil
|
return ctxVars, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
func system(program string, args ...string) ([]byte, error)
|
func system(program string, args ...string) ([]byte, []byte, error)
|
||||||
{
|
{
|
||||||
|
klog.Infof("Running "+program+" "+strings.Join(args, " "))
|
||||||
c := exec.Command(program, args...)
|
c := exec.Command(program, args...)
|
||||||
var stdout, stderr bytes.Buffer
|
var stdout, stderr bytes.Buffer
|
||||||
c.Stdout, c.Stderr = &stdout, &stderr
|
c.Stdout, c.Stderr = &stdout, &stderr
|
||||||
|
@ -115,9 +125,9 @@ func system(program string, args ...string) ([]byte, error)
|
||||||
{
|
{
|
||||||
stdoutStr, stderrStr := string(stdout.Bytes()), string(stderr.Bytes())
|
stdoutStr, stderrStr := string(stdout.Bytes()), string(stderr.Bytes())
|
||||||
klog.Errorf(program+" "+strings.Join(args, " ")+" failed: %s, status %s\n", stdoutStr+stderrStr, err)
|
klog.Errorf(program+" "+strings.Join(args, " ")+" failed: %s, status %s\n", stdoutStr+stderrStr, err)
|
||||||
return nil, status.Error(codes.Internal, stdoutStr+stderrStr+" (status "+err.Error()+")")
|
return nil, nil, status.Error(codes.Internal, stdoutStr+stderrStr+" (status "+err.Error()+")")
|
||||||
}
|
}
|
||||||
return stdout.Bytes(), nil
|
return stdout.Bytes(), stderr.Bytes(), nil
|
||||||
}
|
}
|
||||||
|
|
||||||
func invokeCLI(ctxVars map[string]string, args []string) ([]byte, error)
|
func invokeCLI(ctxVars map[string]string, args []string) ([]byte, error)
|
||||||
|
@ -126,7 +136,8 @@ func invokeCLI(ctxVars map[string]string, args []string) ([]byte, error)
|
||||||
{
|
{
|
||||||
args = append(args, "--config_path", ctxVars["configPath"])
|
args = append(args, "--config_path", ctxVars["configPath"])
|
||||||
}
|
}
|
||||||
return system("/usr/bin/vitastor-cli", args...)
|
stdout, _, err := system("/usr/bin/vitastor-cli", args...)
|
||||||
|
return stdout, err
|
||||||
}
|
}
|
||||||
|
|
||||||
// Create the volume
|
// Create the volume
|
||||||
|
|
|
@ -7,6 +7,7 @@ import (
|
||||||
"context"
|
"context"
|
||||||
"errors"
|
"errors"
|
||||||
"encoding/json"
|
"encoding/json"
|
||||||
|
"fmt"
|
||||||
"os"
|
"os"
|
||||||
"os/exec"
|
"os/exec"
|
||||||
"path/filepath"
|
"path/filepath"
|
||||||
|
@ -154,8 +155,13 @@ func (ns *NodeServer) mapNbd(volName string, ctxVars map[string]string, readonly
|
||||||
{
|
{
|
||||||
args = append(args, "--readonly", "1")
|
args = append(args, "--readonly", "1")
|
||||||
}
|
}
|
||||||
dev, err := system("/usr/bin/vitastor-nbd", args...)
|
stdout, stderr, err := system("/usr/bin/vitastor-nbd", args...)
|
||||||
return strings.TrimSpace(string(dev)), err
|
dev := strings.TrimSpace(string(stdout))
|
||||||
|
if (dev == "")
|
||||||
|
{
|
||||||
|
return "", fmt.Errorf("vitastor-nbd did not return the name of NBD device. output: %s", stderr)
|
||||||
|
}
|
||||||
|
return dev, err
|
||||||
}
|
}
|
||||||
|
|
||||||
func (ns *NodeServer) unmapNbd(devicePath string)
|
func (ns *NodeServer) unmapNbd(devicePath string)
|
||||||
|
@ -221,7 +227,7 @@ func startStorageDaemon(vdpaId, volName, pidFile, configPath string, readonly bo
|
||||||
{
|
{
|
||||||
writable = "false"
|
writable = "false"
|
||||||
}
|
}
|
||||||
_, err := system(
|
_, _, err := system(
|
||||||
"/usr/bin/qemu-storage-daemon", "--daemonize", "--pidfile", pidFile, "--blockdev", string(blockSpecJson),
|
"/usr/bin/qemu-storage-daemon", "--daemonize", "--pidfile", pidFile, "--blockdev", string(blockSpecJson),
|
||||||
"--export", "vduse-blk,id="+vdpaId+",node-name=disk1,name="+vdpaId+",num-queues=16,queue-size=128,writable="+writable,
|
"--export", "vduse-blk,id="+vdpaId+",node-name=disk1,name="+vdpaId+",num-queues=16,queue-size=128,writable="+writable,
|
||||||
)
|
)
|
||||||
|
@ -234,7 +240,7 @@ func (ns *NodeServer) mapVduse(volName string, ctxVars map[string]string, readon
|
||||||
stateFd, err := os.CreateTemp(ns.stateDir, "vitastor-vduse-*.json")
|
stateFd, err := os.CreateTemp(ns.stateDir, "vitastor-vduse-*.json")
|
||||||
if (err != nil)
|
if (err != nil)
|
||||||
{
|
{
|
||||||
return "", "", status.Error(codes.Internal, err.Error())
|
return "", "", err
|
||||||
}
|
}
|
||||||
stateFile := stateFd.Name()
|
stateFile := stateFd.Name()
|
||||||
stateFd.Close()
|
stateFd.Close()
|
||||||
|
@ -246,7 +252,7 @@ func (ns *NodeServer) mapVduse(volName string, ctxVars map[string]string, readon
|
||||||
if (err == nil)
|
if (err == nil)
|
||||||
{
|
{
|
||||||
// Add device to VDPA bus
|
// Add device to VDPA bus
|
||||||
_, err = system("/sbin/vdpa", "-j", "dev", "add", "name", vdpaId, "mgmtdev", "vduse")
|
_, _, err = system("/sbin/vdpa", "-j", "dev", "add", "name", vdpaId, "mgmtdev", "vduse")
|
||||||
if (err == nil)
|
if (err == nil)
|
||||||
{
|
{
|
||||||
// Find block device name
|
// Find block device name
|
||||||
|
@ -277,21 +283,14 @@ func (ns *NodeServer) mapVduse(volName string, ctxVars map[string]string, readon
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
if (err != nil)
|
|
||||||
{
|
|
||||||
err = status.Error(codes.Internal, err.Error())
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
if (err != nil)
|
killErr := killByPidFile(pidFile)
|
||||||
|
if (killErr != nil)
|
||||||
{
|
{
|
||||||
killErr := killByPidFile(pidFile)
|
klog.Errorf("Failed to kill started qemu-storage-daemon: %v", killErr)
|
||||||
if (killErr != nil)
|
|
||||||
{
|
|
||||||
klog.Errorf("Failed to kill started qemu-storage-daemon: %v", killErr)
|
|
||||||
}
|
|
||||||
os.Remove(stateFile)
|
|
||||||
os.Remove(pidFile)
|
|
||||||
}
|
}
|
||||||
|
os.Remove(stateFile)
|
||||||
|
os.Remove(pidFile)
|
||||||
}
|
}
|
||||||
return "", "", err
|
return "", "", err
|
||||||
}
|
}
|
||||||
|
@ -337,7 +336,7 @@ func (ns *NodeServer) unmapVduseById(vdpaId string)
|
||||||
}
|
}
|
||||||
else
|
else
|
||||||
{
|
{
|
||||||
_, _ = system("/sbin/vdpa", "-j", "dev", "del", vdpaId)
|
_, _, _ = system("/sbin/vdpa", "-j", "dev", "del", vdpaId)
|
||||||
}
|
}
|
||||||
stateFile := ns.stateDir + vdpaId + ".json"
|
stateFile := ns.stateDir + vdpaId + ".json"
|
||||||
os.Remove(stateFile)
|
os.Remove(stateFile)
|
||||||
|
@ -377,7 +376,7 @@ func (ns *NodeServer) restoreVduseDaemons()
|
||||||
}
|
}
|
||||||
devList := make(map[string]interface{})
|
devList := make(map[string]interface{})
|
||||||
// example output: {"dev":{"test1":{"type":"block","mgmtdev":"vduse","vendor_id":0,"max_vqs":16,"max_vq_size":128}}}
|
// example output: {"dev":{"test1":{"type":"block","mgmtdev":"vduse","vendor_id":0,"max_vqs":16,"max_vq_size":128}}}
|
||||||
devListJSON, err := system("/sbin/vdpa", "-j", "dev", "list")
|
devListJSON, _, err := system("/sbin/vdpa", "-j", "dev", "list")
|
||||||
if (err != nil)
|
if (err != nil)
|
||||||
{
|
{
|
||||||
return
|
return
|
||||||
|
@ -456,13 +455,13 @@ func (ns *NodeServer) NodePublishVolume(ctx context.Context, req *csi.NodePublis
|
||||||
if (err != nil)
|
if (err != nil)
|
||||||
{
|
{
|
||||||
klog.Errorf("failed to create block device mount target %s with error: %v", targetPath, err)
|
klog.Errorf("failed to create block device mount target %s with error: %v", targetPath, err)
|
||||||
return nil, status.Error(codes.Internal, err.Error())
|
return nil, err
|
||||||
}
|
}
|
||||||
err = pathFile.Close()
|
err = pathFile.Close()
|
||||||
if (err != nil)
|
if (err != nil)
|
||||||
{
|
{
|
||||||
klog.Errorf("failed to close %s with error: %v", targetPath, err)
|
klog.Errorf("failed to close %s with error: %v", targetPath, err)
|
||||||
return nil, status.Error(codes.Internal, err.Error())
|
return nil, err
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
else
|
else
|
||||||
|
@ -471,13 +470,13 @@ func (ns *NodeServer) NodePublishVolume(ctx context.Context, req *csi.NodePublis
|
||||||
if (err != nil)
|
if (err != nil)
|
||||||
{
|
{
|
||||||
klog.Errorf("failed to create fs mount target %s with error: %v", targetPath, err)
|
klog.Errorf("failed to create fs mount target %s with error: %v", targetPath, err)
|
||||||
return nil, status.Error(codes.Internal, err.Error())
|
return nil, err
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
else
|
else
|
||||||
{
|
{
|
||||||
return nil, status.Error(codes.Internal, err.Error())
|
return nil, err
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -597,7 +596,7 @@ unmap:
|
||||||
{
|
{
|
||||||
ns.unmapVduseById(vdpaId)
|
ns.unmapVduseById(vdpaId)
|
||||||
}
|
}
|
||||||
return nil, status.Error(codes.Internal, err.Error())
|
return nil, err
|
||||||
}
|
}
|
||||||
|
|
||||||
// NodeUnpublishVolume unmounts the volume from the target path
|
// NodeUnpublishVolume unmounts the volume from the target path
|
||||||
|
@ -612,7 +611,7 @@ func (ns *NodeServer) NodeUnpublishVolume(ctx context.Context, req *csi.NodeUnpu
|
||||||
{
|
{
|
||||||
return nil, status.Error(codes.NotFound, "Target path not found")
|
return nil, status.Error(codes.NotFound, "Target path not found")
|
||||||
}
|
}
|
||||||
return nil, status.Error(codes.Internal, err.Error())
|
return nil, err
|
||||||
}
|
}
|
||||||
if (devicePath == "")
|
if (devicePath == "")
|
||||||
{
|
{
|
||||||
|
@ -625,7 +624,7 @@ func (ns *NodeServer) NodeUnpublishVolume(ctx context.Context, req *csi.NodeUnpu
|
||||||
err = mount.CleanupMountPoint(targetPath, ns.mounter, false)
|
err = mount.CleanupMountPoint(targetPath, ns.mounter, false)
|
||||||
if (err != nil)
|
if (err != nil)
|
||||||
{
|
{
|
||||||
return nil, status.Error(codes.Internal, err.Error())
|
return nil, err
|
||||||
}
|
}
|
||||||
// unmap NBD device
|
// unmap NBD device
|
||||||
if (refCount == 1)
|
if (refCount == 1)
|
||||||
|
|
Loading…
Reference in New Issue