Skip to content

Commit

Permalink
Add Block volume support for CSI provisioner
Browse files Browse the repository at this point in the history 
In CSI provisioner, below three logics need to be implemented,
to add Block volume support to CSI provisioner:

  1. Add SupportsBlock that properly returns whether Storage
     Provider's plugin supports block (this is checked
     by using ValidateVolumeCapabilities),
  2. Pass BlockVolume instead of MountVolume to CreateVolume
     if volumeMode is set to be Block on Provision,
  3. Set volumeMode to PV returned by Provision.

Also, below 4 test cases for TestSupportsBlock and 2 test cases for TestProvision
are added.

TestSupportsBlock:
 1. ValidateVolumeCapabilities return (true, nil) case: return true expected
 2. ValidateVolumeCapabilities return (false, nil) case: return false expected
 3. ValidateVolumeCapabilities return (true, err) case: return false expected
 4. ValidateVolumeCapabilities return (false, err) case: return false expected

TestProvision:
 1. volumeMode=Filesystem PVC case: return Filesystem PV expected
 2. volumeMode=Block PVC case: return Block PV expected

Fixes kubernetes-csi#110
  • Loading branch information
@mkimuram
mkimuram committed Sep 13, 2018
1 parent 7ffa3dd commit da09b7c
Show file tree
Hide file tree
Showing 2 changed files with 197 additions and 25 deletions.
112 changes: 88 additions & 24 deletions pkg/controller/controller.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -31,6 +31,7 @@ import (
"github.com/golang/glog"

"github.com/kubernetes-incubator/external-storage/lib/controller"
"github.com/kubernetes-incubator/external-storage/lib/util"

snapapi "github.com/kubernetes-csi/external-snapshotter/pkg/apis/volumesnapshot/v1alpha1"
snapclientset "github.com/kubernetes-csi/external-snapshotter/pkg/client/clientset/versioned"
Expand Down Expand Up @@ -93,9 +94,48 @@ type csiProvisioner struct {
var _ controller.Provisioner = &csiProvisioner{}

var (
accessType = &csi.VolumeCapability_Mount{
// AccessType
accessTypeMount = &csi.VolumeCapability_Mount{
Mount: &csi.VolumeCapability_MountVolume{},
}
accessTypeBlock = &csi.VolumeCapability_Block{
Block: &csi.VolumeCapability_BlockVolume{},
}
// AccessMode
accessModeReadWriteOnce = &csi.VolumeCapability_AccessMode{
Mode: csi.VolumeCapability_AccessMode_SINGLE_NODE_WRITER,
}
accessModeReadWriteMany = &csi.VolumeCapability_AccessMode{
Mode: csi.VolumeCapability_AccessMode_MULTI_NODE_MULTI_WRITER,
}
accessModeReadOnlyMany = &csi.VolumeCapability_AccessMode{
Mode: csi.VolumeCapability_AccessMode_MULTI_NODE_READER_ONLY,
}
// VolumeCapability
volumeCapabilityMountReadWriteOnce = &csi.VolumeCapability{
AccessType: accessTypeMount,
AccessMode: accessModeReadWriteOnce,
}
volumeCapabilityMountReadWriteMany = &csi.VolumeCapability{
AccessType: accessTypeMount,
AccessMode: accessModeReadWriteMany,
}
volumeCapabilityMountReadOnlyMany = &csi.VolumeCapability{
AccessType: accessTypeMount,
AccessMode: accessModeReadOnlyMany,
}
volumeCapabilityBlockReadWriteOnce = &csi.VolumeCapability{
AccessType: accessTypeBlock,
AccessMode: accessModeReadWriteOnce,
}
volumeCapabilityBlockReadWriteMany = &csi.VolumeCapability{
AccessType: accessTypeBlock,
AccessMode: accessModeReadWriteMany,
}
volumeCapabilityBlockReadOnlyMany = &csi.VolumeCapability{
AccessType: accessTypeBlock,
AccessMode: accessModeReadOnlyMany,
}
// Each provisioner have a identify string to distinguish with others. This
// identify string will be added in PV annoations under this key.
provisionerIDKey = "storage.kubernetes.io/csiProvisionerIdentity"
Expand Down Expand Up @@ -370,28 +410,24 @@ func (p *csiProvisioner) Provision(options controller.VolumeOptions) (*v1.Persis
// Get access mode
volumeCaps := make([]*csi.VolumeCapability, 0)
for _, cap := range options.PVC.Spec.AccessModes {
switch cap {
case v1.ReadWriteOnce:
volumeCaps = append(volumeCaps, &csi.VolumeCapability{
AccessMode: &csi.VolumeCapability_AccessMode{
Mode: csi.VolumeCapability_AccessMode_SINGLE_NODE_WRITER,
},
AccessType: accessType,
})
case v1.ReadWriteMany:
volumeCaps = append(volumeCaps, &csi.VolumeCapability{
AccessMode: &csi.VolumeCapability_AccessMode{
Mode: csi.VolumeCapability_AccessMode_MULTI_NODE_MULTI_WRITER,
},
AccessType: accessType,
})
case v1.ReadOnlyMany:
volumeCaps = append(volumeCaps, &csi.VolumeCapability{
AccessMode: &csi.VolumeCapability_AccessMode{
Mode: csi.VolumeCapability_AccessMode_MULTI_NODE_READER_ONLY,
},
AccessType: accessType,
})
if util.CheckPersistentVolumeClaimModeBlock(options.PVC) {
switch cap {
case v1.ReadWriteOnce:
volumeCaps = append(volumeCaps, volumeCapabilityBlockReadWriteOnce)
case v1.ReadWriteMany:
volumeCaps = append(volumeCaps, volumeCapabilityBlockReadWriteMany)
case v1.ReadOnlyMany:
volumeCaps = append(volumeCaps, volumeCapabilityBlockReadOnlyMany)
}
} else {
switch cap {
case v1.ReadWriteOnce:
volumeCaps = append(volumeCaps, volumeCapabilityMountReadWriteOnce)
case v1.ReadWriteMany:
volumeCaps = append(volumeCaps, volumeCapabilityMountReadWriteMany)
case v1.ReadOnlyMany:
volumeCaps = append(volumeCaps, volumeCapabilityMountReadOnlyMany)
}
}
}

Expand Down Expand Up @@ -517,7 +553,6 @@ func (p *csiProvisioner) Provision(options controller.VolumeOptions) (*v1.Persis
CSI: &v1.CSIPersistentVolumeSource{
Driver: driverName,
VolumeHandle: p.volumeIdToHandle(rep.Volume.Id),
FSType: fsType,
VolumeAttributes: volumeAttributes,
ControllerPublishSecretRef: controllerPublishSecretRef,
NodeStageSecretRef: nodeStageSecretRef,
Expand All @@ -527,6 +562,16 @@ func (p *csiProvisioner) Provision(options controller.VolumeOptions) (*v1.Persis
},
}

// Set VolumeMode to PV if it is passed via PVC spec when Block feature is enabled
if options.PVC.Spec.VolumeMode != nil {
pv.Spec.VolumeMode = options.PVC.Spec.VolumeMode
}

// Set FSType if PV is not Block Volume
if !util.CheckPersistentVolumeClaimModeBlock(options.PVC) {
pv.Spec.PersistentVolumeSource.CSI.FSType = fsType
}

glog.Infof("successfully created PV %+v", pv.Spec.PersistentVolumeSource)

return pv, nil
Expand Down Expand Up @@ -624,6 +669,25 @@ func (p *csiProvisioner) Delete(volume *v1.PersistentVolume) error {
return err
}

func (p *csiProvisioner) SupportsBlock() bool {
ctx, cancel := context.WithTimeout(context.Background(), p.timeout)
defer cancel()

client := csi.NewControllerClient(p.grpcClient)
req := csi.ValidateVolumeCapabilitiesRequest{
VolumeCapabilities: []*csi.VolumeCapability{
volumeCapabilityMountReadWriteOnce,
},
}

rsp, err := client.ValidateVolumeCapabilities(ctx, &req)
if err != nil {
return false
}

return rsp.Supported
}

//TODO use a unique volume handle from and to Id
func (p *csiProvisioner) volumeIdToHandle(id string) string {
return id
Expand Down
110 changes: 109 additions & 1 deletion pkg/controller/controller_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -676,6 +676,13 @@ func createFakePVC(requestBytes int64) *v1.PersistentVolumeClaim {
}
}

// createFakePVCWithVolumeMode returns PVC with VolumeMode
func createFakePVCWithVolumeMode(requestBytes int64, volumeMode v1.PersistentVolumeMode) *v1.PersistentVolumeClaim {
claim := createFakePVC(requestBytes)
claim.Spec.VolumeMode = &volumeMode
return claim
}

func TestGetSecretReference(t *testing.T) {
testcases := map[string]struct {
nameKey string
Expand Down Expand Up @@ -799,13 +806,67 @@ func TestGetSecretReference(t *testing.T) {
}
}

func TestSupportsBlock(t *testing.T) {
var (
volCapTrue = csi.ValidateVolumeCapabilitiesResponse{
Supported: true,
}
volCapFalse = csi.ValidateVolumeCapabilitiesResponse{
Supported: false,
}
generalError = fmt.Errorf("")
)

testcases := []struct {
name string
volCapResp csi.ValidateVolumeCapabilitiesResponse
volCapErr error
expectSupportsBlock bool
}{
{"ValidateVolumeCapabilities returns (true, nil): expects true", volCapTrue, nil, true},
{"ValidateVolumeCapabilities returns (false, nil): expects false", volCapFalse, nil, false},
{"ValidateVolumeCapabilities returns (true, error): expects false", volCapTrue, generalError, false}, // This won't happen.
{"ValidateVolumeCapabilities returns (false, error): expects false", volCapFalse, generalError, false},
}

mockController, driver, _, controllerServer, csiConn, err := createMockServer(t)
if err != nil {
t.Fatal(err)
}
defer mockController.Finish()
defer driver.Stop()

for _, tc := range testcases {
var clientSet kubernetes.Interface

clientSet = fakeclientset.NewSimpleClientset()
csiProvisioner := NewCSIProvisioner(clientSet, driver.Address(), 5*time.Second, "test-provisioner", "test", 5, csiConn.conn, nil)

controllerServer.EXPECT().ValidateVolumeCapabilities(gomock.Any(), gomock.Any()).Return(&tc.volCapResp, tc.volCapErr).Times(1)

if csiBlockProvisioner, ok := csiProvisioner.(controller.BlockProvisioner); ok {
supportsBlock := csiBlockProvisioner.SupportsBlock()
if tc.expectSupportsBlock != supportsBlock {
t.Errorf("test %q: Expected %v got %v", tc.name, tc.expectSupportsBlock, supportsBlock)
}
} else {
t.Errorf("test %q: Expected csiProvisioner implements BlockProvisioner interface got %v", tc.name, ok)
}
}
}

func TestProvision(t *testing.T) {
var requestedBytes int64 = 100
var (
requestedBytes int64 = 100
volumeModeFileSystem = v1.PersistentVolumeFilesystem
volumeModeBlock = v1.PersistentVolumeBlock
)

type pvSpec struct {
Name string
ReclaimPolicy v1.PersistentVolumeReclaimPolicy
AccessModes []v1.PersistentVolumeAccessMode
VolumeMode *v1.PersistentVolumeMode
Capacity v1.ResourceList
CSIPVS *v1.CSIPersistentVolumeSource
}
Expand Down Expand Up @@ -1080,6 +1141,49 @@ func TestProvision(t *testing.T) {
},
},
},
"provision with volume mode(Filesystem)": {
volOpts: controller.VolumeOptions{
PVName: "test-name",
PVC: createFakePVCWithVolumeMode(requestedBytes, volumeModeFileSystem),
Parameters: map[string]string{},
},
expectedPVSpec: &pvSpec{
Name: "test-testi",
Capacity: v1.ResourceList{
v1.ResourceName(v1.ResourceStorage): bytesToGiQuantity(requestedBytes),
},
VolumeMode: &volumeModeFileSystem,
CSIPVS: &v1.CSIPersistentVolumeSource{
Driver: "test-driver",
VolumeHandle: "test-volume-id",
FSType: "ext4",
VolumeAttributes: map[string]string{
"storage.kubernetes.io/csiProvisionerIdentity": "test-provisioner",
},
},
},
},
"provision with volume mode(Block)": {
volOpts: controller.VolumeOptions{
PVName: "test-name",
PVC: createFakePVCWithVolumeMode(requestedBytes, volumeModeBlock),
Parameters: map[string]string{},
},
expectedPVSpec: &pvSpec{
Name: "test-testi",
Capacity: v1.ResourceList{
v1.ResourceName(v1.ResourceStorage): bytesToGiQuantity(requestedBytes),
},
VolumeMode: &volumeModeBlock,
CSIPVS: &v1.CSIPersistentVolumeSource{
Driver: "test-driver",
VolumeHandle: "test-volume-id",
VolumeAttributes: map[string]string{
"storage.kubernetes.io/csiProvisionerIdentity": "test-provisioner",
},
},
},
},
"fail to get secret reference": {
volOpts: controller.VolumeOptions{
PVName: "test-name",
Expand Down Expand Up @@ -1231,6 +1335,10 @@ func TestProvision(t *testing.T) {
t.Errorf("test %q: expected access modes: %v, got: %v", k, tc.expectedPVSpec.AccessModes, pv.Spec.AccessModes)
}

if !reflect.DeepEqual(pv.Spec.VolumeMode, tc.expectedPVSpec.VolumeMode) {
t.Errorf("test %q: expected volumeMode: %v, got: %v", k, tc.expectedPVSpec.VolumeMode, pv.Spec.VolumeMode)
}

if !reflect.DeepEqual(pv.Spec.Capacity, tc.expectedPVSpec.Capacity) {
t.Errorf("test %q: expected capacity: %v, got: %v", k, tc.expectedPVSpec.Capacity, pv.Spec.Capacity)
}
Expand Down

0 comments on commit da09b7c

Please sign in to comment.