[xeon-phi-kernel-module] / host / vhost / mic_blk.c

 /*
  * Copyright (C) 2009 Red Hat, Inc.
  * Author: Michael S. Tsirkin <mst@redhat.com>
  *
  * This work is licensed under the terms of the GNU GPL, version 2.

  * (C) Badari Pulavarty pbadari@us.ibm.com 2010 with the following comment.
  * He posted on http://lwn.net/Articles/382543/

  * virtio-block server in host kernel.
  * Inspired by vhost-net and shamlessly ripped code from it :)

  * For adapting to MIC
  * (C) Copyright 2012 Intel Corporation
  * Author: Caz Yokoyama <Caz.Yokoyama@intel.com>
  */
#include <linux/version.h>
#if LINUX_VERSION_CODE >= KERNEL_VERSION(2,6,34) || \
	defined(RHEL_RELEASE_CODE)

#include <linux/compat.h>
#include <linux/eventfd.h>
#include <linux/vhost.h>
#include <linux/virtio_ring.h>
#include <linux/virtio_net.h>
#include <linux/virtio_blk.h>
#include <linux/mmu_context.h>
#include <linux/miscdevice.h>
#include <linux/module.h>
#include <linux/mutex.h>
#include <linux/workqueue.h>
#include <linux/rcupdate.h>
#include <linux/file.h>
#include <linux/fdtable.h>

#ifndef VIRTIO_RING_F_EVENT_IDX  /* virtio_ring.h of rhel6.0 does not define */
#define VIRTIO_RING_F_EVENT_IDX		29
#endif
#include "mic_common.h"
#include "mic/micveth_dma.h"
#include "vhost.h"
#include "mic/mic_virtio.h"

#define MIC_SECTOR_SHIFT		9
#define MIC_SECTOR_SIZE			(1UL << MIC_SECTOR_SHIFT)
#define VIRTIO_BLK_QUEUE_SIZE		128
#define DISK_SEG_MAX			(VIRTIO_BLK_QUEUE_SIZE - 2)

#define VHOST_BLK_VQ_MAX 1
#define WQNAME_SIZE 16

struct vhost_blk {
	struct vhost_dev dev;
	struct vhost_virtqueue vqs[VHOST_BLK_VQ_MAX];
	struct vhost_poll poll[VHOST_BLK_VQ_MAX];
	struct workqueue_struct *vb_wq;
	char vb_wqname[WQNAME_SIZE];
	struct work_struct vb_ws_bh;
	struct workqueue_struct *vblk_workqueue;
	struct board_info *bd_info;
	char *file_name;
	struct file *virtblk_file;
};

struct vhost_blk_io {
	struct list_head list;
	struct work_struct work;
	struct vhost_blk *blk;
	struct file *file;
	int head;
	uint32_t type;
	uint32_t nvecs;
	uint64_t sector;
	uint64_t len;
	struct iovec iov[0];
};

#define mic_addr_in_host(va, pa) ((u8 *)(va) + (u64)(pa))

static LIST_HEAD(write_queue);
static LIST_HEAD(read_queue);

static void
cleanup_vblk_workqueue(struct vhost_blk_io *vbio, struct vhost_virtqueue *vq)
{
	struct list_head single, *head, *node, *tmp;
	int need_free;
	struct vhost_blk_io *entry;

	if (vbio->head != -1) {
		INIT_LIST_HEAD(&single);
		list_add(&vbio->list, &single);
		head = &single;
		need_free = 0;
	} else {
		head = &vbio->list;
		need_free = 1;
	}

	mutex_lock(&vq->mutex);
	list_for_each_safe(node, tmp, head) {
		entry = list_entry(node, struct vhost_blk_io, list);
		list_del(node);
		kfree(entry);
	}
	mutex_unlock(&vq->mutex);

	if (need_free)
		kfree(vbio);
}

static void handle_io_work(struct work_struct *work)
{
	struct vhost_blk_io *vbio, *entry;
	struct vhost_virtqueue *vq;
	struct vhost_blk *blk;
	struct list_head single, *head, *node, *tmp;
	struct iovec *iov;
	struct iov_iter iter;
	uint8_t *aper_va;
	struct vring *vring;
	unsigned int num;

	int need_free, ret = 0;
	loff_t pos;
	uint8_t status = 0;

	vbio = container_of(work, struct vhost_blk_io, work);
	blk = vbio->blk;
	vq = &blk->dev.vqs[0];
	pos = vbio->sector << MIC_SECTOR_SHIFT;
	aper_va = blk->bd_info->bi_ctx.aper.va;

	vring = &((struct mic_virtblk *)blk->bd_info->bi_virtio)->vb_shared.vring;
	num = readl(&vring->num);
	if (num == 0 || micpm_get_reference(&blk->bd_info->bi_ctx, true)) {
		cleanup_vblk_workqueue(vbio, vq);
		return;
	}

	if (atomic64_read(&vbio->file->f_count) == 0) {  /* file is closed */
		ret = -1;
	} else if (vbio->type & VIRTIO_BLK_T_FLUSH)  {
#ifdef RHEL_RELEASE_CODE
#if (LINUX_VERSION_CODE >= KERNEL_VERSION(3,10,0))
		ret = vfs_fsync(vbio->file, 1);
#else
		ret = vfs_fsync(vbio->file, vbio->file->f_path.dentry, 1);
#endif
#else
		ret = vfs_fsync(vbio->file, 1);
#endif
	} else if (vbio->type & VIRTIO_BLK_T_OUT) {
		for (iov = vbio->iov; iov < &vbio->iov[vbio->nvecs]; iov++) {
			iov->iov_base = mic_addr_in_host(aper_va, iov->iov_base);
		}
		iov_iter_init(&iter, WRITE, vbio->iov, vbio->nvecs, iov_length(vbio->iov, vbio->nvecs));
		ret = vfs_iter_write(vbio->file, &iter, &pos, 0);
	} else {
		for (iov = vbio->iov; iov < &vbio->iov[vbio->nvecs]; iov++) {
			iov->iov_base = mic_addr_in_host(aper_va, iov->iov_base);
		}
		iov_iter_init(&iter, READ, vbio->iov, vbio->nvecs, iov_length(vbio->iov, vbio->nvecs));
		ret = vfs_iter_read(vbio->file, &iter, &pos, 0);
	}
	status = (ret < 0) ? VIRTIO_BLK_S_IOERR : VIRTIO_BLK_S_OK;
	if (vbio->head != -1) {
		INIT_LIST_HEAD(&single);
		list_add(&vbio->list, &single);
		head = &single;
		need_free = 0;
	} else {
		head = &vbio->list;
		need_free = 1;
	}
	list_for_each_entry(entry, head, list) {
		memcpy_toio(mic_addr_in_host(aper_va, entry->iov[entry->nvecs].iov_base), &status, sizeof(status));
	}
	mutex_lock(&vq->mutex);
	list_for_each_safe(node, tmp, head) {
		entry = list_entry(node, struct vhost_blk_io, list);
		vhost_add_used_and_signal(&blk->dev, vq, entry->head, ret);
		list_del(node);
		kfree(entry);
	}
	mutex_unlock(&vq->mutex);
	if (need_free)
		kfree(vbio);
	micpm_put_reference(&blk->bd_info->bi_ctx);
}

static struct vhost_blk_io *allocate_vbio(int nvecs)
{
	struct vhost_blk_io *vbio;
	int size = sizeof(struct vhost_blk_io) + nvecs * sizeof(struct iovec);
	vbio = kmalloc(size, GFP_KERNEL);
	if (vbio) {
		INIT_WORK(&vbio->work, handle_io_work);
		INIT_LIST_HEAD(&vbio->list);
	}
	return vbio;
}

static void merge_and_handoff_work(struct list_head *queue)
{
	struct vhost_blk_io *vbio, *entry;
	int nvecs = 0;
	int entries = 0;

	list_for_each_entry(entry, queue, list) {
		nvecs += entry->nvecs;
		entries++;
	}

	if (entries == 1) {
		vbio = list_first_entry(queue, struct vhost_blk_io, list);
		list_del(&vbio->list);
		queue_work(vbio->blk->vblk_workqueue, &vbio->work);
		return;
	}

	vbio = allocate_vbio(nvecs);
	if (!vbio) {
		/* Unable to allocate memory - submit IOs individually */
		list_for_each_entry(vbio, queue, list) {
			queue_work(vbio->blk->vblk_workqueue, &vbio->work);
		}
		INIT_LIST_HEAD(queue);
		return;
	}

	entry = list_first_entry(queue, struct vhost_blk_io, list);
	vbio->nvecs = nvecs;
	vbio->blk = entry->blk;
	vbio->file = entry->file;
	vbio->type = entry->type;
	vbio->sector = entry->sector;
	vbio->head = -1;
	vbio->len = 0;
	nvecs = 0;

	list_for_each_entry(entry, queue, list) {
		memcpy(&vbio->iov[nvecs], entry->iov, entry->nvecs * sizeof(struct iovec));
		nvecs += entry->nvecs;
		vbio->len += entry->len;
	}
	list_replace_init(queue, &vbio->list);
	queue_work(vbio->blk->vblk_workqueue, &vbio->work);
}

static void start_io(struct list_head *queue)
{
	struct list_head start;
	struct vhost_blk_io *vbio = NULL, *entry;

	if (list_empty(queue))
                return;

	list_for_each_entry(entry, queue, list) {
		if (!vbio) {
			vbio = entry;
			continue;
		}
		if (vbio->sector + (vbio->len >> MIC_SECTOR_SHIFT) == entry->sector) {
			vbio = entry;
		} else {
			INIT_LIST_HEAD(&start);
			list_cut_position(&start, queue, &vbio->list);
			merge_and_handoff_work(&start);
			vbio = entry;
		}
	}
	if (!list_empty(queue))
		merge_and_handoff_work(queue);
}

static uint64_t calculate_len(struct iovec *iov, int nvecs)
{
	uint64_t len = 0;
	int i;

	for (i=0; i<nvecs; i++)
		len += iov[i].iov_len;
	return len;
}

static void insert_to_queue(struct vhost_blk_io *vbio,
			struct list_head *queue)
{
	struct vhost_blk_io *entry;

	list_for_each_entry(entry, queue, list) {
		if (entry->sector > vbio->sector)
			break;
	}
	list_add_tail(&vbio->list, &entry->list);
}

static int handoff_io(struct vhost_blk *blk, int head,
			uint32_t type, uint64_t sector,
			struct iovec *iov, int nvecs)
{
	struct vhost_virtqueue *vq = &blk->dev.vqs[0];
	struct vhost_blk_io *vbio;

	vbio = allocate_vbio(nvecs+1);
	if (!vbio) {
		return -ENOMEM;
	}
	vbio->blk = blk;
	vbio->head = head;
	vbio->file = vq->private_data;
	vbio->type = type;
	vbio->sector = sector;
	vbio->nvecs = nvecs;
	vbio->len = calculate_len(iov, nvecs);
	memcpy(vbio->iov, iov, (nvecs + 1) * sizeof(struct iovec));

	if (vbio->type & VIRTIO_BLK_T_FLUSH) {
#if 0
		/* Sync called - do I need to submit IOs in the queue ? */
		start_io(&read_queue);
		start_io(&write_queue);
#endif
		queue_work(blk->vblk_workqueue, &vbio->work);
	} else if (vbio->type & VIRTIO_BLK_T_OUT) {
		insert_to_queue(vbio, &write_queue);
	} else {
		insert_to_queue(vbio, &read_queue);
	}
	return 0;
}

static void handle_blk(struct vhost_blk *blk)
{
	struct vhost_virtqueue *vq = &blk->dev.vqs[0];
	unsigned head, out, in;
	struct virtio_blk_outhdr hdr;
	int nvecs;
	struct board_info *bd_info = blk->bd_info;
	struct vring *vring;

	vring = &((struct mic_virtblk *)bd_info->bi_virtio)->vb_shared.vring;
	if (vring == 0 || readl(&vring->num) == 0) {
		printk("request comes in while card side driver is not loaded yet. Ignore\n");
		return;
	}
	/* the first time since the card side driver becomes ready */
	if (vq->desc == NULL || readb(&((struct mic_virtblk *)bd_info->bi_virtio)->vb_shared.update)) {
	  vq->num = readl(&vring->num);
	  vq->desc = (struct vring_desc *)readq(&vring->desc);
	  vq->avail = (struct vring_avail *)readq(&vring->avail);
	  vq->used = (struct vring_used *)readq(&vring->used);
	  vq->last_avail_idx = 0;
	  vq->avail_idx = 0;
	  vq->last_used_idx = 0;
	  vq->signalled_used = 0;
	  vq->signalled_used_valid = false;
	  vq->done_idx = 0;
	  writeb(false, &((struct mic_virtblk *)bd_info->bi_virtio)->vb_shared.update);
	}

	if (micpm_get_reference(&blk->bd_info->bi_ctx,  true))
		return;

	mutex_lock(&vq->mutex);

	vhost_disable_notify(&blk->dev, vq);

	for (;;) {
		head = vhost_get_vq_desc(&blk->dev, vq, vq->iov,
					 ARRAY_SIZE(vq->iov),
					 &out, &in, NULL, NULL);
		if ((head == vq->num) || (head == -EFAULT) || (head == -EINVAL)) {
			if (unlikely(vhost_enable_notify(&blk->dev, vq))) {
				vhost_disable_notify(&blk->dev, vq);
				continue;
			}
			start_io(&read_queue);
			start_io(&write_queue);
			break;
		}

		BUG_ON(vq->iov[0].iov_len != 16);

		memcpy_fromio(&hdr, mic_addr_in_host(bd_info->bi_ctx.aper.va, vq->iov[0].iov_base),
					  sizeof(hdr));

		nvecs = out - 1;
		if (hdr.type == VIRTIO_BLK_T_IN)
			nvecs = in - 1;

		BUG_ON(vq->iov[nvecs+1].iov_len != 1);
		if (handoff_io(blk, head, hdr.type, hdr.sector, &vq->iov[1], nvecs) < 0) {
		  vhost_discard_vq_desc(vq, 1);
			continue;
		}
	}
	mutex_unlock(&vq->mutex);
	micpm_put_reference(&blk->bd_info->bi_ctx);
}

static void handle_blk_kick(struct work_struct *work)
{
	struct vhost_blk *vblk;

	vblk = container_of(work, struct vhost_blk, vb_ws_bh);
	handle_blk(vblk);
}

#if 0
static void handle_rq_blk(struct vhost_work *work)
{
	struct vhost_blk *blk;

	blk = container_of(work, struct vhost_blk, poll[0].work);
	handle_blk(blk);
}
#endif

static int
vhost_doorbell_intr_handler(mic_ctx_t *mic_ctx, int doorbell)
{
	struct board_info *bi;
	struct vhost_blk *vblk;

	bi = container_of(mic_ctx, struct board_info, bi_ctx);
	vblk = ((struct mic_virtblk *)bi->bi_virtio)->vblk;
	queue_work(vblk->vb_wq, &vblk->vb_ws_bh);

	return 0;
}

static long vhost_blk_set_backend(struct vhost_blk *vblk)
{
	struct vhost_virtqueue *vq;
	struct board_info *bd_info = vblk->bd_info;
	unsigned index = bd_info->bi_ctx.bi_id;
	struct vb_shared *vb_shared;
	int ret = 0;
	struct kstat stat;
	unsigned int virtio_blk_features = (1U << VIRTIO_BLK_F_SEG_MAX) |
					   (1U << VIRTIO_BLK_F_BLK_SIZE);

	if (index >= MAX_BOARD_SUPPORTED) {
		ret = -ENOBUFS;
		goto _exit_;
	}
	if (vblk->virtblk_file == NULL) {
		ret = -EBADF;
		goto _exit_;
	}

	vq = &vblk->vqs[0];
	mutex_lock(&vq->mutex);
	rcu_assign_pointer(vq->private_data, vblk->virtblk_file);
	mutex_unlock(&vq->mutex);

	snprintf(vblk->vb_wqname, sizeof(vblk->vb_wqname),
		 "virtblk wq %d", index);
	vblk->vb_wq = __mic_create_singlethread_workqueue(vblk->vb_wqname);
	if (vblk->vb_wq == NULL) {
		ret = -ENOMEM;
		goto _exit_;
	}
	INIT_WORK(&vblk->vb_ws_bh, handle_blk_kick);

	/* They have to be accessed from "struct vhost_virtqueue *vq" in mic_vhost.c.
	   They are not used in vhost block. I don't modify vhost.h. */
	vq->log_base = (void __user *)&bd_info->bi_ctx;
	vq->log_addr = (u64)bd_info->bi_ctx.aper.va;

	vb_shared = &((struct mic_virtblk *)bd_info->bi_virtio)->vb_shared;
#if (LINUX_VERSION_CODE < KERNEL_VERSION(3,6,0))
	virtio_blk_features |= (1U << VIRTIO_BLK_F_FLUSH);
#endif
	writel(virtio_blk_features, &vb_shared->host_features);
	writel(DISK_SEG_MAX, &vb_shared->blk_config.seg_max);
	writel(MIC_SECTOR_SIZE, &vb_shared->blk_config.blk_size);

	ret = vfs_getattr(&vblk->virtblk_file->f_path, &stat, STATX_BASIC_STATS, AT_STATX_SYNC_AS_STAT);
	if (ret < 0)
		goto _exit_;

	if (S_ISBLK(stat.mode)) {
	  writel(i_size_read(I_BDEV(vblk->virtblk_file->f_mapping->host)->bd_inode) / MIC_SECTOR_SIZE,
			 &vb_shared->blk_config.capacity);
	} else {
	  writel(stat.size / MIC_SECTOR_SIZE, &vb_shared->blk_config.capacity);
	}

	ret = mic_reg_irqhandler(&bd_info->bi_ctx, MIC_IRQ_DB2, "Host DoorBell 2",
				 vhost_doorbell_intr_handler);

_exit_:
	return ret;
}

void
mic_vhost_blk_stop(bd_info_t *bd_info)
{
	struct vring *vring = &((struct mic_virtblk *)bd_info->bi_virtio)->vb_shared.vring;

	writel(0, &vring->num);  /* reject subsequent request from MIC card */
}

extern bd_info_t *dev_to_bdi(struct device *dev);

ssize_t
show_virtblk_file(struct device *dev, struct device_attribute *attr, char *buf)
{
	struct board_info *bd_info = dev_to_bdi(dev);
	struct mic_virtblk *mic_virtblk;
	struct vhost_blk *vblk;

	BUG_ON(bd_info == NULL);
	mic_virtblk = bd_info->bi_virtio;
	BUG_ON(mic_virtblk == NULL);
	vblk = mic_virtblk->vblk;
	BUG_ON(vblk == NULL);

	if (vblk->file_name != NULL)
		return snprintf(buf, PAGE_SIZE, "%s\n", vblk->file_name);
	else
		return 0;
}

ssize_t
store_virtblk_file(struct device *dev, struct device_attribute *attr,
				   const char *buf, size_t count)
{
	int ret = 0;
	struct board_info *bd_info = dev_to_bdi(dev);
	struct mic_virtblk *mic_virtblk;
	struct vhost_blk *vblk;
	struct vhost_virtqueue *vq;
	char *p;
	struct file *virtblk_file;

	BUG_ON(bd_info == NULL);
	mic_virtblk = bd_info->bi_virtio;
	BUG_ON(mic_virtblk == NULL);
	vblk = mic_virtblk->vblk;
	BUG_ON(vblk == NULL);
	vq = &vblk->vqs[0];
	BUG_ON(vq == NULL);

	if (buf == NULL) {
		ret = -EINVAL;
		goto _return_;
	}
	if (count <= 1) {
		ret = -EINVAL;
		goto _return_;
	}

	p = strchr(buf, '\n');
	if (p != NULL)
		*p = '\0';

	mutex_lock(&vq->mutex);
	if (vblk->virtblk_file != NULL) {  /* if virtblk file is already assigned */
		printk(KERN_ALERT "you are changing virtblk file: %s -> %s.\n", vblk->file_name, buf);
		kfree(vblk->file_name);
		vblk->file_name = NULL;
		filp_close(vblk->virtblk_file, current->files);
		vblk->virtblk_file = NULL;
	}

	vblk->file_name = kmalloc(count + 1, GFP_KERNEL);
	strcpy(vblk->file_name, buf);
	virtblk_file = filp_open(vblk->file_name, O_RDWR|O_LARGEFILE, 0);
	if (IS_ERR(virtblk_file)) {
		ret = PTR_ERR(virtblk_file);
		mutex_unlock(&vq->mutex);
		goto free_file_name;
	}
	vblk->virtblk_file = virtblk_file;
	mutex_unlock(&vq->mutex);

	ret = vhost_blk_set_backend(vblk);
	if (ret < 0)
		goto close_virtblk_file;

	return count;

 close_virtblk_file:
	filp_close(vblk->virtblk_file, current->files);
 free_file_name:
	kfree(vblk->file_name);
 _return_:
	return ret;
}

int mic_vhost_blk_probe(bd_info_t *bd_info)
{
	int ret = 0;
	char wq_name[8];
	struct mic_virtblk *mic_virtblk;
	struct vhost_blk *vblk;

	mic_virtblk = kzalloc(sizeof(*mic_virtblk), GFP_KERNEL);
	if (mic_virtblk == NULL) {
		ret = -ENOMEM;
		goto err_vblk;
	}
	bd_info->bi_virtio = mic_virtblk;

	vblk = kzalloc(sizeof *vblk, GFP_KERNEL);
	if (vblk == NULL) {
		ret = -ENOMEM;
		goto free_mic_virtblk;
	}
	mic_virtblk->vblk = vblk;
	vblk->bd_info = bd_info;

	ret = vhost_dev_init(&vblk->dev, vblk->vqs, VHOST_BLK_VQ_MAX);
	if (ret < 0)
		goto free_vblk;

#if 0
	vhost_poll_init(vblk->poll, handle_rq_blk, POLLOUT|POLLIN, &vblk->dev);
#endif

	BUG_ON(bd_info->bi_ctx.bi_id >= 1000);
	snprintf(wq_name, ARRAY_SIZE(wq_name), "vblk%03d", bd_info->bi_ctx.bi_id);
	vblk->vblk_workqueue = __mic_create_singlethread_workqueue(wq_name);
	if (vblk->vblk_workqueue == NULL) {
		ret = -ENOMEM;
		goto free_vblk;
	}

	return ret;

 free_vblk:
	kfree(vblk);
 free_mic_virtblk:
	kfree(mic_virtblk);
 err_vblk:
	return ret;
}

void mic_vhost_blk_remove(bd_info_t *bd_info)
{
	struct mic_virtblk *mic_virtblk = bd_info->bi_virtio;
	struct vhost_blk *vblk = mic_virtblk->vblk;
	struct vb_shared *vb_shared = &mic_virtblk->vb_shared;

	if (vblk->virtblk_file != NULL) {
		mic_unreg_irqhandler(&bd_info->bi_ctx, MIC_IRQ_DB2, "Host DoorBell 2");
		memset(&vb_shared->blk_config, 0, sizeof(vb_shared->blk_config));
		destroy_workqueue(vblk->vb_wq);
		if (vblk->vqs[0].private_data != NULL)
			fput(vblk->vqs[0].private_data);
		kfree(vblk->file_name);
		filp_close(vblk->virtblk_file, current->files);
	}
	vhost_dev_cleanup(&vblk->dev);
	destroy_workqueue(vblk->vblk_workqueue);
	kfree(vblk);
	kfree(mic_virtblk);
}
#endif
Commit	Line	Data
800f879a AT	1	/*
	2	* Copyright (C) 2009 Red Hat, Inc.
	3	* Author: Michael S. Tsirkin <mst@redhat.com>
	4	*
	5	* This work is licensed under the terms of the GNU GPL, version 2.
	6
	7	* (C) Badari Pulavarty pbadari@us.ibm.com 2010 with the following comment.
	8	* He posted on http://lwn.net/Articles/382543/
	9
	10	* virtio-block server in host kernel.
	11	* Inspired by vhost-net and shamlessly ripped code from it :)
	12
	13	* For adapting to MIC
	14	* (C) Copyright 2012 Intel Corporation
	15	* Author: Caz Yokoyama <Caz.Yokoyama@intel.com>
	16	*/
	17	#include <linux/version.h>
	18	#if LINUX_VERSION_CODE >= KERNEL_VERSION(2,6,34) \|\| \
	19	defined(RHEL_RELEASE_CODE)
	20
	21	#include <linux/compat.h>
	22	#include <linux/eventfd.h>
	23	#include <linux/vhost.h>
	24	#include <linux/virtio_ring.h>
	25	#include <linux/virtio_net.h>
	26	#include <linux/virtio_blk.h>
	27	#include <linux/mmu_context.h>
	28	#include <linux/miscdevice.h>
	29	#include <linux/module.h>
	30	#include <linux/mutex.h>
	31	#include <linux/workqueue.h>
	32	#include <linux/rcupdate.h>
	33	#include <linux/file.h>
	34	#include <linux/fdtable.h>
	35
	36	#ifndef VIRTIO_RING_F_EVENT_IDX /* virtio_ring.h of rhel6.0 does not define */
	37	#define VIRTIO_RING_F_EVENT_IDX 29
	38	#endif
	39	#include "mic_common.h"
	40	#include "mic/micveth_dma.h"
	41	#include "vhost.h"
	42	#include "mic/mic_virtio.h"
	43
263db953 AT	44	#define MIC_SECTOR_SHIFT 9
263db953 AT	45	#define MIC_SECTOR_SIZE (1UL << MIC_SECTOR_SHIFT)
800f879a AT	46	#define VIRTIO_BLK_QUEUE_SIZE 128
	47	#define DISK_SEG_MAX (VIRTIO_BLK_QUEUE_SIZE - 2)
	48
	49	#define VHOST_BLK_VQ_MAX 1
	50	#define WQNAME_SIZE 16
	51
	52	struct vhost_blk {
	53	struct vhost_dev dev;
	54	struct vhost_virtqueue vqs[VHOST_BLK_VQ_MAX];
	55	struct vhost_poll poll[VHOST_BLK_VQ_MAX];
	56	struct workqueue_struct *vb_wq;
	57	char vb_wqname[WQNAME_SIZE];
	58	struct work_struct vb_ws_bh;
	59	struct workqueue_struct *vblk_workqueue;
	60	struct board_info *bd_info;
	61	char *file_name;
	62	struct file *virtblk_file;
	63	};
	64
	65	struct vhost_blk_io {
	66	struct list_head list;
	67	struct work_struct work;
	68	struct vhost_blk *blk;
	69	struct file *file;
	70	int head;
	71	uint32_t type;
	72	uint32_t nvecs;
	73	uint64_t sector;
	74	uint64_t len;
	75	struct iovec iov[0];
	76	};
	77
	78	#define mic_addr_in_host(va, pa) ((u8 *)(va) + (u64)(pa))
	79
	80	static LIST_HEAD(write_queue);
	81	static LIST_HEAD(read_queue);
	82
	83	static void
	84	cleanup_vblk_workqueue(struct vhost_blk_io vbio, struct vhost_virtqueue vq)
	85	{
	86	struct list_head single, head, node, *tmp;
	87	int need_free;
	88	struct vhost_blk_io *entry;
	89
	90	if (vbio->head != -1) {
	91	INIT_LIST_HEAD(&single);
	92	list_add(&vbio->list, &single);
	93	head = &single;
	94	need_free = 0;
	95	} else {
	96	head = &vbio->list;
	97	need_free = 1;
	98	}
	99
	100	mutex_lock(&vq->mutex);
	101	list_for_each_safe(node, tmp, head) {
	102	entry = list_entry(node, struct vhost_blk_io, list);
	103	list_del(node);
	104	kfree(entry);
	105	}
	106	mutex_unlock(&vq->mutex);
	107
	108	if (need_free)
	109	kfree(vbio);
110	}
111
112	static void handle_io_work(struct work_struct *work)
113	{
114	struct vhost_blk_io vbio, entry;
115	struct vhost_virtqueue *vq;
116	struct vhost_blk *blk;
117	struct list_head single, head, node, *tmp;
118	struct iovec *iov;
26721891	119	struct iov_iter iter;
800f879a AT	120	uint8_t *aper_va;
	121	struct vring *vring;
	122	unsigned int num;
	123
	124	int need_free, ret = 0;
	125	loff_t pos;
	126	uint8_t status = 0;
	127
	128	vbio = container_of(work, struct vhost_blk_io, work);
	129	blk = vbio->blk;
	130	vq = &blk->dev.vqs[0];
263db953	131	pos = vbio->sector << MIC_SECTOR_SHIFT;
800f879a AT	132	aper_va = blk->bd_info->bi_ctx.aper.va;
	133
	134	vring = &((struct mic_virtblk *)blk->bd_info->bi_virtio)->vb_shared.vring;
	135	num = readl(&vring->num);
	136	if (num == 0 \|\| micpm_get_reference(&blk->bd_info->bi_ctx, true)) {
	137	cleanup_vblk_workqueue(vbio, vq);
	138	return;
	139	}
	140
	141	if (atomic64_read(&vbio->file->f_count) == 0) { /* file is closed */
	142	ret = -1;
	143	} else if (vbio->type & VIRTIO_BLK_T_FLUSH) {
	144	#ifdef RHEL_RELEASE_CODE
	145	#if (LINUX_VERSION_CODE >= KERNEL_VERSION(3,10,0))
	146	ret = vfs_fsync(vbio->file, 1);
	147	#else
	148	ret = vfs_fsync(vbio->file, vbio->file->f_path.dentry, 1);
	149	#endif
	150	#else
	151	ret = vfs_fsync(vbio->file, 1);
	152	#endif
	153	} else if (vbio->type & VIRTIO_BLK_T_OUT) {
26721891 AT	154	for (iov = vbio->iov; iov < &vbio->iov[vbio->nvecs]; iov++) {
	155	iov->iov_base = mic_addr_in_host(aper_va, iov->iov_base);
	156	}
	157	iov_iter_init(&iter, WRITE, vbio->iov, vbio->nvecs, iov_length(vbio->iov, vbio->nvecs));
	158	ret = vfs_iter_write(vbio->file, &iter, &pos, 0);
800f879a	159	} else {
26721891 AT	160	for (iov = vbio->iov; iov < &vbio->iov[vbio->nvecs]; iov++) {
	161	iov->iov_base = mic_addr_in_host(aper_va, iov->iov_base);
	162	}
	163	iov_iter_init(&iter, READ, vbio->iov, vbio->nvecs, iov_length(vbio->iov, vbio->nvecs));
	164	ret = vfs_iter_read(vbio->file, &iter, &pos, 0);
800f879a AT	165	}
	166	status = (ret < 0) ? VIRTIO_BLK_S_IOERR : VIRTIO_BLK_S_OK;
	167	if (vbio->head != -1) {
	168	INIT_LIST_HEAD(&single);
	169	list_add(&vbio->list, &single);
	170	head = &single;
	171	need_free = 0;
	172	} else {
	173	head = &vbio->list;
	174	need_free = 1;
	175	}
	176	list_for_each_entry(entry, head, list) {
	177	memcpy_toio(mic_addr_in_host(aper_va, entry->iov[entry->nvecs].iov_base), &status, sizeof(status));
	178	}
	179	mutex_lock(&vq->mutex);
	180	list_for_each_safe(node, tmp, head) {
	181	entry = list_entry(node, struct vhost_blk_io, list);
	182	vhost_add_used_and_signal(&blk->dev, vq, entry->head, ret);
	183	list_del(node);
	184	kfree(entry);
	185	}
	186	mutex_unlock(&vq->mutex);
	187	if (need_free)
	188	kfree(vbio);
	189	micpm_put_reference(&blk->bd_info->bi_ctx);
	190	}
	191
	192	static struct vhost_blk_io *allocate_vbio(int nvecs)
	193	{
	194	struct vhost_blk_io *vbio;
	195	int size = sizeof(struct vhost_blk_io) + nvecs * sizeof(struct iovec);
	196	vbio = kmalloc(size, GFP_KERNEL);
	197	if (vbio) {
	198	INIT_WORK(&vbio->work, handle_io_work);
	199	INIT_LIST_HEAD(&vbio->list);
	200	}
	201	return vbio;
	202	}
	203
	204	static void merge_and_handoff_work(struct list_head *queue)
	205	{
	206	struct vhost_blk_io vbio, entry;
	207	int nvecs = 0;
	208	int entries = 0;
	209
	210	list_for_each_entry(entry, queue, list) {
	211	nvecs += entry->nvecs;
	212	entries++;
	213	}
	214
	215	if (entries == 1) {
	216	vbio = list_first_entry(queue, struct vhost_blk_io, list);
	217	list_del(&vbio->list);
	218	queue_work(vbio->blk->vblk_workqueue, &vbio->work);
	219	return;
	220	}
	221
	222	vbio = allocate_vbio(nvecs);
	223	if (!vbio) {
	224	/* Unable to allocate memory - submit IOs individually */
	225	list_for_each_entry(vbio, queue, list) {
	226	queue_work(vbio->blk->vblk_workqueue, &vbio->work);
	227	}
	228	INIT_LIST_HEAD(queue);
229	return;
230	}
231
232	entry = list_first_entry(queue, struct vhost_blk_io, list);
233	vbio->nvecs = nvecs;
234	vbio->blk = entry->blk;
235	vbio->file = entry->file;
236	vbio->type = entry->type;
237	vbio->sector = entry->sector;
238	vbio->head = -1;
239	vbio->len = 0;
240	nvecs = 0;
241
242	list_for_each_entry(entry, queue, list) {
243	memcpy(&vbio->iov[nvecs], entry->iov, entry->nvecs * sizeof(struct iovec));
244	nvecs += entry->nvecs;
245	vbio->len += entry->len;
246	}
247	list_replace_init(queue, &vbio->list);
248	queue_work(vbio->blk->vblk_workqueue, &vbio->work);
249	}
250
251	static void start_io(struct list_head *queue)
252	{
253	struct list_head start;
254	struct vhost_blk_io vbio = NULL, entry;
255
256	if (list_empty(queue))
257	return;
258
259	list_for_each_entry(entry, queue, list) {
260	if (!vbio) {
261	vbio = entry;
262	continue;
263	}
263db953	264	if (vbio->sector + (vbio->len >> MIC_SECTOR_SHIFT) == entry->sector) {
800f879a AT	265	vbio = entry;
	266	} else {
	267	INIT_LIST_HEAD(&start);
	268	list_cut_position(&start, queue, &vbio->list);
	269	merge_and_handoff_work(&start);
	270	vbio = entry;
	271	}
	272	}
	273	if (!list_empty(queue))
	274	merge_and_handoff_work(queue);
	275	}
	276
	277	static uint64_t calculate_len(struct iovec *iov, int nvecs)
	278	{
	279	uint64_t len = 0;
	280	int i;
	281
	282	for (i=0; i<nvecs; i++)
	283	len += iov[i].iov_len;
	284	return len;
	285	}
	286
	287	static void insert_to_queue(struct vhost_blk_io *vbio,
	288	struct list_head *queue)
	289	{
	290	struct vhost_blk_io *entry;
	291
	292	list_for_each_entry(entry, queue, list) {
	293	if (entry->sector > vbio->sector)
	294	break;
	295	}
	296	list_add_tail(&vbio->list, &entry->list);
	297	}
	298
	299	static int handoff_io(struct vhost_blk *blk, int head,
	300	uint32_t type, uint64_t sector,
	301	struct iovec *iov, int nvecs)
	302	{
	303	struct vhost_virtqueue *vq = &blk->dev.vqs[0];
	304	struct vhost_blk_io *vbio;
	305
	306	vbio = allocate_vbio(nvecs+1);
	307	if (!vbio) {
	308	return -ENOMEM;
	309	}
	310	vbio->blk = blk;
	311	vbio->head = head;
	312	vbio->file = vq->private_data;
	313	vbio->type = type;
	314	vbio->sector = sector;
	315	vbio->nvecs = nvecs;
	316	vbio->len = calculate_len(iov, nvecs);
	317	memcpy(vbio->iov, iov, (nvecs + 1) * sizeof(struct iovec));
	318
	319	if (vbio->type & VIRTIO_BLK_T_FLUSH) {
	320	#if 0
	321	/* Sync called - do I need to submit IOs in the queue ? */
	322	start_io(&read_queue);
	323	start_io(&write_queue);
	324	#endif
	325	queue_work(blk->vblk_workqueue, &vbio->work);
	326	} else if (vbio->type & VIRTIO_BLK_T_OUT) {
	327	insert_to_queue(vbio, &write_queue);
	328	} else {
329	insert_to_queue(vbio, &read_queue);
330	}
331	return 0;
332	}
333
334	static void handle_blk(struct vhost_blk *blk)
335	{
336	struct vhost_virtqueue *vq = &blk->dev.vqs[0];
337	unsigned head, out, in;
338	struct virtio_blk_outhdr hdr;
339	int nvecs;
340	struct board_info *bd_info = blk->bd_info;
341	struct vring *vring;
342
343	vring = &((struct mic_virtblk *)bd_info->bi_virtio)->vb_shared.vring;
344	if (vring == 0 \|\| readl(&vring->num) == 0) {
345	printk("request comes in while card side driver is not loaded yet. Ignore\n");
346	return;
347	}
348	/* the first time since the card side driver becomes ready */
349	if (vq->desc == NULL \|\| readb(&((struct mic_virtblk *)bd_info->bi_virtio)->vb_shared.update)) {
350	vq->num = readl(&vring->num);
351	vq->desc = (struct vring_desc *)readq(&vring->desc);
352	vq->avail = (struct vring_avail *)readq(&vring->avail);
353	vq->used = (struct vring_used *)readq(&vring->used);
354	vq->last_avail_idx = 0;
355	vq->avail_idx = 0;
356	vq->last_used_idx = 0;
357	vq->signalled_used = 0;
358	vq->signalled_used_valid = false;
359	vq->done_idx = 0;
360	writeb(false, &((struct mic_virtblk *)bd_info->bi_virtio)->vb_shared.update);
361	}
362
363	if (micpm_get_reference(&blk->bd_info->bi_ctx, true))
364	return;
365
366	mutex_lock(&vq->mutex);
367
368	vhost_disable_notify(&blk->dev, vq);
369
370	for (;;) {
371	head = vhost_get_vq_desc(&blk->dev, vq, vq->iov,
372	ARRAY_SIZE(vq->iov),
373	&out, &in, NULL, NULL);
374	if ((head == vq->num) \|\| (head == -EFAULT) \|\| (head == -EINVAL)) {
375	if (unlikely(vhost_enable_notify(&blk->dev, vq))) {
376	vhost_disable_notify(&blk->dev, vq);
377	continue;
378	}
379	start_io(&read_queue);
380	start_io(&write_queue);
381	break;
382	}
383
384	BUG_ON(vq->iov[0].iov_len != 16);
385
386	memcpy_fromio(&hdr, mic_addr_in_host(bd_info->bi_ctx.aper.va, vq->iov[0].iov_base),
387	sizeof(hdr));
388
389	nvecs = out - 1;
390	if (hdr.type == VIRTIO_BLK_T_IN)
391	nvecs = in - 1;
392
393	BUG_ON(vq->iov[nvecs+1].iov_len != 1);
394	if (handoff_io(blk, head, hdr.type, hdr.sector, &vq->iov[1], nvecs) < 0) {
395	vhost_discard_vq_desc(vq, 1);
396	continue;
397	}
398	}
399	mutex_unlock(&vq->mutex);
400	micpm_put_reference(&blk->bd_info->bi_ctx);
401	}
402
403	static void handle_blk_kick(struct work_struct *work)
404	{
405	struct vhost_blk *vblk;
406
407	vblk = container_of(work, struct vhost_blk, vb_ws_bh);
408	handle_blk(vblk);
409	}
410
411	#if 0
412	static void handle_rq_blk(struct vhost_work *work)
413	{
414	struct vhost_blk *blk;
415
416	blk = container_of(work, struct vhost_blk, poll[0].work);
417	handle_blk(blk);
418	}
419	#endif
420
421	static int
422	vhost_doorbell_intr_handler(mic_ctx_t *mic_ctx, int doorbell)
423	{
424	struct board_info *bi;
425	struct vhost_blk *vblk;
426
427	bi = container_of(mic_ctx, struct board_info, bi_ctx);
428	vblk = ((struct mic_virtblk *)bi->bi_virtio)->vblk;
429	queue_work(vblk->vb_wq, &vblk->vb_ws_bh);
430
431	return 0;
432	}
433
434	static long vhost_blk_set_backend(struct vhost_blk *vblk)
435	{
436	struct vhost_virtqueue *vq;
437	struct board_info *bd_info = vblk->bd_info;
438	unsigned index = bd_info->bi_ctx.bi_id;
439	struct vb_shared *vb_shared;
440	int ret = 0;
441	struct kstat stat;
442	unsigned int virtio_blk_features = (1U << VIRTIO_BLK_F_SEG_MAX) \|
443	(1U << VIRTIO_BLK_F_BLK_SIZE);
444
445	if (index >= MAX_BOARD_SUPPORTED) {
446	ret = -ENOBUFS;
447	goto _exit_;
448	}
449	if (vblk->virtblk_file == NULL) {
450	ret = -EBADF;
451	goto _exit_;
452	}
453
454	vq = &vblk->vqs[0];
455	mutex_lock(&vq->mutex);
456	rcu_assign_pointer(vq->private_data, vblk->virtblk_file);
457	mutex_unlock(&vq->mutex);
458
459	snprintf(vblk->vb_wqname, sizeof(vblk->vb_wqname),
460	"virtblk wq %d", index);
461	vblk->vb_wq = __mic_create_singlethread_workqueue(vblk->vb_wqname);
462	if (vblk->vb_wq == NULL) {
463	ret = -ENOMEM;
464	goto _exit_;
465	}
466	INIT_WORK(&vblk->vb_ws_bh, handle_blk_kick);
467
468	/* They have to be accessed from "struct vhost_virtqueue *vq" in mic_vhost.c.
469	They are not used in vhost block. I don't modify vhost.h. */
470	vq->log_base = (void __user *)&bd_info->bi_ctx;
471	vq->log_addr = (u64)bd_info->bi_ctx.aper.va;
472
473	vb_shared = &((struct mic_virtblk *)bd_info->bi_virtio)->vb_shared;
474	#if (LINUX_VERSION_CODE < KERNEL_VERSION(3,6,0))
475	virtio_blk_features \|= (1U << VIRTIO_BLK_F_FLUSH);
476	#endif
477	writel(virtio_blk_features, &vb_shared->host_features);
478	writel(DISK_SEG_MAX, &vb_shared->blk_config.seg_max);
263db953	479	writel(MIC_SECTOR_SIZE, &vb_shared->blk_config.blk_size);
800f879a	480
26721891	481	ret = vfs_getattr(&vblk->virtblk_file->f_path, &stat, STATX_BASIC_STATS, AT_STATX_SYNC_AS_STAT);
800f879a AT	482	if (ret < 0)
	483	goto _exit_;
	484
	485	if (S_ISBLK(stat.mode)) {
263db953	486	writel(i_size_read(I_BDEV(vblk->virtblk_file->f_mapping->host)->bd_inode) / MIC_SECTOR_SIZE,
800f879a AT	487	&vb_shared->blk_config.capacity);
800f879a AT	488	} else {
263db953	489	writel(stat.size / MIC_SECTOR_SIZE, &vb_shared->blk_config.capacity);
800f879a AT	490	}
	491
	492	ret = mic_reg_irqhandler(&bd_info->bi_ctx, MIC_IRQ_DB2, "Host DoorBell 2",
	493	vhost_doorbell_intr_handler);
	494
	495	_exit_:
	496	return ret;
	497	}
	498
	499	void
	500	mic_vhost_blk_stop(bd_info_t *bd_info)
	501	{
	502	struct vring vring = &((struct mic_virtblk )bd_info->bi_virtio)->vb_shared.vring;
	503
	504	writel(0, &vring->num); /* reject subsequent request from MIC card */
	505	}
	506
	507	extern bd_info_t dev_to_bdi(struct device dev);
	508
	509	ssize_t
	510	show_virtblk_file(struct device dev, struct device_attribute attr, char *buf)
	511	{
	512	struct board_info *bd_info = dev_to_bdi(dev);
	513	struct mic_virtblk *mic_virtblk;
	514	struct vhost_blk *vblk;
	515
	516	BUG_ON(bd_info == NULL);
	517	mic_virtblk = bd_info->bi_virtio;
	518	BUG_ON(mic_virtblk == NULL);
	519	vblk = mic_virtblk->vblk;
	520	BUG_ON(vblk == NULL);
	521
	522	if (vblk->file_name != NULL)
	523	return snprintf(buf, PAGE_SIZE, "%s\n", vblk->file_name);
	524	else
	525	return 0;
	526	}
	527
	528	ssize_t
	529	store_virtblk_file(struct device dev, struct device_attribute attr,
	530	const char *buf, size_t count)
	531	{
	532	int ret = 0;
	533	struct board_info *bd_info = dev_to_bdi(dev);
	534	struct mic_virtblk *mic_virtblk;
	535	struct vhost_blk *vblk;
	536	struct vhost_virtqueue *vq;
	537	char *p;
	538	struct file *virtblk_file;
	539
	540	BUG_ON(bd_info == NULL);
	541	mic_virtblk = bd_info->bi_virtio;
	542	BUG_ON(mic_virtblk == NULL);
	543	vblk = mic_virtblk->vblk;
	544	BUG_ON(vblk == NULL);
	545	vq = &vblk->vqs[0];
	546	BUG_ON(vq == NULL);
	547
	548	if (buf == NULL) {
	549	ret = -EINVAL;
	550	goto _return_;
	551	}
	552	if (count <= 1) {
	553	ret = -EINVAL;
554	goto _return_;
555	}
556
557	p = strchr(buf, '\n');
558	if (p != NULL)
559	*p = '\0';
560
561	mutex_lock(&vq->mutex);
562	if (vblk->virtblk_file != NULL) { /* if virtblk file is already assigned */
563	printk(KERN_ALERT "you are changing virtblk file: %s -> %s.\n", vblk->file_name, buf);
564	kfree(vblk->file_name);
565	vblk->file_name = NULL;
566	filp_close(vblk->virtblk_file, current->files);
567	vblk->virtblk_file = NULL;
568	}
569
570	vblk->file_name = kmalloc(count + 1, GFP_KERNEL);
571	strcpy(vblk->file_name, buf);
572	virtblk_file = filp_open(vblk->file_name, O_RDWR\|O_LARGEFILE, 0);
573	if (IS_ERR(virtblk_file)) {
574	ret = PTR_ERR(virtblk_file);
575	mutex_unlock(&vq->mutex);
576	goto free_file_name;
577	}
578	vblk->virtblk_file = virtblk_file;
579	mutex_unlock(&vq->mutex);
580
581	ret = vhost_blk_set_backend(vblk);
582	if (ret < 0)
583	goto close_virtblk_file;
584
585	return count;
586
587	close_virtblk_file:
588	filp_close(vblk->virtblk_file, current->files);
589	free_file_name:
590	kfree(vblk->file_name);
591	_return_:
592	return ret;
593	}
594
595	int mic_vhost_blk_probe(bd_info_t *bd_info)
596	{
597	int ret = 0;
598	char wq_name[8];
599	struct mic_virtblk *mic_virtblk;
600	struct vhost_blk *vblk;
601
602	mic_virtblk = kzalloc(sizeof(*mic_virtblk), GFP_KERNEL);
603	if (mic_virtblk == NULL) {
604	ret = -ENOMEM;
605	goto err_vblk;
606	}
607	bd_info->bi_virtio = mic_virtblk;
608
609	vblk = kzalloc(sizeof *vblk, GFP_KERNEL);
610	if (vblk == NULL) {
611	ret = -ENOMEM;
612	goto free_mic_virtblk;
613	}
614	mic_virtblk->vblk = vblk;
615	vblk->bd_info = bd_info;
616
617	ret = vhost_dev_init(&vblk->dev, vblk->vqs, VHOST_BLK_VQ_MAX);
618	if (ret < 0)
619	goto free_vblk;
620
621	#if 0
622	vhost_poll_init(vblk->poll, handle_rq_blk, POLLOUT\|POLLIN, &vblk->dev);
623	#endif
624
625	BUG_ON(bd_info->bi_ctx.bi_id >= 1000);
626	snprintf(wq_name, ARRAY_SIZE(wq_name), "vblk%03d", bd_info->bi_ctx.bi_id);
627	vblk->vblk_workqueue = __mic_create_singlethread_workqueue(wq_name);
628	if (vblk->vblk_workqueue == NULL) {
629	ret = -ENOMEM;
630	goto free_vblk;
631	}
632
633	return ret;
634
635	free_vblk:
636	kfree(vblk);
637	free_mic_virtblk:
638	kfree(mic_virtblk);
639	err_vblk:
640	return ret;
641	}
642
643	void mic_vhost_blk_remove(bd_info_t *bd_info)
644	{
645	struct mic_virtblk *mic_virtblk = bd_info->bi_virtio;
646	struct vhost_blk *vblk = mic_virtblk->vblk;
647	struct vb_shared *vb_shared = &mic_virtblk->vb_shared;
648
649	if (vblk->virtblk_file != NULL) {
650	mic_unreg_irqhandler(&bd_info->bi_ctx, MIC_IRQ_DB2, "Host DoorBell 2");
651	memset(&vb_shared->blk_config, 0, sizeof(vb_shared->blk_config));
652	destroy_workqueue(vblk->vb_wq);
653	if (vblk->vqs[0].private_data != NULL)
654	fput(vblk->vqs[0].private_data);
655	kfree(vblk->file_name);
656	filp_close(vblk->virtblk_file, current->files);
657	}
658	vhost_dev_cleanup(&vblk->dev);
659	destroy_workqueue(vblk->vblk_workqueue);
660	kfree(vblk);
661	kfree(mic_virtblk);
662	}
663	#endif