[xeon-phi-kernel-module] / host / vhost / mic_blk.c

 /*
  * Copyright (C) 2009 Red Hat, Inc.
  * Author: Michael S. Tsirkin <mst@redhat.com>
  *
  * This work is licensed under the terms of the GNU GPL, version 2.

  * (C) Badari Pulavarty pbadari@us.ibm.com 2010 with the following comment.
  * He posted on http://lwn.net/Articles/382543/

  * virtio-block server in host kernel.
  * Inspired by vhost-net and shamlessly ripped code from it :)

  * For adapting to MIC
  * (C) Copyright 2012 Intel Corporation
  * Author: Caz Yokoyama <Caz.Yokoyama@intel.com>
  */
#include <linux/version.h>
#if LINUX_VERSION_CODE >= KERNEL_VERSION(2,6,34) || \
	defined(RHEL_RELEASE_CODE)

#include <linux/compat.h>
#include <linux/eventfd.h>
#include <linux/vhost.h>
#include <linux/virtio_ring.h>
#include <linux/virtio_net.h>
#include <linux/virtio_blk.h>
#include <linux/mmu_context.h>
#include <linux/miscdevice.h>
#include <linux/module.h>
#include <linux/mutex.h>
#include <linux/workqueue.h>
#include <linux/rcupdate.h>
#include <linux/file.h>
#include <linux/fdtable.h>

#ifndef VIRTIO_RING_F_EVENT_IDX  /* virtio_ring.h of rhel6.0 does not define */
#define VIRTIO_RING_F_EVENT_IDX		29
#endif
#include "mic_common.h"
#include "mic/micveth_dma.h"
#include "vhost.h"
#include "mic/mic_virtio.h"

#define MIC_SECTOR_SHIFT		9
#define MIC_SECTOR_SIZE			(1UL << MIC_SECTOR_SHIFT)
#define VIRTIO_BLK_QUEUE_SIZE		128
#define DISK_SEG_MAX			(VIRTIO_BLK_QUEUE_SIZE - 2)

#define VHOST_BLK_VQ_MAX 1
#define WQNAME_SIZE 16

struct vhost_blk {
	struct vhost_dev dev;
	struct vhost_virtqueue vqs[VHOST_BLK_VQ_MAX];
	struct vhost_poll poll[VHOST_BLK_VQ_MAX];
	struct workqueue_struct *vb_wq;
	char vb_wqname[WQNAME_SIZE];
	struct work_struct vb_ws_bh;
	struct workqueue_struct *vblk_workqueue;
	struct board_info *bd_info;
	char *file_name;
	struct file *virtblk_file;
};

struct vhost_blk_io {
	struct list_head list;
	struct work_struct work;
	struct vhost_blk *blk;
	struct file *file;
	int head;
	uint32_t type;
	uint32_t nvecs;
	uint64_t sector;
	uint64_t len;
	struct iovec iov[0];
};

#define mic_addr_in_host(va, pa) ((u8 *)(va) + (u64)(pa))

static LIST_HEAD(write_queue);
static LIST_HEAD(read_queue);

static void
cleanup_vblk_workqueue(struct vhost_blk_io *vbio, struct vhost_virtqueue *vq)
{
	struct list_head single, *head, *node, *tmp;
	int need_free;
	struct vhost_blk_io *entry;

	if (vbio->head != -1) {
		INIT_LIST_HEAD(&single);
		list_add(&vbio->list, &single);
		head = &single;
		need_free = 0;
	} else {
		head = &vbio->list;
		need_free = 1;
	}

	mutex_lock(&vq->mutex);
	list_for_each_safe(node, tmp, head) {
		entry = list_entry(node, struct vhost_blk_io, list);
		list_del(node);
		kfree(entry);
	}
	mutex_unlock(&vq->mutex);

	if (need_free)
		kfree(vbio);
}

static void handle_io_work(struct work_struct *work)
{
	struct vhost_blk_io *vbio, *entry;
	struct vhost_virtqueue *vq;
	struct vhost_blk *blk;
	struct list_head single, *head, *node, *tmp;
	struct iovec *iov;
	uint8_t *aper_va;
	struct vring *vring;
	unsigned int num;

	int need_free, ret = 0;
	loff_t pos;
	uint8_t status = 0;

	vbio = container_of(work, struct vhost_blk_io, work);
	blk = vbio->blk;
	vq = &blk->dev.vqs[0];
	pos = vbio->sector << MIC_SECTOR_SHIFT;
	aper_va = blk->bd_info->bi_ctx.aper.va;

	vring = &((struct mic_virtblk *)blk->bd_info->bi_virtio)->vb_shared.vring;
	num = readl(&vring->num);
	if (num == 0 || micpm_get_reference(&blk->bd_info->bi_ctx, true)) {
		cleanup_vblk_workqueue(vbio, vq);
		return;
	}

	if (atomic64_read(&vbio->file->f_count) == 0) {  /* file is closed */
		ret = -1;
	} else if (vbio->type & VIRTIO_BLK_T_FLUSH)  {
#ifdef RHEL_RELEASE_CODE
#if (LINUX_VERSION_CODE >= KERNEL_VERSION(3,10,0))
		ret = vfs_fsync(vbio->file, 1);
#else
		ret = vfs_fsync(vbio->file, vbio->file->f_path.dentry, 1);
#endif
#else
		ret = vfs_fsync(vbio->file, 1);
#endif
	} else if (vbio->type & VIRTIO_BLK_T_OUT) {
	  for (iov = vbio->iov; iov < &vbio->iov[vbio->nvecs]; iov++) {
		iov->iov_base = mic_addr_in_host(aper_va, iov->iov_base);
	  }
		ret = vfs_writev(vbio->file, vbio->iov, vbio->nvecs, &pos);
	} else {
	  for (iov = vbio->iov; iov < &vbio->iov[vbio->nvecs]; iov++) {
		iov->iov_base = mic_addr_in_host(aper_va, iov->iov_base);
	  }
		ret = vfs_readv(vbio->file, vbio->iov, vbio->nvecs, &pos);
	}
	status = (ret < 0) ? VIRTIO_BLK_S_IOERR : VIRTIO_BLK_S_OK;
	if (vbio->head != -1) {
		INIT_LIST_HEAD(&single);
		list_add(&vbio->list, &single);
		head = &single;
		need_free = 0;
	} else {
		head = &vbio->list;
		need_free = 1;
	}
	list_for_each_entry(entry, head, list) {
		memcpy_toio(mic_addr_in_host(aper_va, entry->iov[entry->nvecs].iov_base), &status, sizeof(status));
	}
	mutex_lock(&vq->mutex);
	list_for_each_safe(node, tmp, head) {
		entry = list_entry(node, struct vhost_blk_io, list);
		vhost_add_used_and_signal(&blk->dev, vq, entry->head, ret);
		list_del(node);
		kfree(entry);
	}
	mutex_unlock(&vq->mutex);
	if (need_free)
		kfree(vbio);
	micpm_put_reference(&blk->bd_info->bi_ctx);
}

static struct vhost_blk_io *allocate_vbio(int nvecs)
{
	struct vhost_blk_io *vbio;
	int size = sizeof(struct vhost_blk_io) + nvecs * sizeof(struct iovec);
	vbio = kmalloc(size, GFP_KERNEL);
	if (vbio) {
		INIT_WORK(&vbio->work, handle_io_work);
		INIT_LIST_HEAD(&vbio->list);
	}
	return vbio;
}

static void merge_and_handoff_work(struct list_head *queue)
{
	struct vhost_blk_io *vbio, *entry;
	int nvecs = 0;
	int entries = 0;

	list_for_each_entry(entry, queue, list) {
		nvecs += entry->nvecs;
		entries++;
	}

	if (entries == 1) {
		vbio = list_first_entry(queue, struct vhost_blk_io, list);
		list_del(&vbio->list);
		queue_work(vbio->blk->vblk_workqueue, &vbio->work);
		return;
	}

	vbio = allocate_vbio(nvecs);
	if (!vbio) {
		/* Unable to allocate memory - submit IOs individually */
		list_for_each_entry(vbio, queue, list) {
			queue_work(vbio->blk->vblk_workqueue, &vbio->work);
		}
		INIT_LIST_HEAD(queue);
		return;
	}

	entry = list_first_entry(queue, struct vhost_blk_io, list);
	vbio->nvecs = nvecs;
	vbio->blk = entry->blk;
	vbio->file = entry->file;
	vbio->type = entry->type;
	vbio->sector = entry->sector;
	vbio->head = -1;
	vbio->len = 0;
	nvecs = 0;

	list_for_each_entry(entry, queue, list) {
		memcpy(&vbio->iov[nvecs], entry->iov, entry->nvecs * sizeof(struct iovec));
		nvecs += entry->nvecs;
		vbio->len += entry->len;
	}
	list_replace_init(queue, &vbio->list);
	queue_work(vbio->blk->vblk_workqueue, &vbio->work);
}

static void start_io(struct list_head *queue)
{
	struct list_head start;
	struct vhost_blk_io *vbio = NULL, *entry;

	if (list_empty(queue))
                return;

	list_for_each_entry(entry, queue, list) {
		if (!vbio) {
			vbio = entry;
			continue;
		}
		if (vbio->sector + (vbio->len >> MIC_SECTOR_SHIFT) == entry->sector) {
			vbio = entry;
		} else {
			INIT_LIST_HEAD(&start);
			list_cut_position(&start, queue, &vbio->list);
			merge_and_handoff_work(&start);
			vbio = entry;
		}
	}
	if (!list_empty(queue))
		merge_and_handoff_work(queue);
}

static uint64_t calculate_len(struct iovec *iov, int nvecs)
{
	uint64_t len = 0;
	int i;

	for (i=0; i<nvecs; i++)
		len += iov[i].iov_len;
	return len;
}

static void insert_to_queue(struct vhost_blk_io *vbio,
			struct list_head *queue)
{
	struct vhost_blk_io *entry;

	list_for_each_entry(entry, queue, list) {
		if (entry->sector > vbio->sector)
			break;
	}
	list_add_tail(&vbio->list, &entry->list);
}

static int handoff_io(struct vhost_blk *blk, int head,
			uint32_t type, uint64_t sector,
			struct iovec *iov, int nvecs)
{
	struct vhost_virtqueue *vq = &blk->dev.vqs[0];
	struct vhost_blk_io *vbio;

	vbio = allocate_vbio(nvecs+1);
	if (!vbio) {
		return -ENOMEM;
	}
	vbio->blk = blk;
	vbio->head = head;
	vbio->file = vq->private_data;
	vbio->type = type;
	vbio->sector = sector;
	vbio->nvecs = nvecs;
	vbio->len = calculate_len(iov, nvecs);
	memcpy(vbio->iov, iov, (nvecs + 1) * sizeof(struct iovec));

	if (vbio->type & VIRTIO_BLK_T_FLUSH) {
#if 0
		/* Sync called - do I need to submit IOs in the queue ? */
		start_io(&read_queue);
		start_io(&write_queue);
#endif
		queue_work(blk->vblk_workqueue, &vbio->work);
	} else if (vbio->type & VIRTIO_BLK_T_OUT) {
		insert_to_queue(vbio, &write_queue);
	} else {
		insert_to_queue(vbio, &read_queue);
	}
	return 0;
}

static void handle_blk(struct vhost_blk *blk)
{
	struct vhost_virtqueue *vq = &blk->dev.vqs[0];
	unsigned head, out, in;
	struct virtio_blk_outhdr hdr;
	int nvecs;
	struct board_info *bd_info = blk->bd_info;
	struct vring *vring;

	vring = &((struct mic_virtblk *)bd_info->bi_virtio)->vb_shared.vring;
	if (vring == 0 || readl(&vring->num) == 0) {
		printk("request comes in while card side driver is not loaded yet. Ignore\n");
		return;
	}
	/* the first time since the card side driver becomes ready */
	if (vq->desc == NULL || readb(&((struct mic_virtblk *)bd_info->bi_virtio)->vb_shared.update)) {
	  vq->num = readl(&vring->num);
	  vq->desc = (struct vring_desc *)readq(&vring->desc);
	  vq->avail = (struct vring_avail *)readq(&vring->avail);
	  vq->used = (struct vring_used *)readq(&vring->used);
	  vq->last_avail_idx = 0;
	  vq->avail_idx = 0;
	  vq->last_used_idx = 0;
	  vq->signalled_used = 0;
	  vq->signalled_used_valid = false;
	  vq->done_idx = 0;
	  writeb(false, &((struct mic_virtblk *)bd_info->bi_virtio)->vb_shared.update);
	}

	if (micpm_get_reference(&blk->bd_info->bi_ctx,  true))
		return;

	mutex_lock(&vq->mutex);

	vhost_disable_notify(&blk->dev, vq);

	for (;;) {
		head = vhost_get_vq_desc(&blk->dev, vq, vq->iov,
					 ARRAY_SIZE(vq->iov),
					 &out, &in, NULL, NULL);
		if ((head == vq->num) || (head == -EFAULT) || (head == -EINVAL)) {
			if (unlikely(vhost_enable_notify(&blk->dev, vq))) {
				vhost_disable_notify(&blk->dev, vq);
				continue;
			}
			start_io(&read_queue);
			start_io(&write_queue);
			break;
		}

		BUG_ON(vq->iov[0].iov_len != 16);

		memcpy_fromio(&hdr, mic_addr_in_host(bd_info->bi_ctx.aper.va, vq->iov[0].iov_base),
					  sizeof(hdr));

		nvecs = out - 1;
		if (hdr.type == VIRTIO_BLK_T_IN)
			nvecs = in - 1;

		BUG_ON(vq->iov[nvecs+1].iov_len != 1);
		if (handoff_io(blk, head, hdr.type, hdr.sector, &vq->iov[1], nvecs) < 0) {
		  vhost_discard_vq_desc(vq, 1);
			continue;
		}
	}
	mutex_unlock(&vq->mutex);
	micpm_put_reference(&blk->bd_info->bi_ctx);
}

static void handle_blk_kick(struct work_struct *work)
{
	struct vhost_blk *vblk;

	vblk = container_of(work, struct vhost_blk, vb_ws_bh);
	handle_blk(vblk);
}

#if 0
static void handle_rq_blk(struct vhost_work *work)
{
	struct vhost_blk *blk;

	blk = container_of(work, struct vhost_blk, poll[0].work);
	handle_blk(blk);
}
#endif

static int
vhost_doorbell_intr_handler(mic_ctx_t *mic_ctx, int doorbell)
{
	struct board_info *bi;
	struct vhost_blk *vblk;

	bi = container_of(mic_ctx, struct board_info, bi_ctx);
	vblk = ((struct mic_virtblk *)bi->bi_virtio)->vblk;
	queue_work(vblk->vb_wq, &vblk->vb_ws_bh);

	return 0;
}

static long vhost_blk_set_backend(struct vhost_blk *vblk)
{
	struct vhost_virtqueue *vq;
	struct board_info *bd_info = vblk->bd_info;
	unsigned index = bd_info->bi_ctx.bi_id;
	struct vb_shared *vb_shared;
	int ret = 0;
	struct kstat stat;
	unsigned int virtio_blk_features = (1U << VIRTIO_BLK_F_SEG_MAX) |
					   (1U << VIRTIO_BLK_F_BLK_SIZE);

	if (index >= MAX_BOARD_SUPPORTED) {
		ret = -ENOBUFS;
		goto _exit_;
	}
	if (vblk->virtblk_file == NULL) {
		ret = -EBADF;
		goto _exit_;
	}

	vq = &vblk->vqs[0];
	mutex_lock(&vq->mutex);
	rcu_assign_pointer(vq->private_data, vblk->virtblk_file);
	mutex_unlock(&vq->mutex);

	snprintf(vblk->vb_wqname, sizeof(vblk->vb_wqname),
		 "virtblk wq %d", index);
	vblk->vb_wq = __mic_create_singlethread_workqueue(vblk->vb_wqname);
	if (vblk->vb_wq == NULL) {
		ret = -ENOMEM;
		goto _exit_;
	}
	INIT_WORK(&vblk->vb_ws_bh, handle_blk_kick);

	/* They have to be accessed from "struct vhost_virtqueue *vq" in mic_vhost.c.
	   They are not used in vhost block. I don't modify vhost.h. */
	vq->log_base = (void __user *)&bd_info->bi_ctx;
	vq->log_addr = (u64)bd_info->bi_ctx.aper.va;

	vb_shared = &((struct mic_virtblk *)bd_info->bi_virtio)->vb_shared;
#if (LINUX_VERSION_CODE < KERNEL_VERSION(3,6,0))
	virtio_blk_features |= (1U << VIRTIO_BLK_F_FLUSH);
#endif
	writel(virtio_blk_features, &vb_shared->host_features);
	writel(DISK_SEG_MAX, &vb_shared->blk_config.seg_max);
	writel(MIC_SECTOR_SIZE, &vb_shared->blk_config.blk_size);

#if (LINUX_VERSION_CODE >= KERNEL_VERSION(3,9,0))
	ret = vfs_getattr(&vblk->virtblk_file->f_path, &stat);
#else
	ret = vfs_getattr(vblk->virtblk_file->f_path.mnt,
					  vblk->virtblk_file->f_path.dentry, &stat);
#endif
	if (ret < 0)
		goto _exit_;

	if (S_ISBLK(stat.mode)) {
	  writel(i_size_read(I_BDEV(vblk->virtblk_file->f_mapping->host)->bd_inode) / MIC_SECTOR_SIZE,
			 &vb_shared->blk_config.capacity);
	} else {
	  writel(stat.size / MIC_SECTOR_SIZE, &vb_shared->blk_config.capacity);
	}

	ret = mic_reg_irqhandler(&bd_info->bi_ctx, MIC_IRQ_DB2, "Host DoorBell 2",
				 vhost_doorbell_intr_handler);

_exit_:
	return ret;
}

void
mic_vhost_blk_stop(bd_info_t *bd_info)
{
	struct vring *vring = &((struct mic_virtblk *)bd_info->bi_virtio)->vb_shared.vring;

	writel(0, &vring->num);  /* reject subsequent request from MIC card */
}

extern bd_info_t *dev_to_bdi(struct device *dev);

ssize_t
show_virtblk_file(struct device *dev, struct device_attribute *attr, char *buf)
{
	struct board_info *bd_info = dev_to_bdi(dev);
	struct mic_virtblk *mic_virtblk;
	struct vhost_blk *vblk;

	BUG_ON(bd_info == NULL);
	mic_virtblk = bd_info->bi_virtio;
	BUG_ON(mic_virtblk == NULL);
	vblk = mic_virtblk->vblk;
	BUG_ON(vblk == NULL);

	if (vblk->file_name != NULL)
		return snprintf(buf, PAGE_SIZE, "%s\n", vblk->file_name);
	else
		return 0;
}

ssize_t
store_virtblk_file(struct device *dev, struct device_attribute *attr,
				   const char *buf, size_t count)
{
	int ret = 0;
	struct board_info *bd_info = dev_to_bdi(dev);
	struct mic_virtblk *mic_virtblk;
	struct vhost_blk *vblk;
	struct vhost_virtqueue *vq;
	char *p;
	struct file *virtblk_file;

	BUG_ON(bd_info == NULL);
	mic_virtblk = bd_info->bi_virtio;
	BUG_ON(mic_virtblk == NULL);
	vblk = mic_virtblk->vblk;
	BUG_ON(vblk == NULL);
	vq = &vblk->vqs[0];
	BUG_ON(vq == NULL);

	if (buf == NULL) {
		ret = -EINVAL;
		goto _return_;
	}
	if (count <= 1) {
		ret = -EINVAL;
		goto _return_;
	}

	p = strchr(buf, '\n');
	if (p != NULL)
		*p = '\0';

	mutex_lock(&vq->mutex);
	if (vblk->virtblk_file != NULL) {  /* if virtblk file is already assigned */
		printk(KERN_ALERT "you are changing virtblk file: %s -> %s.\n", vblk->file_name, buf);
		kfree(vblk->file_name);
		vblk->file_name = NULL;
		filp_close(vblk->virtblk_file, current->files);
		vblk->virtblk_file = NULL;
	}

	vblk->file_name = kmalloc(count + 1, GFP_KERNEL);
	strcpy(vblk->file_name, buf);
	virtblk_file = filp_open(vblk->file_name, O_RDWR|O_LARGEFILE, 0);
	if (IS_ERR(virtblk_file)) {
		ret = PTR_ERR(virtblk_file);
		mutex_unlock(&vq->mutex);
		goto free_file_name;
	}
	vblk->virtblk_file = virtblk_file;
	mutex_unlock(&vq->mutex);

	ret = vhost_blk_set_backend(vblk);
	if (ret < 0)
		goto close_virtblk_file;

	return count;

 close_virtblk_file:
	filp_close(vblk->virtblk_file, current->files);
 free_file_name:
	kfree(vblk->file_name);
 _return_:
	return ret;
}

int mic_vhost_blk_probe(bd_info_t *bd_info)
{
	int ret = 0;
	char wq_name[8];
	struct mic_virtblk *mic_virtblk;
	struct vhost_blk *vblk;

	mic_virtblk = kzalloc(sizeof(*mic_virtblk), GFP_KERNEL);
	if (mic_virtblk == NULL) {
		ret = -ENOMEM;
		goto err_vblk;
	}
	bd_info->bi_virtio = mic_virtblk;

	vblk = kzalloc(sizeof *vblk, GFP_KERNEL);
	if (vblk == NULL) {
		ret = -ENOMEM;
		goto free_mic_virtblk;
	}
	mic_virtblk->vblk = vblk;
	vblk->bd_info = bd_info;

	ret = vhost_dev_init(&vblk->dev, vblk->vqs, VHOST_BLK_VQ_MAX);
	if (ret < 0)
		goto free_vblk;

#if 0
	vhost_poll_init(vblk->poll, handle_rq_blk, POLLOUT|POLLIN, &vblk->dev);
#endif

	BUG_ON(bd_info->bi_ctx.bi_id >= 1000);
	snprintf(wq_name, ARRAY_SIZE(wq_name), "vblk%03d", bd_info->bi_ctx.bi_id);
	vblk->vblk_workqueue = __mic_create_singlethread_workqueue(wq_name);
	if (vblk->vblk_workqueue == NULL) {
		ret = -ENOMEM;
		goto free_vblk;
	}

	return ret;

 free_vblk:
	kfree(vblk);
 free_mic_virtblk:
	kfree(mic_virtblk);
 err_vblk:
	return ret;
}

void mic_vhost_blk_remove(bd_info_t *bd_info)
{
	struct mic_virtblk *mic_virtblk = bd_info->bi_virtio;
	struct vhost_blk *vblk = mic_virtblk->vblk;
	struct vb_shared *vb_shared = &mic_virtblk->vb_shared;

	if (vblk->virtblk_file != NULL) {
		mic_unreg_irqhandler(&bd_info->bi_ctx, MIC_IRQ_DB2, "Host DoorBell 2");
		memset(&vb_shared->blk_config, 0, sizeof(vb_shared->blk_config));
		destroy_workqueue(vblk->vb_wq);
		if (vblk->vqs[0].private_data != NULL)
			fput(vblk->vqs[0].private_data);
		kfree(vblk->file_name);
		filp_close(vblk->virtblk_file, current->files);
	}
	vhost_dev_cleanup(&vblk->dev);
	destroy_workqueue(vblk->vblk_workqueue);
	kfree(vblk);
	kfree(mic_virtblk);
}
#endif
Commit	Line	Data
800f879a AT	1	/*
	2	* Copyright (C) 2009 Red Hat, Inc.
	3	* Author: Michael S. Tsirkin <mst@redhat.com>
	4	*
	5	* This work is licensed under the terms of the GNU GPL, version 2.
	6
	7	* (C) Badari Pulavarty pbadari@us.ibm.com 2010 with the following comment.
	8	* He posted on http://lwn.net/Articles/382543/
	9
	10	* virtio-block server in host kernel.
	11	* Inspired by vhost-net and shamlessly ripped code from it :)
	12
	13	* For adapting to MIC
	14	* (C) Copyright 2012 Intel Corporation
	15	* Author: Caz Yokoyama <Caz.Yokoyama@intel.com>
	16	*/
	17	#include <linux/version.h>
	18	#if LINUX_VERSION_CODE >= KERNEL_VERSION(2,6,34) \|\| \
	19	defined(RHEL_RELEASE_CODE)
	20
	21	#include <linux/compat.h>
	22	#include <linux/eventfd.h>
	23	#include <linux/vhost.h>
	24	#include <linux/virtio_ring.h>
	25	#include <linux/virtio_net.h>
	26	#include <linux/virtio_blk.h>
	27	#include <linux/mmu_context.h>
	28	#include <linux/miscdevice.h>
	29	#include <linux/module.h>
	30	#include <linux/mutex.h>
	31	#include <linux/workqueue.h>
	32	#include <linux/rcupdate.h>
	33	#include <linux/file.h>
	34	#include <linux/fdtable.h>
	35
	36	#ifndef VIRTIO_RING_F_EVENT_IDX /* virtio_ring.h of rhel6.0 does not define */
	37	#define VIRTIO_RING_F_EVENT_IDX 29
	38	#endif
	39	#include "mic_common.h"
	40	#include "mic/micveth_dma.h"
	41	#include "vhost.h"
	42	#include "mic/mic_virtio.h"
	43
263db953 AT	44	#define MIC_SECTOR_SHIFT 9
263db953 AT	45	#define MIC_SECTOR_SIZE (1UL << MIC_SECTOR_SHIFT)
800f879a AT	46	#define VIRTIO_BLK_QUEUE_SIZE 128
	47	#define DISK_SEG_MAX (VIRTIO_BLK_QUEUE_SIZE - 2)
	48
	49	#define VHOST_BLK_VQ_MAX 1
	50	#define WQNAME_SIZE 16
	51
	52	struct vhost_blk {
	53	struct vhost_dev dev;
	54	struct vhost_virtqueue vqs[VHOST_BLK_VQ_MAX];
	55	struct vhost_poll poll[VHOST_BLK_VQ_MAX];
	56	struct workqueue_struct *vb_wq;
	57	char vb_wqname[WQNAME_SIZE];
	58	struct work_struct vb_ws_bh;
	59	struct workqueue_struct *vblk_workqueue;
	60	struct board_info *bd_info;
	61	char *file_name;
	62	struct file *virtblk_file;
	63	};
	64
	65	struct vhost_blk_io {
	66	struct list_head list;
	67	struct work_struct work;
	68	struct vhost_blk *blk;
	69	struct file *file;
	70	int head;
	71	uint32_t type;
	72	uint32_t nvecs;
	73	uint64_t sector;
	74	uint64_t len;
	75	struct iovec iov[0];
	76	};
	77
	78	#define mic_addr_in_host(va, pa) ((u8 *)(va) + (u64)(pa))
	79
	80	static LIST_HEAD(write_queue);
	81	static LIST_HEAD(read_queue);
	82
	83	static void
	84	cleanup_vblk_workqueue(struct vhost_blk_io vbio, struct vhost_virtqueue vq)
	85	{
	86	struct list_head single, head, node, *tmp;
	87	int need_free;
	88	struct vhost_blk_io *entry;
	89
	90	if (vbio->head != -1) {
	91	INIT_LIST_HEAD(&single);
	92	list_add(&vbio->list, &single);
	93	head = &single;
	94	need_free = 0;
	95	} else {
	96	head = &vbio->list;
	97	need_free = 1;
	98	}
	99
	100	mutex_lock(&vq->mutex);
	101	list_for_each_safe(node, tmp, head) {
	102	entry = list_entry(node, struct vhost_blk_io, list);
	103	list_del(node);
	104	kfree(entry);
	105	}
	106	mutex_unlock(&vq->mutex);
	107
	108	if (need_free)
	109	kfree(vbio);
110	}
111
112	static void handle_io_work(struct work_struct *work)
113	{
114	struct vhost_blk_io vbio, entry;
115	struct vhost_virtqueue *vq;
116	struct vhost_blk *blk;
117	struct list_head single, head, node, *tmp;
118	struct iovec *iov;
119	uint8_t *aper_va;
120	struct vring *vring;
121	unsigned int num;
122
123	int need_free, ret = 0;
124	loff_t pos;
125	uint8_t status = 0;
126
127	vbio = container_of(work, struct vhost_blk_io, work);
128	blk = vbio->blk;
129	vq = &blk->dev.vqs[0];
263db953	130	pos = vbio->sector << MIC_SECTOR_SHIFT;
800f879a AT	131	aper_va = blk->bd_info->bi_ctx.aper.va;
	132
	133	vring = &((struct mic_virtblk *)blk->bd_info->bi_virtio)->vb_shared.vring;
	134	num = readl(&vring->num);
	135	if (num == 0 \|\| micpm_get_reference(&blk->bd_info->bi_ctx, true)) {
	136	cleanup_vblk_workqueue(vbio, vq);
	137	return;
	138	}
	139
	140	if (atomic64_read(&vbio->file->f_count) == 0) { /* file is closed */
	141	ret = -1;
	142	} else if (vbio->type & VIRTIO_BLK_T_FLUSH) {
	143	#ifdef RHEL_RELEASE_CODE
	144	#if (LINUX_VERSION_CODE >= KERNEL_VERSION(3,10,0))
	145	ret = vfs_fsync(vbio->file, 1);
	146	#else
	147	ret = vfs_fsync(vbio->file, vbio->file->f_path.dentry, 1);
	148	#endif
	149	#else
	150	ret = vfs_fsync(vbio->file, 1);
	151	#endif
	152	} else if (vbio->type & VIRTIO_BLK_T_OUT) {
	153	for (iov = vbio->iov; iov < &vbio->iov[vbio->nvecs]; iov++) {
	154	iov->iov_base = mic_addr_in_host(aper_va, iov->iov_base);
	155	}
	156	ret = vfs_writev(vbio->file, vbio->iov, vbio->nvecs, &pos);
	157	} else {
	158	for (iov = vbio->iov; iov < &vbio->iov[vbio->nvecs]; iov++) {
	159	iov->iov_base = mic_addr_in_host(aper_va, iov->iov_base);
	160	}
	161	ret = vfs_readv(vbio->file, vbio->iov, vbio->nvecs, &pos);
	162	}
	163	status = (ret < 0) ? VIRTIO_BLK_S_IOERR : VIRTIO_BLK_S_OK;
	164	if (vbio->head != -1) {
	165	INIT_LIST_HEAD(&single);
	166	list_add(&vbio->list, &single);
	167	head = &single;
	168	need_free = 0;
	169	} else {
	170	head = &vbio->list;
	171	need_free = 1;
	172	}
	173	list_for_each_entry(entry, head, list) {
	174	memcpy_toio(mic_addr_in_host(aper_va, entry->iov[entry->nvecs].iov_base), &status, sizeof(status));
	175	}
	176	mutex_lock(&vq->mutex);
	177	list_for_each_safe(node, tmp, head) {
	178	entry = list_entry(node, struct vhost_blk_io, list);
	179	vhost_add_used_and_signal(&blk->dev, vq, entry->head, ret);
	180	list_del(node);
	181	kfree(entry);
	182	}
	183	mutex_unlock(&vq->mutex);
	184	if (need_free)
	185	kfree(vbio);
	186	micpm_put_reference(&blk->bd_info->bi_ctx);
	187	}
	188
	189	static struct vhost_blk_io *allocate_vbio(int nvecs)
	190	{
	191	struct vhost_blk_io *vbio;
	192	int size = sizeof(struct vhost_blk_io) + nvecs * sizeof(struct iovec);
	193	vbio = kmalloc(size, GFP_KERNEL);
	194	if (vbio) {
195	INIT_WORK(&vbio->work, handle_io_work);
196	INIT_LIST_HEAD(&vbio->list);
197	}
198	return vbio;
199	}
200
201	static void merge_and_handoff_work(struct list_head *queue)
202	{
203	struct vhost_blk_io vbio, entry;
204	int nvecs = 0;
205	int entries = 0;
206
207	list_for_each_entry(entry, queue, list) {
208	nvecs += entry->nvecs;
209	entries++;
210	}
211
212	if (entries == 1) {
213	vbio = list_first_entry(queue, struct vhost_blk_io, list);
214	list_del(&vbio->list);
215	queue_work(vbio->blk->vblk_workqueue, &vbio->work);
216	return;
217	}
218
219	vbio = allocate_vbio(nvecs);
220	if (!vbio) {
221	/* Unable to allocate memory - submit IOs individually */
222	list_for_each_entry(vbio, queue, list) {
223	queue_work(vbio->blk->vblk_workqueue, &vbio->work);
224	}
225	INIT_LIST_HEAD(queue);
226	return;
227	}
228
229	entry = list_first_entry(queue, struct vhost_blk_io, list);
230	vbio->nvecs = nvecs;
231	vbio->blk = entry->blk;
232	vbio->file = entry->file;
233	vbio->type = entry->type;
234	vbio->sector = entry->sector;
235	vbio->head = -1;
236	vbio->len = 0;
237	nvecs = 0;
238
239	list_for_each_entry(entry, queue, list) {
240	memcpy(&vbio->iov[nvecs], entry->iov, entry->nvecs * sizeof(struct iovec));
241	nvecs += entry->nvecs;
242	vbio->len += entry->len;
243	}
244	list_replace_init(queue, &vbio->list);
245	queue_work(vbio->blk->vblk_workqueue, &vbio->work);
246	}
247
248	static void start_io(struct list_head *queue)
249	{
250	struct list_head start;
251	struct vhost_blk_io vbio = NULL, entry;
252
253	if (list_empty(queue))
254	return;
255
256	list_for_each_entry(entry, queue, list) {
257	if (!vbio) {
258	vbio = entry;
259	continue;
260	}
263db953	261	if (vbio->sector + (vbio->len >> MIC_SECTOR_SHIFT) == entry->sector) {
800f879a AT	262	vbio = entry;
	263	} else {
	264	INIT_LIST_HEAD(&start);
	265	list_cut_position(&start, queue, &vbio->list);
	266	merge_and_handoff_work(&start);
	267	vbio = entry;
	268	}
	269	}
	270	if (!list_empty(queue))
	271	merge_and_handoff_work(queue);
	272	}
	273
	274	static uint64_t calculate_len(struct iovec *iov, int nvecs)
	275	{
	276	uint64_t len = 0;
	277	int i;
	278
	279	for (i=0; i<nvecs; i++)
	280	len += iov[i].iov_len;
	281	return len;
	282	}
	283
	284	static void insert_to_queue(struct vhost_blk_io *vbio,
	285	struct list_head *queue)
	286	{
	287	struct vhost_blk_io *entry;
	288
	289	list_for_each_entry(entry, queue, list) {
	290	if (entry->sector > vbio->sector)
	291	break;
	292	}
	293	list_add_tail(&vbio->list, &entry->list);
	294	}
	295
	296	static int handoff_io(struct vhost_blk *blk, int head,
	297	uint32_t type, uint64_t sector,
	298	struct iovec *iov, int nvecs)
	299	{
	300	struct vhost_virtqueue *vq = &blk->dev.vqs[0];
	301	struct vhost_blk_io *vbio;
	302
	303	vbio = allocate_vbio(nvecs+1);
	304	if (!vbio) {
	305	return -ENOMEM;
	306	}
	307	vbio->blk = blk;
	308	vbio->head = head;
	309	vbio->file = vq->private_data;
	310	vbio->type = type;
	311	vbio->sector = sector;
	312	vbio->nvecs = nvecs;
	313	vbio->len = calculate_len(iov, nvecs);
	314	memcpy(vbio->iov, iov, (nvecs + 1) * sizeof(struct iovec));
	315
	316	if (vbio->type & VIRTIO_BLK_T_FLUSH) {
	317	#if 0
	318	/* Sync called - do I need to submit IOs in the queue ? */
	319	start_io(&read_queue);
	320	start_io(&write_queue);
	321	#endif
	322	queue_work(blk->vblk_workqueue, &vbio->work);
	323	} else if (vbio->type & VIRTIO_BLK_T_OUT) {
	324	insert_to_queue(vbio, &write_queue);
	325	} else {
326	insert_to_queue(vbio, &read_queue);
327	}
328	return 0;
329	}
330
331	static void handle_blk(struct vhost_blk *blk)
332	{
333	struct vhost_virtqueue *vq = &blk->dev.vqs[0];
334	unsigned head, out, in;
335	struct virtio_blk_outhdr hdr;
336	int nvecs;
337	struct board_info *bd_info = blk->bd_info;
338	struct vring *vring;
339
340	vring = &((struct mic_virtblk *)bd_info->bi_virtio)->vb_shared.vring;
341	if (vring == 0 \|\| readl(&vring->num) == 0) {
342	printk("request comes in while card side driver is not loaded yet. Ignore\n");
343	return;
344	}
345	/* the first time since the card side driver becomes ready */
346	if (vq->desc == NULL \|\| readb(&((struct mic_virtblk *)bd_info->bi_virtio)->vb_shared.update)) {
347	vq->num = readl(&vring->num);
348	vq->desc = (struct vring_desc *)readq(&vring->desc);
349	vq->avail = (struct vring_avail *)readq(&vring->avail);
350	vq->used = (struct vring_used *)readq(&vring->used);
351	vq->last_avail_idx = 0;
352	vq->avail_idx = 0;
353	vq->last_used_idx = 0;
354	vq->signalled_used = 0;
355	vq->signalled_used_valid = false;
356	vq->done_idx = 0;
357	writeb(false, &((struct mic_virtblk *)bd_info->bi_virtio)->vb_shared.update);
358	}
359
360	if (micpm_get_reference(&blk->bd_info->bi_ctx, true))
361	return;
362
363	mutex_lock(&vq->mutex);
364
365	vhost_disable_notify(&blk->dev, vq);
366
367	for (;;) {
368	head = vhost_get_vq_desc(&blk->dev, vq, vq->iov,
369	ARRAY_SIZE(vq->iov),
370	&out, &in, NULL, NULL);
371	if ((head == vq->num) \|\| (head == -EFAULT) \|\| (head == -EINVAL)) {
372	if (unlikely(vhost_enable_notify(&blk->dev, vq))) {
373	vhost_disable_notify(&blk->dev, vq);
374	continue;
375	}
376	start_io(&read_queue);
377	start_io(&write_queue);
378	break;
379	}
380
381	BUG_ON(vq->iov[0].iov_len != 16);
382
383	memcpy_fromio(&hdr, mic_addr_in_host(bd_info->bi_ctx.aper.va, vq->iov[0].iov_base),
384	sizeof(hdr));
385
386	nvecs = out - 1;
387	if (hdr.type == VIRTIO_BLK_T_IN)
388	nvecs = in - 1;
389
390	BUG_ON(vq->iov[nvecs+1].iov_len != 1);
391	if (handoff_io(blk, head, hdr.type, hdr.sector, &vq->iov[1], nvecs) < 0) {
392	vhost_discard_vq_desc(vq, 1);
393	continue;
394	}
395	}
396	mutex_unlock(&vq->mutex);
397	micpm_put_reference(&blk->bd_info->bi_ctx);
398	}
399
400	static void handle_blk_kick(struct work_struct *work)
401	{
402	struct vhost_blk *vblk;
403
404	vblk = container_of(work, struct vhost_blk, vb_ws_bh);
405	handle_blk(vblk);
406	}
407
408	#if 0
409	static void handle_rq_blk(struct vhost_work *work)
410	{
411	struct vhost_blk *blk;
412
413	blk = container_of(work, struct vhost_blk, poll[0].work);
414	handle_blk(blk);
415	}
416	#endif
417
418	static int
419	vhost_doorbell_intr_handler(mic_ctx_t *mic_ctx, int doorbell)
420	{
421	struct board_info *bi;
422	struct vhost_blk *vblk;
423
424	bi = container_of(mic_ctx, struct board_info, bi_ctx);
425	vblk = ((struct mic_virtblk *)bi->bi_virtio)->vblk;
426	queue_work(vblk->vb_wq, &vblk->vb_ws_bh);
427
428	return 0;
429	}
430
431	static long vhost_blk_set_backend(struct vhost_blk *vblk)
432	{
433	struct vhost_virtqueue *vq;
434	struct board_info *bd_info = vblk->bd_info;
435	unsigned index = bd_info->bi_ctx.bi_id;
436	struct vb_shared *vb_shared;
437	int ret = 0;
438	struct kstat stat;
439	unsigned int virtio_blk_features = (1U << VIRTIO_BLK_F_SEG_MAX) \|
440	(1U << VIRTIO_BLK_F_BLK_SIZE);
441
442	if (index >= MAX_BOARD_SUPPORTED) {
443	ret = -ENOBUFS;
444	goto _exit_;
445	}
446	if (vblk->virtblk_file == NULL) {
447	ret = -EBADF;
448	goto _exit_;
449	}
450
451	vq = &vblk->vqs[0];
452	mutex_lock(&vq->mutex);
453	rcu_assign_pointer(vq->private_data, vblk->virtblk_file);
454	mutex_unlock(&vq->mutex);
455
456	snprintf(vblk->vb_wqname, sizeof(vblk->vb_wqname),
457	"virtblk wq %d", index);
458	vblk->vb_wq = __mic_create_singlethread_workqueue(vblk->vb_wqname);
459	if (vblk->vb_wq == NULL) {
460	ret = -ENOMEM;
461	goto _exit_;
462	}
463	INIT_WORK(&vblk->vb_ws_bh, handle_blk_kick);
464
465	/* They have to be accessed from "struct vhost_virtqueue *vq" in mic_vhost.c.
466	They are not used in vhost block. I don't modify vhost.h. */
467	vq->log_base = (void __user *)&bd_info->bi_ctx;
468	vq->log_addr = (u64)bd_info->bi_ctx.aper.va;
469
470	vb_shared = &((struct mic_virtblk *)bd_info->bi_virtio)->vb_shared;
471	#if (LINUX_VERSION_CODE < KERNEL_VERSION(3,6,0))
472	virtio_blk_features \|= (1U << VIRTIO_BLK_F_FLUSH);
473	#endif
474	writel(virtio_blk_features, &vb_shared->host_features);
475	writel(DISK_SEG_MAX, &vb_shared->blk_config.seg_max);
263db953	476	writel(MIC_SECTOR_SIZE, &vb_shared->blk_config.blk_size);
800f879a AT	477
	478	#if (LINUX_VERSION_CODE >= KERNEL_VERSION(3,9,0))
	479	ret = vfs_getattr(&vblk->virtblk_file->f_path, &stat);
	480	#else
	481	ret = vfs_getattr(vblk->virtblk_file->f_path.mnt,
	482	vblk->virtblk_file->f_path.dentry, &stat);
	483	#endif
	484	if (ret < 0)
	485	goto _exit_;
	486
	487	if (S_ISBLK(stat.mode)) {
263db953	488	writel(i_size_read(I_BDEV(vblk->virtblk_file->f_mapping->host)->bd_inode) / MIC_SECTOR_SIZE,
800f879a AT	489	&vb_shared->blk_config.capacity);
800f879a AT	490	} else {
263db953	491	writel(stat.size / MIC_SECTOR_SIZE, &vb_shared->blk_config.capacity);
800f879a AT	492	}
	493
	494	ret = mic_reg_irqhandler(&bd_info->bi_ctx, MIC_IRQ_DB2, "Host DoorBell 2",
	495	vhost_doorbell_intr_handler);
	496
	497	_exit_:
	498	return ret;
	499	}
	500
	501	void
	502	mic_vhost_blk_stop(bd_info_t *bd_info)
	503	{
	504	struct vring vring = &((struct mic_virtblk )bd_info->bi_virtio)->vb_shared.vring;
	505
	506	writel(0, &vring->num); /* reject subsequent request from MIC card */
	507	}
	508
	509	extern bd_info_t dev_to_bdi(struct device dev);
	510
	511	ssize_t
	512	show_virtblk_file(struct device dev, struct device_attribute attr, char *buf)
	513	{
	514	struct board_info *bd_info = dev_to_bdi(dev);
	515	struct mic_virtblk *mic_virtblk;
	516	struct vhost_blk *vblk;
	517
	518	BUG_ON(bd_info == NULL);
	519	mic_virtblk = bd_info->bi_virtio;
	520	BUG_ON(mic_virtblk == NULL);
	521	vblk = mic_virtblk->vblk;
	522	BUG_ON(vblk == NULL);
	523
	524	if (vblk->file_name != NULL)
	525	return snprintf(buf, PAGE_SIZE, "%s\n", vblk->file_name);
	526	else
	527	return 0;
	528	}
	529
	530	ssize_t
	531	store_virtblk_file(struct device dev, struct device_attribute attr,
	532	const char *buf, size_t count)
	533	{
	534	int ret = 0;
	535	struct board_info *bd_info = dev_to_bdi(dev);
	536	struct mic_virtblk *mic_virtblk;
	537	struct vhost_blk *vblk;
	538	struct vhost_virtqueue *vq;
	539	char *p;
	540	struct file *virtblk_file;
	541
	542	BUG_ON(bd_info == NULL);
	543	mic_virtblk = bd_info->bi_virtio;
	544	BUG_ON(mic_virtblk == NULL);
	545	vblk = mic_virtblk->vblk;
	546	BUG_ON(vblk == NULL);
	547	vq = &vblk->vqs[0];
	548	BUG_ON(vq == NULL);
	549
	550	if (buf == NULL) {
	551	ret = -EINVAL;
	552	goto _return_;
	553	}
	554	if (count <= 1) {
	555	ret = -EINVAL;
556	goto _return_;
557	}
558
559	p = strchr(buf, '\n');
560	if (p != NULL)
561	*p = '\0';
562
563	mutex_lock(&vq->mutex);
564	if (vblk->virtblk_file != NULL) { /* if virtblk file is already assigned */
565	printk(KERN_ALERT "you are changing virtblk file: %s -> %s.\n", vblk->file_name, buf);
566	kfree(vblk->file_name);
567	vblk->file_name = NULL;
568	filp_close(vblk->virtblk_file, current->files);
569	vblk->virtblk_file = NULL;
570	}
571
572	vblk->file_name = kmalloc(count + 1, GFP_KERNEL);
573	strcpy(vblk->file_name, buf);
574	virtblk_file = filp_open(vblk->file_name, O_RDWR\|O_LARGEFILE, 0);
575	if (IS_ERR(virtblk_file)) {
576	ret = PTR_ERR(virtblk_file);
577	mutex_unlock(&vq->mutex);
578	goto free_file_name;
579	}
580	vblk->virtblk_file = virtblk_file;
581	mutex_unlock(&vq->mutex);
582
583	ret = vhost_blk_set_backend(vblk);
584	if (ret < 0)
585	goto close_virtblk_file;
586
587	return count;
588
589	close_virtblk_file:
590	filp_close(vblk->virtblk_file, current->files);
591	free_file_name:
592	kfree(vblk->file_name);
593	_return_:
594	return ret;
595	}
596
597	int mic_vhost_blk_probe(bd_info_t *bd_info)
598	{
599	int ret = 0;
600	char wq_name[8];
601	struct mic_virtblk *mic_virtblk;
602	struct vhost_blk *vblk;
603
604	mic_virtblk = kzalloc(sizeof(*mic_virtblk), GFP_KERNEL);
605	if (mic_virtblk == NULL) {
606	ret = -ENOMEM;
607	goto err_vblk;
608	}
609	bd_info->bi_virtio = mic_virtblk;
610
611	vblk = kzalloc(sizeof *vblk, GFP_KERNEL);
612	if (vblk == NULL) {
613	ret = -ENOMEM;
614	goto free_mic_virtblk;
615	}
616	mic_virtblk->vblk = vblk;
617	vblk->bd_info = bd_info;
618
619	ret = vhost_dev_init(&vblk->dev, vblk->vqs, VHOST_BLK_VQ_MAX);
620	if (ret < 0)
621	goto free_vblk;
622
623	#if 0
624	vhost_poll_init(vblk->poll, handle_rq_blk, POLLOUT\|POLLIN, &vblk->dev);
625	#endif
626
627	BUG_ON(bd_info->bi_ctx.bi_id >= 1000);
628	snprintf(wq_name, ARRAY_SIZE(wq_name), "vblk%03d", bd_info->bi_ctx.bi_id);
629	vblk->vblk_workqueue = __mic_create_singlethread_workqueue(wq_name);
630	if (vblk->vblk_workqueue == NULL) {
631	ret = -ENOMEM;
632	goto free_vblk;
633	}
634
635	return ret;
636
637	free_vblk:
638	kfree(vblk);
639	free_mic_virtblk:
640	kfree(mic_virtblk);
641	err_vblk:
642	return ret;
643	}
644
645	void mic_vhost_blk_remove(bd_info_t *bd_info)
646	{
647	struct mic_virtblk *mic_virtblk = bd_info->bi_virtio;
648	struct vhost_blk *vblk = mic_virtblk->vblk;
649	struct vb_shared *vb_shared = &mic_virtblk->vb_shared;
650
651	if (vblk->virtblk_file != NULL) {
652	mic_unreg_irqhandler(&bd_info->bi_ctx, MIC_IRQ_DB2, "Host DoorBell 2");
653	memset(&vb_shared->blk_config, 0, sizeof(vb_shared->blk_config));
654	destroy_workqueue(vblk->vb_wq);
655	if (vblk->vqs[0].private_data != NULL)
656	fput(vblk->vqs[0].private_data);
657	kfree(vblk->file_name);
658	filp_close(vblk->virtblk_file, current->files);
659	}
660	vhost_dev_cleanup(&vblk->dev);
661	destroy_workqueue(vblk->vblk_workqueue);
662	kfree(vblk);
663	kfree(mic_virtblk);
664	}
665	#endif