Updated `README.md` with instructions for building/using the kernel module.

[xeon-phi-kernel-module] / dma / mic_dma_md.c

/*
 * Copyright 2010-2017 Intel Corporation.
 *
 * This program is free software; you can redistribute it and/or modify
 * it under the terms of the GNU General Public License, version 2,
 * as published by the Free Software Foundation.
 *
 * This program is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
 * General Public License for more details.
 *
 * Disclaimer: The codes contained in these modules may be specific to
 * the Intel Software Development Platform codenamed Knights Ferry,
 * and the Intel product codenamed Knights Corner, and are not backward
 * compatible with other Intel products. Additionally, Intel will NOT
 * support the codes or instruction set in future products.
 *
 * Intel offers no warranty of any kind regarding the code. This code is
 * licensed on an "AS IS" basis and Intel is not obligated to provide
 * any support, assistance, installation, training, or other services
 * of any kind. Intel is also not obligated to provide any updates,
 * enhancements or extensions. Intel specifically disclaims any warranty
 * of merchantability, non-infringement, fitness for any particular
 * purpose, and any other warranty.
 *
 * Further, Intel disclaims all liability of any kind, including but
 * not limited to liability for infringement of any proprietary rights,
 * relating to the use of the code, even if Intel is notified of the
 * possibility of such liability. Except as expressly stated in an Intel
 * license agreement provided with this code and agreed upon with Intel,
 * no license, express or implied, by estoppel or otherwise, to any
 * intellectual property rights is granted herein.
 */

#include<linux/module.h>
#include<linux/slab.h>
#include<asm/io.h>
#include<linux/kernel.h>

#include <mic/micscif_smpt.h>
#include <mic/mic_dma_md.h>
#include <mic/mic_dma_api.h>

#define PR_PREFIX "DMA_LIB_MD:"

#ifdef CONFIG_ML1OM
#define MIC_DMA_AES_CHAN_NUM	7
#define is_AES_channel(n) ((n) == MIC_DMA_AES_CHAN_NUM)
#else
#define is_AES_channel(n) ((void)(n), 0)
#endif

#define DMA_CHAN_COOKIE		0xdeadc0d

#define SBOX_DCAR_IM0		(0x1 << 24)	// APIC Interrupt mask bit
#define SBOX_DCAR_IM1		(0x1 << 25)	// MSI-X Interrupt mask bit
#define SBOX_DCAR_IS0		(0x1 << 26)	// Interrupt status

#define SBOX_DRARHI_SYS_MASK	(0x1 << 26)

#ifdef _MIC_SCIF_
static inline uint32_t chan_to_dcr_mask(uint32_t dcr, struct md_mic_dma_chan *chan, struct mic_dma_device *dma_dev)
{
	uint32_t chan_num = chan->ch_num;
	uint32_t owner;

	if (!is_AES_channel(chan_num))
		owner = chan->owner;
	else
		owner = chan->endianness;

	return ((dcr & ~(0x1 << (chan_num * 2))) | (owner << (chan_num * 2)));
}
#endif

static inline uint32_t drar_hi_to_ba_bits(uint32_t drar_hi)
{
	/*
	 * Setting bits 3:2 should generate a DESC_ADDR_ERR but the hardware ignores
	 * these bits currently and doesn't generate the error.
	 */
#ifdef _MIC_SCIF_
	return drar_hi & 0xf;
#else
	return drar_hi & 0x3;
#endif
}

static inline uint32_t physaddr_to_drarhi_ba(phys_addr_t phys_addr)
{
	return drar_hi_to_ba_bits((uint32_t)(phys_addr >> 32));
}

static inline uint32_t size_to_drar_hi_size(uint32_t size)
{
	return (size & 0x1ffff) << 4;
}

static inline uint32_t addr_to_drar_hi_smpt_bits(phys_addr_t mic_phys_addr)
{
	return ((mic_phys_addr >> MIC_SYSTEM_PAGE_SHIFT) & 0x1f) << 21;
}

static inline uint32_t drar_hi_to_smpt(uint32_t drar_hi, uint32_t chan_num)
{
	return ((drar_hi >> 21) & 0x1f);
}

void md_mic_dma_enable_chan(struct mic_dma_device *dma_dev, uint32_t chan_num, bool enable);


#ifdef _MIC_SCIF_
/**
 * md_mic_dma_chan_init_attr - Set channel attributes like owner and endianness
 * @chan: The DMA channel handle
 */
void md_mic_dma_chan_init_attr(struct mic_dma_device *dma_dev,
				      struct md_mic_dma_chan *chan)
{
	uint32_t dcr;

	CHECK_CHAN(chan);

	dcr = mic_sbox_read_mmio(dma_dev->mm_sbox, SBOX_DCR);
	dcr = chan_to_dcr_mask(dcr, chan, dma_dev);
	mic_sbox_write_mmio(dma_dev->mm_sbox, SBOX_DCR, dcr);
}
#endif

/* One time DMA Init API */
void md_mic_dma_init(struct mic_dma_device *dma_dev, uint8_t *mmio_va_base)
{
	int i;
#ifdef _MIC_SCIF_
	dma_dev->mm_sbox = mic_sbox_md_init();
#else
	dma_dev->mm_sbox = mmio_va_base;
#endif
	//pr_debug("sbox: va=%p\n", dma_dev.mm_sbox);

	for (i = 0; i < MAX_NUM_DMA_CHAN; i++) {
		atomic_set(&(dma_dev->chan_info[i].in_use), CHAN_AVAILABLE);
		dma_dev->chan_info[i].cookie = DMA_CHAN_COOKIE;
		dma_dev->chan_info[i].dstat_wb_phys = 0;
		dma_dev->chan_info[i].dstat_wb_loc = NULL;
	}
	return;
}

/* One time DMA Uninit API */
void md_mic_dma_uninit(struct mic_dma_device *dma_dev)
{
	return;
}

/**
 * md_mic_dma_request_chan
 * @owner: DMA channel owner: MIC or Host
 *
 * Return - The DMA channel handle or NULL if failed
 *
 * Note: Allocating a Host owned channel is not allowed currently
 */
struct md_mic_dma_chan *md_mic_dma_request_chan(struct mic_dma_device *dma_dev,
						enum md_mic_dma_chan_owner owner)
{
	struct md_mic_dma_chan *tmp = NULL;
	int i;

	for (i = 0; i < MAX_NUM_DMA_CHAN; i++) {
		if (CHAN_AVAILABLE == atomic_cmpxchg(&(dma_dev->chan_info[i].in_use), 
							   CHAN_AVAILABLE, CHAN_INUSE)) {
			tmp = &dma_dev->chan_info[i];
			tmp->owner = owner;
			tmp->ch_num = i;
			/*
			 * Setting endianness by default to MIC_LITTLE_ENDIAN
			 * in case the AES channel is used for clear transfers
			 * This is a don't care for clear transfers.
			 */
			tmp->endianness = MIC_LITTLE_ENDIAN;
#ifdef _MIC_SCIF_
			md_mic_dma_chan_init_attr(dma_dev, tmp);
#endif
			break;
		}
	}
	return tmp;
}

/**
 * md_mic_dma_free_chan - Frees up a DMA channel
 * @chan: The DMA channel handle
 */
void md_mic_dma_free_chan(struct mic_dma_device *dma_dev,
			  struct md_mic_dma_chan *chan)
{
	CHECK_CHAN(chan);
	atomic_set(&(chan->in_use), CHAN_AVAILABLE);
	md_mic_dma_enable_chan(dma_dev, chan->ch_num, false);
}

/**
 * md_mic_dma_enable_chan - Enable/disable the DMA channel
 * @chan_num: The DMA channel
 * @enable: enable/disable
 *
 * Must set desc ring and update head pointer only
 * after disabling the channel
 */
void md_mic_dma_enable_chan(struct mic_dma_device *dma_dev,
			    uint32_t chan_num, bool enable)
{
	uint32_t dcr = mic_sbox_read_mmio(dma_dev->mm_sbox, SBOX_DCR);

	/*
	 * There is a separate bit for every channel.
	 * Look up sboxDcrReg.
	 */
	if (enable) {
		dcr |= 2 << (chan_num << 1);
	} else {
		dcr &= ~(2 << (chan_num << 1));
	}
	mic_sbox_write_mmio(dma_dev->mm_sbox, SBOX_DCR, dcr);
}

#if 0
uint32_t md_mic_dma_chan_read_completion_count(struct mic_dma_device *dma_dev, struct md_mic_dma_chan *chan)
{
	CHECK_CHAN(chan);

	return (md_mic_dma_read_mmio(dma_dev, chan->ch_num, REG_DSTAT) & 0xffff);
}


/* This function needs to be used only in error case */
void update_compcount_and_tail(struct mic_dma_device *dma_dev, struct md_mic_dma_chan *chan)
{
	chan->completion_count = md_mic_dma_chan_read_completion_count(dma_dev, chan);
	chan->cached_tail = md_mic_dma_chan_read_tail(dma_dev, chan);
}
#endif
void md_mic_dma_chan_set_dstat_wb(struct mic_dma_device *dma_dev,
					struct md_mic_dma_chan *chan)
{
	uint32_t dstat_wb, dstat_wb_hi;
	CHECK_CHAN(chan);

	dstat_wb = (uint32_t)chan->dstat_wb_phys;
	dstat_wb_hi = chan->dstat_wb_phys >> 32;
	md_mic_dma_write_mmio(dma_dev, chan->ch_num, REG_DSTATWB_LO, dstat_wb);
	md_mic_dma_write_mmio(dma_dev, chan->ch_num, REG_DSTATWB_HI, dstat_wb_hi);
}

void md_mic_dma_chan_set_dcherr_msk(struct mic_dma_device *dma_dev,
				struct md_mic_dma_chan *chan, uint32_t mask)
{
	CHECK_CHAN(chan);
	md_mic_dma_write_mmio(dma_dev, chan->ch_num, REG_DCHERRMSK, mask);
}
#if 0
uint32_t md_mic_dma_chan_get_dcherr_msk(struct mic_dma_device *dma_dev,
				struct md_mic_dma_chan *chan)
{
	CHECK_CHAN(chan);
	return md_mic_dma_read_mmio(dma_dev, chan->ch_num, REG_DCHERRMSK);
}

uint32_t md_mic_dma_chan_get_dcherr(struct mic_dma_device *dma_dev,
				struct md_mic_dma_chan *chan)
{
	CHECK_CHAN(chan);
	return md_mic_dma_read_mmio(dma_dev, chan->ch_num, REG_DCHERR);
}

void md_mic_dma_chan_set_dcherr(struct mic_dma_device *dma_dev,
				struct md_mic_dma_chan *chan, uint32_t value)
{
	CHECK_CHAN(chan);
	md_mic_dma_write_mmio(dma_dev, chan->ch_num, REG_DCHERR, value);
	printk("dcherr = %d\n", md_mic_dma_read_mmio(dma_dev, chan->ch_num, REG_DCHERR));
}
#endif

/**
 * md_mic_dma_chan_set_desc_ring - Configures the DMA channel desc ring
 * @chan: The DMA channel handle
 * @desc_ring_phys_addr: Physical address of the desc ring base. Needs to be
 *                       physically contiguous and wired down memory.
 * @num_desc: Number of descriptors must be a multiple of cache line size.
 * Descriptor size should be determined using sizeof(union md_mic_dma_desc).
 *            The maximum number of descriptors is defined by
 *            MIC_MAX_NUM_DESC_PER_RING.
 */
void md_mic_dma_chan_set_desc_ring(struct mic_dma_device *dma_dev,
				   struct md_mic_dma_chan *chan,
				   phys_addr_t desc_ring_phys_addr,
				   uint32_t num_desc)
{
	uint32_t chan_num;
	uint32_t drar_lo = 0;
	uint32_t drar_hi = 0;

	CHECK_CHAN(chan);
	chan_num = chan->ch_num;
	/*
	 * TODO: Maybe the 2nd condition should be different considering the
	 * size of union md_mic_dma_desc?
	 */
	KASSERT((((num_desc) <= MIC_MAX_NUM_DESC_PER_RING) &&
		(ALIGN((num_desc - (L1_CACHE_BYTES - 1)), L1_CACHE_BYTES) == num_desc)),
		"num_desc > max or not multiple of cache line num 0x%x", num_desc);

	md_mic_dma_enable_chan(dma_dev, chan_num, false);

	drar_hi = size_to_drar_hi_size(num_desc);

	if (MIC_DMA_CHAN_HOST_OWNED == chan->owner) {
		drar_hi |= SBOX_DRARHI_SYS_MASK;
		drar_hi |= addr_to_drar_hi_smpt_bits(desc_ring_phys_addr);
	}
	drar_lo = (uint32_t)desc_ring_phys_addr;
	drar_hi |= physaddr_to_drarhi_ba(desc_ring_phys_addr);
	md_mic_dma_write_mmio(dma_dev, chan_num, REG_DRAR_LO, drar_lo);
	md_mic_dma_write_mmio(dma_dev, chan_num, REG_DRAR_HI, drar_hi);
	chan->num_desc_in_ring = num_desc;
	pr_debug("md_mic_dma_chan_set_desc_ring addr=0x%llx num=%d drar_hi.bits.pageno 0x%x\n", 
			desc_ring_phys_addr, num_desc, 
			(uint32_t)(desc_ring_phys_addr >> MIC_SYSTEM_PAGE_SHIFT));
	chan->cached_tail = md_mic_dma_chan_read_tail(dma_dev, chan);

	md_mic_dma_enable_chan(dma_dev, chan_num, true);
}

uint32_t md_mic_dma_chan_read_head(struct mic_dma_device *dma_dev, struct md_mic_dma_chan *chan)
{
	CHECK_CHAN(chan);

	return md_mic_dma_read_mmio(dma_dev, chan->ch_num, REG_DHPR);
}

uint32_t md_mic_dma_chan_read_tail(struct mic_dma_device *dma_dev, struct md_mic_dma_chan *chan)
{
	CHECK_CHAN(chan);

	return md_mic_dma_read_mmio(dma_dev, chan->ch_num, REG_DTPR);
}

/**
 * md_mic_dma_chan_intr_pending - Reads interrupt status to figure out
 *                              if an interrupt is pending.
 * @chan: The DMA channel handle.
 */
bool md_mic_dma_chan_intr_pending(struct mic_dma_device *dma_dev, struct md_mic_dma_chan *chan)
{
	uint32_t dcar;
	CHECK_CHAN(chan);

	dcar = md_mic_dma_read_mmio(dma_dev, chan->ch_num, REG_DCAR);
	return (dcar >> 26) & 0x1;
}

/**
 * md_mic_dma_chan_mask_intr - Mask or disable interrupts
 * @chan: The DMA channel handle
 *
 * Masking interrupts will also acknowledge any pending
 * interrupts on the channel.
 */
void md_mic_dma_chan_mask_intr(struct mic_dma_device *dma_dev, struct md_mic_dma_chan *chan)
{
	uint32_t dcar;
	uint32_t chan_num;
	CHECK_CHAN(chan);
	chan_num = chan->ch_num;

	dcar = md_mic_dma_read_mmio(dma_dev, chan_num, REG_DCAR);

	if (MIC_DMA_CHAN_MIC_OWNED == chan->owner)
		dcar |= SBOX_DCAR_IM0;
	else
		dcar |= SBOX_DCAR_IM1;

	md_mic_dma_write_mmio(dma_dev, chan_num, REG_DCAR, dcar);
	/*
	 * This read is completed only after previous write is completed.
	 * It guarantees that, interrupts has been acknowledged to SBOX DMA
	 * This read forces previous write to be commited in memory.
	 * This is the actual fix for HSD# 3497216 based on theoretical
	 * hypothesis that somehow previous write is not truly completed
	 * since for writes as long as transactions are accepted by SBOX
	 * ( not necessarily commited in memory) those write transactions
	 * reported as complete.
	 */
	dcar = md_mic_dma_read_mmio(dma_dev, chan_num, REG_DCAR);
}

/**
 * md_mic_dma_chan_unmask_intr - Unmask or enable interrupts
 * @chan: The DMA channel handle
 */
void md_mic_dma_chan_unmask_intr(struct mic_dma_device *dma_dev, struct md_mic_dma_chan *chan)
{
	uint32_t dcar;
	uint32_t chan_num;
	CHECK_CHAN(chan);
	chan_num = chan->ch_num;

	dcar = md_mic_dma_read_mmio(dma_dev, chan_num, REG_DCAR);

	if (MIC_DMA_CHAN_MIC_OWNED == chan->owner)
		dcar &= ~SBOX_DCAR_IM0;
	else
		dcar &= ~SBOX_DCAR_IM1;

	md_mic_dma_write_mmio(dma_dev, chan_num, REG_DCAR, dcar);
	/*
	 * This read is completed only after previous write is completed.
	 * It guarantees that, interrupts has been acknowledged to SBOX DMA
	 * This read forces previous write to be commited in memory.
	 * This is the actual fix for HSD# 3497216 based on theoretical
	 * hypothesis that somehow previous write is not truly completed
	 * since for writes as long as transactions are accepted by SBOX
	 * ( not necessarily commited in memory) those write transactions
	 * reported as complete.
	 */
	dcar = md_mic_dma_read_mmio(dma_dev, chan_num, REG_DCAR);
}

/**
 * md_mic_dma_chan_get_desc_ring_phys - Compute the value of the descriptor ring
 * base physical address from the descriptor ring attributes register.
 * @dma_dev: DMA device.
 * @chan: The DMA channel handle
 */
phys_addr_t
md_mic_dma_chan_get_desc_ring_phys(struct mic_dma_device *dma_dev, struct md_mic_dma_chan *chan)
{
	phys_addr_t phys, phys_hi;
	uint32_t phys_lo, chan_num, drar_hi;

	CHECK_CHAN(chan);
	chan_num = chan->ch_num;
	phys_lo = md_mic_dma_read_mmio(dma_dev, chan_num, REG_DRAR_LO);
	drar_hi = md_mic_dma_read_mmio(dma_dev, chan_num, REG_DRAR_HI);
	phys_hi = drar_hi_to_ba_bits(drar_hi);
	phys_hi |= drar_hi_to_smpt(drar_hi, chan_num) << 2;

	phys = phys_lo | (phys_hi << 32);
	return phys;
}

/**
 * md_mic_dma_chan_get_dstatwb_phys - Compute the value of the DSTAT write back
 * physical address.
 * @dma_dev: DMA device.
 * @chan: The DMA channel handle
 */
phys_addr_t md_mic_dma_chan_get_dstatwb_phys(struct mic_dma_device *dma_dev,
			struct md_mic_dma_chan *chan)
{
	uint32_t reg, chan_num;
	phys_addr_t phys;

	CHECK_CHAN(chan);
	chan_num = chan->ch_num;
	reg = md_mic_dma_read_mmio(dma_dev, chan_num, REG_DSTATWB_HI);
	phys = reg;
	reg = md_mic_dma_read_mmio(dma_dev, chan_num, REG_DSTATWB_LO);

	phys = phys << 32 | reg;
	return phys;
}

/**
 * md_mic_dma_prep_nop_desc - Prepares a NOP descriptor.
 * @desc: Descriptor to be populated.
 *
 * This descriptor is used to pad a cacheline if the previous
 * descriptor does not end on a cacheline boundary.
 */
void md_mic_dma_prep_nop_desc(union md_mic_dma_desc *desc)
{
	KASSERT((desc != 0), ("NULL desc"));

	desc->qwords.qw0 = 0;
	desc->qwords.qw1 = 0;
	desc->desc.nop.type = 0;
}

/* Only Debug Code Below */

/**
 * md_mic_dma_print_debug - Print channel debug information
 * @chan: The DMA channel handle
 * @sbuf: Print to an sbuf if not NULL else prints to console
 */
void md_mic_dma_print_debug(struct mic_dma_device *dma_dev, struct md_mic_dma_chan *chan)
{
	uint32_t dcr;
	uint32_t dcar;
	uint32_t dtpr;
	uint32_t dhpr;
	uint32_t drar_lo;
	uint32_t drar_hi;
	uint32_t dstat;
	uint32_t chan_num = chan->ch_num;

	dcr  = mic_sbox_read_mmio(dma_dev->mm_sbox, SBOX_DCR);
	dcar = md_mic_dma_read_mmio(dma_dev, chan_num, REG_DCAR);
	dtpr = md_mic_dma_read_mmio(dma_dev, chan_num, REG_DTPR);
	dhpr = md_mic_dma_read_mmio(dma_dev, chan_num, REG_DHPR);
	drar_lo = md_mic_dma_read_mmio(dma_dev, chan_num, REG_DRAR_LO);
	drar_hi = md_mic_dma_read_mmio(dma_dev, chan_num, REG_DRAR_HI);
	dstat = md_mic_dma_read_mmio(dma_dev, chan_num, REG_DSTAT);
	pr_debug(PR_PREFIX "Chan_Num 0x%x DCR 0x%x DCAR 0x%x DTPR 0x%x" 
			      "DHPR 0x%x DRAR_HI 0x%x DRAR_LO 0x%x DSTAT 0x%x\n", 
			      chan_num, dcr, dcar, dtpr, dhpr, drar_hi, drar_lo, dstat);
	pr_debug(PR_PREFIX "DCR 0x%x\n", dcr);
}