Updated `README.md` with instructions for building/using the kernel module.

[xeon-phi-kernel-module] / micscif / micscif_smpt.c

/*
 * Copyright 2010-2017 Intel Corporation.
 *
 * This program is free software; you can redistribute it and/or modify
 * it under the terms of the GNU General Public License, version 2,
 * as published by the Free Software Foundation.
 *
 * This program is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
 * General Public License for more details.
 *
 * Disclaimer: The codes contained in these modules may be specific to
 * the Intel Software Development Platform codenamed Knights Ferry,
 * and the Intel product codenamed Knights Corner, and are not backward
 * compatible with other Intel products. Additionally, Intel will NOT
 * support the codes or instruction set in future products.
 *
 * Intel offers no warranty of any kind regarding the code. This code is
 * licensed on an "AS IS" basis and Intel is not obligated to provide
 * any support, assistance, installation, training, or other services
 * of any kind. Intel is also not obligated to provide any updates,
 * enhancements or extensions. Intel specifically disclaims any warranty
 * of merchantability, non-infringement, fitness for any particular
 * purpose, and any other warranty.
 *
 * Further, Intel disclaims all liability of any kind, including but
 * not limited to liability for infringement of any proprietary rights,
 * relating to the use of the code, even if Intel is notified of the
 * possibility of such liability. Except as expressly stated in an Intel
 * license agreement provided with this code and agreed upon with Intel,
 * no license, express or implied, by estoppel or otherwise, to any
 * intellectual property rights is granted herein.
 */

#include <mic/micscif.h>
#include <mic/micscif_smpt.h>
#if defined(HOST) || defined(WINDOWS)
#include "mic_common.h"
#endif

struct _mic_ctx_t;
// Figure out which SMPT entry based on the host addr
#define SYSTEM_ADDR_TO_SMPT(sysaddr) 	((sysaddr) >> (MIC_SYSTEM_PAGE_SHIFT))
#define HOSTMIC_PA_TO_SMPT(hostmic_pa) (((hostmic_pa) - MIC_SYSTEM_BASE)\
						 >> MIC_SYSTEM_PAGE_SHIFT)

#define	NUM_SMPT_ENTRIES_IN_USE		32
#define SMPT_TO_MIC_PA(smpt_index) 	(MIC_SYSTEM_BASE + ((smpt_index) * \
						MIC_SYSTEM_PAGE_SIZE))
#define MAX_HOST_MEMORY 		((NUM_SMPT_ENTRIES_IN_USE) * MIC_SYSTEM_PAGE_SIZE)
#define MAX_SYSTEM_ADDR 		((MIC_SYSTEM_BASE) + (MAX_HOST_MEMORY) - (1))
#define IS_MIC_SYSTEM_ADDR(addr)	(((addr) >=  MIC_SYSTEM_BASE) && \
						 ((addr) <= MAX_SYSTEM_ADDR))

#define _PAGE_OFFSET(x) 	((x) & ((PAGE_SIZE) - (1ULL)))
#define SMPT_OFFSET(x) 		((x) & MIC_SYSTEM_PAGE_MASK)
#define PAGE_ALIGN_LOW(x)	ALIGN(((x) - ((PAGE_SIZE) - 1ULL)), (PAGE_SIZE))
#define PAGE_ALIGN_HIGH(x) 	ALIGN((x), (PAGE_SIZE))
#define SMPT_ALIGN_LOW(x) 	ALIGN(((x) - (MIC_SYSTEM_PAGE_MASK)), \
								(MIC_SYSTEM_PAGE_SIZE))
#define SMPT_ALIGN_HIGH(x) 	ALIGN((x), (MIC_SYSTEM_PAGE_SIZE))

#if defined(HOST)
#define SMPT_LOGGING			0
#if SMPT_LOGGING
static int64_t smpt_ref_count_g[MAX_BOARD_SUPPORTED];
static int64_t map_count_g;
static int64_t unmap_count_g;
#endif
#endif

void mic_smpt_set(volatile void *mm_sbox, uint64_t dma_addr, uint64_t index)
{
	uint32_t smpt_reg_val = BUILD_SMPT(SNOOP_ON, dma_addr >> MIC_SYSTEM_PAGE_SHIFT);
	writel(smpt_reg_val, (uint8_t*)mm_sbox + SBOX_SMPT00 + (4 * index));
}

#if defined(HOST)
/*
 * Called once per board as part of starting a MIC
 * to restore the SMPT state to the previous values
 * as stored in SMPT SW data structures.
 */
void mic_smpt_restore(mic_ctx_t *mic_ctx)
{
	int i;
	dma_addr_t dma_addr;
	uint32_t *smpt = (uint32_t*)(mic_ctx->mmio.va +
			 HOST_SBOX_BASE_ADDRESS + SBOX_SMPT00);
	uint32_t smpt_reg_val;

	for (i = 0; i < NUM_SMPT_ENTRIES_IN_USE; i++) {
		dma_addr = mic_ctx->mic_smpt[i].dma_addr;
		if (mic_ctx->bi_family == FAMILY_KNC) {
			smpt_reg_val = BUILD_SMPT(SNOOP_ON,
					dma_addr >> MIC_SYSTEM_PAGE_SHIFT);
			writel(smpt_reg_val, &smpt[i]);
		}
	}
}

/*
 * Called once per board as part of smpt init
 * This does a 0-512G smpt mapping, 
 */
void mic_smpt_init(mic_ctx_t *mic_ctx)
{
	int i;
	dma_addr_t dma_addr;
	uint32_t *smpt = (uint32_t*)(mic_ctx->mmio.va +
			 HOST_SBOX_BASE_ADDRESS + SBOX_SMPT00);
	uint32_t smpt_reg_val;
#if SMPT_LOGGING
	smpt_ref_count_g[mic_ctx->bi_id] = 0;
#endif

	spin_lock_init(&mic_ctx->smpt_lock);
	mic_ctx->mic_smpt = kmalloc(sizeof(mic_smpt_t)
					* NUM_SMPT_ENTRIES_IN_USE, GFP_KERNEL);

	for (i = 0; i < NUM_SMPT_ENTRIES_IN_USE; i++) {
		dma_addr = i * MIC_SYSTEM_PAGE_SIZE;
		mic_ctx->mic_smpt[i].dma_addr = dma_addr;
		mic_ctx->mic_smpt[i].ref_count = 0;
		if (mic_ctx->bi_family == FAMILY_KNC) {
			smpt_reg_val = BUILD_SMPT(SNOOP_ON,
					dma_addr >> MIC_SYSTEM_PAGE_SHIFT);
			writel(smpt_reg_val, &smpt[i]);
		}
	}
}

/*
 * Called during mic exit per ctx (i.e once for every board)
 * If ref count is non-zero, then it means that some module
 * did not call mic_unmap_single/mic_ctx_unmap_single correctly.
 */
void
mic_smpt_uninit(mic_ctx_t *mic_ctx)
{
#if SMPT_LOGGING
	printk("global ref count for node = %d is %lld\n", 
		mic_ctx->bi_id+1, smpt_ref_count_g[mic_ctx->bi_id]);
	printk("mic map calls = %lld, mic unmap calls = %lld \n", 
					map_count_g, unmap_count_g);

	for (i = 0; i < NUM_SMPT_ENTRIES_IN_USE; i++) {
		printk("[smpt_san%d] smpt_entry[%d]  dma_addr = 0x%llX"
			" ref_count = %lld \n", mic_ctx->bi_id+1, i, 
			mic_ctx->mic_smpt[i].dma_addr, 
			mic_ctx->mic_smpt[i].ref_count);
	}
#endif
#ifdef DEBUG
	{
		int i;
		for (i = 0; i < NUM_SMPT_ENTRIES_IN_USE; i++)
			WARN_ON(mic_ctx->mic_smpt[i].ref_count);
	}
#endif

	kfree(mic_ctx->mic_smpt);
	mic_ctx->mic_smpt = NULL;
	;
}

dma_addr_t mic_ctx_map_single(mic_ctx_t *mic_ctx, void *p, size_t size)
{
	struct pci_dev *hwdev = mic_ctx->bi_pdev;
	int bid = mic_ctx->bi_id;

	return mic_map_single(bid, hwdev, p, size);
}

void mic_unmap_single(int bid, struct pci_dev *hwdev, dma_addr_t mic_addr,
		size_t size)
{
	dma_addr_t dma_addr = mic_to_dma_addr(bid, mic_addr);
	mic_unmap(bid, mic_addr, size);
	pci_unmap_single(hwdev, dma_addr, size, PCI_DMA_BIDIRECTIONAL);
}

void mic_ctx_unmap_single(mic_ctx_t *mic_ctx, dma_addr_t dma_addr,
		size_t size)
{
	struct pci_dev *hwdev = mic_ctx->bi_pdev;
	int bid = mic_ctx->bi_id;
	mic_unmap_single(bid, hwdev, dma_addr, size);
}

dma_addr_t mic_map_single(int bid, struct pci_dev *hwdev, void *p,
		size_t size)
{
	dma_addr_t mic_addr = 0;
	dma_addr_t dma_addr;

	dma_addr = pci_map_single(hwdev, p, size, PCI_DMA_BIDIRECTIONAL);

	if (!pci_dma_mapping_error(hwdev, dma_addr))
		if (!(mic_addr = mic_map(bid, dma_addr, size))) {
				printk(KERN_ERR "mic_map failed board id %d\
					     addr %#016llx size %#016zx\n", 
					     bid, dma_addr, size);
				pci_unmap_single(hwdev, dma_addr, 
						       size, PCI_DMA_BIDIRECTIONAL);
	}
	return mic_addr;
}

void add_smpt_entry(int spt, int64_t *ref, uint64_t dma_addr, int entries, mic_ctx_t *mic_ctx)
{

	struct nodemsg msg;
	dma_addr_t addr = dma_addr;
	mic_smpt_t *mic_smpt = mic_ctx->mic_smpt;
	int dev_id = mic_ctx->bi_id + 1;
	void *mm_sbox = mic_ctx->mmio.va + HOST_SBOX_BASE_ADDRESS;
	int i;

	for (i = spt; i < spt + entries; i++, addr += MIC_SYSTEM_PAGE_SIZE) {
#ifdef CONFIG_ML1OM
		/*
		 * For KNF if the ref count is 0 and the entry number is greater
		 * than 16 then we must resend a SMPT_SET message in case the uOS
		 * was rebooted and lost SMPT register state (example during host
		 * suspend/hibernate.
		 */
		if (!mic_smpt[i].ref_count && i >= (NUM_SMPT_ENTRIES_IN_USE >> 1)) {
#else
		if (!mic_smpt[i].ref_count && (mic_smpt[i].dma_addr != addr)) {
#endif
			/*
 			 * ref count was zero and dma_addr requested did not
 			 * match the dma address in the table. So, this is a
 			 * new entry in the table. 
 			 * KNF: Send a message to the card
 			 * to update its smpt table with a new value.
 			 * KNC: write to the SMPT registers from host since
 			 * they are accessible. 
 			 */
			if (mic_ctx->bi_family == FAMILY_ABR) {
				msg.uop = SMPT_SET;
				msg.payload[0] = addr;
				msg.payload[1] = i;
				msg.dst.node  = scif_dev[dev_id].sd_node;
				msg.src.node  = 0;
#if SMPT_LOGGING
				printk("[smpt_node%d] ==> sending msg to "
					" node = %d dma_addr = 0x%llX, entry =" 
					"0x%llX\n" , mic_ctx->bi_id + 1, 
					scif_dev[dev_id].sd_node, 
					msg.payload[0], msg.payload[1]);
#endif
				micscif_inc_node_refcnt(&scif_dev[dev_id], 1);
				micscif_nodeqp_send(&scif_dev[dev_id], &msg, NULL);
				micscif_dec_node_refcnt(&scif_dev[dev_id], 1);
			}
			else
				mic_smpt_set(mm_sbox, addr, i);
			mic_smpt[i].dma_addr = addr;
		}
		mic_smpt[i].ref_count += ref[i - spt];
	}
}

dma_addr_t smpt_op(int bid, uint64_t dma_addr,
				int entries, int64_t *ref)
{
	int spt = -1;   /* smpt index */
	int ee = 0; 	/* existing entries */
	int fe = 0;     /* free entries */
	int i;
	unsigned long flags;
	dma_addr_t mic_addr = 0;
	dma_addr_t addr = dma_addr;
	mic_ctx_t *mic_ctx = get_per_dev_ctx(bid);
	mic_smpt_t *mic_smpt = mic_ctx->mic_smpt;

	if (micpm_get_reference(mic_ctx, true))
		goto exit;
	spin_lock_irqsave(&mic_ctx->smpt_lock, flags);

	/* find existing entries */
	for (i = 0; i < NUM_SMPT_ENTRIES_IN_USE; i++) {
		if (mic_smpt[i].dma_addr == addr) {
			ee++;
			addr += MIC_SYSTEM_PAGE_SIZE;
		}
		else if (ee) /* cannot find contiguous entries */
			goto not_found;

		if (ee == entries)
			goto found;
	}

	/* find free entry */
#ifdef CONFIG_ML1OM
	/*
	 * For KNF the SMPT registers are not host accessible so we maintain a
	 * 1:1 map for SMPT registers from 0-256GB i.e. the first 16 entries and
	 * look for SMPT entries for P2P and IB etc from the 16th entry onwards.
	 * This allows the KNF card to boot on Host systems with < 256GB system
	 * memory and access VNET/SCIF buffers without crashing. P2P and IB SMPT
	 * entries are setup after SCIF driver load/reload via SCIF Node QP
	 * SMPT_SET messages.
	 */
	for (i = NUM_SMPT_ENTRIES_IN_USE / 2 ; i < NUM_SMPT_ENTRIES_IN_USE; i++) {
#else
	for (i = 0 ; i < NUM_SMPT_ENTRIES_IN_USE; i++) {
#endif
		fe = (mic_smpt[i].ref_count == 0) ? fe + 1: 0;
		if (fe == entries)
			goto found;
	}

not_found:
	spin_unlock_irqrestore(&mic_ctx->smpt_lock, flags);
	micpm_put_reference(mic_ctx);
exit:
	return mic_addr;
found:
	spt = i - entries + 1;
	mic_addr = SMPT_TO_MIC_PA(spt);
	add_smpt_entry(spt, ref, dma_addr, entries, mic_ctx);
	spin_unlock_irqrestore(&mic_ctx->smpt_lock, flags);
	micpm_put_reference(mic_ctx);
	return mic_addr;
}


/*
 * Returns number of smpt entries needed for dma_addr to dma_addr + size
 * also returns the reference count array for each of those entries
 * and the starting smpt address
 */
int get_smpt_ref_count(int64_t *ref, dma_addr_t dma_addr, size_t size,
						uint64_t *smpt_start)
{
	uint64_t start =  dma_addr;
	uint64_t end = dma_addr + size;
	int i = 0;

	while (start < end) {
		ref[i++] = min(SMPT_ALIGN_HIGH(start + 1), end) - start;
		start = SMPT_ALIGN_HIGH(start + 1);
	}

	if (smpt_start)
		*smpt_start = SMPT_ALIGN_LOW(dma_addr);

	return i;
}

/*
 * Maps dma_addr to dma_addr + size memory in the smpt table
 * of board bid
 */
dma_addr_t mic_map(int bid, dma_addr_t dma_addr, size_t size)
{
	dma_addr_t mic_addr = 0;
	int entries;
	int64_t ref[NUM_SMPT_ENTRIES_IN_USE];
	uint64_t smpt_start;
#if SMPT_LOGGING
	unsigned long flags;
	mic_ctx_t *mic_ctx = get_per_dev_ctx(bid);
	spin_lock_irqsave(&mic_ctx->smpt_lock, flags);
	map_count_g++;
	smpt_ref_count_g[bid] += (int64_t)size;
	spin_unlock_irqrestore(&mic_ctx->smpt_lock, flags);
#endif
	if (!size)
		return mic_addr;

	/*
 	 * Get number of smpt entries to be mapped, ref count array
 	 * and the starting smpt address to start the search for
 	 * free or existing smpt entries.
 	 */
	entries = get_smpt_ref_count(ref, dma_addr, size, &smpt_start);

	/* Set the smpt table appropriately and get 16G aligned mic address */
	mic_addr =  smpt_op(bid, smpt_start, entries, ref);

	/*
 	 * If mic_addr is zero then its a error case
 	 * since mic_addr can never be zero.
 	 * else generate mic_addr by adding the 16G offset in dma_addr
 	 */
	if (!mic_addr) {
		WARN_ON(1);
		return mic_addr;
	}
	else
		return (mic_addr + (dma_addr & MIC_SYSTEM_PAGE_MASK));
}

/*
 * Unmaps mic_addr to mic_addr + size memory in the smpt table
 * of board bid
 */
void mic_unmap(int bid, dma_addr_t mic_addr, size_t size)
{
	mic_ctx_t *mic_ctx = get_per_dev_ctx(bid);
	mic_smpt_t *mic_smpt = mic_ctx->mic_smpt;
	int64_t ref[NUM_SMPT_ENTRIES_IN_USE];
	int num_smpt;
	int spt = HOSTMIC_PA_TO_SMPT(mic_addr);
	int i;
	unsigned long flags;

	if (!size)
		return;

	if (!IS_MIC_SYSTEM_ADDR(mic_addr)) {
		WARN_ON(1);
		return;
	}

 	/* Get number of smpt entries to be mapped, ref count array */
	num_smpt = get_smpt_ref_count(ref, mic_addr, size, NULL);

	spin_lock_irqsave(&mic_ctx->smpt_lock, flags);

#if SMPT_LOGGING
	unmap_count_g++;
	smpt_ref_count_g[bid] -= (int64_t)size;
#endif

	for (i = spt; i < spt + num_smpt; i++) {
         	mic_smpt[i].ref_count -= ref[i - spt];
		WARN_ON(mic_smpt[i].ref_count < 0);
	}
	spin_unlock_irqrestore(&mic_ctx->smpt_lock, flags);
}

dma_addr_t mic_to_dma_addr(int bid, dma_addr_t mic_addr)
{
	mic_ctx_t *mic_ctx = get_per_dev_ctx(bid);
	int spt = HOSTMIC_PA_TO_SMPT(mic_addr);
	dma_addr_t dma_addr;

	if (!IS_MIC_SYSTEM_ADDR(mic_addr)) {
		WARN_ON(1);
		return 0;
	}
	dma_addr = mic_ctx->mic_smpt[spt].dma_addr + SMPT_OFFSET(mic_addr);
	return dma_addr;
}

#endif

bool is_syspa(dma_addr_t pa)
{
	return IS_MIC_SYSTEM_ADDR(pa);
}