// SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note OR MIT

/*
 * (C) COPYRIGHT 2023-2024 ARM Limited or its affiliates. All rights reserved.
 */

/*
 * Part of the Mali reference arbiter
 */

#include <linux/hrtimer.h>
#include <linux/module.h>
#include <linux/mutex.h>
#include <linux/platform_device.h>
#include <linux/slab.h>
#include <linux/workqueue.h>

#include <mali_arbiter/mali_arbiter.h>
#include "mali_arbiter_core.h"
#include "mali_arbiter_gpu_subinstance.h"
#include "mali_arbiter_internal.h"
#include "mali_arbiter_power.h"
#include "mali_arbiter_sysfs.h"
#include <ptm/mali_gpu_resource_group.h>
#include <ptm/mali_gpu_partition_config.h>
#include <ptm/mali_gpu_assign.h>


/* Module parameters */
static int slice_power_off_wait_time = 100;
module_param(slice_power_off_wait_time, int, 0644);
MODULE_PARM_DESC(slice_power_off_wait_time, "Max time in ms to wait before powering off slices");

/* Arbiter version against which was implemented this file */
#define MALI_ARBITER_IMPLEMENTATION_VERSION 2
#if MALI_ARBITER_IMPLEMENTATION_VERSION != MALI_ARBITER_VERSION
#error "Unsupported Mali Arbiter version."
#endif

/* Partition config version against which was implemented this module */
#define MALI_REQUIRED_PARTITION_CONFIG_VERSION 1
#if MALI_REQUIRED_PARTITION_CONFIG_VERSION != MALI_PARTITION_CONFIG_VERSION
#error "Unsupported partition config API version."
#endif

/* Resource group version against which was implemented this module */
#define MALI_REQUIRED_RESOURCE_GROUP_VERSION 1
#if MALI_REQUIRED_RESOURCE_GROUP_VERSION != MALI_RESOURCE_GROUP_VERSION
#error "Unsupported resource group API version."
#endif

/* GPU Power version against which was implemented this module */
#define MALI_REQUIRED_GPU_POWER_VERSION 1
#if MALI_REQUIRED_GPU_POWER_VERSION != MALI_GPU_POWER_VERSION
#error "Unsupported gpu power version."
#endif

/* Maximum access window mask
 */
#define MAX_AW_MASK 0xFFFF

/**
 * struct registered_vm - Stores a pointer to a registered VM
 * @entry:    List entry for reg_vms_list.
 * @vm:       private Arbiter VM data used to identify and work with each VM.
 *
 * Keep track of each VM registered with the arbiter.
 */
struct registered_vm {
	struct list_head entry;
	struct mali_arb_priv_data *vm;
};

/**
 * struct slice - Arbiter representation of a slice
 * @index: Index of the slice
 * @slice_tmr: Timer to control the slice power
 * @slice_timeout_work: Job to power down the slice
 * @timer_running: Represents whether the time is running
 * @arb: Arbiter handle
 */
struct slice {
	int index;
	struct hrtimer slice_tmr;
	struct work_struct slice_timeout_work;
	bool timer_running;
	struct mali_arb *arb;
};

/**
 * struct max_config - contain max config data
 * @l2_slices: Number of l2 slices assigned.
 * @core_mask: Mask of number of cores in each slice assigned.
 * @updated: True if the max config data has been calculated.
 */
struct max_config {
	u32 l2_slices;
	u32 core_mask;
	bool updated;
};

/**
 * struct gpu_subinstance_info - contain information around gpu-subinstances
 * @gsi_enabled: true when the gpu-subinstance is initialized.
 * @gpu_subinstance: gpu-subinstance representations
 * @gsi_mutex: Mutex to protect gpu-subinstance state
 * @gsi_access_windows: An access window mask assigned to gpu-subinstance.
 *                      Set via sysfs.
 * @gsi_slices: A slice mask assigned to gpu-subinstance.
 *              Set via sysfs when configurable.
 * @repart_if: Re-partition interfaces to change slices on gpu-subinstances
 *             This will be NULL when repartition is not supported.
 */
struct gpu_subinstance_info {
	bool gsi_enabled;
	struct mali_arb_gpu_subinstance *gpu_subinstance;
	struct mutex gsi_mutex;
	u32 gsi_access_windows;
	u32 gsi_slices;
	struct repartition_interface repart_if;
};


/**
 * struct mali_arb - Internal Arbiter state information
 * @arb_data: Embedded public arbiter device data.
 * @dev: Device for arbiter (only single Arbiter device supported).
 * @sysfs: SysFS data
 * @is_pv: A flag that tells if it's Paravirtualization.
 * @gsi_info: gpu-subinstance information structure.
 * @max_config_info: Max config data computed by the Arbiter
 * @slices: Slices representations
 * @slices_mutex: Mutex to protect slice power and clock data
 * @slices_wq: Work queue to push slice power off jobs
 * @pwr_priv: Arbiter power sub-module private data
 * @wait_list: Global GPU request list for all VMs that are currently
 *             not assigned to any gpu-subinstances.
 * @reg_vms_list: List of VMs registered with the Arbiter
 * @rg_ops: Resource group operations,
 *          available when hardware separation is supported.
 */
struct mali_arb {
	struct mali_arb_data arb_data;
	struct device *dev;
	struct mali_arb_sysfs *sysfs;
	bool is_pv;

	/* gpu-subinstance information */
	struct gpu_subinstance_info gsi_info[MALI_PTM_PARTITION_COUNT];

	/* slice information */
	struct max_config max_config_info;
	struct slice *slices[MALI_PTM_SLICES_COUNT];
	struct mutex slices_mutex;
	struct workqueue_struct *slices_wq;

	/* power information */
	void *pwr_priv;

	/* vm information */
	struct list_head wait_list;
	struct list_head reg_vms_list;
	struct mali_ptm_rg_ops *rg_ops;

};

/**
 * mali_gpu_get_powered_slices_mask() - Get a mask of the currently powered on
 *                                      slices
 * @arb: Arbiter handle
 * @slice_mask: Mask of slices powered up
 *
 * Return: 0 on success and non-zero on failure
 */
static int mali_gpu_get_powered_slices_mask(struct mali_arb *arb, u32 *slice_mask)
{
	int ret;

	if (!arb || !slice_mask)
		return -EINVAL;

	/* Check if resource operation is supported or not. */
	if (!arb->rg_ops || !arb->rg_ops->get_powered_slices_mask)
		return -ENODEV;

	ret = arb->rg_ops->get_powered_slices_mask(arb->dev, slice_mask);
	return ret;
}

/**
 * mali_gpu_get_enabled_slices_mask() - Get a mask of currently enabled slices
 * @arb: Arbiter handle
 * @slice_mask: Mask of slices enabled
 *
 * Return: 0 on success and non-zero on failure
 */
static int mali_gpu_get_enabled_slices_mask(struct mali_arb *arb, u32 *slice_mask)
{
	int ret;

	if (!arb || !slice_mask)
		return -EINVAL;

	/* Check if resource operation is supported or not. */
	if (!arb->rg_ops || !arb->rg_ops->get_enabled_slices_mask)
		return -ENODEV;

	ret = arb->rg_ops->get_enabled_slices_mask(arb->dev, slice_mask);
	return ret;
}

/**
 * mali_gpu_arbiter_poweron_slices() - Turn on the power to a set of slices
 * @arb: Arbiter handle
 * @slice_mask: mask of slices to be powered up
 *
 * Requests resource group module to power on the requested slices. And if the
 * power down timer is currently running for any requested slice, cancels the
 * timer.
 *
 * Return: 0 on success and non-zero on failure
 */
static int mali_gpu_arbiter_poweron_slices(struct mali_arb *arb, u32 slice_mask)
{
	int ret = 0;
	u32 i;
	u32 powered_mask = 0;
	const unsigned long mask = slice_mask;

	/* Check if resource operation is supported or not. */
	if (!arb->rg_ops || !arb->rg_ops->poweron_slices)
		return 0;

	if (!slice_mask)
		return ret;

	mutex_lock(&arb->slices_mutex);
	for_each_set_bit(i, &mask, MALI_PTM_SLICES_COUNT) {
		if (arb->slices[i]->timer_running) {
			arb->slices[i]->timer_running = false;
			hrtimer_cancel(&arb->slices[i]->slice_tmr);
			cancel_work_sync(&arb->slices[i]->slice_timeout_work);
		}
	}

	ret = mali_gpu_get_powered_slices_mask(arb, &powered_mask);
	if (ret)
		goto cleanup;

	if ((powered_mask & slice_mask) == slice_mask)
		goto cleanup;

	ret = arb->rg_ops->poweron_slices(arb->dev, slice_mask);

cleanup:
	mutex_unlock(&arb->slices_mutex);
	return ret;
}

/**
 * mali_gpu_arbiter_poweroff_slices() - Turn off the power to a set of slices
 * @arb: Arbiter handle
 * @slice_mask: mask of slices to be powered down
 *
 * This function iteratively checks and cancels the power down timer if running
 * and if the slice is not already OFF, start a timer to power it down at a
 * later point of time.
 *
 * Return: 0 on success and non-zero on failure
 */
static int mali_gpu_arbiter_poweroff_slices(struct mali_arb *arb, u32 slice_mask)
{
	int ret = 0;
	u32 i;
	u32 powered_mask = 0;
	const unsigned long mask = slice_mask;

	/* Check if resource operation is supported or not. */
	if (!arb->rg_ops || !arb->rg_ops->poweroff_slices)
		return 0;

	if (!slice_mask)
		return ret;

	mutex_lock(&arb->slices_mutex);
	ret = mali_gpu_get_powered_slices_mask(arb, &powered_mask);
	if (ret)
		goto cleanup;

	for_each_set_bit(i, &mask, MALI_PTM_SLICES_COUNT) {
		if (arb->slices[i]->timer_running) {
			arb->slices[i]->timer_running = false;
			hrtimer_cancel(&arb->slices[i]->slice_tmr);
			cancel_work_sync(&arb->slices[i]->slice_timeout_work);
		}
		if (powered_mask & BIT(i)) {
			hrtimer_start(&arb->slices[i]->slice_tmr,
				      ms_to_ktime(slice_power_off_wait_time), HRTIMER_MODE_REL);
			arb->slices[i]->timer_running = true;
		}
	}

cleanup:
	mutex_unlock(&arb->slices_mutex);
	return ret;
}

/**
 * mali_gpu_arbiter_nodelay_poweroff_slices() - Turn off the power to a set
 *                                              of slices with no delay
 * @arb: Arbiter handle
 * @slice_mask: mask of slices to be powered down
 *
 * This function turns off the power to a set of slices immediately with no
 * delay
 *
 * Return: 0 on success and non-zero on failure
 */
static int mali_gpu_arbiter_nodelay_poweroff_slices(struct mali_arb *arb, u32 slice_mask)
{
	u32 powered_mask = 0;
	int res = 0;

	/* Check if resource operation is supported or not. */
	if (!arb->rg_ops || !arb->rg_ops->poweroff_slices)
		return 0;

	if (!slice_mask)
		return res;

	mutex_lock(&arb->slices_mutex);

	res = mali_gpu_get_powered_slices_mask(arb, &powered_mask);
	if (!res && ((powered_mask & slice_mask) == 0))
		goto cleanup;

	res = arb->rg_ops->poweroff_slices(arb->dev, slice_mask);

cleanup:
	mutex_unlock(&arb->slices_mutex);
	if (res)
		dev_err(arb->dev, "Failed to power down slices");
	return res;
}

/**
 * mali_gpu_arbiter_enable_slices() - Turn on the clock to a set of slices
 *                                    and bring them out of reset
 * @arb: Arbiter handle
 * @slice_mask: mask of slices to be enabled
 *
 * Requests resource group module to enable slices.
 *
 * Return: 0 on success and non-zero on failure
 */
static int mali_gpu_arbiter_enable_slices(struct mali_arb *arb, u32 slice_mask)
{
	int i, ret = 0;
	u32 powered_mask = 0, enabled_mask = 0;
	bool slice_enabled, slice_powered;

	/* Check if resource operation is supported or not. */
	if (!arb->rg_ops || !arb->rg_ops->enable_slices)
		return 0;

	if (!slice_mask)
		return ret;

	mutex_lock(&arb->slices_mutex);
	ret = mali_gpu_get_powered_slices_mask(arb, &powered_mask);
	if (ret)
		goto cleanup;

	for (i = 0; i < MALI_PTM_SLICES_COUNT; i++) {
		slice_enabled = (slice_mask & (0x1 << i)) > 0;
		slice_powered = (powered_mask & (0x1 << i)) > 0;

		if (slice_enabled && !slice_powered) {
			dev_err(arb->dev, "S%u: Can't enable this slice\n", arb->slices[i]->index);
			ret = -EPERM;
			goto cleanup;
		}
	}

	ret = mali_gpu_get_enabled_slices_mask(arb, &enabled_mask);
	if (ret)
		goto cleanup;

	if ((enabled_mask & slice_mask) == slice_mask)
		goto cleanup;

	ret = arb->rg_ops->enable_slices(arb->dev, slice_mask);

cleanup:
	mutex_unlock(&arb->slices_mutex);
	return ret;
}

/**
 * mali_gpu_arbiter_disable_slices() - Turn off the clock to a set of slices
 *                                     and put them in reset
 * @arb: Arbiter handle
 * @slice_mask: mask of slices to be disabled
 *
 * This function iteratively checks and disables the slices if
 * they are not already disabled. Requests resource group
 * module to disable required slices, post which sets the clock
 * state of each slice if it goes successful.
 *
 * Return: 0 on success and non-zero on failure
 */
static int mali_gpu_arbiter_disable_slices(struct mali_arb *arb, u32 slice_mask)
{
	int ret = 0;
	u32 enabled_mask = 0;

	/* Check if resource operation is supported or not. */
	if (!arb->rg_ops || !arb->rg_ops->reset_slices)
		return 0;

	if (!slice_mask)
		return ret;

	mutex_lock(&arb->slices_mutex);
	ret = mali_gpu_get_enabled_slices_mask(arb, &enabled_mask);
	if (ret)
		goto cleanup;

	if ((~enabled_mask & slice_mask) == slice_mask)
		goto cleanup;

	ret = arb->rg_ops->reset_slices(arb->dev, slice_mask);

cleanup:
	mutex_unlock(&arb->slices_mutex);
	return ret;
}

/**
 * slice_timer_isr() - Slice timer ISR routine
 * @timer: pointer to the timer object
 *
 * This function queues the slice power down job to the
 * specific slice work queue.
 *
 * Return: HRTIMER_NORESTART, as we don't need this timer to restart
 */
static enum hrtimer_restart slice_timer_isr(struct hrtimer *timer)
{
	struct slice *this_slice;

	this_slice = container_of(timer, struct slice, slice_tmr);

	if (WARN_ON(!this_slice) || WARN_ON(!this_slice->arb))
		return HRTIMER_NORESTART;

	dev_dbg(this_slice->arb->dev, "S%u: hrtimer_isr\n", this_slice->index);

	queue_work(this_slice->arb->slices_wq, &this_slice->slice_timeout_work);
	return HRTIMER_NORESTART;
}

/**
 * slice_timeout_worker() - Job to power down the slice
 * @data: pointer to the work
 *
 * This function requests the resource group module to
 * power off a slice on which the timer had expired and
 * changes the power state if it goes successful.
 *
 * Return: void
 */
static void slice_timeout_worker(struct work_struct *data)
{
	struct slice *this_slice;
	struct mali_arb *arb;
	int ret;

	this_slice = container_of(data, struct slice, slice_timeout_work);
	arb = this_slice->arb;
	dev_dbg(arb->dev, "S%u: Timer has expired...\n", this_slice->index);

	ret = mali_gpu_arbiter_nodelay_poweroff_slices(arb, (0x1 << this_slice->index));
	if (ret)
		dev_err(arb->dev, "S%u: Failed to power off slice\n", this_slice->index);

	mutex_lock(&arb->slices_mutex);
	this_slice->timer_running = false;
	mutex_unlock(&arb->slices_mutex);
}

/**
 * arb_from_data() - Convert arb_data to arb
 * @arb_data: The arbiter device
 *
 * Return: arb data or NULL if input parameter was NULL
 */
static inline struct mali_arb *arb_from_data(struct mali_arb_data *arb_data)
{
	if (likely(arb_data))
		return container_of(arb_data, struct mali_arb, arb_data);
	return NULL;
}

/**
 * get_slice_assignment() - Get the slice assignment for a gpu-subinstance
 * @arb: Arbiter data
 * @gsi_idx: gpu-subinstance index
 * @buf: Pointer to a u32 for the resulting assignment mask
 *
 * Return: 0 on success, or error code
 */
static int get_slice_assignment(struct mali_arb *arb, u8 gsi_idx, u32 *buf)
{
	if (!arb || !buf || gsi_idx >= MALI_PTM_PARTITION_COUNT)
		return -EINVAL;

	if (arb->gsi_info[gsi_idx].gsi_enabled) {
		mutex_lock(&arb->gsi_info[gsi_idx].gsi_mutex);
		*buf = arb->gsi_info[gsi_idx].gsi_slices;
		mutex_unlock(&arb->gsi_info[gsi_idx].gsi_mutex);
	}

	return 0;
}

/**
 * set_slice_assignment() - Set the slice assignment for a gpu-subinstance
 * @arb: Arbiter data
 * @gsi_idx: gpu-subinstance index
 * @new_slices: Assigned slice mask
 * @old_slices: Previous slice mask
 *
 * Return:
 * * 0		- success.
 * * -EINVAL	- invalid argument.
 * * -ENODEV	- expected interface not available.
 * * -EIO	- failed with hardware fault.
 */
static int set_slice_assignment(struct mali_arb *arb, u8 gsi_idx, u32 new_slices, u32 *old_slices)
{
	struct mali_arb_gpu_subinstance *gsi_ptr;
	u32 all_slices = 0, slice_mask = 0;
	int ret = 0;
	u32 i, j;

	if (!arb || !arb->dev || !old_slices) {
		pr_err("%s : Invalid argument.\n", __func__);
		return -EINVAL;
	}

	if (gsi_idx >= MALI_PTM_PARTITION_COUNT) {
		dev_err(arb->dev, "gpu-subinstance index out of range.\n");
		return -EINVAL;
	}

	/* Basic check that the mask only contains bits for the whole GPU */
	if (new_slices >= (1 << MALI_PTM_SLICES_COUNT)) {
		dev_err(arb->dev, "Slice mask out of range.\n");
		return -EINVAL;
	}

	if (!arb->gsi_info[gsi_idx].gsi_enabled) {
		dev_err(arb->dev, "No gpu-subinstance built for the interface ID %d\n", gsi_idx);
		return -EINVAL;
	}

	if (!arb->rg_ops || !arb->rg_ops->get_slice_mask) {
		dev_err(arb->dev, "NULL function pointers for Resource Group.\n");
		return -EINVAL;
	}

	/* Get lock */
	mutex_lock(&arb->gsi_info[gsi_idx].gsi_mutex);
	gsi_ptr = arb->gsi_info[gsi_idx].gpu_subinstance;

	*old_slices = arb->gsi_info[gsi_idx].gsi_slices;
	all_slices = arb->gsi_info[gsi_idx].gsi_slices | new_slices;

	if (*old_slices == new_slices)
		goto exit;

	ret = arb->rg_ops->get_slice_mask(arb->dev, &slice_mask);
	if (ret) {
		dev_err(arb->dev, "Get Slice Mask failed.\n");
		ret = -EIO;
		goto exit;
	}

	/* Check the request is allowed */
	for (i = 0; i < MALI_PTM_SLICES_COUNT; i++) {
		u32 cur_slice_bit = 1 << i;

		if (!(new_slices & cur_slice_bit))
			continue;

		/* Check that the requested slice is in this RG */
		if (!(slice_mask & cur_slice_bit)) {
			dev_err(arb->dev, "Slice not in resource group.\n");
			ret = -EINVAL;
			goto exit;
		}

		/* Check that the requested slice is not in another gpu-subinstance */
		for (j = 0; j < MALI_PTM_PARTITION_COUNT; j++) {
			if (!arb->gsi_info[j].gsi_enabled)
				continue;

			if (j != gsi_idx && (arb->gsi_info[j].gsi_slices & cur_slice_bit)) {
				dev_err(arb->dev, "Slice in another gpu-subinstance.\n");
				ret = -EINVAL;
				goto exit;
			}
		}
	}

	/* Stop gpu-subinstance */
	mali_arbiter_gsi_stop(gsi_ptr);

	/* Disable the slice clocks, put into reset and turn off the slices */
	ret = mali_gpu_arbiter_disable_slices(arb, all_slices);
	if (ret) {
		dev_err(arb->dev, "Failed to disable slices.\n");
		ret = -EIO;
		goto exit;
	}

	ret = mali_gpu_arbiter_poweroff_slices(arb, all_slices);
	if (ret) {
		dev_err(arb->dev, "Failed to poweroff slices.\n");
		/* Enable the slices again */
		mali_gpu_arbiter_enable_slices(arb, all_slices);
		ret = -EIO;
		goto exit;
	}

	/* Assign slices to partition with repart_if operations */
	if (!arb->gsi_info[gsi_idx].repart_if.ops ||
	    !arb->gsi_info[gsi_idx].repart_if.ops->assign_slices) {
		dev_err(arb->dev, "P%u: Re-partition interface not available.\n", gsi_idx);
		ret = -ENODEV;
		goto exit;
	}
	ret = arb->gsi_info[gsi_idx].repart_if.ops->assign_slices(
		arb->gsi_info[gsi_idx].repart_if.dev, new_slices);
	if (ret) {
		dev_err(arb->dev, "Failed to assign slices.\n");
		/* Power the slices and bring out of reset on again */
		mali_gpu_arbiter_poweron_slices(arb, all_slices);
		mali_gpu_arbiter_enable_slices(arb, all_slices);
		ret = -EIO;
		goto exit;
	}

	/* Update gsi_info accordingly */
	arb->gsi_info[gsi_idx].gsi_slices = new_slices;
	if (new_slices) {
		/* Power the slices and bring out of reset on again */
		ret = mali_gpu_arbiter_poweron_slices(arb, new_slices);
		if (ret) {
			dev_err(arb->dev, "Failed to poweron slices.\n");
			ret = -EIO;
			goto exit;
		}

		/* Enable the slices again */
		ret = mali_gpu_arbiter_enable_slices(arb, new_slices);
		if (ret) {
			dev_err(arb->dev, "Failed to enable slices.\n");
			ret = -EIO;
		}

		/* Set GSI_FLAG_SLICE_ASSIGNED flag */
		mali_arbiter_gsi_flag_set(arb->gsi_info[gsi_idx].gpu_subinstance,
					  GSI_FLAG_SLICE_ASSIGNED);
	} else {
		/* Clear GSI_FLAG_SLICE_ASSIGNED flag */
		mali_arbiter_gsi_flag_clear(arb->gsi_info[gsi_idx].gpu_subinstance,
					    GSI_FLAG_SLICE_ASSIGNED);
	}

exit:
	/* Start gpu-subinstance */
	mali_arbiter_gsi_start(gsi_ptr);

	/* Release lock */
	mutex_unlock(&arb->gsi_info[gsi_idx].gsi_mutex);

	return ret;
}

/**
 * get_aw_assignment() - Get the per-subinstance AW assignment
 * @arb: Arbiter data
 * @gsi_idx: gpu-subinstance index
 * @buf: Pointer to a u32 for the resulting assignment mask
 *
 * Return: 0 on success, or error code
 */
static int get_aw_assignment(struct mali_arb *arb, u8 gsi_idx, u32 *buf)
{
	if (!arb || !buf || gsi_idx >= MALI_PTM_PARTITION_COUNT)
		return -EINVAL;

	if (arb->gsi_info[gsi_idx].gsi_enabled) {
		mutex_lock(&arb->gsi_info[gsi_idx].gsi_mutex);
		*buf = arb->gsi_info[gsi_idx].gsi_access_windows;
		mutex_unlock(&arb->gsi_info[gsi_idx].gsi_mutex);
	}

	return 0;
}

/**
 * set_aw_assignment() - Set the AW assignment for a gpu-subinstance
 * @arb: Arbiter data
 * @gsi_idx: gpu-subinstance index
 * @new_access_windows: Assigned AW mask
 * @old_access_windows: Previously assigned AW mask
 *
 * Return:
 * * 0		- success.
 * * -EINVAL	- invalid argument.
 * * -EIO	- failed with hardware fault.
 * * -EFAULT	- failed with software fault.
 */
static int set_aw_assignment(struct mali_arb *arb, u8 gsi_idx, u32 new_access_windows,
			     u32 *old_access_windows)
{
	struct mali_arb_gpu_subinstance *gsi_ptr;
	int ret = 0;
	u32 current_aws = 0, aw_mask = 0;
	u32 i, j;
	struct mali_arb_priv_data *cur_vm, *tmp_vm;

	if (!arb || !arb->dev || !old_access_windows) {
		pr_err("%s : Invalid argument.\n", __func__);
		return -EINVAL;
	}

	if (new_access_windows > MAX_AW_MASK) {
		dev_err(arb->dev, "Invalid AW mask = %d\n", new_access_windows);
		return -EINVAL;
	}

	if (gsi_idx >= MALI_PTM_PARTITION_COUNT) {
		dev_err(arb->dev, "gpu-subinstance index out of range.\n");
		return -EINVAL;
	}

	if (!arb->gsi_info[gsi_idx].gsi_enabled) {
		dev_err(arb->dev, "No gpu-subinstance built for the interface ID %d\n", gsi_idx);
		return -EINVAL;
	}

	if (!arb->rg_ops || !arb->rg_ops->get_aw_mask) {
		dev_err(arb->dev, "NULL function pointers for Resource Group.\n");
		return -EINVAL;
	}

	/* Get lock */
	mutex_lock(&arb->gsi_info[gsi_idx].gsi_mutex);

	gsi_ptr = arb->gsi_info[gsi_idx].gpu_subinstance;

	current_aws = arb->gsi_info[gsi_idx].gsi_access_windows;
	*old_access_windows = current_aws;

	if (current_aws == new_access_windows)
		goto exit;

	ret = arb->rg_ops->get_aw_mask(arb->dev, &aw_mask);
	if (ret) {
		dev_err(arb->dev, "Get Access Window Mask failed.\n");
		ret = -EIO;
		goto exit;
	}

	/* Check the request is allowed */
	for (i = 0; i < MALI_PTM_ACCESS_WINDOW_COUNT; i++) {
		u32 cur_aw_bit = 1 << i;

		if (!(new_access_windows & cur_aw_bit))
			continue;

		/* Check that the requested AW is in this RG */
		if (!(aw_mask & cur_aw_bit)) {
			ret = -EINVAL;
			dev_err(arb->dev, "AW%u not in RG\n", i);
			goto exit;
		}

		/* Check that the requested AW is not in another gpu-subinstance */
		for (j = 0; j < MALI_PTM_PARTITION_COUNT; j++) {
			if (!arb->gsi_info[j].gsi_enabled)
				continue;

			if (j != gsi_idx && (arb->gsi_info[j].gsi_access_windows & cur_aw_bit)) {
				dev_err(arb->dev, "AW%u already assigned to P%u\n", i, j);
				ret = -EINVAL;
				goto exit;
			}
		}
	}

	/* Stop gpu-subinstance */
	mali_arbiter_gsi_stop(gsi_ptr);

	/* Update gsi_access_window */
	arb->gsi_info[gsi_idx].gsi_access_windows = new_access_windows;

	/* Check arbiter wait list with new AW_MASK and add them to new gpu-subinstance */
	list_for_each_entry_safe(cur_vm, tmp_vm, &arb->wait_list, entry) {
		if ((1 << cur_vm->info.id) & new_access_windows) {
			/* Remove the VM from Arbiter wait-queue */
			list_del_init(&cur_vm->entry);

			/* Add the VM to new gpu-subinstance */
			mali_arbiter_gsi_gpu_request(cur_vm, gsi_ptr);
		}
	}

	/* Move requested VMs in AWs being unassigned into the arbiter wait_list */
	for (i = 0; i < MALI_PTM_ACCESS_WINDOW_COUNT; i++) {
		u32 cur_aw_bit = 1 << i;

		if ((*old_access_windows & cur_aw_bit) && !(new_access_windows & cur_aw_bit)) {
			struct registered_vm *reg_vm;

			list_for_each_entry(reg_vm, &arb->reg_vms_list, entry) {
				if (reg_vm->vm->info.id == i) {
					bool requested;

					/* Remove the VM from the gpu-subinstance */
					requested = mali_arbiter_gsi_remove_vm(gsi_ptr, reg_vm->vm);

					/* Add the VM to the wait-list if it was in req-list */
					if (requested)
						list_add_tail(&reg_vm->vm->entry, &arb->wait_list);

					break;
				}
			}
		}
	}

	/* Start gpu-subinstance */
	mali_arbiter_gsi_start(gsi_ptr);

exit:
	/* Release lock */
	mutex_unlock(&arb->gsi_info[gsi_idx].gsi_mutex);

	return ret;
}

/**
 * reg_assign_if() - Register VM assign interface
 * @arb_data:  Arbiter public data exposed (see mali_arb_data).
 * @vm_assign: GPU to VM assign interfaces.
 *
 * Register a VM assign interface to be used by the Arbiter to assign a gpu-
 * subinstance to a specific VM. If hardware separation is not supported, just the
 * interface index zero (see vm_assign_interface) should be registered.
 *
 * Return:
 * * 0		- successful.
 * * -EINVAL	- invalid argument.
 * * -EFAULT	- failed with software fault.
 */
static int reg_assign_if(struct mali_arb_data *arb_data, struct vm_assign_interface vm_assign)
{
	struct mali_arb *arb;
	int ret = 0;

	if (WARN_ON(!arb_data || !vm_assign.ops)) {
		pr_err("%s: Invalid argument.\n", __func__);
		return -EINVAL;
	}

	if (unlikely(!vm_assign.ops->assign_vm || !vm_assign.ops->unassign_vm)) {
		pr_err("%s: Missing VM assign interface callbacks\n", __func__);
		return -EINVAL;
	}

	arb = arb_from_data(arb_data);
	if (!arb) {
		pr_err("%s: Arbiter not initialized or arb_data invalid\n", __func__);
		return -EINVAL;
	}

	/* register VM Assign interface */
	mutex_lock(&arb->gsi_info[vm_assign.if_id].gsi_mutex);

	if (arb->gsi_info[vm_assign.if_id].gsi_enabled) {
		ret = mali_arbiter_gsi_register_assign_if(
			arb->gsi_info[vm_assign.if_id].gpu_subinstance, vm_assign);
	} else {
		dev_err(arb->dev, "No gpu-subinstance built for the interface ID %d\n",
			vm_assign.if_id);
		ret = -EINVAL;
	}

	mutex_unlock(&arb->gsi_info[vm_assign.if_id].gsi_mutex);

	return ret;
}

/**
 * unreg_assign_if() - Unregister VM assign interface
 * @arb_data: Arbiter public data exposed (see mali_arb_data).
 * @if_id:    Index of the interface to be unregistered. Should be zero in case
 *            hardware separation is not supported.
 *
 * Unregister the VM assign interface of index if_id from the Arbiter. If
 * hardware separation is not supported, just the interface index zero (see
 * vm_assign_interface) should have been registered (see arb_reg_assign_if).
 */
static void unreg_assign_if(struct mali_arb_data *arb_data, u32 if_id)
{
	struct mali_arb *arb;

	if (WARN_ON(!arb_data || if_id >= MALI_PTM_PARTITION_COUNT)) {
		pr_err("%s: Invalid argument.\n", __func__);
		return;
	}

	arb = arb_from_data(arb_data);
	if (!arb) {
		pr_err("%s: Arbiter not initialized or arb_data invalid\n", __func__);
		return;
	}

	/* unregister VM Assign interface */
	mutex_lock(&arb->gsi_info[if_id].gsi_mutex);

	if (arb->gsi_info[if_id].gsi_enabled)
		mali_arbiter_gsi_unregister_assign_if(arb->gsi_info[if_id].gpu_subinstance);
	else
		dev_err(arb->dev, "No gpu-subinstance built for the interface %d\n", if_id);

	mutex_unlock(&arb->gsi_info[if_id].gsi_mutex);
}

/**
 * locked_instance_from_vm_id() - Returns the assigned gpu-subinstance from the
 *                                 given VM id.
 * @arb: Arbiter data
 * @vm_id: VM index
 *
 * Be aware that if found, the gpu-subinstance will be returned locked so the
 * assigned AWs cannot change.
 *
 * Return: Locked owning gpu-subinstance index, or -1 if no gpu-subinstance is found
 */
static int locked_instance_from_vm_id(struct mali_arb *arb, u32 vm_id)
{
	int i;

	for (i = 0; i < MALI_PTM_PARTITION_COUNT; i++) {
		if (!arb->gsi_info[i].gsi_enabled)
			continue;

		mutex_lock(&arb->gsi_info[i].gsi_mutex);

		if (arb->is_pv)
			return i;
		else if (arb->gsi_info[i].gsi_access_windows & (1 << vm_id))
			return i;

		mutex_unlock(&arb->gsi_info[i].gsi_mutex);
	}

	return -1;
}

/**
 * gpu_stopped() - VM KBase driver is in stopped state GPU is not being used
 * @arb_vm: Opaque pointer to the Arbiter VM data populated by the arbiter
 *          during the registration.
 * @req_again: VM has work pending and still wants the GPU
 *
 * VM must no longer use the GPU once this function has been called until
 * gpu_granted has been called by the Arbiter.
 */
static void gpu_stopped(struct mali_arb_priv_data *arb_vm, bool req_again)
{
	struct mali_arb *arb;
	struct mali_arb_priv_data *cur_vm;
	int gsi_idx;

	if (WARN_ON(!arb_vm) || WARN_ON(!arb_vm->arb))
		return;
	arb = arb_vm->arb;

	/* Get gpu-subinstance index with lock */
	gsi_idx = locked_instance_from_vm_id(arb, arb_vm->info.id);

	/* We may have received this stopped after an arbiter restart
	 * and before a gpu-subinstance has been re-assigned to vm.
	 * In such case, use wait-list of arbiter, not req-list of the gpu-subinstance.
	 */
	if (gsi_idx < 0) {
		if (req_again) {
			list_for_each_entry(cur_vm, &arb->wait_list, entry) {
				if (cur_vm == arb_vm)
					return;
			}
			dev_dbg(arb->dev,
				"Adding VM %u to partitionless wait list (after a restart)\n",
				arb_vm->info.id);
			list_add_tail(&arb_vm->entry, &arb->wait_list);
		} else {
			dev_warn(arb->dev,
				 "GPU_STOPPED from VM %u not assigned to a gpu-subinstance\n",
				 arb_vm->info.id);
		}
		return;
	}

	dev_dbg(arb->dev, "P%u: %u, req_again: %d\n", gsi_idx, arb_vm->info.id, req_again);


	/* Call a handler function on gpu-subinstance */
	mali_arbiter_gsi_gpu_stopped(arb_vm, arb->gsi_info[gsi_idx].gpu_subinstance, req_again);

	/* Unlock gpu-subinstance mutex */
	mutex_unlock(&arb->gsi_info[gsi_idx].gsi_mutex);
}

/**
 * gpu_request() - Request GPU time
 * @arb_vm: Opaque pointer to the Arbiter VM data populated by the arbiter
 *          during the registration.
 *
 * Called by a VM comms device to request GPU time. The Arbiter should respond
 * when ready with a callback to gpu_granted.
 */
static void gpu_request(struct mali_arb_priv_data *arb_vm)
{
	struct mali_arb *arb;
	struct mali_arb_priv_data *cur_vm;
	int gsi_idx;

	if (WARN_ON(!arb_vm) || WARN_ON(!arb_vm->arb))
		return;
	arb = arb_vm->arb;

	/* Get gpu-subinstance index with lock */
	gsi_idx = locked_instance_from_vm_id(arb, arb_vm->info.id);

	/* VM is not assigned to any gpu-subinstance. Add it to arbiter wait-list. */
	if (gsi_idx < 0) {
		list_for_each_entry(cur_vm, &arb->wait_list, entry) {
			if (cur_vm == arb_vm)
				return;
		}
		list_add_tail(&arb_vm->entry, &arb->wait_list);
		dev_warn(arb->dev, "GPU_REQUEST from VM %u not assigned to a gpu-subinstance.\n",
			 arb_vm->info.id);
		return;
	}


	/* Call a handler function on gpu-subinstance */
	mali_arbiter_gsi_gpu_request(arb_vm, arb->gsi_info[gsi_idx].gpu_subinstance);


	/* Unlock gpu-subinstance mutex */
	mutex_unlock(&arb->gsi_info[gsi_idx].gsi_mutex);
}

/**
 * gpu_active() - Notify Arbiter that GPU is being used
 * @arb_vm: Opaque pointer to the Arbiter VM data populated by the arbiter
 *          during the registration.
 *
 * Called by a VM comms device to notify the Arbiter that the VM is busy using
 * the GPU. This information can be useful in making scheduling decisions.
 */
static void gpu_active(struct mali_arb_priv_data *arb_vm)
{
	struct mali_arb *arb;
	int gsi_idx;

	if (WARN_ON(!arb_vm) || WARN_ON(!arb_vm->arb))
		return;
	arb = arb_vm->arb;

	/* Get gpu-subinstance index with lock */
	gsi_idx = locked_instance_from_vm_id(arb, arb_vm->info.id);

	/* VM is not assigned to any gpu-subinstance. No operation in such case. */
	if (gsi_idx < 0) {
		dev_warn(arb->dev, "GPU_ACTIVE from VM %u not assigned to a gpu-subinstance\n",
			 arb_vm->info.id);
		return;
	}

	dev_dbg(arb->dev, "P%u: GPU %u is active\n", gsi_idx, arb_vm->info.id);

	/* Call a handler function on gpu-subinstance */
	mali_arbiter_gsi_gpu_active(arb_vm, arb->gsi_info[gsi_idx].gpu_subinstance);

	/* Unlock gpu-subinstance mutex */
	mutex_unlock(&arb->gsi_info[gsi_idx].gsi_mutex);
}

/**
 * gpu_idle() - Notify Arbiter that GPU is not being used
 * @arb_vm: Opaque pointer to the Arbiter VM data populated by the arbiter
 *          during the registration.
 *
 * Called by a VM to notify the Arbiter that it is no longer using the GPU. This
 * information can be useful in making scheduling decisions.
 */
static void gpu_idle(struct mali_arb_priv_data *arb_vm)
{
	struct mali_arb *arb;
	struct mali_arb_priv_data *cur_vm, *tmp_vm;
	int gsi_idx;

	if (WARN_ON(!arb_vm) || WARN_ON(!arb_vm->arb))
		return;
	arb = arb_vm->arb;

	/* Get gpu-subinstance index with lock */
	gsi_idx = locked_instance_from_vm_id(arb, arb_vm->info.id);

	/* VM is not assigned to any gpu-subinstance.
	 * Remove from arbiter wait list if exists.
	 */
	if (gsi_idx < 0) {
		list_for_each_entry_safe(cur_vm, tmp_vm, &arb->wait_list, entry) {
			if (cur_vm == arb_vm) {
				list_del_init(&cur_vm->entry);
				break;
			}
		}
		dev_warn(arb->dev, "GPU_IDLE from VM %u not assigned to a gpu-subinstance\n",
			 arb_vm->info.id);
		return;
	}


	/* Call a handler function on gpu-subinstance */
	mali_arbiter_gsi_gpu_idle(arb_vm, arb->gsi_info[gsi_idx].gpu_subinstance);

	/* Unlock gpu-subinstance mutex */
	mutex_unlock(&arb->gsi_info[gsi_idx].gsi_mutex);
}

/**
 * register_vm() - Register a VM comms device
 * @arb_data: Arbiter public data exposed (see mali_arb_data).
 * @vm_info: The VM info data populated by caller (see mali_vm_data).
 * @vm: Opaque pointer to the private VM data used by the backend to identify
 *      the VMs. The Arbiter must pass this pointer back in the VM callbacks
 *      every time it needs to communicate to this VM being registered by this
 *      function.
 * @arb_vm: Opaque pointer to the private Arbiter VM data used by the Arbiter to
 *          identify and work with each VM. This is populated by the Arbiter in
 *          this function. The backend must pass this pointer back in the
 *          Arbiter callbacks every time this VM being registered needs to
 *          communicate with the Arbiter.
 *
 * This function must be called first before a VM comms device instance can use
 * the arbiter functionality.
 *
 * Return:
 * * 0		- successful.
 * * -EINVAL	- invalid argument.
 * * -ENOMEM	- out of memory.
 */
static int register_vm(struct mali_arb_data *arb_data, struct mali_vm_data *vm_info,
		       struct mali_vm_priv_data *vm, struct mali_arb_priv_data **arb_vm)
{
	struct mali_arb *arb;
	struct mali_arb_priv_data *priv_vm;
	struct registered_vm *reg_vm;
	int i;

	if (WARN_ON(!arb_data || !vm_info || !vm || !arb_vm)) {
		if (arb_vm)
			*arb_vm = NULL;
		pr_err("%s : Invalid argument.\n", __func__);
		return -EINVAL;
	}

	if (unlikely(vm_info->dev == NULL || vm_info->ops.gpu_granted == NULL ||
		     vm_info->ops.gpu_stop == NULL || vm_info->ops.gpu_lost == NULL)) {
		pr_err("%s : VM ops functions are invalid.\n", __func__);
		return -EINVAL;
	}

	arb = arb_from_data(arb_data);

	list_for_each_entry(reg_vm, &arb->reg_vms_list, entry) {
		if (reg_vm->vm->info.id == vm_info->id) {
			dev_warn(arb->dev, "VM %u already registered.\n", vm_info->id);
			*arb_vm = reg_vm->vm;
			return 0;
		}
	}

	dev_dbg(arb->dev, "Registering VM %u\n", vm_info->id);

	priv_vm = devm_kzalloc(arb->dev, sizeof(struct mali_arb_priv_data), GFP_KERNEL);
	if (!priv_vm)
		return -ENOMEM;

	priv_vm->info = *vm_info;
	priv_vm->arb = arb;
	priv_vm->gpu_lost = false;
	priv_vm->vm = vm;
	INIT_LIST_HEAD(&priv_vm->entry);

	reg_vm = devm_kzalloc(arb->dev, sizeof(struct registered_vm), GFP_KERNEL);
	if (!reg_vm)
		return -ENOMEM;

	reg_vm->vm = priv_vm;
	INIT_LIST_HEAD(&reg_vm->entry);
	list_add_tail(&reg_vm->entry, &arb->reg_vms_list);

	*arb_vm = priv_vm;

	/* Call remove_vm function for all gpu-subinstances.
	 * This is for arbiter restart case to synchronize active VMs.
	 */
	for (i = 0; i < MALI_PTM_PARTITION_COUNT; i++) {
		bool requested;

		if (!arb->gsi_info[i].gsi_enabled)
			continue;

		mutex_lock(&arb->gsi_info[i].gsi_mutex);
		requested = mali_arbiter_gsi_remove_vm(arb->gsi_info[i].gpu_subinstance, priv_vm);
		mutex_unlock(&arb->gsi_info[i].gsi_mutex);

		if (unlikely(requested))
			dev_err(arb->dev, "VM-%u is in req_list of GSI-%d.\n", vm_info->id, i);
	}

	return 0;
}

/**
 * unregister_vm() - Unregister a VM comms device
 * @arb_vm: Opaque pointer to the Arbiter VM data populated by the arbiter
 * during the registration.
 *
 * This function must be called before the VM comms device is unloaded to sever
 * the connection.
 */
static void unregister_vm(struct mali_arb_priv_data *arb_vm)
{
	struct mali_arb *arb;
	int gsi_idx;
	struct registered_vm *reg_vm;
	bool requested;

	if (WARN_ON(!arb_vm) || WARN_ON(!arb_vm->arb))
		return;
	arb = arb_vm->arb;

	/* Get gpu-subinstance index with lock */
	gsi_idx = locked_instance_from_vm_id(arb, arb_vm->info.id);

	/* VM is not assigned to any gpu-subinstance. No operation in such case. */
	if (gsi_idx < 0) {
		dev_warn(arb->dev, "VM %u is not registered to a gpu-subinstance\n",
			 arb_vm->info.id);
		return;
	}

	/* Remove VM from the gpu-subinstance */
	requested = mali_arbiter_gsi_remove_vm(arb->gsi_info[gsi_idx].gpu_subinstance, arb_vm);
	if (requested)
		dev_warn(arb->dev, "VM %u requested GPU when it's unregistered.\n",
			 arb_vm->info.id);

	mutex_unlock(&arb->gsi_info[gsi_idx].gsi_mutex);

	/* Release resources for VM structures */
	list_for_each_entry(reg_vm, &arb->reg_vms_list, entry) {
		if (reg_vm->vm->info.id == arb_vm->info.id) {
			list_del(&reg_vm->entry);
			devm_kfree(arb->dev, reg_vm);
			break;
		}
	}
	devm_kfree(arb->dev, arb_vm);

	dev_dbg(arb->dev, "%s %u\n", __func__, arb_vm->info.id);
}

/**
 * get_utilisation() - Get utilization information
 * @dev: Pointer to mali_arb structure
 * @gpu_busytime: will contain the GPU busy time
 * @gpu_totaltime: will contain the GPU total time
 *
 * Called from GPU power integration driver for DVFS to get GPU busytime and
 * totaltime since last request
 * This callback should not call back the power module or it will cause
 * deadlocks.
 */
static void get_utilisation(void *dev, u32 *gpu_busytime, u32 *gpu_totaltime)
{
	struct mali_arb *arb = (struct mali_arb *)dev;
	u64 last_percent;
	u32 i;

	if (WARN_ON(!dev || !gpu_busytime || !gpu_totaltime))
		return;

	*gpu_busytime = 0;
	*gpu_totaltime = 0;

	/* Find the gpu-subinstance with the maximum utilization */
	last_percent = 0;
	for (i = 0; i < MALI_PTM_PARTITION_COUNT; i++) {
		u64 percent = 0;
		u32 busytime, totaltime;

		if (!arb->gsi_info[i].gsi_enabled)
			continue;

		mutex_lock(&arb->gsi_info[i].gsi_mutex);

		mali_arbiter_gsi_get_utilisation(arb->gsi_info[i].gpu_subinstance, &busytime,
						 &totaltime);

		/* Do not divide by zero... */
		if (totaltime == 0) {
			mutex_unlock(&arb->gsi_info[i].gsi_mutex);
			continue;
		}

		percent = div_u64((u64)busytime * 100, totaltime);
		if (last_percent < percent) {
			last_percent = percent;
			*gpu_busytime = busytime;
			*gpu_totaltime = totaltime;
		}

		dev_dbg(arb->dev, "Arbiter P%u reporting busy %u, total %u (~ %llu%%)\n", i,
			busytime, totaltime, percent);

		mutex_unlock(&arb->gsi_info[i].gsi_mutex);
	}

	WARN_ON(*gpu_busytime > *gpu_totaltime);
}

/**
 * update_freq() - Get notified on frequency change
 * @dev: Pointer to mali_arb structure
 * @new_freq: updated frequency
 *
 * Called from GPU power integration driver for DVFS to get
 * notification when frequency changes.
 * This callback should not call back the power module or it will cause
 * deadlocks.
 */
static void update_freq(void *dev, u32 new_freq)
{
	struct mali_arb *arb = (struct mali_arb *)dev;
	int i;

	if (WARN_ON(!arb))
		return;

	for (i = 0; i < MALI_PTM_PARTITION_COUNT; i++) {
		if (!arb->gsi_info[i].gsi_enabled)
			continue;

		mutex_lock(&arb->gsi_info[i].gsi_mutex);

		/* Update clock frequency of gpu-subinstance */
		mali_arbiter_gsi_update_freq(arb->gsi_info[i].gpu_subinstance, new_freq);

		mutex_unlock(&arb->gsi_info[i].gsi_mutex);
	}
}

/**
 * get_max_l2_slices() - Calculates max l2 slices data
 * @dev: Resource Group module data
 * @max_l2_slices: pointer to variable to receive max l2 slices data
 *
 * Calculate max_l2_slices value which is considered the number of L2 slices
 * allocated to the resource group which this arbiter is associated with
 * (1 to 1 relationship between arbiter and resource group)
 *
 * Return: 0 on success, standard linux error code otherwise
 */
static int get_max_l2_slices(struct device *dev, u32 *max_l2_slices)
{
	u32 slice_mask = 0;
	u32 block_l2_num = 0;
	struct mali_ptm_rg_ops *rg_ops;
	int err;

	if (!dev || !max_l2_slices)
		return -EINVAL;

	rg_ops = dev_get_drvdata(dev);
	if (!rg_ops || !rg_ops->get_slice_mask) {
		dev_dbg(dev, "Setup does not support variable L2 slices number.\n");
		*max_l2_slices = 0;
		return 0;
	}

	err = rg_ops->get_slice_mask(dev, &slice_mask);
	if (err)
		return err;

	*max_l2_slices = 0;

	while (slice_mask) {
		if (slice_mask & 0x1) {
			block_l2_num++;
		} else {
			if (block_l2_num > *max_l2_slices)
				*max_l2_slices = block_l2_num;
			block_l2_num = 0;
		}
		slice_mask >>= 1;
	}

	if (block_l2_num > *max_l2_slices)
		*max_l2_slices = block_l2_num;

	return 0;
}

/**
 * get_max_core_mask() - Calculates max core mask data
 * @dev: Resource Group module data
 * @max_core_mask: pointer to variable to receive max core mask data
 *
 * Calculate max_core_mask value which is considered to be all the l2 slices
 * assigned to the resource group and the shader_cores on each of them and we
 * calculate the max_core_mask bitmap as the smallest superset that would
 * represent all of them. So any possible combination of contiguous
 * l2 slices word have a core_mask that would be a subgroup of
 * the max_core_mask.
 *
 * Return: 0 on success, standard linux error code otherwise
 */
static int get_max_core_mask(struct device *dev, u32 *max_core_mask)
{
	u32 slice_mask = 0;
	u8 core_mask_stride = 0;
	u32 block_core_mask = 0;
	u32 core_mask = 0;
	u64 slices_core_mask = 0;
	struct mali_ptm_rg_ops *rg_ops;
	int err;
	int slice_count = 0;

	if (!max_core_mask)
		return -EINVAL;

	rg_ops = dev_get_drvdata(dev);
	if (!rg_ops || !rg_ops->get_slice_mask || !rg_ops->get_slices_core_mask) {
		dev_dbg(dev, "Setup does not support variable Core Mask.\n");
		*max_core_mask = 0;
		return 0;
	}

	err = rg_ops->get_slice_mask(dev, &slice_mask);
	if (err)
		return err;

	err = rg_ops->get_slices_core_mask(dev, &slices_core_mask, &core_mask_stride);
	if (err)
		return err;

	*max_core_mask = 0;

	while (slice_mask) {
		core_mask = slices_core_mask & ((0x1ll << core_mask_stride) - 1);
		slices_core_mask >>= core_mask_stride;

		if (slice_mask & 0x1) {
			core_mask <<= core_mask_stride * slice_count;
			block_core_mask |= core_mask;
			slice_count++;
		} else {
			*max_core_mask |= block_core_mask;
			block_core_mask = 0;
			slice_count = 0;
		}
		slice_mask >>= 1;
	}
	*max_core_mask |= block_core_mask;

	return 0;
}

/**
 * calculate_max_config() - Calculates max config data
 * @arb: Arbiter data
 *
 * Calculate and save max_l2_slices and max_core_mask values
 *
 * Return: 0 on success, error code otherwise
 */
static int calculate_max_config(struct mali_arb *arb)
{
	int err;

	if (!arb)
		return -EINVAL;

	err = get_max_l2_slices(arb->dev, &arb->max_config_info.l2_slices);
	if (err) {
		dev_err(arb->dev, "Cannot calculate max l2 slices\n");
		return err;
	}

	err = get_max_core_mask(arb->dev, &arb->max_config_info.core_mask);
	if (err) {
		dev_err(arb->dev, "Cannot calculate max core mask\n");
		return err;
	}

	arb->max_config_info.updated = true;
	return 0;
}

/**
 * get_max_config() - Notify Arbiter of a request for max config
 * @arb_vm: Opaque pointer to the Arbiter VM data populated by the arbiter
 *          during the registration.
 * @max_l2_slices: Returns the maximum number of GPU slices that can be assigned
 *                 to a gpu-subinstance. Caller must ensure not NULL.
 * @max_core_mask: Returns intersection of any possible gpu-subinstance core mask
 *                 for that resource group.
 *
 * Called by a Resource group module to Notify Arbiter that a request for
 * getting max config have been made, and that the arbiter needs to supply max
 * config data.
 *
 * Return:
 * * 0		- successful.
 * * -EINVAL	- invalid argument.
 * * -EFAULT	- max config calculation failed.
 */
static int get_max_config(struct mali_arb_priv_data *arb_vm, u32 *max_l2_slices, u32 *max_core_mask)
{
	int err;

	if (!arb_vm || !arb_vm->arb || !max_l2_slices || !max_core_mask) {
		pr_err("%s : Invalid argument.\n", __func__);
		return -EINVAL;
	}

	if (!arb_vm->arb->max_config_info.updated) {
		err = calculate_max_config(arb_vm->arb);
		if (err)
			return -EFAULT;
	}
	*max_l2_slices = arb_vm->arb->max_config_info.l2_slices;
	*max_core_mask = arb_vm->arb->max_config_info.core_mask;

	return 0;
}

static void handle_gsi_cb_lock(struct mali_arb *arb, u32 gsi_idx)
{
	if (unlikely(!arb || gsi_idx >= MALI_PTM_PARTITION_COUNT)) {
		pr_err("%s: Invalid parameter! (arb = %pK, gsi_idx = %d\n)", __func__, arb,
		       gsi_idx);
		return;
	}

	if (arb->gsi_info[gsi_idx].gsi_enabled)
		mutex_lock(&arb->gsi_info[gsi_idx].gsi_mutex);
}

static void handle_gsi_cb_lock_assert_held(struct mali_arb *arb, u32 gsi_idx)
{
	if (unlikely(!arb || gsi_idx >= MALI_PTM_PARTITION_COUNT)) {
		pr_err("%s: Invalid parameter! (arb = %pK, gsi_idx = %d\n)", __func__, arb,
		       gsi_idx);
		return;
	}

	if (arb->gsi_info[gsi_idx].gsi_enabled)
		lockdep_assert_held(&arb->gsi_info[gsi_idx].gsi_mutex);
}

static void handle_gsi_cb_unlock(struct mali_arb *arb, u32 gsi_idx)
{
	if (unlikely(!arb || gsi_idx >= MALI_PTM_PARTITION_COUNT)) {
		pr_err("%s: Invalid parameter! (arb = %pK, gsi_idx = %d\n)", __func__, arb,
		       gsi_idx);
		return;
	}

	if (arb->gsi_info[gsi_idx].gsi_enabled)
		mutex_unlock(&arb->gsi_info[gsi_idx].gsi_mutex);
}

static void handle_gsi_cb_active(struct mali_arb *arb, u32 gsi_idx)
{
	if (unlikely(!arb || gsi_idx >= MALI_PTM_PARTITION_COUNT)) {
		pr_err("%s: Invalid parameter! (arb = %pK, gsi_idx = %d\n)", __func__, arb,
		       gsi_idx);
		return;
	}

	if (arb->is_pv)
		mali_arbiter_power_enable(arb->pwr_priv);
	else
		mali_gpu_arbiter_poweron_slices(arb, arb->gsi_info[gsi_idx].gsi_slices);
}

static void handle_gsi_cb_idle(struct mali_arb *arb, u32 gsi_idx)
{
	if (unlikely(!arb || gsi_idx >= MALI_PTM_PARTITION_COUNT)) {
		pr_err("%s: Invalid parameter! (arb = %pK, gsi_idx = %d\n)", __func__, arb,
		       gsi_idx);
		return;
	}

	if (arb->is_pv)
		mali_arbiter_power_disable(arb->pwr_priv);
	else
		mali_gpu_arbiter_nodelay_poweroff_slices(arb, arb->gsi_info[gsi_idx].gsi_slices);
}

/**
 * term_slices - Frees allocated slice data structures
 * @arb: Internal Arbiter data
 */
static void term_slices(struct mali_arb *arb)
{
	u32 i = 0;

	for (i = 0; i < MALI_PTM_SLICES_COUNT; i++) {
		kfree(arb->slices[i]);
		arb->slices[i] = NULL;
	}
	mali_arb_sysfs_free(arb->sysfs);
	arb->sysfs = NULL;
}

/**
 * init_slices - Initialize slice data structures.
 * @arb: Internal Arbiter data
 *
 * Return: 0 on success, otherwise negative linux error code.
 */
static int init_slices(struct mali_arb *arb)
{
	int err = 0;
	u32 i = 0;
	u32 slice_mask = 0, aw_mask = 0;
	unsigned long mask;

	/* Check if rg_ops has a mandatory functions for repartitioning */
	if (!arb->rg_ops || !arb->rg_ops->get_slice_mask || !arb->rg_ops->get_aw_mask) {
		dev_err(arb->dev, "NULL function pointers in rg_ops\n");
		return -EINVAL;
	}

	/* Get the Resource Group masks so we can work out the global
	 * indices of the resources
	 */
	err = arb->rg_ops->get_slice_mask(arb->dev, &slice_mask);
	if (err) {
		dev_err(arb->dev, "Cannot get slice-mask\n");
		return -EIO;
	}
	err = arb->rg_ops->get_aw_mask(arb->dev, &aw_mask);
	if (err) {
		dev_err(arb->dev, "Cannot get aw-mask\n");
		return -EIO;
	}

	mask = slice_mask;
	for_each_set_bit(i, &mask, MALI_PTM_SLICES_COUNT) {
		arb->slices[i] = kzalloc(sizeof(*arb->slices[i]), GFP_KERNEL);
		if (!arb->slices[i]) {
			err = -ENOMEM;
			goto clean_slices;
		}

		arb->slices[i]->arb = arb;
		arb->slices[i]->index = i;
		hrtimer_init(&arb->slices[i]->slice_tmr, CLOCK_MONOTONIC, HRTIMER_MODE_REL);
		arb->slices[i]->slice_tmr.function = slice_timer_isr;
		arb->slices[i]->timer_running = false;
		INIT_WORK(&arb->slices[i]->slice_timeout_work, slice_timeout_worker);

		/* Create SysFs entry for each slice */
		if (mali_arb_sysfs_add_slice(arb->sysfs, i, arb->dev)) {
			dev_err(arb->dev, "Failed to create slice %u SysFS entry\n", i);
			err = -EFAULT;
			goto clean_slices;
		}
	}

	/* Allocate slice workqueue. */
	arb->slices_wq = alloc_ordered_workqueue("mali_arb_slices", WQ_HIGHPRI);
	if (!arb->slices_wq) {
		dev_err(arb->dev, "Failed to create slices wq\n");
		err = -EFAULT;
		goto clean_slices;
	}

	/* Initialize slice mutex. */
	mutex_init(&arb->slices_mutex);

	return 0;

clean_slices:
	term_slices(arb);

	return err;
}

/**
 * update_arbiter_resources() - Updated the Arbiter internal data based on the
 *                              resources managed by the Arbiter.
 * @arb: Internal Arbiter data
 * @res: Resources interfaces provided by the backend
 *
 * This function takes the resource information and updates the Arbiter internal
 * data. It builds the gpu-subinstances, initialize the slices' information and
 * creates sysfs entries for all resources.
 *
 * Return:
 * * 0		- success.
 * * -EINVAL	- invalid argument.
 * * -ENODEV	- expected interface not available.
 * * -EFAULT	- failed with software fault.
 * * -EIO	- failed with hardware fault.
 * * -ENOMEM	- out of memory.
 */
static int update_arbiter_resources(struct mali_arb *arb, struct resource_interfaces res)
{
	int err;
	u32 i;
	struct mali_arb_gsi_cb callbacks = { .cb_lock = handle_gsi_cb_lock,
					     .cb_lock_assert_held = handle_gsi_cb_lock_assert_held,
					     .cb_unlock = handle_gsi_cb_unlock,
					     .cb_active = handle_gsi_cb_active,
					     .cb_idle = handle_gsi_cb_idle };
	u32 num_gsi;

	/* Create the SysFS root */
	arb->sysfs = mali_arb_sysfs_create_root(arb, arb->dev, get_slice_assignment,
						set_slice_assignment, get_aw_assignment,
						set_aw_assignment);
	if (!arb->sysfs) {
		dev_err(arb->dev, "Cannot create SysFS root\n");
		err = -EFAULT;
		goto fail;
	}

	/* Initialize slices, when repartition interface is given */
	if (res.repartition) {
		dev_dbg(arb->dev, "Repartition-interface is provided\n");

		err = init_slices(arb);
		if (err)
			goto clean_root;
	}

	/* When resource interfaces are not provided, it is a single GSI. */
	if (res.num_if)
		num_gsi = res.num_if;
	else
		num_gsi = 1;

	/* Iterate over the resource information, and create gpu-subinstances
	 * that are in the related Resource Group.
	 */
	for (i = 0; i < num_gsi; i++) {
		u32 gsi_id = i;
		bool has_assign_if = (res.vm_assign && res.vm_assign[i]);
		bool has_repart_if = (res.repartition && res.repartition[i]);

		/* Try to get gsi_id from given resources-interfaces.
		 * When not given, gsi_id will be i.
		 */
		if (has_assign_if) {
			gsi_id = res.vm_assign[i]->if_id;

			/* if-id check */
			if (has_repart_if) {
				if (res.repartition[i]->if_id != gsi_id) {
					dev_err(arb->dev,
						"if-id doesn't match between assign and repart (%u != %u)\n",
						gsi_id, res.repartition[i]->if_id);
					err = -EINVAL;
					goto clean_instances;
				}
			}

			/* assign_vm and unassign_vm are mandatory for vm_assign interface */
			if (unlikely(!res.vm_assign[i]->ops->assign_vm ||
				     !res.vm_assign[i]->ops->unassign_vm)) {
				pr_err("%s: Missing VM assign interface callbacks", __func__);
				err = -EINVAL;
				goto clean_instances;
			}
		} else if (has_repart_if) {
			gsi_id = res.repartition[i]->if_id;
		}

		/* check if given gsi_id is not already in use */
		if (arb->gsi_info[gsi_id].gsi_enabled) {
			dev_err(arb->dev, "P%u: gpu-subinstance is already enabled.\n", gsi_id);
			err = -EFAULT;
			goto clean_instances;
		}

		/* Make it enabled */
		arb->gsi_info[gsi_id].gsi_enabled = true;

		/* Create gpu-subinstance */
		arb->gsi_info[gsi_id].gpu_subinstance =
			mali_arbiter_gsi_create(arb->dev, gsi_id, callbacks, arb);
		if (!arb->gsi_info[gsi_id].gpu_subinstance) {
			err = -ENOMEM;
			goto clean_instances;
		}

		/* Initialize mutex */
		mutex_init(&arb->gsi_info[gsi_id].gsi_mutex);

		/* Set access-window mask to 0 */
		arb->gsi_info[gsi_id].gsi_access_windows = 0x0;

		/* Register repartition interface when available and initialize slice mask */
		if (has_repart_if) {
			arb->gsi_info[gsi_id].repart_if = *res.repartition[i];

			/* Configure slice to 0x0 */
			arb->gsi_info[gsi_id].gsi_slices = 0x0;
		} else {
			struct repartition_interface dummy_repart_if = {
				0,
			};
			arb->gsi_info[gsi_id].repart_if = dummy_repart_if;

			/* Configure slice to single and set flag */
			arb->gsi_info[gsi_id].gsi_slices = 0x1;
			mali_arbiter_gsi_flag_set(arb->gsi_info[gsi_id].gpu_subinstance,
						  GSI_FLAG_SLICE_ASSIGNED);
		}

		/* SysFS partition entry setup */
		err = mali_arb_sysfs_add_partition(arb->sysfs, gsi_id, arb->dev, has_repart_if);
		if (err) {
			dev_err(arb->dev, "P%u: Cannot create SysFS entry\n", gsi_id);
			err = -EFAULT;
			goto clean_instances;
		}

		/* When it is provided, register assign interface to partition */
		if (has_assign_if) {
			mali_arbiter_gsi_register_assign_if(arb->gsi_info[gsi_id].gpu_subinstance,
							    *res.vm_assign[i]);
		} else {
			/* When it is not provided, add assign-registration interface to ops */
			arb->arb_data.ops.arb_reg_assign_if = reg_assign_if;
			arb->arb_data.ops.arb_unreg_assign_if = unreg_assign_if;
		}
	}

	/* Initialize list heads */
	INIT_LIST_HEAD(&arb->wait_list);
	INIT_LIST_HEAD(&arb->reg_vms_list);

	return 0;

clean_instances:
	for (i = 0; i < num_gsi; i++) {
		if (arb->gsi_info[i].gsi_enabled) {
			mali_arbiter_gsi_destroy(arb->gsi_info[i].gpu_subinstance);
			arb->gsi_info[i].gpu_subinstance = NULL;
			mutex_destroy(&arb->gsi_info[i].gsi_mutex);
		}
		arb->gsi_info[i].gsi_enabled = false;
	}

	if (arb->slices_wq)
		destroy_workqueue(arb->slices_wq);

	term_slices(arb);

clean_root:
	if (arb->sysfs)
		mali_arb_sysfs_destroy_root(arb->sysfs);

fail:
	return err;
}


/**
 * init_arbiter_data() - Initializes the Arbiter internal data.
 * @arb: Internal Arbiter data
 *
 * Initializes the arbiter internal data with all the common internal data to
 * any Arbiter configuration.
 *
 * Return: 0 on success, or error code
 */
static int init_arbiter_data(struct mali_arb *arb)
{
	/* Callbacks common to Partition Manager and Xen/PV */
	arb->arb_data.ops.gpu_request = gpu_request;
	arb->arb_data.ops.gpu_active = gpu_active;
	arb->arb_data.ops.gpu_idle = gpu_idle;
	arb->arb_data.ops.gpu_stopped = gpu_stopped;
	arb->arb_data.ops.get_max = get_max_config;
	arb->arb_data.ops.register_vm = register_vm;
	arb->arb_data.ops.unregister_vm = unregister_vm;

	return 0;
}

/** Initializes the Arbiter internal data.
 */
int arbiter_core_create(struct device *dev, struct power_interface pwr,
			struct resource_interfaces res, struct mali_arb_data **arb_data)
{
	struct mali_gpu_power_arbiter_cb_ops pwr_cb = {
		.arb_get_utilisation = get_utilisation,
		.update_freq = update_freq,
	};
	struct mali_arb *arb;
	int err;

	if (!dev || !arb_data) {
		pr_err("%s : Invalid argument.\n", __func__);
		return -EINVAL;
	}

	/* Allocate arbiter instance and initiate fundamental fields */
	arb = devm_kzalloc(dev, sizeof(struct mali_arb), GFP_KERNEL);
	if (!arb)
		return -ENOMEM;
	arb->dev = dev;
	arb->rg_ops = dev_get_drvdata(arb->dev);
	if (!arb->rg_ops)
		arb->is_pv = true;

	/* Initialize arbiter data */
	err = init_arbiter_data(arb);
	if (err)
		goto cleanup;

	/* Initialize arbiter resources */
	err = update_arbiter_resources(arb, res);
	if (err)
		goto cleanup;

	/* Initialize arbiter power sub-module */
	err = mali_arbiter_power_init(arb->dev, &arb->arb_data, pwr, pwr_cb, &arb->pwr_priv);
	if (err)
		goto cleanup;

	/* PTM specific initialization */
	if (!arb->is_pv) {
		/* After initializing the arbiter it is necessary to send the
		 * max config information to all VMs assigned to this resource group.
		 * Sending it automatically in the beginning helps to avoid round trip
		 * during the kbase initialization.
		 */
		arb->max_config_info.updated = false;
		err = calculate_max_config(arb);
		if (err)
			dev_dbg(dev, "Failed to calculate max config.\n");

	}

	*arb_data = &arb->arb_data;
	return 0;

cleanup:
	devm_kfree(dev, arb);
	return err;
}

/** Terminates the arbiter device and release its resources
 */
void arbiter_core_destroy(struct device *dev, struct mali_arb_data *arb_data)
{
	struct mali_arb *arb;
	u32 i;

	if (WARN_ON(!dev || !arb_data))
		return;

	arb = arb_from_data(arb_data);

	/* Clean-up sysfs */
	if (arb->sysfs)
		mali_arb_sysfs_free(arb->sysfs);

	/* Clean-up power sub-module */
	mali_arbiter_power_deinit(arb->pwr_priv);

	/* Clean-up gpu-subinstance info */
	for (i = 0; i < MALI_PTM_PARTITION_COUNT; i++) {
		if (arb->gsi_info[i].gsi_enabled) {
			arb->gsi_info[i].gsi_enabled = false;
			mali_arbiter_gsi_destroy(arb->gsi_info[i].gpu_subinstance);
			arb->gsi_info[i].gpu_subinstance = NULL;
			arb->gsi_info[i].gsi_slices = 0x0;
			arb->gsi_info[i].gsi_access_windows = 0x0;
			mutex_destroy(&arb->gsi_info[i].gsi_mutex);
		}
	}

	/* Clean-up slices and wq */
	for (i = 0; i < MALI_PTM_SLICES_COUNT; i++) {
		if (arb->slices[i]) {
			if (arb->slices[i]->timer_running)
				hrtimer_cancel(&arb->slices[i]->slice_tmr);
			kfree(arb->slices[i]);
		}
	}
	if (!arb->is_pv)
		mutex_destroy(&arb->slices_mutex);
	if (arb->slices_wq)
		destroy_workqueue(arb->slices_wq);


	/* Clean-up arbiter instance */
	devm_kfree(dev, arb);
}
