/*
 * Copyright 2003 Sun Microsystems, Inc.  All rights reserved.
 * Use is subject to license terms.
 */

#ifndef	_LGRP_H
#define	_LGRP_H

#pragma ident	"@(#)lgrp.h	1.12	03/09/17 SMI"

/*
 * latency group definitions for kernel
 */

#include <sys/types.h>

#ifdef	__cplusplus
extern "C" {
#endif

#define	LGRP_NONE	(-1)	/* non-existent lgroup ID */

typedef	id_t	lgrp_id_t;	/* lgroup ID */

#if (!defined(_KERNEL) && !defined(_KMEMUSER))
typedef struct lgrp_mem_policy_info { int opaque[2]; }	lgrp_mem_policy_info_t;
#endif	/* !_KERNEL && !_KMEMUSER */

#if (defined(_KERNEL) || defined(_KMEMUSER))
#include <sys/cpuvar.h>
#include <sys/bitmap.h>
#include <sys/vnode.h>
#include <vm/anon.h>
#include <vm/seg.h>
#include <sys/lgrp_user.h>

#define	LGRP_NONE_SUCH		LGRP_NONE	/* non-existent lgroup ID */
#define	LGRP_NULL_HANDLE	((void *)0xbadbad) /* null platform handle */
#define	LGRP_DEFAULT_HANDLE	((void *)0xbabecafe) /* uma handle */
#define	LGRP_ROOTID		(lgrp_root->lgrp_id)	/* root lgroup ID */

/*
 * Maximum number of lgrps a platform may define.
 */

#define	NLGRPS_MAX	19

/*
 * lgroup statistics.  Most of these are counters that are updated
 * dynamically so they are hashed to CPU buckets to reduce cache
 * interference.  The remaining statistics are snapshots of kernel
 * data, so they aren't stored in the array of counter stats.
 *
 * For the hashed stats to make sense, you have to sum all the buckets for
 * that stat, hence macros are provided to read the stats.
 */

#define	LGRP_NUM_CPU_BUCKETS	8	/* must be power of 2 */
#define	LGRP_CPU_BUCKET_MASK	(LGRP_NUM_CPU_BUCKETS - 1)

/*
 * Flags for what to do with lgroup memory policy
 * Used for heap and stack where policy is extended to new segments added to
 * the end
 */
#define	LGRP_MP_FLAG_EXTEND_UP		0x1	/* policy should extend up */
#define	LGRP_MP_FLAG_EXTEND_DOWN	0x2	/* policy should extend down */

#define	LGRP_STAT(stats, bucket, whichstat) \
	((stats)->ls_data[bucket][whichstat])

/* Return a pointer suitable for an atomic 64-bit op on the bucket */
#define	LGRP_STAT_WRITE_PTR(stats, whichstat) \
	(&LGRP_STAT(stats, (CPU->cpu_id) & LGRP_CPU_BUCKET_MASK, \
	    whichstat))

/* Sum up all the buckets and return the value in 'val' */
#define	LGRP_STAT_READ(stats, whichstat, val) {				\
	int bkt;							\
	for (val = 0, bkt = 0; bkt < LGRP_NUM_CPU_BUCKETS; bkt++)	\
		val += LGRP_STAT(stats, bkt, whichstat);		\
}

/* Reset all buckets for the stat to 0 */
#define	LGRP_STAT_RESET(stats, stat) {					\
	int i;								\
	for (i = 0; i < LGRP_NUM_CPU_BUCKETS; i++)			\
		LGRP_STAT(stats, i, stat) = 0;				\
}

/*
 * Define all of the statistics that are kept for lgrp kstats,
 * and their corresponding text names.
 */

typedef enum lgrp_stat_types {
	LGRP_NUM_MIGR,		/* # migrations away from this lgrp */
	LGRP_NUM_ALLOC_FAIL,	/* # times alloc fails for chosen lgrp */
	LGRP_PM_SRC_PGS,	/* # pages migrated from this lgrp */
	LGRP_PM_DEST_PGS,	/* # pages migrated to this lgrp */
	LGRP_PM_FAIL_ALLOC_PGS,	/* # pages failed to migrate to this lgrp */
	LGRP_PM_FAIL_LOCK_PGS,	/* # pages failed to migrate from this lgrp */
	LGRP_PMM_PGS,		/* # pages marked to migrate from this lgrp */
	LGRP_PMM_FAIL_PGS,	/* # pages marked to migrate from this lgrp */
	LGRP_NUM_DEFAULT,	/* # of times default policy applied */
	LGRP_NUM_NEXT,		/* # of times next touch policy applied */
	LGRP_NUM_RANDOM,	/* # of times random policy applied */
	LGRP_NUM_RANDOM_PROC,	/* # of times random proc policy applied */
	LGRP_NUM_RANDOM_PSET,	/* # of times random pset policy applied */
	LGRP_NUM_ROUNDROBIN,	/* # of times round robin policy applied */
	LGRP_NUM_COUNTER_STATS,	/* always last */
	LGRP_CTR_STATS_ALLOC = 16	/* cache-align pad - multiple of 8 */
				/* always keep >= LGRP_NUM_COUNTER_STATS */
} lgrp_stat_t;

typedef enum lgrp_snap_stat_types {
	LGRP_NUM_CPUS,		/* number of CPUs */
	LGRP_NUM_PG_FREE,	/* # of free pages */
	LGRP_NUM_PG_AVAIL,	/* # of allocatable physical pages */
	LGRP_NUM_PG_INSTALL,	/* # of installed physical pages */
	LGRP_LOADAVG,		/* "load average" of this lgrp */
	LGRP_NUM_SNAPSHOT_STATS	/* always last */
} lgrp_snap_stat_t;

#define	LGRP_KSTAT_NAMES		\
static char *lgrp_kstat_names[] = {	\
					\
	/* Counter stats */		\
	"lwp migrations",		\
	"alloc fail",			\
	"pages migrated from",		\
	"pages migrated to",		\
	"pages failed to migrate to",	\
	"pages failed to migrate from",	\
	"pages marked for migration",	\
	"pages failed to mark",		\
	"default policy",		\
	"next-touch policy",		\
	"random policy",		\
	"span process policy",		\
	"span psrset policy",		\
	"round robin policy",		\
					\
	/* Snapshot stats */		\
	"cpus",				\
	"pages free",			\
	"pages avail",			\
	"pages installed",		\
	"load average"			\
}

#define	LGRP_NUM_STATS	(LGRP_NUM_COUNTER_STATS + LGRP_NUM_SNAPSHOT_STATS)

/*
 * The contents of this structure are opaque and should only be
 * accessed through the LGRP_STAT macro.
 */
struct lgrp_stats {
	int64_t ls_data[LGRP_NUM_CPU_BUCKETS][LGRP_CTR_STATS_ALLOC];
};


/*
 * lgroup structure
 *
 * Visible to generic code and contains the lgroup ID, CPUs in this lgroup,
 * and a platform handle used to identify this lgroup to the lgroup platform
 * support code
 */
typedef struct lgrp {

	struct cpu	*lgrp_cpu;	/* pointer to a cpu may be null */
	void *   	lgrp_plathand;	/* handle for platform calls	*/
	lgrp_id_t 	lgrp_id;	/* which group this is		*/
	uint_t		lgrp_cpucnt;	/* number of cpus in this lgrp	*/

	/*
	 * per-lgrp kstats
	 */
	struct kstat_named lgrp_kstat_data[LGRP_NUM_STATS];
	kstat_t		*lgrp_kstat;
	kmutex_t	lgrp_kstat_mutex;

} lgrp_t;

/*
 * lgroup load average structure
 */
typedef struct lgrp_ld {
	int32_t		lpl_loadavg;	/* load average		*/
	uint_t		lpl_ncpu;	/* how many cpus  	*/
	lgrp_id_t 	lpl_lgrpid;	/* which group this lpl part of */
} lpl_t;

/*
 * Operations handled by lgrp_config()
 */
typedef enum lgrp_config_flag {
	LGRP_CONFIG_CPU_ADD,
	LGRP_CONFIG_CPU_DEL,
	LGRP_CONFIG_MEM_ADD,
	LGRP_CONFIG_MEM_DEL,
	LGRP_CONFIG_MEM_RENAME,
	LGRP_CONFIG_GEN_UPDATE
} lgrp_config_flag_t;

/*
 * Memory allocation policies
 */
typedef enum lgrp_mem_policy {
	LGRP_MEM_POLICY_DEFAULT,
	LGRP_MEM_POLICY_NEXT,		/* near LWP to next touch */
	LGRP_MEM_POLICY_RANDOM_PROC,	/* randomly across process */
	LGRP_MEM_POLICY_RANDOM_PSET,	/* randomly across processor set */
	LGRP_MEM_POLICY_RANDOM,		/* randomly across all lgroups */
	LGRP_MEM_POLICY_ROUNDROBIN,	/* round robin across all lgroups */
	LGRP_NUM_MEM_POLICIES
} lgrp_mem_policy_t;

/*
 * Information needed to implement memory allocation policy
 */
typedef struct lgrp_mem_policy_info {
	int	mem_policy;			/* memory allocation policy */
	int	mem_reserved;			/* reserved */
} lgrp_mem_policy_info_t;

/*
 * Shared memory policy segment
 */
typedef struct lgrp_shm_policy_seg {
	u_offset_t		shm_off;	/* offset into shared object */
	size_t			shm_size;	/* size of segment */
	lgrp_mem_policy_info_t	shm_policy;	/* memory allocation policy */
	avl_node_t		shm_tree;	/* AVL tree */
} lgrp_shm_policy_seg_t;

/*
 * Shared memory locality info
 */
typedef struct lgrp_shm_locality {
	size_t		loc_count;		/* reference count */
	avl_tree_t	*loc_tree;		/* policy segment tree */
	krwlock_t	loc_lock;		/* protects tree */
} lgrp_shm_locality_t;

/*
 * Queries that may be made to determine lgroup memory size
 */
#define	LGRP_MEM_SIZE_FREE	1	/* number of free pages */
#define	LGRP_MEM_SIZE_AVAIL	2	/* number of pages in phys_avail */
#define	LGRP_MEM_SIZE_INSTALL	3	/* number of pages in phys_install */

/* The kernel's version of a bitmap of lgroups */
typedef uint64_t klgrpset_t;

/* Macro to clear an lgroup bitmap */
#define	klgrpset_clear(klgrpset) \
	(klgrpset) = 0

/* Macro to add an lgroup to an lgroup bitmap */
#define	klgrpset_add(klgrpset, lgrpid) \
	(klgrpset) |= (1 << (lgrpid))

/* Macro to delete an lgroup from an lgroup bitmap */
#define	klgrpset_del(klgrpset, lgrpid) \
	(klgrpset) &= ~(1 << (lgrpid))

/* Macro to copy a klgrpset into another klgrpset */
#define	klgrpset_copy(klgrpset_to, klgrpset_from) \
	(klgrpset_to) = (klgrpset_from)

/* Macro to perform an 'and' operation on a pair of lgroup bitmaps */
#define	klgrpset_and(klgrpset_rslt, klgrpset_arg) \
	(klgrpset_rslt) &= (klgrpset_arg)

/* Macro to perform an 'or' operation on a pair of lgroup bitmaps */
#define	klgrpset_or(klgrpset_rslt, klgrpset_arg) \
	(klgrpset_rslt) |= (klgrpset_arg)

/* Macro to perform a 'diff' operation on a pair of lgroup bitmaps */
#define	klgrpset_diff(klgrpset_rslt, klgrpset_arg) \
	(klgrpset_rslt) &= ~(klgrpset_arg)

/* Macro to check if an lgroup is a member of an lgrpset */
#define	klgrpset_ismember(klgrpset, lgrpid) \
	((klgrpset) & (1 << (lgrpid)))

/* Macro to check if an lgroup bitmap is empty */
#define	klgrpset_isempty(klgrpset) \
	((klgrpset) == 0)

/* Macro to count the number of members in an lgrpset */
#define	klgrpset_nlgrps(klgrpset, count)				\
{									\
	lgrp_id_t	lgrpid;						\
	for (lgrpid = 0, count = 0; lgrpid < nlgrps; lgrpid++) {	\
		if (klgrpset_ismember(klgrpset, lgrpid))		\
			count++;					\
	}								\
}

/* Macro to get total memory size (in bytes) of a given set of lgroups */
#define	klgrpset_totalsize(klgrpset, size)				\
{									\
	void		*hand;						\
	lgrp_id_t	lgrpid;						\
									\
	for (lgrpid = 0, size = 0; lgrpid < nlgrps; lgrpid++) {		\
		if (klgrpset_ismember(klgrpset, lgrpid) &&		\
		    lgrp_table[lgrpid])	{				\
			hand = lgrp_table[lgrpid]->lgrp_plathand;	\
			size += lgrp_plat_mem_size(hand,		\
			    LGRP_MEM_SIZE_AVAIL) * PAGESIZE;		\
		}							\
	}								\
}

/*
 * Macro to see whether memory allocation policy can be reapplied
 */
#define	LGRP_MEM_POLICY_REAPPLICABLE(p) \
	(p == LGRP_MEM_POLICY_NEXT)


extern lgrp_t	*lgrp_table[NLGRPS_MAX];	/* indexed by lgrp_id */
extern int	nlgrps;			/* number of lgroups in machine */
extern int	nlgrpsmax;		/* max number of lgroups on platform */
extern lgrp_gen_t	lgrp_gen;	/* generation of lgroup hierarchy */
extern lgrp_t		*lgrp_root;	/* root lgroup */

/* generic interfaces */
lpl_t	*lgrp_affinity_best(kthread_t *, struct cpupart *, lgrp_id_t);
void	lgrp_affinity_init(lgrp_affinity_t **);
void	lgrp_affinity_free(lgrp_affinity_t **);
void	lgrp_part_add_cpu(struct cpu *, lgrp_id_t);
void	lgrp_part_del_cpu(struct cpu *);
void	lgrp_loadavg(lpl_t *, uint_t, int);
void	lgrp_config(lgrp_config_flag_t, void *);
void	lgrp_kstat_create(cpu_t *);
void	lgrp_kstat_destroy(cpu_t *);
void	lgrp_cpu_init(struct cpu *);
void	lgrp_cpu_fini(struct cpu *, lgrp_id_t);
lgrp_t	*lgrp_hand_to_lgrp(void *);
lgrp_id_t	lgrp_home_id(kthread_t *);
void	*lgrp_home_handle(kthread_t *);
lgrp_t	*lgrp_phys_to_lgrp(u_longlong_t); /* used by numat driver */
lgrp_t  *lgrp_pfn_to_lgrp(pfn_t);
int	lgrp_query_cpu(processorid_t, lgrp_id_t *);
int	lgrp_query_load(processorid_t, uint_t *);
lpl_t	*lgrp_choose(kthread_t *, struct cpupart *);
void	lgrp_move_thread(kthread_t *, lpl_t *, int);
lgrp_mem_policy_t	lgrp_madv_to_policy(uchar_t, size_t, int);
pgcnt_t	lgrp_mem_size(lgrp_id_t, int);
void	*lgrp_mem_hand(struct seg *, caddr_t, size_t);
lgrp_mem_policy_t	lgrp_mem_policy_default(size_t, int);
int	lgrp_optimizations(void);
int	lgrp_privm_policy_set(lgrp_mem_policy_t, lgrp_mem_policy_info_t *,
    size_t);
void	lgrp_root_init(void);
void	lgrp_shm_policy_init(struct anon_map *, vnode_t *);
void	lgrp_shm_policy_fini(struct anon_map *, vnode_t *);
lgrp_mem_policy_info_t	*lgrp_shm_policy_get(struct anon_map *, ulong_t,
    vnode_t *, u_offset_t);
int	lgrp_shm_policy_set(lgrp_mem_policy_t, struct anon_map *, ulong_t,
    vnode_t *, u_offset_t, size_t);
void	lgrp_stat_add(lgrp_id_t, lgrp_stat_t, int64_t);
void	lgrp_stat_add_hand(void *, lgrp_stat_t, int64_t);
int64_t lgrp_stat_read(lgrp_id_t, lgrp_stat_t);

/* platform interfaces */
void	lgrp_plat_init(void);
lgrp_t	*lgrp_plat_alloc(lgrp_id_t lgrpid);
void	lgrp_plat_config(lgrp_config_flag_t, void *);
void	*lgrp_plat_cpu_to_hand(processorid_t);
void	*lgrp_plat_pfn_to_hand(pfn_t);
int	lgrp_plat_max_lgrps(void);
pgcnt_t	lgrp_plat_mem_size(void *, int);
int	lgrp_plat_latency(void *, void *);
void	*lgrp_plat_root_hand(void);

#endif	/* _KERNEL && _KMEMUSER */

#ifdef	__cplusplus
}
#endif

#endif /* _LGRP_H */
