llvm-project/polly/lib/External/ppcg/grouping.c

/*
 * Copyright 2016      Sven Verdoolaege
 *
 * Use of this software is governed by the MIT license
 *
 * Written by Sven Verdoolaege.
 */

#include <isl/ctx.h>
#include <isl/id.h>
#include <isl/val.h>
#include <isl/space.h>
#include <isl/aff.h>
#include <isl/set.h>
#include <isl/map.h>
#include <isl/union_set.h>
#include <isl/union_map.h>
#include <isl/schedule.h>
#include <isl/schedule_node.h>

#include "ppcg.h"

/* Internal data structure for use during the detection of statements
 * that can be grouped.
 *
 * "sc" contains the original schedule constraints (not a copy).
 * "dep" contains the intersection of the validity and the proximity
 * constraints in "sc".  It may be NULL if it has not been computed yet.
 * "group_id" is the identifier for the next group that is extracted.
 *
 * "domain" is the set of statement instances that belong to any of the groups.
 * "contraction" maps the elements of "domain" to the corresponding group
 * instances.
 * "schedule" schedules the statements in each group relatively to each other.
 * These last three fields are NULL if no groups have been found so far.
 */
struct ppcg_grouping {
	isl_schedule_constraints *sc;

	isl_union_map *dep;
	int group_id;

	isl_union_set *domain;
	isl_union_pw_multi_aff *contraction;
	isl_schedule *schedule;
};

/* Clear all memory allocated by "grouping".
 */
static void ppcg_grouping_clear(struct ppcg_grouping *grouping)
{
	isl_union_map_free(grouping->dep);
	isl_union_set_free(grouping->domain);
	isl_union_pw_multi_aff_free(grouping->contraction);
	isl_schedule_free(grouping->schedule);
}

/* Compute the intersection of the proximity and validity dependences
 * in grouping->sc and store the result in grouping->dep, unless
 * this intersection has been computed before.
 */
static isl_stat ppcg_grouping_compute_dep(struct ppcg_grouping *grouping)
{
	isl_union_map *validity, *proximity;

	if (grouping->dep)
		return isl_stat_ok;

	validity = isl_schedule_constraints_get_validity(grouping->sc);
	proximity = isl_schedule_constraints_get_proximity(grouping->sc);
	grouping->dep = isl_union_map_intersect(validity, proximity);

	if (!grouping->dep)
		return isl_stat_error;

	return isl_stat_ok;
}

/* Information extracted from one or more consecutive leaves
 * in the input schedule.
 *
 * "list" contains the sets of statement instances in the leaves,
 * one element in the list for each original leaf.
 * "domain" contains the union of the sets in "list".
 * "prefix" contains the prefix schedule of these elements.
 */
struct ppcg_grouping_leaf {
	isl_union_set *domain;
	isl_union_set_list *list;
	isl_multi_union_pw_aff *prefix;
};

/* Free all memory allocated for "leaves".
 */
static void ppcg_grouping_leaf_free(int n, struct ppcg_grouping_leaf leaves[])
{
	int i;

	if (!leaves)
		return;

	for (i = 0; i < n; ++i) {
		isl_union_set_free(leaves[i].domain);
		isl_union_set_list_free(leaves[i].list);
		isl_multi_union_pw_aff_free(leaves[i].prefix);
	}

	free(leaves);
}

/* Short-hand for retrieving the prefix schedule at "node"
 * in the form of an isl_multi_union_pw_aff.
 */
static __isl_give isl_multi_union_pw_aff *get_prefix(
	__isl_keep isl_schedule_node *node)
{
	return isl_schedule_node_get_prefix_schedule_multi_union_pw_aff(node);
}

/* Return an array of "n" elements with information extracted from
 * the "n" children of "node" starting at "first", all of which
 * are known to be filtered leaves.
 */
struct ppcg_grouping_leaf *extract_leaves(__isl_keep isl_schedule_node *node,
	int first, int n)
{
	int i;
	isl_ctx *ctx;
	struct ppcg_grouping_leaf *leaves;

	if (!node)
		return NULL;

	ctx = isl_schedule_node_get_ctx(node);
	leaves = isl_calloc_array(ctx, struct ppcg_grouping_leaf, n);
	if (!leaves)
		return NULL;

	for (i = 0; i < n; ++i) {
		isl_schedule_node *child;
		isl_union_set *domain;

		child = isl_schedule_node_get_child(node, first + i);
		child = isl_schedule_node_child(child, 0);
		domain = isl_schedule_node_get_domain(child);
		leaves[i].domain = isl_union_set_copy(domain);
		leaves[i].list = isl_union_set_list_from_union_set(domain);
		leaves[i].prefix = get_prefix(child);
		isl_schedule_node_free(child);
	}

	return leaves;
}

/* Internal data structure used by merge_leaves.
 *
 * "src" and "dst" point to the two consecutive leaves that are
 * under investigation for being merged.
 * "merge" is initially set to 0 and is set to 1 as soon as
 * it turns out that it is useful to merge the two leaves.
 */
struct ppcg_merge_leaves_data {
	int merge;
	struct ppcg_grouping_leaf *src;
	struct ppcg_grouping_leaf *dst;
};

/* Given a relation "map" between instances of two statements A and B,
 * does it relate every instance of A (according to the domain of "src")
 * to every instance of B (according to the domain of "dst")?
 */
static isl_bool covers_src_and_dst(__isl_keep isl_map *map,
	struct ppcg_grouping_leaf *src, struct ppcg_grouping_leaf *dst)
{
	isl_space *space;
	isl_set *set1, *set2;
	isl_bool is_subset;

	space = isl_space_domain(isl_map_get_space(map));
	set1 = isl_union_set_extract_set(src->domain, space);
	set2 = isl_map_domain(isl_map_copy(map));
	is_subset = isl_set_is_subset(set1, set2);
	isl_set_free(set1);
	isl_set_free(set2);
	if (is_subset < 0 || !is_subset)
		return is_subset;

	space = isl_space_range(isl_map_get_space(map));
	set1 = isl_union_set_extract_set(dst->domain, space);
	set2 = isl_map_range(isl_map_copy(map));
	is_subset = isl_set_is_subset(set1, set2);
	isl_set_free(set1);
	isl_set_free(set2);

	return is_subset;
}

/* Given a relation "map" between instances of two statements A and B,
 * are pairs of related instances executed together in the input schedule?
 * That is, is each pair of instances assigned the same value
 * by the corresponding prefix schedules?
 *
 * In particular, select the subset of "map" that has pairs of elements
 * with the same value for the prefix schedules and then check
 * if "map" is still a subset of the result.
 */
static isl_bool matches_prefix(__isl_keep isl_map *map,
	struct ppcg_grouping_leaf *src, struct ppcg_grouping_leaf *dst)
{
	isl_union_map *umap, *equal;
	isl_multi_union_pw_aff *src_prefix, *dst_prefix, *prefix;
	isl_bool is_subset;

	src_prefix = isl_multi_union_pw_aff_copy(src->prefix);
	dst_prefix = isl_multi_union_pw_aff_copy(dst->prefix);
	prefix = isl_multi_union_pw_aff_union_add(src_prefix, dst_prefix);

	umap = isl_union_map_from_map(isl_map_copy(map));
	equal = isl_union_map_copy(umap);
	equal = isl_union_map_eq_at_multi_union_pw_aff(equal, prefix);

	is_subset = isl_union_map_is_subset(umap, equal);

	isl_union_map_free(umap);
	isl_union_map_free(equal);

	return is_subset;
}

/* Given a set of validity and proximity schedule constraints "map"
 * between statements in consecutive leaves in a valid schedule,
 * should the two leaves be merged into one?
 *
 * In particular, the two are merged if the constraints form
 * a bijection between every instance of the first statement and
 * every instance of the second statement.  Moreover, each
 * pair of such dependent instances needs to be executed consecutively
 * in the input schedule.  That is, they need to be assigned
 * the same value by their prefix schedules.
 *
 * What this means is that for each instance of the first statement
 * there is exactly one instance of the second statement that
 * is executed immediately after the instance of the first statement and
 * that, moreover, both depends on this statement instance and
 * should be brought as close as possible to this statement instance.
 * In other words, it is both possible to execute the two instances
 * together (according to the input schedule) and desirable to do so
 * (according to the validity and proximity schedule constraints).
 */
static isl_stat check_merge(__isl_take isl_map *map, void *user)
{
	struct ppcg_merge_leaves_data *data = user;
	isl_bool ok;

	ok = covers_src_and_dst(map, data->src, data->dst);
	if (ok >= 0 && ok)
		ok = isl_map_is_bijective(map);
	if (ok >= 0 && ok)
		ok = matches_prefix(map, data->src, data->dst);

	isl_map_free(map);

	if (ok < 0)
		return isl_stat_error;
	if (!ok)
		return isl_stat_ok;

	data->merge = 1;
	return isl_stat_error;
}

/* Merge the leaves at position "pos" and "pos + 1" in "leaves".
 */
static isl_stat merge_pair(int n, struct ppcg_grouping_leaf leaves[], int pos)
{
	int i;

	leaves[pos].domain = isl_union_set_union(leaves[pos].domain,
						leaves[pos + 1].domain);
	leaves[pos].list = isl_union_set_list_concat(leaves[pos].list,
						leaves[pos + 1].list);
	leaves[pos].prefix = isl_multi_union_pw_aff_union_add(
				leaves[pos].prefix, leaves[pos + 1].prefix);
	for (i = pos + 1; i + 1 < n; ++i)
		leaves[i] = leaves[i + 1];
	leaves[n - 1].domain = NULL;
	leaves[n - 1].list = NULL;
	leaves[n - 1].prefix = NULL;

	if (!leaves[pos].domain || !leaves[pos].list || !leaves[pos].prefix)
		return isl_stat_error;

	return isl_stat_ok;
}

/* Merge pairs of consecutive leaves in "leaves" taking into account
 * the intersection of validity and proximity schedule constraints "dep".
 *
 * If a leaf has been merged with the next leaf, then the combination
 * is checked again for merging with the next leaf.
 * That is, if the leaves are A, B and C, then B may not have been
 * merged with C, but after merging A and B, it could still be useful
 * to merge the combination AB with C.
 *
 * Two leaves A and B are merged if there are instances of at least
 * one pair of statements, one statement in A and one B, such that
 * the validity and proximity schedule constraints between them
 * make them suitable for merging according to check_merge.
 *
 * Return the final number of leaves in the sequence, or -1 on error.
 */
static int merge_leaves(int n, struct ppcg_grouping_leaf leaves[],
	__isl_keep isl_union_map *dep)
{
	int i;
	struct ppcg_merge_leaves_data data;

	for (i = n - 1; i >= 0; --i) {
		isl_union_map *dep_i;
		isl_stat ok;

		if (i + 1 >= n)
			continue;

		dep_i = isl_union_map_copy(dep);
		dep_i = isl_union_map_intersect_domain(dep_i,
				isl_union_set_copy(leaves[i].domain));
		dep_i = isl_union_map_intersect_range(dep_i,
				isl_union_set_copy(leaves[i + 1].domain));
		data.merge = 0;
		data.src = &leaves[i];
		data.dst = &leaves[i + 1];
		ok = isl_union_map_foreach_map(dep_i, &check_merge, &data);
		isl_union_map_free(dep_i);
		if (ok < 0 && !data.merge)
			return -1;
		if (!data.merge)
			continue;
		if (merge_pair(n, leaves, i) < 0)
			return -1;
		--n;
		++i;
	}

	return n;
}

/* Construct a schedule with "domain" as domain, that executes
 * the elements of "list" in order (as a sequence).
 */
static __isl_give isl_schedule *schedule_from_domain_and_list(
	__isl_keep isl_union_set *domain, __isl_keep isl_union_set_list *list)
{
	isl_schedule *schedule;
	isl_schedule_node *node;

	schedule = isl_schedule_from_domain(isl_union_set_copy(domain));
	node = isl_schedule_get_root(schedule);
	isl_schedule_free(schedule);
	node = isl_schedule_node_child(node, 0);
	list = isl_union_set_list_copy(list);
	node = isl_schedule_node_insert_sequence(node, list);
	schedule = isl_schedule_node_get_schedule(node);
	isl_schedule_node_free(node);

	return schedule;
}

/* Construct a unique identifier for a group in "grouping".
 *
 * The name is of the form G_n, with n the first value starting at
 * grouping->group_id that does not result in an identifier
 * that is already in use in the domain of the original schedule
 * constraints.
 */
static isl_id *construct_group_id(struct ppcg_grouping *grouping,
	__isl_take isl_space *space)
{
	isl_ctx *ctx;
	isl_id *id;
	isl_bool empty;
	isl_union_set *domain;

	if (!space)
		return NULL;

	ctx = isl_space_get_ctx(space);
	domain = isl_schedule_constraints_get_domain(grouping->sc);

	do {
		char buffer[20];
		isl_id *id;
		isl_set *set;

		snprintf(buffer, sizeof(buffer), "G_%d", grouping->group_id);
		grouping->group_id++;
		id = isl_id_alloc(ctx, buffer, NULL);
		space = isl_space_set_tuple_id(space, isl_dim_set, id);
		set = isl_union_set_extract_set(domain, isl_space_copy(space));
		empty = isl_set_plain_is_empty(set);
		isl_set_free(set);
	} while (empty >= 0 && !empty);

	if (empty < 0)
		space = isl_space_free(space);

	id = isl_space_get_tuple_id(space, isl_dim_set);

	isl_space_free(space);
	isl_union_set_free(domain);

	return id;
}

/* Construct a contraction from "prefix" and "domain" for a new group
 * in "grouping".
 *
 * The values of the prefix schedule "prefix" are used as instances
 * of the new group.  The identifier of the group is constructed
 * in such a way that it does not conflict with those of earlier
 * groups nor with statements in the domain of the original
 * schedule constraints.
 * The isl_multi_union_pw_aff "prefix" then simply needs to be
 * converted to an isl_union_pw_multi_aff.  However, this is not
 * possible if "prefix" is zero-dimensional, so in this case,
 * a contraction is constructed from "domain" instead.
 */
static isl_union_pw_multi_aff *group_contraction_from_prefix_and_domain(
	struct ppcg_grouping *grouping,
	__isl_keep isl_multi_union_pw_aff *prefix,
	__isl_keep isl_union_set *domain)
{
	isl_id *id;
	isl_space *space;
	int dim;

	space = isl_multi_union_pw_aff_get_space(prefix);
	if (!space)
		return NULL;
	dim = isl_space_dim(space, isl_dim_set);
	id = construct_group_id(grouping, space);
	if (dim == 0) {
		isl_multi_val *mv;

		space = isl_multi_union_pw_aff_get_space(prefix);
		space = isl_space_set_tuple_id(space, isl_dim_set, id);
		mv = isl_multi_val_zero(space);
		domain = isl_union_set_copy(domain);
		return isl_union_pw_multi_aff_multi_val_on_domain(domain, mv);
	}
	prefix = isl_multi_union_pw_aff_copy(prefix);
	prefix = isl_multi_union_pw_aff_set_tuple_id(prefix, isl_dim_out, id);
	return isl_union_pw_multi_aff_from_multi_union_pw_aff(prefix);
}

/* Extend "grouping" with groups corresponding to merged
 * leaves in the list of potentially merged leaves "leaves".
 *
 * The "list" field of each element in "leaves" contains a list
 * of the instances sets of the original leaves that have been
 * merged into this element.  If at least two of the original leaves
 * have been merged into a given element, then add the corresponding
 * group to "grouping".
 * In particular, the domain is extended with the statement instances
 * of the merged leaves, the contraction is extended with a mapping
 * of these statement instances to instances of a new group and
 * the schedule is extended with a schedule that executes
 * the statement instances according to the order of the leaves
 * in which they appear.
 * Since the instances of the groups should already be scheduled apart
 * in the schedule into which this schedule will be plugged in,
 * the schedules of the individual groups are combined independently
 * of each other (as a set).
 */
static isl_stat add_groups(struct ppcg_grouping *grouping,
	int n, struct ppcg_grouping_leaf leaves[])
{
	int i;

	for (i = 0; i < n; ++i) {
		int n_leaf;
		isl_schedule *schedule;
		isl_union_set *domain;
		isl_union_pw_multi_aff *upma;

		n_leaf = isl_union_set_list_n_union_set(leaves[i].list);
		if (n_leaf < 0)
			return isl_stat_error;
		if (n_leaf <= 1)
			continue;
		schedule = schedule_from_domain_and_list(leaves[i].domain,
							leaves[i].list);
		upma = group_contraction_from_prefix_and_domain(grouping,
					leaves[i].prefix, leaves[i].domain);

		domain = isl_union_set_copy(leaves[i].domain);
		if (grouping->domain) {
			domain = isl_union_set_union(domain, grouping->domain);
			upma = isl_union_pw_multi_aff_union_add(upma,
						grouping->contraction);
			schedule = isl_schedule_set(schedule,
						grouping->schedule);
		}
		grouping->domain = domain;
		grouping->contraction = upma;
		grouping->schedule = schedule;

		if (!grouping->domain || !grouping->contraction ||
		    !grouping->schedule)
			return isl_stat_error;
	}

	return isl_stat_ok;
}

/* Look for any pairs of consecutive leaves among the "n" children of "node"
 * starting at "first" that should be merged together.
 * Store the results in "grouping".
 *
 * First make sure the intersection of validity and proximity
 * schedule constraints is available and extract the required
 * information from the "n" leaves.
 * Then try and merge consecutive leaves based on the validity
 * and proximity constraints.
 * If any pairs were successfully merged, then add groups
 * corresponding to the merged leaves to "grouping".
 */
static isl_stat group_subsequence(__isl_keep isl_schedule_node *node,
	int first, int n, struct ppcg_grouping *grouping)
{
	int n_merge;
	struct ppcg_grouping_leaf *leaves;

	if (ppcg_grouping_compute_dep(grouping) < 0)
		return isl_stat_error;

	leaves = extract_leaves(node, first, n);
	if (!leaves)
		return isl_stat_error;

	n_merge = merge_leaves(n, leaves, grouping->dep);
	if (n_merge >= 0 && n_merge < n &&
	    add_groups(grouping, n_merge, leaves) < 0)
		return isl_stat_error;

	ppcg_grouping_leaf_free(n, leaves);

	return isl_stat_ok;
}

/* If "node" is a sequence, then check if it has any consecutive
 * leaves that should be merged together and store the results
 * in "grouping".
 *
 * In particular, call group_subsequence on each consecutive
 * sequence of (filtered) leaves among the children of "node".
 */
static isl_bool detect_groups(__isl_keep isl_schedule_node *node, void *user)
{
	int i, n, first;
	struct ppcg_grouping *grouping = user;

	if (isl_schedule_node_get_type(node) != isl_schedule_node_sequence)
		return isl_bool_true;

	n = isl_schedule_node_n_children(node);
	if (n < 0)
		return isl_bool_error;

	first = -1;
	for (i = 0; i < n; ++i) {
		isl_schedule_node *child;
		enum isl_schedule_node_type type;

		child = isl_schedule_node_get_child(node, i);
		child = isl_schedule_node_child(child, 0);
		type = isl_schedule_node_get_type(child);
		isl_schedule_node_free(child);

		if (first >= 0 && type != isl_schedule_node_leaf) {
			if (group_subsequence(node, first, i - first,
						grouping) < 0)
				return isl_bool_error;
			first = -1;
		}
		if (first < 0 && type == isl_schedule_node_leaf)
			first = i;
	}
	if (first >= 0) {
		if (group_subsequence(node, first, n - first, grouping) < 0)
			return isl_bool_error;
	}

	return isl_bool_true;
}

/* Complete "grouping" to cover all statement instances in the domain
 * of grouping->sc.
 *
 * In particular, grouping->domain is set to the full set of statement
 * instances; group->contraction is extended with an identity
 * contraction on the additional instances and group->schedule
 * is extended with an independent schedule on those additional instances.
 * In the extension of group->contraction, the additional instances
 * are split into those belong to different statements and those
 * that belong to some of the same statements.  The first group
 * is replaced by its universe in order to simplify the contraction extension.
 */
static void complete_grouping(struct ppcg_grouping *grouping)
{
	isl_union_set *domain, *left, *overlap;
	isl_union_pw_multi_aff *upma;
	isl_schedule *schedule;

	domain = isl_schedule_constraints_get_domain(grouping->sc);
	left = isl_union_set_subtract(isl_union_set_copy(domain),
				    isl_union_set_copy(grouping->domain));
	schedule = isl_schedule_from_domain(isl_union_set_copy(left));
	schedule = isl_schedule_set(schedule, grouping->schedule);
	grouping->schedule = schedule;

	overlap = isl_union_set_universe(grouping->domain);
	grouping->domain = domain;
	overlap = isl_union_set_intersect(isl_union_set_copy(left), overlap);
	left = isl_union_set_subtract(left, isl_union_set_copy(overlap));
	left = isl_union_set_universe(left);
	left = isl_union_set_union(left, overlap);
	upma = isl_union_set_identity_union_pw_multi_aff(left);
	upma = isl_union_pw_multi_aff_union_add(upma, grouping->contraction);
	grouping->contraction = upma;
}

/* Compute a schedule on the domain of "sc" that respects the schedule
 * constraints in "sc".
 *
 * "schedule" is a known correct schedule that is used to combine
 * groups of statements if options->group_chains is set.
 * In particular, statements that are executed consecutively in a sequence
 * in this schedule and where all instances of the second depend on
 * the instance of the first that is executed in the same iteration
 * of outer band nodes are grouped together into a single statement.
 * The schedule constraints are then mapped to these groups of statements
 * and the resulting schedule is expanded again to refer to the original
 * statements.
 */
__isl_give isl_schedule *ppcg_compute_schedule(
	__isl_take isl_schedule_constraints *sc,
	__isl_keep isl_schedule *schedule, struct ppcg_options *options)
{
	struct ppcg_grouping grouping = { sc };
	isl_union_pw_multi_aff *contraction;
	isl_union_map *umap;
	isl_schedule *res, *expansion;

	if (!options->group_chains)
		return isl_schedule_constraints_compute_schedule(sc);

	grouping.group_id = 0;
	if (isl_schedule_foreach_schedule_node_top_down(schedule,
			&detect_groups, &grouping) < 0)
		goto error;
	if (!grouping.contraction) {
		ppcg_grouping_clear(&grouping);
		return isl_schedule_constraints_compute_schedule(sc);
	}
	complete_grouping(&grouping);
	contraction = isl_union_pw_multi_aff_copy(grouping.contraction);
	umap = isl_union_map_from_union_pw_multi_aff(contraction);

	sc = isl_schedule_constraints_apply(sc, umap);

	res = isl_schedule_constraints_compute_schedule(sc);

	contraction = isl_union_pw_multi_aff_copy(grouping.contraction);
	expansion = isl_schedule_copy(grouping.schedule);
	res = isl_schedule_expand(res, contraction, expansion);

	ppcg_grouping_clear(&grouping);
	return res;
error:
	ppcg_grouping_clear(&grouping);
	isl_schedule_constraints_free(sc);
	return NULL;
}