| /* Copyright (C) 2015-2022 Free Software Foundation, Inc. |
| Contributed by Aldy Hernandez <aldyh@redhat.com>. |
| |
| This file is part of the GNU Offloading and Multi Processing Library |
| (libgomp). |
| |
| Libgomp is free software; you can redistribute it and/or modify it |
| under the terms of the GNU General Public License as published by |
| the Free Software Foundation; either version 3, or (at your option) |
| any later version. |
| |
| Libgomp is distributed in the hope that it will be useful, but WITHOUT ANY |
| WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS |
| FOR A PARTICULAR PURPOSE. See the GNU General Public License for |
| more details. |
| |
| Under Section 7 of GPL version 3, you are granted additional |
| permissions described in the GCC Runtime Library Exception, version |
| 3.1, as published by the Free Software Foundation. |
| |
| You should have received a copy of the GNU General Public License and |
| a copy of the GCC Runtime Library Exception along with this program; |
| see the files COPYING3 and COPYING.RUNTIME respectively. If not, see |
| <http://www.gnu.org/licenses/>. */ |
| |
| /* Header file for a priority queue of GOMP tasks. */ |
| |
| /* ?? Perhaps all the priority_tree_* functions are complex and rare |
| enough to go out-of-line and be moved to priority_queue.c. ?? */ |
| |
| #ifndef _PRIORITY_QUEUE_H_ |
| #define _PRIORITY_QUEUE_H_ |
| |
| /* One task. */ |
| |
| struct priority_node |
| { |
| /* Next and previous chains in a circular doubly linked list for |
| tasks within this task's priority. */ |
| struct priority_node *next, *prev; |
| }; |
| |
| /* All tasks within the same priority. */ |
| |
| struct priority_list |
| { |
| /* Priority of the tasks in this set. */ |
| int priority; |
| |
| /* Tasks. */ |
| struct priority_node *tasks; |
| |
| /* This points to the last of the higher priority WAITING tasks. |
| Remember that for the children queue, we have: |
| |
| parent_depends_on WAITING tasks. |
| !parent_depends_on WAITING tasks. |
| TIED tasks. |
| |
| This is a pointer to the last of the parent_depends_on WAITING |
| tasks which are essentially, higher priority items within their |
| priority. */ |
| struct priority_node *last_parent_depends_on; |
| }; |
| |
| /* Another splay tree instantiation, for priority_list's. */ |
| typedef struct prio_splay_tree_node_s *prio_splay_tree_node; |
| typedef struct prio_splay_tree_s *prio_splay_tree; |
| typedef struct prio_splay_tree_key_s *prio_splay_tree_key; |
| struct prio_splay_tree_key_s { |
| /* This structure must only containing a priority_list, as we cast |
| prio_splay_tree_key to priority_list throughout. */ |
| struct priority_list l; |
| }; |
| #define splay_tree_prefix prio |
| #include "splay-tree.h" |
| |
| /* The entry point into a priority queue of tasks. |
| |
| There are two alternate implementations with which to store tasks: |
| as a balanced tree of sorts, or as a simple list of tasks. If |
| there are only priority-0 items (ROOT is NULL), we use the simple |
| list, otherwise (ROOT is non-NULL) we use the tree. */ |
| |
| struct priority_queue |
| { |
| /* If t.root != NULL, this is a splay tree of priority_lists to hold |
| all tasks. This is only used if multiple priorities are in play, |
| otherwise we use the priority_list `l' below to hold all |
| (priority-0) tasks. */ |
| struct prio_splay_tree_s t; |
| |
| /* If T above is NULL, only priority-0 items exist, so keep them |
| in a simple list. */ |
| struct priority_list l; |
| }; |
| |
| enum priority_insert_type { |
| /* Insert at the beginning of a priority list. */ |
| PRIORITY_INSERT_BEGIN, |
| /* Insert at the end of a priority list. */ |
| PRIORITY_INSERT_END |
| }; |
| |
| /* Used to determine in which queue a given priority node belongs in. |
| See pnode field of gomp_task. */ |
| |
| enum priority_queue_type |
| { |
| PQ_TEAM, /* Node belongs in gomp_team's task_queue. */ |
| PQ_CHILDREN, /* Node belongs in parent's children_queue. */ |
| PQ_TASKGROUP, /* Node belongs in taskgroup->taskgroup_queue. */ |
| PQ_IGNORED = 999 |
| }; |
| |
| typedef bool (*priority_queue_predicate) (struct gomp_task *); |
| |
| /* Priority queue implementation prototypes. */ |
| |
| extern bool priority_queue_task_in_queue_p (enum priority_queue_type, |
| struct priority_queue *, |
| struct gomp_task *); |
| extern void priority_queue_dump (enum priority_queue_type, |
| struct priority_queue *); |
| extern void priority_queue_verify (enum priority_queue_type, |
| struct priority_queue *, bool); |
| extern struct gomp_task *priority_queue_find (enum priority_queue_type, |
| struct priority_queue *, |
| priority_queue_predicate); |
| extern void priority_tree_remove (enum priority_queue_type, |
| struct priority_queue *, |
| struct priority_node *); |
| extern struct gomp_task *priority_tree_next_task (enum priority_queue_type, |
| struct priority_queue *, |
| enum priority_queue_type, |
| struct priority_queue *, |
| bool *); |
| |
| /* Return TRUE if there is more than one priority in HEAD. This is |
| used throughout to to choose between the fast path (priority 0 only |
| items) and a world with multiple priorities. */ |
| |
| static inline bool |
| priority_queue_multi_p (struct priority_queue *head) |
| { |
| return __builtin_expect (head->t.root != NULL, 0); |
| } |
| |
| /* Initialize a priority queue. */ |
| |
| static inline void |
| priority_queue_init (struct priority_queue *head) |
| { |
| head->t.root = NULL; |
| /* To save a few microseconds, we don't initialize head->l.priority |
| to 0 here. It is implied that priority will be 0 if head->t.root |
| == NULL. |
| |
| priority_tree_insert() will fix this when we encounter multiple |
| priorities. */ |
| head->l.tasks = NULL; |
| head->l.last_parent_depends_on = NULL; |
| } |
| |
| static inline void |
| priority_queue_free (struct priority_queue *head) |
| { |
| /* There's nothing to do, as tasks were freed as they were removed |
| in priority_queue_remove. */ |
| } |
| |
| /* Forward declarations. */ |
| static inline size_t priority_queue_offset (enum priority_queue_type); |
| static inline struct gomp_task *priority_node_to_task |
| (enum priority_queue_type, |
| struct priority_node *); |
| static inline struct priority_node *task_to_priority_node |
| (enum priority_queue_type, |
| struct gomp_task *); |
| |
| /* Return TRUE if priority queue HEAD is empty. |
| |
| MODEL IS MEMMODEL_ACQUIRE if we should use an acquire atomic to |
| read from the root of the queue, otherwise MEMMODEL_RELAXED if we |
| should use a plain load. */ |
| |
| static inline _Bool |
| priority_queue_empty_p (struct priority_queue *head, enum memmodel model) |
| { |
| /* Note: The acquire barriers on the loads here synchronize with |
| the write of a NULL in gomp_task_run_post_remove_parent. It is |
| not necessary that we synchronize with other non-NULL writes at |
| this point, but we must ensure that all writes to memory by a |
| child thread task work function are seen before we exit from |
| GOMP_taskwait. */ |
| if (priority_queue_multi_p (head)) |
| { |
| if (model == MEMMODEL_ACQUIRE) |
| return __atomic_load_n (&head->t.root, MEMMODEL_ACQUIRE) == NULL; |
| return head->t.root == NULL; |
| } |
| if (model == MEMMODEL_ACQUIRE) |
| return __atomic_load_n (&head->l.tasks, MEMMODEL_ACQUIRE) == NULL; |
| return head->l.tasks == NULL; |
| } |
| |
| /* Look for a given PRIORITY in HEAD. Return it if found, otherwise |
| return NULL. This only applies to the tree variant in HEAD. There |
| is no point in searching for priorities in HEAD->L. */ |
| |
| static inline struct priority_list * |
| priority_queue_lookup_priority (struct priority_queue *head, int priority) |
| { |
| if (head->t.root == NULL) |
| return NULL; |
| struct prio_splay_tree_key_s k; |
| k.l.priority = priority; |
| return (struct priority_list *) |
| prio_splay_tree_lookup (&head->t, &k); |
| } |
| |
| /* Insert task in DATA, with PRIORITY, in the priority list in LIST. |
| LIST contains items of type TYPE. |
| |
| If POS is PRIORITY_INSERT_BEGIN, the new task is inserted at the |
| top of its respective priority. If POS is PRIORITY_INSERT_END, the |
| task is inserted at the end of its priority. |
| |
| If ADJUST_PARENT_DEPENDS_ON is TRUE, LIST is a children queue, and |
| we must keep track of higher and lower priority WAITING tasks by |
| keeping the queue's last_parent_depends_on field accurate. This |
| only applies to the children queue, and the caller must ensure LIST |
| is a children queue in this case. |
| |
| If ADJUST_PARENT_DEPENDS_ON is TRUE, TASK_IS_PARENT_DEPENDS_ON is |
| set to the task's parent_depends_on field. If |
| ADJUST_PARENT_DEPENDS_ON is FALSE, this field is irrelevant. |
| |
| Return the new priority_node. */ |
| |
| static inline void |
| priority_list_insert (enum priority_queue_type type, |
| struct priority_list *list, |
| struct gomp_task *task, |
| int priority, |
| enum priority_insert_type pos, |
| bool adjust_parent_depends_on, |
| bool task_is_parent_depends_on) |
| { |
| struct priority_node *node = task_to_priority_node (type, task); |
| if (list->tasks) |
| { |
| /* If we are keeping track of higher/lower priority items, |
| but this is a lower priority WAITING task |
| (parent_depends_on != NULL), put it after all ready to |
| run tasks. See the comment in |
| priority_queue_upgrade_task for a visual on how tasks |
| should be organized. */ |
| if (adjust_parent_depends_on |
| && pos == PRIORITY_INSERT_BEGIN |
| && list->last_parent_depends_on |
| && !task_is_parent_depends_on) |
| { |
| struct priority_node *last_parent_depends_on |
| = list->last_parent_depends_on; |
| node->next = last_parent_depends_on->next; |
| node->prev = last_parent_depends_on; |
| } |
| /* Otherwise, put it at the top/bottom of the queue. */ |
| else |
| { |
| node->next = list->tasks; |
| node->prev = list->tasks->prev; |
| if (pos == PRIORITY_INSERT_BEGIN) |
| list->tasks = node; |
| } |
| node->next->prev = node; |
| node->prev->next = node; |
| } |
| else |
| { |
| node->next = node; |
| node->prev = node; |
| list->tasks = node; |
| } |
| if (adjust_parent_depends_on |
| && list->last_parent_depends_on == NULL |
| && task_is_parent_depends_on) |
| list->last_parent_depends_on = node; |
| } |
| |
| /* Tree version of priority_list_insert. */ |
| |
| static inline void |
| priority_tree_insert (enum priority_queue_type type, |
| struct priority_queue *head, |
| struct gomp_task *task, |
| int priority, |
| enum priority_insert_type pos, |
| bool adjust_parent_depends_on, |
| bool task_is_parent_depends_on) |
| { |
| if (__builtin_expect (head->t.root == NULL, 0)) |
| { |
| /* The first time around, transfer any priority 0 items to the |
| tree. */ |
| if (head->l.tasks != NULL) |
| { |
| prio_splay_tree_node k = gomp_malloc (sizeof (*k)); |
| k->left = NULL; |
| k->right = NULL; |
| k->key.l.priority = 0; |
| k->key.l.tasks = head->l.tasks; |
| k->key.l.last_parent_depends_on = head->l.last_parent_depends_on; |
| prio_splay_tree_insert (&head->t, k); |
| head->l.tasks = NULL; |
| } |
| } |
| struct priority_list *list |
| = priority_queue_lookup_priority (head, priority); |
| if (!list) |
| { |
| prio_splay_tree_node k = gomp_malloc (sizeof (*k)); |
| k->left = NULL; |
| k->right = NULL; |
| k->key.l.priority = priority; |
| k->key.l.tasks = NULL; |
| k->key.l.last_parent_depends_on = NULL; |
| prio_splay_tree_insert (&head->t, k); |
| list = &k->key.l; |
| } |
| priority_list_insert (type, list, task, priority, pos, |
| adjust_parent_depends_on, |
| task_is_parent_depends_on); |
| } |
| |
| /* Generic version of priority_*_insert. */ |
| |
| static inline void |
| priority_queue_insert (enum priority_queue_type type, |
| struct priority_queue *head, |
| struct gomp_task *task, |
| int priority, |
| enum priority_insert_type pos, |
| bool adjust_parent_depends_on, |
| bool task_is_parent_depends_on) |
| { |
| #if _LIBGOMP_CHECKING_ |
| if (priority_queue_task_in_queue_p (type, head, task)) |
| gomp_fatal ("Attempt to insert existing task %p", task); |
| #endif |
| if (priority_queue_multi_p (head) || __builtin_expect (priority > 0, 0)) |
| priority_tree_insert (type, head, task, priority, pos, |
| adjust_parent_depends_on, |
| task_is_parent_depends_on); |
| else |
| priority_list_insert (type, &head->l, task, priority, pos, |
| adjust_parent_depends_on, |
| task_is_parent_depends_on); |
| } |
| |
| /* If multiple priorities are in play, return the highest priority |
| task from within Q1 and Q2, while giving preference to tasks from |
| Q1. If the returned task is chosen from Q1, *Q1_CHOSEN_P is set to |
| TRUE, otherwise it is set to FALSE. |
| |
| If multiple priorities are not in play (only 0 priorities are |
| available), the next task is chosen exclusively from Q1. |
| |
| As a special case, Q2 can be NULL, in which case, we just choose |
| the highest priority WAITING task in Q1. This is an optimization |
| to speed up looking through only one queue. |
| |
| We assume Q1 has at least one item. */ |
| |
| static inline struct gomp_task * |
| priority_queue_next_task (enum priority_queue_type t1, |
| struct priority_queue *q1, |
| enum priority_queue_type t2, |
| struct priority_queue *q2, |
| bool *q1_chosen_p) |
| { |
| #if _LIBGOMP_CHECKING_ |
| if (priority_queue_empty_p (q1, MEMMODEL_RELAXED)) |
| gomp_fatal ("priority_queue_next_task: Q1 is empty"); |
| #endif |
| if (priority_queue_multi_p (q1)) |
| { |
| struct gomp_task *t |
| = priority_tree_next_task (t1, q1, t2, q2, q1_chosen_p); |
| /* If T is NULL, there are no WAITING tasks in Q1. In which |
| case, return any old (non-waiting) task which will cause the |
| caller to do the right thing when checking T->KIND == |
| GOMP_TASK_WAITING. */ |
| if (!t) |
| { |
| #if _LIBGOMP_CHECKING_ |
| if (*q1_chosen_p == false) |
| gomp_fatal ("priority_queue_next_task inconsistency"); |
| #endif |
| return priority_node_to_task (t1, q1->t.root->key.l.tasks); |
| } |
| return t; |
| } |
| else |
| { |
| *q1_chosen_p = true; |
| return priority_node_to_task (t1, q1->l.tasks); |
| } |
| } |
| |
| /* Remove NODE from LIST. |
| |
| If we are removing the one and only item in the list, and MODEL is |
| MEMMODEL_RELEASE, use an atomic release to clear the list. |
| |
| If the list becomes empty after the remove, return TRUE. */ |
| |
| static inline bool |
| priority_list_remove (struct priority_list *list, |
| struct priority_node *node, |
| enum memmodel model) |
| { |
| bool empty = false; |
| node->prev->next = node->next; |
| node->next->prev = node->prev; |
| if (list->tasks == node) |
| { |
| if (node->next != node) |
| list->tasks = node->next; |
| else |
| { |
| /* We access task->children in GOMP_taskwait outside of |
| the task lock mutex region, so need a release barrier |
| here to ensure memory written by child_task->fn above |
| is flushed before the NULL is written. */ |
| if (model == MEMMODEL_RELEASE) |
| __atomic_store_n (&list->tasks, NULL, MEMMODEL_RELEASE); |
| else |
| list->tasks = NULL; |
| empty = true; |
| goto remove_out; |
| } |
| } |
| remove_out: |
| #if _LIBGOMP_CHECKING_ |
| memset (node, 0xaf, sizeof (*node)); |
| #endif |
| return empty; |
| } |
| |
| /* This is the generic version of priority_list_remove. |
| |
| Remove NODE from priority queue HEAD. HEAD contains tasks of type TYPE. |
| |
| If we are removing the one and only item in the priority queue and |
| MODEL is MEMMODEL_RELEASE, use an atomic release to clear the queue. |
| |
| If the queue becomes empty after the remove, return TRUE. */ |
| |
| static inline bool |
| priority_queue_remove (enum priority_queue_type type, |
| struct priority_queue *head, |
| struct gomp_task *task, |
| enum memmodel model) |
| { |
| #if _LIBGOMP_CHECKING_ |
| if (!priority_queue_task_in_queue_p (type, head, task)) |
| gomp_fatal ("Attempt to remove missing task %p", task); |
| #endif |
| if (priority_queue_multi_p (head)) |
| { |
| priority_tree_remove (type, head, task_to_priority_node (type, task)); |
| if (head->t.root == NULL) |
| { |
| if (model == MEMMODEL_RELEASE) |
| /* Errr, we store NULL twice, the alternative would be to |
| use an atomic release directly in the splay tree |
| routines. Worth it? */ |
| __atomic_store_n (&head->t.root, NULL, MEMMODEL_RELEASE); |
| return true; |
| } |
| return false; |
| } |
| else |
| return priority_list_remove (&head->l, |
| task_to_priority_node (type, task), model); |
| } |
| |
| #endif /* _PRIORITY_QUEUE_H_ */ |