libgomp/taskloop.c - gcc - Git at Google

 /* Copyright (C) 2015-2022 Free Software Foundation, Inc.
    Contributed by Jakub Jelinek <jakub@redhat.com>.

    This file is part of the GNU Offloading and Multi Processing Library
    (libgomp).

    Libgomp is free software; you can redistribute it and/or modify it
    under the terms of the GNU General Public License as published by
    the Free Software Foundation; either version 3, or (at your option)
    any later version.

    Libgomp is distributed in the hope that it will be useful, but WITHOUT ANY
    WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
    FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
    more details.

    Under Section 7 of GPL version 3, you are granted additional
    permissions described in the GCC Runtime Library Exception, version
    3.1, as published by the Free Software Foundation.

    You should have received a copy of the GNU General Public License and
    a copy of the GCC Runtime Library Exception along with this program;
    see the files COPYING3 and COPYING.RUNTIME respectively.  If not, see
    <http://www.gnu.org/licenses/>.  */

 /* This file handles the taskloop construct.  It is included twice, once
    for the long and once for unsigned long long variant.  */

 /* Called when encountering an explicit task directive.  If IF_CLAUSE is
    false, then we must not delay in executing the task.  If UNTIED is true,
    then the task may be executed by any member of the team.  */

 void
 GOMP_taskloop (void (*fn) (void *), void *data, void (*cpyfn) (void *, void *),
 	       long arg_size, long arg_align, unsigned flags,
 	       unsigned long num_tasks, int priority,
 	       TYPE start, TYPE end, TYPE step)
 {
   struct gomp_thread *thr = gomp_thread ();
   struct gomp_team *team = thr->ts.team;

 #ifdef HAVE_BROKEN_POSIX_SEMAPHORES
   /* If pthread_mutex_* is used for omp_*lock*, then each task must be
      tied to one thread all the time.  This means UNTIED tasks must be
      tied and if CPYFN is non-NULL IF(0) must be forced, as CPYFN
      might be running on different thread than FN.  */
   if (cpyfn)
     flags &= ~GOMP_TASK_FLAG_IF;
   flags &= ~GOMP_TASK_FLAG_UNTIED;
 #endif

   /* If parallel or taskgroup has been cancelled, don't start new tasks.  */
   if (team && gomp_team_barrier_cancelled (&team->barrier))
     {
     early_return:
       if ((flags & (GOMP_TASK_FLAG_NOGROUP | GOMP_TASK_FLAG_REDUCTION))
 	  == GOMP_TASK_FLAG_REDUCTION)
 	{
 	  struct gomp_data_head { TYPE t1, t2; uintptr_t *ptr; };
 	  uintptr_t *ptr = ((struct gomp_data_head *) data)->ptr;
 	  /* Tell callers GOMP_taskgroup_reduction_register has not been
 	     called.  */
 	  ptr[2] = 0;
 	}
       return;
     }

 #ifdef TYPE_is_long
   TYPE s = step;
   if (step > 0)
     {
       if (start >= end)
 	goto early_return;
       s--;
     }
   else
     {
       if (start <= end)
 	goto early_return;
       s++;
     }
   UTYPE n = (end - start + s) / step;
 #else
   UTYPE n;
   if (flags & GOMP_TASK_FLAG_UP)
     {
       if (start >= end)
 	goto early_return;
       n = (end - start + step - 1) / step;
     }
   else
     {
       if (start <= end)
 	goto early_return;
       n = (start - end - step - 1) / -step;
     }
 #endif

   TYPE task_step = step;
   TYPE nfirst_task_step = step;
   unsigned long nfirst = n;
   if (flags & GOMP_TASK_FLAG_GRAINSIZE)
     {
       unsigned long grainsize = num_tasks;
 #ifdef TYPE_is_long
       num_tasks = n / grainsize;
 #else
       UTYPE ndiv = n / grainsize;
       num_tasks = ndiv;
       if (num_tasks != ndiv)
 	num_tasks = ~0UL;
 #endif
       if ((flags & GOMP_TASK_FLAG_STRICT)
 	  && num_tasks != ~0ULL)
 	{
 	  UTYPE mod = n % grainsize;
 	  task_step = (TYPE) grainsize * step;
 	  if (mod)
 	    {
 	      num_tasks++;
 	      nfirst_task_step = (TYPE) mod * step;
 	      if (num_tasks == 1)
 		task_step = nfirst_task_step;
 	      else
 		nfirst = num_tasks - 2;
 	    }
 	}
       else if (num_tasks <= 1)
 	{
 	  num_tasks = 1;
 	  task_step = end - start;
 	}
       else if (num_tasks >= grainsize
 #ifndef TYPE_is_long
 	       && num_tasks != ~0UL
 #endif
 	      )
 	{
 	  UTYPE mul = num_tasks * grainsize;
 	  task_step = (TYPE) grainsize * step;
 	  if (mul != n)
 	    {
 	      nfirst_task_step = task_step;
 	      task_step += step;
 	      nfirst = n - mul - 1;
 	    }
 	}
       else
 	{
 	  UTYPE div = n / num_tasks;
 	  UTYPE mod = n % num_tasks;
 	  task_step = (TYPE) div * step;
 	  if (mod)
 	    {
 	      nfirst_task_step = task_step;
 	      task_step += step;
 	      nfirst = mod - 1;
 	    }
 	}
     }
   else
     {
       if (num_tasks == 0)
 	num_tasks = team ? team->nthreads : 1;
       if (num_tasks >= n)
 	num_tasks = n;
       else
 	{
 	  UTYPE div = n / num_tasks;
 	  UTYPE mod = n % num_tasks;
 	  task_step = (TYPE) div * step;
 	  if (mod)
 	    {
 	      nfirst_task_step = task_step;
 	      task_step += step;
 	      nfirst = mod - 1;
 	    }
 	}
     }

   if (flags & GOMP_TASK_FLAG_NOGROUP)
     {
       if (__builtin_expect (gomp_cancel_var, 0)
 	  && thr->task
 	  && thr->task->taskgroup)
 	{
 	  if (thr->task->taskgroup->cancelled)
 	    return;
 	  if (thr->task->taskgroup->workshare
 	      && thr->task->taskgroup->prev
 	      && thr->task->taskgroup->prev->cancelled)
 	    return;
 	}
     }
   else
     {
       ialias_call (GOMP_taskgroup_start) ();
       if (flags & GOMP_TASK_FLAG_REDUCTION)
 	{
 	  struct gomp_data_head { TYPE t1, t2; uintptr_t *ptr; };
 	  uintptr_t *ptr = ((struct gomp_data_head *) data)->ptr;
 	  ialias_call (GOMP_taskgroup_reduction_register) (ptr);
 	}
     }

   if (priority > gomp_max_task_priority_var)
     priority = gomp_max_task_priority_var;

   if ((flags & GOMP_TASK_FLAG_IF) == 0 || team == NULL
       || (thr->task && thr->task->final_task)
       || team->task_count + num_tasks > 64 * team->nthreads)
     {
       unsigned long i;
       if (__builtin_expect (cpyfn != NULL, 0))
 	{
 	  struct gomp_task task[num_tasks];
 	  struct gomp_task *parent = thr->task;
 	  arg_size = (arg_size + arg_align - 1) & ~(arg_align - 1);
 	  char buf[num_tasks * arg_size + arg_align - 1];
 	  char *arg = (char *) (((uintptr_t) buf + arg_align - 1)
 				& ~(uintptr_t) (arg_align - 1));
 	  char *orig_arg = arg;
 	  for (i = 0; i < num_tasks; i++)
 	    {
 	      gomp_init_task (&task[i], parent, gomp_icv (false));
 	      task[i].priority = priority;
 	      task[i].kind = GOMP_TASK_UNDEFERRED;
 	      task[i].final_task = (thr->task && thr->task->final_task)
 				   || (flags & GOMP_TASK_FLAG_FINAL);
 	      if (thr->task)
 		{
 		  task[i].in_tied_task = thr->task->in_tied_task;
 		  task[i].taskgroup = thr->task->taskgroup;
 		}
 	      thr->task = &task[i];
 	      cpyfn (arg, data);
 	      arg += arg_size;
 	    }
 	  arg = orig_arg;
 	  for (i = 0; i < num_tasks; i++)
 	    {
 	      thr->task = &task[i];
 	      ((TYPE *)arg)[0] = start;
 	      start += task_step;
 	      ((TYPE *)arg)[1] = start;
 	      if (i == nfirst)
 		task_step = nfirst_task_step;
 	      fn (arg);
 	      arg += arg_size;
 	      if (!priority_queue_empty_p (&task[i].children_queue,
 					   MEMMODEL_RELAXED))
 		{
 		  gomp_mutex_lock (&team->task_lock);
 		  gomp_clear_parent (&task[i].children_queue);
 		  gomp_mutex_unlock (&team->task_lock);
 		}
 	      gomp_end_task ();
 	    }
 	}
       else
 	for (i = 0; i < num_tasks; i++)
 	  {
 	    struct gomp_task task;

 	    gomp_init_task (&task, thr->task, gomp_icv (false));
 	    task.priority = priority;
 	    task.kind = GOMP_TASK_UNDEFERRED;
 	    task.final_task = (thr->task && thr->task->final_task)
 			      || (flags & GOMP_TASK_FLAG_FINAL);
 	    if (thr->task)
 	      {
 		task.in_tied_task = thr->task->in_tied_task;
 		task.taskgroup = thr->task->taskgroup;
 	      }
 	    thr->task = &task;
 	    ((TYPE *)data)[0] = start;
 	    start += task_step;
 	    ((TYPE *)data)[1] = start;
 	    if (i == nfirst)
 	      task_step = nfirst_task_step;
 	    fn (data);
 	    if (!priority_queue_empty_p (&task.children_queue,
 					 MEMMODEL_RELAXED))
 	      {
 		gomp_mutex_lock (&team->task_lock);
 		gomp_clear_parent (&task.children_queue);
 		gomp_mutex_unlock (&team->task_lock);
 	      }
 	    gomp_end_task ();
 	  }
     }
   else
     {
       struct gomp_task *tasks[num_tasks];
       struct gomp_task *parent = thr->task;
       struct gomp_taskgroup *taskgroup = parent->taskgroup;
       char *arg;
       int do_wake;
       unsigned long i;

       for (i = 0; i < num_tasks; i++)
 	{
 	  struct gomp_task *task
 	    = gomp_malloc (sizeof (*task) + arg_size + arg_align - 1);
 	  tasks[i] = task;
 	  arg = (char *) (((uintptr_t) (task + 1) + arg_align - 1)
 			  & ~(uintptr_t) (arg_align - 1));
 	  gomp_init_task (task, parent, gomp_icv (false));
 	  task->priority = priority;
 	  task->kind = GOMP_TASK_UNDEFERRED;
 	  task->in_tied_task = parent->in_tied_task;
 	  task->taskgroup = taskgroup;
 	  thr->task = task;
 	  if (cpyfn)
 	    {
 	      cpyfn (arg, data);
 	      task->copy_ctors_done = true;
 	    }
 	  else
 	    memcpy (arg, data, arg_size);
 	  ((TYPE *)arg)[0] = start;
 	  start += task_step;
 	  ((TYPE *)arg)[1] = start;
 	  if (i == nfirst)
 	    task_step = nfirst_task_step;
 	  thr->task = parent;
 	  task->kind = GOMP_TASK_WAITING;
 	  task->fn = fn;
 	  task->fn_data = arg;
 	  task->final_task = (flags & GOMP_TASK_FLAG_FINAL) >> 1;
 	}
       gomp_mutex_lock (&team->task_lock);
       /* If parallel or taskgroup has been cancelled, don't start new
 	 tasks.  */
       if (__builtin_expect (gomp_cancel_var, 0)
 	  && cpyfn == NULL)
 	{
 	  if (gomp_team_barrier_cancelled (&team->barrier))
 	    {
 	    do_cancel:
 	      gomp_mutex_unlock (&team->task_lock);
 	      for (i = 0; i < num_tasks; i++)
 		{
 		  gomp_finish_task (tasks[i]);
 		  free (tasks[i]);
 		}
 	      if ((flags & GOMP_TASK_FLAG_NOGROUP) == 0)
 		ialias_call (GOMP_taskgroup_end) ();
 	      return;
 	    }
 	  if (taskgroup)
 	    {
 	      if (taskgroup->cancelled)
 		goto do_cancel;
 	      if (taskgroup->workshare
 		  && taskgroup->prev
 		  && taskgroup->prev->cancelled)
 		goto do_cancel;
 	    }
 	}
       if (taskgroup)
 	taskgroup->num_children += num_tasks;
       for (i = 0; i < num_tasks; i++)
 	{
 	  struct gomp_task *task = tasks[i];
 	  priority_queue_insert (PQ_CHILDREN, &parent->children_queue,
 				 task, priority,
 				 PRIORITY_INSERT_BEGIN,
 				 /*last_parent_depends_on=*/false,
 				 task->parent_depends_on);
 	  if (taskgroup)
 	    priority_queue_insert (PQ_TASKGROUP, &taskgroup->taskgroup_queue,
 				   task, priority, PRIORITY_INSERT_BEGIN,
 				   /*last_parent_depends_on=*/false,
 				   task->parent_depends_on);
 	  priority_queue_insert (PQ_TEAM, &team->task_queue, task, priority,
 				 PRIORITY_INSERT_END,
 				 /*last_parent_depends_on=*/false,
 				 task->parent_depends_on);
 	  ++team->task_count;
 	  ++team->task_queued_count;
 	}
       gomp_team_barrier_set_task_pending (&team->barrier);
       if (team->task_running_count + !parent->in_tied_task
 	  < team->nthreads)
 	{
 	  do_wake = team->nthreads - team->task_running_count
 		    - !parent->in_tied_task;
 	  if ((unsigned long) do_wake > num_tasks)
 	    do_wake = num_tasks;
 	}
       else
 	do_wake = 0;
       gomp_mutex_unlock (&team->task_lock);
       if (do_wake)
 	gomp_team_barrier_wake (&team->barrier, do_wake);
     }
   if ((flags & GOMP_TASK_FLAG_NOGROUP) == 0)
     ialias_call (GOMP_taskgroup_end) ();
 }
	/* Copyright (C) 2015-2022 Free Software Foundation, Inc.
	Contributed by Jakub Jelinek <jakub@redhat.com>.

	This file is part of the GNU Offloading and Multi Processing Library
	(libgomp).

	Libgomp is free software; you can redistribute it and/or modify it
	under the terms of the GNU General Public License as published by
	the Free Software Foundation; either version 3, or (at your option)
	any later version.

	Libgomp is distributed in the hope that it will be useful, but WITHOUT ANY
	WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
	FOR A PARTICULAR PURPOSE. See the GNU General Public License for
	more details.

	Under Section 7 of GPL version 3, you are granted additional
	permissions described in the GCC Runtime Library Exception, version
	3.1, as published by the Free Software Foundation.

	You should have received a copy of the GNU General Public License and
	a copy of the GCC Runtime Library Exception along with this program;
	see the files COPYING3 and COPYING.RUNTIME respectively. If not, see
	<http://www.gnu.org/licenses/>. */

	/* This file handles the taskloop construct. It is included twice, once
	for the long and once for unsigned long long variant. */

	/* Called when encountering an explicit task directive. If IF_CLAUSE is
	false, then we must not delay in executing the task. If UNTIED is true,
	then the task may be executed by any member of the team. */

	void
	GOMP_taskloop (void (fn) (void ), void data, void (cpyfn) (void , void ),
	long arg_size, long arg_align, unsigned flags,
	unsigned long num_tasks, int priority,
	TYPE start, TYPE end, TYPE step)
	{
	struct gomp_thread *thr = gomp_thread ();
	struct gomp_team *team = thr->ts.team;

	#ifdef HAVE_BROKEN_POSIX_SEMAPHORES
	/* If pthread_mutex_* is used for omp_lock, then each task must be
	tied to one thread all the time. This means UNTIED tasks must be
	tied and if CPYFN is non-NULL IF(0) must be forced, as CPYFN
	might be running on different thread than FN. */
	if (cpyfn)
	flags &= ~GOMP_TASK_FLAG_IF;
	flags &= ~GOMP_TASK_FLAG_UNTIED;
	#endif

	/* If parallel or taskgroup has been cancelled, don't start new tasks. */
	if (team && gomp_team_barrier_cancelled (&team->barrier))
	{
	early_return:
	if ((flags & (GOMP_TASK_FLAG_NOGROUP \| GOMP_TASK_FLAG_REDUCTION))
	== GOMP_TASK_FLAG_REDUCTION)
	{
	struct gomp_data_head { TYPE t1, t2; uintptr_t *ptr; };
	uintptr_t ptr = ((struct gomp_data_head ) data)->ptr;
	/* Tell callers GOMP_taskgroup_reduction_register has not been
	called. */
	ptr[2] = 0;
	}
	return;
	}

	#ifdef TYPE_is_long
	TYPE s = step;
	if (step > 0)
	{
	if (start >= end)
	goto early_return;
	s--;
	}
	else
	{
	if (start <= end)
	goto early_return;
	s++;
	}
	UTYPE n = (end - start + s) / step;
	#else
	UTYPE n;
	if (flags & GOMP_TASK_FLAG_UP)
	{
	if (start >= end)
	goto early_return;
	n = (end - start + step - 1) / step;
	}
	else
	{
	if (start <= end)
	goto early_return;
	n = (start - end - step - 1) / -step;
	}
	#endif

	TYPE task_step = step;
	TYPE nfirst_task_step = step;
	unsigned long nfirst = n;
	if (flags & GOMP_TASK_FLAG_GRAINSIZE)
	{
	unsigned long grainsize = num_tasks;
	#ifdef TYPE_is_long
	num_tasks = n / grainsize;
	#else
	UTYPE ndiv = n / grainsize;
	num_tasks = ndiv;
	if (num_tasks != ndiv)
	num_tasks = ~0UL;
	#endif
	if ((flags & GOMP_TASK_FLAG_STRICT)
	&& num_tasks != ~0ULL)
	{
	UTYPE mod = n % grainsize;
	task_step = (TYPE) grainsize * step;
	if (mod)
	{
	num_tasks++;
	nfirst_task_step = (TYPE) mod * step;
	if (num_tasks == 1)
	task_step = nfirst_task_step;
	else
	nfirst = num_tasks - 2;
	}
	}
	else if (num_tasks <= 1)
	{
	num_tasks = 1;
	task_step = end - start;
	}
	else if (num_tasks >= grainsize
	#ifndef TYPE_is_long
	&& num_tasks != ~0UL
	#endif
	)
	{
	UTYPE mul = num_tasks * grainsize;
	task_step = (TYPE) grainsize * step;
	if (mul != n)
	{
	nfirst_task_step = task_step;
	task_step += step;
	nfirst = n - mul - 1;
	}
	}
	else
	{
	UTYPE div = n / num_tasks;
	UTYPE mod = n % num_tasks;
	task_step = (TYPE) div * step;
	if (mod)
	{
	nfirst_task_step = task_step;
	task_step += step;
	nfirst = mod - 1;
	}
	}
	}
	else
	{
	if (num_tasks == 0)
	num_tasks = team ? team->nthreads : 1;
	if (num_tasks >= n)
	num_tasks = n;
	else
	{
	UTYPE div = n / num_tasks;
	UTYPE mod = n % num_tasks;
	task_step = (TYPE) div * step;
	if (mod)
	{
	nfirst_task_step = task_step;
	task_step += step;
	nfirst = mod - 1;
	}
	}
	}

	if (flags & GOMP_TASK_FLAG_NOGROUP)
	{
	if (__builtin_expect (gomp_cancel_var, 0)
	&& thr->task
	&& thr->task->taskgroup)
	{
	if (thr->task->taskgroup->cancelled)
	return;
	if (thr->task->taskgroup->workshare
	&& thr->task->taskgroup->prev
	&& thr->task->taskgroup->prev->cancelled)
	return;
	}
	}
	else
	{
	ialias_call (GOMP_taskgroup_start) ();
	if (flags & GOMP_TASK_FLAG_REDUCTION)
	{
	struct gomp_data_head { TYPE t1, t2; uintptr_t *ptr; };
	uintptr_t ptr = ((struct gomp_data_head ) data)->ptr;
	ialias_call (GOMP_taskgroup_reduction_register) (ptr);
	}
	}

	if (priority > gomp_max_task_priority_var)
	priority = gomp_max_task_priority_var;

	if ((flags & GOMP_TASK_FLAG_IF) == 0 \|\| team == NULL
	\|\| (thr->task && thr->task->final_task)
	\|\| team->task_count + num_tasks > 64 * team->nthreads)
	{
	unsigned long i;
	if (__builtin_expect (cpyfn != NULL, 0))
	{
	struct gomp_task task[num_tasks];
	struct gomp_task *parent = thr->task;
	arg_size = (arg_size + arg_align - 1) & ~(arg_align - 1);
	char buf[num_tasks * arg_size + arg_align - 1];
	char arg = (char ) (((uintptr_t) buf + arg_align - 1)
	& ~(uintptr_t) (arg_align - 1));
	char *orig_arg = arg;
	for (i = 0; i < num_tasks; i++)
	{
	gomp_init_task (&task[i], parent, gomp_icv (false));
	task[i].priority = priority;
	task[i].kind = GOMP_TASK_UNDEFERRED;
	task[i].final_task = (thr->task && thr->task->final_task)
	\|\| (flags & GOMP_TASK_FLAG_FINAL);
	if (thr->task)
	{
	task[i].in_tied_task = thr->task->in_tied_task;
	task[i].taskgroup = thr->task->taskgroup;
	}
	thr->task = &task[i];
	cpyfn (arg, data);
	arg += arg_size;
	}
	arg = orig_arg;
	for (i = 0; i < num_tasks; i++)
	{
	thr->task = &task[i];
	((TYPE *)arg)[0] = start;
	start += task_step;
	((TYPE *)arg)[1] = start;
	if (i == nfirst)
	task_step = nfirst_task_step;
	fn (arg);
	arg += arg_size;
	if (!priority_queue_empty_p (&task[i].children_queue,
	MEMMODEL_RELAXED))
	{
	gomp_mutex_lock (&team->task_lock);
	gomp_clear_parent (&task[i].children_queue);
	gomp_mutex_unlock (&team->task_lock);
	}
	gomp_end_task ();
	}
	}
	else
	for (i = 0; i < num_tasks; i++)
	{
	struct gomp_task task;

	gomp_init_task (&task, thr->task, gomp_icv (false));
	task.priority = priority;
	task.kind = GOMP_TASK_UNDEFERRED;
	task.final_task = (thr->task && thr->task->final_task)
	\|\| (flags & GOMP_TASK_FLAG_FINAL);
	if (thr->task)
	{
	task.in_tied_task = thr->task->in_tied_task;
	task.taskgroup = thr->task->taskgroup;
	}
	thr->task = &task;
	((TYPE *)data)[0] = start;
	start += task_step;
	((TYPE *)data)[1] = start;
	if (i == nfirst)
	task_step = nfirst_task_step;
	fn (data);
	if (!priority_queue_empty_p (&task.children_queue,
	MEMMODEL_RELAXED))
	{
	gomp_mutex_lock (&team->task_lock);
	gomp_clear_parent (&task.children_queue);
	gomp_mutex_unlock (&team->task_lock);
	}
	gomp_end_task ();
	}
	}
	else
	{
	struct gomp_task *tasks[num_tasks];
	struct gomp_task *parent = thr->task;
	struct gomp_taskgroup *taskgroup = parent->taskgroup;
	char *arg;
	int do_wake;
	unsigned long i;

	for (i = 0; i < num_tasks; i++)
	{
	struct gomp_task *task
	= gomp_malloc (sizeof (*task) + arg_size + arg_align - 1);
	tasks[i] = task;
	arg = (char *) (((uintptr_t) (task + 1) + arg_align - 1)
	& ~(uintptr_t) (arg_align - 1));
	gomp_init_task (task, parent, gomp_icv (false));
	task->priority = priority;
	task->kind = GOMP_TASK_UNDEFERRED;
	task->in_tied_task = parent->in_tied_task;
	task->taskgroup = taskgroup;
	thr->task = task;
	if (cpyfn)
	{
	cpyfn (arg, data);
	task->copy_ctors_done = true;
	}
	else
	memcpy (arg, data, arg_size);
	((TYPE *)arg)[0] = start;
	start += task_step;
	((TYPE *)arg)[1] = start;
	if (i == nfirst)
	task_step = nfirst_task_step;
	thr->task = parent;
	task->kind = GOMP_TASK_WAITING;
	task->fn = fn;
	task->fn_data = arg;
	task->final_task = (flags & GOMP_TASK_FLAG_FINAL) >> 1;
	}
	gomp_mutex_lock (&team->task_lock);
	/* If parallel or taskgroup has been cancelled, don't start new
	tasks. */
	if (__builtin_expect (gomp_cancel_var, 0)
	&& cpyfn == NULL)
	{
	if (gomp_team_barrier_cancelled (&team->barrier))
	{
	do_cancel:
	gomp_mutex_unlock (&team->task_lock);
	for (i = 0; i < num_tasks; i++)
	{
	gomp_finish_task (tasks[i]);
	free (tasks[i]);
	}
	if ((flags & GOMP_TASK_FLAG_NOGROUP) == 0)
	ialias_call (GOMP_taskgroup_end) ();
	return;
	}
	if (taskgroup)
	{
	if (taskgroup->cancelled)
	goto do_cancel;
	if (taskgroup->workshare
	&& taskgroup->prev
	&& taskgroup->prev->cancelled)
	goto do_cancel;
	}
	}
	if (taskgroup)
	taskgroup->num_children += num_tasks;
	for (i = 0; i < num_tasks; i++)
	{
	struct gomp_task *task = tasks[i];
	priority_queue_insert (PQ_CHILDREN, &parent->children_queue,
	task, priority,
	PRIORITY_INSERT_BEGIN,
	/last_parent_depends_on=/false,
	task->parent_depends_on);
	if (taskgroup)
	priority_queue_insert (PQ_TASKGROUP, &taskgroup->taskgroup_queue,
	task, priority, PRIORITY_INSERT_BEGIN,
	/last_parent_depends_on=/false,
	task->parent_depends_on);
	priority_queue_insert (PQ_TEAM, &team->task_queue, task, priority,
	PRIORITY_INSERT_END,
	/last_parent_depends_on=/false,
	task->parent_depends_on);
	++team->task_count;
	++team->task_queued_count;
	}
	gomp_team_barrier_set_task_pending (&team->barrier);
	if (team->task_running_count + !parent->in_tied_task
	< team->nthreads)
	{
	do_wake = team->nthreads - team->task_running_count
	- !parent->in_tied_task;
	if ((unsigned long) do_wake > num_tasks)
	do_wake = num_tasks;
	}
	else
	do_wake = 0;
	gomp_mutex_unlock (&team->task_lock);
	if (do_wake)
	gomp_team_barrier_wake (&team->barrier, do_wake);
	}
	if ((flags & GOMP_TASK_FLAG_NOGROUP) == 0)
	ialias_call (GOMP_taskgroup_end) ();
	}