[OpenMP] Fix for nested proc_bind affinity bug

Using proc_bind clause on a nested #pragma omp parallel region
with KMP_AFFINITY set causes an assertion error. This assertion occurs because
the place-partition-var is not properly initialized in the nested master threads.
Trying to get an intuitive result with KMP_AFFINITY + proc_bind is difficult
because of how the KMP_AFFINITY gtid-to-place mapping occurs. This
patch creates an initial place list no matter what affinity mechanism is used.
For KMP_AFFINITY, the place-partition-var is initialized to all the places.

Differential Revision: https://reviews.llvm.org/D55795

3 files changed, 44 insertions, 13 deletions

diff --git a/openmp/runtime/src/kmp_affinity.cpp b/openmp/runtime/src/kmp_affinity.cpp
index 775862ebb7e..f14cdf68c08 100644
--- a/openmp/runtime/src/kmp_affinity.cpp
+++ b/openmp/runtime/src/kmp_affinity.cpp
@@ -4505,6 +4505,7 @@ static void __kmp_aux_affinity_initialize(void) {
         KMP_WARNING(AffNoValidProcID);
       }
       __kmp_affinity_type = affinity_none;
+      __kmp_create_affinity_none_places();
       return;
     }
     break;
@@ -4557,11 +4558,9 @@ static void __kmp_aux_affinity_initialize(void) {
         KMP_WARNING(AffBalancedNotAvail, "KMP_AFFINITY");
       }
       __kmp_affinity_type = affinity_none;
+      __kmp_create_affinity_none_places();
       return;
-    } else if (__kmp_affinity_uniform_topology()) {
-      break;
-    } else { // Non-uniform topology
-
+    } else if (!__kmp_affinity_uniform_topology()) {
       // Save the depth for further usage
       __kmp_aff_depth = depth;
 
@@ -4602,8 +4601,9 @@ static void __kmp_aux_affinity_initialize(void) {
 
         procarr[core * maxprocpercore + inlastcore] = proc;
       }
-
-      break;
+    }
+    if (__kmp_affinity_compact >= depth) {
+      __kmp_affinity_compact = depth - 1;
     }
 
   sortAddresses:
@@ -4781,6 +4781,11 @@ void __kmp_affinity_set_init_mask(int gtid, int isa_root) {
     th->th.th_new_place = i;
     th->th.th_first_place = 0;
     th->th.th_last_place = __kmp_affinity_num_masks - 1;
+  } else if (KMP_AFFINITY_NON_PROC_BIND) {
+    // When using a Non-OMP_PROC_BIND affinity method,
+    // set all threads' place-partition-var to the entire place list
+    th->th.th_first_place = 0;
+    th->th.th_last_place = __kmp_affinity_num_masks - 1;
   }
 
   if (i == KMP_PLACE_ALL) {
diff --git a/openmp/runtime/src/kmp_ftn_entry.h b/openmp/runtime/src/kmp_ftn_entry.h
index c14f73cb4ae..c08e31b034d 100644
--- a/openmp/runtime/src/kmp_ftn_entry.h
+++ b/openmp/runtime/src/kmp_ftn_entry.h
@@ -858,9 +858,6 @@ int FTN_STDCALL KMP_EXPAND_NAME(FTN_GET_PARTITION_NUM_PLACES)(void) {
   }
   if (!KMP_AFFINITY_CAPABLE())
     return 0;
-  if (KMP_AFFINITY_NON_PROC_BIND) {
-    return 1;
-  }
   gtid = __kmp_entry_gtid();
   thread = __kmp_thread_from_gtid(gtid);
   first_place = thread->th.th_first_place;
@@ -889,10 +886,6 @@ void
     return;
   gtid = __kmp_entry_gtid();
   thread = __kmp_thread_from_gtid(gtid);
-  if (KMP_AFFINITY_NON_PROC_BIND) {
-    place_nums[0] = thread->th.th_current_place;
-    return;
-  }
   first_place = thread->th.th_first_place;
   last_place = thread->th.th_last_place;
   if (first_place < 0 || last_place < 0)
diff --git a/openmp/runtime/test/affinity/bug-nested.c b/openmp/runtime/test/affinity/bug-nested.c
new file mode 100644
index 00000000000..a81b5f36292
--- /dev/null
+++ b/openmp/runtime/test/affinity/bug-nested.c
@@ -0,0 +1,33 @@
+// RUN: %libomp-compile && env KMP_AFFINITY=compact %libomp-run
+// REQUIRES: openmp-4.0
+
+#include <stdio.h>
+#include <stdint.h>
+#include <omp.h>
+#include "omp_testsuite.h"
+
+int test_nested_affinity_bug() {
+  int a = 0;
+  omp_set_nested(1);
+  #pragma omp parallel num_threads(2) shared(a)
+  {
+    #pragma omp parallel num_threads(2) shared(a) proc_bind(close)
+    {
+      #pragma omp atomic
+      a++;
+    }
+  }
+  return 1;
+}
+
+int main() {
+  int i;
+  int num_failed = 0;
+
+  for (i = 0; i < REPETITIONS; i++) {
+    if (!test_nested_affinity_bug()) {
+      num_failed++;
+    }
+  }
+  return num_failed;
+}