2019-10-25 Richard Biener <rguenther@suse.de>

	* tree-vect-loop.c (vectorizable_reduction): Verify
	STMT_VINFO_REDUC_IDX on the to be vectorized stmts is set up
	correctly.
	* tree-vect-patterns.c (vect_mark_pattern_stmts): Transfer
	STMT_VINFO_REDUC_IDX from the original stmts to the pattern
	stmts.


git-svn-id: https://gcc.gnu.org/svn/gcc/trunk@277437 138bc75d-0d04-0410-961f-82ee72b054a4

3 files changed, 74 insertions, 1 deletions

diff --git a/gcc/ChangeLog b/gcc/ChangeLog
index 4d9e4709da2..bb64e932ae8 100644
--- a/gcc/ChangeLog
+++ b/gcc/ChangeLog
@@ -1,3 +1,12 @@
+2019-10-25  Richard Biener  <rguenther@suse.de>
+
+	* tree-vect-loop.c (vectorizable_reduction): Verify
+	STMT_VINFO_REDUC_IDX on the to be vectorized stmts is set up
+	correctly.
+	* tree-vect-patterns.c (vect_mark_pattern_stmts): Transfer
+	STMT_VINFO_REDUC_IDX from the original stmts to the pattern
+	stmts.
+
 2019-10-24  Jakub Jelinek  <jakub@redhat.com>
 
 	* gimplify.h (omp_construct_selector_matches): Declare.
diff --git a/gcc/tree-vect-loop.c b/gcc/tree-vect-loop.c
index 9c5ce47249c..d0fd7bdbf80 100644
--- a/gcc/tree-vect-loop.c
+++ b/gcc/tree-vect-loop.c
@@ -5719,7 +5719,19 @@ vectorizable_reduction (stmt_vec_info stmt_info, slp_tree slp_node,
   /* PHIs should not participate in patterns.  */
   gcc_assert (!STMT_VINFO_RELATED_STMT (phi_info));
   gphi *reduc_def_phi = as_a <gphi *> (phi_info->stmt);
-  tree reduc_def = PHI_RESULT (reduc_def_phi);
+
+  /* Verify following REDUC_IDX from the latch def leads us back to the PHI.  */
+  tree reduc_def = PHI_ARG_DEF_FROM_EDGE (reduc_def_phi,
+					  loop_latch_edge (loop));
+  while (reduc_def != PHI_RESULT (reduc_def_phi))
+    {
+      stmt_vec_info def = loop_vinfo->lookup_def (reduc_def);
+      def = vect_stmt_to_vectorize (def);
+      gcc_assert (STMT_VINFO_REDUC_IDX (def) != -1);
+      reduc_def = gimple_op (def->stmt, 1 + STMT_VINFO_REDUC_IDX (def));
+    }
+
+  reduc_def = PHI_RESULT (reduc_def_phi);
   int reduc_index = -1;
   for (i = 0; i < op_type; i++)
     {
diff --git a/gcc/tree-vect-patterns.c b/gcc/tree-vect-patterns.c
index b497ec908d9..31e9e2a0859 100644
--- a/gcc/tree-vect-patterns.c
+++ b/gcc/tree-vect-patterns.c
@@ -5075,6 +5075,7 @@ static inline void
 vect_mark_pattern_stmts (stmt_vec_info orig_stmt_info, gimple *pattern_stmt,
                          tree pattern_vectype)
 {
+  stmt_vec_info orig_stmt_info_saved = orig_stmt_info;
   gimple *def_seq = STMT_VINFO_PATTERN_DEF_SEQ (orig_stmt_info);
 
   gimple *orig_pattern_stmt = NULL;
@@ -5134,6 +5135,57 @@ vect_mark_pattern_stmts (stmt_vec_info orig_stmt_info, gimple *pattern_stmt,
     }
   else
     vect_set_pattern_stmt (pattern_stmt, orig_stmt_info, pattern_vectype);
+
+  /* Transfer reduction path info to the pattern.  */
+  if (STMT_VINFO_REDUC_IDX (orig_stmt_info_saved) != -1)
+    {
+      vec_info *vinfo = orig_stmt_info_saved->vinfo;
+      tree lookfor = gimple_op (orig_stmt_info_saved->stmt,
+				1 + STMT_VINFO_REDUC_IDX (orig_stmt_info));
+      /* Search the pattern def sequence and the main pattern stmt.  Note
+         we may have inserted all into a containing pattern def sequence
+	 so the following is a bit awkward.  */
+      gimple_stmt_iterator si;
+      gimple *s;
+      if (def_seq)
+	{
+	  si = gsi_start (def_seq);
+	  s = gsi_stmt (si);
+	  gsi_next (&si);
+	}
+      else
+	{
+	  si = gsi_none ();
+	  s = pattern_stmt;
+	}
+      do
+	{
+	  bool found = false;
+	  for (unsigned i = 1; i < gimple_num_ops (s); ++i)
+	    if (gimple_op (s, i) == lookfor)
+	      {
+		STMT_VINFO_REDUC_IDX (vinfo->lookup_stmt (s)) = i - 1;
+		lookfor = gimple_get_lhs (s);
+		found = true;
+		break;
+	      }
+	  if (found && s == pattern_stmt)
+	    break;
+	  if (s == pattern_stmt)
+	    gcc_unreachable ();
+	  if (gsi_end_p (si))
+	    s = pattern_stmt;
+	  else
+	    {
+	      s = gsi_stmt (si);
+	      if (s == pattern_stmt)
+		/* Found the end inside a bigger pattern def seq.  */
+		si = gsi_none ();
+	      else
+		gsi_next (&si);
+	    }
+	} while (1);
+    }
 }
 
 /* Function vect_pattern_recog_1