2012-07-13 Bill Schmidt <wschmidt@linux.ibm.com>

	* doc/tm.texi: Regenerate.
	* doc/tm.texi.in (TARGET_VECTORIZE_INIT_COST): New hook.
	(TARGET_VECTORIZE_ADD_STMT_COST): Likewise.
	(TARGET_VECTORIZE_FINISH_COST): Likewise.
	(TARGET_VECTORIZE_DESTROY_COST_DATA): Likewise.
	* targhooks.c (default_init_cost): New function.
	(default_add_stmt_cost): Likewise.
	(default_finish_cost): Likewise.
	(default_destroy_cost_data): Likewise.
	* targhooks.h (default_init_cost): New decl.
	(default_add_stmt_cost): Likewise.
	(default_finish_cost): Likewise.
	(default_destroy_cost_data): Likewise.
	* target.def (init_cost): New DEFHOOK.
	(add_stmt_cost): Likewise.
	(finish_cost): Likewise.
	(destroy_cost_data): Likewise.
	* target.h (struct _stmt_vec_info): New extern decl.
	(stmt_vectype): Likewise.
	(stmt_in_inner_loop_p): Likewise.
	* tree-vectorizer.h (stmt_info_for_cost): New struct/typedef.
	(stmt_vector_for_cost): New VEC/typedef.
	(add_stmt_info_to_vec): New function.
	(struct _slp_tree): Remove cost.inside_of_loop field.
	(struct _slp_instance): Remove cost.inside_of_loop field; add
	stmt_cost_vec field.
	(SLP_INSTANCE_INSIDE_OF_LOOP_COST): Remove macro.
	(SLP_INSTANCE_STMT_COST_VEC): New accessor macro.
	(SLP_TREE_INSIDE_OF_LOOP_COST): Remove macro.
	(struct _vect_peel_extended_info): Add stmt_cost_vec field.
	(struct _loop_vec_info): Add target_cost_data field.
	(LOOP_VINFO_TARGET_COST_DATA): New accessor macro.
	(struct _bb_vec_info): Add target_cost_data field.
	(BB_VINFO_TARGET_COST_DATA): New accessor macro.
	(struct _stmt_vec_info): Remove cost.inside_of_loop field.
	(STMT_VINFO_INSIDE_OF_LOOP_COST): Remove macro.
	(stmt_vinfo_set_inside_of_loop_cost): Remove function.
	(init_cost): New function.
	(add_stmt_cost): Likewise.
	(finish_cost): Likewise.
	(destroy_cost_data): Likewise.
	(vect_model_simple_cost): Change parameter list.
	(vect_model_store_cost): Likewise.
	(vect_model_load_cost): Likewise.
	(record_stmt_cost): New extern decl.
	(vect_get_load_cost): Change parameter list.
	(vect_get_store_cost): Likewise.
	* tree-vect-loop.c (new_loop_vec_info): Call init_cost.
	(destroy_loop_vec_info): Call destroy_cost_data.
	(vect_estimate_min_profitable_iters): Remove old calculation of
	inside costs; call finish_cost instead.
	(vect_model_reduction_cost): Call add_stmt_cost instead of old
	inside-costs calculation.
	(vect_model_induction_cost): Likewise.
	* tree-vect-data-refs.c (vect_get_data_access_cost): Change to
	return a stmt_vector_for_cost; modify calls to vect_get_load_cost
	and vect_get_store_cost to obtain the value to return.
	(vect_peeling_hash_get_lowest_cost): Obtain a stmt_cost_vec from
	vect_get_data_access_cost and store it in the minimum peeling
	structure.
	(vect_peeling_hash_choose_best_peeling): Change the parameter list
	to add a (stmt_vector_for_cost *) output parameter, and set its value.
	(vect_enhance_data_refs_alignment): Ignore the new return value from
	calls to vect_get_data_access_cost; obtain stmt_cost_vec from
	vect_peeling_hash_choose_best_peeling and pass its contents to the
	target cost model.
	* tree-vect-stmts.c (stmt_vectype): New function.
	(stmt_in_inner_loop_p): Likewise.
	(record_stmt_cost): Likewise.
	(vect_model_simple_cost): Add stmt_cost_vec parameter; call
	record_stmt_cost instead of old calculation; don't call
	stmt_vinfo_set_inside_of_loop_cost.
	(vect_model_promotion_demotion_cost): Call add_stmt_cost instead of
	old calculation; don't call stmt_vinfo_set_inside_of_loop_cost.
	(vect_model_store_cost): Add stmt_cost_vec parameter; call
	record_stmt_cost instead of old calculation; add stmt_cost_vec
	parameter to vect_get_store_cost call; don't call
	stmt_vinfo_set_inside_of_loop_cost.
	(vect_get_store_cost): Add stmt_cost_vec parameter; call
	record_stmt_cost instead of old calculation.
	(vect_model_load_cost): Add stmt_cost_vec parameter; call
	record_stmt_cost instead of old calculation; add stmt_cost_vec
	parameter to vect_get_load_cost call; don't call
	stmt_vinfo_set_inside_of_loop_cost.
	(vect_get_load_cost): Add stmt_cost_vec parameter; call
	record_stmt_cost instead of old calculation.
	(vectorizable_call): Add NULL parameter to vect_model_simple_cost call.
	(vectorizable_conversion): Likewise.
	(vectorizable_assignment): Likewise.
	(vectorizable_shift): Likewise.
	(vectorizable_operation): Likewise.
	(vectorizable_store): Add NULL parameter to vect_model_store_cost call.
	(vectorizable_load): Add NULL parameter to vect_model_load_cost call.
	(new_stmt_vec_info): Don't set STMT_VINFO_INSIDE_OF_LOOP_COST.
	* config/spu/spu.c (TARGET_VECTORIZE_INIT_COST): New macro def.
	(TARGET_VECTORIZE_ADD_STMT_COST): Likewise.
	(TARGET_VECTORIZE_FINISH_COST): Likewise.
	(TARGET_VECTORIZE_DESTROY_COST_DATA): Likewise.
	(spu_init_cost): New function.
	(spu_add_stmt_cost): Likewise.
	(spu_finish_cost): Likewise.
	(spu_destroy_cost_data): Likewise.
	* config/i386/i386.c (ix86_init_cost): New function.
	(ix86_add_stmt_cost): Likewise.
	(ix86_finish_cost): Likewise.
	(ix86_destroy_cost_data): Likewise.
	(TARGET_VECTORIZE_INIT_COST): New macro def.
	(TARGET_VECTORIZE_ADD_STMT_COST): Likewise.
	(TARGET_VECTORIZE_FINISH_COST): Likewise.
	(TARGET_VECTORIZE_DESTROY_COST_DATA): Likewise.
	* config/rs6000/rs6000.c (TARGET_VECTORIZE_INIT_COST): New macro def.
	(TARGET_VECTORIZE_ADD_STMT_COST): Likewise.
	(TARGET_VECTORIZE_FINISH_COST): Likewise.
	(TARGET_VECTORIZE_DESTROY_COST_DATA): Likewise.
	(rs6000_init_cost): New function.
	(rs6000_add_stmt_cost): Likewise.
	(rs6000_finish_cost): Likewise.
	(rs6000_destroy_cost_data): Likewise.
	* tree-vect-slp.c (vect_free_slp_instance): Free stmt_cost_vec.
	(vect_create_new_slp_node): Don't set SLP_TREE_INSIDE_OF_LOOP_COST.
	(vect_get_and_check_slp_defs): Add stmt_cost_vec parameter; add
	stmt_cost_vec parameter to vect_model_store_cost and
	vect_model_simple_cost calls.
	(vect_build_slp_tree): Remove inside_cost parameter; add stmt_cost_vec
	parameter; add stmt_cost_vec parameter to vect_get_and_check_slp_defs,
	vect_model_load_cost, and recursive vect_build_slp_tree calls; prevent
	calculating cost more than once for loads; remove inside_cost
	parameter from recursive vect_build_slp_tree calls; call
	record_stmt_cost instead of old calculation.
	(vect_analyze_slp_instance): Allocate stmt_cost_vec and save it with
	the instance; free it on premature exit; remove inside_cost parameter
	from vect_build_slp_tree call; add stmt_cost_vec parameter to
	vect_build_slp_tree call; don't set SLP_INSTANCE_INSIDE_OF_LOOP_COST.
	(new_bb_vec_info): Call init_cost.
	(destroy_bb_vec_info): Call destroy_cost_data.
	(vect_bb_vectorization_profitable_p): Call add_stmt_cost for each
	statement recorded with an SLP instance; call finish_cost instead of
	the old calculation.
	(vect_update_slp_costs_according_to_vf): Record statement costs from
	SLP instances, multiplying by the appropriate number of copies; don't
	update SLP_INSTANCE_INSIDE_OF_LOOP_COST.


git-svn-id: https://gcc.gnu.org/svn/gcc/trunk@189460 138bc75d-0d04-0410-961f-82ee72b054a4

1 files changed, 133 insertions, 58 deletions

diff --git a/gcc/tree-vect-stmts.c b/gcc/tree-vect-stmts.c
index fe6a997a1aa..19e36e01322 100644
--- a/gcc/tree-vect-stmts.c
+++ b/gcc/tree-vect-stmts.c
@@ -41,6 +41,66 @@ along with GCC; see the file COPYING3.  If not see
 #include "langhooks.h"
 
 
+/* Return the vectorized type for the given statement.  */
+
+tree
+stmt_vectype (struct _stmt_vec_info *stmt_info)
+{
+  return STMT_VINFO_VECTYPE (stmt_info);
+}
+
+/* Return TRUE iff the given statement is in an inner loop relative to
+   the loop being vectorized.  */
+bool
+stmt_in_inner_loop_p (struct _stmt_vec_info *stmt_info)
+{
+  gimple stmt = STMT_VINFO_STMT (stmt_info);
+  basic_block bb = gimple_bb (stmt);
+  loop_vec_info loop_vinfo = STMT_VINFO_LOOP_VINFO (stmt_info);
+  struct loop* loop;
+
+  if (!loop_vinfo)
+    return false;
+
+  loop = LOOP_VINFO_LOOP (loop_vinfo);
+
+  return (bb->loop_father == loop->inner);
+}
+
+/* Record the cost of a statement, either by directly informing the 
+   target model or by saving it in a vector for later processing.
+   Return a preliminary estimate of the statement's cost.  */
+
+unsigned
+record_stmt_cost (stmt_vector_for_cost *stmt_cost_vec, int count,
+		  enum vect_cost_for_stmt kind, stmt_vec_info stmt_info,
+		  int misalign)
+{
+  if (stmt_cost_vec)
+    {
+      tree vectype = stmt_vectype (stmt_info);
+      add_stmt_info_to_vec (stmt_cost_vec, count, kind,
+			    STMT_VINFO_STMT (stmt_info), misalign);
+      return (unsigned)
+	(targetm.vectorize.builtin_vectorization_cost (kind, vectype, misalign)
+	 * count);
+	 
+    }
+  else
+    {
+      loop_vec_info loop_vinfo = STMT_VINFO_LOOP_VINFO (stmt_info);
+      bb_vec_info bb_vinfo = STMT_VINFO_BB_VINFO (stmt_info);
+      void *target_cost_data;
+
+      if (loop_vinfo)
+	target_cost_data = LOOP_VINFO_TARGET_COST_DATA (loop_vinfo);
+      else
+	target_cost_data = BB_VINFO_TARGET_COST_DATA (bb_vinfo);
+
+      return add_stmt_cost (target_cost_data, count, kind, stmt_info, misalign);
+    }
+}
+
 /* Return a variable of type ELEM_TYPE[NELEMS].  */
 
 static tree
@@ -735,7 +795,8 @@ vect_mark_stmts_to_be_vectorized (loop_vec_info loop_vinfo)
 
 void
 vect_model_simple_cost (stmt_vec_info stmt_info, int ncopies,
-			enum vect_def_type *dt, slp_tree slp_node)
+			enum vect_def_type *dt, slp_tree slp_node,
+			stmt_vector_for_cost *stmt_cost_vec)
 {
   int i;
   int inside_cost = 0, outside_cost = 0;
@@ -744,8 +805,6 @@ vect_model_simple_cost (stmt_vec_info stmt_info, int ncopies,
   if (PURE_SLP_STMT (stmt_info))
     return;
 
-  inside_cost = ncopies * vect_get_stmt_cost (vector_stmt); 
-
   /* FORNOW: Assuming maximum 2 args per stmts.  */
   for (i = 0; i < 2; i++)
     {
@@ -753,13 +812,16 @@ vect_model_simple_cost (stmt_vec_info stmt_info, int ncopies,
 	outside_cost += vect_get_stmt_cost (vector_stmt); 
     }
 
+  /* Set the costs either in STMT_INFO or SLP_NODE (if exists).  */
+  stmt_vinfo_set_outside_of_loop_cost (stmt_info, slp_node, outside_cost);
+
+  /* Pass the inside-of-loop statements to the target-specific cost model.  */
+  inside_cost = record_stmt_cost (stmt_cost_vec, ncopies, vector_stmt,
+				  stmt_info, 0);
+
   if (vect_print_dump_info (REPORT_COST))
     fprintf (vect_dump, "vect_model_simple_cost: inside_cost = %d, "
              "outside_cost = %d .", inside_cost, outside_cost);
-
-  /* Set the costs either in STMT_INFO or SLP_NODE (if exists).  */
-  stmt_vinfo_set_inside_of_loop_cost (stmt_info, slp_node, inside_cost);
-  stmt_vinfo_set_outside_of_loop_cost (stmt_info, slp_node, outside_cost);
 }
 
 
@@ -773,18 +835,26 @@ vect_model_promotion_demotion_cost (stmt_vec_info stmt_info,
 				    enum vect_def_type *dt, int pwr)
 {
   int i, tmp;
-  int inside_cost = 0, outside_cost = 0, single_stmt_cost;
+  int inside_cost = 0, outside_cost = 0;
+  loop_vec_info loop_vinfo = STMT_VINFO_LOOP_VINFO (stmt_info);
+  bb_vec_info bb_vinfo = STMT_VINFO_BB_VINFO (stmt_info);
+  void *target_cost_data;
 
   /* The SLP costs were already calculated during SLP tree build.  */
   if (PURE_SLP_STMT (stmt_info))
     return;
 
-  single_stmt_cost = vect_get_stmt_cost (vec_promote_demote);
+  if (loop_vinfo)
+    target_cost_data = LOOP_VINFO_TARGET_COST_DATA (loop_vinfo);
+  else
+    target_cost_data = BB_VINFO_TARGET_COST_DATA (bb_vinfo);
+
   for (i = 0; i < pwr + 1; i++)
     {
       tmp = (STMT_VINFO_TYPE (stmt_info) == type_promotion_vec_info_type) ?
 	(i + 1) : i;
-      inside_cost += vect_pow2 (tmp) * single_stmt_cost;
+      inside_cost += add_stmt_cost (target_cost_data, vect_pow2 (tmp),
+				    vec_promote_demote, stmt_info, 0);
     }
 
   /* FORNOW: Assuming maximum 2 args per stmts.  */
@@ -799,7 +869,6 @@ vect_model_promotion_demotion_cost (stmt_vec_info stmt_info,
              "outside_cost = %d .", inside_cost, outside_cost);
 
   /* Set the costs in STMT_INFO.  */
-  stmt_vinfo_set_inside_of_loop_cost (stmt_info, NULL, inside_cost);
   stmt_vinfo_set_outside_of_loop_cost (stmt_info, NULL, outside_cost);
 }
 
@@ -829,7 +898,7 @@ vect_cost_group_size (stmt_vec_info stmt_info)
 void
 vect_model_store_cost (stmt_vec_info stmt_info, int ncopies,
 		       bool store_lanes_p, enum vect_def_type dt,
-		       slp_tree slp_node)
+		       slp_tree slp_node, stmt_vector_for_cost *stmt_cost_vec)
 {
   int group_size;
   unsigned int inside_cost = 0, outside_cost = 0;
@@ -873,8 +942,10 @@ vect_model_store_cost (stmt_vec_info stmt_info, int ncopies,
   if (!store_lanes_p && group_size > 1)
     {
       /* Uses a high and low interleave operation for each needed permute.  */
-      inside_cost = ncopies * exact_log2(group_size) * group_size
-        * vect_get_stmt_cost (vec_perm);
+      
+      int nstmts = ncopies * exact_log2 (group_size) * group_size;
+      inside_cost = record_stmt_cost (stmt_cost_vec, nstmts, vec_perm,
+				      stmt_info, 0);
 
       if (vect_print_dump_info (REPORT_COST))
         fprintf (vect_dump, "vect_model_store_cost: strided group_size = %d .",
@@ -882,14 +953,13 @@ vect_model_store_cost (stmt_vec_info stmt_info, int ncopies,
     }
 
   /* Costs of the stores.  */
-  vect_get_store_cost (first_dr, ncopies, &inside_cost);
+  vect_get_store_cost (first_dr, ncopies, &inside_cost, stmt_cost_vec);
 
   if (vect_print_dump_info (REPORT_COST))
     fprintf (vect_dump, "vect_model_store_cost: inside_cost = %d, "
              "outside_cost = %d .", inside_cost, outside_cost);
 
   /* Set the costs either in STMT_INFO or SLP_NODE (if exists).  */
-  stmt_vinfo_set_inside_of_loop_cost (stmt_info, slp_node, inside_cost);
   stmt_vinfo_set_outside_of_loop_cost (stmt_info, slp_node, outside_cost);
 }
 
@@ -897,15 +967,19 @@ vect_model_store_cost (stmt_vec_info stmt_info, int ncopies,
 /* Calculate cost of DR's memory access.  */
 void
 vect_get_store_cost (struct data_reference *dr, int ncopies,
-                     unsigned int *inside_cost)
+		     unsigned int *inside_cost,
+		     stmt_vector_for_cost *stmt_cost_vec)
 {
   int alignment_support_scheme = vect_supportable_dr_alignment (dr, false);
+  gimple stmt = DR_STMT (dr);
+  stmt_vec_info stmt_info = vinfo_for_stmt (stmt);
 
   switch (alignment_support_scheme)
     {
     case dr_aligned:
       {
-        *inside_cost += ncopies * vect_get_stmt_cost (vector_store);
+	*inside_cost += record_stmt_cost (stmt_cost_vec, ncopies,
+					  vector_store, stmt_info, 0);
 
         if (vect_print_dump_info (REPORT_COST))
           fprintf (vect_dump, "vect_model_store_cost: aligned.");
@@ -915,14 +989,10 @@ vect_get_store_cost (struct data_reference *dr, int ncopies,
 
     case dr_unaligned_supported:
       {
-        gimple stmt = DR_STMT (dr);
-        stmt_vec_info stmt_info = vinfo_for_stmt (stmt);
-        tree vectype = STMT_VINFO_VECTYPE (stmt_info);
-
         /* Here, we assign an additional cost for the unaligned store.  */
-        *inside_cost += ncopies
-          * targetm.vectorize.builtin_vectorization_cost (unaligned_store,
-                                 vectype, DR_MISALIGNMENT (dr));
+	*inside_cost += record_stmt_cost (stmt_cost_vec, ncopies,
+					  unaligned_store, stmt_info,
+					  DR_MISALIGNMENT (dr));
 
         if (vect_print_dump_info (REPORT_COST))
           fprintf (vect_dump, "vect_model_store_cost: unaligned supported by "
@@ -956,7 +1026,7 @@ vect_get_store_cost (struct data_reference *dr, int ncopies,
 
 void
 vect_model_load_cost (stmt_vec_info stmt_info, int ncopies, bool load_lanes_p,
-		      slp_tree slp_node)
+		      slp_tree slp_node, stmt_vector_for_cost *stmt_cost_vec)
 {
   int group_size;
   gimple first_stmt;
@@ -988,8 +1058,9 @@ vect_model_load_cost (stmt_vec_info stmt_info, int ncopies, bool load_lanes_p,
   if (!load_lanes_p && group_size > 1)
     {
       /* Uses an even and odd extract operations for each needed permute.  */
-      inside_cost = ncopies * exact_log2(group_size) * group_size
-	* vect_get_stmt_cost (vec_perm);
+      int nstmts = ncopies * exact_log2 (group_size) * group_size;
+      inside_cost += record_stmt_cost (stmt_cost_vec, nstmts, vec_perm,
+				       stmt_info, 0);
 
       if (vect_print_dump_info (REPORT_COST))
         fprintf (vect_dump, "vect_model_load_cost: strided group_size = %d .",
@@ -1001,24 +1072,23 @@ vect_model_load_cost (stmt_vec_info stmt_info, int ncopies, bool load_lanes_p,
     {
       /* N scalar loads plus gathering them into a vector.  */
       tree vectype = STMT_VINFO_VECTYPE (stmt_info);
-      inside_cost += (vect_get_stmt_cost (scalar_load) * ncopies
-		      * TYPE_VECTOR_SUBPARTS (vectype));
-      inside_cost += ncopies
-	* targetm.vectorize.builtin_vectorization_cost (vec_construct,
-							vectype, 0);
+      inside_cost += record_stmt_cost (stmt_cost_vec,
+				       ncopies * TYPE_VECTOR_SUBPARTS (vectype),
+				       scalar_load, stmt_info, 0);
+      inside_cost += record_stmt_cost (stmt_cost_vec, ncopies, vec_construct,
+				       stmt_info, 0);
     }
   else
     vect_get_load_cost (first_dr, ncopies,
 			((!STMT_VINFO_GROUPED_ACCESS (stmt_info))
 			 || group_size > 1 || slp_node),
-			&inside_cost, &outside_cost);
+			&inside_cost, &outside_cost, stmt_cost_vec);
 
   if (vect_print_dump_info (REPORT_COST))
     fprintf (vect_dump, "vect_model_load_cost: inside_cost = %d, "
              "outside_cost = %d .", inside_cost, outside_cost);
 
   /* Set the costs either in STMT_INFO or SLP_NODE (if exists).  */
-  stmt_vinfo_set_inside_of_loop_cost (stmt_info, slp_node, inside_cost);
   stmt_vinfo_set_outside_of_loop_cost (stmt_info, slp_node, outside_cost);
 }
 
@@ -1026,16 +1096,20 @@ vect_model_load_cost (stmt_vec_info stmt_info, int ncopies, bool load_lanes_p,
 /* Calculate cost of DR's memory access.  */
 void
 vect_get_load_cost (struct data_reference *dr, int ncopies,
-                    bool add_realign_cost, unsigned int *inside_cost,
-                    unsigned int *outside_cost)
+		    bool add_realign_cost, unsigned int *inside_cost,
+		    unsigned int *outside_cost,
+		    stmt_vector_for_cost *stmt_cost_vec)
 {
   int alignment_support_scheme = vect_supportable_dr_alignment (dr, false);
+  gimple stmt = DR_STMT (dr);
+  stmt_vec_info stmt_info = vinfo_for_stmt (stmt);
 
   switch (alignment_support_scheme)
     {
     case dr_aligned:
       {
-        *inside_cost += ncopies * vect_get_stmt_cost (vector_load); 
+	*inside_cost += record_stmt_cost (stmt_cost_vec, ncopies,
+					  vector_load, stmt_info, 0);
 
         if (vect_print_dump_info (REPORT_COST))
           fprintf (vect_dump, "vect_model_load_cost: aligned.");
@@ -1044,14 +1118,11 @@ vect_get_load_cost (struct data_reference *dr, int ncopies,
       }
     case dr_unaligned_supported:
       {
-        gimple stmt = DR_STMT (dr);
-        stmt_vec_info stmt_info = vinfo_for_stmt (stmt);
-        tree vectype = STMT_VINFO_VECTYPE (stmt_info);
-
         /* Here, we assign an additional cost for the unaligned load.  */
-        *inside_cost += ncopies
-          * targetm.vectorize.builtin_vectorization_cost (unaligned_load,
-                                           vectype, DR_MISALIGNMENT (dr));
+	*inside_cost += record_stmt_cost (stmt_cost_vec, ncopies,
+					  unaligned_load, stmt_info,
+					  DR_MISALIGNMENT (dr));
+
         if (vect_print_dump_info (REPORT_COST))
           fprintf (vect_dump, "vect_model_load_cost: unaligned supported by "
                    "hardware.");
@@ -1060,14 +1131,17 @@ vect_get_load_cost (struct data_reference *dr, int ncopies,
       }
     case dr_explicit_realign:
       {
-        *inside_cost += ncopies * (2 * vect_get_stmt_cost (vector_load)
-				   + vect_get_stmt_cost (vec_perm));
+	*inside_cost += record_stmt_cost (stmt_cost_vec, ncopies * 2,
+					  vector_load, stmt_info, 0);
+	*inside_cost += record_stmt_cost (stmt_cost_vec, ncopies,
+					  vec_perm, stmt_info, 0);
 
         /* FIXME: If the misalignment remains fixed across the iterations of
            the containing loop, the following cost should be added to the
            outside costs.  */
         if (targetm.vectorize.builtin_mask_for_load)
-          *inside_cost += vect_get_stmt_cost (vector_stmt);
+	  *inside_cost += record_stmt_cost (stmt_cost_vec, 1, vector_stmt,
+					    stmt_info, 0);
 
         if (vect_print_dump_info (REPORT_COST))
           fprintf (vect_dump, "vect_model_load_cost: explicit realign");
@@ -1094,8 +1168,10 @@ vect_get_load_cost (struct data_reference *dr, int ncopies,
               *outside_cost += vect_get_stmt_cost (vector_stmt);
           }
 
-        *inside_cost += ncopies * (vect_get_stmt_cost (vector_load)
-				   + vect_get_stmt_cost (vec_perm));
+	*inside_cost += record_stmt_cost (stmt_cost_vec, ncopies,
+					  vector_load, stmt_info, 0);
+	*inside_cost += record_stmt_cost (stmt_cost_vec, ncopies,
+					  vec_perm, stmt_info, 0);
 
         if (vect_print_dump_info (REPORT_COST))
           fprintf (vect_dump,
@@ -1719,7 +1795,7 @@ vectorizable_call (gimple stmt, gimple_stmt_iterator *gsi, gimple *vec_stmt,
       STMT_VINFO_TYPE (stmt_info) = call_vec_info_type;
       if (vect_print_dump_info (REPORT_DETAILS))
         fprintf (vect_dump, "=== vectorizable_call ===");
-      vect_model_simple_cost (stmt_info, ncopies, dt, NULL);
+      vect_model_simple_cost (stmt_info, ncopies, dt, NULL, NULL);
       return true;
     }
 
@@ -2433,7 +2509,7 @@ vectorizable_conversion (gimple stmt, gimple_stmt_iterator *gsi,
       if (code == FIX_TRUNC_EXPR || code == FLOAT_EXPR)
         {
 	  STMT_VINFO_TYPE (stmt_info) = type_conversion_vec_info_type;
-	  vect_model_simple_cost (stmt_info, ncopies, dt, NULL);
+	  vect_model_simple_cost (stmt_info, ncopies, dt, NULL, NULL);
 	}
       else if (modifier == NARROW)
 	{
@@ -2841,7 +2917,7 @@ vectorizable_assignment (gimple stmt, gimple_stmt_iterator *gsi,
       STMT_VINFO_TYPE (stmt_info) = assignment_vec_info_type;
       if (vect_print_dump_info (REPORT_DETAILS))
         fprintf (vect_dump, "=== vectorizable_assignment ===");
-      vect_model_simple_cost (stmt_info, ncopies, dt, NULL);
+      vect_model_simple_cost (stmt_info, ncopies, dt, NULL, NULL);
       return true;
     }
 
@@ -3187,7 +3263,7 @@ vectorizable_shift (gimple stmt, gimple_stmt_iterator *gsi,
       STMT_VINFO_TYPE (stmt_info) = shift_vec_info_type;
       if (vect_print_dump_info (REPORT_DETAILS))
         fprintf (vect_dump, "=== vectorizable_shift ===");
-      vect_model_simple_cost (stmt_info, ncopies, dt, NULL);
+      vect_model_simple_cost (stmt_info, ncopies, dt, NULL, NULL);
       return true;
     }
 
@@ -3501,7 +3577,7 @@ vectorizable_operation (gimple stmt, gimple_stmt_iterator *gsi,
       STMT_VINFO_TYPE (stmt_info) = op_vec_info_type;
       if (vect_print_dump_info (REPORT_DETAILS))
         fprintf (vect_dump, "=== vectorizable_operation ===");
-      vect_model_simple_cost (stmt_info, ncopies, dt, NULL);
+      vect_model_simple_cost (stmt_info, ncopies, dt, NULL, NULL);
       return true;
     }
 
@@ -3805,7 +3881,7 @@ vectorizable_store (gimple stmt, gimple_stmt_iterator *gsi, gimple *vec_stmt,
   if (!vec_stmt) /* transformation not required.  */
     {
       STMT_VINFO_TYPE (stmt_info) = store_vec_info_type;
-      vect_model_store_cost (stmt_info, ncopies, store_lanes_p, dt, NULL);
+      vect_model_store_cost (stmt_info, ncopies, store_lanes_p, dt, NULL, NULL);
       return true;
     }
 
@@ -4361,7 +4437,7 @@ vectorizable_load (gimple stmt, gimple_stmt_iterator *gsi, gimple *vec_stmt,
   if (!vec_stmt) /* transformation not required.  */
     {
       STMT_VINFO_TYPE (stmt_info) = load_vec_info_type;
-      vect_model_load_cost (stmt_info, ncopies, load_lanes_p, NULL);
+      vect_model_load_cost (stmt_info, ncopies, load_lanes_p, NULL, NULL);
       return true;
     }
 
@@ -5801,7 +5877,6 @@ new_stmt_vec_info (gimple stmt, loop_vec_info loop_vinfo,
     STMT_VINFO_DEF_TYPE (res) = vect_internal_def;
 
   STMT_VINFO_SAME_ALIGN_REFS (res) = VEC_alloc (dr_p, heap, 5);
-  STMT_VINFO_INSIDE_OF_LOOP_COST (res) = 0;
   STMT_VINFO_OUTSIDE_OF_LOOP_COST (res) = 0;
   STMT_SLP_TYPE (res) = loop_vect;
   GROUP_FIRST_ELEMENT (res) = NULL;