diff options
Diffstat (limited to 'gcc/tree-vectorizer.c')
| -rw-r--r-- | gcc/tree-vectorizer.c | 283 |
1 files changed, 263 insertions, 20 deletions
diff --git a/gcc/tree-vectorizer.c b/gcc/tree-vectorizer.c index 1b01ac2a110..0c7586bf501 100644 --- a/gcc/tree-vectorizer.c +++ b/gcc/tree-vectorizer.c @@ -209,20 +209,32 @@ static bool vectorizable_operation (tree, block_stmt_iterator *, tree *); static bool vectorizable_assignment (tree, block_stmt_iterator *, tree *); static enum dr_alignment_support vect_supportable_dr_alignment (struct data_reference *); +/* APPLE LOCAL begin AV cond expr. -dpatel */ +/* Patch is waiting FSF review since mid Sep, 2004. + New function, vectoriable_select. */ +static bool vectorizable_select (tree, block_stmt_iterator *, tree *); +/* APPLE LOCAL end AV cond expr. -dpatel */ static void vect_align_data_ref (tree); static void vect_enhance_data_refs_alignment (loop_vec_info); /* Utility functions for the analyses. */ static bool vect_is_simple_use (tree , loop_vec_info, tree *); +/* APPLE LOCAL begin AV cond expr. -dpatel */ +/* Patch is waiting FSF review since mid Sep, 2004. + New function, vect_is_simple_cond. */ +static bool vect_is_simple_cond (tree, loop_vec_info); +/* APPLE LOCAL end AV cond expr. -dpatel */ static bool exist_non_indexing_operands_for_use_p (tree, tree); static bool vect_is_simple_iv_evolution (unsigned, tree, tree *, tree *); static void vect_mark_relevant (varray_type *, tree); static bool vect_stmt_relevant_p (tree, loop_vec_info); -static tree vect_get_loop_niters (struct loop *, tree *); +/* APPLE LOCAL loops-to-memset */ +extern tree vect_get_loop_niters (struct loop *, tree *); static bool vect_compute_data_ref_alignment (struct data_reference *); static bool vect_analyze_data_ref_access (struct data_reference *); static bool vect_can_force_dr_alignment_p (tree, unsigned int); -static struct data_reference * vect_analyze_pointer_ref_access +/* APPLE LOCAL loops-to-memset */ +extern struct data_reference * vect_analyze_pointer_ref_access (tree, tree, bool, tree, tree *, tree *); static bool vect_can_advance_ivs_p (loop_vec_info); static tree vect_get_ptr_offset (tree, tree, tree *); @@ -286,6 +298,11 @@ static void vect_set_dump_settings (void); void vect_set_verbosity_level (const char *); +/* APPLE LOCAL begin AV data dependence. -dpatel */ +/* Patch is waiting FSF review since mid Sep, 2004. + New variable, loops_num. */ +static unsigned int loops_num; +/* APPLE LOCAL end AV data dependence. -dpatel */ /************************************************************************* Simple Loop Peeling Utilities @@ -293,6 +310,13 @@ void vect_set_verbosity_level (const char *); Utilities to support loop peeling for vectorization purposes. *************************************************************************/ +/* APPLE LOCAL begin AV data dependence. -dpatel */ +/* Utilities for dependence analysis. */ +/* Patch is waiting FSF review since mid Sep, 2004. + New function, vect_build_dist_vector. */ +static unsigned int vect_build_dist_vector (struct loop *, + struct data_dependence_relation *); +/* APPLE LOCAL end AV data dependence. -dpatel */ /* For each definition in DEFINITIONS this function allocates new ssa name. */ @@ -2793,6 +2817,103 @@ vect_supportable_dr_alignment (struct data_reference *dr) } +/* APPLE LOCAL begin AV cond expr. -dpatel */ +/* Patch is waiting FSF review since mid Sep, 2004. */ +/* vectorizable_select. + + Check if STMT is conditional modify expression that can be vectorized. + If VEC_STMT is also passed, vectorize the STMT: create a vectorized + stmt using VEC_COND_EXPR to replace it, put it in VEC_STMT, and insert it + at BSI. + + Return FALSE if not a vectorizable STMT, TRUE otherwise. */ + +static bool +vectorizable_select (tree stmt, block_stmt_iterator *bsi, tree *vec_stmt) +{ + tree scalar_dest = NULL_TREE; + tree vec_dest = NULL_TREE; + tree op = NULL_TREE; + tree cond_expr, then_clause, else_clause; + stmt_vec_info stmt_info = vinfo_for_stmt (stmt); + tree vectype = STMT_VINFO_VECTYPE (stmt_info); + tree vec_cond_lhs, vec_cond_rhs, vec_then_clause, vec_else_clause; + tree vec_compare, vec_cond_expr; + tree new_temp; + loop_vec_info loop_vinfo = STMT_VINFO_LOOP_VINFO (stmt_info); + enum machine_mode vec_mode; + + if (TREE_CODE (stmt) != MODIFY_EXPR) + return false; + + op = TREE_OPERAND (stmt, 1); + + if (TREE_CODE (op) != COND_EXPR) + return false; + + cond_expr = TREE_OPERAND (op, 0); + then_clause = TREE_OPERAND (op, 1); + else_clause = TREE_OPERAND (op, 2); + + if (!vect_is_simple_cond (cond_expr, loop_vinfo)) + return false; + + if (TREE_CODE (then_clause) == SSA_NAME) + { + tree then_def_stmt = SSA_NAME_DEF_STMT (then_clause); + if (!vect_is_simple_use (then_clause, loop_vinfo, &then_def_stmt)) + return false; + } + else if (TREE_CODE (then_clause) != INTEGER_CST + && TREE_CODE (then_clause) != REAL_CST) + return false; + + if (TREE_CODE (else_clause) == SSA_NAME) + { + tree else_def_stmt = SSA_NAME_DEF_STMT (else_clause); + if (!vect_is_simple_use (else_clause, loop_vinfo, &else_def_stmt)) + return false; + } + else if (TREE_CODE (else_clause) != INTEGER_CST + && TREE_CODE (else_clause) != REAL_CST) + return false; + + + vec_mode = TYPE_MODE (vectype); + + if (!vec_stmt) + { + STMT_VINFO_TYPE (stmt_info) = select_vec_info_type; + return expand_vec_cond_expr_p (op, vec_mode); + } + + /* Transform */ + + /* Handle def. */ + scalar_dest = TREE_OPERAND (stmt, 0); + vec_dest = vect_create_destination_var (scalar_dest, vectype); + + /* Handle cond expr. */ + vec_cond_lhs = vect_get_vec_def_for_operand (TREE_OPERAND (cond_expr, 0), stmt); + vec_cond_rhs = vect_get_vec_def_for_operand (TREE_OPERAND (cond_expr, 1), stmt); + vec_then_clause = vect_get_vec_def_for_operand (then_clause, stmt); + vec_else_clause = vect_get_vec_def_for_operand (else_clause, stmt); + + /* Arguments are ready. create the new vector stmt. */ + vec_compare = build2 (TREE_CODE (cond_expr), vectype, + vec_cond_lhs, vec_cond_rhs); + vec_cond_expr = build (VEC_COND_EXPR, vectype, + vec_compare, vec_then_clause, vec_else_clause); + + *vec_stmt = build2 (MODIFY_EXPR, vectype, vec_dest, vec_cond_expr); + new_temp = make_ssa_name (vec_dest, *vec_stmt); + TREE_OPERAND (*vec_stmt, 0) = new_temp; + vect_finish_stmt_generation (stmt, *vec_stmt, bsi); + + return true; +} +/* APPLE LOCAL end AV cond expr. -dpatel */ + /* Function vect_transform_stmt. Create a vectorized stmt to replace STMT, and insert it at BSI. */ @@ -2827,6 +2948,15 @@ vect_transform_stmt (tree stmt, block_stmt_iterator *bsi) gcc_assert (done); is_store = true; break; + +/* APPLE LOCAL begin AV cond expr. -dpatel */ +/* Patch is waiting FSF review since mid Sep, 2004. */ + case select_vec_info_type: + if (!vectorizable_select (stmt, bsi, &vec_stmt)) + abort (); + break; +/* APPLE LOCAL end AV cond expr. -dpatel */ + default: if (vect_print_dump_info (REPORT_DETAILS, UNKNOWN_LOC)) fprintf (vect_dump, "stmt not supported."); @@ -3408,6 +3538,49 @@ vect_transform_loop (loop_vec_info loop_vinfo, fprintf (vect_dump, "LOOP VECTORIZED."); } +/* APPLE LOCAL begin AV cond expr. -dpatel */ +/* Patch is waiting FSF review since mid Sep, 2004. */ +/* FUnction vect_is_simple_cond. + + Input: + LOOP - the loop that is being vectorized. + COND - Condition that is checked for simple use. + + Returns whether a COND can be vectorized. Checkes whether + condition operands are supportable using vec_is_simple_use. */ + +static bool +vect_is_simple_cond (tree cond, loop_vec_info loop_vinfo) +{ + tree lhs, rhs; + + if (TREE_CODE_CLASS (TREE_CODE (cond)) != tcc_comparison) + return false; + + lhs = TREE_OPERAND (cond, 0); + rhs = TREE_OPERAND (cond, 1); + + if (TREE_CODE (lhs) == SSA_NAME) + { + tree lhs_def_stmt = SSA_NAME_DEF_STMT (lhs); + if (!vect_is_simple_use (lhs, loop_vinfo, &lhs_def_stmt)) + return false; + } + else if (TREE_CODE (lhs) != INTEGER_CST && TREE_CODE (lhs) != REAL_CST) + return false; + + if (TREE_CODE (rhs) == SSA_NAME) + { + tree rhs_def_stmt = SSA_NAME_DEF_STMT (rhs); + if (!vect_is_simple_use (rhs, loop_vinfo, &rhs_def_stmt)) + return false; + } + else if (TREE_CODE (rhs) != INTEGER_CST && TREE_CODE (rhs) != REAL_CST) + return false; + + return true; +} +/* APPLE LOCAL end AV cond expr. -dpatel */ /* Function vect_is_simple_use. @@ -3583,6 +3756,8 @@ vect_analyze_operations (loop_vec_info loop_vinfo) ok = (vectorizable_operation (stmt, NULL, NULL) || vectorizable_assignment (stmt, NULL, NULL) || vectorizable_load (stmt, NULL, NULL) + /* APPLE LOCAL AV cond expr. -dpatel */ + || vectorizable_select (stmt, NULL, NULL) || vectorizable_store (stmt, NULL, NULL)); if (!ok) @@ -3879,6 +4054,39 @@ vect_analyze_scalar_cycles (loop_vec_info loop_vinfo) return true; } +/* APPLE LOCAL begin AV data dependence. -dpatel */ +/* Patch is waiting FSF review since mid Sep, 2004. */ +/* Function vect_build_dist_vector. + + Build classic dist vector for dependence relation DDR using LOOP's loop + nest. Return LOOP's depth in its loop nest. */ + +static unsigned int +vect_build_dist_vector (struct loop *loop, + struct data_dependence_relation *ddr) +{ + struct loop *loop_nest = loop; + unsigned int loop_depth = 1; + + /* Find loop nest and loop depth. */ + while (loop_nest) + { + if (loop_nest->outer && loop_nest->outer->outer) + { + loop_nest = loop_nest->outer; + loop_depth++; + } + else + break; + } + + /* Compute distance vector. */ + compute_subscript_distance (ddr); + build_classic_dist_vector (ddr, loops_num, loop_nest->depth); + + return loop_depth - 1; +} +/* APPLE LOCAL end AV data dependence. -dpatel */ /* Function vect_base_addr_differ_p. @@ -3947,6 +4155,12 @@ vect_analyze_data_ref_dependence (struct data_reference *dra, { bool differ_p; struct data_dependence_relation *ddr; + /* APPLE LOCAL begin AV data dependence. -dpatel */ + int vectorization_factor = LOOP_VINFO_VECT_FACTOR (loop_vinfo); + struct loop *loop = LOOP_VINFO_LOOP (loop_vinfo); + unsigned int loop_depth = 0; + int dist; + /* APPLE LOCAL end AV data dependence. -dpatel */ if (!vect_base_addr_differ_p (dra, drb, &differ_p)) { @@ -3970,6 +4184,27 @@ vect_analyze_data_ref_dependence (struct data_reference *dra, if (DDR_ARE_DEPENDENT (ddr) == chrec_known) return false; + + /* APPLE LOCAL begin AV data dependence. -dpatel */ + if (DDR_ARE_DEPENDENT (ddr) == chrec_dont_know) + return true; + + loop_depth = vect_build_dist_vector (loop, ddr); + + if (!DDR_DIST_VECT (ddr)) + return true; + + dist = DDR_DIST_VECT (ddr)[loop_depth]; + + /* Same loop iteration. */ + if (dist == 0) + return false; + + if (dist >= vectorization_factor) + /* Dependence distance does not create dependence, as far as vectorization + is concerned, in this case. */ + return false; + /* APPLE LOCAL end AV data dependence. -dpatel */ if (vect_print_dump_info (REPORT_UNVECTORIZED_LOOPS, LOOP_LOC (loop_vinfo))) @@ -4510,7 +4745,8 @@ vect_analyze_data_ref_accesses (loop_vec_info loop_vinfo) INIT - the initial condition of MEMREF in the loop. */ -static struct data_reference * +/* APPLE LOCAL loops-to-memset */ +struct data_reference * vect_analyze_pointer_ref_access (tree memref, tree stmt, bool is_read, tree access_fn, tree *ptr_init, tree *ptr_step) { @@ -5492,7 +5728,8 @@ vect_can_advance_ivs_p (loop_vec_info loop_vinfo) can be constructed, place it in NUMBER_OF_ITERATIONS. Return the loop exit condition. */ -static tree +/* APPLE LOCAL loops-to-memset */ +tree vect_get_loop_niters (struct loop *loop, tree *number_of_iterations) { tree niters; @@ -5720,6 +5957,21 @@ vect_analyze_loop (struct loop *loop) return NULL; } + /* APPLE LOCAL begin AV data dependence. -dpatel */ + /* Analyze operations before data dependence. */ + + /* Scan all the operations in the loop and make sure they are + vectorizable. */ + + ok = vect_analyze_operations (loop_vinfo); + if (!ok) + { + if (vect_print_dump_info (REPORT_DETAILS, LOOP_LOC (loop_vinfo))) + fprintf (vect_dump, "bad data access."); + destroy_loop_vec_info (loop_vinfo); + return NULL; + } + /* Analyze data dependences between the data-refs in the loop. FORNOW: fail at the first data dependence that we encounter. */ @@ -5727,7 +5979,7 @@ vect_analyze_loop (struct loop *loop) if (!ok) { if (vect_print_dump_info (REPORT_DETAILS, LOOP_LOC (loop_vinfo))) - fprintf (vect_dump, "bad data dependence."); + fprintf (vect_dump, "bad data alignment."); destroy_loop_vec_info (loop_vinfo); return NULL; } @@ -5739,7 +5991,7 @@ vect_analyze_loop (struct loop *loop) if (!ok) { if (vect_print_dump_info (REPORT_DETAILS, LOOP_LOC (loop_vinfo))) - fprintf (vect_dump, "bad data access."); + fprintf (vect_dump, "bad operation or unsupported loop bound."); destroy_loop_vec_info (loop_vinfo); return NULL; } @@ -5751,23 +6003,13 @@ vect_analyze_loop (struct loop *loop) if (!ok) { if (vect_print_dump_info (REPORT_DETAILS, LOOP_LOC (loop_vinfo))) - fprintf (vect_dump, "bad data alignment."); - destroy_loop_vec_info (loop_vinfo); - return NULL; - } - - /* Scan all the operations in the loop and make sure they are - vectorizable. */ - - ok = vect_analyze_operations (loop_vinfo); - if (!ok) - { - if (vect_print_dump_info (REPORT_DETAILS, LOOP_LOC (loop_vinfo))) - fprintf (vect_dump, "bad operation or unsupported loop bound."); + fprintf (vect_dump, "bad data dependence."); destroy_loop_vec_info (loop_vinfo); return NULL; } + /* APPLE LOCAL end AV data dependence. -dpatel */ + LOOP_VINFO_VECTORIZABLE_P (loop_vinfo) = 1; return loop_vinfo; @@ -5794,7 +6036,8 @@ need_imm_uses_for (tree var) void vectorize_loops (struct loops *loops) { - unsigned int i, loops_num; + /* APPLE LOCAL AV data dependence. -dpatel */ + unsigned int i; unsigned int num_vectorized_loops = 0; /* Fix the verbosity level if not defined explicitly by the user. */ |