Enable VBMI2 support [3/7]

gcc/
	config/i386/avx512vbmi2intrin.h (_mm512_mask_expand_epi8,
	_mm512_maskz_expand_epi8, _mm512_mask_expandloadu_epi8,
	_mm512_maskz_expandloadu_epi8, _mm512_mask_expand_epi16,
	_mm512_maskz_expand_epi16, _mm512_mask_expandloadu_epi16,
	_mm512_maskz_expandloadu_epi16): New intrinsics.
	config/i386/avx512vbmi2vlintrin.h (_mm_mask_expand_epi8,
	_mm_maskz_expand_epi8, _mm_mask_expandloadu_epi8,
	_mm_maskz_expandloadu_epi8, _mm_mask_expand_epi16,
	_mm_maskz_expand_epi16, _mm_mask_expandloadu_epi16,
	_mm_maskz_expandloadu_epi16, _mm256_mask_expand_epi16,
	_mm256_maskz_expand_epi16, _mm256_mask_expandloadu_epi16,
	_mm256_maskz_expandloadu_epi16, _mm256_mask_expand_epi8,
	_mm256_maskz_expand_epi8, _mm256_mask_expandloadu_epi8,
	_mm256_maskz_expandloadu_epi8): New intrinsics.
	config/i386/i386-builtin-types.def (V64QI_FTYPE_PCV64QI_V64QI_UDI,
	V32HI_FTYPE_PCV32HI_V32HI_USI, V32QI_FTYPE_PCV32QI_V32QI_USI,
	V16HI_FTYPE_PCV16HI_V16HI_UHI, V16QI_FTYPE_PCV16QI_V16QI_UHI,
	V8HI_FTYPE_PCV8HI_V8HI_UQI): New types.
	config/i386/i386.c (ix86_expand_special_args_builtin): Use new types.
	config/i386/sse.md (VI248_VLBW): New iterator.
	(expand<mode>_mask, expand<mode>_maskz): New patterns.

gcc/testsuite/
	gcc.target/i386/avx512f-vpexpandb-1.c: New test.
	gcc.target/i386/avx512f-vpexpandb-2.c: Ditto.
	gcc.target/i386/avx512f-vpexpandw-1.c: Ditto.
	gcc.target/i386/avx512f-vpexpandw-2.c: Ditto.
	gcc.target/i386/avx512vl-vpexpandb-1.c: Ditto.
	gcc.target/i386/avx512vl-vpexpandb-2.c: Ditto.
	gcc.target/i386/avx512vl-vpexpandw-1.c: Ditto.
	gcc.target/i386/avx512vl-vpexpandw-2.c: Ditto.



git-svn-id: https://gcc.gnu.org/svn/gcc/trunk@255121 138bc75d-0d04-0410-961f-82ee72b054a4

14 files changed, 546 insertions, 0 deletions

diff --git a/gcc/ChangeLog b/gcc/ChangeLog
index c25ee4b9f81..7b40bd09b27 100644
--- a/gcc/ChangeLog
+++ b/gcc/ChangeLog
@@ -1,5 +1,29 @@
 2017-11-23  Julia Koval  <julia.koval@intel.com>
 
+	config/i386/avx512vbmi2intrin.h (_mm512_mask_expand_epi8,
+	_mm512_maskz_expand_epi8, _mm512_mask_expandloadu_epi8,
+	_mm512_maskz_expandloadu_epi8, _mm512_mask_expand_epi16,
+	_mm512_maskz_expand_epi16, _mm512_mask_expandloadu_epi16,
+	_mm512_maskz_expandloadu_epi16): New intrinsics.
+	config/i386/avx512vbmi2vlintrin.h (_mm_mask_expand_epi8,
+	_mm_maskz_expand_epi8, _mm_mask_expandloadu_epi8,
+	_mm_maskz_expandloadu_epi8, _mm_mask_expand_epi16,
+	_mm_maskz_expand_epi16, _mm_mask_expandloadu_epi16,
+	_mm_maskz_expandloadu_epi16, _mm256_mask_expand_epi16,
+	_mm256_maskz_expand_epi16, _mm256_mask_expandloadu_epi16,
+	_mm256_maskz_expandloadu_epi16, _mm256_mask_expand_epi8,
+	_mm256_maskz_expand_epi8, _mm256_mask_expandloadu_epi8,
+	_mm256_maskz_expandloadu_epi8): New intrinsics.
+	config/i386/i386-builtin-types.def (V64QI_FTYPE_PCV64QI_V64QI_UDI,
+	V32HI_FTYPE_PCV32HI_V32HI_USI, V32QI_FTYPE_PCV32QI_V32QI_USI,
+	V16HI_FTYPE_PCV16HI_V16HI_UHI, V16QI_FTYPE_PCV16QI_V16QI_UHI,
+	V8HI_FTYPE_PCV8HI_V8HI_UQI): New types.
+	config/i386/i386.c (ix86_expand_special_args_builtin): Use new types.
+	config/i386/sse.md (VI248_VLBW): New iterator.
+	(expand<mode>_mask, expand<mode>_maskz): New patterns.
+
+2017-11-23  Julia Koval  <julia.koval@intel.com>
+
 	config.gcc (avx512vbmi2intrin.h, avx512vbmi2vlintrin): New headers.
 	config/i386/avx512vbmi2intrin.h (_mm512_mask_compress_epi8,
 	_mm512_maskz_compress_epi8, _mm512_mask_compressstoreu_epi8,
diff --git a/gcc/config/i386/avx512vbmi2intrin.h b/gcc/config/i386/avx512vbmi2intrin.h
index 70c3257342a..aa936ce5546 100644
--- a/gcc/config/i386/avx512vbmi2intrin.h
+++ b/gcc/config/i386/avx512vbmi2intrin.h
@@ -83,6 +83,72 @@ _mm512_mask_compressstoreu_epi16 (void * __A, __mmask32 __B, __m512i __C)
   __builtin_ia32_compressstoreuhi512_mask ((__v32hi *) __A, (__v32hi) __C,
 							(__mmask32) __B);
 }
+
+extern __inline __m512i
+__attribute__((__gnu_inline__, __always_inline__, __artificial__))
+_mm512_mask_expand_epi8 (__m512i __A, __mmask64 __B, __m512i __C)
+{
+  return (__m512i) __builtin_ia32_expandqi512_mask ((__v64qi) __C,
+						    (__v64qi) __A,
+						    (__mmask64) __B);
+}
+
+extern __inline __m512i
+__attribute__((__gnu_inline__, __always_inline__, __artificial__))
+_mm512_maskz_expand_epi8 (__mmask64 __A, __m512i __B)
+{
+  return (__m512i) __builtin_ia32_expandqi512_maskz ((__v64qi) __B,
+			(__v64qi) _mm512_setzero_si512 (), (__mmask64) __A);
+}
+
+extern __inline __m512i
+__attribute__((__gnu_inline__, __always_inline__, __artificial__))
+_mm512_mask_expandloadu_epi8 (__m512i __A, __mmask64 __B, const void * __C)
+{
+  return (__m512i) __builtin_ia32_expandloadqi512_mask ((const __v64qi *) __C,
+					(__v64qi) __A, (__mmask64) __B);
+}
+
+extern __inline __m512i
+__attribute__((__gnu_inline__, __always_inline__, __artificial__))
+_mm512_maskz_expandloadu_epi8 (__mmask64 __A, const void * __B)
+{
+  return (__m512i) __builtin_ia32_expandloadqi512_maskz ((const __v64qi *) __B,
+			(__v64qi) _mm512_setzero_si512 (), (__mmask64) __A);
+}
+
+extern __inline __m512i
+__attribute__((__gnu_inline__, __always_inline__, __artificial__))
+_mm512_mask_expand_epi16 (__m512i __A, __mmask32 __B, __m512i __C)
+{
+  return (__m512i) __builtin_ia32_expandhi512_mask ((__v32hi) __C,
+						    (__v32hi) __A,
+						    (__mmask32) __B);
+}
+
+extern __inline __m512i
+__attribute__((__gnu_inline__, __always_inline__, __artificial__))
+_mm512_maskz_expand_epi16 (__mmask32 __A, __m512i __B)
+{
+  return (__m512i) __builtin_ia32_expandhi512_maskz ((__v32hi) __B,
+			(__v32hi) _mm512_setzero_si512 (), (__mmask32) __A);
+}
+
+extern __inline __m512i
+__attribute__((__gnu_inline__, __always_inline__, __artificial__))
+_mm512_mask_expandloadu_epi16 (__m512i __A, __mmask32 __B, const void * __C)
+{
+  return (__m512i) __builtin_ia32_expandloadhi512_mask ((const __v32hi *) __C,
+					(__v32hi) __A, (__mmask32) __B);
+}
+
+extern __inline __m512i
+__attribute__((__gnu_inline__, __always_inline__, __artificial__))
+_mm512_maskz_expandloadu_epi16 (__mmask32 __A, const void * __B)
+{
+  return (__m512i) __builtin_ia32_expandloadhi512_maskz ((const __v32hi *) __B,
+			(__v32hi) _mm512_setzero_si512 (), (__mmask32) __A);
+}
 #ifdef __DISABLE_AVX512VBMI2BW__
 #undef __DISABLE_AVX512VBMI2BW__
 
diff --git a/gcc/config/i386/avx512vbmi2vlintrin.h b/gcc/config/i386/avx512vbmi2vlintrin.h
index 6d8bbb62858..f47f3d0c6a6 100644
--- a/gcc/config/i386/avx512vbmi2vlintrin.h
+++ b/gcc/config/i386/avx512vbmi2vlintrin.h
@@ -106,6 +106,104 @@ _mm_mask_compressstoreu_epi16 (void * __A, __mmask8 __B, __m128i __C)
   __builtin_ia32_compressstoreuhi128_mask ((__v8hi *) __A, (__v8hi) __C,
 							(__mmask8) __B);
 }
+
+extern __inline __m128i
+__attribute__((__gnu_inline__, __always_inline__, __artificial__))
+_mm_mask_expand_epi8 (__m128i __A, __mmask16 __B, __m128i __C)
+{
+  return (__m128i) __builtin_ia32_expandqi128_mask ((__v16qi) __C,
+						    (__v16qi) __A,
+						    (__mmask16) __B);
+}
+
+extern __inline __m128i
+__attribute__((__gnu_inline__, __always_inline__, __artificial__))
+_mm_maskz_expand_epi8 (__mmask16 __A, __m128i __B)
+{
+  return (__m128i) __builtin_ia32_expandqi128_maskz ((__v16qi) __B,
+			(__v16qi) _mm_setzero_si128 (), (__mmask16) __A);
+}
+
+extern __inline __m128i
+__attribute__((__gnu_inline__, __always_inline__, __artificial__))
+_mm_mask_expandloadu_epi8 (__m128i __A, __mmask16 __B, const void * __C)
+{
+  return (__m128i) __builtin_ia32_expandloadqi128_mask ((const __v16qi *) __C,
+					(__v16qi) __A, (__mmask16) __B);
+}
+
+extern __inline __m128i
+__attribute__((__gnu_inline__, __always_inline__, __artificial__))
+_mm_maskz_expandloadu_epi8 (__mmask16 __A, const void * __B)
+{
+  return (__m128i) __builtin_ia32_expandloadqi128_maskz ((const __v16qi *) __B,
+			(__v16qi) _mm_setzero_si128 (), (__mmask16) __A);
+}
+
+extern __inline __m128i
+__attribute__((__gnu_inline__, __always_inline__, __artificial__))
+_mm_mask_expand_epi16 (__m128i __A, __mmask8 __B, __m128i __C)
+{
+  return (__m128i) __builtin_ia32_expandhi128_mask ((__v8hi) __C,
+						    (__v8hi) __A,
+						    (__mmask8) __B);
+}
+
+extern __inline __m128i
+__attribute__((__gnu_inline__, __always_inline__, __artificial__))
+_mm_maskz_expand_epi16 (__mmask8 __A, __m128i __B)
+{
+  return (__m128i) __builtin_ia32_expandhi128_maskz ((__v8hi) __B,
+				(__v8hi) _mm_setzero_si128 (), (__mmask8) __A);
+}
+
+extern __inline __m128i
+__attribute__((__gnu_inline__, __always_inline__, __artificial__))
+_mm_mask_expandloadu_epi16 (__m128i __A, __mmask8 __B, const void * __C)
+{
+  return (__m128i) __builtin_ia32_expandloadhi128_mask ((const __v8hi *) __C,
+						(__v8hi) __A, (__mmask8) __B);
+}
+
+extern __inline __m128i
+__attribute__((__gnu_inline__, __always_inline__, __artificial__))
+_mm_maskz_expandloadu_epi16 (__mmask8 __A, const void * __B)
+{
+  return (__m128i) __builtin_ia32_expandloadhi128_maskz ((const __v8hi *) __B,
+				(__v8hi) _mm_setzero_si128 (), (__mmask8) __A);
+}
+extern __inline __m256i
+__attribute__((__gnu_inline__, __always_inline__, __artificial__))
+_mm256_mask_expand_epi16 (__m256i __A, __mmask16 __B, __m256i __C)
+{
+  return (__m256i) __builtin_ia32_expandhi256_mask ((__v16hi) __C,
+						    (__v16hi) __A,
+						    (__mmask16) __B);
+}
+
+extern __inline __m256i
+__attribute__((__gnu_inline__, __always_inline__, __artificial__))
+_mm256_maskz_expand_epi16 (__mmask16 __A, __m256i __B)
+{
+  return (__m256i) __builtin_ia32_expandhi256_maskz ((__v16hi) __B,
+			(__v16hi) _mm256_setzero_si256 (), (__mmask16) __A);
+}
+
+extern __inline __m256i
+__attribute__((__gnu_inline__, __always_inline__, __artificial__))
+_mm256_mask_expandloadu_epi16 (__m256i __A, __mmask16 __B, const void * __C)
+{
+  return (__m256i) __builtin_ia32_expandloadhi256_mask ((const __v16hi *) __C,
+					(__v16hi) __A, (__mmask16) __B);
+}
+
+extern __inline __m256i
+__attribute__((__gnu_inline__, __always_inline__, __artificial__))
+_mm256_maskz_expandloadu_epi16 (__mmask16 __A, const void * __B)
+{
+  return (__m256i) __builtin_ia32_expandloadhi256_maskz ((const __v16hi *) __B,
+			(__v16hi) _mm256_setzero_si256 (), (__mmask16) __A);
+}
 #ifdef __DISABLE_AVX512VBMI2VL__
 #undef __DISABLE_AVX512VBMI2VL__
 #pragma GCC pop_options
@@ -142,6 +240,39 @@ _mm256_mask_compressstoreu_epi8 (void * __A, __mmask32 __B, __m256i __C)
 							(__mmask32) __B);
 }
 
+extern __inline __m256i
+__attribute__((__gnu_inline__, __always_inline__, __artificial__))
+_mm256_mask_expand_epi8 (__m256i __A, __mmask32 __B, __m256i __C)
+{
+  return (__m256i) __builtin_ia32_expandqi256_mask ((__v32qi) __C,
+						    (__v32qi) __A,
+						    (__mmask32) __B);
+}
+
+extern __inline __m256i
+__attribute__((__gnu_inline__, __always_inline__, __artificial__))
+_mm256_maskz_expand_epi8 (__mmask32 __A, __m256i __B)
+{
+  return (__m256i) __builtin_ia32_expandqi256_maskz ((__v32qi) __B,
+			(__v32qi) _mm256_setzero_si256 (), (__mmask32) __A);
+}
+
+extern __inline __m256i
+__attribute__((__gnu_inline__, __always_inline__, __artificial__))
+_mm256_mask_expandloadu_epi8 (__m256i __A, __mmask32 __B, const void * __C)
+{
+  return (__m256i) __builtin_ia32_expandloadqi256_mask ((const __v32qi *) __C,
+					(__v32qi) __A, (__mmask32) __B);
+}
+
+extern __inline __m256i
+__attribute__((__gnu_inline__, __always_inline__, __artificial__))
+_mm256_maskz_expandloadu_epi8 (__mmask32 __A, const void * __B)
+{
+  return (__m256i) __builtin_ia32_expandloadqi256_maskz ((const __v32qi *) __B,
+			(__v32qi) _mm256_setzero_si256 (), (__mmask32) __A);
+}
+
 #ifdef __DISABLE_AVX512VBMI2VLBW__
 #undef __DISABLE_AVX512VBMI2VLBW__
 #pragma GCC pop_options
diff --git a/gcc/config/i386/i386.c b/gcc/config/i386/i386.c
index 0e00351a3c7..2e528396982 100644
--- a/gcc/config/i386/i386.c
+++ b/gcc/config/i386/i386.c
@@ -34914,6 +34914,12 @@ ix86_expand_special_args_builtin (const struct builtin_description *d,
     case V2DI_FTYPE_PCV2DI_V2DI_UQI:
     case V4DI_FTYPE_PCV4DI_V4DI_UQI:
     case V8DI_FTYPE_PCV8DI_V8DI_UQI:
+    case V64QI_FTYPE_PCV64QI_V64QI_UDI:
+    case V32HI_FTYPE_PCV32HI_V32HI_USI:
+    case V32QI_FTYPE_PCV32QI_V32QI_USI:
+    case V16QI_FTYPE_PCV16QI_V16QI_UHI:
+    case V16HI_FTYPE_PCV16HI_V16HI_UHI:
+    case V8HI_FTYPE_PCV8HI_V8HI_UQI:
       switch (icode)
 	{
 	/* These builtins and instructions require the memory
diff --git a/gcc/config/i386/sse.md b/gcc/config/i386/sse.md
index 5e67ea5fe38..93efd275997 100644
--- a/gcc/config/i386/sse.md
+++ b/gcc/config/i386/sse.md
@@ -418,6 +418,12 @@
 (define_mode_iterator VI2_AVX2_AVX512BW
   [(V32HI "TARGET_AVX512BW") (V16HI "TARGET_AVX2") V8HI])
 
+(define_mode_iterator VI248_VLBW
+  [(V32HI "TARGET_AVX512BW") V16SI V8DI
+   (V16HI "TARGET_AVX512VL") (V8SI "TARGET_AVX512VL")
+   (V4DI "TARGET_AVX512VL") (V8HI "TARGET_AVX512VL")
+   (V4SI "TARGET_AVX512VL") (V2DI "TARGET_AVX512VL")])
+
 (define_mode_iterator VI48_AVX2
   [(V8SI "TARGET_AVX2") V4SI
    (V4DI "TARGET_AVX2") V2DI])
@@ -19369,6 +19375,30 @@
    (set_attr "memory" "none,load")
    (set_attr "mode" "<sseinsnmode>")])
 
+(define_insn "expand<mode>_mask"
+  [(set (match_operand:VI12_AVX512VLBW 0 "register_operand" "=v,v")
+	(unspec:VI12_AVX512VLBW
+	  [(match_operand:VI12_AVX512VLBW 1 "nonimmediate_operand" "v,m")
+	   (match_operand:VI12_AVX512VLBW 2 "vector_move_operand" "0C,0C")
+	   (match_operand:<avx512fmaskmode> 3 "register_operand" "Yk,Yk")]
+	  UNSPEC_EXPAND))]
+  "TARGET_AVX512VBMI2"
+  "v<sseintprefix>expand<ssemodesuffix>\t{%1, %0%{%3%}%N2|%0%{%3%}%N2, %1}"
+  [(set_attr "type" "ssemov")
+   (set_attr "prefix" "evex")
+   (set_attr "memory" "none,load")
+   (set_attr "mode" "<sseinsnmode>")])
+
+(define_expand "expand<mode>_maskz"
+  [(set (match_operand:VI12_AVX512VLBW 0 "register_operand")
+	(unspec:VI12_AVX512VLBW
+	  [(match_operand:VI12_AVX512VLBW 1 "nonimmediate_operand")
+	   (match_operand:VI12_AVX512VLBW 2 "vector_move_operand")
+	   (match_operand:<avx512fmaskmode> 3 "register_operand")]
+	  UNSPEC_EXPAND))]
+  "TARGET_AVX512VBMI2"
+  "operands[2] = CONST0_RTX (<MODE>mode);")
+
 (define_insn "avx512dq_rangep<mode><mask_name><round_saeonly_name>"
   [(set (match_operand:VF_AVX512VL 0 "register_operand" "=v")
 	(unspec:VF_AVX512VL
diff --git a/gcc/testsuite/ChangeLog b/gcc/testsuite/ChangeLog
index 89f180b52ec..943f5fe84c0 100644
--- a/gcc/testsuite/ChangeLog
+++ b/gcc/testsuite/ChangeLog
@@ -1,3 +1,14 @@
+2017-11-23  Julia Koval  <julia.koval@intel.com>
+
+	gcc.target/i386/avx512f-vpexpandb-1.c: New test.
+	gcc.target/i386/avx512f-vpexpandb-2.c: Ditto.
+	gcc.target/i386/avx512f-vpexpandw-1.c: Ditto.
+	gcc.target/i386/avx512f-vpexpandw-2.c: Ditto.
+	gcc.target/i386/avx512vl-vpexpandb-1.c: Ditto.
+	gcc.target/i386/avx512vl-vpexpandb-2.c: Ditto.
+	gcc.target/i386/avx512vl-vpexpandw-1.c: Ditto.
+	gcc.target/i386/avx512vl-vpexpandw-2.c: Ditto.
+
 2017-11-16  Julia Koval  <julia.koval@intel.com>
 
 	gcc.target/i386/avx512-check.h: Handle AVX512VBMI2 bit.
diff --git a/gcc/testsuite/gcc.target/i386/avx512f-vpexpandb-1.c b/gcc/testsuite/gcc.target/i386/avx512f-vpexpandb-1.c
new file mode 100644
index 00000000000..fb0c58e428f
--- /dev/null
+++ b/gcc/testsuite/gcc.target/i386/avx512f-vpexpandb-1.c
@@ -0,0 +1,22 @@
+/* { dg-do compile } */
+/* { dg-options "-mavx512vbmi2 -mavx512bw -mavx512f -O2" } */
+/* { dg-final { scan-assembler-times "vpexpandb\[ \\t\]+\[^\{\n\]*\\(\[^\n\]*%zmm\[0-9\]+\{%k\[1-7\]\}(?:\n|\[ \\t\]+#)"  1 } } */
+/* { dg-final { scan-assembler-times "vpexpandb\[ \\t\]+\[^\{\n\]*\\(\[^\n\]*%zmm\[0-9\]+\{%k\[1-7\]\}\{z\}(?:\n|\[ \\t\]+#)"  1 } } */
+/* { dg-final { scan-assembler-times "vpexpandb\[ \\t\]+\[^\{\n\(]*%zmm\[0-9\]+\{%k\[1-7\]\}(?:\n|\[ \\t\]+#)"  1 } } */
+/* { dg-final { scan-assembler-times "vpexpandb\[ \\t\]+\[^\{\n\(]*%zmm\[0-9\]+\{%k\[1-7\]\}\{z\}(?:\n|\[ \\t\]+#)"  1 } } */
+
+#include <immintrin.h>
+
+int *p;
+volatile __m512i x;
+volatile __mmask16 m;
+
+void extern
+avx512f_test (void)
+{
+  x = _mm512_mask_expand_epi8 (x, m, x);
+  x = _mm512_maskz_expand_epi8 (m, x);
+
+  x = _mm512_mask_expandloadu_epi8 (x, m, p);
+  x = _mm512_maskz_expandloadu_epi8 (m, p);
+}
diff --git a/gcc/testsuite/gcc.target/i386/avx512f-vpexpandb-2.c b/gcc/testsuite/gcc.target/i386/avx512f-vpexpandb-2.c
new file mode 100644
index 00000000000..0105ddbe20e
--- /dev/null
+++ b/gcc/testsuite/gcc.target/i386/avx512f-vpexpandb-2.c
@@ -0,0 +1,68 @@
+/* { dg-do run } */
+/* { dg-options "-O2 -mavx512f -mavx512vbmi2 -mavx512bw" } */
+/* { dg-require-effective-target avx512f } */
+/* { dg-require-effective-target avx512vbmi2 } */
+
+#define AVX512F
+
+#define AVX512VBMI2
+#include "avx512f-helper.h"
+
+#define SIZE (AVX512F_LEN / 8)
+#include "avx512f-mask-type.h"
+
+static void
+CALC (char *s, char *r, MASK_TYPE mask)
+{
+  int i, k;
+
+  for (i = 0, k = 0; i < SIZE; i++)
+    {
+      if (mask & ((long long)1 << i))
+	r[i] = s[k++];
+    }
+}
+
+void
+TEST (void)
+{
+  UNION_TYPE (AVX512F_LEN, i_b) s1, res1, res2, res3, res4;
+  MASK_TYPE mask = MASK_VALUE;
+  char s2[SIZE];
+  char res_ref1[SIZE];
+  char res_ref2[SIZE];
+  int i, sign = 1;
+
+  for (i = 0; i < SIZE; i++)
+    {
+      s1.a[i] = 12345 * (i + 200) * sign;
+      s2[i] = 67890 * (i + 300) * sign;
+      res1.a[i] = DEFAULT_VALUE;
+      res3.a[i] = DEFAULT_VALUE;
+      sign = -sign;
+    }
+
+  res1.x = INTRINSIC (_mask_expand_epi8) (res1.x, mask, s1.x);
+  res2.x = INTRINSIC (_maskz_expand_epi8) (mask, s1.x);
+  res3.x = INTRINSIC (_mask_expandloadu_epi8) (res3.x, mask, s2);
+  res4.x = INTRINSIC (_maskz_expandloadu_epi8) (mask, s2);
+
+  CALC (s1.a, res_ref1, mask);
+  CALC (s2, res_ref2, mask);
+
+  MASK_MERGE (i_b) (res_ref1, mask, SIZE);
+  if (UNION_CHECK (AVX512F_LEN, i_b) (res1, res_ref1))
+    abort ();
+
+  MASK_ZERO (i_b) (res_ref1, mask, SIZE);
+  if (UNION_CHECK (AVX512F_LEN, i_b) (res2, res_ref1))
+    abort ();
+
+  MASK_MERGE (i_b) (res_ref2, mask, SIZE);
+  if (UNION_CHECK (AVX512F_LEN, i_b) (res3, res_ref2))
+    abort ();
+
+  MASK_ZERO (i_b) (res_ref2, mask, SIZE);
+  if (UNION_CHECK (AVX512F_LEN, i_b) (res4, res_ref2))
+    abort ();
+}
diff --git a/gcc/testsuite/gcc.target/i386/avx512f-vpexpandw-1.c b/gcc/testsuite/gcc.target/i386/avx512f-vpexpandw-1.c
new file mode 100644
index 00000000000..49d9fb89acf
--- /dev/null
+++ b/gcc/testsuite/gcc.target/i386/avx512f-vpexpandw-1.c
@@ -0,0 +1,22 @@
+/* { dg-do compile } */
+/* { dg-options "-mavx512vbmi2 -mavx512bw -mavx512f -O2" } */
+/* { dg-final { scan-assembler-times "vpexpandw\[ \\t\]+\[^\{\n\]*\\(\[^\n\]*%zmm\[0-9\]+\{%k\[1-7\]\}(?:\n|\[ \\t\]+#)"  1 } } */
+/* { dg-final { scan-assembler-times "vpexpandw\[ \\t\]+\[^\{\n\]*\\(\[^\n\]*%zmm\[0-9\]+\{%k\[1-7\]\}\{z\}(?:\n|\[ \\t\]+#)"  1 } } */
+/* { dg-final { scan-assembler-times "vpexpandw\[ \\t\]+\[^\{\n\(]*%zmm\[0-9\]+\{%k\[1-7\]\}(?:\n|\[ \\t\]+#)"  1 } } */
+/* { dg-final { scan-assembler-times "vpexpandw\[ \\t\]+\[^\{\n\(]*%zmm\[0-9\]+\{%k\[1-7\]\}\{z\}(?:\n|\[ \\t\]+#)"  1 } } */
+
+#include <immintrin.h>
+
+int *p;
+volatile __m512i x;
+volatile __mmask16 m;
+
+void extern
+avx512f_test (void)
+{
+  x = _mm512_mask_expand_epi16 (x, m, x);
+  x = _mm512_maskz_expand_epi16 (m, x);
+
+  x = _mm512_mask_expandloadu_epi16 (x, m, p);
+  x = _mm512_maskz_expandloadu_epi16 (m, p);
+}
diff --git a/gcc/testsuite/gcc.target/i386/avx512f-vpexpandw-2.c b/gcc/testsuite/gcc.target/i386/avx512f-vpexpandw-2.c
new file mode 100644
index 00000000000..fdad38b6813
--- /dev/null
+++ b/gcc/testsuite/gcc.target/i386/avx512f-vpexpandw-2.c
@@ -0,0 +1,68 @@
+/* { dg-do run } */
+/* { dg-options "-O2 -mavx512f -mavx512vbmi2 -mavx512bw" } */
+/* { dg-require-effective-target avx512f } */
+/* { dg-require-effective-target avx512vbmi2 } */
+
+#define AVX512F
+
+#define AVX512VBMI2
+#include "avx512f-helper.h"
+
+#define SIZE (AVX512F_LEN / 16)
+#include "avx512f-mask-type.h"
+
+static void
+CALC (short *s, short *r, MASK_TYPE mask)
+{
+  int i, k;
+
+  for (i = 0, k = 0; i < SIZE; i++)
+    {
+      if (mask & (1 << i))
+	r[i] = s[k++];
+    }
+}
+
+void
+TEST (void)
+{
+  UNION_TYPE (AVX512F_LEN, i_w) s1, res1, res2, res3, res4;
+  MASK_TYPE mask = MASK_VALUE;
+  short s2[SIZE];
+  short res_ref1[SIZE];
+  short res_ref2[SIZE];
+  int i, sign = 1;
+
+  for (i = 0; i < SIZE; i++)
+    {
+      s1.a[i] = 12345 * (i + 200) * sign;
+      s2[i] = 67890 * (i + 300) * sign;
+      res1.a[i] = DEFAULT_VALUE;
+      res3.a[i] = DEFAULT_VALUE;
+      sign = -sign;
+    }
+
+  res1.x = INTRINSIC (_mask_expand_epi16) (res1.x, mask, s1.x);
+  res2.x = INTRINSIC (_maskz_expand_epi16) (mask, s1.x);
+  res3.x = INTRINSIC (_mask_expandloadu_epi16) (res3.x, mask, s2);
+  res4.x = INTRINSIC (_maskz_expandloadu_epi16) (mask, s2);
+
+  CALC (s1.a, res_ref1, mask);
+  CALC (s2, res_ref2, mask);
+
+  MASK_MERGE (i_w) (res_ref1, mask, SIZE);
+  if (UNION_CHECK (AVX512F_LEN, i_w) (res1, res_ref1))
+    abort ();
+
+  MASK_ZERO (i_w) (res_ref1, mask, SIZE);
+  if (UNION_CHECK (AVX512F_LEN, i_w) (res2, res_ref1))
+    abort ();
+
+  MASK_MERGE (i_w) (res_ref2, mask, SIZE);
+  if (UNION_CHECK (AVX512F_LEN, i_w) (res3, res_ref2))
+    abort ();
+
+  MASK_ZERO (i_w) (res_ref2, mask, SIZE);
+  if (UNION_CHECK (AVX512F_LEN, i_w) (res4, res_ref2))
+    abort ();
+}
diff --git a/gcc/testsuite/gcc.target/i386/avx512vl-vpexpandb-1.c b/gcc/testsuite/gcc.target/i386/avx512vl-vpexpandb-1.c
new file mode 100644
index 00000000000..96e0d815f13
--- /dev/null
+++ b/gcc/testsuite/gcc.target/i386/avx512vl-vpexpandb-1.c
@@ -0,0 +1,33 @@
+/* { dg-do compile } */
+/* { dg-options "-mavx512vbmi2 -mavx512bw -mavx512vl -O2" } */
+/* { dg-final { scan-assembler-times "vpexpandb\[ \\t\]+\[^\{\n\]*%ymm\[0-9\]+\[^\n\]*%ymm\[0-9\]+\{%k\[1-7\]\}(?:\n|\[ \\t\]+#)"  1 } } */
+/* { dg-final { scan-assembler-times "vpexpandb\[ \\t\]+\[^\{\n\]*%xmm\[0-9\]+\[^\n\]*%xmm\[0-9\]+\{%k\[1-7\]\}(?:\n|\[ \\t\]+#)"  1 } } */
+/* { dg-final { scan-assembler-times "vpexpandb\[ \\t\]+\[^\{\n\]*%ymm\[0-9\]+\[^\n\]*%ymm\[0-9\]+\{%k\[1-7\]\}\{z\}(?:\n|\[ \\t\]+#)"  1 } } */
+/* { dg-final { scan-assembler-times "vpexpandb\[ \\t\]+\[^\{\n\]*%xmm\[0-9\]+\[^\n\]*%xmm\[0-9\]+\{%k\[1-7\]\}\{z\}(?:\n|\[ \\t\]+#)"  1 } } */
+/* { dg-final { scan-assembler-times "vpexpandb\[ \\t\]+\[^\{\n\]*\\(\[^\n\]*%ymm\[0-9\]+\{%k\[1-7\]\}(?:\n|\[ \\t\]+#)"  1 } } */
+/* { dg-final { scan-assembler-times "vpexpandb\[ \\t\]+\[^\{\n\]*\\(\[^\n\]*%xmm\[0-9\]+\{%k\[1-7\]\}(?:\n|\[ \\t\]+#)"  1 } } */
+/* { dg-final { scan-assembler-times "vpexpandb\[ \\t\]+\[^\{\n\]*\\(\[^\n\]*%ymm\[0-9\]+\{%k\[1-7\]\}\{z\}(?:\n|\[ \\t\]+#)"  1 } } */
+/* { dg-final { scan-assembler-times "vpexpandb\[ \\t\]+\[^\{\n\]*\\(\[^\n\]*%xmm\[0-9\]+\{%k\[1-7\]\}\{z\}(?:\n|\[ \\t\]+#)"  1 } } */
+
+#include <immintrin.h>
+
+int *p;
+volatile __m256i x1;
+volatile __m128i x2;
+volatile __mmask8 m;
+
+void extern
+avx512vl_test (void)
+{
+  x1 = _mm256_mask_expand_epi8 (x1, m, x1);
+  x2 = _mm_mask_expand_epi8 (x2, m, x2);
+
+  x1 = _mm256_maskz_expand_epi8 (m, x1);
+  x2 = _mm_maskz_expand_epi8 (m, x2);
+
+  x1 = _mm256_mask_expandloadu_epi8 (x1, m, p);
+  x2 = _mm_mask_expandloadu_epi8 (x2, m, p);
+
+  x1 = _mm256_maskz_expandloadu_epi8 (m, p);
+  x2 = _mm_maskz_expandloadu_epi8 (m, p);
+}
diff --git a/gcc/testsuite/gcc.target/i386/avx512vl-vpexpandb-2.c b/gcc/testsuite/gcc.target/i386/avx512vl-vpexpandb-2.c
new file mode 100644
index 00000000000..280aedad135
--- /dev/null
+++ b/gcc/testsuite/gcc.target/i386/avx512vl-vpexpandb-2.c
@@ -0,0 +1,16 @@
+/* { dg-do run } */
+/* { dg-options "-O2 -mavx512vl -mavx512vbmi2 -mavx512bw" } */
+/* { dg-require-effective-target avx512vl } */
+/* { dg-require-effective-target avx512vbmi2 } */
+
+#define AVX512VL
+#define AVX512F_LEN 256
+#define AVX512F_LEN_HALF 128
+#include "avx512f-vpexpandb-2.c"
+
+#undef AVX512F_LEN
+#undef AVX512F_LEN_HALF
+
+#define AVX512F_LEN 128
+#define AVX512F_LEN_HALF 128
+#include "avx512f-vpexpandb-2.c"
diff --git a/gcc/testsuite/gcc.target/i386/avx512vl-vpexpandw-1.c b/gcc/testsuite/gcc.target/i386/avx512vl-vpexpandw-1.c
new file mode 100644
index 00000000000..ac5c34a0f42
--- /dev/null
+++ b/gcc/testsuite/gcc.target/i386/avx512vl-vpexpandw-1.c
@@ -0,0 +1,33 @@
+/* { dg-do compile } */
+/* { dg-options "-mavx512vbmi2 -mavx512bw -mavx512vl -O2" } */
+/* { dg-final { scan-assembler-times "vpexpandw\[ \\t\]+\[^\{\n\]*%ymm\[0-9\]+\[^\n\]*%ymm\[0-9\]+\{%k\[1-7\]\}(?:\n|\[ \\t\]+#)"  1 } } */
+/* { dg-final { scan-assembler-times "vpexpandw\[ \\t\]+\[^\{\n\]*%xmm\[0-9\]+\[^\n\]*%xmm\[0-9\]+\{%k\[1-7\]\}(?:\n|\[ \\t\]+#)"  1 } } */
+/* { dg-final { scan-assembler-times "vpexpandw\[ \\t\]+\[^\{\n\]*%ymm\[0-9\]+\[^\n\]*%ymm\[0-9\]+\{%k\[1-7\]\}\{z\}(?:\n|\[ \\t\]+#)"  1 } } */
+/* { dg-final { scan-assembler-times "vpexpandw\[ \\t\]+\[^\{\n\]*%xmm\[0-9\]+\[^\n\]*%xmm\[0-9\]+\{%k\[1-7\]\}\{z\}(?:\n|\[ \\t\]+#)"  1 } } */
+/* { dg-final { scan-assembler-times "vpexpandw\[ \\t\]+\[^\{\n\]*\\(\[^\n\]*%ymm\[0-9\]+\{%k\[1-7\]\}(?:\n|\[ \\t\]+#)"  1 } } */
+/* { dg-final { scan-assembler-times "vpexpandw\[ \\t\]+\[^\{\n\]*\\(\[^\n\]*%xmm\[0-9\]+\{%k\[1-7\]\}(?:\n|\[ \\t\]+#)"  1 } } */
+/* { dg-final { scan-assembler-times "vpexpandw\[ \\t\]+\[^\{\n\]*\\(\[^\n\]*%ymm\[0-9\]+\{%k\[1-7\]\}\{z\}(?:\n|\[ \\t\]+#)"  1 } } */
+/* { dg-final { scan-assembler-times "vpexpandw\[ \\t\]+\[^\{\n\]*\\(\[^\n\]*%xmm\[0-9\]+\{%k\[1-7\]\}\{z\}(?:\n|\[ \\t\]+#)"  1 } } */
+
+#include <immintrin.h>
+
+int *p;
+volatile __m256i x1;
+volatile __m128i x2;
+volatile __mmask8 m;
+
+void extern
+avx512vl_test (void)
+{
+  x1 = _mm256_mask_expand_epi16 (x1, m, x1);
+  x2 = _mm_mask_expand_epi16 (x2, m, x2);
+
+  x1 = _mm256_maskz_expand_epi16 (m, x1);
+  x2 = _mm_maskz_expand_epi16 (m, x2);
+
+  x1 = _mm256_mask_expandloadu_epi16 (x1, m, p);
+  x2 = _mm_mask_expandloadu_epi16 (x2, m, p);
+
+  x1 = _mm256_maskz_expandloadu_epi16 (m, p);
+  x2 = _mm_maskz_expandloadu_epi16 (m, p);
+}
diff --git a/gcc/testsuite/gcc.target/i386/avx512vl-vpexpandw-2.c b/gcc/testsuite/gcc.target/i386/avx512vl-vpexpandw-2.c
new file mode 100644
index 00000000000..2c1e00457cc
--- /dev/null
+++ b/gcc/testsuite/gcc.target/i386/avx512vl-vpexpandw-2.c
@@ -0,0 +1,16 @@
+/* { dg-do run } */
+/* { dg-options "-O2 -mavx512vl -mavx512vbmi2 -mavx512bw" } */
+/* { dg-require-effective-target avx512vl } */
+/* { dg-require-effective-target avx512vbmi2 } */
+
+#define AVX512VL
+#define AVX512F_LEN 256
+#define AVX512F_LEN_HALF 128
+#include "avx512f-vpexpandw-2.c"
+
+#undef AVX512F_LEN
+#undef AVX512F_LEN_HALF
+
+#define AVX512F_LEN 128
+#define AVX512F_LEN_HALF 128
+#include "avx512f-vpexpandw-2.c"