Add tests for literal overloads of fmod, length, and normalize (#6437)

Certain HL ops with no double overload will generate double overloads
for literal types. These are lowered to double DXIL overloads, some of
which are not legal for these ops. We currently rely on illegal
intermediate DXIL op overloads for constant evaluation. If constant
evaluation fails, we end up with illegal DXIL overloads in final DXIL,
which is caught by the validator.

The prior revert restored the ability to run these scenarios (with
asserts disabled).

This change adds tests for #6419 so automated testing will test across
configurations and prevent regressions of this scenario in this branch.

Author: tex3d

tools/clang/test/CodeGenDXIL/literal/fmod_const_eval.hlsl

@@ -0,0 +1,44 @@
+// RUN: %dxc -T vs_6_0 %s -E main | FileCheck %s
+// RUN: not %dxc -T vs_6_0 %s -E main -DNO_FOLD 2>&1 | FileCheck %s --check-prefixes=NO_FOLD
+
+// The code path for this case asserts, but the assert does not indicate a
+// serious problem with internal state.  The invalid overload will either be
+// constant folded away, or caught by the validator.
+// UNSUPPORTED: asserts
+
+// Ensure fmod is constant evaluated during codegen, or dxil const eval
+// TODO: handle fp specials properly!
+
+RWBuffer<float4> results : register(u0);
+
+[shader("vertex")]
+void main(bool b : B) {
+    uint i = 0;
+
+    // Literal float
+    // 2.5, -2.5, 2.5, -2.5
+    // CHECK: call void @dx.op.bufferStore.f32(i32 69, %dx.types.Handle %{{.+}}, i32 0, i32 undef, float 2.500000e+00, float -2.500000e+00, float 2.500000e+00, float -2.500000e+00, i8 15)
+    results[i++] = float4(fmod(5.5, 3.0),
+                          fmod(-5.5, 3.0),
+                          fmod(5.5, -3.0),
+                          fmod(-5.5, -3.0));
+
+    // Explicit float
+    // 2.5, -2.5, 2.5, -2.5
+    // CHECK: call void @dx.op.bufferStore.f32(i32 69, %dx.types.Handle %{{.+}}, i32 1, i32 undef, float 2.500000e+00, float -2.500000e+00, float 2.500000e+00, float -2.500000e+00, i8 15)
+    results[i++] = float4(fmod(5.5f, 3.0f),
+                          fmod(-5.5f, 3.0f),
+                          fmod(5.5f, -3.0f),
+                          fmod(-5.5f, -3.0f));
+
+#ifdef NO_FOLD
+    // Currently, we rely on constant folding of DXIL ops to get rid of illegal
+    // double overloads. If this doesn't happen, we expect a validation error.
+    // Ternary operator can return literal type, while not being foldable due
+    // non-constant condition.
+    // NO_FOLD: error: validation errors
+    // NO_FOLD: error: DXIL intrinsic overload must be valid.
+    float result = fmod(-5.5, b ? 1.5 : 0.5);
+    results[i++] = float4(result, 0, 0, 0);
+#endif // NO_FOLD
+}

tools/clang/test/CodeGenDXIL/literal/length_const_eval.hlsl

@@ -0,0 +1,42 @@
+// RUN: %dxc -T vs_6_0 %s -E main | FileCheck %s
+// RUN: not %dxc -T vs_6_0 %s -E main -DNO_FOLD 2>&1 | FileCheck %s --check-prefixes=NO_FOLD
+
+// The code path for this case asserts, but the assert does not indicate a
+// serious problem with internal state.  The invalid overload will either be
+// constant folded away, or caught by the validator.
+// UNSUPPORTED: asserts
+
+// Ensure length is constant evaluated during codegen, or dxil const eval
+// TODO: handle fp specials properly!
+
+RWBuffer<float4> results : register(u0);
+
+[shader("vertex")]
+void main(bool b : B) {
+    uint i = 0;
+
+    // Literal float
+    // CHECK: call void @dx.op.bufferStore.f32(i32 69, %dx.types.Handle %{{.+}}, i32 0, i32 undef, float 0x3FE6A09E60000000, float 0x4004C8DC20000000, float 0.000000e+00, float 0.000000e+00, i8 15)
+    results[i++] = float4(length(0.5.xx),
+                          length(-1.5.xxx),
+                          length(0.0.xxxx),
+                          length(-0.0.xxxx));
+
+    // Explicit float
+    // CHECK: call void @dx.op.bufferStore.f32(i32 69, %dx.types.Handle %{{.+}}, i32 1, i32 undef, float 0x3FE6A09E60000000, float 0x4004C8DC20000000, float 0.000000e+00, float 0.000000e+00, i8 15)
+    results[i++] = float4(length(0.5F.xx),
+                          length(-1.5F.xxx),
+                          length(0.0F.xxxx),
+                          length(-0.0F.xxxx));
+
+#ifdef NO_FOLD
+    // Currently, we rely on constant folding of DXIL ops to get rid of illegal
+    // double overloads. If this doesn't happen, we expect a validation error.
+    // Ternary operator can return literal type, while not being foldable due
+    // non-constant condition.
+    // NO_FOLD: error: validation errors
+    // NO_FOLD: error: DXIL intrinsic overload must be valid.
+    float result = length((b ? 1.5 : 0.5).xxx);
+    results[i++] = float4(result, 0, 0, 0);
+#endif // NO_FOLD
+}

tools/clang/test/CodeGenDXIL/literal/normalize_const_eval.hlsl

@@ -0,0 +1,41 @@
+// RUN: %dxc -T vs_6_0 %s -E main | FileCheck %s
+// RUN: not %dxc -T vs_6_0 %s -E main -DNO_FOLD 2>&1 | FileCheck %s --check-prefixes=NO_FOLD
+
+// The code path for this case asserts, but the assert does not indicate a
+// serious problem with internal state.  The invalid overload will either be
+// constant folded away, or caught by the validator.
+// UNSUPPORTED: asserts
+
+// Ensure normalize is constant evaluated during codegen, or dxil const eval
+// TODO: handle fp specials properly!
+
+RWBuffer<float4> results : register(u0);
+
+[shader("vertex")]
+void main(bool b : B) {
+    uint i = 0;
+
+    // Literal float
+    // CHECK: call void @dx.op.bufferStore.f32(i32 69, %dx.types.Handle %{{.+}}, i32 0, i32 undef, float 0x3FE6A09E60000000, float 0xBFE279A740000000, float 0x7FF8000000000000, float 0x7FF8000000000000, i8 15)
+    results[i++] = float4(normalize(0.5.xx).x,
+                          normalize(-1.5.xxx).x,
+                          normalize(0.0.xxxx).x,
+                          normalize(-0.0.xxxx).x);
+
+    // Explicit float
+    // CHECK: call void @dx.op.bufferStore.f32(i32 69, %dx.types.Handle %{{.+}}, i32 1, i32 undef, float 0x3FE6A09E60000000, float 0xBFE279A740000000, float 0x7FF8000000000000, float 0x7FF8000000000000, i8 15)
+    results[i++] = float4(normalize(0.5F.xx).x,
+                          normalize(-1.5F.xxx).x,
+                          normalize(0.0F.xxxx).x,
+                          normalize(-0.0F.xxxx).x);
+
+#ifdef NO_FOLD
+    // Currently, we rely on constant folding of DXIL ops to get rid of illegal
+    // double overloads. If this doesn't happen, we expect a validation error.
+    // Ternary operator can return literal type, while not being foldable due
+    // non-constant condition.
+    // NO_FOLD: error: validation errors
+    // NO_FOLD: error: DXIL intrinsic overload must be valid.
+    results[i++] = normalize((b ? 1.25 : 2.5).xxxx);
+#endif // NO_FOLD
+}