Make the HWND one clear the same way as composition. Identify and temporarily resolve issue where device resources aren't created by conhost early enough for intersects to happen against the actual screen size. Mitigate issue where Present1 doesn't like being called if there isn't already ONE frame presented ever on the chain.

src/renderer/dx/DxRenderer.cpp

@@ -20,6 +20,11 @@
 
 using namespace DirectX;
 
+TRACELOGGING_DEFINE_PROVIDER(g_hDxRenderProvider,
+                             "Microsoft.Windows.Terminal.Renderer.DirectX",
+                             // {c93e739e-ae50-5a14-78e7-f171e947535d}
+                             (0xc93e739e, 0xae50, 0x5a14, 0x78, 0xe7, 0xf1, 0x71, 0xe9, 0x47, 0x53, 0x5d), );
+
 // Quad where we draw the terminal.
 // pos is world space coordinates where origin is at the center of screen.
 // tex is texel coordinates where origin is top left.
@@ -80,8 +85,11 @@ DxEngine::DxEngine() :
     _dpi{ USER_DEFAULT_SCREEN_DPI },
     _scale{ 1.0f },
     _chainMode{ SwapChainMode::ForComposition },
-    _customRenderer{ ::Microsoft::WRL::Make<CustomTextRenderer>() }
+    _customRenderer{ ::Microsoft::WRL::Make<CustomTextRenderer>() },
+    _hasEverPresented{ false }
 {
+    TraceLoggingRegister(g_hDxRenderProvider);
+
     THROW_IF_FAILED(D2D1CreateFactory(D2D1_FACTORY_TYPE_SINGLE_THREADED, IID_PPV_ARGS(&_d2dFactory)));
 
     THROW_IF_FAILED(DWriteCreateFactory(
@@ -99,6 +107,8 @@ DxEngine::DxEngine() :
 DxEngine::~DxEngine()
 {
     _ReleaseDeviceResources();
+
+    TraceLoggingUnregister(g_hDxRenderProvider);
 }
 
 // Routine Description:
@@ -144,6 +154,10 @@ DxEngine::~DxEngine()
     {
         _ReleaseDeviceResources();
     }
+    else
+    {
+        RETURN_IF_FAILED(_CreateDeviceResources(true));
+    }
 
     return S_OK;
 }
@@ -322,7 +336,7 @@ HRESULT DxEngine::_SetupTerminalEffects()
 // You can find out how to install it here:
 // https://docs.microsoft.com/en-us/windows/uwp/gaming/use-the-directx-runtime-and-visual-studio-graphics-diagnostic-features
                               // clang-format on
-                              // D3D11_CREATE_DEVICE_DEBUG |
+                              D3D11_CREATE_DEVICE_DEBUG |
                               D3D11_CREATE_DEVICE_SINGLETHREADED;
 
     const std::array<D3D_FEATURE_LEVEL, 5> FeatureLevels{ D3D_FEATURE_LEVEL_11_1,
@@ -376,8 +390,7 @@ HRESULT DxEngine::_SetupTerminalEffects()
         {
             switch (_chainMode)
             {
-            case SwapChainMode::ForHwnd:
+            case SwapChainMode::ForHwnd: {
-            {
                 // use the HWND's dimensions for the swap chain dimensions.
                 RECT rect = { 0 };
                 RETURN_IF_WIN32_BOOL_FALSE(GetClientRect(_hwndTarget, &rect));
@@ -406,8 +419,7 @@ HRESULT DxEngine::_SetupTerminalEffects()
 
                 break;
             }
-            case SwapChainMode::ForComposition:
+            case SwapChainMode::ForComposition: {
-            {
                 // Use the given target size for compositions.
                 SwapChainDesc.Width = _displaySizePixels.cx;
                 SwapChainDesc.Height = _displaySizePixels.cy;
@@ -427,6 +439,8 @@ HRESULT DxEngine::_SetupTerminalEffects()
                 THROW_HR(E_NOTIMPL);
             }
 
+            _hasEverPresented = false;
+
             if (_retroTerminalEffects)
             {
                 const HRESULT hr = _SetupTerminalEffects();
@@ -518,6 +532,9 @@ HRESULT DxEngine::_SetupTerminalEffects()
 
             RETURN_IF_FAILED(sc2->SetMatrixTransform(&inverseScale));
         }
+
+        _hasEverPresented = false;
+
         return S_OK;
     }
     CATCH_RETURN();
@@ -546,6 +563,7 @@ void DxEngine::_ReleaseDeviceResources() noexcept
 
         _dxgiSurface.Reset();
         _dxgiSwapChain.Reset();
+        _hasEverPresented = false;
 
         if (nullptr != _d3dDeviceContext.Get())
         {
@@ -777,8 +795,7 @@ Microsoft::WRL::ComPtr<IDXGISwapChain1> DxEngine::GetSwapChain()
 {
     switch (_chainMode)
     {
-    case SwapChainMode::ForHwnd:
+    case SwapChainMode::ForHwnd: {
-    {
         RECT clientRect = { 0 };
         LOG_IF_WIN32_BOOL_FALSE(GetClientRect(_hwndTarget, &clientRect));
 
@@ -788,8 +805,7 @@ Microsoft::WRL::ComPtr<IDXGISwapChain1> DxEngine::GetSwapChain()
 
         return clientSize;
     }
-    case SwapChainMode::ForComposition:
+    case SwapChainMode::ForComposition: {
-    {
         SIZE size = _sizeTarget;
         size.cx = static_cast<LONG>(size.cx * _scale);
         size.cy = static_cast<LONG>(size.cy * _scale);
@@ -922,9 +938,20 @@ void DxEngine::_InvalidOr(RECT rc) noexcept
 // - Any DirectX error, a memory error, etc.
 [[nodiscard]] HRESULT DxEngine::StartPaint() noexcept
 {
-    FAIL_FAST_IF_FAILED(InvalidateAll());
     RETURN_HR_IF(E_NOT_VALID_STATE, _isPainting); // invalid to start a paint while painting.
 
+    // TODO: not sure why this is happening at all, but it is
+    RETURN_HR_IF(S_FALSE, IsRectEmpty(&_invalidRect) && _invalidScroll.cx == 0 && _invalidScroll.cy == 0);
+
+    TraceLoggingWrite(g_hDxRenderProvider,
+                      "Invalid",
+                      TraceLoggingInt32(_invalidRect.bottom - _invalidRect.top, "InvalidHeight"),
+                      TraceLoggingInt32(_invalidRect.right - _invalidRect.left, "InvalidWidth"),
+                      TraceLoggingInt32(_invalidRect.left, "InvalidX"),
+                      TraceLoggingInt32(_invalidRect.top, "InvalidY"),
+                      TraceLoggingInt32(_invalidScroll.cx, "ScrollWidth"),
+                      TraceLoggingInt32(_invalidScroll.cy, "ScrollHeight"));
+
     if (_isEnabled)
     {
         try
@@ -987,18 +1014,18 @@ void DxEngine::_InvalidOr(RECT rc) noexcept
 
         if (SUCCEEDED(hr))
         {
+            _presentDirty = _invalidRect;
+
+            _presentParams.DirtyRectsCount = 1;
+            _presentParams.pDirtyRects = &_presentDirty;
+
             if (_invalidScroll.cy != 0 || _invalidScroll.cx != 0)
             {
-                _presentDirty = _invalidRect;
-
                 const RECT display = _GetDisplayRect();
                 SubtractRect(&_presentScroll, &display, &_presentDirty);
                 _presentOffset.x = _invalidScroll.cx;
                 _presentOffset.y = _invalidScroll.cy;
 
-                _presentParams.DirtyRectsCount = 1;
-                _presentParams.pDirtyRects = &_presentDirty;
-
                 _presentParams.pScrollOffset = &_presentOffset;
                 _presentParams.pScrollRect = &_presentScroll;
 
@@ -1076,8 +1103,15 @@ void DxEngine::_InvalidOr(RECT rc) noexcept
         {
             HRESULT hr = S_OK;
 
-            hr = _dxgiSwapChain->Present(1, 0);
+            if (_hasEverPresented)
-            /*hr = _dxgiSwapChain->Present1(1, 0, &_presentParams);*/
+            {
+                hr = _dxgiSwapChain->Present1(1, 0, &_presentParams);
+            }
+            else
+            {
+                hr = _dxgiSwapChain->Present(1, 0);
+                _hasEverPresented = true;
+            }
 
             if (FAILED(hr))
             {
@@ -1126,19 +1160,23 @@ void DxEngine::_InvalidOr(RECT rc) noexcept
 // - S_OK
 [[nodiscard]] HRESULT DxEngine::PaintBackground() noexcept
 {
+    const auto rect = D2D1::RectF(static_cast<float>(_invalidRect.left),
+                                  static_cast<float>(_invalidRect.top),
+                                  static_cast<float>(_invalidRect.right),
+                                  static_cast<float>(_invalidRect.bottom));
+
     switch (_chainMode)
     {
     case SwapChainMode::ForHwnd:
-        _d2dRenderTarget->FillRectangle(D2D1::RectF(static_cast<float>(_invalidRect.left),
-                                                    static_cast<float>(_invalidRect.top),
-                                                    static_cast<float>(_invalidRect.right),
-                                                    static_cast<float>(_invalidRect.bottom)),
-                                        _d2dBrushBackground.Get());
-        break;
     case SwapChainMode::ForComposition:
-        D2D1_COLOR_F nothing = { 0 };
+        _d2dRenderTarget->PushAxisAlignedClip(rect,
+                                              D2D1_ANTIALIAS_MODE::D2D1_ANTIALIAS_MODE_PER_PRIMITIVE);
 
+        D2D1_COLOR_F nothing = { 0 };
         _d2dRenderTarget->Clear(nothing);
+
+        _d2dRenderTarget->PopAxisAlignedClip();
+
         break;
     }
 
@@ -1345,8 +1383,7 @@ enum class CursorPaintType
 
     switch (options.cursorType)
     {
-    case CursorType::Legacy:
+    case CursorType::Legacy: {
-    {
         // Enforce min/max cursor height
         ULONG ulHeight = std::clamp(options.ulCursorHeightPercent, s_ulMinCursorHeightPercent, s_ulMaxCursorHeightPercent);
 
@@ -1354,26 +1391,22 @@ enum class CursorPaintType
         rect.top = rect.bottom - ulHeight;
         break;
     }
-    case CursorType::VerticalBar:
+    case CursorType::VerticalBar: {
-    {
         // It can't be wider than one cell or we'll have problems in invalidation, so restrict here.
         // It's either the left + the proposed width from the ease of access setting, or
         // it's the right edge of the block cursor as a maximum.
         rect.right = std::min(rect.right, rect.left + options.cursorPixelWidth);
         break;
     }
-    case CursorType::Underscore:
+    case CursorType::Underscore: {
-    {
         rect.top = rect.bottom - 1;
         break;
     }
-    case CursorType::EmptyBox:
+    case CursorType::EmptyBox: {
-    {
         paintType = CursorPaintType::Outline;
         break;
     }
-    case CursorType::FullBox:
+    case CursorType::FullBox: {
-    {
         break;
     }
     default:
@@ -1390,13 +1423,11 @@ enum class CursorPaintType
 
     switch (paintType)
     {
-    case CursorPaintType::Fill:
+    case CursorPaintType::Fill: {
-    {
         _d2dRenderTarget->FillRectangle(rect, brush.Get());
         break;
     }
-    case CursorPaintType::Outline:
+    case CursorPaintType::Outline: {
-    {
         // DrawRectangle in straddles physical pixels in an attempt to draw a line
         // between them. To avoid this, bump the rectangle around by half the stroke width.
         rect.top += 0.5f;
@@ -2050,12 +2081,10 @@ float DxEngine::GetScaling() const noexcept
 
     switch (_chainMode)
     {
-    case SwapChainMode::ForHwnd:
+    case SwapChainMode::ForHwnd: {
-    {
         return D2D1::ColorF(rgb);
     }
-    case SwapChainMode::ForComposition:
+    case SwapChainMode::ForComposition: {
-    {
         // Get the A value we've snuck into the highest byte
         const BYTE a = ((color >> 24) & 0xFF);
         const float aFloat = a / 255.0f;

src/renderer/dx/DxRenderer.hpp

@@ -25,6 +25,10 @@
 
 #include "../../types/inc/Viewport.hpp"
 
+#include <TraceLoggingProvider.h>
+
+TRACELOGGING_DECLARE_PROVIDER(g_hDxRenderProvider);
+
 namespace Microsoft::Console::Render
 {
     class DxEngine final : public RenderEngineBase
@@ -145,6 +149,7 @@ namespace Microsoft::Console::Render
 
         void _InvalidOffset(POINT pt);
 
+        bool _hasEverPresented;
         bool _presentReady;
         RECT _presentDirty;
         RECT _presentScroll;