Absolutely everything is awful and I have no idea how to fix this. Better just not then I guess

Manually picking a tab color doesn't seem to work right, so let's sort that out.

  But otherwise the selected / deselected tab colors seem right. It was the `SelectedBackgroundPath.Fill` that was basically totally overriding the background color.

  No idea where to go next with the manual color overrides. Maybe I missed something. Probably did.

build/packages.config

@@ -4,7 +4,8 @@
   <package id="Microsoft.Taef" version="10.60.210621002" targetFramework="native" />
   <package id="Microsoft.Internal.PGO-Helpers.Cpp" version="0.2.34" targetFramework="native" />
   <!-- This cannot be included in another project that depends on XAML (as it would be a duplicate package ID) -->
-  <package id="Microsoft.UI.Xaml" version="2.7.3" targetFramework="native" />
+  <package id="Microsoft.UI.Xaml" version="2.8.2" targetFramework="native" />
+  <package id="Microsoft.Web.WebView2" version="1.0.1587.40" targetFramework="native" />
   <package id="Microsoft.Debugging.Tools.PdbStr" version="20220617.1556.0" targetFramework="native" />
   <package id="Microsoft.Debugging.Tools.SrcTool" version="20220617.1556.0" targetFramework="native" />
 </packages>

build/pipelines/templates/helix-runtests-job.yml

@@ -14,8 +14,8 @@ parameters:
   platform: ''
   # if 'useBuildOutputFromBuildId' is set, we will default to using a build from this pipeline:
   useBuildOutputFromPipeline: $(System.DefinitionId)
-  openHelixTargetQueues: 'windows.10.amd64.client21h1.open.xaml'
-  closedHelixTargetQueues: 'windows.10.amd64.client21h1.xaml'
+  openHelixTargetQueues: 'windows.11.amd64.client.open.reunion'
+  closedHelixTargetQueues: 'windows.11.amd64.client.reunion'
 
 jobs:
 - job: ${{ parameters.name }}

common.openconsole.props

@@ -18,14 +18,14 @@
 
     This version should be tracked in all project packages.config files for projects that depend on Xaml.
     -->
-    <TerminalMUXVersion>2.7.3-prerelease.220816001</TerminalMUXVersion>
+    <TerminalMUXVersion>2.8.2-prerelease.220830001</TerminalMUXVersion>
     <!--
     For the Windows 11-specific build, we're targeting the public version of Microsoft.UI.Xaml.
     This version emits a package dependency instead of embedding the dependency in our own package.
 
     This version should be tracked in build/packages.config.
     -->
-    <TerminalMUXVersion Condition="'$(TerminalTargetWindowsVersion)'=='Win11'">2.7.3</TerminalMUXVersion>
+    <TerminalMUXVersion Condition="'$(TerminalTargetWindowsVersion)'=='Win11'">2.8.2</TerminalMUXVersion>
   </PropertyGroup>
 
 </Project>

src/buffer/out/cursor.cpp

@@ -286,9 +286,9 @@ void Cursor::CopyProperties(const Cursor& OtherCursor) noexcept
     _cursorType = OtherCursor._cursorType;
 }
 
-void Cursor::DelayEOLWrap(const til::point coordDelayedAt) noexcept
+void Cursor::DelayEOLWrap() noexcept
 {
-    _coordDelayedAt = coordDelayedAt;
+    _coordDelayedAt = _cPosition;
     _fDelayedEolWrap = true;
 }
 

src/buffer/out/cursor.h

@@ -76,7 +76,7 @@ public:
 
     void CopyProperties(const Cursor& OtherCursor) noexcept;
 
-    void DelayEOLWrap(const til::point coordDelayedAt) noexcept;
+    void DelayEOLWrap() noexcept;
     void ResetDelayEOLWrap() noexcept;
     til::point GetDelayedAtPosition() const noexcept;
     bool IsDelayedEOLWrap() const noexcept;

src/cascadia/TerminalApp/TabBase.cpp

@@ -462,8 +462,46 @@ namespace winrt::TerminalApp::implementation
         // In GH#11294 we thought we'd still need to set
         // TabViewItemHeaderBackground manually, but GH#11382 discovered that
         // Background() was actually okay after all.
-        TabViewItem().Background(deselectedTabBrush);
-        TabViewItem().Resources().Insert(winrt::box_value(L"TabViewItemHeaderBackgroundSelected"), selectedTabBrush);
+        // TabViewItem().Background(deselectedTabBrush);
+        // TabViewItem().Background(selectedTabBrush); // This does seem to colorize the selected tab correctly, but not the deselected one
+        // TabViewItem().Resources().Insert(winrt::box_value(L"TabViewItemHeaderBackground"), deselectedTabBrush);
+        // TabViewItem().Resources().Insert(winrt::box_value(L"TabViewItemHeaderBackgroundSelected"), selectedTabBrush);
+
+        // {
+        //     // Attempt 1
+        //     TabViewItem().Background(selectedTabBrush); // This does seem to colorize the selected tab correctly, but not the deselected one
+        //     TabViewItem().Resources().Insert(winrt::box_value(L"TabViewItemHeaderBackground"), deselectedTabBrush);
+        //     TabViewItem().Resources().Insert(winrt::box_value(L"TabViewItemHeaderBackgroundSelected"), selectedTabBrush);
+        //     // This sorta worked, but deselected tabs were broken.
+        //     // Everything was the selected tab color. yikes.
+        // }
+        // {
+        //     // Attempt 2
+        //     TabViewItem().Resources().Insert(winrt::box_value(L"TabViewItemHeaderBackground"), deselectedTabBrush);
+        //     TabViewItem().Resources().Insert(winrt::box_value(L"TabViewItemHeaderBackgroundSelected"), selectedTabBrush);
+        //     // This didn't work at all. Deslected tabs and selected ones were not colorized.
+        //     // * Selected were that default blackish color
+        //     // * Deselected were transparent
+        // }
+
+        // {
+        //     // Attempt 3
+        //     TabViewItem().Resources().Insert(winrt::box_value(L"TabViewItemHeaderBackground"), deselectedTabBrush);
+        //     TabViewItem().Resources().Insert(winrt::box_value(L"TabViewItemHeaderBackgroundSelected"), selectedTabBrush);
+        //     // Additionally,
+        //     // <Setter Target="TabContainer.Background" Value="{ThemeResource TabViewItemHeaderBackgroundSelected}" />
+        //     // in <VisualState x:Name="Selected">.Setters
+        // }
+
+        {
+            // Attempt 4
+            // TabViewItem().Resources().Insert(winrt::box_value(L"TabViewItemHeaderBackground"), deselectedTabBrush);
+            TabViewItem().Resources().Insert(winrt::box_value(L"TabViewItemHeaderBackgroundSelected"), selectedTabBrush);
+
+            TabViewItem().Background(deselectedTabBrush);
+            // Additionally, changes to the XAML
+        }
+
         TabViewItem().Resources().Insert(winrt::box_value(L"TabViewItemHeaderBackgroundPointerOver"), hoverTabBrush);
         TabViewItem().Resources().Insert(winrt::box_value(L"TabViewItemHeaderBackgroundPressed"), selectedTabBrush);
 
@@ -471,7 +509,7 @@ namespace winrt::TerminalApp::implementation
         // when the TabViewItem isn't selected, but not when it is hovered,
         // pressed, dragged, or selected, so we'll need to just set them all
         // anyways.
-        TabViewItem().Foreground(deselectedFontBrush);
+        // TabViewItem().Foreground(deselectedFontBrush); // TODO!
         TabViewItem().Resources().Insert(winrt::box_value(L"TabViewItemHeaderForeground"), deselectedFontBrush);
         TabViewItem().Resources().Insert(winrt::box_value(L"TabViewItemHeaderForegroundSelected"), fontBrush);
         TabViewItem().Resources().Insert(winrt::box_value(L"TabViewItemHeaderForegroundPointerOver"), fontBrush);
@@ -537,7 +575,7 @@ namespace winrt::TerminalApp::implementation
         // GH#11382 DON'T set the background to null. If you do that, then the
         // tab won't be hit testable at all. Transparent, however, is a totally
         // valid hit test target. That makes sense.
-        TabViewItem().Background(WUX::Media::SolidColorBrush{ Windows::UI::Colors::Transparent() });
+        // TabViewItem().Background(WUX::Media::SolidColorBrush{ Windows::UI::Colors::Transparent() });
 
         _RefreshVisualState();
     }

src/cascadia/TerminalApp/TabRowControl.xaml

@@ -14,6 +14,15 @@
 
     <!--  GH#13143: Make sure that the Background is actually TabViewBackground here, not Transparent. This is load bearing, for showTabsInTitlebar=false.  -->
 
+
+    <ContentPresenter.Resources>
+        <ResourceDictionary>
+            <ResourceDictionary.MergedDictionaries>
+                <ResourceDictionary Source="TabViewStyles.xaml" />
+            </ResourceDictionary.MergedDictionaries>
+        </ResourceDictionary>
+    </ContentPresenter.Resources>
+
     <mux:TabView x:Name="TabView"
                  VerticalAlignment="Bottom"
                  HorizontalContentAlignment="Stretch"
@@ -22,6 +31,7 @@
                  CanDragTabs="True"
                  CanReorderTabs="True"
                  IsAddTabButtonVisible="false"
+                 Style="{StaticResource MyTabViewStyle}"
                  TabWidthMode="Equal">
 
         <mux:TabView.TabStripHeader>

src/cascadia/TerminalApp/TerminalAppLib.vcxproj

@@ -60,6 +60,9 @@
     <Page Include="CommandPalette.xaml">
       <SubType>Designer</SubType>
     </Page>
+    <Page Include="TabViewStyles.xaml">
+      <SubType>Designer</SubType>
+    </Page>
   </ItemGroup>
   <!-- ========================= Headers ======================== -->
   <ItemGroup>

src/cascadia/TerminalApp/packages.config

@@ -1,4 +1,4 @@
 <?xml version="1.0" encoding="utf-8"?>
 <packages>
-  <package id="Microsoft.UI.Xaml" version="2.7.3-prerelease.220816001" targetFramework="native" />
+  <package id="Microsoft.UI.Xaml" version="2.8.2-prerelease.220830001" targetFramework="native" />
 </packages>

src/cascadia/TerminalSettingsEditor/Appearances.h

@@ -96,7 +96,6 @@ namespace winrt::Microsoft::Terminal::Settings::Editor::implementation
     private:
         Model::AppearanceConfig _appearance;
         winrt::hstring _lastBgImagePath;
-        float _cachedLineHeight = 0;
     };
 
     struct Appearances : AppearancesT<Appearances>

src/cascadia/WindowsTerminal/WindowsTerminal.vcxproj

@@ -27,6 +27,14 @@
   <Import Project="$(OpenConsoleDir)src\common.nugetversions.props" />
   <Import Project="$(OpenConsoleDir)src\cppwinrt.build.pre.props" />
 
+  <PropertyGroup>
+    <!-- This magic property will let WebVeiw2 know that it needs to reveal its
+    winmd to us. Without it, the WebView2 targets don't think we're a WinRT
+    context, and our build will fail when MUX fails midl compile (due to missing
+    webview components). -->
+    <WebView2UseWinRT>true</WebView2UseWinRT>
+  </PropertyGroup>
+
   <ItemDefinitionGroup>
     <ClCompile>
       <SDLCheck>true</SDLCheck>

src/common.nugetversions.targets

@@ -54,6 +54,7 @@
 
     <!-- WinUI (which depends on WebView2 as of 2.8.0) -->
     <Import Project="$(MSBuildThisFileDirectory)..\packages\Microsoft.UI.Xaml.$(TerminalMUXVersion)\build\native\Microsoft.UI.Xaml.targets" Condition="'$(TerminalMUX)' == 'true' and Exists('$(MSBuildThisFileDirectory)..\packages\Microsoft.UI.Xaml.$(TerminalMUXVersion)\build\native\Microsoft.UI.Xaml.targets')" />
+    <Import Project="$(MSBuildThisFileDirectory)..\packages\Microsoft.Web.WebView2.1.0.1587.40\build\native\Microsoft.Web.WebView2.targets" Condition="'$(TerminalMUX)' == 'true' and Exists('$(MSBuildThisFileDirectory)..\packages\Microsoft.Web.WebView2.1.0.1587.40\build\native\Microsoft.Web.WebView2.targets')" />
 
     <!-- WIL (so widely used that this one does not have a TerminalWIL opt-in property; it is automatic) -->
     <Import Project="$(MSBuildThisFileDirectory)..\packages\Microsoft.Windows.ImplementationLibrary.1.0.220201.1\build\native\Microsoft.Windows.ImplementationLibrary.targets" Condition="Exists('$(MSBuildThisFileDirectory)..\packages\Microsoft.Windows.ImplementationLibrary.1.0.220201.1\build\native\Microsoft.Windows.ImplementationLibrary.targets')" />
@@ -87,6 +88,7 @@
 
     <!-- WinUI (which depends on WebView2 as of 2.8.0) -->
     <Error Condition="'$(TerminalMUX)' == 'true' AND !Exists('$(MSBuildThisFileDirectory)..\packages\Microsoft.UI.Xaml.$(TerminalMUXVersion)\build\native\Microsoft.UI.Xaml.targets')" Text="$([System.String]::Format('$(ErrorText)', '$(MSBuildThisFileDirectory)..\packages\Microsoft.UI.Xaml.$(TerminalMUXVersion)\build\native\Microsoft.UI.Xaml.targets'))" />
+    <Error Condition="'$(TerminalMUX)' == 'true' AND !Exists('$(MSBuildThisFileDirectory)..\packages\Microsoft.Web.WebView2.1.0.1587.40\build\native\Microsoft.Web.WebView2.targets')" Text="$([System.String]::Format('$(ErrorText)', '$(MSBuildThisFileDirectory)..\packages\Microsoft.Web.WebView2.1.0.1587.40\build\native\Microsoft.Web.WebView2.targets'))" />
 
     <!-- WIL (so widely used that this one does not have a TerminalWIL opt-in property; it is automatic) -->
     <Error Condition="!Exists('$(MSBuildThisFileDirectory)..\packages\Microsoft.Windows.ImplementationLibrary.1.0.220201.1\build\native\Microsoft.Windows.ImplementationLibrary.targets')" Text="$([System.String]::Format('$(ErrorText)', '$(MSBuildThisFileDirectory)..\packages\Microsoft.Windows.ImplementationLibrary.1.0.220201.1\build\native\Microsoft.Windows.ImplementationLibrary.targets'))" />

src/host/dbcs.cpp

@@ -177,77 +177,3 @@ BOOL IsAvailableEastAsianCodePage(const UINT uiCodePage)
         return false;
     }
 }
-
-_Ret_range_(0, cbAnsi)
-    ULONG TranslateUnicodeToOem(_In_reads_(cchUnicode) PCWCHAR pwchUnicode,
-                                const ULONG cchUnicode,
-                                _Out_writes_bytes_(cbAnsi) PCHAR pchAnsi,
-                                const ULONG cbAnsi,
-                                _Out_ std::unique_ptr<IInputEvent>& partialEvent)
-{
-    const auto& gci = ServiceLocator::LocateGlobals().getConsoleInformation();
-    const auto TmpUni = new (std::nothrow) WCHAR[cchUnicode];
-    if (TmpUni == nullptr)
-    {
-        return 0;
-    }
-
-    memcpy(TmpUni, pwchUnicode, cchUnicode * sizeof(WCHAR));
-
-    CHAR AsciiDbcs[2];
-    AsciiDbcs[1] = 0;
-
-    ULONG i, j;
-    for (i = 0, j = 0; i < cchUnicode && j < cbAnsi; i++, j++)
-    {
-        if (IsGlyphFullWidth(TmpUni[i]))
-        {
-            const auto NumBytes = sizeof(AsciiDbcs);
-            ConvertToOem(gci.CP, &TmpUni[i], 1, (LPSTR)&AsciiDbcs[0], NumBytes);
-            if (IsDBCSLeadByteConsole(AsciiDbcs[0], &gci.CPInfo))
-            {
-                if (j < cbAnsi - 1)
-                { // -1 is safe DBCS in buffer
-                    pchAnsi[j] = AsciiDbcs[0];
-                    j++;
-                    pchAnsi[j] = AsciiDbcs[1];
-                    AsciiDbcs[1] = 0;
-                }
-                else
-                {
-                    pchAnsi[j] = AsciiDbcs[0];
-                    break;
-                }
-            }
-            else
-            {
-                pchAnsi[j] = AsciiDbcs[0];
-                AsciiDbcs[1] = 0;
-            }
-        }
-        else
-        {
-            ConvertToOem(gci.CP, &TmpUni[i], 1, &pchAnsi[j], 1);
-        }
-    }
-
-    if (AsciiDbcs[1])
-    {
-        try
-        {
-            auto keyEvent = std::make_unique<KeyEvent>();
-            if (keyEvent.get())
-            {
-                keyEvent->SetCharData(AsciiDbcs[1]);
-                partialEvent.reset(static_cast<IInputEvent* const>(keyEvent.release()));
-            }
-        }
-        catch (...)
-        {
-            LOG_HR(wil::ResultFromCaughtException());
-        }
-    }
-
-    delete[] TmpUni;
-    return j;
-}

src/host/dbcs.h

@@ -29,10 +29,3 @@ bool IsDBCSLeadByteConsole(const CHAR ch, const CPINFO* const pCPInfo);
 BYTE CodePageToCharSet(const UINT uiCodePage);
 
 BOOL IsAvailableEastAsianCodePage(const UINT uiCodePage);
-
-_Ret_range_(0, cbAnsi)
-    ULONG TranslateUnicodeToOem(_In_reads_(cchUnicode) PCWCHAR pwchUnicode,
-                                const ULONG cchUnicode,
-                                _Out_writes_bytes_(cbAnsi) PCHAR pchAnsi,
-                                const ULONG cbAnsi,
-                                _Out_ std::unique_ptr<IInputEvent>& partialEvent);

src/host/directio.cpp

@@ -169,75 +169,21 @@ void EventsToUnicode(_Inout_ std::deque<std::unique_ptr<IInputEvent>>& inEvents,
         LockConsole();
         auto Unlock = wil::scope_exit([&] { UnlockConsole(); });
 
-        std::deque<std::unique_ptr<IInputEvent>> partialEvents;
-        if (!IsUnicode)
-        {
-            if (inputBuffer.IsReadPartialByteSequenceAvailable())
-            {
-                partialEvents.push_back(inputBuffer.FetchReadPartialByteSequence(IsPeek));
-            }
-        }
-
-        size_t amountToRead;
-        if (FAILED(SizeTSub(eventReadCount, partialEvents.size(), &amountToRead)))
-        {
-            return STATUS_INTEGER_OVERFLOW;
-        }
-        std::deque<std::unique_ptr<IInputEvent>> readEvents;
-        auto Status = inputBuffer.Read(readEvents,
-                                       amountToRead,
-                                       IsPeek,
-                                       true,
-                                       IsUnicode,
-                                       false);
+        const auto Status = inputBuffer.Read(outEvents,
+                                             eventReadCount,
+                                             IsPeek,
+                                             true,
+                                             IsUnicode,
+                                             false);
 
         if (CONSOLE_STATUS_WAIT == Status)
         {
-            FAIL_FAST_IF(!(readEvents.empty()));
             // If we're told to wait until later, move all of our context
             // to the read data object and send it back up to the server.
             waiter = std::make_unique<DirectReadData>(&inputBuffer,
                                                       &readHandleState,
                                                       eventReadCount,
-                                                      std::move(partialEvents));
-        }
-        else if (SUCCEEDED_NTSTATUS(Status))
-        {
-            // split key events to oem chars if necessary
-            if (!IsUnicode)
-            {
-                try
-                {
-                    SplitToOem(readEvents);
-                }
-                CATCH_LOG();
-            }
-
-            // combine partial and readEvents
-            while (!partialEvents.empty())
-            {
-                readEvents.push_front(std::move(partialEvents.back()));
-                partialEvents.pop_back();
-            }
-
-            // move events over
-            for (size_t i = 0; i < eventReadCount; ++i)
-            {
-                if (readEvents.empty())
-                {
-                    break;
-                }
-                outEvents.push_back(std::move(readEvents.front()));
-                readEvents.pop_front();
-            }
-
-            // store partial event if necessary
-            if (!readEvents.empty())
-            {
-                inputBuffer.StoreReadPartialByteSequence(std::move(readEvents.front()));
-                readEvents.pop_front();
-                FAIL_FAST_IF(!(readEvents.empty()));
-            }
+                                                      std::move(outEvents));
         }
         return Status;
     }

src/host/inputBuffer.cpp

@@ -3,12 +3,13 @@
 
 #include "precomp.h"
 #include "inputBuffer.hpp"
-#include "dbcs.h"
+
 #include "stream.h"
 #include "../types/inc/GlyphWidth.hpp"
 
-#include <functional>
+#include <til/bytes.h>
 
+#include "misc.h"
 #include "../interactivity/inc/ServiceLocator.hpp"
 
 #define INPUT_BUFFER_DEFAULT_INPUT_MODE (ENABLE_LINE_INPUT | ENABLE_PROCESSED_INPUT | ENABLE_ECHO_INPUT | ENABLE_MOUSE_INPUT)
@@ -36,59 +37,199 @@ InputBuffer::InputBuffer() :
     fInComposition = false;
 }
 
-// Routine Description:
-// - This routine frees the resources associated with an input buffer.
-// Arguments:
-// - None
-// Return Value:
-InputBuffer::~InputBuffer() = default;
-
-// Routine Description:
-// - checks if any partial char data is available for reading operation
-// Arguments:
-// - None
-// Return Value:
-// - true if partial char data is available, false otherwise
-bool InputBuffer::IsReadPartialByteSequenceAvailable()
+// Transfer as many `wchar_t`s from source over to the `char`/`wchar_t` buffer `target`. After it returns,
+// the start of the `source` and `target` slices will be offset by as many bytes as have been copied
+// over, so that if you call this function again it'll continue copying from wherever it left off.
+//
+// It performs the necessary `WideCharToMultiByte` conversion if `isUnicode` is `false`.
+// Since not all converted `char`s might fit into `target` it'll cache the remainder. The next
+// time this function is called those cached `char`s will then be the first to be copied over.
+void InputBuffer::Consume(bool isUnicode, std::wstring_view& source, std::span<char>& target)
 {
-    return _readPartialByteSequence.get() != nullptr;
-}
+    // `_cachedTextReaderA` might still contain target data from a previous invocation.
+    // `ConsumeCached` calls `_switchReadingMode` for us.
+    ConsumeCached(isUnicode, target);
 
-// Routine Description:
-// - reads any read partial char data available
-// Arguments:
-// - peek - if true, data will not be removed after being fetched
-// Return Value:
-// - the partial char data. may be nullptr if no data is available
-std::unique_ptr<IInputEvent> InputBuffer::FetchReadPartialByteSequence(_In_ bool peek)
-{
-    if (!IsReadPartialByteSequenceAvailable())
+    if (source.empty() || target.empty())
     {
-        return std::unique_ptr<IInputEvent>();
+        return;
     }
 
-    if (peek)
+    if (isUnicode)
     {
-        return IInputEvent::Create(_readPartialByteSequence->ToInputRecord());
+        // The above block should either leave `target` or `_cachedTextReaderW` empty (or both).
+        // If we're here, `_cachedTextReaderW` should be empty.
+        assert(_cachedTextReaderW.empty());
+
+        til::bytes_transfer(target, source);
     }
     else
     {
-        std::unique_ptr<IInputEvent> outEvent;
-        outEvent.swap(_readPartialByteSequence);
-        return outEvent;
+        // The above block should either leave `target` or `_cachedTextReaderA` empty (or both).
+        // If we're here, `_cachedTextReaderA` should be empty.
+        assert(_cachedTextReaderA.empty());
+
+        const auto cp = ServiceLocator::LocateGlobals().getConsoleInformation().CP;
+
+        // Fast path: Batch convert all data in case the user provided buffer is large enough.
+        {
+            const auto wideLength = gsl::narrow<ULONG>(source.size());
+            const auto narrowLength = gsl::narrow<ULONG>(target.size());
+
+            const auto length = WideCharToMultiByte(cp, 0, source.data(), wideLength, target.data(), narrowLength, nullptr, nullptr);
+            if (length > 0)
+            {
+                source = {};
+                til::bytes_advance(target, gsl::narrow_cast<size_t>(length));
+                return;
+            }
+
+            const auto error = GetLastError();
+            THROW_HR_IF(HRESULT_FROM_WIN32(error), error != ERROR_INSUFFICIENT_BUFFER);
+        }
+
+        // Slow path: Character-wise conversion otherwise. We do this in order to only
+        // consume as many characters from `source` as necessary to fill `target`.
+        {
+            size_t read = 0;
+
+            for (const auto& wch : source)
+            {
+                char buffer[8];
+                const auto length = WideCharToMultiByte(cp, 0, &wch, 1, &buffer[0], sizeof(buffer), nullptr, nullptr);
+                THROW_LAST_ERROR_IF(length <= 0);
+
+                std::string_view slice{ &buffer[0], gsl::narrow_cast<size_t>(length) };
+                til::bytes_transfer(target, slice);
+
+                ++read;
+
+                if (!slice.empty())
+                {
+                    _cachedTextA = slice;
+                    _cachedTextReaderA = _cachedTextA;
+                    break;
+                }
+            }
+
+            source = source.substr(read);
+        }
     }
 }
 
-// Routine Description:
-// - stores partial read char data for a later read. will overwrite
-// any previously stored data.
-// Arguments:
-// - event - The event to store
-// Return Value:
-// - None
-void InputBuffer::StoreReadPartialByteSequence(std::unique_ptr<IInputEvent> event)
+// Same as `Consume`, but without any `source` characters.
+void InputBuffer::ConsumeCached(bool isUnicode, std::span<char>& target)
 {
-    _readPartialByteSequence.swap(event);
+    _switchReadingMode(isUnicode ? ReadingMode::StringW : ReadingMode::StringA);
+
+    if (isUnicode)
+    {
+        if (!_cachedTextReaderW.empty())
+        {
+            til::bytes_transfer(target, _cachedTextReaderW);
+
+            if (_cachedTextReaderW.empty())
+            {
+                // This is just so that we release memory eagerly.
+                _cachedTextW = std::wstring{};
+            }
+        }
+    }
+    else
+    {
+        if (!_cachedTextReaderA.empty())
+        {
+            til::bytes_transfer(target, _cachedTextReaderA);
+
+            if (_cachedTextReaderA.empty())
+            {
+                // This is just so that we release memory eagerly.
+                _cachedTextA = std::string{};
+            }
+        }
+    }
+}
+
+void InputBuffer::Cache(std::wstring_view source)
+{
+    const auto off = _cachedTextW.empty() ? 0 : _cachedTextReaderW.data() - _cachedTextW.data();
+    _cachedTextW.append(source);
+    _cachedTextReaderW = std::wstring_view{ _cachedTextW }.substr(off);
+}
+
+// Moves up to `count`, previously cached events into `target`.
+size_t InputBuffer::ConsumeCached(bool isUnicode, size_t count, InputEventQueue& target)
+{
+    _switchReadingMode(isUnicode ? ReadingMode::InputEventsW : ReadingMode::InputEventsA);
+
+    size_t i = 0;
+
+    while (i < count && !_cachedInputEvents.empty())
+    {
+        target.push_back(std::move(_cachedInputEvents.front()));
+        _cachedInputEvents.pop_front();
+        i++;
+    }
+
+    return i;
+}
+
+// Copies up to `count`, previously cached events into `target`.
+size_t InputBuffer::PeekCached(bool isUnicode, size_t count, InputEventQueue& target)
+{
+    _switchReadingMode(isUnicode ? ReadingMode::InputEventsW : ReadingMode::InputEventsA);
+
+    size_t i = 0;
+
+    for (const auto& e : _cachedInputEvents)
+    {
+        if (i >= count)
+        {
+            break;
+        }
+
+        target.push_back(IInputEvent::Create(e->ToInputRecord()));
+        i++;
+    }
+
+    return i;
+}
+
+// Trims `source` to have a size below or equal to `expectedSourceSize` by
+// storing any extra events in `_cachedInputEvents` for later retrieval.
+void InputBuffer::Cache(bool isUnicode, InputEventQueue& source, size_t expectedSourceSize)
+{
+    _switchReadingMode(isUnicode ? ReadingMode::InputEventsW : ReadingMode::InputEventsA);
+
+    if (source.size() > expectedSourceSize)
+    {
+        _cachedInputEvents.insert(
+            _cachedInputEvents.end(),
+            std::make_move_iterator(source.begin() + expectedSourceSize),
+            std::make_move_iterator(source.end()));
+        source.resize(expectedSourceSize);
+    }
+}
+
+void InputBuffer::_switchReadingMode(ReadingMode mode)
+{
+    if (_readingMode != mode)
+    {
+        _switchReadingModeSlowPath(mode);
+    }
+}
+
+void InputBuffer::_switchReadingModeSlowPath(ReadingMode mode)
+{
+    _cachedTextA = std::string{};
+    _cachedTextReaderA = {};
+
+    _cachedTextW = std::wstring{};
+    _cachedTextReaderW = {};
+
+    _cachedInputEvents = std::deque<std::unique_ptr<IInputEvent>>{};
+
+    _readingMode = mode;
 }
 
 // Routine Description:
@@ -261,47 +402,105 @@ void InputBuffer::PassThroughWin32MouseRequest(bool enable)
                                          const bool WaitForData,
                                          const bool Unicode,
                                          const bool Stream)
+try
 {
-    try
+    assert(OutEvents.empty());
+
+    const auto cp = ServiceLocator::LocateGlobals().getConsoleInformation().CP;
+
+    if (Peek)
     {
-        if (_storage.empty())
+        PeekCached(Unicode, AmountToRead, OutEvents);
+    }
+    else
+    {
+        ConsumeCached(Unicode, AmountToRead, OutEvents);
+    }
+
+    auto it = _storage.begin();
+    const auto end = _storage.end();
+
+    while (it != end && OutEvents.size() < AmountToRead)
+    {
+        auto event = IInputEvent::Create((*it)->ToInputRecord());
+
+        if (event->EventType() == InputEventType::KeyEvent)
         {
-            if (!WaitForData)
+            const auto keyEvent = static_cast<KeyEvent*>(event.get());
+            WORD repeat = 1;
+
+            // for stream reads we need to split any key events that have been coalesced
+            if (Stream)
             {
-                return STATUS_SUCCESS;
+                repeat = keyEvent->GetRepeatCount();
+                keyEvent->SetRepeatCount(1);
+            }
+
+            if (Unicode)
+            {
+                do
+                {
+                    OutEvents.push_back(std::make_unique<KeyEvent>(*keyEvent));
+                    repeat--;
+                } while (repeat > 0 && OutEvents.size() < AmountToRead);
+            }
+            else
+            {
+                const auto wch = keyEvent->GetCharData();
+
+                char buffer[8];
+                const auto length = WideCharToMultiByte(cp, 0, &wch, 1, &buffer[0], sizeof(buffer), nullptr, nullptr);
+                THROW_LAST_ERROR_IF(length <= 0);
+
+                const std::string_view str{ &buffer[0], gsl::narrow_cast<size_t>(length) };
+
+                do
+                {
+                    for (const auto& ch : str)
+                    {
+                        auto tempEvent = std::make_unique<KeyEvent>(*keyEvent);
+                        tempEvent->SetCharData(ch);
+                        OutEvents.push_back(std::move(tempEvent));
+                    }
+                    repeat--;
+                } while (repeat > 0 && OutEvents.size() < AmountToRead);
+            }
+
+            if (repeat && !Peek)
+            {
+                const auto originalKeyEvent = static_cast<KeyEvent*>((*it).get());
+                originalKeyEvent->SetRepeatCount(repeat);
+                break;
             }
-            return CONSOLE_STATUS_WAIT;
         }
-
-        // read from buffer
-        std::deque<std::unique_ptr<IInputEvent>> events;
-        size_t eventsRead;
-        bool resetWaitEvent;
-        _ReadBuffer(events,
-                    AmountToRead,
-                    eventsRead,
-                    Peek,
-                    resetWaitEvent,
-                    Unicode,
-                    Stream);
-
-        // copy events to outEvents
-        while (!events.empty())
+        else
         {
-            OutEvents.push_back(std::move(events.front()));
-            events.pop_front();
+            OutEvents.push_back(std::move(event));
         }
 
-        if (resetWaitEvent)
-        {
-            ServiceLocator::LocateGlobals().hInputEvent.ResetEvent();
-        }
-        return STATUS_SUCCESS;
+        ++it;
     }
-    catch (...)
+
+    if (!Peek)
     {
-        return NTSTATUS_FROM_HRESULT(wil::ResultFromCaughtException());
+        _storage.erase(_storage.begin(), it);
     }
+
+    Cache(Unicode, OutEvents, AmountToRead);
+
+    if (OutEvents.empty())
+    {
+        return WaitForData ? CONSOLE_STATUS_WAIT : STATUS_SUCCESS;
+    }
+    if (_storage.empty())
+    {
+        ServiceLocator::LocateGlobals().hInputEvent.ResetEvent();
+    }
+    return STATUS_SUCCESS;
+}
+catch (...)
+{
+    return NTSTATUS_FROM_HRESULT(wil::ResultFromCaughtException());
 }
 
 // Routine Description:
@@ -349,130 +548,6 @@ void InputBuffer::PassThroughWin32MouseRequest(bool enable)
     return Status;
 }
 
-// Routine Description:
-// - This routine reads from a buffer. It does the buffer manipulation.
-// Arguments:
-// - outEvents - where read events are placed
-// - readCount - amount of events to read
-// - eventsRead - where to store number of events read
-// - peek - if true , don't remove data from buffer, just copy it.
-// - resetWaitEvent - on exit, true if buffer became empty.
-// - unicode - true if read should be done in unicode mode
-// - streamRead - true if read should unpack KeyEvents that have a >1 repeat count. readCount must be 1 if streamRead is true.
-// Return Value:
-// - <none>
-// Note:
-// - The console lock must be held when calling this routine.
-void InputBuffer::_ReadBuffer(_Out_ std::deque<std::unique_ptr<IInputEvent>>& outEvents,
-                              const size_t readCount,
-                              _Out_ size_t& eventsRead,
-                              const bool peek,
-                              _Out_ bool& resetWaitEvent,
-                              const bool unicode,
-                              const bool streamRead)
-{
-    // when stream reading, the previous behavior was to only allow reading of a single
-    // event at a time.
-    FAIL_FAST_IF(streamRead && readCount != 1);
-
-    resetWaitEvent = false;
-
-    std::deque<std::unique_ptr<IInputEvent>> readEvents;
-    // we need another var to keep track of how many we've read
-    // because dbcs records count for two when we aren't doing a
-    // unicode read but the eventsRead count should return the number
-    // of events actually put into outRecords.
-    size_t virtualReadCount = 0;
-
-    while (!_storage.empty() && virtualReadCount < readCount)
-    {
-        auto performNormalRead = true;
-        // for stream reads we need to split any key events that have been coalesced
-        if (streamRead)
-        {
-            if (_storage.front()->EventType() == InputEventType::KeyEvent)
-            {
-                const auto pKeyEvent = static_cast<KeyEvent* const>(_storage.front().get());
-                if (pKeyEvent->GetRepeatCount() > 1)
-                {
-                    // split the key event
-                    auto streamKeyEvent = std::make_unique<KeyEvent>(*pKeyEvent);
-                    streamKeyEvent->SetRepeatCount(1);
-                    readEvents.push_back(std::move(streamKeyEvent));
-                    pKeyEvent->SetRepeatCount(pKeyEvent->GetRepeatCount() - 1);
-                    performNormalRead = false;
-                }
-            }
-        }
-
-        if (performNormalRead)
-        {
-            readEvents.push_back(std::move(_storage.front()));
-            _storage.pop_front();
-        }
-
-        ++virtualReadCount;
-        if (!unicode)
-        {
-            if (readEvents.back()->EventType() == InputEventType::KeyEvent)
-            {
-                const auto pKeyEvent = static_cast<const KeyEvent* const>(readEvents.back().get());
-                if (IsGlyphFullWidth(pKeyEvent->GetCharData()))
-                {
-                    ++virtualReadCount;
-                }
-            }
-        }
-    }
-
-    // the amount of events that were actually read
-    eventsRead = readEvents.size();
-
-    // copy the events back if we were supposed to peek
-    if (peek)
-    {
-        if (streamRead)
-        {
-            // we need to check and see if the event was split from a coalesced key event
-            // or if it was unrelated to the current front event in storage
-            if (!readEvents.empty() &&
-                !_storage.empty() &&
-                readEvents.back()->EventType() == InputEventType::KeyEvent &&
-                _storage.front()->EventType() == InputEventType::KeyEvent &&
-                _CanCoalesce(static_cast<const KeyEvent&>(*readEvents.back()),
-                             static_cast<const KeyEvent&>(*_storage.front())))
-            {
-                auto& keyEvent = static_cast<KeyEvent&>(*_storage.front());
-                keyEvent.SetRepeatCount(keyEvent.GetRepeatCount() + 1);
-            }
-            else
-            {
-                _storage.push_front(IInputEvent::Create(readEvents.back()->ToInputRecord()));
-            }
-        }
-        else
-        {
-            for (auto it = readEvents.crbegin(); it != readEvents.crend(); ++it)
-            {
-                _storage.push_front(IInputEvent::Create((*it)->ToInputRecord()));
-            }
-        }
-    }
-
-    // move events read to proper deque
-    while (!readEvents.empty())
-    {
-        outEvents.push_back(std::move(readEvents.front()));
-        readEvents.pop_front();
-    }
-
-    // signal if we emptied the buffer
-    if (_storage.empty())
-    {
-        resetWaitEvent = true;
-    }
-}
-
 // Routine Description:
 // -  Writes events to the beginning of the input buffer.
 // Arguments:

src/host/inputBuffer.hpp

@@ -41,12 +41,15 @@ public:
     bool fInComposition; // specifies if there's an ongoing text composition
 
     InputBuffer();
-    ~InputBuffer();
 
-    // storage API for partial dbcs bytes being read from the buffer
-    bool IsReadPartialByteSequenceAvailable();
-    std::unique_ptr<IInputEvent> FetchReadPartialByteSequence(_In_ bool peek);
-    void StoreReadPartialByteSequence(std::unique_ptr<IInputEvent> event);
+    // String oriented APIs
+    void Consume(bool isUnicode, std::wstring_view& source, std::span<char>& target);
+    void ConsumeCached(bool isUnicode, std::span<char>& target);
+    void Cache(std::wstring_view source);
+    // INPUT_RECORD oriented APIs
+    size_t ConsumeCached(bool isUnicode, size_t count, InputEventQueue& target);
+    size_t PeekCached(bool isUnicode, size_t count, InputEventQueue& target);
+    void Cache(bool isUnicode, InputEventQueue& source, size_t expectedSourceSize);
 
     // storage API for partial dbcs bytes being written to the buffer
     bool IsWritePartialByteSequenceAvailable();
@@ -84,8 +87,22 @@ public:
     void PassThroughWin32MouseRequest(bool enable);
 
 private:
+    enum class ReadingMode : uint8_t
+    {
+        StringA,
+        StringW,
+        InputEventsA,
+        InputEventsW,
+    };
+
+    std::string _cachedTextA;
+    std::string_view _cachedTextReaderA;
+    std::wstring _cachedTextW;
+    std::wstring_view _cachedTextReaderW;
+    std::deque<std::unique_ptr<IInputEvent>> _cachedInputEvents;
+    ReadingMode _readingMode = ReadingMode::StringA;
+
     std::deque<std::unique_ptr<IInputEvent>> _storage;
-    std::unique_ptr<IInputEvent> _readPartialByteSequence;
     std::unique_ptr<IInputEvent> _writePartialByteSequence;
     Microsoft::Console::VirtualTerminal::TerminalInput _termInput;
     Microsoft::Console::Render::VtEngine* _pTtyConnection;
@@ -96,13 +113,8 @@ private:
     // Otherwise, we should be calling them.
     bool _vtInputShouldSuppress{ false };
 
-    void _ReadBuffer(_Out_ std::deque<std::unique_ptr<IInputEvent>>& outEvents,
-                     const size_t readCount,
-                     _Out_ size_t& eventsRead,
-                     const bool peek,
-                     _Out_ bool& resetWaitEvent,
-                     const bool unicode,
-                     const bool streamRead);
+    void _switchReadingMode(ReadingMode mode);
+    void _switchReadingModeSlowPath(ReadingMode mode);
 
     void _WriteBuffer(_Inout_ std::deque<std::unique_ptr<IInputEvent>>& inRecords,
                       _Out_ size_t& eventsWritten,

src/host/misc.cpp

@@ -2,13 +2,12 @@
 // Licensed under the MIT license.
 
 #include "precomp.h"
-
 #include "misc.h"
+
+#include <til/unicode.h>
+
 #include "dbcs.h"
-
-#include "../types/inc/convert.hpp"
 #include "../types/inc/GlyphWidth.hpp"
-
 #include "../interactivity/inc/ServiceLocator.hpp"
 
 #pragma hdrstop
@@ -182,47 +181,6 @@ BOOL CheckBisectProcessW(const SCREEN_INFORMATION& ScreenInfo,
     }
 }
 
-// Routine Description:
-// - Converts all key events in the deque to the oem char data and adds
-// them back to events.
-// Arguments:
-// - events - on input the IInputEvents to convert. on output, the
-// converted input events
-// Note: may throw on error
-void SplitToOem(std::deque<std::unique_ptr<IInputEvent>>& events)
-{
-    const auto cp = ServiceLocator::LocateGlobals().getConsoleInformation().CP;
-    std::deque<std::unique_ptr<IInputEvent>> convertedEvents;
-
-    for (auto& currentEvent : events)
-    {
-        if (currentEvent->EventType() == InputEventType::KeyEvent)
-        {
-            const auto pKeyEvent = static_cast<const KeyEvent* const>(currentEvent.get());
-            const auto wch = pKeyEvent->GetCharData();
-
-            char buffer[8];
-            const auto length = WideCharToMultiByte(cp, 0, &wch, 1, &buffer[0], sizeof(buffer), nullptr, nullptr);
-            THROW_LAST_ERROR_IF(length <= 0);
-
-            const std::string_view str{ &buffer[0], gsl::narrow_cast<size_t>(length) };
-
-            for (const auto& ch : str)
-            {
-                auto tempEvent = std::make_unique<KeyEvent>(*pKeyEvent);
-                tempEvent->SetCharData(ch);
-                convertedEvents.push_back(std::move(tempEvent));
-            }
-        }
-        else
-        {
-            convertedEvents.push_back(std::move(currentEvent));
-        }
-    }
-
-    events = std::move(convertedEvents);
-}
-
 // Routine Description:
 // - Converts unicode characters to ANSI given a destination codepage
 // Arguments:

src/host/misc.h

@@ -44,8 +44,6 @@ int ConvertToOem(const UINT uiCodePage,
                  _Out_writes_(cchTarget) CHAR* const pchTarget,
                  const UINT cchTarget) noexcept;
 
-void SplitToOem(std::deque<std::unique_ptr<IInputEvent>>& events);
-
 int ConvertInputToUnicode(const UINT uiCodePage,
                           _In_reads_(cchSource) const CHAR* const pchSource,
                           const UINT cchSource,

src/host/readDataCooked.cpp

@@ -47,7 +47,7 @@ COOKED_READ_DATA::COOKED_READ_DATA(_In_ InputBuffer* const pInputBuffer,
                                    _In_ INPUT_READ_HANDLE_DATA* const pInputReadHandleData,
                                    SCREEN_INFORMATION& screenInfo,
                                    _In_ size_t UserBufferSize,
-                                   _In_ PWCHAR UserBuffer,
+                                   _In_ char* UserBuffer,
                                    _In_ ULONG CtrlWakeupMask,
                                    _In_ const std::wstring_view exeName,
                                    _In_ const std::string_view initialData,
@@ -62,7 +62,7 @@ COOKED_READ_DATA::COOKED_READ_DATA(_In_ InputBuffer* const pInputBuffer,
     _exeName{ exeName },
     _pdwNumBytes{ nullptr },
 
-    _commandHistory{ CommandHistory::s_Find((HANDLE)pClientProcess) },
+    _commandHistory{ CommandHistory::s_Find(pClientProcess) },
     _controlKeyState{ 0 },
     _ctrlWakeupMask{ CtrlWakeupMask },
     _visibleCharCount{ 0 },
@@ -881,10 +881,10 @@ size_t COOKED_READ_DATA::SavePromptToUserBuffer(const size_t cch)
         try
         {
             const auto& gci = ServiceLocator::LocateGlobals().getConsoleInformation();
-            const auto wstr = ConvertToW(gci.CP, { reinterpret_cast<char*>(_userBuffer), cch });
-            const auto copyAmount = std::min(wstr.size(), _userBufferSize / sizeof(wchar_t));
-            std::copy_n(wstr.begin(), copyAmount, _userBuffer);
-            return copyAmount * sizeof(wchar_t);
+            const auto wstr = ConvertToW(gci.CP, { _userBuffer, cch });
+            const auto copyAmount = std::min(wstr.size() * sizeof(wchar_t), _userBufferSize);
+            std::copy_n(reinterpret_cast<const char*>(wstr.data()), copyAmount, _userBuffer);
+            return copyAmount;
         }
         CATCH_LOG();
     }
@@ -1003,211 +1003,55 @@ void COOKED_READ_DATA::SavePendingInput(const size_t index, const bool multiline
 // - Status code that indicates success, out of memory, etc.
 [[nodiscard]] NTSTATUS COOKED_READ_DATA::_handlePostCharInputLoop(const bool isUnicode, size_t& numBytes, ULONG& controlKeyState) noexcept
 {
+    std::span writer{ _userBuffer, _userBufferSize };
+    std::wstring_view input{ _backupLimit, _bytesRead / sizeof(wchar_t) };
     DWORD LineCount = 1;
 
     if (_echoInput)
     {
-        // Figure out where real string ends (at carriage return or end of buffer).
-        auto StringPtr = _backupLimit;
-        auto StringLength = _bytesRead / sizeof(WCHAR);
-        auto FoundCR = false;
-        for (size_t i = 0; i < StringLength; i++)
-        {
-            if (*StringPtr++ == UNICODE_CARRIAGERETURN)
-            {
-                StringLength = i;
-                FoundCR = true;
-                break;
-            }
-        }
-
-        if (FoundCR)
+        const auto idx = input.find(UNICODE_CARRIAGERETURN);
+        if (idx != decltype(input)::npos)
         {
             if (_commandHistory)
             {
-                // add to command line recall list if we have a history list.
                 auto& gci = ServiceLocator::LocateGlobals().getConsoleInformation();
-                LOG_IF_FAILED(_commandHistory->Add({ _backupLimit, StringLength },
-                                                   WI_IsFlagSet(gci.Flags, CONSOLE_HISTORY_NODUP)));
+                LOG_IF_FAILED(_commandHistory->Add({ _backupLimit, idx }, WI_IsFlagSet(gci.Flags, CONSOLE_HISTORY_NODUP)));
             }
 
-            Tracing::s_TraceCookedRead(_clientProcess,
-                                       _backupLimit,
-                                       base::saturated_cast<ULONG>(StringLength));
-
-            // check for alias
+            Tracing::s_TraceCookedRead(_clientProcess, _backupLimit, base::saturated_cast<ULONG>(idx));
             ProcessAliases(LineCount);
+
+            if (LineCount > 1)
+            {
+                input = input.substr(0, idx + 1);
+            }
         }
     }
 
-    auto fAddDbcsLead = false;
-    size_t NumBytes = 0;
-    // at this point, a->NumBytes contains the number of bytes in
-    // the UNICODE string read.  UserBufferSize contains the converted
-    // size of the app's buffer.
-    if (_bytesRead > _userBufferSize || LineCount > 1)
+    GetInputBuffer()->Consume(isUnicode, input, writer);
+
+    if (!input.empty())
     {
         if (LineCount > 1)
         {
-            PWSTR Tmp;
-            if (!isUnicode)
-            {
-                if (_pInputBuffer->IsReadPartialByteSequenceAvailable())
-                {
-                    fAddDbcsLead = true;
-                    auto event = GetInputBuffer()->FetchReadPartialByteSequence(false);
-                    const auto pKeyEvent = static_cast<const KeyEvent* const>(event.get());
-                    *_userBuffer = static_cast<char>(pKeyEvent->GetCharData());
-                    _userBuffer++;
-                    _userBufferSize -= sizeof(wchar_t);
-                }
-
-                NumBytes = 0;
-                for (Tmp = _backupLimit;
-                     *Tmp != UNICODE_LINEFEED && _userBufferSize / sizeof(WCHAR) > NumBytes;
-                     Tmp++)
-                {
-                    NumBytes += IsGlyphFullWidth(*Tmp) ? 2 : 1;
-                }
-            }
-
-            // clang-format off
-#pragma prefast(suppress: __WARNING_BUFFER_OVERFLOW, "LineCount > 1 means there's a UNICODE_LINEFEED")
-            // clang-format on
-            for (Tmp = _backupLimit; *Tmp != UNICODE_LINEFEED; Tmp++)
-            {
-                FAIL_FAST_IF(!(Tmp < (_backupLimit + _bytesRead)));
-            }
-
-            numBytes = (ULONG)(Tmp - _backupLimit + 1) * sizeof(*Tmp);
+            GetInputReadHandleData()->SaveMultilinePendingInput(input);
         }
         else
         {
-            if (!isUnicode)
-            {
-                PWSTR Tmp;
-
-                if (_pInputBuffer->IsReadPartialByteSequenceAvailable())
-                {
-                    fAddDbcsLead = true;
-                    auto event = GetInputBuffer()->FetchReadPartialByteSequence(false);
-                    const auto pKeyEvent = static_cast<const KeyEvent* const>(event.get());
-                    *_userBuffer = static_cast<char>(pKeyEvent->GetCharData());
-                    _userBuffer++;
-                    _userBufferSize -= sizeof(wchar_t);
-                }
-                NumBytes = 0;
-                auto NumToWrite = _bytesRead;
-                for (Tmp = _backupLimit;
-                     NumToWrite && _userBufferSize / sizeof(WCHAR) > NumBytes;
-                     Tmp++, NumToWrite -= sizeof(WCHAR))
-                {
-                    NumBytes += IsGlyphFullWidth(*Tmp) ? 2 : 1;
-                }
-            }
-            numBytes = _userBufferSize;
-        }
-
-        __analysis_assume(numBytes <= _userBufferSize);
-        memmove(_userBuffer, _backupLimit, numBytes);
-
-        const auto pInputReadHandleData = GetInputReadHandleData();
-        const std::wstring_view pending{ _backupLimit + (numBytes / sizeof(wchar_t)), (_bytesRead - numBytes) / sizeof(wchar_t) };
-        if (LineCount > 1)
-        {
-            pInputReadHandleData->SaveMultilinePendingInput(pending);
-        }
-        else
-        {
-            pInputReadHandleData->SavePendingInput(pending);
+            GetInputReadHandleData()->SavePendingInput(input);
         }
     }
-    else
-    {
-        if (!isUnicode)
-        {
-            PWSTR Tmp;
 
-            if (_pInputBuffer->IsReadPartialByteSequenceAvailable())
-            {
-                fAddDbcsLead = true;
-                auto event = GetInputBuffer()->FetchReadPartialByteSequence(false);
-                const auto pKeyEvent = static_cast<const KeyEvent* const>(event.get());
-                *_userBuffer = static_cast<char>(pKeyEvent->GetCharData());
-                _userBuffer++;
-                _userBufferSize -= sizeof(wchar_t);
-
-                if (_userBufferSize == 0)
-                {
-                    numBytes = 1;
-                    return STATUS_SUCCESS;
-                }
-            }
-            NumBytes = 0;
-            auto NumToWrite = _bytesRead;
-            for (Tmp = _backupLimit;
-                 NumToWrite && _userBufferSize / sizeof(WCHAR) > NumBytes;
-                 Tmp++, NumToWrite -= sizeof(WCHAR))
-            {
-                NumBytes += IsGlyphFullWidth(*Tmp) ? 2 : 1;
-            }
-        }
-
-        numBytes = _bytesRead;
-
-        if (numBytes > _userBufferSize)
-        {
-            return STATUS_BUFFER_OVERFLOW;
-        }
-
-        memmove(_userBuffer, _backupLimit, numBytes);
-    }
+    numBytes = _userBufferSize - writer.size();
     controlKeyState = _controlKeyState;
-
-    if (!isUnicode)
-    {
-        // if ansi, translate string.
-        std::unique_ptr<char[]> tempBuffer;
-        try
-        {
-            tempBuffer = std::make_unique<char[]>(NumBytes);
-        }
-        catch (...)
-        {
-            return STATUS_NO_MEMORY;
-        }
-
-        std::unique_ptr<IInputEvent> partialEvent;
-        numBytes = TranslateUnicodeToOem(_userBuffer,
-                                         gsl::narrow<ULONG>(numBytes / sizeof(wchar_t)),
-                                         tempBuffer.get(),
-                                         gsl::narrow<ULONG>(NumBytes),
-                                         partialEvent);
-
-        if (partialEvent.get())
-        {
-            GetInputBuffer()->StoreReadPartialByteSequence(std::move(partialEvent));
-        }
-
-        if (numBytes > _userBufferSize)
-        {
-            return STATUS_BUFFER_OVERFLOW;
-        }
-
-        memmove(_userBuffer, tempBuffer.get(), numBytes);
-        if (fAddDbcsLead)
-        {
-            numBytes++;
-        }
-    }
     return STATUS_SUCCESS;
 }
 
 void COOKED_READ_DATA::MigrateUserBuffersOnTransitionToBackgroundWait(const void* oldBuffer, void* newBuffer)
 {
     // See the comment in WaitBlock.cpp for more information.
-    if (_userBuffer == reinterpret_cast<const wchar_t*>(oldBuffer))
+    if (_userBuffer == oldBuffer)
     {
-        _userBuffer = reinterpret_cast<wchar_t*>(newBuffer);
+        _userBuffer = static_cast<char*>(newBuffer);
     }
 }

src/host/readDataCooked.hpp

@@ -37,7 +37,7 @@ public:
                      _In_ INPUT_READ_HANDLE_DATA* const pInputReadHandleData,
                      SCREEN_INFORMATION& screenInfo,
                      _In_ size_t UserBufferSize,
-                     _In_ PWCHAR UserBuffer,
+                     _In_ char* UserBuffer,
                      _In_ ULONG CtrlWakeupMask,
                      _In_ const std::wstring_view exeName,
                      _In_ const std::string_view initialData,
@@ -129,7 +129,7 @@ private:
     wchar_t* _backupLimit;
 
     size_t _userBufferSize; // doubled size in ansi case
-    wchar_t* _userBuffer;
+    char* _userBuffer;
 
     size_t* _pdwNumBytes;
 

src/host/readDataDirect.cpp

@@ -10,9 +10,7 @@
 
 // Routine Description:
 // - Constructs direct read data class to hold context across sessions
-// generally when there's not enough data to return. Also used a bit
-// internally to just pass some information along the stack
-// (regardless of wait necessity).
+// generally when there's not enough data to return.
 // Arguments:
 // - pInputBuffer - Buffer that data will be read from.
 // - pInputReadHandleData - Context stored across calls from the same
@@ -27,17 +25,10 @@ DirectReadData::DirectReadData(_In_ InputBuffer* const pInputBuffer,
                                const size_t eventReadCount,
                                _In_ std::deque<std::unique_ptr<IInputEvent>> partialEvents) :
     ReadData(pInputBuffer, pInputReadHandleData),
-    _eventReadCount{ eventReadCount },
-    _partialEvents{ std::move(partialEvents) },
-    _outEvents{}
+    _eventReadCount{ eventReadCount }
 {
 }
 
-// Routine Description:
-// - Destructs a read data class.
-// - Decrements count of readers waiting on the given handle.
-DirectReadData::~DirectReadData() = default;
-
 // Routine Description:
 // - This routine is called to complete a direct read that blocked in
 //   ReadInputBuffer. The context of the read was saved in the DirectReadData
@@ -68,18 +59,17 @@ bool DirectReadData::Notify(const WaitTerminationReason TerminationReason,
                             _Out_ size_t* const pNumBytes,
                             _Out_ DWORD* const pControlKeyState,
                             _Out_ void* const pOutputData)
+try
 {
     FAIL_FAST_IF_NULL(pOutputData);
-
     FAIL_FAST_IF(_pInputReadHandleData->GetReadCount() == 0);
 
-    const auto& gci = Microsoft::Console::Interactivity::ServiceLocator::LocateGlobals().getConsoleInformation();
-    FAIL_FAST_IF(!gci.IsConsoleLocked());
+    assert(Microsoft::Console::Interactivity::ServiceLocator::LocateGlobals().getConsoleInformation().IsConsoleLocked());
 
     *pReplyStatus = STATUS_SUCCESS;
     *pControlKeyState = 0;
     *pNumBytes = 0;
-    auto retVal = true;
+
     std::deque<std::unique_ptr<IInputEvent>> readEvents;
 
     // If ctrl-c or ctrl-break was seen, ignore it.
@@ -88,105 +78,58 @@ bool DirectReadData::Notify(const WaitTerminationReason TerminationReason,
         return false;
     }
 
-    // check if a partial byte is already stored that we should read
-    if (!fIsUnicode &&
-        _pInputBuffer->IsReadPartialByteSequenceAvailable() &&
-        _eventReadCount == 1)
-    {
-        _partialEvents.push_back(_pInputBuffer->FetchReadPartialByteSequence(false));
-    }
-
     // See if called by CsrDestroyProcess or CsrDestroyThread
     // via ConsoleNotifyWaitBlock. If so, just decrement the ReadCount and return.
     if (WI_IsFlagSet(TerminationReason, WaitTerminationReason::ThreadDying))
     {
         *pReplyStatus = STATUS_THREAD_IS_TERMINATING;
+        return true;
     }
+
     // We must see if we were woken up because the handle is being
     // closed. If so, we decrement the read count. If it goes to
     // zero, we wake up the close thread. Otherwise, we wake up any
     // other thread waiting for data.
-    else if (WI_IsFlagSet(TerminationReason, WaitTerminationReason::HandleClosing))
+    if (WI_IsFlagSet(TerminationReason, WaitTerminationReason::HandleClosing))
     {
         *pReplyStatus = STATUS_ALERTED;
+        return true;
     }
-    else
+
+    // if we get to here, this routine was called either by the input
+    // thread or a write routine.  both of these callers grab the
+    // current console lock.
+
+    size_t amountToRead;
+    if (FAILED(SizeTSub(_eventReadCount, _outEvents.size(), &amountToRead)))
     {
-        // if we get to here, this routine was called either by the input
-        // thread or a write routine.  both of these callers grab the
-        // current console lock.
-
-        // calculate how many events we need to read
-        size_t amountAlreadyRead;
-        if (FAILED(SizeTAdd(_partialEvents.size(), _outEvents.size(), &amountAlreadyRead)))
-        {
-            *pReplyStatus = STATUS_INTEGER_OVERFLOW;
-            return retVal;
-        }
-        size_t amountToRead;
-        if (FAILED(SizeTSub(_eventReadCount, amountAlreadyRead, &amountToRead)))
-        {
-            *pReplyStatus = STATUS_INTEGER_OVERFLOW;
-            return retVal;
-        }
-
-        *pReplyStatus = _pInputBuffer->Read(readEvents,
-                                            amountToRead,
-                                            false,
-                                            false,
-                                            fIsUnicode,
-                                            false);
-
-        if (*pReplyStatus == CONSOLE_STATUS_WAIT)
-        {
-            retVal = false;
-        }
+        *pReplyStatus = STATUS_INTEGER_OVERFLOW;
+        return true;
     }
 
-    if (*pReplyStatus != CONSOLE_STATUS_WAIT)
+    *pReplyStatus = _pInputBuffer->Read(_outEvents,
+                                        amountToRead,
+                                        false,
+                                        false,
+                                        fIsUnicode,
+                                        false);
+
+    if (*pReplyStatus == CONSOLE_STATUS_WAIT)
     {
-        // split key events to oem chars if necessary
-        if (*pReplyStatus == STATUS_SUCCESS && !fIsUnicode)
-        {
-            try
-            {
-                SplitToOem(readEvents);
-            }
-            CATCH_LOG();
-        }
-
-        // combine partial and whole events
-        while (!_partialEvents.empty())
-        {
-            readEvents.push_front(std::move(_partialEvents.back()));
-            _partialEvents.pop_back();
-        }
-
-        // move read events to out storage
-        for (size_t i = 0; i < _eventReadCount; ++i)
-        {
-            if (readEvents.empty())
-            {
-                break;
-            }
-            _outEvents.push_back(std::move(readEvents.front()));
-            readEvents.pop_front();
-        }
-
-        // store partial event if necessary
-        if (!readEvents.empty())
-        {
-            _pInputBuffer->StoreReadPartialByteSequence(std::move(readEvents.front()));
-            readEvents.pop_front();
-            FAIL_FAST_IF(!(readEvents.empty()));
-        }
-
-        // move events to pOutputData
-        const auto pOutputDeque = reinterpret_cast<std::deque<std::unique_ptr<IInputEvent>>* const>(pOutputData);
-        *pNumBytes = _outEvents.size() * sizeof(INPUT_RECORD);
-        pOutputDeque->swap(_outEvents);
+        return false;
     }
-    return retVal;
+
+    // move events to pOutputData
+    const auto pOutputDeque = static_cast<std::deque<std::unique_ptr<IInputEvent>>* const>(pOutputData);
+    *pNumBytes = _outEvents.size() * sizeof(INPUT_RECORD);
+    *pOutputDeque = std::move(_outEvents);
+
+    return true;
+}
+catch (...)
+{
+    *pReplyStatus = wil::StatusFromCaughtException();
+    return true;
 }
 
 void DirectReadData::MigrateUserBuffersOnTransitionToBackgroundWait(const void* /*oldBuffer*/, void* /*newBuffer*/)

src/host/readDataDirect.hpp

@@ -38,8 +38,6 @@ public:
 
     DirectReadData(DirectReadData&&) = default;
 
-    ~DirectReadData() override;
-
     void MigrateUserBuffersOnTransitionToBackgroundWait(const void* oldBuffer, void* newBuffer) override;
     bool Notify(const WaitTerminationReason TerminationReason,
                 const bool fIsUnicode,
@@ -50,6 +48,5 @@ public:
 
 private:
     const size_t _eventReadCount;
-    std::deque<std::unique_ptr<IInputEvent>> _partialEvents;
     std::deque<std::unique_ptr<IInputEvent>> _outEvents;
 };

src/host/readDataRaw.cpp

@@ -31,7 +31,6 @@ RAW_READ_DATA::RAW_READ_DATA(_In_ InputBuffer* const pInputBuffer,
     _BufferSize{ BufferSize },
     _BufPtr{ THROW_HR_IF_NULL(E_INVALIDARG, BufPtr) }
 {
-    THROW_HR_IF(E_INVALIDARG, _BufferSize % sizeof(wchar_t) != 0);
     THROW_HR_IF(E_INVALIDARG, _BufferSize == 0);
 }
 
@@ -82,146 +81,50 @@ bool RAW_READ_DATA::Notify(const WaitTerminationReason TerminationReason,
     *pControlKeyState = 0;
 
     *pNumBytes = 0;
-    size_t NumBytes = 0;
-
-    PWCHAR lpBuffer;
-    auto RetVal = true;
-    auto fAddDbcsLead = false;
-    auto fSkipFinally = false;
 
     // If a ctrl-c is seen, don't terminate read. If ctrl-break is seen, terminate read.
     if (WI_IsFlagSet(TerminationReason, WaitTerminationReason::CtrlC))
     {
         return false;
     }
-    else if (WI_IsFlagSet(TerminationReason, WaitTerminationReason::CtrlBreak))
+
+    if (WI_IsFlagSet(TerminationReason, WaitTerminationReason::CtrlBreak))
     {
         *pReplyStatus = STATUS_ALERTED;
+        return true;
     }
+
     // See if we were called because the thread that owns this wait block is exiting.
-    else if (WI_IsFlagSet(TerminationReason, WaitTerminationReason::ThreadDying))
+    if (WI_IsFlagSet(TerminationReason, WaitTerminationReason::ThreadDying))
     {
         *pReplyStatus = STATUS_THREAD_IS_TERMINATING;
+        return true;
     }
+
     // We must see if we were woken up because the handle is being
     // closed. If so, we decrement the read count. If it goes to zero,
     // we wake up the close thread. Otherwise, we wake up any other
     // thread waiting for data.
-    else if (WI_IsFlagSet(TerminationReason, WaitTerminationReason::HandleClosing))
+    if (WI_IsFlagSet(TerminationReason, WaitTerminationReason::HandleClosing))
     {
         *pReplyStatus = STATUS_ALERTED;
-    }
-    else
-    {
-        // If we get to here, this routine was called either by the input
-        // thread or a write routine. Both of these callers grab the current
-        // console lock.
-
-        lpBuffer = _BufPtr;
-
-        if (!fIsUnicode && _pInputBuffer->IsReadPartialByteSequenceAvailable())
-        {
-            auto event = _pInputBuffer->FetchReadPartialByteSequence(false);
-            const auto pKeyEvent = static_cast<const KeyEvent* const>(event.get());
-            *lpBuffer = static_cast<char>(pKeyEvent->GetCharData());
-            _BufferSize -= sizeof(wchar_t);
-            *pReplyStatus = STATUS_SUCCESS;
-            fAddDbcsLead = true;
-
-            if (_BufferSize == 0)
-            {
-                *pNumBytes = 1;
-                RetVal = false;
-                fSkipFinally = true;
-            }
-        }
-        else
-        {
-            // This call to GetChar may block.
-            *pReplyStatus = GetChar(_pInputBuffer,
-                                    lpBuffer,
-                                    true,
-                                    nullptr,
-                                    nullptr,
-                                    nullptr);
-        }
-
-        if (FAILED_NTSTATUS(*pReplyStatus) || fSkipFinally)
-        {
-            if (*pReplyStatus == CONSOLE_STATUS_WAIT)
-            {
-                RetVal = false;
-            }
-        }
-        else
-        {
-            NumBytes += IsGlyphFullWidth(*lpBuffer) ? 2 : 1;
-            lpBuffer++;
-            *pNumBytes += sizeof(WCHAR);
-            while (*pNumBytes < _BufferSize)
-            {
-                // This call to GetChar won't block.
-                *pReplyStatus = GetChar(_pInputBuffer,
-                                        lpBuffer,
-                                        false,
-                                        nullptr,
-                                        nullptr,
-                                        nullptr);
-                if (FAILED_NTSTATUS(*pReplyStatus))
-                {
-                    *pReplyStatus = STATUS_SUCCESS;
-                    break;
-                }
-                NumBytes += IsGlyphFullWidth(*lpBuffer) ? 2 : 1;
-                lpBuffer++;
-                *pNumBytes += sizeof(WCHAR);
-            }
-        }
+        return true;
     }
 
-    // If the read was completed (status != wait), free the raw read data.
-    if (*pReplyStatus != CONSOLE_STATUS_WAIT &&
-        !fSkipFinally &&
-        !fIsUnicode)
-    {
-        // It's ansi, so translate the string.
-        std::unique_ptr<char[]> tempBuffer;
-        try
-        {
-            tempBuffer = std::make_unique<char[]>(NumBytes);
-        }
-        catch (...)
-        {
-            return true;
-        }
+    // If we get to here, this routine was called either by the input
+    // thread or a write routine. Both of these callers grab the current
+    // console lock.
 
-        lpBuffer = _BufPtr;
-        std::unique_ptr<IInputEvent> partialEvent;
-
-        *pNumBytes = TranslateUnicodeToOem(lpBuffer,
-                                           gsl::narrow<ULONG>(*pNumBytes / sizeof(wchar_t)),
-                                           tempBuffer.get(),
-                                           gsl::narrow<ULONG>(NumBytes),
-                                           partialEvent);
-        if (partialEvent.get())
-        {
-            _pInputBuffer->StoreReadPartialByteSequence(std::move(partialEvent));
-        }
-
-        memmove(lpBuffer, tempBuffer.get(), *pNumBytes);
-        if (fAddDbcsLead)
-        {
-            (*pNumBytes)++;
-        }
-    }
-    return RetVal;
+    std::span buffer{ reinterpret_cast<char*>(_BufPtr), _BufferSize };
+    *pReplyStatus = ReadCharacterInput(*_pInputBuffer, buffer, *pNumBytes, *_pInputReadHandleData, fIsUnicode);
+    return *pReplyStatus != CONSOLE_STATUS_WAIT;
 }
 
 void RAW_READ_DATA::MigrateUserBuffersOnTransitionToBackgroundWait(const void* oldBuffer, void* newBuffer)
 {
     // See the comment in WaitBlock.cpp for more information.
-    if (_BufPtr == reinterpret_cast<const wchar_t*>(oldBuffer))
+    if (_BufPtr == static_cast<const wchar_t*>(oldBuffer))
     {
-        _BufPtr = reinterpret_cast<wchar_t*>(newBuffer);
+        _BufPtr = static_cast<wchar_t*>(newBuffer);
     }
 }

src/host/stream.cpp

@@ -6,7 +6,6 @@
 #include "_stream.h"
 #include "stream.h"
 
-#include "dbcs.h"
 #include "handle.h"
 #include "misc.h"
 #include "readDataRaw.hpp"
@@ -285,151 +284,38 @@ til::CoordType RetrieveNumberOfSpaces(_In_ til::CoordType sOriginalCursorPositio
                                                 size_t& bytesRead,
                                                 INPUT_READ_HANDLE_DATA& readHandleState,
                                                 const bool unicode)
+try
 {
-    // TODO: MSFT: 18047766 - Correct this method to not play byte counting games.
-    auto fAddDbcsLead = FALSE;
-    size_t NumToWrite = 0;
-    size_t NumToBytes = 0;
-    auto pBuffer = reinterpret_cast<wchar_t*>(buffer.data());
-    auto bufferRemaining = buffer.size_bytes();
     bytesRead = 0;
 
-    if (buffer.size_bytes() < sizeof(wchar_t))
-    {
-        return STATUS_BUFFER_TOO_SMALL;
-    }
-
-    const auto pending = readHandleState.GetPendingInput();
-    auto pendingBytes = pending.size() * sizeof(wchar_t);
-    auto Tmp = pending.cbegin();
+    auto pending = readHandleState.GetPendingInput();
 
     if (readHandleState.IsMultilineInput())
     {
-        if (!unicode)
+        const auto idx = pending.find(UNICODE_LINEFEED);
+        if (idx != decltype(pending)::npos)
         {
-            if (inputBuffer.IsReadPartialByteSequenceAvailable())
-            {
-                auto event = inputBuffer.FetchReadPartialByteSequence(false);
-                const auto pKeyEvent = static_cast<const KeyEvent* const>(event.get());
-                *pBuffer = static_cast<char>(pKeyEvent->GetCharData());
-                ++pBuffer;
-                bufferRemaining -= sizeof(wchar_t);
-                pendingBytes -= sizeof(wchar_t);
-                fAddDbcsLead = TRUE;
-            }
-
-            if (pendingBytes == 0 || bufferRemaining == 0)
-            {
-                readHandleState.CompletePending();
-                bytesRead = 1;
-                return STATUS_SUCCESS;
-            }
-            else
-            {
-                for (NumToWrite = 0, Tmp = pending.cbegin(), NumToBytes = 0;
-                     NumToBytes < pendingBytes &&
-                     NumToBytes < bufferRemaining / sizeof(wchar_t) &&
-                     *Tmp != UNICODE_LINEFEED;
-                     Tmp++, NumToWrite += sizeof(wchar_t))
-                {
-                    NumToBytes += IsGlyphFullWidth(*Tmp) ? 2 : 1;
-                }
-            }
-        }
-
-        NumToWrite = 0;
-        Tmp = pending.cbegin();
-        while (NumToWrite < pendingBytes &&
-               *Tmp != UNICODE_LINEFEED)
-        {
-            ++Tmp;
-            NumToWrite += sizeof(wchar_t);
-        }
-
-        NumToWrite += sizeof(wchar_t);
-        if (NumToWrite > bufferRemaining)
-        {
-            NumToWrite = bufferRemaining;
+            // +1 to include the newline.
+            pending = pending.substr(0, idx + 1);
         }
     }
-    else
-    {
-        if (!unicode)
-        {
-            if (inputBuffer.IsReadPartialByteSequenceAvailable())
-            {
-                auto event = inputBuffer.FetchReadPartialByteSequence(false);
-                const auto pKeyEvent = static_cast<const KeyEvent* const>(event.get());
-                *pBuffer = static_cast<char>(pKeyEvent->GetCharData());
-                ++pBuffer;
-                bufferRemaining -= sizeof(wchar_t);
-                pendingBytes -= sizeof(wchar_t);
-                fAddDbcsLead = TRUE;
-            }
 
-            if (pendingBytes == 0)
-            {
-                readHandleState.CompletePending();
-                bytesRead = 1;
-                return STATUS_SUCCESS;
-            }
-            else
-            {
-                for (NumToWrite = 0, Tmp = pending.cbegin(), NumToBytes = 0;
-                     NumToBytes < pendingBytes && NumToBytes < bufferRemaining / sizeof(wchar_t);
-                     Tmp++, NumToWrite += sizeof(wchar_t))
-                {
-                    NumToBytes += IsGlyphFullWidth(*Tmp) ? 2 : 1;
-                }
-            }
-        }
+    std::span writer{ buffer };
+    inputBuffer.Consume(unicode, pending, writer);
 
-        NumToWrite = (bufferRemaining < pendingBytes) ? bufferRemaining : pendingBytes;
-    }
-
-    memmove(pBuffer, pending.data(), NumToWrite);
-    pendingBytes -= NumToWrite;
-    if (pendingBytes != 0)
-    {
-        std::wstring_view remainingPending{ pending.data() + (NumToWrite / sizeof(wchar_t)), pendingBytes / sizeof(wchar_t) };
-        readHandleState.UpdatePending(remainingPending);
-    }
-    else
+    if (pending.empty())
     {
         readHandleState.CompletePending();
     }
-
-    if (!unicode)
+    else
     {
-        // if ansi, translate string.  we allocated the capture buffer
-        // large enough to handle the translated string.
-        auto tempBuffer = std::make_unique<char[]>(NumToBytes);
-        std::unique_ptr<IInputEvent> partialEvent;
-
-        NumToWrite = TranslateUnicodeToOem(pBuffer,
-                                           gsl::narrow<ULONG>(NumToWrite / sizeof(wchar_t)),
-                                           tempBuffer.get(),
-                                           gsl::narrow<ULONG>(NumToBytes),
-                                           partialEvent);
-        if (partialEvent.get())
-        {
-            inputBuffer.StoreReadPartialByteSequence(std::move(partialEvent));
-        }
-
-        // clang-format off
-#pragma prefast(suppress: __WARNING_POTENTIAL_BUFFER_OVERFLOW_HIGH_PRIORITY, "This access is fine but prefast can't follow it, evidently")
-        // clang-format on
-        memmove(pBuffer, tempBuffer.get(), NumToWrite);
-
-        if (fAddDbcsLead)
-        {
-            NumToWrite++;
-        }
+        readHandleState.UpdatePending(pending);
     }
 
-    bytesRead = NumToWrite;
+    bytesRead = buffer.size() - writer.size();
     return STATUS_SUCCESS;
 }
+NT_CATCH_RETURN()
 
 // Routine Description:
 // - read in characters until the buffer is full or return is read.
@@ -478,7 +364,7 @@ til::CoordType RetrieveNumberOfSpaces(_In_ til::CoordType sOriginalCursorPositio
                                                                  &readHandleState, // pInputReadHandleData
                                                                  screenInfo, // pScreenInfo
                                                                  buffer.size_bytes(), // UserBufferSize
-                                                                 reinterpret_cast<wchar_t*>(buffer.data()), // UserBuffer
+                                                                 buffer.data(), // UserBuffer
                                                                  ctrlWakeupMask, // CtrlWakeupMask
                                                                  exeName, // exe name
                                                                  initialData,
@@ -521,140 +407,51 @@ til::CoordType RetrieveNumberOfSpaces(_In_ til::CoordType sOriginalCursorPositio
 // populated.
 // - STATUS_SUCCESS on success
 // - Other NTSTATUS codes as necessary
-[[nodiscard]] static NTSTATUS _ReadCharacterInput(InputBuffer& inputBuffer,
-                                                  std::span<char> buffer,
-                                                  size_t& bytesRead,
-                                                  INPUT_READ_HANDLE_DATA& readHandleState,
-                                                  const bool unicode,
-                                                  std::unique_ptr<IWaitRoutine>& waiter)
+[[nodiscard]] NTSTATUS ReadCharacterInput(InputBuffer& inputBuffer,
+                                          std::span<char> buffer,
+                                          size_t& bytesRead,
+                                          INPUT_READ_HANDLE_DATA& readHandleState,
+                                          const bool unicode)
+try
 {
-    size_t NumToWrite = 0;
-    auto addDbcsLead = false;
-    auto Status = STATUS_SUCCESS;
-    auto pBuffer = reinterpret_cast<wchar_t*>(buffer.data());
-    auto bufferRemaining = buffer.size_bytes();
+    UNREFERENCED_PARAMETER(readHandleState);
+
     bytesRead = 0;
 
-    if (buffer.size() < 1)
+    const auto charSize = unicode ? sizeof(wchar_t) : sizeof(char);
+    std::span writer{ buffer };
+
+    if (writer.size() < charSize)
     {
         return STATUS_BUFFER_TOO_SMALL;
     }
 
-    if (bytesRead < bufferRemaining)
+    inputBuffer.ConsumeCached(unicode, writer);
+
+    // We don't need to wait for input if `ConsumeCached` read something already, which is
+    // indicated by the writer having been advanced (= it's shorter than the original buffer).
+    auto wait = writer.size() == buffer.size();
+    auto status = STATUS_SUCCESS;
+
+    while (writer.size() >= charSize)
     {
-        auto pwchBufferTmp = pBuffer;
-
-        NumToWrite = 0;
-
-        if (!unicode && inputBuffer.IsReadPartialByteSequenceAvailable())
+        wchar_t wch;
+        status = GetChar(&inputBuffer, &wch, wait, nullptr, nullptr, nullptr);
+        if (!NT_SUCCESS(status))
         {
-            auto event = inputBuffer.FetchReadPartialByteSequence(false);
-            const auto pKeyEvent = static_cast<const KeyEvent* const>(event.get());
-            *pBuffer = static_cast<char>(pKeyEvent->GetCharData());
-            ++pBuffer;
-            bufferRemaining -= sizeof(wchar_t);
-            addDbcsLead = true;
-
-            if (bufferRemaining == 0)
-            {
-                bytesRead = 1;
-                return STATUS_SUCCESS;
-            }
-        }
-        else
-        {
-            Status = GetChar(&inputBuffer,
-                             pBuffer,
-                             true,
-                             nullptr,
-                             nullptr,
-                             nullptr);
+            break;
         }
 
-        if (Status == CONSOLE_STATUS_WAIT)
-        {
-            waiter = std::make_unique<RAW_READ_DATA>(&inputBuffer,
-                                                     &readHandleState,
-                                                     gsl::narrow<ULONG>(buffer.size_bytes()),
-                                                     reinterpret_cast<wchar_t*>(buffer.data()));
-        }
+        std::wstring_view wchView{ &wch, 1 };
+        inputBuffer.Consume(unicode, wchView, writer);
 
-        if (FAILED_NTSTATUS(Status))
-        {
-            bytesRead = 0;
-            return Status;
-        }
-
-        if (!addDbcsLead)
-        {
-            bytesRead += IsGlyphFullWidth(*pBuffer) ? 2 : 1;
-            NumToWrite += sizeof(wchar_t);
-            pBuffer++;
-        }
-
-        while (NumToWrite < static_cast<ULONG>(bufferRemaining))
-        {
-            Status = GetChar(&inputBuffer,
-                             pBuffer,
-                             false,
-                             nullptr,
-                             nullptr,
-                             nullptr);
-            if (FAILED_NTSTATUS(Status))
-            {
-                break;
-            }
-            bytesRead += IsGlyphFullWidth(*pBuffer) ? 2 : 1;
-            NumToWrite += sizeof(wchar_t);
-            pBuffer++;
-        }
-
-        // if ansi, translate string.  we allocated the capture buffer large enough to handle the translated string.
-        if (!unicode)
-        {
-            std::unique_ptr<char[]> tempBuffer;
-            try
-            {
-                tempBuffer = std::make_unique<char[]>(bytesRead);
-            }
-            catch (...)
-            {
-                return STATUS_NO_MEMORY;
-            }
-
-            pBuffer = pwchBufferTmp;
-            std::unique_ptr<IInputEvent> partialEvent;
-
-            bytesRead = TranslateUnicodeToOem(pBuffer,
-                                              gsl::narrow<ULONG>(NumToWrite / sizeof(wchar_t)),
-                                              tempBuffer.get(),
-                                              gsl::narrow<ULONG>(bytesRead),
-                                              partialEvent);
-
-            if (partialEvent.get())
-            {
-                inputBuffer.StoreReadPartialByteSequence(std::move(partialEvent));
-            }
-
-#pragma prefast(suppress : 26053 26015, "PREfast claims read overflow. *pReadByteCount is the exact size of tempBuffer as allocated above.")
-            memmove(pBuffer, tempBuffer.get(), bytesRead);
-
-            if (addDbcsLead)
-            {
-                ++bytesRead;
-            }
-        }
-        else
-        {
-            // We always return the byte count for A & W modes, so in
-            // the Unicode case where we didn't translate back, set
-            // the return to the byte count that we assembled while
-            // pulling characters from the internal buffers.
-            bytesRead = NumToWrite;
-        }
+        wait = false;
     }
-    return STATUS_SUCCESS;
+
+    bytesRead = buffer.size() - writer.size();
+    return status;
 }
+NT_CATCH_RETURN()
 
 // Routine Description:
 // - This routine reads in characters for stream input and does the
@@ -730,12 +527,16 @@ til::CoordType RetrieveNumberOfSpaces(_In_ til::CoordType sOriginalCursorPositio
         }
         else
         {
-            return _ReadCharacterInput(inputBuffer,
-                                       buffer,
-                                       bytesRead,
-                                       readHandleState,
-                                       unicode,
-                                       waiter);
+            const auto status = ReadCharacterInput(inputBuffer,
+                                                   buffer,
+                                                   bytesRead,
+                                                   readHandleState,
+                                                   unicode);
+            if (status == CONSOLE_STATUS_WAIT)
+            {
+                waiter = std::make_unique<RAW_READ_DATA>(&inputBuffer, &readHandleState, gsl::narrow<ULONG>(buffer.size()), reinterpret_cast<wchar_t*>(buffer.data()));
+            }
+            return status;
         }
     }
     CATCH_RETURN();

src/host/stream.h

@@ -29,6 +29,12 @@ Revision History:
                                _Out_opt_ bool* const pPopupKeys,
                                _Out_opt_ DWORD* const pdwKeyState) noexcept;
 
+[[nodiscard]] NTSTATUS ReadCharacterInput(InputBuffer& inputBuffer,
+                                          std::span<char> buffer,
+                                          size_t& bytesRead,
+                                          INPUT_READ_HANDLE_DATA& readHandleState,
+                                          const bool unicode);
+
 // Routine Description:
 // - This routine returns the total number of screen spaces the characters up to the specified character take up.
 til::CoordType RetrieveTotalNumberOfSpaces(const til::CoordType sOriginalCursorPositionX,

src/host/ut_host/CommandLineTests.cpp

@@ -77,7 +77,7 @@ class CommandLineTests
                             const size_t cchBuffer)
     {
         cookedReadData._commandHistory = pHistory;
-        cookedReadData._userBuffer = pBuffer;
+        cookedReadData._userBuffer = reinterpret_cast<char*>(pBuffer);
         cookedReadData._userBufferSize = cchBuffer * sizeof(wchar_t);
         cookedReadData._bufferSize = cchBuffer * sizeof(wchar_t);
         cookedReadData._backupLimit = pBuffer;

src/host/ut_host/Host.UnitTests.vcxproj

@@ -35,7 +35,6 @@
     <ClCompile Include="UtilsTests.cpp" />
     <ClCompile Include="Utf8ToWideCharParserTests.cpp" />
     <ClCompile Include="InputBufferTests.cpp" />
-    <ClCompile Include="ReadWaitTests.cpp" />
     <ClCompile Include="ViewportTests.cpp" />
     <ClCompile Include="VtIoTests.cpp" />
     <ClCompile Include="VtRendererTests.cpp" />

src/host/ut_host/Host.UnitTests.vcxproj.filters

@@ -54,9 +54,6 @@
     <ClCompile Include="InputBufferTests.cpp">
       <Filter>Source Files</Filter>
     </ClCompile>
-    <ClCompile Include="ReadWaitTests.cpp">
-      <Filter>Source Files</Filter>
-    </ClCompile>
     <ClCompile Include="ConsoleArgumentsTests.cpp">
       <Filter>Source Files</Filter>
     </ClCompile>

src/host/ut_host/InputBufferTests.cpp

@@ -367,7 +367,7 @@ class InputBufferTests
 
     TEST_METHOD(EmptyingBufferDuringReadSetsResetWaitEvent)
     {
-        Log::Comment(L"ResetWaitEvent should be true if a read to the buffer completely empties it");
+        Log::Comment(L"hInputEvent should be reset if a read to the buffer completely empties it");
 
         InputBuffer inputBuffer;
 
@@ -381,48 +381,62 @@ class InputBufferTests
         }
         VERIFY_IS_GREATER_THAN(inputBuffer.Write(inEvents), 0u);
 
-        // read one record, make sure ResetWaitEvent isn't set
-        std::deque<std::unique_ptr<IInputEvent>> outEvents;
-        size_t eventsRead = 0;
-        auto resetWaitEvent = false;
-        inputBuffer._ReadBuffer(outEvents,
-                                1,
-                                eventsRead,
-                                false,
-                                resetWaitEvent,
-                                true,
-                                false);
-        VERIFY_ARE_EQUAL(eventsRead, 1u);
-        VERIFY_IS_FALSE(!!resetWaitEvent);
+        auto& waitEvent = ServiceLocator::LocateGlobals().hInputEvent;
+        waitEvent.SetEvent();
 
-        // read the rest, resetWaitEvent should be set to true
+        // read one record, hInputEvent should still be signaled
+        std::deque<std::unique_ptr<IInputEvent>> outEvents;
+        VERIFY_NT_SUCCESS(inputBuffer.Read(outEvents,
+                                           1,
+                                           false,
+                                           false,
+                                           true,
+                                           false));
+        VERIFY_ARE_EQUAL(outEvents.size(), 1u);
+        VERIFY_IS_TRUE(waitEvent.is_signaled());
+
+        // read the rest, hInputEvent should be reset
+        waitEvent.SetEvent();
         outEvents.clear();
-        inputBuffer._ReadBuffer(outEvents,
-                                RECORD_INSERT_COUNT - 1,
-                                eventsRead,
-                                false,
-                                resetWaitEvent,
-                                true,
-                                false);
-        VERIFY_ARE_EQUAL(eventsRead, RECORD_INSERT_COUNT - 1);
-        VERIFY_IS_TRUE(!!resetWaitEvent);
+        VERIFY_NT_SUCCESS(inputBuffer.Read(outEvents,
+                                           RECORD_INSERT_COUNT - 1,
+                                           false,
+                                           false,
+                                           true,
+                                           false));
+        VERIFY_ARE_EQUAL(outEvents.size(), RECORD_INSERT_COUNT - 1);
+        VERIFY_IS_FALSE(waitEvent.is_signaled());
     }
 
     TEST_METHOD(ReadingDbcsCharsPadsOutputArray)
     {
         Log::Comment(L"During a non-unicode read, the input buffer should count twice for each dbcs key event");
 
+        auto& codepage = ServiceLocator::LocateGlobals().getConsoleInformation().CP;
+        const auto restoreCP = wil::scope_exit([&, previous = codepage]() {
+            codepage = previous;
+        });
+
+        codepage = CP_JAPANESE;
+
         // write a mouse event, key event, dbcs key event, mouse event
         InputBuffer inputBuffer;
-        const unsigned int recordInsertCount = 4;
-        INPUT_RECORD inRecords[recordInsertCount];
+
+        std::array<INPUT_RECORD, 4> inRecords{};
         inRecords[0].EventType = MOUSE_EVENT;
         inRecords[1] = MakeKeyEvent(TRUE, 1, L'A', 0, L'A', 0);
         inRecords[2] = MakeKeyEvent(TRUE, 1, 0x3042, 0, 0x3042, 0); // U+3042 hiragana A
         inRecords[3].EventType = MOUSE_EVENT;
 
+        std::array<INPUT_RECORD, 5> outRecordsExpected{};
+        outRecordsExpected[0].EventType = MOUSE_EVENT;
+        outRecordsExpected[1] = MakeKeyEvent(TRUE, 1, L'A', 0, L'A', 0);
+        outRecordsExpected[2] = MakeKeyEvent(TRUE, 1, 0x3042, 0, 0x82, 0);
+        outRecordsExpected[3] = MakeKeyEvent(TRUE, 1, 0x3042, 0, 0xa0, 0);
+        outRecordsExpected[4].EventType = MOUSE_EVENT;
+
         std::deque<std::unique_ptr<IInputEvent>> inEvents;
-        for (size_t i = 0; i < recordInsertCount; ++i)
+        for (size_t i = 0; i < inRecords.size(); ++i)
         {
             inEvents.push_back(IInputEvent::Create(inRecords[i]));
         }
@@ -432,22 +446,16 @@ class InputBufferTests
 
         // read them out non-unicode style and compare
         std::deque<std::unique_ptr<IInputEvent>> outEvents;
-        size_t eventsRead = 0;
-        auto resetWaitEvent = false;
-        inputBuffer._ReadBuffer(outEvents,
-                                recordInsertCount,
-                                eventsRead,
-                                false,
-                                resetWaitEvent,
-                                false,
-                                false);
-        // the dbcs record should have counted for two elements in
-        // the array, making it so that we get less events read
-        VERIFY_ARE_EQUAL(eventsRead, recordInsertCount - 1);
-        VERIFY_ARE_EQUAL(eventsRead, outEvents.size());
-        for (size_t i = 0; i < eventsRead; ++i)
+        VERIFY_NT_SUCCESS(inputBuffer.Read(outEvents,
+                                           outRecordsExpected.size(),
+                                           false,
+                                           false,
+                                           false,
+                                           false));
+        VERIFY_ARE_EQUAL(outEvents.size(), outRecordsExpected.size());
+        for (size_t i = 0; i < outEvents.size(); ++i)
         {
-            VERIFY_ARE_EQUAL(outEvents[i]->ToInputRecord(), inRecords[i]);
+            VERIFY_ARE_EQUAL(outEvents[i]->ToInputRecord(), outRecordsExpected[i]);
         }
     }
 

src/host/ut_host/ReadWaitTests.cpp

@@ -1,129 +0,0 @@
-// Copyright (c) Microsoft Corporation.
-// Licensed under the MIT license.
-
-#include "precomp.h"
-#include "WexTestClass.h"
-#include "../../inc/consoletaeftemplates.hpp"
-
-#include "misc.h"
-#include "dbcs.h"
-#include "../../types/inc/IInputEvent.hpp"
-
-#include "../interactivity/inc/ServiceLocator.hpp"
-
-#include <deque>
-#include <memory>
-
-using namespace WEX::Logging;
-using Microsoft::Console::Interactivity::ServiceLocator;
-
-class InputRecordConversionTests
-{
-    TEST_CLASS(InputRecordConversionTests);
-
-    static const size_t INPUT_RECORD_COUNT = 10;
-    UINT savedCodepage = 0;
-
-    TEST_CLASS_SETUP(ClassSetup)
-    {
-        auto& gci = ServiceLocator::LocateGlobals().getConsoleInformation();
-        savedCodepage = gci.CP;
-        gci.CP = CP_JAPANESE;
-        VERIFY_IS_TRUE(!!GetCPInfo(gci.CP, &gci.CPInfo));
-        return true;
-    }
-
-    TEST_CLASS_CLEANUP(ClassCleanup)
-    {
-        auto& gci = ServiceLocator::LocateGlobals().getConsoleInformation();
-        gci.CP = savedCodepage;
-        VERIFY_IS_TRUE(!!GetCPInfo(gci.CP, &gci.CPInfo));
-        return true;
-    }
-
-    TEST_METHOD(SplitToOemLeavesNonKeyEventsAlone)
-    {
-        Log::Comment(L"nothing should happen to input events that aren't key events");
-
-        std::deque<std::unique_ptr<IInputEvent>> inEvents;
-        INPUT_RECORD inRecords[INPUT_RECORD_COUNT] = { 0 };
-        for (size_t i = 0; i < INPUT_RECORD_COUNT; ++i)
-        {
-            inRecords[i].EventType = MOUSE_EVENT;
-            inRecords[i].Event.MouseEvent.dwMousePosition.X = static_cast<SHORT>(i);
-            inRecords[i].Event.MouseEvent.dwMousePosition.Y = static_cast<SHORT>(i * 2);
-            inEvents.push_back(IInputEvent::Create(inRecords[i]));
-        }
-
-        SplitToOem(inEvents);
-        VERIFY_ARE_EQUAL(INPUT_RECORD_COUNT, inEvents.size());
-
-        for (size_t i = 0; i < INPUT_RECORD_COUNT; ++i)
-        {
-            VERIFY_ARE_EQUAL(inRecords[i], inEvents[i]->ToInputRecord());
-        }
-    }
-
-    TEST_METHOD(SplitToOemLeavesNonDbcsCharsAlone)
-    {
-        Log::Comment(L"non-dbcs chars shouldn't be split");
-
-        std::deque<std::unique_ptr<IInputEvent>> inEvents;
-        INPUT_RECORD inRecords[INPUT_RECORD_COUNT] = { 0 };
-        for (size_t i = 0; i < INPUT_RECORD_COUNT; ++i)
-        {
-            inRecords[i].EventType = KEY_EVENT;
-            inRecords[i].Event.KeyEvent.uChar.UnicodeChar = static_cast<wchar_t>(L'a' + i);
-            inEvents.push_back(IInputEvent::Create(inRecords[i]));
-        }
-
-        SplitToOem(inEvents);
-        VERIFY_ARE_EQUAL(INPUT_RECORD_COUNT, inEvents.size());
-
-        for (size_t i = 0; i < INPUT_RECORD_COUNT; ++i)
-        {
-            VERIFY_ARE_EQUAL(inRecords[i], inEvents[i]->ToInputRecord());
-        }
-    }
-
-    TEST_METHOD(SplitToOemSplitsDbcsChars)
-    {
-        Log::Comment(L"dbcs chars should be split");
-
-        const auto codepage = ServiceLocator::LocateGlobals().getConsoleInformation().CP;
-
-        INPUT_RECORD inRecords[INPUT_RECORD_COUNT * 2] = { 0 };
-        std::deque<std::unique_ptr<IInputEvent>> inEvents;
-        // U+3042 hiragana letter A
-        wchar_t hiraganaA = 0x3042;
-        wchar_t inChars[INPUT_RECORD_COUNT];
-        for (size_t i = 0; i < INPUT_RECORD_COUNT; ++i)
-        {
-            auto currentChar = static_cast<wchar_t>(hiraganaA + (i * 2));
-            inRecords[i].EventType = KEY_EVENT;
-            inRecords[i].Event.KeyEvent.uChar.UnicodeChar = currentChar;
-            inChars[i] = currentChar;
-            inEvents.push_back(IInputEvent::Create(inRecords[i]));
-        }
-
-        SplitToOem(inEvents);
-        VERIFY_ARE_EQUAL(INPUT_RECORD_COUNT * 2, inEvents.size());
-
-        // create the data to compare the output to
-        char dbcsChars[INPUT_RECORD_COUNT * 2] = { 0 };
-        auto writtenBytes = WideCharToMultiByte(codepage,
-                                                0,
-                                                inChars,
-                                                INPUT_RECORD_COUNT,
-                                                dbcsChars,
-                                                INPUT_RECORD_COUNT * 2,
-                                                nullptr,
-                                                FALSE);
-        VERIFY_ARE_EQUAL(writtenBytes, static_cast<int>(INPUT_RECORD_COUNT * 2));
-        for (size_t i = 0; i < INPUT_RECORD_COUNT * 2; ++i)
-        {
-            const auto pKeyEvent = static_cast<const KeyEvent* const>(inEvents[i].get());
-            VERIFY_ARE_EQUAL(static_cast<char>(pKeyEvent->GetCharData()), dbcsChars[i]);
-        }
-    }
-};

src/host/ut_host/ScreenBufferTests.cpp

@@ -252,6 +252,8 @@ class ScreenBufferTests
     TEST_METHOD(TestDeferredMainBufferResize);
 
     TEST_METHOD(RectangularAreaOperations);
+
+    TEST_METHOD(DelayedWrapReset);
 };
 
 void ScreenBufferTests::SingleAlternateBufferCreationTest()
@@ -6266,33 +6268,38 @@ void ScreenBufferTests::CursorSaveRestore()
     VERIFY_SUCCEEDED(si.SetViewportOrigin(true, til::point(0, 0), true));
 
     Log::Comment(L"Restore after save.");
-    // Set the cursor position, attributes, and character set.
+    // Set the cursor position, delayed wrap, attributes, and character set.
     cursor.SetPosition({ 20, 10 });
+    cursor.DelayEOLWrap();
     si.SetAttributes(colorAttrs);
     stateMachine.ProcessString(selectGraphicsChars);
     // Save state.
     stateMachine.ProcessString(saveCursor);
-    // Reset the cursor position, attributes, and character set.
+    // Reset the cursor position, delayed wrap, attributes, and character set.
     cursor.SetPosition({ 0, 0 });
     si.SetAttributes(defaultAttrs);
     stateMachine.ProcessString(selectAsciiChars);
     // Restore state.
     stateMachine.ProcessString(restoreCursor);
-    // Verify initial position, colors, and graphic character set.
+    // Verify initial position, delayed wrap, colors, and graphic character set.
     VERIFY_ARE_EQUAL(til::point(20, 10), cursor.GetPosition());
+    VERIFY_IS_TRUE(cursor.IsDelayedEOLWrap());
+    cursor.ResetDelayEOLWrap();
     VERIFY_ARE_EQUAL(colorAttrs, si.GetAttributes());
     stateMachine.ProcessString(asciiText);
     VERIFY_IS_TRUE(_ValidateLineContains({ 20, 10 }, graphicText, colorAttrs));
 
     Log::Comment(L"Restore again without save.");
-    // Reset the cursor position, attributes, and character set.
+    // Reset the cursor position, delayed wrap, attributes, and character set.
     cursor.SetPosition({ 0, 0 });
     si.SetAttributes(defaultAttrs);
     stateMachine.ProcessString(selectAsciiChars);
     // Restore state.
     stateMachine.ProcessString(restoreCursor);
-    // Verify initial saved position, colors, and graphic character set.
+    // Verify initial saved position, delayed wrap, colors, and graphic character set.
     VERIFY_ARE_EQUAL(til::point(20, 10), cursor.GetPosition());
+    VERIFY_IS_TRUE(cursor.IsDelayedEOLWrap());
+    cursor.ResetDelayEOLWrap();
     VERIFY_ARE_EQUAL(colorAttrs, si.GetAttributes());
     stateMachine.ProcessString(asciiText);
     VERIFY_IS_TRUE(_ValidateLineContains({ 20, 10 }, graphicText, colorAttrs));
@@ -6300,14 +6307,16 @@ void ScreenBufferTests::CursorSaveRestore()
     Log::Comment(L"Restore after reset.");
     // Soft reset.
     stateMachine.ProcessString(L"\x1b[!p");
-    // Set the cursor position, attributes, and character set.
+    // Set the cursor position, delayed wrap, attributes, and character set.
     cursor.SetPosition({ 20, 10 });
+    cursor.DelayEOLWrap();
     si.SetAttributes(colorAttrs);
     stateMachine.ProcessString(selectGraphicsChars);
     // Restore state.
     stateMachine.ProcessString(restoreCursor);
-    // Verify home position, default attributes, and ascii character set.
+    // Verify home position, no delayed wrap, default attributes, and ascii character set.
     VERIFY_ARE_EQUAL(til::point(0, 0), cursor.GetPosition());
+    VERIFY_IS_FALSE(cursor.IsDelayedEOLWrap());
     VERIFY_ARE_EQUAL(defaultAttrs, si.GetAttributes());
     stateMachine.ProcessString(asciiText);
     VERIFY_IS_TRUE(_ValidateLineContains(til::point(0, 0), asciiText, defaultAttrs));
@@ -7538,3 +7547,99 @@ void ScreenBufferTests::RectangularAreaOperations()
     VERIFY_IS_TRUE(_ValidateLinesContain(targetArea.top, targetArea.bottom, bufferChars, bufferAttr));
     VERIFY_IS_TRUE(_ValidateLinesContain(targetArea.right, targetArea.top, targetArea.bottom, bufferChar, bufferAttr));
 }
+
+void ScreenBufferTests::DelayedWrapReset()
+{
+    BEGIN_TEST_METHOD_PROPERTIES()
+        TEST_METHOD_PROPERTY(L"IsolationLevel", L"Method")
+        TEST_METHOD_PROPERTY(L"Data:op", L"{0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25,26,27,28,29,30,31,32,33,34}")
+    END_TEST_METHOD_PROPERTIES();
+
+    int opIndex;
+    VERIFY_SUCCEEDED(TestData::TryGetValue(L"op", opIndex));
+
+    auto& gci = ServiceLocator::LocateGlobals().getConsoleInformation();
+    auto& si = gci.GetActiveOutputBuffer().GetActiveBuffer();
+    auto& stateMachine = si.GetStateMachine();
+    const auto width = si.GetTextBuffer().GetSize().Width();
+    const auto halfWidth = width / 2;
+    WI_SetFlag(si.OutputMode, ENABLE_VIRTUAL_TERMINAL_PROCESSING);
+    WI_SetFlag(si.OutputMode, DISABLE_NEWLINE_AUTO_RETURN);
+    stateMachine.ProcessString(L"\033[?40h"); // Make sure DECCOLM is allowed
+
+    const auto startRow = 5;
+    const auto startCol = width - 1;
+
+    // The operations below are all those that the DEC STD 070 reference has
+    // documented as needing to reset the Last Column Flag (see page D-13). The
+    // only controls that we haven't included are HT and SUB, because most DEC
+    // terminals did *not* trigger a reset after executing those sequences, and
+    // most modern terminals also seem to have agreed that that is the correct
+    // approach to take.
+    const struct
+    {
+        std::wstring_view name;
+        std::wstring_view sequence;
+        til::point expectedPos = {};
+        bool absolutePos = false;
+    } ops[] = {
+        { L"DECSTBM", L"\033[1;10r", { 0, 0 }, true },
+        { L"DECSWL", L"\033#5" },
+        { L"DECDWL", L"\033#6", { halfWidth - 1, startRow }, true },
+        { L"DECDHL (top)", L"\033#3", { halfWidth - 1, startRow }, true },
+        { L"DECDHL (bottom)", L"\033#4", { halfWidth - 1, startRow }, true },
+        { L"DECCOLM set", L"\033[?3h", { 0, 0 }, true },
+        { L"DECOM set", L"\033[?6h", { 0, 0 }, true },
+        { L"DECCOLM set", L"\033[?3l", { 0, 0 }, true },
+        { L"DECOM reset", L"\033[?6l", { 0, 0 }, true },
+        { L"DECAWM reset", L"\033[?7l" },
+        { L"CUU", L"\033[A", { 0, -1 } },
+        { L"CUD", L"\033[B", { 0, 1 } },
+        { L"CUF", L"\033[C" },
+        { L"CUB", L"\033[D", { -1, 0 } },
+        { L"CUP", L"\033[3;7H", { 6, 2 }, true },
+        { L"HVP", L"\033[3;7f", { 6, 2 }, true },
+        { L"BS", L"\b", { -1, 0 } },
+        { L"LF", L"\n", { 0, 1 } },
+        { L"VT", L"\v", { 0, 1 } },
+        { L"FF", L"\f", { 0, 1 } },
+        { L"CR", L"\r", { 0, startRow }, true },
+        { L"IND", L"\033D", { 0, 1 } },
+        { L"RI", L"\033M", { 0, -1 } },
+        { L"NEL", L"\033E", { 0, startRow + 1 }, true },
+        { L"ECH", L"\033[X" },
+        { L"DCH", L"\033[P" },
+        { L"ICH", L"\033[@" },
+        { L"EL", L"\033[K" },
+        { L"DECSEL", L"\033[?K" },
+        { L"DL", L"\033[M", { 0, startRow }, true },
+        { L"IL", L"\033[L", { 0, startRow }, true },
+        { L"ED", L"\033[J" },
+        { L"ED (all)", L"\033[2J" },
+        { L"ED (scrollback)", L"\033[3J" },
+        { L"DECSED", L"\033[?J" },
+    };
+    const auto& op = gsl::at(ops, opIndex);
+
+    Log::Comment(L"Writing a character at the end of the line should set delayed EOL wrap");
+    const auto startPos = til::point{ startCol, startRow };
+    si.GetTextBuffer().GetCursor().SetPosition(startPos);
+    stateMachine.ProcessCharacter(L'X');
+    {
+        auto& cursor = si.GetTextBuffer().GetCursor();
+        VERIFY_IS_TRUE(cursor.IsDelayedEOLWrap());
+        VERIFY_ARE_EQUAL(startPos, cursor.GetPosition());
+    }
+
+    Log::Comment(NoThrowString().Format(
+        L"Executing a %s control should reset delayed EOL wrap",
+        op.name.data()));
+    const auto expectedPos = op.absolutePos ? op.expectedPos : startPos + op.expectedPos;
+    stateMachine.ProcessString(op.sequence);
+    {
+        auto& cursor = si.GetTextBuffer().GetCursor();
+        const auto actualPos = cursor.GetPosition() - si.GetViewport().Origin();
+        VERIFY_IS_FALSE(cursor.IsDelayedEOLWrap());
+        VERIFY_ARE_EQUAL(expectedPos, actualPos);
+    }
+}

src/inc/til/bytes.h

@@ -0,0 +1,57 @@
+// Copyright (c) Microsoft Corporation.
+// Licensed under the MIT license.
+
+#pragma once
+
+#include "type_traits.h"
+
+namespace til
+{
+    template<ContiguousBytes Target>
+    constexpr void bytes_advance(Target& target, size_t count)
+    {
+        if (count > target.size())
+        {
+            throw std::length_error{ "insufficient space left" };
+        }
+
+        target = { target.data() + count, target.size() - count };
+    }
+
+    template<TriviallyCopyable T, ContiguousBytes Target>
+    constexpr bool bytes_can_put(const Target& target)
+    {
+        return target.size() >= sizeof(T);
+    }
+
+    template<TriviallyCopyable T, ContiguousBytes Target>
+    constexpr void bytes_put(Target& target, const T& value)
+    {
+        using TargetType = typename Target::value_type;
+        constexpr auto size = sizeof(value);
+
+        if (size > target.size())
+        {
+            throw std::length_error{ "insufficient space left" };
+        }
+
+        std::copy_n(reinterpret_cast<const TargetType*>(&value), size, target.data());
+        target = { target.data() + size, target.size() - size };
+    }
+
+    template<ContiguousBytes Target, ContiguousView Source>
+    requires TriviallyCopyable<typename Source::value_type>
+    constexpr void bytes_transfer(Target& target, Source& source)
+    {
+        using TargetType = typename Target::value_type;
+        constexpr auto elementSize = sizeof(typename Source::value_type);
+
+        const auto sourceCount = std::min(source.size(), target.size() / elementSize);
+        const auto targetCount = sourceCount * elementSize;
+
+        std::copy_n(reinterpret_cast<const TargetType*>(source.data()), targetCount, target.data());
+
+        target = { target.data() + targetCount, target.size() - targetCount };
+        source = { source.data() + sourceCount, source.size() - sourceCount };
+    }
+}

src/inc/til/type_traits.h

@@ -0,0 +1,48 @@
+// Copyright (c) Microsoft Corporation.
+// Licensed under the MIT license.
+
+#pragma once
+
+namespace til
+{
+    namespace details
+    {
+        template<typename T>
+        struct is_contiguous_view : std::false_type
+        {
+        };
+        template<typename U, std::size_t E>
+        struct is_contiguous_view<std::span<U, E>> : std::true_type
+        {
+        };
+        template<typename U, typename V>
+        struct is_contiguous_view<std::basic_string_view<U, V>> : std::true_type
+        {
+        };
+
+        template<typename T>
+        struct is_byte : std::false_type
+        {
+        };
+        template<>
+        struct is_byte<char> : std::true_type
+        {
+        };
+        template<>
+        struct is_byte<std::byte> : std::true_type
+        {
+        };
+    }
+
+    template<typename T>
+    concept Byte = details::is_byte<std::remove_cv_t<T>>::value;
+
+    template<typename T>
+    concept ContiguousView = details::is_contiguous_view<std::remove_cv_t<T>>::value;
+
+    template<typename T>
+    concept ContiguousBytes = ContiguousView<T> && Byte<typename T::value_type>;
+
+    template<typename T>
+    concept TriviallyCopyable = std::is_trivially_copyable_v<T>;
+}

src/renderer/atlas/AtlasEngine.api.cpp

@@ -711,7 +711,7 @@ void AtlasEngine::_resolveFontMetrics(const wchar_t* requestedFaceName, const Fo
         fontMetrics->fontName = std::move(fontName);
         fontMetrics->fontSizeInDIP = fontSizeInDIP;
         fontMetrics->baselineInDIP = baseline / static_cast<float>(_api.dpi) * 96.0f;
-        fontMetrics->advanceScale = cellWidth / adjustedWidth;
+        fontMetrics->advanceScale = cellWidth / advanceWidth;
         fontMetrics->cellSize = { cellWidth, cellHeight };
         fontMetrics->fontWeight = fontWeightU16;
         fontMetrics->underlinePos = underlinePosU16;

src/renderer/base/CSSLengthPercentage.cpp

@@ -52,16 +52,6 @@ CSSLengthPercentage CSSLengthPercentage::FromString(const wchar_t* str)
     return obj;
 }
 
-CSSLengthPercentage::ReferenceFrame CSSLengthPercentage::GetReferenceFrame() const noexcept
-{
-    return _referenceFrame;
-}
-
-float CSSLengthPercentage::Resolve(float factor) const noexcept
-{
-    return _value * factor;
-}
-
 float CSSLengthPercentage::Resolve(float fallback, float dpi, float fontSize, float advanceWidth) const noexcept
 {
     switch (_referenceFrame)

src/renderer/base/sources.inc

@@ -24,6 +24,7 @@ PRECOMPILED_CXX         = 1
 PRECOMPILED_INCLUDE     = ..\precomp.h
 
 SOURCES = \
+    ..\CSSLengthPercentage.cpp \
     ..\FontInfo.cpp \
     ..\FontInfoBase.cpp \
     ..\FontInfoDesired.cpp \

src/renderer/inc/CSSLengthPercentage.h

@@ -23,8 +23,6 @@ struct CSSLengthPercentage
 
     static CSSLengthPercentage FromString(const wchar_t* str);
 
-    ReferenceFrame GetReferenceFrame() const noexcept;
-    float Resolve(float factor) const noexcept;
     float Resolve(float fallback, float dpi, float fontSize, float advanceWidth) const noexcept;
 
 private:

src/renderer/vt/paint.cpp

@@ -604,7 +604,11 @@ using namespace Microsoft::Console::Types;
         {
             RETURN_IF_FAILED(_EraseCharacter(numSpaces));
         }
-        else
+        // If we're past the end of the row (i.e. in the "delayed EOL wrap"
+        // state), then there is no need to erase the rest of line. In fact
+        // if we did output an EL sequence at this point, it could reset the
+        // "delayed EOL wrap" state, breaking subsequent output.
+        else if (_lastText.x <= _lastViewport.RightInclusive())
         {
             RETURN_IF_FAILED(_EraseLine());
         }

src/server/ApiDispatchers.cpp

@@ -270,13 +270,8 @@
     // Get output parameter buffer.
     PVOID pvBuffer;
     ULONG cbBufferSize;
-    // TODO: This is dumb. We should find out how much we need, not guess.
-    // If the request is not in Unicode mode, we must allocate an output buffer that is twice as big as the actual caller buffer.
-    RETURN_IF_FAILED(m->GetAugmentedOutputBuffer((a->Unicode != FALSE) ? 1 : 2,
-                                                 &pvBuffer,
-                                                 &cbBufferSize));
+    RETURN_IF_FAILED(m->GetOutputBuffer(&pvBuffer, &cbBufferSize));
 
-    // TODO: This is also rather strange and will also probably make more sense if we stop guessing that we need 2x buffer to convert.
     // This might need to go on the other side of the fence (inside host) because the server doesn't know what we're going to do with initial num bytes.
     // (This restriction exists because it's going to copy initial into the final buffer, but we don't know that.)
     RETURN_HR_IF(E_INVALIDARG, a->InitialNumBytes > cbBufferSize);

src/terminal/adapter/adaptDispatch.cpp

@@ -149,7 +149,7 @@ void AdaptDispatch::_WriteToBuffer(const std::wstring_view string)
             cursor.SetPosition(cursorPosition);
             if (wrapAtEOL)
             {
-                cursor.DelayEOLWrap(cursorPosition);
+                cursor.DelayEOLWrap();
             }
         }
         else
@@ -446,6 +446,7 @@ bool AdaptDispatch::CursorSaveState()
     auto& savedCursorState = _savedCursorState.at(_usingAltBuffer);
     savedCursorState.Column = cursorPosition.x + 1;
     savedCursorState.Row = cursorPosition.y + 1;
+    savedCursorState.IsDelayedEOLWrap = textBuffer.GetCursor().IsDelayedEOLWrap();
     savedCursorState.IsOriginModeRelative = _modes.test(Mode::Origin);
     savedCursorState.Attributes = attributes;
     savedCursorState.TermOutput = _termOutput;
@@ -484,6 +485,12 @@ bool AdaptDispatch::CursorRestoreState()
     _modes.reset(Mode::Origin);
     CursorPosition(row, col);
 
+    // If the delayed wrap flag was set when the cursor was saved, we need to restore that now.
+    if (savedCursorState.IsDelayedEOLWrap)
+    {
+        _api.GetTextBuffer().GetCursor().DelayEOLWrap();
+    }
+
     // Once the cursor position is restored, we can then restore the actual origin mode.
     _modes.set(Mode::Origin, savedCursorState.IsOriginModeRelative);
 
@@ -600,6 +607,8 @@ void AdaptDispatch::_InsertDeleteCharacterHelper(const VTInt delta)
     const auto startCol = textBuffer.GetCursor().GetPosition().x;
     const auto endCol = textBuffer.GetLineWidth(row);
     _ScrollRectHorizontally(textBuffer, { startCol, row, endCol, row + 1 }, delta);
+    // The ICH and DCH controls are expected to reset the delayed wrap flag.
+    textBuffer.GetCursor().ResetDelayEOLWrap();
 }
 
 // Routine Description:
@@ -668,6 +677,9 @@ bool AdaptDispatch::EraseCharacters(const VTInt numChars)
     const auto startCol = textBuffer.GetCursor().GetPosition().x;
     const auto endCol = std::min<VTInt>(startCol + numChars, textBuffer.GetLineWidth(row));
 
+    // The ECH control is expected to reset the delayed wrap flag.
+    textBuffer.GetCursor().ResetDelayEOLWrap();
+
     auto eraseAttributes = textBuffer.GetCurrentAttributes();
     eraseAttributes.SetStandardErase();
     _FillRect(textBuffer, { startCol, row, endCol, row + 1 }, L' ', eraseAttributes);
@@ -721,6 +733,11 @@ bool AdaptDispatch::EraseInDisplay(const DispatchTypes::EraseType eraseType)
     const auto row = textBuffer.GetCursor().GetPosition().y;
     const auto col = textBuffer.GetCursor().GetPosition().x;
 
+    // The ED control is expected to reset the delayed wrap flag.
+    // The special case variants above ("erase all" and "erase scrollback")
+    // take care of that themselves when they set the cursor position.
+    textBuffer.GetCursor().ResetDelayEOLWrap();
+
     auto eraseAttributes = textBuffer.GetCurrentAttributes();
     eraseAttributes.SetStandardErase();
 
@@ -758,6 +775,9 @@ bool AdaptDispatch::EraseInLine(const DispatchTypes::EraseType eraseType)
     const auto row = textBuffer.GetCursor().GetPosition().y;
     const auto col = textBuffer.GetCursor().GetPosition().x;
 
+    // The EL control is expected to reset the delayed wrap flag.
+    textBuffer.GetCursor().ResetDelayEOLWrap();
+
     auto eraseAttributes = textBuffer.GetCurrentAttributes();
     eraseAttributes.SetStandardErase();
     switch (eraseType)
@@ -822,6 +842,9 @@ bool AdaptDispatch::SelectiveEraseInDisplay(const DispatchTypes::EraseType erase
     const auto row = textBuffer.GetCursor().GetPosition().y;
     const auto col = textBuffer.GetCursor().GetPosition().x;
 
+    // The DECSED control is expected to reset the delayed wrap flag.
+    textBuffer.GetCursor().ResetDelayEOLWrap();
+
     switch (eraseType)
     {
     case DispatchTypes::EraseType::FromBeginning:
@@ -855,6 +878,9 @@ bool AdaptDispatch::SelectiveEraseInLine(const DispatchTypes::EraseType eraseTyp
     const auto row = textBuffer.GetCursor().GetPosition().y;
     const auto col = textBuffer.GetCursor().GetPosition().x;
 
+    // The DECSEL control is expected to reset the delayed wrap flag.
+    textBuffer.GetCursor().ResetDelayEOLWrap();
+
     switch (eraseType)
     {
     case DispatchTypes::EraseType::FromBeginning:
@@ -1251,7 +1277,13 @@ bool AdaptDispatch::SelectAttributeChangeExtent(const DispatchTypes::ChangeExten
 // - True.
 bool AdaptDispatch::SetLineRendition(const LineRendition rendition)
 {
-    _api.GetTextBuffer().SetCurrentLineRendition(rendition);
+    auto& textBuffer = _api.GetTextBuffer();
+    textBuffer.SetCurrentLineRendition(rendition);
+    // There is some variation in how this was handled by the different DEC
+    // terminals, but the STD 070 reference (on page D-13) makes it clear that
+    // the delayed wrap (aka the Last Column Flag) was expected to be reset when
+    // line rendition controls were executed.
+    textBuffer.GetCursor().ResetDelayEOLWrap();
     return true;
 }
 
@@ -1606,6 +1638,11 @@ bool AdaptDispatch::_ModeParamsHelper(const DispatchTypes::ModeParams param, con
         return true;
     case DispatchTypes::ModeParams::DECAWM_AutoWrapMode:
         _api.SetAutoWrapMode(enable);
+        // Resetting DECAWM should also reset the delayed wrap flag.
+        if (!enable)
+        {
+            _api.GetTextBuffer().GetCursor().ResetDelayEOLWrap();
+        }
         return true;
     case DispatchTypes::ModeParams::DECARM_AutoRepeatMode:
         _terminalInput.SetInputMode(TerminalInput::Mode::AutoRepeat, enable);
@@ -2097,8 +2134,20 @@ bool AdaptDispatch::ForwardTab(const VTInt numTabs)
         }
     }
 
+    // While the STD 070 reference suggests that horizontal tabs should reset
+    // the delayed wrap, almost none of the DEC terminals actually worked that
+    // way, and most modern terminal emulators appear to have taken the same
+    // approach (i.e. they don't reset). For us this is a bit messy, since all
+    // cursor movement resets the flag automatically, so we need to save the
+    // original state here, and potentially reapply it after the move.
+    const auto delayedWrapOriginallySet = cursor.IsDelayedEOLWrap();
     cursor.SetXPosition(column);
     _ApplyCursorMovementFlags(cursor);
+    if (delayedWrapOriginallySet)
+    {
+        cursor.DelayEOLWrap();
+    }
+
     return true;
 }
 

src/terminal/adapter/adaptDispatch.hpp

@@ -170,6 +170,7 @@ namespace Microsoft::Console::VirtualTerminal
         {
             VTInt Row = 1;
             VTInt Column = 1;
+            bool IsDelayedEOLWrap = false;
             bool IsOriginModeRelative = false;
             TextAttribute Attributes = {};
             TerminalOutput TermOutput = {};

src/types/inc/IInputEvent.hpp

@@ -64,6 +64,8 @@ public:
 
 inline IInputEvent::~IInputEvent() = default;
 
+using InputEventQueue = std::deque<std::unique_ptr<IInputEvent>>;
+
 #ifdef UNIT_TESTING
 std::wostream& operator<<(std::wostream& stream, const IInputEvent* pEvent);
 #endif