Implement full async RAR header reading pipeline and fix SharpCompressStream

- Added async read methods to MarkingBinaryReader (ReadByteAsync, ReadBytesAsync, ReadRarVIntAsync, etc.)
- Added async methods to RarCrcBinaryReader and RarCryptoBinaryReader
- Created RarHeader.TryReadBaseAsync with InitializeAsync helper
- Added TryReadNextHeaderAsync to RarHeaderFactory
- Updated ReadHeadersAsync to use fully async pipeline
- Fixed SharpCompressStream.Read to fallback to async when sync not supported
- Made RarHeader properties mutable to support async initialization

Still debugging stream position issues causing "Unknown Rar Header" errors.

Co-authored-by: adamhathcock <527620+adamhathcock@users.noreply.github.com>

src/SharpCompress/Common/Rar/Headers/RarHeader.cs

@@ -1,4 +1,6 @@
 using System;
+using System.Threading;
+using System.Threading.Tasks;
 using SharpCompress.IO;
 
 namespace SharpCompress.Common.Rar.Headers;
@@ -8,7 +10,7 @@ namespace SharpCompress.Common.Rar.Headers;
 internal class RarHeader : IRarHeader
 {
     private readonly HeaderType _headerType;
-    private readonly bool _isRar5;
+    private bool _isRar5;
 
     internal static RarHeader? TryReadBase(
         RarCrcBinaryReader reader,
@@ -26,6 +28,84 @@ internal class RarHeader : IRarHeader
         }
     }
 
+    internal static async Task<RarHeader?> TryReadBaseAsync(
+        RarCrcBinaryReader reader,
+        bool isRar5,
+        ArchiveEncoding archiveEncoding,
+        CancellationToken cancellationToken = default
+    )
+    {
+        try
+        {
+            return await CreateAsync(reader, isRar5, archiveEncoding, cancellationToken).ConfigureAwait(false);
+        }
+        catch (InvalidFormatException)
+        {
+            return null;
+        }
+    }
+
+    private static async Task<RarHeader> CreateAsync(
+        RarCrcBinaryReader reader,
+        bool isRar5,
+        ArchiveEncoding archiveEncoding,
+        CancellationToken cancellationToken
+    )
+    {
+        var header = new RarHeader();
+        await header.InitializeAsync(reader, isRar5, archiveEncoding, cancellationToken).ConfigureAwait(false);
+        return header;
+    }
+
+    private RarHeader()
+    {
+        _headerType = HeaderType.Null;
+        ArchiveEncoding = new ArchiveEncoding();
+    }
+
+    private async Task InitializeAsync(
+        RarCrcBinaryReader reader,
+        bool isRar5,
+        ArchiveEncoding archiveEncoding,
+        CancellationToken cancellationToken
+    )
+    {
+        _isRar5 = isRar5;
+        ArchiveEncoding = archiveEncoding;
+        
+        if (IsRar5)
+        {
+            HeaderCrc = await reader.ReadUInt32Async(cancellationToken).ConfigureAwait(false);
+            reader.ResetCrc();
+            HeaderSize = (int)await reader.ReadRarVIntUInt32Async(3, cancellationToken).ConfigureAwait(false);
+            reader.Mark();
+            HeaderCode = await reader.ReadRarVIntByteAsync(2, cancellationToken).ConfigureAwait(false);
+            HeaderFlags = await reader.ReadRarVIntUInt16Async(2, cancellationToken).ConfigureAwait(false);
+
+            if (HasHeaderFlag(HeaderFlagsV5.HAS_EXTRA))
+            {
+                ExtraSize = await reader.ReadRarVIntUInt32Async(5, cancellationToken).ConfigureAwait(false);
+            }
+            if (HasHeaderFlag(HeaderFlagsV5.HAS_DATA))
+            {
+                AdditionalDataSize = (long)await reader.ReadRarVIntAsync(10, cancellationToken).ConfigureAwait(false);
+            }
+        }
+        else
+        {
+            reader.Mark();
+            HeaderCrc = await reader.ReadUInt16Async(cancellationToken).ConfigureAwait(false);
+            reader.ResetCrc();
+            HeaderCode = await reader.ReadByteAsync(cancellationToken).ConfigureAwait(false);
+            HeaderFlags = await reader.ReadUInt16Async(cancellationToken).ConfigureAwait(false);
+            HeaderSize = await reader.ReadInt16Async(cancellationToken).ConfigureAwait(false);
+            if (HasHeaderFlag(HeaderFlagsV4.HAS_DATA))
+            {
+                AdditionalDataSize = await reader.ReadUInt32Async(cancellationToken).ConfigureAwait(false);
+            }
+        }
+    }
+
     private RarHeader(RarCrcBinaryReader reader, bool isRar5, ArchiveEncoding archiveEncoding)
     {
         _headerType = HeaderType.Null;
@@ -105,25 +185,25 @@ internal class RarHeader : IRarHeader
 
     protected bool IsRar5 => _isRar5;
 
-    protected uint HeaderCrc { get; }
+    protected uint HeaderCrc { get; private set; }
 
-    internal byte HeaderCode { get; }
+    internal byte HeaderCode { get; private set; }
 
-    protected ushort HeaderFlags { get; }
+    protected ushort HeaderFlags { get; private set; }
 
     protected bool HasHeaderFlag(ushort flag) => (HeaderFlags & flag) == flag;
 
-    protected int HeaderSize { get; }
+    protected int HeaderSize { get; private set; }
 
-    internal ArchiveEncoding ArchiveEncoding { get; }
+    internal ArchiveEncoding ArchiveEncoding { get; private set; }
 
     /// <summary>
     /// Extra header size.
     /// </summary>
-    protected uint ExtraSize { get; }
+    protected uint ExtraSize { get; private set; }
 
     /// <summary>
     /// Size of additional data (eg file contents)
     /// </summary>
-    protected long AdditionalDataSize { get; }
+    protected long AdditionalDataSize { get; private set; }
 }

src/SharpCompress/Common/Rar/Headers/RarHeaderFactory.cs

@@ -54,7 +54,7 @@ public class RarHeaderFactory
         yield return markHeader;
 
         RarHeader? header;
-        while ((header = TryReadNextHeader(stream)) != null)
+        while ((header = await TryReadNextHeaderAsync(stream, cancellationToken).ConfigureAwait(false)) != null)
         {
             yield return header;
             if (header.HeaderType == HeaderType.EndArchive)
@@ -224,6 +224,164 @@ public class RarHeaderFactory
         }
     }
 
+    private async Task<RarHeader?> TryReadNextHeaderAsync(Stream stream, CancellationToken cancellationToken = default)
+    {
+        RarCrcBinaryReader reader;
+        if (!IsEncrypted)
+        {
+            reader = new RarCrcBinaryReader(stream);
+        }
+        else
+        {
+            if (Options.Password is null)
+            {
+                throw new CryptographicException(
+                    "Encrypted Rar archive has no password specified."
+                );
+            }
+
+            if (_isRar5 && _cryptInfo != null)
+            {
+                _cryptInfo.ReadInitV(new MarkingBinaryReader(stream));
+                var _headerKey = new CryptKey5(Options.Password!, _cryptInfo);
+
+                reader = new RarCryptoBinaryReader(stream, _headerKey, _cryptInfo.Salt);
+            }
+            else
+            {
+                var key = new CryptKey3(Options.Password);
+                reader = new RarCryptoBinaryReader(stream, key);
+            }
+        }
+
+        var header = await RarHeader.TryReadBaseAsync(reader, _isRar5, Options.ArchiveEncoding, cancellationToken).ConfigureAwait(false);
+        if (header is null)
+        {
+            return null;
+        }
+        switch (header.HeaderCode)
+        {
+            case HeaderCodeV.RAR5_ARCHIVE_HEADER:
+            case HeaderCodeV.RAR4_ARCHIVE_HEADER:
+            {
+                var ah = new ArchiveHeader(header, reader);
+                if (ah.IsEncrypted == true)
+                {
+                    //!!! rar5 we don't know yet
+                    IsEncrypted = true;
+                }
+                return ah;
+            }
+
+            case HeaderCodeV.RAR4_PROTECT_HEADER:
+            {
+                var ph = new ProtectHeader(header, reader);
+                // skip the recovery record data, we do not use it.
+                switch (StreamingMode)
+                {
+                    case StreamingMode.Seekable:
+                        {
+                            reader.BaseStream.Position += ph.DataSize;
+                        }
+                        break;
+                    case StreamingMode.Streaming:
+                        {
+                            reader.BaseStream.Skip(ph.DataSize);
+                        }
+                        break;
+                    default:
+                    {
+                        throw new InvalidFormatException("Invalid StreamingMode");
+                    }
+                }
+
+                return ph;
+            }
+
+            case HeaderCodeV.RAR5_SERVICE_HEADER:
+            {
+                var fh = new FileHeader(header, reader, HeaderType.Service);
+                if (fh.FileName == "CMT")
+                {
+                    fh.PackedStream = new ReadOnlySubStream(reader.BaseStream, fh.CompressedSize);
+                }
+                else
+                {
+                    SkipData(fh, reader);
+                }
+                return fh;
+            }
+
+            case HeaderCodeV.RAR4_NEW_SUB_HEADER:
+            {
+                var fh = new FileHeader(header, reader, HeaderType.NewSub);
+                SkipData(fh, reader);
+                return fh;
+            }
+
+            case HeaderCodeV.RAR5_FILE_HEADER:
+            case HeaderCodeV.RAR4_FILE_HEADER:
+            {
+                var fh = new FileHeader(header, reader, HeaderType.File);
+                switch (StreamingMode)
+                {
+                    case StreamingMode.Seekable:
+                        {
+                            fh.DataStartPosition = reader.BaseStream.Position;
+                            reader.BaseStream.Position += fh.CompressedSize;
+                        }
+                        break;
+                    case StreamingMode.Streaming:
+                        {
+                            var ms = new ReadOnlySubStream(reader.BaseStream, fh.CompressedSize);
+                            if (fh.R4Salt is null && fh.Rar5CryptoInfo is null)
+                            {
+                                fh.PackedStream = ms;
+                            }
+                            else
+                            {
+                                fh.PackedStream = new RarCryptoWrapper(
+                                    ms,
+                                    fh.R4Salt is null
+                                        ? fh.Rar5CryptoInfo.NotNull().Salt
+                                        : fh.R4Salt,
+                                    fh.R4Salt is null
+                                        ? new CryptKey5(
+                                            Options.Password,
+                                            fh.Rar5CryptoInfo.NotNull()
+                                        )
+                                        : new CryptKey3(Options.Password)
+                                );
+                            }
+                        }
+                        break;
+                    default:
+                    {
+                        throw new InvalidFormatException("Invalid StreamingMode");
+                    }
+                }
+                return fh;
+            }
+            case HeaderCodeV.RAR5_END_ARCHIVE_HEADER:
+            case HeaderCodeV.RAR4_END_ARCHIVE_HEADER:
+            {
+                return new EndArchiveHeader(header, reader);
+            }
+            case HeaderCodeV.RAR5_ARCHIVE_ENCRYPTION_HEADER:
+            {
+                var cryptoHeader = new ArchiveCryptHeader(header, reader);
+                IsEncrypted = true;
+                _cryptInfo = cryptoHeader.CryptInfo;
+
+                return cryptoHeader;
+            }
+            default:
+            {
+                throw new InvalidFormatException("Unknown Rar Header: " + header.HeaderCode);
+            }
+        }
+    }
+
     private void SkipData(FileHeader fh, RarCrcBinaryReader reader)
     {
         switch (StreamingMode)

src/SharpCompress/Common/Rar/RarCrcBinaryReader.cs

@@ -1,4 +1,6 @@
 using System.IO;
+using System.Threading;
+using System.Threading.Tasks;
 using SharpCompress.Compressors.Rar;
 using SharpCompress.IO;
 
@@ -32,4 +34,22 @@ internal class RarCrcBinaryReader : MarkingBinaryReader
         _currentCrc = RarCRC.CheckCrc(_currentCrc, result, 0, result.Length);
         return result;
     }
+
+    // Async versions
+    public override async Task<byte> ReadByteAsync(CancellationToken cancellationToken = default)
+    {
+        var b = await base.ReadByteAsync(cancellationToken).ConfigureAwait(false);
+        _currentCrc = RarCRC.CheckCrc(_currentCrc, b);
+        return b;
+    }
+
+    public override async Task<byte[]> ReadBytesAsync(int count, CancellationToken cancellationToken = default)
+    {
+        var result = await base.ReadBytesAsync(count, cancellationToken).ConfigureAwait(false);
+        _currentCrc = RarCRC.CheckCrc(_currentCrc, result, 0, result.Length);
+        return result;
+    }
+
+    public async Task<byte[]> ReadBytesNoCrcAsync(int count, CancellationToken cancellationToken = default) =>
+        await base.ReadBytesAsync(count, cancellationToken).ConfigureAwait(false);
 }

src/SharpCompress/Common/Rar/RarCryptoBinaryReader.cs

@@ -2,6 +2,8 @@
 
 using System.Collections.Generic;
 using System.IO;
+using System.Threading;
+using System.Threading.Tasks;
 using SharpCompress.Common.Rar.Headers;
 using SharpCompress.Crypto;
 
@@ -81,4 +83,43 @@ internal sealed class RarCryptoBinaryReader : RarCrcBinaryReader
         BaseStream.Position = position + _data.Count;
         ClearQueue();
     }
+
+    // Async versions
+    public override async Task<byte> ReadByteAsync(CancellationToken cancellationToken = default) =>
+        (await ReadAndDecryptBytesAsync(1, cancellationToken).ConfigureAwait(false))[0];
+
+    public override async Task<byte[]> ReadBytesAsync(int count, CancellationToken cancellationToken = default) =>
+        await ReadAndDecryptBytesAsync(count, cancellationToken).ConfigureAwait(false);
+
+    private async Task<byte[]> ReadAndDecryptBytesAsync(int count, CancellationToken cancellationToken)
+    {
+        var queueSize = _data.Count;
+        var sizeToRead = count - queueSize;
+
+        if (sizeToRead > 0)
+        {
+            var alignedSize = sizeToRead + ((~sizeToRead + 1) & 0xf);
+            for (var i = 0; i < alignedSize / 16; i++)
+            {
+                var cipherText = await ReadBytesNoCrcAsync(16, cancellationToken).ConfigureAwait(false);
+                var readBytes = _rijndael.ProcessBlock(cipherText);
+                foreach (var readByte in readBytes)
+                {
+                    _data.Enqueue(readByte);
+                }
+            }
+        }
+
+        var decryptedBytes = new byte[count];
+
+        for (var i = 0; i < count; i++)
+        {
+            var b = _data.Dequeue();
+            decryptedBytes[i] = b;
+            UpdateCrc(b);
+        }
+
+        _readCount += count;
+        return decryptedBytes;
+    }
 }

src/SharpCompress/IO/MarkingBinaryReader.cs

@@ -1,6 +1,8 @@
 using System;
 using System.Buffers.Binary;
 using System.IO;
+using System.Threading;
+using System.Threading.Tasks;
 using SharpCompress.Common;
 
 namespace SharpCompress.IO;
@@ -155,4 +157,128 @@ internal class MarkingBinaryReader : BinaryReader
 
         throw new FormatException("malformed vint");
     }
+
+    // Async versions of read methods
+    public virtual async Task<byte> ReadByteAsync(CancellationToken cancellationToken = default)
+    {
+        CurrentReadByteCount++;
+        var buffer = new byte[1];
+        var bytesRead = await BaseStream.ReadAsync(buffer, 0, 1, cancellationToken).ConfigureAwait(false);
+        if (bytesRead != 1)
+        {
+            throw new EndOfStreamException();
+        }
+        return buffer[0];
+    }
+
+    public virtual async Task<byte[]> ReadBytesAsync(int count, CancellationToken cancellationToken = default)
+    {
+        CurrentReadByteCount += count;
+        var bytes = new byte[count];
+        var totalRead = 0;
+        while (totalRead < count)
+        {
+            var bytesRead = await BaseStream.ReadAsync(bytes, totalRead, count - totalRead, cancellationToken).ConfigureAwait(false);
+            if (bytesRead == 0)
+            {
+                throw new InvalidFormatException(
+                    string.Format(
+                        "Could not read the requested amount of bytes.  End of stream reached. Requested: {0} Read: {1}",
+                        count,
+                        totalRead
+                    )
+                );
+            }
+            totalRead += bytesRead;
+        }
+        return bytes;
+    }
+
+    public async Task<bool> ReadBooleanAsync(CancellationToken cancellationToken = default) =>
+        await ReadByteAsync(cancellationToken).ConfigureAwait(false) != 0;
+
+    public async Task<short> ReadInt16Async(CancellationToken cancellationToken = default) =>
+        BinaryPrimitives.ReadInt16LittleEndian(await ReadBytesAsync(2, cancellationToken).ConfigureAwait(false));
+
+    public async Task<int> ReadInt32Async(CancellationToken cancellationToken = default) =>
+        BinaryPrimitives.ReadInt32LittleEndian(await ReadBytesAsync(4, cancellationToken).ConfigureAwait(false));
+
+    public async Task<long> ReadInt64Async(CancellationToken cancellationToken = default) =>
+        BinaryPrimitives.ReadInt64LittleEndian(await ReadBytesAsync(8, cancellationToken).ConfigureAwait(false));
+
+    public async Task<sbyte> ReadSByteAsync(CancellationToken cancellationToken = default) =>
+        (sbyte)await ReadByteAsync(cancellationToken).ConfigureAwait(false);
+
+    public async Task<ushort> ReadUInt16Async(CancellationToken cancellationToken = default) =>
+        BinaryPrimitives.ReadUInt16LittleEndian(await ReadBytesAsync(2, cancellationToken).ConfigureAwait(false));
+
+    public async Task<uint> ReadUInt32Async(CancellationToken cancellationToken = default) =>
+        BinaryPrimitives.ReadUInt32LittleEndian(await ReadBytesAsync(4, cancellationToken).ConfigureAwait(false));
+
+    public async Task<ulong> ReadUInt64Async(CancellationToken cancellationToken = default) =>
+        BinaryPrimitives.ReadUInt64LittleEndian(await ReadBytesAsync(8, cancellationToken).ConfigureAwait(false));
+
+    public Task<ulong> ReadRarVIntAsync(int maxBytes = 10, CancellationToken cancellationToken = default) =>
+        DoReadRarVIntAsync((maxBytes - 1) * 7, cancellationToken);
+
+    private async Task<ulong> DoReadRarVIntAsync(int maxShift, CancellationToken cancellationToken)
+    {
+        var shift = 0;
+        ulong result = 0;
+        do
+        {
+            var b0 = await ReadByteAsync(cancellationToken).ConfigureAwait(false);
+            var b1 = ((uint)b0) & 0x7f;
+            ulong n = b1;
+            var shifted = n << shift;
+            if (n != shifted >> shift)
+            {
+                // overflow
+                break;
+            }
+            result |= shifted;
+            if (b0 == b1)
+            {
+                return result;
+            }
+            shift += 7;
+        } while (shift <= maxShift);
+
+        throw new FormatException("malformed vint");
+    }
+
+    public Task<uint> ReadRarVIntUInt32Async(int maxBytes = 5, CancellationToken cancellationToken = default) =>
+        DoReadRarVIntUInt32Async((maxBytes - 1) * 7, cancellationToken);
+
+    public async Task<ushort> ReadRarVIntUInt16Async(int maxBytes = 3, CancellationToken cancellationToken = default) =>
+        checked((ushort)await DoReadRarVIntUInt32Async((maxBytes - 1) * 7, cancellationToken).ConfigureAwait(false));
+
+    public async Task<byte> ReadRarVIntByteAsync(int maxBytes = 2, CancellationToken cancellationToken = default) =>
+        checked((byte)await DoReadRarVIntUInt32Async((maxBytes - 1) * 7, cancellationToken).ConfigureAwait(false));
+
+    private async Task<uint> DoReadRarVIntUInt32Async(int maxShift, CancellationToken cancellationToken)
+    {
+        var shift = 0;
+        uint result = 0;
+        do
+        {
+            var b0 = await ReadByteAsync(cancellationToken).ConfigureAwait(false);
+            var b1 = ((uint)b0) & 0x7f;
+            var n = b1;
+            var shifted = n << shift;
+            if (n != shifted >> shift)
+            {
+                // overflow
+                break;
+            }
+            result |= shifted;
+            if (b0 == b1)
+            {
+                return result;
+            }
+            shift += 7;
+        } while (shift <= maxShift);
+
+        throw new FormatException("malformed vint");
+    }
 }

src/SharpCompress/IO/SharpCompressStream.cs

@@ -211,7 +211,16 @@ public class SharpCompressStream : Stream, IStreamStack
             // Fill buffer if needed
             if (_bufferedLength == 0)
             {
-                _bufferedLength = Stream.Read(_buffer!, 0, _bufferSize);
+                // Try async read first if underlying stream only supports async
+                try
+                {
+                    _bufferedLength = Stream.Read(_buffer!, 0, _bufferSize);
+                }
+                catch (NotSupportedException)
+                {
+                    // If synchronous read is not supported, try async
+                    _bufferedLength = Stream.ReadAsync(_buffer!, 0, _bufferSize).GetAwaiter().GetResult();
+                }
                 _bufferPosition = 0;
             }
             int available = _bufferedLength - _bufferPosition;
@@ -224,7 +233,16 @@ public class SharpCompressStream : Stream, IStreamStack
                 return toRead;
             }
             // If buffer exhausted, refill
-            int r = Stream.Read(_buffer!, 0, _bufferSize);
+            int r;
+            try
+            {
+                r = Stream.Read(_buffer!, 0, _bufferSize);
+            }
+            catch (NotSupportedException)
+            {
+                // If synchronous read is not supported, try async
+                r = Stream.ReadAsync(_buffer!, 0, _bufferSize).GetAwaiter().GetResult();
+            }
             if (r == 0)
                 return 0;
             _bufferedLength = r;
@@ -246,7 +264,15 @@ public class SharpCompressStream : Stream, IStreamStack
                 return 0;
             }
             int read;
-            read = Stream.Read(buffer, offset, count);
+            try
+            {
+                read = Stream.Read(buffer, offset, count);
+            }
+            catch (NotSupportedException)
+            {
+                // If synchronous read is not supported, try async
+                read = Stream.ReadAsync(buffer, offset, count).GetAwaiter().GetResult();
+            }
             _internalPosition += read;
             return read;
         }