Merge pull request #133 from qmfrederik/fixes/tarball

Test & fix extracting .tar.gz packages on macOS

.azure-pipelines-shared.yml

@@ -0,0 +1,31 @@
+steps:
+- task: DownloadBuildArtifacts@0
+  displayName: Download Build Artifacts
+  inputs:
+    artifactName: nuget
+    downloadPath: $(Build.ArtifactStagingDirectory)
+- bash: |
+    set -e
+    export PATH="$PATH:/$HOME/.dotnet/tools"
+
+    mkdir test-$(command)/
+    cd test-$(command)
+
+    echo "<configuration><packageSources><add key='local' value='$(Build.ArtifactStagingDirectory)/nuget' /></packageSources></configuration>" > NuGet.config
+    version=$(cat $(Build.ArtifactStagingDirectory)/nuget/version.txt)
+
+    # Install the dotnet-$(command) tool
+    dotnet tool install --global dotnet-$(command) --version $version --add-source $(Build.ArtifactStagingDirectory)/nuget
+
+    # Create a new console application
+    dotnet new console
+
+    # Install and use dotnet $(command)
+    dotnet $(command) install
+    dotnet $(command) -o $(Build.ArtifactStagingDirectory)/packages/$(container)
+  workingDirectory: $(Pipeline.Workspace)
+  displayName: Run dotnet $(command)
+- task: PublishBuildArtifacts@1
+  inputs:
+    pathToPublish: $(Build.ArtifactStagingDirectory)/packages/
+    artifactName: packages

.azure-pipelines.yml

@@ -1,3 +1,6 @@
+trigger:
+- master
+
 resources:
   containers:
   - container: 2.2-bionic
@@ -60,36 +63,7 @@ stages:
 
     container: $[ variables['container'] ]
     steps:
-    - task: DownloadBuildArtifacts@0
-      displayName: Download Build Artifacts
-      inputs:
-        artifactName: nuget
-        downloadPath: $(Build.ArtifactStagingDirectory)
-    - bash: |
-        set -e
-        export PATH="$PATH:/$HOME/.dotnet/tools"
-
-        mkdir test-$(command)/
-        cd test-$(command)
-
-        echo "<configuration><packageSources><add key='local' value='$(Build.ArtifactStagingDirectory)/nuget' /></packageSources></configuration>" > NuGet.config
-        version=$(cat $(Build.ArtifactStagingDirectory)/nuget/version.txt)
-
-        # Install the dotnet-$(command) tool
-        dotnet tool install --global dotnet-$(command) --version $version --add-source $(Build.ArtifactStagingDirectory)/nuget
-
-        # Create a new console application
-        dotnet new console
-
-        # Install and use dotnet $(command)
-        dotnet $(command) install
-        dotnet $(command) -o $(Build.ArtifactStagingDirectory)/packages/$(container)
-      workingDirectory: $(Pipeline.Workspace)
-      displayName: Run dotnet $(command)
-    - task: PublishBuildArtifacts@1
-      inputs:
-        pathToPublish: $(Build.ArtifactStagingDirectory)/packages/
-        artifactName: packages
+    - template: .azure-pipelines-shared.yml
 
   - job: molecule
     pool:
@@ -151,3 +125,42 @@ stages:
         searchFolder: '$(Build.SourcesDirectory)/molecule/$(suite)/molecule/default/'
       condition: always()
       displayName: Publish test results
+
+  - job: build_macos
+    pool:
+      vmImage: 'macOS-10.14'
+    variables:
+      container: macos
+      command: tarball
+    steps:
+    - task: UseDotNet@2
+      displayName: 'Use .NET Core 3.0'
+      inputs:
+        packageType: sdk
+        version: 3.0.100
+    - template: .azure-pipelines-shared.yml
+
+  - job: test_macos
+    pool:
+      vmImage: 'macOS-10.14'
+    dependsOn: build_macos
+    steps:
+    - task: DownloadBuildArtifacts@0
+      displayName: Download Build Artifacts
+      inputs:
+        artifactName: packages
+        downloadPath: $(Build.ArtifactStagingDirectory)/
+    - task: UseDotNet@2
+      displayName: 'Use .NET Core 3.0'
+      inputs:
+        packageType: sdk
+        version: 3.0.100
+    - bash: |
+        set -e
+        tar xvzf $(Build.ArtifactStagingDirectory)/packages/macos/test-tarball.1.0.0.tar.gz
+      displayName: Extract tarball
+    - bash: |
+        set -e
+        chmod +x ./test-tarball
+        ./test-tarball
+      displayName: Execute program

Packaging.Targets.Tests/Deb/DebTaskTests.cs

@@ -1,4 +1,8 @@
-using Xunit;
+using Packaging.Targets.IO;
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using Xunit;
 
 namespace Packaging.Targets.Tests.Deb
 {
@@ -32,5 +36,70 @@ namespace Packaging.Targets.Tests.Deb
         {
             Assert.Equal(packageAchitecture, DebTask.GetPackageArchitecture(runtimeIdentifier));
         }
+
+        [Fact]
+        public void EnsureDirectoriesTest()
+        {
+            List<ArchiveEntry> archiveEntries = new List<ArchiveEntry>();
+            archiveEntries.Add(
+                new ArchiveEntry()
+                {
+                    Mode = LinuxFileMode.S_IROTH | LinuxFileMode.S_IRGRP | LinuxFileMode.S_IRUSR | LinuxFileMode.S_IFREG,
+                    TargetPath = "./ConsoleApp1.deps.json",
+                });
+
+            DebTask.EnsureDirectories(archiveEntries, includeRoot: false);
+
+            // This example contains one entry in the current directory, so no new directory entries should
+            // have been created
+            Assert.Single(archiveEntries);
+        }
+
+        [Fact]
+        public void EnsureDirectoriesTest2()
+        {
+            List<ArchiveEntry> archiveEntries = new List<ArchiveEntry>();
+            archiveEntries.Add(
+                new ArchiveEntry()
+                {
+                    Mode = LinuxFileMode.S_IROTH | LinuxFileMode.S_IRGRP | LinuxFileMode.S_IRUSR | LinuxFileMode.S_IFREG,
+                    TargetPath = "/usr/local/share/consoleapp/ConsoleApp1.deps.json",
+                });
+
+            DebTask.EnsureDirectories(archiveEntries, includeRoot: true);
+
+            archiveEntries = archiveEntries
+                .OrderBy(e => e.TargetPathWithFinalSlash, StringComparer.Ordinal)
+                .ToList();
+
+            // This example contains one entry in the current directory, so no new directory entries should
+            // have been created
+            Assert.Collection(
+                archiveEntries,
+                (e) => Assert.Equal("/", e.TargetPath),
+                (e) => Assert.Equal("/usr", e.TargetPath),
+                (e) => Assert.Equal("/usr/local", e.TargetPath),
+                (e) => Assert.Equal("/usr/local/share", e.TargetPath),
+                (e) => Assert.Equal("/usr/local/share/consoleapp", e.TargetPath),
+                (e) => Assert.Equal("/usr/local/share/consoleapp/ConsoleApp1.deps.json", e.TargetPath));
+        }
+
+        [Fact]
+        public void EnsureDirectoriesTest3()
+        {
+            List<ArchiveEntry> archiveEntries = new List<ArchiveEntry>();
+            archiveEntries.Add(
+                new ArchiveEntry()
+                {
+                    Mode = LinuxFileMode.S_IROTH | LinuxFileMode.S_IRGRP | LinuxFileMode.S_IRUSR | LinuxFileMode.S_IFREG,
+                    TargetPath = "ConsoleApp1.deps.json",
+                });
+
+            DebTask.EnsureDirectories(archiveEntries, includeRoot: false);
+
+            // This example contains one entry in the current directory, so no new directory entries should
+            // have been created
+            Assert.Single(archiveEntries);
+        }
     }
 }

Packaging.Targets/ArchiveBuilder.cs

@@ -293,7 +293,14 @@ namespace Packaging.Targets
 
                 if (name == null)
                 {
-                    name = prefix + "/" + fileName;
+                    if (!string.IsNullOrEmpty(prefix))
+                    {
+                        name = prefix + "/" + fileName;
+                    }
+                    else
+                    {
+                        name = fileName;
+                    }
                 }
 
                 string linkTo = string.Empty;

Packaging.Targets/DebTask.cs

@@ -192,7 +192,7 @@ namespace Packaging.Targets
                     this.Content);
 
                 archiveEntries.AddRange(archiveBuilder.FromLinuxFolders(this.LinuxFolders));
-                this.EnsureDirectories(archiveEntries);
+                EnsureDirectories(archiveEntries);
 
                 archiveEntries = archiveEntries
                     .OrderBy(e => e.TargetPathWithFinalSlash, StringComparer.Ordinal)
@@ -268,7 +268,7 @@ namespace Packaging.Targets
             }
         }
 
-        private void EnsureDirectories(List<ArchiveEntry> entries)
+        internal static void EnsureDirectories(List<ArchiveEntry> entries, bool includeRoot = true)
         {
             var dirs = new HashSet<string>(entries.Where(x => x.Mode.HasFlag(LinuxFileMode.S_IFDIR))
                 .Select(d => d.TargetPathWithFinalSlash));
@@ -285,7 +285,7 @@ namespace Packaging.Targets
 
                 if (!path.Contains("/"))
                 {
-                    return "/";
+                    return string.Empty;
                 }
 
                 return path.Substring(0, path.LastIndexOf('/'));
@@ -293,7 +293,7 @@ namespace Packaging.Targets
 
             void EnsureDir(string dirPath)
             {
-                if (dirPath == string.Empty)
+                if (dirPath == string.Empty || dirPath == ".")
                 {
                     return;
                 }
@@ -325,7 +325,11 @@ namespace Packaging.Targets
                 EnsureDir(GetDirPath(entry.TargetPathWithFinalSlash));
             }
 
-            EnsureDir("/");
+            if (includeRoot)
+            {
+                EnsureDir("/");
+            }
+
             entries.AddRange(toAdd);
         }
     }

Packaging.Targets/IO/TarFileCreator.cs

@@ -31,7 +31,14 @@ namespace Packaging.Targets.IO
 
         public static void WriteTrailer(Stream stream)
         {
-            Align(stream);
+            // The stream should already be aligned; as it is aligned after every entry.
+            // As a safety measure, for streams which can report on .Position, align the
+            // stream again.
+            if (stream.CanSeek)
+            {
+                Align(stream);
+            }
+
             var trailer = new byte[1024];
             stream.Write(trailer, 0, trailer.Length);
         }
@@ -39,10 +46,10 @@ namespace Packaging.Targets.IO
         public static void WriteEntry(Stream stream, TarHeader header, Stream data)
         {
             header.Checksum = header.ComputeChecksum();
-            stream.WriteStruct(header);
-            Align(stream);
+            int written = stream.WriteStruct(header);
+            Align(stream, written);
             data.CopyTo(stream);
-            Align(stream);
+            Align(stream, data.Length);
         }
 
         public static void WriteEntry(Stream stream, ArchiveEntry entry, Stream data = null)
@@ -151,7 +158,12 @@ namespace Packaging.Targets.IO
 
         private static void Align(Stream stream)
         {
-            var spos = stream.Position % 512;
+            Align(stream, stream.Position);
+        }
+
+        private static void Align(Stream stream, long position)
+        {
+            var spos = position % 512;
             if (spos == 0)
             {
                 return;

Packaging.Targets/Packaging.Targets.csproj

@@ -30,15 +30,12 @@
     <PackageReference Include="Portable.BouncyCastle" Version="1.8.1.3">
       <PrivateAssets>all</PrivateAssets>
     </PackageReference>
-    <PackageReference Include="SharpZipLib.NETStandard" Version="0.86.0.1">
+    <PackageReference Include="SharpZipLib" Version="1.2.0">
       <PrivateAssets>all</PrivateAssets>
     </PackageReference>
     <PackageReference Include="StyleCop.Analyzers" Version="1.0.2">
       <PrivateAssets>all</PrivateAssets>
     </PackageReference>
-    <PackageReference Include="SharpZipLib.NETStandard" Version="0.86.0.1">
-      <PrivateAssets>all</PrivateAssets>
-    </PackageReference>
     <PackageReference Include="System.Buffers" Version="4.5.0" />
   </ItemGroup>
   <ItemGroup>
@@ -46,9 +43,9 @@
       <Pack>true</Pack>
       <PackagePath>build\</PackagePath>
     </Content>
-    
+
     <!-- SharpZipLib -->
-    <Content Include="$(NuGetPackageRoot)\sharpziplib.netstandard\0.86.0.1\lib\netstandard1.3\SharpZipLib.NETStandard.dll" Link="SharpZipLib.NETStandard.dll">
+    <Content Include="$(NuGetPackageRoot)\sharpziplib\1.2.0\lib\netstandard2.0\ICSharpCode.SharpZipLib.dll" Link="ICSharpCode.SharpZipLib.dll">
       <Pack>true</Pack>
       <PackagePath>tools\netstandard2.0\</PackagePath>
     </Content>

Packaging.Targets/StreamExtensions.cs

@@ -81,7 +81,7 @@ namespace Packaging.Targets
         /// <param name="data">
         /// The struct to write to the stram.
         /// </param>
-        public static void WriteStruct<T>(this Stream stream, T data)
+        public static int WriteStruct<T>(this Stream stream, T data)
             where T : struct
         {
             if (stream == null)
@@ -105,6 +105,7 @@ namespace Packaging.Targets
             RespectEndianness<T>(bytes);
 
             stream.Write(bytes, 0, bytes.Length);
+            return bytes.Length;
         }
 
         /// <summary>

Packaging.Targets/TarballTask.cs

@@ -1,11 +1,10 @@
-using ICSharpCode.SharpZipLib.GZip;
-using ICSharpCode.SharpZipLib.Tar;
-using ICSharpCode.SharpZipLib.Zip;
-using Microsoft.Build.Framework;
+using Microsoft.Build.Framework;
 using Microsoft.Build.Utilities;
+using Packaging.Targets.IO;
 using System;
-using System.Diagnostics;
 using System.IO;
+using System.IO.Compression;
+using System.Linq;
 
 namespace Packaging.Targets
 {
@@ -13,17 +12,18 @@ namespace Packaging.Targets
     {
         [Required]
         public string PublishDir
-        {
-            get;
-            set;
-        }
+        { get;  set; }
 
         [Required]
         public string TarballPath
-        {
-            get;
-            set;
-        }
+        { get; set; }
+
+        [Required]
+        public ITaskItem[] Content
+        { get; set; }
+
+        public string Prefix
+        { get; set; }
 
         public override bool Execute()
         {
@@ -37,14 +37,22 @@ namespace Packaging.Targets
 
         private void CreateLinuxTarball()
         {
-            using (var stream = File.Create(this.TarballPath))
-            using (var gzipStream = new GZipOutputStream(stream))
-            using (var archive = TarArchive.CreateOutputTarArchive(gzipStream))
-            {
-                archive.RootPath = this.PublishDir.Replace("\\", "/");
-                var entry = TarEntry.CreateEntryFromFile(this.PublishDir);
+            ArchiveBuilder archiveBuilder = new ArchiveBuilder();
+            var archiveEntries = archiveBuilder.FromDirectory(
+                this.PublishDir,
+                this.Prefix,
+                this.Content);
 
-                archive.WriteEntry(entry, true);
+            DebTask.EnsureDirectories(archiveEntries, includeRoot: false);
+
+            archiveEntries = archiveEntries
+                .OrderBy(e => e.TargetPathWithFinalSlash, StringComparer.Ordinal)
+                .ToList();
+
+            using (var stream = File.Create(this.TarballPath))
+            using (var gzipStream = new GZipStream(stream, CompressionMode.Compress))
+            {
+                TarFileCreator.FromArchiveEntries(archiveEntries, gzipStream);
             }
         }
     }

Packaging.Targets/build/Packaging.Targets.targets

@@ -182,6 +182,7 @@
   <Target Name="CreateTarball" DependsOnTargets="CreatePackageProperties">
     <PropertyGroup>
       <TarballPath Condition="'$(TarballPath)' == ''">$(PackagePath).tar.gz</TarballPath>
+      <TarballPrefix Condition="'$(TarballPrefix)' == ''">.</TarballPrefix>
     </PropertyGroup>
 
     <Message Text="Creating tarball $(TarballPath)" Importance="high"/>
@@ -189,7 +190,9 @@
     <MakeDir Directories="$(PackageDir)"/>
     
     <TarballTask PublishDir="$(PublishDir)"
-             TarballPath="$(TarballPath)"/>
+                 TarballPath="$(TarballPath)"
+                 Prefix="$(TarballPrefix)"
+                 Content="@(Content)" />
   </Target>
 
   <Target Name="CreateZip" DependsOnTargets="CreatePackageProperties">