Compare commits

..

3 Commits

Author SHA1 Message Date
Adam Hathcock
862fa50fcb Don't use SkipLocalsInit 2021-11-29 09:29:40 +00:00
Adam Hathcock
7b87924172 Add zstd classes for initial work 2021-11-28 10:11:54 +00:00
Adam Hathcock
d9d7ea8ec5 Add zstandard compression 2021-11-28 09:52:09 +00:00
325 changed files with 42473 additions and 3418 deletions

View File

@@ -1,543 +0,0 @@
# Version: 2.0.1 (Using https://semver.org/)
# Updated: 2020-12-11
# See https://github.com/RehanSaeed/EditorConfig/releases for release notes.
# See https://github.com/RehanSaeed/EditorConfig for updates to this file.
# See http://EditorConfig.org for more information about .editorconfig files.
##########################################
# Common Settings
##########################################
# This file is the top-most EditorConfig file
root = true
# All Files
[*]
charset = utf-8
indent_style = space
indent_size = 4
insert_final_newline = true
trim_trailing_whitespace = true
##########################################
# File Extension Settings
##########################################
# Visual Studio Solution Files
[*.sln]
indent_style = tab
# Visual Studio XML Project Files
[*.{csproj,vbproj,vcxproj.filters,proj,projitems,shproj}]
indent_size = 2
# XML Configuration Files
[*.{xml,config,props,targets,nuspec,resx,ruleset,vsixmanifest,vsct}]
indent_size = 2
# JSON Files
[*.{json,json5,webmanifest}]
indent_size = 2
# YAML Files
[*.{yml,yaml}]
indent_size = 2
# Markdown Files
[*.md]
trim_trailing_whitespace = false
# Web Files
[*.{htm,html,js,jsm,ts,tsx,css,sass,scss,less,svg,vue}]
indent_size = 2
# Batch Files
[*.{cmd,bat}]
end_of_line = crlf
# Bash Files
[*.sh]
end_of_line = lf
# Makefiles
[Makefile]
indent_style = tab
##########################################
# Default .NET Code Style Severities
# https://docs.microsoft.com/dotnet/fundamentals/code-analysis/configuration-options#scope
##########################################
[*.{cs,csx,cake,vb,vbx}]
# Default Severity for all .NET Code Style rules below
dotnet_analyzer_diagnostic.severity = warning
##########################################
# File Header (Uncomment to support file headers)
# https://docs.microsoft.com/visualstudio/ide/reference/add-file-header
##########################################
# [*.{cs,csx,cake,vb,vbx}]
# file_header_template = <copyright file="{fileName}" company="PROJECT-AUTHOR">\n© PROJECT-AUTHOR\n</copyright>
# SA1636: File header copyright text should match
# Justification: .editorconfig supports file headers. If this is changed to a value other than "none", a stylecop.json file will need to added to the project.
# dotnet_diagnostic.SA1636.severity = none
##########################################
# .NET Language Conventions
# https://docs.microsoft.com/visualstudio/ide/editorconfig-language-conventions
##########################################
# .NET Code Style Settings
# https://docs.microsoft.com/visualstudio/ide/editorconfig-language-conventions#net-code-style-settings
[*.{cs,csx,cake,vb,vbx}]
# "this." and "Me." qualifiers
# https://docs.microsoft.com/visualstudio/ide/editorconfig-language-conventions#this-and-me
#dotnet_style_qualification_for_field = true:warning
#dotnet_style_qualification_for_property = true:warning
#dotnet_style_qualification_for_method = true:warning
#dotnet_style_qualification_for_event = true:warning
# Language keywords instead of framework type names for type references
# https://docs.microsoft.com/visualstudio/ide/editorconfig-language-conventions#language-keywords
dotnet_style_predefined_type_for_locals_parameters_members = true:warning
dotnet_style_predefined_type_for_member_access = true:warning
# Modifier preferences
# https://docs.microsoft.com/visualstudio/ide/editorconfig-language-conventions#normalize-modifiers
dotnet_style_require_accessibility_modifiers = always:warning
csharp_preferred_modifier_order = public,private,protected,internal,static,extern,new,virtual,abstract,sealed,override,readonly,unsafe,volatile,async:warning
visual_basic_preferred_modifier_order = Partial,Default,Private,Protected,Public,Friend,NotOverridable,Overridable,MustOverride,Overloads,Overrides,MustInherit,NotInheritable,Static,Shared,Shadows,ReadOnly,WriteOnly,Dim,Const,WithEvents,Widening,Narrowing,Custom,Async:warning
dotnet_style_readonly_field = true:warning
# Parentheses preferences
# https://docs.microsoft.com/visualstudio/ide/editorconfig-language-conventions#parentheses-preferences
dotnet_style_parentheses_in_arithmetic_binary_operators = always_for_clarity:warning
dotnet_style_parentheses_in_relational_binary_operators = always_for_clarity:warning
dotnet_style_parentheses_in_other_binary_operators = always_for_clarity:warning
dotnet_style_parentheses_in_other_operators = always_for_clarity:suggestion
# Expression-level preferences
# https://docs.microsoft.com/visualstudio/ide/editorconfig-language-conventions#expression-level-preferences
dotnet_style_object_initializer = true:warning
dotnet_style_collection_initializer = true:warning
dotnet_style_explicit_tuple_names = true:warning
dotnet_style_prefer_inferred_tuple_names = true:warning
dotnet_style_prefer_inferred_anonymous_type_member_names = true:warning
dotnet_style_prefer_auto_properties = true:warning
dotnet_style_prefer_is_null_check_over_reference_equality_method = true:warning
dotnet_style_prefer_conditional_expression_over_assignment = false:suggestion
dotnet_diagnostic.IDE0045.severity = suggestion
dotnet_style_prefer_conditional_expression_over_return = false:suggestion
dotnet_diagnostic.IDE0046.severity = suggestion
dotnet_style_prefer_compound_assignment = true:warning
# Null-checking preferences
# https://docs.microsoft.com/visualstudio/ide/editorconfig-language-conventions#null-checking-preferences
dotnet_style_coalesce_expression = true:warning
dotnet_style_null_propagation = true:warning
# Parameter preferences
# https://docs.microsoft.com/visualstudio/ide/editorconfig-language-conventions#parameter-preferences
dotnet_code_quality_unused_parameters = all:warning
# More style options (Undocumented)
# https://github.com/MicrosoftDocs/visualstudio-docs/issues/3641
dotnet_style_operator_placement_when_wrapping = end_of_line
# https://github.com/dotnet/roslyn/pull/40070
dotnet_style_prefer_simplified_interpolation = true:warning
# C# Code Style Settings
# https://docs.microsoft.com/visualstudio/ide/editorconfig-language-conventions#c-code-style-settings
[*.{cs,csx,cake}]
# Implicit and explicit types
# https://docs.microsoft.com/visualstudio/ide/editorconfig-language-conventions#implicit-and-explicit-types
csharp_style_var_for_built_in_types = true:warning
csharp_style_var_when_type_is_apparent = true:warning
csharp_style_var_elsewhere = true:warning
# Expression-bodied members
# https://docs.microsoft.com/visualstudio/ide/editorconfig-language-conventions#expression-bodied-members
csharp_style_expression_bodied_methods = true:warning
csharp_style_expression_bodied_constructors = true:warning
csharp_style_expression_bodied_operators = true:warning
csharp_style_expression_bodied_properties = true:warning
csharp_style_expression_bodied_indexers = true:warning
csharp_style_expression_bodied_accessors = true:warning
csharp_style_expression_bodied_lambdas = true:warning
csharp_style_expression_bodied_local_functions = true:warning
# Pattern matching
# https://docs.microsoft.com/visualstudio/ide/editorconfig-language-conventions#pattern-matching
csharp_style_pattern_matching_over_is_with_cast_check = true:warning
csharp_style_pattern_matching_over_as_with_null_check = true:warning
# Inlined variable declarations
# https://docs.microsoft.com/visualstudio/ide/editorconfig-language-conventions#inlined-variable-declarations
csharp_style_inlined_variable_declaration = true:warning
# Expression-level preferences
# https://docs.microsoft.com/visualstudio/ide/editorconfig-language-conventions#expression-level-preferences
csharp_prefer_simple_default_expression = true:warning
# "Null" checking preferences
# https://docs.microsoft.com/visualstudio/ide/editorconfig-language-conventions#c-null-checking-preferences
csharp_style_throw_expression = true:warning
csharp_style_conditional_delegate_call = true:warning
# Code block preferences
# https://docs.microsoft.com/visualstudio/ide/editorconfig-language-conventions#code-block-preferences
csharp_prefer_braces = true:warning
# Unused value preferences
# https://docs.microsoft.com/visualstudio/ide/editorconfig-language-conventions#unused-value-preferences
csharp_style_unused_value_expression_statement_preference = discard_variable:suggestion
dotnet_diagnostic.IDE0058.severity = suggestion
csharp_style_unused_value_assignment_preference = discard_variable:suggestion
dotnet_diagnostic.IDE0059.severity = suggestion
# Index and range preferences
# https://docs.microsoft.com/visualstudio/ide/editorconfig-language-conventions#index-and-range-preferences
csharp_style_prefer_index_operator = true:warning
csharp_style_prefer_range_operator = true:warning
# Miscellaneous preferences
# https://docs.microsoft.com/visualstudio/ide/editorconfig-language-conventions#miscellaneous-preferences
csharp_style_deconstructed_variable_declaration = true:warning
csharp_style_pattern_local_over_anonymous_function = true:warning
csharp_using_directive_placement = outside_namespace:warning
csharp_prefer_static_local_function = true:warning
csharp_prefer_simple_using_statement = true:suggestion
dotnet_diagnostic.IDE0063.severity = suggestion
csharp_style_namespace_declarations = file_scoped
##########################################
# .NET Formatting Conventions
# https://docs.microsoft.com/visualstudio/ide/editorconfig-code-style-settings-reference#formatting-conventions
##########################################
# Organize usings
# https://docs.microsoft.com/visualstudio/ide/editorconfig-formatting-conventions#organize-using-directives
dotnet_sort_system_directives_first = true
# Newline options
# https://docs.microsoft.com/visualstudio/ide/editorconfig-formatting-conventions#new-line-options
csharp_new_line_before_open_brace = all
csharp_new_line_before_else = true
csharp_new_line_before_catch = true
csharp_new_line_before_finally = true
csharp_new_line_before_members_in_object_initializers = true
csharp_new_line_before_members_in_anonymous_types = true
csharp_new_line_between_query_expression_clauses = true
# Indentation options
# https://docs.microsoft.com/visualstudio/ide/editorconfig-formatting-conventions#indentation-options
csharp_indent_case_contents = true
csharp_indent_switch_labels = true
csharp_indent_labels = no_change
csharp_indent_block_contents = true
csharp_indent_braces = false
csharp_indent_case_contents_when_block = false
# Spacing options
# https://docs.microsoft.com/visualstudio/ide/editorconfig-formatting-conventions#spacing-options
csharp_space_after_cast = false
csharp_space_after_keywords_in_control_flow_statements = true
csharp_space_between_parentheses = false
csharp_space_before_colon_in_inheritance_clause = true
csharp_space_after_colon_in_inheritance_clause = true
csharp_space_around_binary_operators = before_and_after
csharp_space_between_method_declaration_parameter_list_parentheses = false
csharp_space_between_method_declaration_empty_parameter_list_parentheses = false
csharp_space_between_method_declaration_name_and_open_parenthesis = false
csharp_space_between_method_call_parameter_list_parentheses = false
csharp_space_between_method_call_empty_parameter_list_parentheses = false
csharp_space_between_method_call_name_and_opening_parenthesis = false
csharp_space_after_comma = true
csharp_space_before_comma = false
csharp_space_after_dot = false
csharp_space_before_dot = false
csharp_space_after_semicolon_in_for_statement = true
csharp_space_before_semicolon_in_for_statement = false
csharp_space_around_declaration_statements = false
csharp_space_before_open_square_brackets = false
csharp_space_between_empty_square_brackets = false
csharp_space_between_square_brackets = false
# Wrapping options
# https://docs.microsoft.com/visualstudio/ide/editorconfig-formatting-conventions#wrap-options
csharp_preserve_single_line_statements = false
csharp_preserve_single_line_blocks = true
##########################################
# .NET Naming Conventions
# https://docs.microsoft.com/visualstudio/ide/editorconfig-naming-conventions
##########################################
[*.{cs,csx,cake,vb,vbx}]
dotnet_diagnostic.CA1000.severity = suggestion
dotnet_diagnostic.CA1001.severity = error
dotnet_diagnostic.CA1018.severity = error
dotnet_diagnostic.CA1051.severity = suggestion
dotnet_diagnostic.CA1068.severity = error
dotnet_diagnostic.CA1069.severity = error
dotnet_diagnostic.CA1304.severity = error
dotnet_diagnostic.CA1305.severity = suggestion
dotnet_diagnostic.CA1309.severity = suggestion
dotnet_diagnostic.CA1310.severity = error
dotnet_diagnostic.CA1707.severity = suggestion
dotnet_diagnostic.CA1708.severity = suggestion
dotnet_diagnostic.CA1711.severity = suggestion
dotnet_diagnostic.CA1716.severity = suggestion
dotnet_diagnostic.CA1720.severity = suggestion
dotnet_diagnostic.CA1725.severity = suggestion
dotnet_diagnostic.CA1805.severity = suggestion
dotnet_diagnostic.CA1816.severity = suggestion
dotnet_diagnostic.CA1822.severity = suggestion
dotnet_diagnostic.CA1825.severity = error
dotnet_diagnostic.CA1826.severity = silent
dotnet_diagnostic.CA1827.severity = error
dotnet_diagnostic.CA1829.severity = suggestion
dotnet_diagnostic.CA1834.severity = error
dotnet_diagnostic.CA1845.severity = suggestion
dotnet_diagnostic.CA1848.severity = suggestion
dotnet_diagnostic.CA2016.severity = suggestion
dotnet_diagnostic.CA2201.severity = error
dotnet_diagnostic.CA2206.severity = error
dotnet_diagnostic.CA2208.severity = error
dotnet_diagnostic.CA2211.severity = error
dotnet_diagnostic.CA2249.severity = error
dotnet_diagnostic.CA2251.severity = error
dotnet_diagnostic.CA2252.severity = none
dotnet_diagnostic.CA2254.severity = suggestion
dotnet_diagnostic.CS0169.severity = error
dotnet_diagnostic.CS0219.severity = error
dotnet_diagnostic.CS1998.severity = error
dotnet_diagnostic.CS8602.severity = error
dotnet_diagnostic.CS8604.severity = error
dotnet_diagnostic.CS8618.severity = error
dotnet_diagnostic.CS0618.severity = error
dotnet_diagnostic.CS1998.severity = error
dotnet_diagnostic.CS4014.severity = error
dotnet_diagnostic.CS8600.severity = error
dotnet_diagnostic.CS8603.severity = error
dotnet_diagnostic.CS8625.severity = error
dotnet_diagnostic.BL0005.severity = suggestion
dotnet_diagnostic.MVC1000.severity = suggestion
dotnet_diagnostic.IDE0055.severity = suggestion # Fix formatting
dotnet_diagnostic.IDE0023.severity = suggestion # use expression body for operators
dotnet_diagnostic.IDE0025.severity = suggestion # use expression body for properties
dotnet_diagnostic.IDE1006.severity = suggestion # Naming rule violation: These words cannot contain lower case characters
dotnet_diagnostic.IDE0072.severity = suggestion # Populate switch - forces population of all cases even when default specified
dotnet_diagnostic.IDE0027.severity = suggestion # Use expression body for accessors
dotnet_diagnostic.IDE0032.severity = suggestion # Use auto property
dotnet_diagnostic.IDE0007.severity = error # Use var
dotnet_diagnostic.IDE0160.severity = suggestion # Use block scoped
dotnet_diagnostic.IDE0011.severity = error # Use braces on if statements
dotnet_diagnostic.IDE0057.severity = suggestion # substring can be simplified
dotnet_diagnostic.IDE0004.severity = error # redundant cast
dotnet_diagnostic.IDE0010.severity = silent # populate switch
dotnet_diagnostic.IDE0021.severity = silent # expression body for constructors
dotnet_diagnostic.IDE0022.severity = silent # expression body for methods
dotnet_diagnostic.IDE0024.severity = silent # expression body for operators
dotnet_diagnostic.IDE0028.severity = silent
dotnet_diagnostic.IDE0033.severity = error # prefer tuple name
dotnet_diagnostic.IDE0040.severity = error # modifiers required
dotnet_diagnostic.IDE0041.severity = error # simplify null
dotnet_diagnostic.IDE0042.severity = error # deconstruct variable
dotnet_diagnostic.IDE0044.severity = error # make field only when possible
dotnet_diagnostic.IDE0047.severity = suggestion # paratemeter name
dotnet_diagnostic.IDE0051.severity = error # unused field
dotnet_diagnostic.IDE0052.severity = error # unused member
dotnet_diagnostic.IDE0060.severity = suggestion # unused parameters
dotnet_diagnostic.IDE0066.severity = suggestion # switch expression
dotnet_diagnostic.IDE0078.severity = suggestion # use pattern matching
dotnet_diagnostic.IDE0090.severity = suggestion # new can be simplified
dotnet_diagnostic.IDE0130.severity = suggestion # namespace folder structure
dotnet_diagnostic.IDE0160.severity = silent # Use block namespaces ARE NOT required
dotnet_diagnostic.IDE0161.severity = suggestion # Please use file namespaces
##########################################
# Styles
##########################################
# camel_case_style - Define the camelCase style
dotnet_naming_style.camel_case_style.capitalization = camel_case
# pascal_case_style - Define the PascalCase style
dotnet_naming_style.pascal_case_style.capitalization = pascal_case
# constant_case - Define the CONSTANT_CASE style
dotnet_naming_style.constant_case.capitalization = all_upper
dotnet_naming_style.constant_case.word_separator = _
# first_upper_style - The first character must start with an upper-case character
dotnet_naming_style.first_upper_style.capitalization = first_word_upper
# prefix_interface_with_i_style - Interfaces must be PascalCase and the first character of an interface must be an 'I'
dotnet_naming_style.prefix_interface_with_i_style.capitalization = pascal_case
dotnet_naming_style.prefix_interface_with_i_style.required_prefix = I
# prefix_type_parameters_with_t_style - Generic Type Parameters must be PascalCase and the first character must be a 'T'
dotnet_naming_style.prefix_type_parameters_with_t_style.capitalization = pascal_case
dotnet_naming_style.prefix_type_parameters_with_t_style.required_prefix = T
# disallowed_style - Anything that has this style applied is marked as disallowed
dotnet_naming_style.disallowed_style.capitalization = pascal_case
dotnet_naming_style.disallowed_style.required_prefix = ____RULE_VIOLATION____
dotnet_naming_style.disallowed_style.required_suffix = ____RULE_VIOLATION____
# internal_error_style - This style should never occur... if it does, it indicates a bug in file or in the parser using the file
dotnet_naming_style.internal_error_style.capitalization = pascal_case
dotnet_naming_style.internal_error_style.required_prefix = ____INTERNAL_ERROR____
dotnet_naming_style.internal_error_style.required_suffix = ____INTERNAL_ERROR____
# prefix_interface_with_i_style - Interfaces must be PascalCase and the first character of an interface must be an 'I'
dotnet_naming_style.underscore_camel_case_style.capitalization = camel_case
dotnet_naming_style.underscore_camel_case_style.required_prefix = _
##########################################
# .NET Design Guideline Field Naming Rules
# Naming rules for fields follow the .NET Framework design guidelines
# https://docs.microsoft.com/dotnet/standard/design-guidelines/index
##########################################
# All public/protected/protected_internal constant fields must be constant_case
# https://docs.microsoft.com/dotnet/standard/design-guidelines/field
dotnet_naming_symbols.public_protected_constant_fields_group.applicable_accessibilities = public, protected, protected_internal
dotnet_naming_symbols.public_protected_constant_fields_group.required_modifiers = const
dotnet_naming_symbols.public_protected_constant_fields_group.applicable_kinds = field
dotnet_naming_rule.public_protected_constant_fields_must_be_pascal_case_rule.symbols = public_protected_constant_fields_group
dotnet_naming_rule.public_protected_constant_fields_must_be_pascal_case_rule.style = constant_case
dotnet_naming_rule.public_protected_constant_fields_must_be_pascal_case_rule.severity = warning
# All public/protected/protected_internal static readonly fields must be constant_case
# https://docs.microsoft.com/dotnet/standard/design-guidelines/field
dotnet_naming_symbols.public_protected_static_readonly_fields_group.applicable_accessibilities = public, protected, protected_internal
dotnet_naming_symbols.public_protected_static_readonly_fields_group.required_modifiers = static, readonly
dotnet_naming_symbols.public_protected_static_readonly_fields_group.applicable_kinds = field
dotnet_naming_rule.public_protected_static_readonly_fields_must_be_pascal_case_rule.symbols = public_protected_static_readonly_fields_group
dotnet_naming_rule.public_protected_static_readonly_fields_must_be_pascal_case_rule.style = constant_case
dotnet_naming_rule.public_protected_static_readonly_fields_must_be_pascal_case_rule.severity = warning
# No other public/protected/protected_internal fields are allowed
# https://docs.microsoft.com/dotnet/standard/design-guidelines/field
dotnet_naming_symbols.other_public_protected_fields_group.applicable_accessibilities = public, protected, protected_internal
dotnet_naming_symbols.other_public_protected_fields_group.applicable_kinds = field
dotnet_naming_rule.other_public_protected_fields_disallowed_rule.symbols = other_public_protected_fields_group
dotnet_naming_rule.other_public_protected_fields_disallowed_rule.style = disallowed_style
dotnet_naming_rule.other_public_protected_fields_disallowed_rule.severity = error
##########################################
# StyleCop Field Naming Rules
# Naming rules for fields follow the StyleCop analyzers
# This does not override any rules using disallowed_style above
# https://github.com/DotNetAnalyzers/StyleCopAnalyzers
##########################################
# All constant fields must be constant_case
# https://github.com/DotNetAnalyzers/StyleCopAnalyzers/blob/master/documentation/SA1303.md
dotnet_naming_symbols.stylecop_constant_fields_group.applicable_accessibilities = public, internal, protected_internal, protected, private_protected, private
dotnet_naming_symbols.stylecop_constant_fields_group.required_modifiers = const
dotnet_naming_symbols.stylecop_constant_fields_group.applicable_kinds = field
dotnet_naming_rule.stylecop_constant_fields_must_be_pascal_case_rule.symbols = stylecop_constant_fields_group
dotnet_naming_rule.stylecop_constant_fields_must_be_pascal_case_rule.style = constant_case
dotnet_naming_rule.stylecop_constant_fields_must_be_pascal_case_rule.severity = warning
# All static readonly fields must be constant_case
# https://github.com/DotNetAnalyzers/StyleCopAnalyzers/blob/master/documentation/SA1311.md
dotnet_naming_symbols.stylecop_static_readonly_fields_group.applicable_accessibilities = public, internal, protected_internal, protected, private_protected, private
dotnet_naming_symbols.stylecop_static_readonly_fields_group.required_modifiers = static, readonly
dotnet_naming_symbols.stylecop_static_readonly_fields_group.applicable_kinds = field
dotnet_naming_rule.stylecop_static_readonly_fields_must_be_pascal_case_rule.symbols = stylecop_static_readonly_fields_group
dotnet_naming_rule.stylecop_static_readonly_fields_must_be_pascal_case_rule.style = constant_case
dotnet_naming_rule.stylecop_static_readonly_fields_must_be_pascal_case_rule.severity = warning
# No non-private instance fields are allowed
# https://github.com/DotNetAnalyzers/StyleCopAnalyzers/blob/master/documentation/SA1401.md
dotnet_naming_symbols.stylecop_fields_must_be_private_group.applicable_accessibilities = public, internal, protected_internal, protected, private_protected
dotnet_naming_symbols.stylecop_fields_must_be_private_group.applicable_kinds = field
dotnet_naming_rule.stylecop_instance_fields_must_be_private_rule.symbols = stylecop_fields_must_be_private_group
dotnet_naming_rule.stylecop_instance_fields_must_be_private_rule.style = disallowed_style
dotnet_naming_rule.stylecop_instance_fields_must_be_private_rule.severity = error
# Private fields must be camelCase
# https://github.com/DotNetAnalyzers/StyleCopAnalyzers/blob/master/documentation/SA1306.md
dotnet_naming_symbols.stylecop_private_fields_group.applicable_accessibilities = private
dotnet_naming_symbols.stylecop_private_fields_group.applicable_kinds = field
dotnet_naming_rule.stylecop_private_fields_must_be_camel_case_rule.symbols = stylecop_private_fields_group
dotnet_naming_rule.stylecop_private_fields_must_be_camel_case_rule.style = underscore_camel_case_style
dotnet_naming_rule.stylecop_private_fields_must_be_camel_case_rule.severity = warning
# Local variables must be camelCase
# https://github.com/DotNetAnalyzers/StyleCopAnalyzers/blob/master/documentation/SA1312.md
dotnet_naming_symbols.stylecop_local_fields_group.applicable_accessibilities = local
dotnet_naming_symbols.stylecop_local_fields_group.applicable_kinds = local
dotnet_naming_rule.stylecop_local_fields_must_be_camel_case_rule.symbols = stylecop_local_fields_group
dotnet_naming_rule.stylecop_local_fields_must_be_camel_case_rule.style = camel_case_style
dotnet_naming_rule.stylecop_local_fields_must_be_camel_case_rule.severity = warning
# This rule should never fire. However, it's included for at least two purposes:
# First, it helps to understand, reason about, and root-case certain types of issues, such as bugs in .editorconfig parsers.
# Second, it helps to raise immediate awareness if a new field type is added (as occurred recently in C#).
dotnet_naming_symbols.sanity_check_uncovered_field_case_group.applicable_accessibilities = *
dotnet_naming_symbols.sanity_check_uncovered_field_case_group.applicable_kinds = field
dotnet_naming_rule.sanity_check_uncovered_field_case_rule.symbols = sanity_check_uncovered_field_case_group
dotnet_naming_rule.sanity_check_uncovered_field_case_rule.style = internal_error_style
dotnet_naming_rule.sanity_check_uncovered_field_case_rule.severity = error
##########################################
# Other Naming Rules
##########################################
# All of the following must be PascalCase:
# - Namespaces
# https://docs.microsoft.com/dotnet/standard/design-guidelines/names-of-namespaces
# https://github.com/DotNetAnalyzers/StyleCopAnalyzers/blob/master/documentation/SA1300.md
# - Classes and Enumerations
# https://docs.microsoft.com/dotnet/standard/design-guidelines/names-of-classes-structs-and-interfaces
# https://github.com/DotNetAnalyzers/StyleCopAnalyzers/blob/master/documentation/SA1300.md
# - Delegates
# https://docs.microsoft.com/dotnet/standard/design-guidelines/names-of-classes-structs-and-interfaces#names-of-common-types
# - Constructors, Properties, Events, Methods
# https://docs.microsoft.com/dotnet/standard/design-guidelines/names-of-type-members
dotnet_naming_symbols.element_group.applicable_kinds = namespace, class, enum, struct, delegate, event, method, property
dotnet_naming_rule.element_rule.symbols = element_group
dotnet_naming_rule.element_rule.style = pascal_case_style
dotnet_naming_rule.element_rule.severity = warning
# Interfaces use PascalCase and are prefixed with uppercase 'I'
# https://docs.microsoft.com/dotnet/standard/design-guidelines/names-of-classes-structs-and-interfaces
dotnet_naming_symbols.interface_group.applicable_kinds = interface
dotnet_naming_rule.interface_rule.symbols = interface_group
dotnet_naming_rule.interface_rule.style = prefix_interface_with_i_style
dotnet_naming_rule.interface_rule.severity = warning
# Generics Type Parameters use PascalCase and are prefixed with uppercase 'T'
# https://docs.microsoft.com/dotnet/standard/design-guidelines/names-of-classes-structs-and-interfaces
dotnet_naming_symbols.type_parameter_group.applicable_kinds = type_parameter
dotnet_naming_rule.type_parameter_rule.symbols = type_parameter_group
dotnet_naming_rule.type_parameter_rule.style = prefix_type_parameters_with_t_style
dotnet_naming_rule.type_parameter_rule.severity = warning
# Function parameters use camelCase
# https://docs.microsoft.com/dotnet/standard/design-guidelines/naming-parameters
dotnet_naming_symbols.parameters_group.applicable_kinds = parameter
dotnet_naming_rule.parameters_rule.symbols = parameters_group
dotnet_naming_rule.parameters_rule.style = camel_case_style
dotnet_naming_rule.parameters_rule.severity = warning
##########################################
# License
##########################################
# The following applies as to the .editorconfig file ONLY, and is
# included below for reference, per the requirements of the license
# corresponding to this .editorconfig file.
# See: https://github.com/RehanSaeed/EditorConfig
#
# MIT License
#
# Copyright (c) 2017-2019 Muhammad Rehan Saeed
# Copyright (c) 2019 Henry Gabryjelski
#
# Permission is hereby granted, free of charge, to any
# person obtaining a copy of this software and associated
# documentation files (the "Software"), to deal in the
# Software without restriction, including without limitation
# the rights to use, copy, modify, merge, publish, distribute,
# sublicense, and/or sell copies of the Software, and to permit
# persons to whom the Software is furnished to do so, subject
# to the following conditions:
#
# The above copyright notice and this permission notice shall be
# included in all copies or substantial portions of the Software.
#
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES
# OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT
# HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
# FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
# OTHER DEALINGS IN THE SOFTWARE.
##########################################

View File

@@ -1,6 +0,0 @@
version: 2
updates:
- package-ecosystem: "github-actions" # search for actions - there are other options available
directory: "/" # search in .github/workflows under root `/`
schedule:
interval: "weekly" # check for action update every week

View File

@@ -9,12 +9,12 @@ jobs:
os: [windows-latest, ubuntu-latest]
steps:
- uses: actions/checkout@v3
- uses: actions/setup-dotnet@v2
- uses: actions/checkout@v1
- uses: actions/setup-dotnet@v1
with:
dotnet-version: 6.0.x
dotnet-version: 5.0.401
- run: dotnet run -p build/build.csproj
- uses: actions/upload-artifact@v3
- uses: actions/upload-artifact@v2
with:
name: ${{ matrix.os }}-sharpcompress.nupkg
path: artifacts/*

1
.gitignore vendored
View File

@@ -18,4 +18,3 @@ tools
.DS_Store
*.snupkg
/tests/TestArchives/6d23a38c-f064-4ef1-ad89-b942396f53b9/Scratch

View File

@@ -182,8 +182,6 @@ I'm always looking for help or ideas. Please submit code or email with ideas. Un
XZ implementation based on: https://github.com/sambott/XZ.NET by @sambott
XZ BCJ filters support contributed by Louis-Michel Bergeron, on behalf of aDolus Technology Inc. - 2022
7Zip implementation based on: https://code.google.com/p/managed-lzma/
LICENSE

View File

@@ -6,78 +6,78 @@ using GlobExpressions;
using static Bullseye.Targets;
using static SimpleExec.Command;
const string Clean = "clean";
const string Format = "format";
const string Build = "build";
const string Test = "test";
const string Publish = "publish";
class Program
{
private const string Clean = "clean";
private const string Format = "format";
private const string Build = "build";
private const string Test = "test";
private const string Publish = "publish";
Target(Clean,
ForEach("**/bin", "**/obj"),
dir =>
{
IEnumerable<string> GetDirectories(string d)
{
return Glob.Directories(".", d);
}
static void Main(string[] args)
{
Target(Clean,
ForEach("**/bin", "**/obj"),
dir =>
{
IEnumerable<string> GetDirectories(string d)
{
return Glob.Directories(".", d);
}
void RemoveDirectory(string d)
{
if (Directory.Exists(d))
void RemoveDirectory(string d)
{
if (Directory.Exists(d))
{
Console.WriteLine(d);
Directory.Delete(d, true);
}
}
foreach (var d in GetDirectories(dir))
{
RemoveDirectory(d);
}
});
Target(Format, () =>
{
Run("dotnet", "tool restore");
Run("dotnet", "format --check");
});
Target(Build, DependsOn(Format),
framework =>
{
Console.WriteLine(d);
Directory.Delete(d, true);
}
}
Run("dotnet", "build src/SharpCompress/SharpCompress.csproj -c Release");
});
foreach (var d in GetDirectories(dir))
{
RemoveDirectory(d);
}
});
Target(Test, DependsOn(Build), ForEach("net5.0", "net461"),
framework =>
{
IEnumerable<string> GetFiles(string d)
{
return Glob.Files(".", d);
}
if (!RuntimeInformation.IsOSPlatform(OSPlatform.Windows) && framework == "net461")
{
return;
}
Target(Format,
() =>
{
Run("dotnet", "tool restore");
Run("dotnet", "format --check");
});
foreach (var file in GetFiles("**/*.Test.csproj"))
{
Run("dotnet", $"test {file} -c Release -f {framework}");
}
});
Target(Build,
DependsOn(Format),
framework =>
{
Run("dotnet", "build src/SharpCompress/SharpCompress.csproj -c Release");
});
Target(Publish, DependsOn(Test),
() =>
{
Run("dotnet", "pack src/SharpCompress/SharpCompress.csproj -c Release -o artifacts/");
});
Target(Test,
DependsOn(Build),
ForEach("net6.0", "net461"),
framework =>
{
IEnumerable<string> GetFiles(string d)
{
return Glob.Files(".", d);
}
Target("default", DependsOn(Publish), () => Console.WriteLine("Done!"));
if (!RuntimeInformation.IsOSPlatform(OSPlatform.Windows) && framework == "net461")
{
return;
}
foreach (var file in GetFiles("**/*.Test.csproj"))
{
Run("dotnet", $"test {file} -c Release -f {framework}");
}
});
Target(Publish,
DependsOn(Test),
() =>
{
Run("dotnet", "pack src/SharpCompress/SharpCompress.csproj -c Release -o artifacts/");
});
Target("default", DependsOn(Publish), () => Console.WriteLine("Done!"));
await RunTargetsAndExitAsync(args);
RunTargetsAndExit(args);
}
}

View File

@@ -2,13 +2,13 @@
<PropertyGroup>
<OutputType>Exe</OutputType>
<TargetFramework>net6.0</TargetFramework>
<TargetFramework>net5.0</TargetFramework>
</PropertyGroup>
<ItemGroup>
<PackageReference Include="Bullseye" Version="4.0.0" />
<PackageReference Include="Glob" Version="1.1.9" />
<PackageReference Include="SimpleExec" Version="10.0.0" />
<PackageReference Include="Bullseye" Version="3.8.0" />
<PackageReference Include="Glob" Version="1.1.8" />
<PackageReference Include="SimpleExec" Version="8.0.0" />
</ItemGroup>
</Project>

View File

@@ -1,6 +1,6 @@
{
"sdk": {
"version": "6.0.200",
"version": "5.0.300",
"rollForward": "latestFeature"
}
}

View File

@@ -1,420 +0,0 @@
// Copyright (c) Six Labors.
// Licensed under the Apache License, Version 2.0.
#if !NETSTANDARD2_0 && !NETSTANDARD2_1 && !NETFRAMEWORK
#define SUPPORTS_RUNTIME_INTRINSICS
#define SUPPORTS_HOTPATH
#endif
using System;
using System.Runtime.CompilerServices;
using System.Runtime.InteropServices;
#if SUPPORTS_RUNTIME_INTRINSICS
using System.Runtime.Intrinsics;
using System.Runtime.Intrinsics.X86;
#endif
#pragma warning disable IDE0007 // Use implicit type
namespace SharpCompress.Algorithms
{
/// <summary>
/// Calculates the 32 bit Adler checksum of a given buffer according to
/// RFC 1950. ZLIB Compressed Data Format Specification version 3.3)
/// </summary>
internal static class Adler32 // From https://github.com/SixLabors/ImageSharp/blob/main/src/ImageSharp/Compression/Zlib/Adler32.cs
{
/// <summary>
/// Global inlining options. Helps temporarily disable inlining for better profiler output.
/// </summary>
private static class InliningOptions // From https://github.com/SixLabors/ImageSharp/blob/main/src/ImageSharp/Common/Helpers/InliningOptions.cs
{
/// <summary>
/// <see cref="MethodImplOptions.AggressiveInlining"/> regardless of the build conditions.
/// </summary>
public const MethodImplOptions AlwaysInline = MethodImplOptions.AggressiveInlining;
#if PROFILING
public const MethodImplOptions HotPath = MethodImplOptions.NoInlining;
public const MethodImplOptions ShortMethod = MethodImplOptions.NoInlining;
#else
#if SUPPORTS_HOTPATH
public const MethodImplOptions HotPath = MethodImplOptions.AggressiveOptimization;
#else
public const MethodImplOptions HotPath = MethodImplOptions.AggressiveInlining;
#endif
public const MethodImplOptions ShortMethod = MethodImplOptions.AggressiveInlining;
#endif
public const MethodImplOptions ColdPath = MethodImplOptions.NoInlining;
}
#if SUPPORTS_RUNTIME_INTRINSICS
/// <summary>
/// Provides optimized static methods for trigonometric, logarithmic,
/// and other common mathematical functions.
/// </summary>
private static class Numerics // From https://github.com/SixLabors/ImageSharp/blob/main/src/ImageSharp/Common/Helpers/Numerics.cs
{
/// <summary>
/// Reduces elements of the vector into one sum.
/// </summary>
/// <param name="accumulator">The accumulator to reduce.</param>
/// <returns>The sum of all elements.</returns>
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public static int ReduceSum(Vector256<int> accumulator)
{
// Add upper lane to lower lane.
Vector128<int> vsum = Sse2.Add(accumulator.GetLower(), accumulator.GetUpper());
// Add odd to even.
vsum = Sse2.Add(vsum, Sse2.Shuffle(vsum, 0b_11_11_01_01));
// Add high to low.
vsum = Sse2.Add(vsum, Sse2.Shuffle(vsum, 0b_11_10_11_10));
return Sse2.ConvertToInt32(vsum);
}
/// <summary>
/// Reduces even elements of the vector into one sum.
/// </summary>
/// <param name="accumulator">The accumulator to reduce.</param>
/// <returns>The sum of even elements.</returns>
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public static int EvenReduceSum(Vector256<int> accumulator)
{
Vector128<int> vsum = Sse2.Add(accumulator.GetLower(), accumulator.GetUpper()); // add upper lane to lower lane
vsum = Sse2.Add(vsum, Sse2.Shuffle(vsum, 0b_11_10_11_10)); // add high to low
// Vector128<int>.ToScalar() isn't optimized pre-net5.0 https://github.com/dotnet/runtime/pull/37882
return Sse2.ConvertToInt32(vsum);
}
}
#endif
/// <summary>
/// The default initial seed value of a Adler32 checksum calculation.
/// </summary>
public const uint SeedValue = 1U;
// Largest prime smaller than 65536
private const uint BASE = 65521;
// NMAX is the largest n such that 255n(n+1)/2 + (n+1)(BASE-1) <= 2^32-1
private const uint NMAX = 5552;
#if SUPPORTS_RUNTIME_INTRINSICS
private const int MinBufferSize = 64;
private const int BlockSize = 1 << 5;
// The C# compiler emits this as a compile-time constant embedded in the PE file.
private static ReadOnlySpan<byte> Tap1Tap2 => new byte[]
{
32, 31, 30, 29, 28, 27, 26, 25, 24, 23, 22, 21, 20, 19, 18, 17, // tap1
16, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1 // tap2
};
#endif
/// <summary>
/// Calculates the Adler32 checksum with the bytes taken from the span.
/// </summary>
/// <param name="buffer">The readonly span of bytes.</param>
/// <returns>The <see cref="uint"/>.</returns>
[MethodImpl(InliningOptions.ShortMethod)]
public static uint Calculate(ReadOnlySpan<byte> buffer)
=> Calculate(SeedValue, buffer);
/// <summary>
/// Calculates the Adler32 checksum with the bytes taken from the span and seed.
/// </summary>
/// <param name="adler">The input Adler32 value.</param>
/// <param name="buffer">The readonly span of bytes.</param>
/// <returns>The <see cref="uint"/>.</returns>
[MethodImpl(InliningOptions.HotPath | InliningOptions.ShortMethod)]
public static uint Calculate(uint adler, ReadOnlySpan<byte> buffer)
{
if (buffer.IsEmpty)
{
return adler;
}
#if SUPPORTS_RUNTIME_INTRINSICS
if (Avx2.IsSupported && buffer.Length >= MinBufferSize)
{
return CalculateAvx2(adler, buffer);
}
if (Ssse3.IsSupported && buffer.Length >= MinBufferSize)
{
return CalculateSse(adler, buffer);
}
return CalculateScalar(adler, buffer);
#else
return CalculateScalar(adler, buffer);
#endif
}
// Based on https://github.com/chromium/chromium/blob/master/third_party/zlib/adler32_simd.c
#if SUPPORTS_RUNTIME_INTRINSICS
[MethodImpl(InliningOptions.HotPath | InliningOptions.ShortMethod)]
private static unsafe uint CalculateSse(uint adler, ReadOnlySpan<byte> buffer)
{
uint s1 = adler & 0xFFFF;
uint s2 = (adler >> 16) & 0xFFFF;
// Process the data in blocks.
uint length = (uint)buffer.Length;
uint blocks = length / BlockSize;
length -= blocks * BlockSize;
fixed (byte* bufferPtr = &MemoryMarshal.GetReference(buffer))
{
fixed (byte* tapPtr = &MemoryMarshal.GetReference(Tap1Tap2))
{
byte* localBufferPtr = bufferPtr;
// _mm_setr_epi8 on x86
Vector128<sbyte> tap1 = Sse2.LoadVector128((sbyte*)tapPtr);
Vector128<sbyte> tap2 = Sse2.LoadVector128((sbyte*)(tapPtr + 0x10));
Vector128<byte> zero = Vector128<byte>.Zero;
var ones = Vector128.Create((short)1);
while (blocks > 0)
{
uint n = NMAX / BlockSize; /* The NMAX constraint. */
if (n > blocks)
{
n = blocks;
}
blocks -= n;
// Process n blocks of data. At most NMAX data bytes can be
// processed before s2 must be reduced modulo BASE.
Vector128<uint> v_ps = Vector128.CreateScalar(s1 * n);
Vector128<uint> v_s2 = Vector128.CreateScalar(s2);
Vector128<uint> v_s1 = Vector128<uint>.Zero;
do
{
// Load 32 input bytes.
Vector128<byte> bytes1 = Sse3.LoadDquVector128(localBufferPtr);
Vector128<byte> bytes2 = Sse3.LoadDquVector128(localBufferPtr + 0x10);
// Add previous block byte sum to v_ps.
v_ps = Sse2.Add(v_ps, v_s1);
// Horizontally add the bytes for s1, multiply-adds the
// bytes by [ 32, 31, 30, ... ] for s2.
v_s1 = Sse2.Add(v_s1, Sse2.SumAbsoluteDifferences(bytes1, zero).AsUInt32());
Vector128<short> mad1 = Ssse3.MultiplyAddAdjacent(bytes1, tap1);
v_s2 = Sse2.Add(v_s2, Sse2.MultiplyAddAdjacent(mad1, ones).AsUInt32());
v_s1 = Sse2.Add(v_s1, Sse2.SumAbsoluteDifferences(bytes2, zero).AsUInt32());
Vector128<short> mad2 = Ssse3.MultiplyAddAdjacent(bytes2, tap2);
v_s2 = Sse2.Add(v_s2, Sse2.MultiplyAddAdjacent(mad2, ones).AsUInt32());
localBufferPtr += BlockSize;
}
while (--n > 0);
v_s2 = Sse2.Add(v_s2, Sse2.ShiftLeftLogical(v_ps, 5));
// Sum epi32 ints v_s1(s2) and accumulate in s1(s2).
const byte S2301 = 0b1011_0001; // A B C D -> B A D C
const byte S1032 = 0b0100_1110; // A B C D -> C D A B
v_s1 = Sse2.Add(v_s1, Sse2.Shuffle(v_s1, S1032));
s1 += v_s1.ToScalar();
v_s2 = Sse2.Add(v_s2, Sse2.Shuffle(v_s2, S2301));
v_s2 = Sse2.Add(v_s2, Sse2.Shuffle(v_s2, S1032));
s2 = v_s2.ToScalar();
// Reduce.
s1 %= BASE;
s2 %= BASE;
}
if (length > 0)
{
HandleLeftOver(localBufferPtr, length, ref s1, ref s2);
}
return s1 | (s2 << 16);
}
}
}
// Based on: https://github.com/zlib-ng/zlib-ng/blob/develop/arch/x86/adler32_avx2.c
[MethodImpl(InliningOptions.HotPath | InliningOptions.ShortMethod)]
public static unsafe uint CalculateAvx2(uint adler, ReadOnlySpan<byte> buffer)
{
uint s1 = adler & 0xFFFF;
uint s2 = (adler >> 16) & 0xFFFF;
uint length = (uint)buffer.Length;
fixed (byte* bufferPtr = &MemoryMarshal.GetReference(buffer))
{
byte* localBufferPtr = bufferPtr;
Vector256<byte> zero = Vector256<byte>.Zero;
var dot3v = Vector256.Create((short)1);
var dot2v = Vector256.Create(32, 31, 30, 29, 28, 27, 26, 25, 24, 23, 22, 21, 20, 19, 18, 17, 16, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1);
// Process n blocks of data. At most NMAX data bytes can be
// processed before s2 must be reduced modulo BASE.
var vs1 = Vector256.CreateScalar(s1);
var vs2 = Vector256.CreateScalar(s2);
while (length >= 32)
{
int k = length < NMAX ? (int)length : (int)NMAX;
k -= k % 32;
length -= (uint)k;
Vector256<uint> vs10 = vs1;
Vector256<uint> vs3 = Vector256<uint>.Zero;
while (k >= 32)
{
// Load 32 input bytes.
Vector256<byte> block = Avx.LoadVector256(localBufferPtr);
// Sum of abs diff, resulting in 2 x int32's
Vector256<ushort> vs1sad = Avx2.SumAbsoluteDifferences(block, zero);
vs1 = Avx2.Add(vs1, vs1sad.AsUInt32());
vs3 = Avx2.Add(vs3, vs10);
// sum 32 uint8s to 16 shorts.
Vector256<short> vshortsum2 = Avx2.MultiplyAddAdjacent(block, dot2v);
// sum 16 shorts to 8 uint32s.
Vector256<int> vsum2 = Avx2.MultiplyAddAdjacent(vshortsum2, dot3v);
vs2 = Avx2.Add(vsum2.AsUInt32(), vs2);
vs10 = vs1;
localBufferPtr += BlockSize;
k -= 32;
}
// Defer the multiplication with 32 to outside of the loop.
vs3 = Avx2.ShiftLeftLogical(vs3, 5);
vs2 = Avx2.Add(vs2, vs3);
s1 = (uint)Numerics.EvenReduceSum(vs1.AsInt32());
s2 = (uint)Numerics.ReduceSum(vs2.AsInt32());
s1 %= BASE;
s2 %= BASE;
vs1 = Vector256.CreateScalar(s1);
vs2 = Vector256.CreateScalar(s2);
}
if (length > 0)
{
HandleLeftOver(localBufferPtr, length, ref s1, ref s2);
}
return s1 | (s2 << 16);
}
}
private static unsafe void HandleLeftOver(byte* localBufferPtr, uint length, ref uint s1, ref uint s2)
{
if (length >= 16)
{
s2 += s1 += localBufferPtr[0];
s2 += s1 += localBufferPtr[1];
s2 += s1 += localBufferPtr[2];
s2 += s1 += localBufferPtr[3];
s2 += s1 += localBufferPtr[4];
s2 += s1 += localBufferPtr[5];
s2 += s1 += localBufferPtr[6];
s2 += s1 += localBufferPtr[7];
s2 += s1 += localBufferPtr[8];
s2 += s1 += localBufferPtr[9];
s2 += s1 += localBufferPtr[10];
s2 += s1 += localBufferPtr[11];
s2 += s1 += localBufferPtr[12];
s2 += s1 += localBufferPtr[13];
s2 += s1 += localBufferPtr[14];
s2 += s1 += localBufferPtr[15];
localBufferPtr += 16;
length -= 16;
}
while (length-- > 0)
{
s2 += s1 += *localBufferPtr++;
}
if (s1 >= BASE)
{
s1 -= BASE;
}
s2 %= BASE;
}
#endif
[MethodImpl(InliningOptions.HotPath | InliningOptions.ShortMethod)]
private static unsafe uint CalculateScalar(uint adler, ReadOnlySpan<byte> buffer)
{
uint s1 = adler & 0xFFFF;
uint s2 = (adler >> 16) & 0xFFFF;
uint k;
fixed (byte* bufferPtr = buffer)
{
var localBufferPtr = bufferPtr;
uint length = (uint)buffer.Length;
while (length > 0)
{
k = length < NMAX ? length : NMAX;
length -= k;
while (k >= 16)
{
s2 += s1 += localBufferPtr[0];
s2 += s1 += localBufferPtr[1];
s2 += s1 += localBufferPtr[2];
s2 += s1 += localBufferPtr[3];
s2 += s1 += localBufferPtr[4];
s2 += s1 += localBufferPtr[5];
s2 += s1 += localBufferPtr[6];
s2 += s1 += localBufferPtr[7];
s2 += s1 += localBufferPtr[8];
s2 += s1 += localBufferPtr[9];
s2 += s1 += localBufferPtr[10];
s2 += s1 += localBufferPtr[11];
s2 += s1 += localBufferPtr[12];
s2 += s1 += localBufferPtr[13];
s2 += s1 += localBufferPtr[14];
s2 += s1 += localBufferPtr[15];
localBufferPtr += 16;
k -= 16;
}
while (k-- > 0)
{
s2 += s1 += *localBufferPtr++;
}
s1 %= BASE;
s2 %= BASE;
}
return (s2 << 16) | s1;
}
}
}
}

View File

@@ -0,0 +1,280 @@
// Copyright (c) Six Labors and contributors.
// Licensed under the GNU Affero General Public License, Version 3.
using System;
using System.Runtime.CompilerServices;
using System.Runtime.InteropServices;
#if !NETSTANDARD2_0 && !NETSTANDARD2_1 && !NETFRAMEWORK
using System.Runtime.Intrinsics;
using System.Runtime.Intrinsics.X86;
#endif
namespace SharpCompress.Algorithms
{
/// <summary>
/// Calculates the 32 bit Adler checksum of a given buffer according to
/// RFC 1950. ZLIB Compressed Data Format Specification version 3.3)
/// </summary>
internal static class Adler32
{
/// <summary>
/// The default initial seed value of a Adler32 checksum calculation.
/// </summary>
public const uint SeedValue = 1U;
#if !NETSTANDARD2_0 && !NETSTANDARD2_1 && !NETFRAMEWORK
private const int MinBufferSize = 64;
#endif
// Largest prime smaller than 65536
private const uint BASE = 65521;
// NMAX is the largest n such that 255n(n+1)/2 + (n+1)(BASE-1) <= 2^32-1
private const uint NMAX = 5552;
/// <summary>
/// Calculates the Adler32 checksum with the bytes taken from the span.
/// </summary>
/// <param name="buffer">The readonly span of bytes.</param>
/// <returns>The <see cref="uint"/>.</returns>
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public static uint Calculate(ReadOnlySpan<byte> buffer)
{
return Calculate(SeedValue, buffer);
}
/// <summary>
/// Calculates the Adler32 checksum with the bytes taken from the span and seed.
/// </summary>
/// <param name="adler">The input Adler32 value.</param>
/// <param name="buffer">The readonly span of bytes.</param>
/// <returns>The <see cref="uint"/>.</returns>
public static uint Calculate(uint adler, ReadOnlySpan<byte> buffer)
{
#if !NETSTANDARD2_0 && !NETSTANDARD2_1 && !NETFRAMEWORK
if (Sse3.IsSupported && buffer.Length >= MinBufferSize)
{
return CalculateSse(adler, buffer);
}
return CalculateScalar(adler, buffer);
#else
return CalculateScalar(adler, buffer);
#endif
}
// Based on https://github.com/chromium/chromium/blob/master/third_party/zlib/adler32_simd.c
#if !NETSTANDARD2_0 && !NETSTANDARD2_1 && !NETFRAMEWORK
private static unsafe uint CalculateSse(uint adler, ReadOnlySpan<byte> buffer)
{
uint s1 = adler & 0xFFFF;
uint s2 = (adler >> 16) & 0xFFFF;
// Process the data in blocks.
const int BLOCK_SIZE = 1 << 5;
uint length = (uint)buffer.Length;
uint blocks = length / BLOCK_SIZE;
length -= blocks * BLOCK_SIZE;
int index = 0;
fixed (byte* bufferPtr = &buffer[0])
{
index += (int)blocks * BLOCK_SIZE;
var localBufferPtr = bufferPtr;
// _mm_setr_epi8 on x86
var tap1 = Vector128.Create(32, 31, 30, 29, 28, 27, 26, 25, 24, 23, 22, 21, 20, 19, 18, 17);
var tap2 = Vector128.Create(16, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1);
Vector128<byte> zero = Vector128<byte>.Zero;
var ones = Vector128.Create((short)1);
while (blocks > 0)
{
uint n = NMAX / BLOCK_SIZE; /* The NMAX constraint. */
if (n > blocks)
{
n = blocks;
}
blocks -= n;
// Process n blocks of data. At most NMAX data bytes can be
// processed before s2 must be reduced modulo BASE.
Vector128<int> v_ps = Vector128.CreateScalar(s1 * n).AsInt32();
Vector128<int> v_s2 = Vector128.CreateScalar(s2).AsInt32();
Vector128<int> v_s1 = Vector128<int>.Zero;
do
{
// Load 32 input bytes.
Vector128<byte> bytes1 = Sse3.LoadDquVector128(localBufferPtr);
Vector128<byte> bytes2 = Sse3.LoadDquVector128(localBufferPtr + 16);
// Add previous block byte sum to v_ps.
v_ps = Sse2.Add(v_ps, v_s1);
// Horizontally add the bytes for s1, multiply-adds the
// bytes by [ 32, 31, 30, ... ] for s2.
v_s1 = Sse2.Add(v_s1, Sse2.SumAbsoluteDifferences(bytes1, zero).AsInt32());
Vector128<short> mad1 = Ssse3.MultiplyAddAdjacent(bytes1, tap1);
v_s2 = Sse2.Add(v_s2, Sse2.MultiplyAddAdjacent(mad1, ones));
v_s1 = Sse2.Add(v_s1, Sse2.SumAbsoluteDifferences(bytes2, zero).AsInt32());
Vector128<short> mad2 = Ssse3.MultiplyAddAdjacent(bytes2, tap2);
v_s2 = Sse2.Add(v_s2, Sse2.MultiplyAddAdjacent(mad2, ones));
localBufferPtr += BLOCK_SIZE;
}
while (--n > 0);
v_s2 = Sse2.Add(v_s2, Sse2.ShiftLeftLogical(v_ps, 5));
// Sum epi32 ints v_s1(s2) and accumulate in s1(s2).
const byte S2301 = 0b1011_0001; // A B C D -> B A D C
const byte S1032 = 0b0100_1110; // A B C D -> C D A B
v_s1 = Sse2.Add(v_s1, Sse2.Shuffle(v_s1, S2301));
v_s1 = Sse2.Add(v_s1, Sse2.Shuffle(v_s1, S1032));
s1 += (uint)v_s1.ToScalar();
v_s2 = Sse2.Add(v_s2, Sse2.Shuffle(v_s2, S2301));
v_s2 = Sse2.Add(v_s2, Sse2.Shuffle(v_s2, S1032));
s2 = (uint)v_s2.ToScalar();
// Reduce.
s1 %= BASE;
s2 %= BASE;
}
}
ref byte bufferRef = ref MemoryMarshal.GetReference(buffer);
if (length > 0)
{
if (length >= 16)
{
s1 += Unsafe.Add(ref bufferRef, index++);
s2 += s1;
s1 += Unsafe.Add(ref bufferRef, index++);
s2 += s1;
s1 += Unsafe.Add(ref bufferRef, index++);
s2 += s1;
s1 += Unsafe.Add(ref bufferRef, index++);
s2 += s1;
s1 += Unsafe.Add(ref bufferRef, index++);
s2 += s1;
s1 += Unsafe.Add(ref bufferRef, index++);
s2 += s1;
s1 += Unsafe.Add(ref bufferRef, index++);
s2 += s1;
s1 += Unsafe.Add(ref bufferRef, index++);
s2 += s1;
s1 += Unsafe.Add(ref bufferRef, index++);
s2 += s1;
s1 += Unsafe.Add(ref bufferRef, index++);
s2 += s1;
s1 += Unsafe.Add(ref bufferRef, index++);
s2 += s1;
s1 += Unsafe.Add(ref bufferRef, index++);
s2 += s1;
s1 += Unsafe.Add(ref bufferRef, index++);
s2 += s1;
s1 += Unsafe.Add(ref bufferRef, index++);
s2 += s1;
s1 += Unsafe.Add(ref bufferRef, index++);
s2 += s1;
s1 += Unsafe.Add(ref bufferRef, index++);
s2 += s1;
length -= 16;
}
while (length-- > 0)
{
s2 += s1 += Unsafe.Add(ref bufferRef, index++);
}
if (s1 >= BASE)
{
s1 -= BASE;
}
s2 %= BASE;
}
return s1 | (s2 << 16);
}
#endif
private static uint CalculateScalar(uint adler, ReadOnlySpan<byte> buffer)
{
uint s1 = adler & 0xFFFF;
uint s2 = (adler >> 16) & 0xFFFF;
uint k;
ref byte bufferRef = ref MemoryMarshal.GetReference<byte>(buffer);
uint length = (uint)buffer.Length;
int index = 0;
while (length > 0)
{
k = length < NMAX ? length : NMAX;
length -= k;
while (k >= 16)
{
s1 += Unsafe.Add(ref bufferRef, index++);
s2 += s1;
s1 += Unsafe.Add(ref bufferRef, index++);
s2 += s1;
s1 += Unsafe.Add(ref bufferRef, index++);
s2 += s1;
s1 += Unsafe.Add(ref bufferRef, index++);
s2 += s1;
s1 += Unsafe.Add(ref bufferRef, index++);
s2 += s1;
s1 += Unsafe.Add(ref bufferRef, index++);
s2 += s1;
s1 += Unsafe.Add(ref bufferRef, index++);
s2 += s1;
s1 += Unsafe.Add(ref bufferRef, index++);
s2 += s1;
s1 += Unsafe.Add(ref bufferRef, index++);
s2 += s1;
s1 += Unsafe.Add(ref bufferRef, index++);
s2 += s1;
s1 += Unsafe.Add(ref bufferRef, index++);
s2 += s1;
s1 += Unsafe.Add(ref bufferRef, index++);
s2 += s1;
s1 += Unsafe.Add(ref bufferRef, index++);
s2 += s1;
s1 += Unsafe.Add(ref bufferRef, index++);
s2 += s1;
s1 += Unsafe.Add(ref bufferRef, index++);
s2 += s1;
s1 += Unsafe.Add(ref bufferRef, index++);
s2 += s1;
k -= 16;
}
if (k != 0)
{
do
{
s1 += Unsafe.Add(ref bufferRef, index++);
s2 += s1;
}
while (--k != 0);
}
s1 %= BASE;
s2 %= BASE;
}
return (s2 << 16) | s1;
}
}
}

View File

@@ -1,9 +1,8 @@
using System;
using System;
using System.Collections.Generic;
using System.IO;
using System.Linq;
using SharpCompress.Common;
using SharpCompress.IO;
using SharpCompress.Readers;
namespace SharpCompress.Archives
@@ -24,14 +23,28 @@ namespace SharpCompress.Archives
protected ReaderOptions ReaderOptions { get; }
private bool disposed;
protected SourceStream SrcStream;
internal AbstractArchive(ArchiveType type, SourceStream srcStream)
internal AbstractArchive(ArchiveType type, FileInfo fileInfo, ReaderOptions readerOptions)
{
Type = type;
ReaderOptions = srcStream.ReaderOptions;
SrcStream = srcStream;
lazyVolumes = new LazyReadOnlyCollection<TVolume>(LoadVolumes(SrcStream));
if (!fileInfo.Exists)
{
throw new ArgumentException("File does not exist: " + fileInfo.FullName);
}
ReaderOptions = readerOptions;
readerOptions.LeaveStreamOpen = false;
lazyVolumes = new LazyReadOnlyCollection<TVolume>(LoadVolumes(fileInfo));
lazyEntries = new LazyReadOnlyCollection<TEntry>(LoadEntries(Volumes));
}
protected abstract IEnumerable<TVolume> LoadVolumes(FileInfo file);
internal AbstractArchive(ArchiveType type, IEnumerable<Stream> streams, ReaderOptions readerOptions)
{
Type = type;
ReaderOptions = readerOptions;
lazyVolumes = new LazyReadOnlyCollection<TVolume>(LoadVolumes(streams.Select(CheckStreams)));
lazyEntries = new LazyReadOnlyCollection<TEntry>(LoadEntries(Volumes));
}
@@ -85,7 +98,7 @@ namespace SharpCompress.Archives
/// </summary>
public virtual long TotalUncompressSize => Entries.Aggregate(0L, (total, cf) => total + cf.Size);
protected abstract IEnumerable<TVolume> LoadVolumes(SourceStream srcStream);
protected abstract IEnumerable<TVolume> LoadVolumes(IEnumerable<Stream> streams);
protected abstract IEnumerable<TEntry> LoadEntries(IEnumerable<TVolume> volumes);
IEnumerable<IArchiveEntry> IArchive.Entries => Entries.Cast<IArchiveEntry>();
@@ -98,8 +111,6 @@ namespace SharpCompress.Archives
{
lazyVolumes.ForEach(v => v.Dispose());
lazyEntries.GetLoaded().Cast<Entry>().ForEach(x => x.Close());
if (SrcStream != null)
SrcStream.Dispose();
disposed = true;
}
}

View File

@@ -1,9 +1,8 @@
using System;
using System;
using System.Collections.Generic;
using System.IO;
using System.Linq;
using SharpCompress.Common;
using SharpCompress.IO;
using SharpCompress.Readers;
using SharpCompress.Writers;
@@ -41,8 +40,13 @@ namespace SharpCompress.Archives
{
}
internal AbstractWritableArchive(ArchiveType type, SourceStream srcStream)
: base(type, srcStream)
internal AbstractWritableArchive(ArchiveType type, Stream stream, ReaderOptions readerFactoryOptions)
: base(type, stream.AsEnumerable(), readerFactoryOptions)
{
}
internal AbstractWritableArchive(ArchiveType type, FileInfo fileInfo, ReaderOptions readerFactoryOptions)
: base(type, fileInfo, readerFactoryOptions)
{
}

View File

@@ -1,14 +1,11 @@
using System;
using System.Collections.Generic;
using System;
using System.IO;
using System.Linq;
using SharpCompress.Archives.GZip;
using SharpCompress.Archives.Rar;
using SharpCompress.Archives.SevenZip;
using SharpCompress.Archives.Tar;
using SharpCompress.Archives.Zip;
using SharpCompress.Common;
using SharpCompress.IO;
using SharpCompress.Readers;
namespace SharpCompress.Archives
@@ -29,25 +26,34 @@ namespace SharpCompress.Archives
throw new ArgumentException("Stream should be readable and seekable");
}
readerOptions ??= new ReaderOptions();
ArchiveType? type;
IsArchive(stream, out type); //test and reset stream position
if (type != null)
if (ZipArchive.IsZipFile(stream, null))
{
switch (type.Value)
{
case ArchiveType.Zip:
return ZipArchive.Open(stream, readerOptions);
case ArchiveType.SevenZip:
return SevenZipArchive.Open(stream, readerOptions);
case ArchiveType.GZip:
return GZipArchive.Open(stream, readerOptions);
case ArchiveType.Rar:
return RarArchive.Open(stream, readerOptions);
case ArchiveType.Tar:
return TarArchive.Open(stream, readerOptions);
}
stream.Seek(0, SeekOrigin.Begin);
return ZipArchive.Open(stream, readerOptions);
}
stream.Seek(0, SeekOrigin.Begin);
if (SevenZipArchive.IsSevenZipFile(stream))
{
stream.Seek(0, SeekOrigin.Begin);
return SevenZipArchive.Open(stream, readerOptions);
}
stream.Seek(0, SeekOrigin.Begin);
if (GZipArchive.IsGZipFile(stream))
{
stream.Seek(0, SeekOrigin.Begin);
return GZipArchive.Open(stream, readerOptions);
}
stream.Seek(0, SeekOrigin.Begin);
if (RarArchive.IsRarFile(stream, readerOptions))
{
stream.Seek(0, SeekOrigin.Begin);
return RarArchive.Open(stream, readerOptions);
}
stream.Seek(0, SeekOrigin.Begin);
if (TarArchive.IsTarFile(stream))
{
stream.Seek(0, SeekOrigin.Begin);
return TarArchive.Open(stream, readerOptions);
}
throw new InvalidOperationException("Cannot determine compressed stream type. Supported Archive Formats: Zip, GZip, Tar, Rar, 7Zip, LZip");
}
@@ -84,108 +90,30 @@ namespace SharpCompress.Archives
fileInfo.CheckNotNull(nameof(fileInfo));
options ??= new ReaderOptions { LeaveStreamOpen = false };
ArchiveType? type;
using (Stream stream = fileInfo.OpenRead())
using var stream = fileInfo.OpenRead();
if (ZipArchive.IsZipFile(stream, null))
{
IsArchive(stream, out type); //test and reset stream position
if (type != null)
{
switch (type.Value)
{
case ArchiveType.Zip:
return ZipArchive.Open(fileInfo, options);
case ArchiveType.SevenZip:
return SevenZipArchive.Open(fileInfo, options);
case ArchiveType.GZip:
return GZipArchive.Open(fileInfo, options);
case ArchiveType.Rar:
return RarArchive.Open(fileInfo, options);
case ArchiveType.Tar:
return TarArchive.Open(fileInfo, options);
}
}
return ZipArchive.Open(fileInfo, options);
}
throw new InvalidOperationException("Cannot determine compressed stream type. Supported Archive Formats: Zip, GZip, Tar, Rar, 7Zip");
}
/// <summary>
/// Constructor with IEnumerable FileInfo objects, multi and split support.
/// </summary>
/// <param name="fileInfos"></param>
/// <param name="options"></param>
public static IArchive Open(IEnumerable<FileInfo> fileInfos, ReaderOptions? options = null)
{
fileInfos.CheckNotNull(nameof(fileInfos));
FileInfo[] files = fileInfos.ToArray();
if (files.Length == 0)
throw new InvalidOperationException("No files to open");
FileInfo fileInfo = files[0];
if (files.Length == 1)
return Open(fileInfo, options);
fileInfo.CheckNotNull(nameof(fileInfo));
options ??= new ReaderOptions { LeaveStreamOpen = false };
ArchiveType? type;
using (Stream stream = fileInfo.OpenRead())
IsArchive(stream, out type); //test and reset stream position
if (type != null)
stream.Seek(0, SeekOrigin.Begin);
if (SevenZipArchive.IsSevenZipFile(stream))
{
switch (type.Value)
{
case ArchiveType.Zip:
return ZipArchive.Open(files, options);
case ArchiveType.SevenZip:
return SevenZipArchive.Open(files, options);
case ArchiveType.GZip:
return GZipArchive.Open(files, options);
case ArchiveType.Rar:
return RarArchive.Open(files, options);
case ArchiveType.Tar:
return TarArchive.Open(files, options);
}
return SevenZipArchive.Open(fileInfo, options);
}
throw new InvalidOperationException("Cannot determine compressed stream type. Supported Archive Formats: Zip, GZip, Tar, Rar, 7Zip");
}
/// <summary>
/// Constructor with IEnumerable FileInfo objects, multi and split support.
/// </summary>
/// <param name="streams"></param>
/// <param name="options"></param>
public static IArchive Open(IEnumerable<Stream> streams, ReaderOptions? options = null)
{
streams.CheckNotNull(nameof(streams));
if (streams.Count() == 0)
throw new InvalidOperationException("No streams");
if (streams.Count() == 1)
return Open(streams.First(), options);
options ??= new ReaderOptions();
ArchiveType? type;
using (Stream stream = streams.First())
IsArchive(stream, out type); //test and reset stream position
if (type != null)
stream.Seek(0, SeekOrigin.Begin);
if (GZipArchive.IsGZipFile(stream))
{
switch (type.Value)
{
case ArchiveType.Zip:
return ZipArchive.Open(streams, options);
case ArchiveType.SevenZip:
return SevenZipArchive.Open(streams, options);
case ArchiveType.GZip:
return GZipArchive.Open(streams, options);
case ArchiveType.Rar:
return RarArchive.Open(streams, options);
case ArchiveType.Tar:
return TarArchive.Open(streams, options);
}
return GZipArchive.Open(fileInfo, options);
}
stream.Seek(0, SeekOrigin.Begin);
if (RarArchive.IsRarFile(stream, options))
{
return RarArchive.Open(fileInfo, options);
}
stream.Seek(0, SeekOrigin.Begin);
if (TarArchive.IsTarFile(stream))
{
return TarArchive.Open(fileInfo, options);
}
throw new InvalidOperationException("Cannot determine compressed stream type. Supported Archive Formats: Zip, GZip, Tar, Rar, 7Zip");
}
@@ -202,95 +130,5 @@ namespace SharpCompress.Archives
entry.WriteToDirectory(destinationDirectory, options);
}
}
public static bool IsArchive(string filePath, out ArchiveType? type)
{
filePath.CheckNotNullOrEmpty(nameof(filePath));
using (Stream s = File.OpenRead(filePath))
return IsArchive(s, out type);
}
private static bool IsArchive(Stream stream, out ArchiveType? type)
{
type = null;
stream.CheckNotNull(nameof(stream));
if (!stream.CanRead || !stream.CanSeek)
{
throw new ArgumentException("Stream should be readable and seekable");
}
if (ZipArchive.IsZipFile(stream, null))
type = ArchiveType.Zip;
stream.Seek(0, SeekOrigin.Begin);
if (type == null)
{
if (SevenZipArchive.IsSevenZipFile(stream))
type = ArchiveType.SevenZip;
stream.Seek(0, SeekOrigin.Begin);
}
if (type == null)
{
if (GZipArchive.IsGZipFile(stream))
type = ArchiveType.GZip;
stream.Seek(0, SeekOrigin.Begin);
}
if (type == null)
{
if (RarArchive.IsRarFile(stream))
type = ArchiveType.Rar;
stream.Seek(0, SeekOrigin.Begin);
}
if (type == null)
{
if (TarArchive.IsTarFile(stream))
type = ArchiveType.Tar;
stream.Seek(0, SeekOrigin.Begin);
}
if (type == null) //test multipartzip as it could find zips in other non compressed archive types?
{
if (ZipArchive.IsZipMulti(stream)) //test the zip (last) file of a multipart zip
type = ArchiveType.Zip;
stream.Seek(0, SeekOrigin.Begin);
}
return type != null;
}
/// <summary>
/// From a passed in archive (zip, rar, 7z, 001), return all parts.
/// </summary>
/// <param name="part1"></param>
/// <returns></returns>
public static IEnumerable<string> GetFileParts(string part1)
{
part1.CheckNotNullOrEmpty(nameof(part1));
return GetFileParts(new FileInfo(part1)).Select(a => a.FullName);
}
/// <summary>
/// From a passed in archive (zip, rar, 7z, 001), return all parts.
/// </summary>
/// <param name="part1"></param>
/// <returns></returns>
public static IEnumerable<FileInfo> GetFileParts(FileInfo part1)
{
part1.CheckNotNull(nameof(part1));
yield return part1;
int i = 1;
FileInfo? part = RarArchiveVolumeFactory.GetFilePart(i++, part1);
if (part != null)
{
yield return part;
while ((part = RarArchiveVolumeFactory.GetFilePart(i++, part1)) != null) //tests split too
yield return part;
}
else
{
i = 1;
while ((part = ZipArchiveVolumeFactory.GetFilePart(i++, part1)) != null) //tests split too
yield return part;
}
}
}
}

View File

@@ -1,29 +0,0 @@
using System;
using System.Collections.Generic;
using System.IO;
using SharpCompress.Readers;
using System.Linq;
using System.Text;
using SharpCompress.Common.Rar.Headers;
using System.Text.RegularExpressions;
namespace SharpCompress.Archives
{
internal abstract class ArchiveVolumeFactory
{
internal static FileInfo? GetFilePart(int index, FileInfo part1) //base the name on the first part
{
FileInfo? item = null;
//split 001, 002 ...
Match m = Regex.Match(part1.Name, @"^(.*\.)([0-9]+)$", RegexOptions.IgnoreCase);
if (m.Success)
item = new FileInfo(Path.Combine(part1.DirectoryName!, String.Concat(m.Groups[1].Value, (index + 1).ToString().PadLeft(m.Groups[2].Value.Length, '0'))));
if (item != null && item.Exists)
return item;
return null;
}
}
}

View File

@@ -1,10 +1,9 @@
using System;
using System;
using System.Collections.Generic;
using System.IO;
using System.Linq;
using SharpCompress.Common;
using SharpCompress.Common.GZip;
using SharpCompress.IO;
using SharpCompress.Readers;
using SharpCompress.Readers.GZip;
using SharpCompress.Writers;
@@ -33,31 +32,7 @@ namespace SharpCompress.Archives.GZip
public static GZipArchive Open(FileInfo fileInfo, ReaderOptions? readerOptions = null)
{
fileInfo.CheckNotNull(nameof(fileInfo));
return new GZipArchive(new SourceStream(fileInfo, i => ArchiveVolumeFactory.GetFilePart(i, fileInfo), readerOptions ?? new ReaderOptions()));
}
/// <summary>
/// Constructor with all file parts passed in
/// </summary>
/// <param name="fileInfos"></param>
/// <param name="readerOptions"></param>
public static GZipArchive Open(IEnumerable<FileInfo> fileInfos, ReaderOptions? readerOptions = null)
{
fileInfos.CheckNotNull(nameof(fileInfos));
FileInfo[] files = fileInfos.ToArray();
return new GZipArchive(new SourceStream(files[0], i => i < files.Length ? files[i] : null, readerOptions ?? new ReaderOptions()));
}
/// <summary>
/// Constructor with all stream parts passed in
/// </summary>
/// <param name="streams"></param>
/// <param name="readerOptions"></param>
public static GZipArchive Open(IEnumerable<Stream> streams, ReaderOptions? readerOptions = null)
{
streams.CheckNotNull(nameof(streams));
Stream[] strms = streams.ToArray();
return new GZipArchive(new SourceStream(strms[0], i => i < strms.Length ? strms[i] : null, readerOptions ?? new ReaderOptions()));
return new GZipArchive(fileInfo, readerOptions ?? new ReaderOptions());
}
/// <summary>
@@ -68,7 +43,7 @@ namespace SharpCompress.Archives.GZip
public static GZipArchive Open(Stream stream, ReaderOptions? readerOptions = null)
{
stream.CheckNotNull(nameof(stream));
return new GZipArchive(new SourceStream(stream, i => null, readerOptions ?? new ReaderOptions()));
return new GZipArchive(stream, readerOptions ?? new ReaderOptions());
}
public static GZipArchive Create()
@@ -77,20 +52,20 @@ namespace SharpCompress.Archives.GZip
}
/// <summary>
/// Constructor with a SourceStream able to handle FileInfo and Streams.
/// Constructor with a FileInfo object to an existing file.
/// </summary>
/// <param name="srcStream"></param>
/// <param name="fileInfo"></param>
/// <param name="options"></param>
internal GZipArchive(SourceStream srcStream)
: base(ArchiveType.Tar, srcStream)
internal GZipArchive(FileInfo fileInfo, ReaderOptions options)
: base(ArchiveType.GZip, fileInfo, options)
{
}
protected override IEnumerable<GZipVolume> LoadVolumes(SourceStream srcStream)
protected override IEnumerable<GZipVolume> LoadVolumes(FileInfo file)
{
srcStream.LoadAllParts();
return srcStream.Streams.Select(a => new GZipVolume(a, ReaderOptions));
return new GZipVolume(file, ReaderOptions).AsEnumerable();
}
public static bool IsGZipFile(string filePath)
{
return IsGZipFile(new FileInfo(filePath));
@@ -139,6 +114,16 @@ namespace SharpCompress.Archives.GZip
return true;
}
/// <summary>
/// Takes multiple seekable Streams for a multi-part archive
/// </summary>
/// <param name="stream"></param>
/// <param name="options"></param>
internal GZipArchive(Stream stream, ReaderOptions options)
: base(ArchiveType.GZip, stream, options)
{
}
internal GZipArchive()
: base(ArchiveType.GZip)
{
@@ -175,6 +160,11 @@ namespace SharpCompress.Archives.GZip
}
}
protected override IEnumerable<GZipVolume> LoadVolumes(IEnumerable<Stream> streams)
{
return new GZipVolume(streams.First(), ReaderOptions).AsEnumerable();
}
protected override IEnumerable<GZipArchiveEntry> LoadEntries(IEnumerable<GZipVolume> volumes)
{
Stream stream = volumes.Single().Stream;

View File

@@ -5,7 +5,6 @@ using SharpCompress.Common;
using SharpCompress.Common.Rar;
using SharpCompress.Common.Rar.Headers;
using SharpCompress.Compressors.Rar;
using SharpCompress.IO;
using SharpCompress.Readers;
using SharpCompress.Readers.Rar;
@@ -17,14 +16,28 @@ namespace SharpCompress.Archives.Rar
internal Lazy<IRarUnpack> UnpackV2017 { get; } = new Lazy<IRarUnpack>(() => new SharpCompress.Compressors.Rar.UnpackV2017.Unpack());
internal Lazy<IRarUnpack> UnpackV1 { get; } = new Lazy<IRarUnpack>(() => new SharpCompress.Compressors.Rar.UnpackV1.Unpack());
/// <summary>
/// Constructor with a FileInfo object to an existing file.
/// </summary>
/// <param name="fileInfo"></param>
/// <param name="options"></param>
internal RarArchive(FileInfo fileInfo, ReaderOptions options)
: base(ArchiveType.Rar, fileInfo, options)
{
}
protected override IEnumerable<RarVolume> LoadVolumes(FileInfo file)
{
return RarArchiveVolumeFactory.GetParts(file, ReaderOptions);
}
/// <summary>
/// Constructor with a SourceStream able to handle FileInfo and Streams.
/// Takes multiple seekable Streams for a multi-part archive
/// </summary>
/// <param name="srcStream"></param>
/// <param name="streams"></param>
/// <param name="options"></param>
internal RarArchive(SourceStream srcStream)
: base(ArchiveType.Rar, srcStream)
internal RarArchive(IEnumerable<Stream> streams, ReaderOptions options)
: base(ArchiveType.Rar, streams, options)
{
}
@@ -33,20 +46,9 @@ namespace SharpCompress.Archives.Rar
return RarArchiveEntryFactory.GetEntries(this, volumes, ReaderOptions);
}
protected override IEnumerable<RarVolume> LoadVolumes(SourceStream srcStream)
protected override IEnumerable<RarVolume> LoadVolumes(IEnumerable<Stream> streams)
{
base.SrcStream.LoadAllParts(); //request all streams
Stream[] streams = base.SrcStream.Streams.ToArray();
if (streams.Length > 1 && IsRarFile(streams[1], ReaderOptions)) //test part 2 - true = multipart not split
{
base.SrcStream.IsVolumes = true;
streams[1].Position = 0;
base.SrcStream.Position = 0;
return srcStream.Streams.Select(a => new StreamRarArchiveVolume(a, ReaderOptions));
}
else //split mode or single file
return new StreamRarArchiveVolume(base.SrcStream, ReaderOptions).AsEnumerable();
return RarArchiveVolumeFactory.GetParts(streams, ReaderOptions);
}
protected override IReader CreateReaderForSolidExtraction()
@@ -67,8 +69,7 @@ namespace SharpCompress.Archives.Rar
public static RarArchive Open(string filePath, ReaderOptions? options = null)
{
filePath.CheckNotNullOrEmpty(nameof(filePath));
FileInfo fileInfo = new FileInfo(filePath);
return new RarArchive(new SourceStream(fileInfo, i => RarArchiveVolumeFactory.GetFilePart(i, fileInfo), options ?? new ReaderOptions()));
return new RarArchive(new FileInfo(filePath), options ?? new ReaderOptions());
}
/// <summary>
@@ -79,7 +80,7 @@ namespace SharpCompress.Archives.Rar
public static RarArchive Open(FileInfo fileInfo, ReaderOptions? options = null)
{
fileInfo.CheckNotNull(nameof(fileInfo));
return new RarArchive(new SourceStream(fileInfo, i => RarArchiveVolumeFactory.GetFilePart(i, fileInfo), options ?? new ReaderOptions()));
return new RarArchive(fileInfo, options ?? new ReaderOptions());
}
/// <summary>
@@ -90,34 +91,20 @@ namespace SharpCompress.Archives.Rar
public static RarArchive Open(Stream stream, ReaderOptions? options = null)
{
stream.CheckNotNull(nameof(stream));
return new RarArchive(new SourceStream(stream, i => null, options ?? new ReaderOptions()));
return Open(stream.AsEnumerable(), options ?? new ReaderOptions());
}
/// <summary>
/// Constructor with all file parts passed in
/// </summary>
/// <param name="fileInfos"></param>
/// <param name="readerOptions"></param>
public static RarArchive Open(IEnumerable<FileInfo> fileInfos, ReaderOptions? readerOptions = null)
{
fileInfos.CheckNotNull(nameof(fileInfos));
FileInfo[] files = fileInfos.ToArray();
return new RarArchive(new SourceStream(files[0], i => i < files.Length ? files[i] : null, readerOptions ?? new ReaderOptions()));
}
/// <summary>
/// Constructor with all stream parts passed in
/// Takes multiple seekable Streams for a multi-part archive
/// </summary>
/// <param name="streams"></param>
/// <param name="readerOptions"></param>
public static RarArchive Open(IEnumerable<Stream> streams, ReaderOptions? readerOptions = null)
/// <param name="options"></param>
public static RarArchive Open(IEnumerable<Stream> streams, ReaderOptions? options = null)
{
streams.CheckNotNull(nameof(streams));
Stream[] strms = streams.ToArray();
return new RarArchive(new SourceStream(strms[0], i => i < strms.Length ? strms[i] : null, readerOptions ?? new ReaderOptions()));
return new RarArchive(streams, options ?? new ReaderOptions());
}
public static bool IsRarFile(string filePath)
{
return IsRarFile(new FileInfo(filePath));

View File

@@ -21,7 +21,6 @@ namespace SharpCompress.Archives.Rar
this.parts = parts.ToList();
this.archive = archive;
this.readerOptions = readerOptions;
this.IsSolid = this.FileHeader.IsSolid;
}
public override CompressionType CompressionType => CompressionType.Rar;

View File

@@ -1,4 +1,4 @@
using System;
using System;
using System.Collections.Generic;
using System.IO;
using SharpCompress.Common.Rar;
@@ -6,35 +6,135 @@ using SharpCompress.Readers;
using System.Linq;
using System.Text;
using SharpCompress.Common.Rar.Headers;
using System.Text.RegularExpressions;
namespace SharpCompress.Archives.Rar
{
internal static class RarArchiveVolumeFactory
{
internal static FileInfo? GetFilePart(int index, FileInfo part1) //base the name on the first part
internal static IEnumerable<RarVolume> GetParts(IEnumerable<Stream> streams, ReaderOptions options)
{
FileInfo? item = null;
//new style rar - ..part1 | /part01 | part001 ....
Match m = Regex.Match(part1.Name, @"^(.*\.part)([0-9]+)(\.rar)$", RegexOptions.IgnoreCase);
if (m.Success)
item = new FileInfo(Path.Combine(part1.DirectoryName!, String.Concat(m.Groups[1].Value, (index + 1).ToString().PadLeft(m.Groups[2].Value.Length, '0'), m.Groups[3].Value)));
else
foreach (Stream s in streams)
{
//old style - ...rar, .r00, .r01 ...
m = Regex.Match(part1.Name, @"^(.*\.r)(ar|[0-9]+)$", RegexOptions.IgnoreCase);
if (m.Success)
item = new FileInfo(Path.Combine(part1.DirectoryName!, String.Concat(m.Groups[1].Value, index == 0 ? "ar" : (index - 1).ToString().PadLeft(m.Groups[2].Value.Length, '0'))));
else //split .001, .002 ....
return ArchiveVolumeFactory.GetFilePart(index, part1);
if (!s.CanRead || !s.CanSeek)
{
throw new ArgumentException("Stream is not readable and seekable");
}
StreamRarArchiveVolume part = new StreamRarArchiveVolume(s, options);
yield return part;
}
if (item != null && item.Exists)
return item;
return null; //no more items
}
internal static IEnumerable<RarVolume> GetParts(FileInfo fileInfo, ReaderOptions options)
{
FileInfoRarArchiveVolume part = new FileInfoRarArchiveVolume(fileInfo, options);
yield return part;
ArchiveHeader ah = part.ArchiveHeader;
if (!ah.IsVolume)
{
yield break; //if file isn't volume then there is no reason to look
}
fileInfo = GetNextFileInfo(ah, part.FileParts.FirstOrDefault() as FileInfoRarFilePart)!;
//we use fileinfo because rar is dumb and looks at file names rather than archive info for another volume
while (fileInfo != null && fileInfo.Exists)
{
part = new FileInfoRarArchiveVolume(fileInfo, options);
fileInfo = GetNextFileInfo(ah, part.FileParts.FirstOrDefault() as FileInfoRarFilePart)!;
yield return part;
}
}
private static FileInfo? GetNextFileInfo(ArchiveHeader ah, FileInfoRarFilePart? currentFilePart)
{
if (currentFilePart is null)
{
return null;
}
bool oldNumbering = ah.OldNumberingFormat
|| currentFilePart.MarkHeader.OldNumberingFormat;
if (oldNumbering)
{
return FindNextFileWithOldNumbering(currentFilePart.FileInfo);
}
else
{
return FindNextFileWithNewNumbering(currentFilePart.FileInfo);
}
}
private static FileInfo FindNextFileWithOldNumbering(FileInfo currentFileInfo)
{
// .rar, .r00, .r01, ...
string extension = currentFileInfo.Extension;
var buffer = new StringBuilder(currentFileInfo.FullName.Length);
buffer.Append(currentFileInfo.FullName.Substring(0,
currentFileInfo.FullName.Length - extension.Length));
if (string.Compare(extension, ".rar", StringComparison.OrdinalIgnoreCase) == 0)
{
buffer.Append(".r00");
}
else
{
if (int.TryParse(extension.Substring(2, 2), out int num))
{
num++;
buffer.Append(".r");
if (num < 10)
{
buffer.Append('0');
}
buffer.Append(num);
}
else
{
ThrowInvalidFileName(currentFileInfo);
}
}
return new FileInfo(buffer.ToString());
}
private static FileInfo FindNextFileWithNewNumbering(FileInfo currentFileInfo)
{
// part1.rar, part2.rar, ...
string extension = currentFileInfo.Extension;
if (string.Compare(extension, ".rar", StringComparison.OrdinalIgnoreCase) != 0)
{
throw new ArgumentException("Invalid extension, expected 'rar': " + currentFileInfo.FullName);
}
int startIndex = currentFileInfo.FullName.LastIndexOf(".part");
if (startIndex < 0)
{
ThrowInvalidFileName(currentFileInfo);
}
StringBuilder buffer = new StringBuilder(currentFileInfo.FullName.Length);
buffer.Append(currentFileInfo.FullName, 0, startIndex);
string numString = currentFileInfo.FullName.Substring(startIndex + 5,
currentFileInfo.FullName.IndexOf('.', startIndex + 5) -
startIndex - 5);
buffer.Append(".part");
if (int.TryParse(numString, out int num))
{
num++;
for (int i = 0; i < numString.Length - num.ToString().Length; i++)
{
buffer.Append('0');
}
buffer.Append(num);
}
else
{
ThrowInvalidFileName(currentFileInfo);
}
buffer.Append(".rar");
return new FileInfo(buffer.ToString());
}
private static void ThrowInvalidFileName(FileInfo fileInfo)
{
throw new ArgumentException("Filename invalid or next archive could not be found:"
+ fileInfo.FullName);
}
}
}
}

View File

@@ -1,4 +1,4 @@
#nullable disable
#nullable disable
using System;
using System.Collections.Generic;
@@ -34,33 +34,8 @@ namespace SharpCompress.Archives.SevenZip
public static SevenZipArchive Open(FileInfo fileInfo, ReaderOptions readerOptions = null)
{
fileInfo.CheckNotNull("fileInfo");
return new SevenZipArchive(new SourceStream(fileInfo, i => ArchiveVolumeFactory.GetFilePart(i, fileInfo), readerOptions ?? new ReaderOptions()));
return new SevenZipArchive(fileInfo, readerOptions ?? new ReaderOptions());
}
/// <summary>
/// Constructor with all file parts passed in
/// </summary>
/// <param name="fileInfos"></param>
/// <param name="readerOptions"></param>
public static SevenZipArchive Open(IEnumerable<FileInfo> fileInfos, ReaderOptions readerOptions = null)
{
fileInfos.CheckNotNull(nameof(fileInfos));
FileInfo[] files = fileInfos.ToArray();
return new SevenZipArchive(new SourceStream(files[0], i => i < files.Length ? files[i] : null, readerOptions ?? new ReaderOptions()));
}
/// <summary>
/// Constructor with all stream parts passed in
/// </summary>
/// <param name="streams"></param>
/// <param name="readerOptions"></param>
public static SevenZipArchive Open(IEnumerable<Stream> streams, ReaderOptions readerOptions = null)
{
streams.CheckNotNull(nameof(streams));
Stream[] strms = streams.ToArray();
return new SevenZipArchive(new SourceStream(strms[0], i => i < strms.Length ? strms[i] : null, readerOptions ?? new ReaderOptions()));
}
/// <summary>
/// Takes a seekable Stream as a source
/// </summary>
@@ -69,23 +44,17 @@ namespace SharpCompress.Archives.SevenZip
public static SevenZipArchive Open(Stream stream, ReaderOptions readerOptions = null)
{
stream.CheckNotNull("stream");
return new SevenZipArchive(new SourceStream(stream, i => null, readerOptions ?? new ReaderOptions()));
return new SevenZipArchive(stream, readerOptions ?? new ReaderOptions());
}
/// <summary>
/// Constructor with a SourceStream able to handle FileInfo and Streams.
/// </summary>
/// <param name="srcStream"></param>
/// <param name="options"></param>
internal SevenZipArchive(SourceStream srcStream)
: base(ArchiveType.SevenZip, srcStream)
internal SevenZipArchive(FileInfo fileInfo, ReaderOptions readerOptions)
: base(ArchiveType.SevenZip, fileInfo, readerOptions)
{
}
protected override IEnumerable<SevenZipVolume> LoadVolumes(SourceStream srcStream)
protected override IEnumerable<SevenZipVolume> LoadVolumes(FileInfo file)
{
base.SrcStream.LoadAllParts(); //request all streams
return new SevenZipVolume(srcStream, ReaderOptions).AsEnumerable(); //simple single volume or split, multivolume not supported
return new SevenZipVolume(file.OpenRead(), ReaderOptions).AsEnumerable();
}
public static bool IsSevenZipFile(string filePath)
@@ -105,32 +74,38 @@ namespace SharpCompress.Archives.SevenZip
}
}
internal SevenZipArchive(Stream stream, ReaderOptions readerOptions)
: base(ArchiveType.SevenZip, stream.AsEnumerable(), readerOptions)
{
}
internal SevenZipArchive()
: base(ArchiveType.SevenZip)
{
}
protected override IEnumerable<SevenZipVolume> LoadVolumes(IEnumerable<Stream> streams)
{
foreach (Stream s in streams)
{
if (!s.CanRead || !s.CanSeek)
{
throw new ArgumentException("Stream is not readable and seekable");
}
SevenZipVolume volume = new SevenZipVolume(s, ReaderOptions);
yield return volume;
}
}
protected override IEnumerable<SevenZipArchiveEntry> LoadEntries(IEnumerable<SevenZipVolume> volumes)
{
var stream = volumes.Single().Stream;
LoadFactory(stream);
var entries = new SevenZipArchiveEntry[database._files.Count];
for (int i = 0; i < database._files.Count; i++)
{
var file = database._files[i];
entries[i] = new SevenZipArchiveEntry(this, new SevenZipFilePart(stream, database, i, file, ReaderOptions.ArchiveEncoding));
yield return new SevenZipArchiveEntry(this, new SevenZipFilePart(stream, database, i, file, ReaderOptions.ArchiveEncoding));
}
foreach (var group in entries.Where(x => !x.IsDirectory).GroupBy(x => x.FilePart.Folder))
{
var isSolid = false;
foreach (var entry in group)
{
entry.IsSolid = isSolid;
isSolid = true; //mark others in this group as solid - same as rar behaviour.
}
}
return entries;
}
private void LoadFactory(Stream stream)

View File

@@ -1,4 +1,4 @@
using System;
using System;
using System.Collections.Generic;
using System.IO;
using System.Linq;
@@ -34,31 +34,7 @@ namespace SharpCompress.Archives.Tar
public static TarArchive Open(FileInfo fileInfo, ReaderOptions? readerOptions = null)
{
fileInfo.CheckNotNull(nameof(fileInfo));
return new TarArchive(new SourceStream(fileInfo, i => ArchiveVolumeFactory.GetFilePart(i, fileInfo), readerOptions ?? new ReaderOptions()));
}
/// <summary>
/// Constructor with all file parts passed in
/// </summary>
/// <param name="fileInfos"></param>
/// <param name="readerOptions"></param>
public static TarArchive Open(IEnumerable<FileInfo> fileInfos, ReaderOptions? readerOptions = null)
{
fileInfos.CheckNotNull(nameof(fileInfos));
FileInfo[] files = fileInfos.ToArray();
return new TarArchive(new SourceStream(files[0], i => i < files.Length ? files[i] : null, readerOptions ?? new ReaderOptions()));
}
/// <summary>
/// Constructor with all stream parts passed in
/// </summary>
/// <param name="streams"></param>
/// <param name="readerOptions"></param>
public static TarArchive Open(IEnumerable<Stream> streams, ReaderOptions? readerOptions = null)
{
streams.CheckNotNull(nameof(streams));
Stream[] strms = streams.ToArray();
return new TarArchive(new SourceStream(strms[0], i => i < strms.Length ? strms[i] : null, readerOptions ?? new ReaderOptions()));
return new TarArchive(fileInfo, readerOptions ?? new ReaderOptions());
}
/// <summary>
@@ -69,7 +45,7 @@ namespace SharpCompress.Archives.Tar
public static TarArchive Open(Stream stream, ReaderOptions? readerOptions = null)
{
stream.CheckNotNull(nameof(stream));
return new TarArchive(new SourceStream(stream, i => null, readerOptions ?? new ReaderOptions()));
return new TarArchive(stream, readerOptions ?? new ReaderOptions());
}
public static bool IsTarFile(string filePath)
@@ -104,19 +80,28 @@ namespace SharpCompress.Archives.Tar
return false;
}
protected override IEnumerable<TarVolume> LoadVolumes(SourceStream srcStream)
/// <summary>
/// Constructor with a FileInfo object to an existing file.
/// </summary>
/// <param name="fileInfo"></param>
/// <param name="readerOptions"></param>
internal TarArchive(FileInfo fileInfo, ReaderOptions readerOptions)
: base(ArchiveType.Tar, fileInfo, readerOptions)
{
base.SrcStream.LoadAllParts(); //request all streams
return new TarVolume(srcStream, ReaderOptions).AsEnumerable(); //simple single volume or split, multivolume not supported
}
protected override IEnumerable<TarVolume> LoadVolumes(FileInfo file)
{
return new TarVolume(file.OpenRead(), ReaderOptions).AsEnumerable();
}
/// <summary>
/// Constructor with a SourceStream able to handle FileInfo and Streams.
/// Takes multiple seekable Streams for a multi-part archive
/// </summary>
/// <param name="srcStream"></param>
/// <param name="options"></param>
internal TarArchive(SourceStream srcStream)
: base(ArchiveType.Tar, srcStream)
/// <param name="stream"></param>
/// <param name="readerOptions"></param>
internal TarArchive(Stream stream, ReaderOptions readerOptions)
: base(ArchiveType.Tar, stream, readerOptions)
{
}
@@ -125,6 +110,11 @@ namespace SharpCompress.Archives.Tar
{
}
protected override IEnumerable<TarVolume> LoadVolumes(IEnumerable<Stream> streams)
{
return new TarVolume(streams.First(), ReaderOptions).AsEnumerable();
}
protected override IEnumerable<TarArchiveEntry> LoadEntries(IEnumerable<TarVolume> volumes)
{
Stream stream = volumes.Single().Stream;

View File

@@ -1,4 +1,4 @@
using System;
using System;
using System.Collections.Generic;
using System.IO;
using System.Linq;
@@ -6,7 +6,6 @@ using SharpCompress.Common;
using SharpCompress.Common.Zip;
using SharpCompress.Common.Zip.Headers;
using SharpCompress.Compressors.Deflate;
using SharpCompress.IO;
using SharpCompress.Readers;
using SharpCompress.Readers.Zip;
using SharpCompress.Writers;
@@ -26,17 +25,6 @@ namespace SharpCompress.Archives.Zip
/// </summary>
public CompressionLevel DeflateCompressionLevel { get; set; }
/// <summary>
/// Constructor with a SourceStream able to handle FileInfo and Streams.
/// </summary>
/// <param name="srcStream"></param>
/// <param name="options"></param>
internal ZipArchive(SourceStream srcStream)
: base(ArchiveType.Zip, srcStream)
{
headerFactory = new SeekableZipHeaderFactory(srcStream.ReaderOptions.Password, srcStream.ReaderOptions.ArchiveEncoding);
}
/// <summary>
/// Constructor expects a filepath to an existing file.
/// </summary>
@@ -56,31 +44,7 @@ namespace SharpCompress.Archives.Zip
public static ZipArchive Open(FileInfo fileInfo, ReaderOptions? readerOptions = null)
{
fileInfo.CheckNotNull(nameof(fileInfo));
return new ZipArchive(new SourceStream(fileInfo, i => ZipArchiveVolumeFactory.GetFilePart(i, fileInfo), readerOptions ?? new ReaderOptions()));
}
/// <summary>
/// Constructor with all file parts passed in
/// </summary>
/// <param name="fileInfos"></param>
/// <param name="readerOptions"></param>
public static ZipArchive Open(IEnumerable<FileInfo> fileInfos, ReaderOptions? readerOptions = null)
{
fileInfos.CheckNotNull(nameof(fileInfos));
FileInfo[] files = fileInfos.ToArray();
return new ZipArchive(new SourceStream(files[0], i => i < files.Length ? files[i] : null, readerOptions ?? new ReaderOptions()));
}
/// <summary>
/// Constructor with all stream parts passed in
/// </summary>
/// <param name="streams"></param>
/// <param name="readerOptions"></param>
public static ZipArchive Open(IEnumerable<Stream> streams, ReaderOptions? readerOptions = null)
{
streams.CheckNotNull(nameof(streams));
Stream[] strms = streams.ToArray();
return new ZipArchive(new SourceStream(strms[0], i => i < strms.Length ? strms[i] : null, readerOptions ?? new ReaderOptions()));
return new ZipArchive(fileInfo, readerOptions ?? new ReaderOptions());
}
/// <summary>
@@ -91,7 +55,7 @@ namespace SharpCompress.Archives.Zip
public static ZipArchive Open(Stream stream, ReaderOptions? readerOptions = null)
{
stream.CheckNotNull(nameof(stream));
return new ZipArchive(new SourceStream(stream, i => null, readerOptions ?? new ReaderOptions()));
return new ZipArchive(stream, readerOptions ?? new ReaderOptions());
}
public static bool IsZipFile(string filePath, string? password = null)
@@ -133,61 +97,20 @@ namespace SharpCompress.Archives.Zip
}
}
public static bool IsZipMulti(Stream stream, string? password = null)
/// <summary>
/// Constructor with a FileInfo object to an existing file.
/// </summary>
/// <param name="fileInfo"></param>
/// <param name="readerOptions"></param>
internal ZipArchive(FileInfo fileInfo, ReaderOptions readerOptions)
: base(ArchiveType.Zip, fileInfo, readerOptions)
{
StreamingZipHeaderFactory headerFactory = new StreamingZipHeaderFactory(password, new ArchiveEncoding());
try
{
ZipHeader? header = headerFactory.ReadStreamHeader(stream).FirstOrDefault(x => x.ZipHeaderType != ZipHeaderType.Split);
if (header is null)
{
if (stream.CanSeek) //could be multipart. Test for central directory - might not be z64 safe
{
SeekableZipHeaderFactory z = new SeekableZipHeaderFactory(password, new ArchiveEncoding());
var x = z.ReadSeekableHeader(stream).FirstOrDefault();
return x?.ZipHeaderType == ZipHeaderType.DirectoryEntry;
}
else
return false;
}
return Enum.IsDefined(typeof(ZipHeaderType), header.ZipHeaderType);
}
catch (CryptographicException)
{
return true;
}
catch
{
return false;
}
headerFactory = new SeekableZipHeaderFactory(readerOptions.Password, readerOptions.ArchiveEncoding);
}
protected override IEnumerable<ZipVolume> LoadVolumes(SourceStream srcStream)
protected override IEnumerable<ZipVolume> LoadVolumes(FileInfo file)
{
base.SrcStream.LoadAllParts(); //request all streams
base.SrcStream.Position = 0;
List<Stream> streams = base.SrcStream.Streams.ToList();
if (streams.Count > 1) //test part 2 - true = multipart not split
{
streams[1].Position += 4; //skip the POST_DATA_DESCRIPTOR to prevent an exception
bool isZip = IsZipFile(streams[1], ReaderOptions.Password);
streams[1].Position -= 4;
if (isZip)
{
base.SrcStream.IsVolumes = true;
var tmp = streams[0]; //arcs as zip, z01 ... swap the zip the end
streams.RemoveAt(0);
streams.Add(tmp);
//streams[0].Position = 4; //skip the POST_DATA_DESCRIPTOR to prevent an exception
return streams.Select(a => new ZipVolume(a, ReaderOptions));
}
}
//split mode or single file
return new ZipVolume(base.SrcStream, ReaderOptions).AsEnumerable();
return new ZipVolume(file.OpenRead(), ReaderOptions).AsEnumerable();
}
internal ZipArchive()
@@ -195,35 +118,46 @@ namespace SharpCompress.Archives.Zip
{
}
/// <summary>
/// Takes multiple seekable Streams for a multi-part archive
/// </summary>
/// <param name="stream"></param>
/// <param name="readerOptions"></param>
internal ZipArchive(Stream stream, ReaderOptions readerOptions)
: base(ArchiveType.Zip, stream, readerOptions)
{
headerFactory = new SeekableZipHeaderFactory(readerOptions.Password, readerOptions.ArchiveEncoding);
}
protected override IEnumerable<ZipVolume> LoadVolumes(IEnumerable<Stream> streams)
{
return new ZipVolume(streams.First(), ReaderOptions).AsEnumerable();
}
protected override IEnumerable<ZipArchiveEntry> LoadEntries(IEnumerable<ZipVolume> volumes)
{
var vols = volumes.ToArray();
foreach (ZipHeader h in headerFactory.ReadSeekableHeader(vols.Last().Stream))
var volume = volumes.Single();
Stream stream = volume.Stream;
foreach (ZipHeader h in headerFactory.ReadSeekableHeader(stream))
{
if (h != null)
{
switch (h.ZipHeaderType)
{
case ZipHeaderType.DirectoryEntry:
{
DirectoryEntryHeader deh = (DirectoryEntryHeader)h;
Stream s;
if (deh.RelativeOffsetOfEntryHeader + deh.CompressedSize > vols[deh.DiskNumberStart].Stream.Length)
{
var v = vols.Skip(deh.DiskNumberStart).ToArray();
s = new SourceStream(v[0].Stream, i => i < v.Length ? v[i].Stream : null, new ReaderOptions() { LeaveStreamOpen = true });
yield return new ZipArchiveEntry(this,
new SeekableZipFilePart(headerFactory,
(DirectoryEntryHeader)h,
stream));
}
else
s = vols[deh.DiskNumberStart].Stream;
yield return new ZipArchiveEntry(this, new SeekableZipFilePart(headerFactory, deh, s));
}
break;
break;
case ZipHeaderType.DirectoryEnd:
{
byte[] bytes = ((DirectoryEndHeader)h).Comment ?? Array.Empty<byte>();
volumes.Last().Comment = ReaderOptions.ArchiveEncoding.Decode(bytes);
yield break;
}
{
byte[] bytes = ((DirectoryEndHeader)h).Comment ?? Array.Empty<byte>();
volume.Comment = ReaderOptions.ArchiveEncoding.Decode(bytes);
yield break;
}
}
}
}

View File

@@ -1,34 +0,0 @@
using System;
using System.Collections.Generic;
using System.IO;
using SharpCompress.Common.Rar;
using SharpCompress.Readers;
using System.Linq;
using System.Text;
using SharpCompress.Common.Rar.Headers;
using System.Text.RegularExpressions;
namespace SharpCompress.Archives.Zip
{
internal static class ZipArchiveVolumeFactory
{
internal static FileInfo? GetFilePart(int index, FileInfo part1) //base the name on the first part
{
FileInfo? item = null;
//load files with zip/zipx first. Swapped to end once loaded in ZipArchive
//new style .zip, z01.. | .zipx, zx01 - if the numbers go beyond 99 then they use 100 ...1000 etc
Match m = Regex.Match(part1.Name, @"^(.*\.)(zipx?|zx?[0-9]+)$", RegexOptions.IgnoreCase);
if (m.Success)
item = new FileInfo(Path.Combine(part1.DirectoryName!, String.Concat(m.Groups[1].Value, Regex.Replace(m.Groups[2].Value, @"[^xz]", ""), index.ToString().PadLeft(2, '0'))));
else //split - 001, 002 ...
return ArchiveVolumeFactory.GetFilePart(index, part1);
if (item != null && item.Exists)
return item;
return null; //no more items
}
}
}

View File

@@ -5,7 +5,7 @@ using System.Runtime.CompilerServices;
[assembly: AssemblyTitle("SharpCompress")]
[assembly: AssemblyProduct("SharpCompress")]
[assembly: InternalsVisibleTo("SharpCompress.Test" + SharpCompress.AssemblyInfo.PublicKeySuffix)]
[assembly: CLSCompliant(true)]
[assembly: CLSCompliant(false)]
namespace SharpCompress
{

View File

@@ -6,6 +6,7 @@
Zip,
Tar,
SevenZip,
GZip
GZip,
ZStandard
}
}

View File

@@ -14,6 +14,7 @@
LZip,
Xz,
Unknown,
Deflate64
Deflate64,
ZStandard,
}
}

View File

@@ -75,7 +75,7 @@ namespace SharpCompress.Common
internal abstract IEnumerable<FilePart> Parts { get; }
public bool IsSolid { get; set; }
internal bool IsSolid { get; set; }
internal virtual void Close()
{

View File

@@ -14,7 +14,6 @@ namespace SharpCompress.Common
bool IsDirectory { get; }
bool IsEncrypted { get; }
bool IsSplitAfter { get; }
bool IsSolid { get; }
DateTime? LastAccessedTime { get; }
DateTime? LastModifiedTime { get; }
long Size { get; }

View File

@@ -10,7 +10,7 @@ namespace SharpCompress.Common.Rar
/// <summary>
/// As the V2017 port isn't complete, add this check to use the legacy Rar code.
/// </summary>
internal bool IsRarV3 => FileHeader.CompressionAlgorithm == 20 || FileHeader.CompressionAlgorithm == 26 || FileHeader.CompressionAlgorithm == 29 || FileHeader.CompressionAlgorithm == 36; //Nanook - Added 20+26 as Test arc from WinRar2.8 (algo 20) was failing with 2017 code
internal bool IsRarV3 => FileHeader.CompressionAlgorithm == 29 || FileHeader.CompressionAlgorithm == 36;
/// <summary>
/// The File's 32 bit CRC Hash

View File

@@ -1248,7 +1248,7 @@ namespace SharpCompress.Common.SevenZip
if (nextHeaderOffset > _streamEnding - db._startPositionAfterHeader)
{
throw new InvalidOperationException("nextHeaderOffset is invalid");
throw new IndexOutOfRangeException();
}
_stream.Seek(nextHeaderOffset, SeekOrigin.Current);

View File

@@ -30,7 +30,7 @@ namespace SharpCompress.Common.SevenZip
}
}
throw new InvalidOperationException();
throw new Exception();
}
public int GetNumOutStreams()
@@ -185,4 +185,4 @@ namespace SharpCompress.Common.SevenZip
return true;
}
}
}
}

View File

@@ -19,9 +19,11 @@ namespace SharpCompress.Common.Tar.Headers
internal string Name { get; set; }
internal string LinkName { get; set; }
internal long Mode { get; set; }
internal long UserId { get; set; }
internal long GroupId { get; set; }
//internal int Mode { get; set; }
//internal int UserId { get; set; }
//internal string UserName { get; set; }
//internal int GroupId { get; set; }
//internal string GroupName { get; set; }
internal long Size { get; set; }
internal DateTime LastModifiedTime { get; set; }
internal EntryType EntryType { get; set; }
@@ -125,12 +127,9 @@ namespace SharpCompress.Common.Tar.Headers
EntryType = ReadEntryType(buffer);
Size = ReadSize(buffer);
Mode = ReadAsciiInt64Base8(buffer, 100, 7);
if(EntryType == EntryType.Directory)
Mode |= 0b1_000_000_000;
UserId = ReadAsciiInt64Base8(buffer, 108, 7);
GroupId = ReadAsciiInt64Base8(buffer, 116, 7);
//Mode = ReadASCIIInt32Base8(buffer, 100, 7);
//UserId = ReadASCIIInt32Base8(buffer, 108, 7);
//GroupId = ReadASCIIInt32Base8(buffer, 116, 7);
long unixTimeStamp = ReadAsciiInt64Base8(buffer, 136, 11);
LastModifiedTime = EPOCH.AddSeconds(unixTimeStamp).ToLocalTime();

View File

@@ -44,12 +44,6 @@ namespace SharpCompress.Common.Tar
public override bool IsSplitAfter => false;
public long Mode => _filePart.Header.Mode;
public long UserID => _filePart.Header.UserId;
public long GroupId => _filePart.Header.GroupId;
internal override IEnumerable<FilePart> Parts => _filePart.AsEnumerable<FilePart>();
internal static IEnumerable<TarEntry> GetEntries(StreamingMode mode, Stream stream,

View File

@@ -0,0 +1,50 @@
using System;
using System.Collections.Generic;
using System.IO;
using SharpCompress.Common.GZip;
namespace SharpCompress.Common.ZStandard
{
public class ZStandardEntry : Entry
{
private readonly ZStandardFilePart _filePart;
internal ZStandardEntry(ZStandardFilePart filePart)
{
_filePart = filePart;
}
public override CompressionType CompressionType => CompressionType.GZip;
public override long Crc => _filePart.Crc ?? 0;
public override string Key => _filePart.FilePartName;
public override string? LinkTarget => null;
public override long CompressedSize => 0;
public override long Size => _filePart.UncompressedSize ?? 0;
public override DateTime? LastModifiedTime => _filePart.DateModified;
public override DateTime? CreatedTime => null;
public override DateTime? LastAccessedTime => null;
public override DateTime? ArchivedTime => null;
public override bool IsEncrypted => false;
public override bool IsDirectory => false;
public override bool IsSplitAfter => false;
internal override IEnumerable<FilePart> Parts => _filePart.AsEnumerable<FilePart>();
internal static IEnumerable<GZipEntry> GetEntries(Stream stream, OptionsBase options)
{
yield return new GZipEntry(new GZipFilePart(stream, options.ArchiveEncoding));
}
}
}

View File

@@ -0,0 +1,37 @@
using System;
using System.IO;
using ZstdSharp;
namespace SharpCompress.Common.ZStandard
{
internal sealed class ZStandardFilePart : FilePart
{
private string _name = "";
private readonly Stream _stream;
internal ZStandardFilePart(Stream stream, ArchiveEncoding archiveEncoding)
: base(archiveEncoding)
{
_stream = stream;
EntryStartPosition = stream.Position;
}
internal long EntryStartPosition { get; }
internal DateTime? DateModified { get; private set; }
internal int? Crc { get; private set; }
internal int? UncompressedSize { get; private set; }
internal override string FilePartName => _name!;
internal override Stream GetCompressedStream()
{
return new DecompressionStream(_stream);
}
internal override Stream GetRawStream()
{
return _stream;
}
}
}

View File

@@ -0,0 +1,23 @@
using System.IO;
using SharpCompress.Readers;
namespace SharpCompress.Common.ZStandard
{
public class ZStandardVolume : Volume
{
public ZStandardVolume(Stream stream, ReaderOptions options)
: base(stream, options)
{
}
public ZStandardVolume(FileInfo fileInfo, ReaderOptions options)
: base(fileInfo.OpenRead(), options)
{
options.LeaveStreamOpen = false;
}
public override bool IsFirstVolume => true;
public override bool IsMultiVolume => true;
}
}

View File

@@ -174,7 +174,7 @@ namespace SharpCompress.Common.Zip
default:
{
throw new InvalidOperationException("Header.CompressionMethod is invalid");
throw new ArgumentOutOfRangeException();
}
}

View File

@@ -1,20 +1,20 @@
// Zlib.cs
// ------------------------------------------------------------------
//
// Copyright (c) 2009 Dino Chiesa and Microsoft Corporation.
// Copyright (c) 2009 Dino Chiesa and Microsoft Corporation.
// All rights reserved.
//
// This code module is part of DotNetZip, a zipfile class library.
//
// ------------------------------------------------------------------
//
// This code is licensed under the Microsoft Public License.
// This code is licensed under the Microsoft Public License.
// See the file License.txt for the license details.
// More info on: http://dotnetzip.codeplex.com
//
// ------------------------------------------------------------------
//
// last saved (in emacs):
// last saved (in emacs):
// Time-stamp: <2009-November-07 05:26:55>
//
// ------------------------------------------------------------------
@@ -27,22 +27,22 @@
// included below.
//
// ------------------------------------------------------------------
//
//
// Copyright (c) 2000,2001,2002,2003 ymnk, JCraft,Inc. All rights reserved.
//
//
// Redistribution and use in source and binary forms, with or without
// modification, are permitted provided that the following conditions are met:
//
//
// 1. Redistributions of source code must retain the above copyright notice,
// this list of conditions and the following disclaimer.
//
// 2. Redistributions in binary form must reproduce the above copyright
// notice, this list of conditions and the following disclaimer in
//
// 2. Redistributions in binary form must reproduce the above copyright
// notice, this list of conditions and the following disclaimer in
// the documentation and/or other materials provided with the distribution.
//
//
// 3. The names of the authors may not be used to endorse or promote products
// derived from this software without specific prior written permission.
//
//
// THIS SOFTWARE IS PROVIDED ``AS IS'' AND ANY EXPRESSED OR IMPLIED WARRANTIES,
// INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND
// FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL JCRAFT,
@@ -53,7 +53,7 @@
// LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
// NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE,
// EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
//
//
// -----------------------------------------------------------------------
//
// This program is based on zlib-1.1.3; credit to authors
@@ -82,7 +82,7 @@ namespace SharpCompress.Compressors.Deflate
/// <summary>
/// Same as None.
/// </summary>
Level0 = None,
Level0 = 0,
/// <summary>
/// The fastest but least effective compression.
@@ -92,7 +92,7 @@ namespace SharpCompress.Compressors.Deflate
/// <summary>
/// A synonym for BestSpeed.
/// </summary>
Level1 = BestSpeed,
Level1 = 1,
/// <summary>
/// A little slower, but better, than level 1.
@@ -115,14 +115,14 @@ namespace SharpCompress.Compressors.Deflate
Level5 = 5,
/// <summary>
/// The default compression level, with a good balance of speed and compression efficiency.
/// The default compression level, with a good balance of speed and compression efficiency.
/// </summary>
Default = 6,
/// <summary>
/// A synonym for Default.
/// </summary>
Level6 = Default,
Level6 = 6,
/// <summary>
/// Pretty good compression!
@@ -135,7 +135,7 @@ namespace SharpCompress.Compressors.Deflate
Level8 = 8,
/// <summary>
/// The "best" compression, where best means greatest reduction in size of the input data stream.
/// The "best" compression, where best means greatest reduction in size of the input data stream.
/// This is also the slowest compression.
/// </summary>
BestCompression = 9,
@@ -143,7 +143,7 @@ namespace SharpCompress.Compressors.Deflate
/// <summary>
/// A synonym for BestCompression.
/// </summary>
Level9 = BestCompression
Level9 = 9
}
/// <summary>
@@ -154,7 +154,7 @@ namespace SharpCompress.Compressors.Deflate
public enum CompressionStrategy
{
/// <summary>
/// The default strategy is probably the best for normal data.
/// The default strategy is probably the best for normal data.
/// </summary>
Default = 0,
@@ -181,7 +181,7 @@ namespace SharpCompress.Compressors.Deflate
{
/// <summary>
/// The ZlibException class captures exception information generated
/// by the Zlib library.
/// by the Zlib library.
/// </summary>
public ZlibException()
{
@@ -233,7 +233,7 @@ namespace SharpCompress.Compressors.Deflate
/// <param name="target">Contains the array of characteres read from the source TextReader.</param>
/// <param name="start">The starting index of the target array.</param>
/// <param name="count">The maximum number of characters to read from the source TextReader.</param>
///
///
/// <returns>
/// The number of characters read. The number will be less than or equal to
/// count depending on the data available in the source TextReader. Returns -1
@@ -405,4 +405,4 @@ namespace SharpCompress.Compressors.Deflate
BitLengths = new StaticTree(null, extra_blbits, 0, InternalConstants.BL_CODES, InternalConstants.MAX_BL_BITS);
}
}
}
}

View File

@@ -1,297 +0,0 @@
/*
* BranchExecFilter.cs -- Converters for executable
* <Contribution by Louis-Michel Bergeron, on behalf of aDolus Technolog Inc., 2022>
* @TODO Encoding
*/
using System;
using System.Collections.Generic;
using System.IO;
using System.Linq;
using System.Text;
using System.Threading.Tasks;
namespace SharpCompress.Compressors.Filters
{
[CLSCompliant(false)]
public sealed class BranchExecFilter
{
public enum Alignment : int
{
ARCH_x86_ALIGNMENT = 1,
ARCH_PowerPC_ALIGNMENT = 4,
ARCH_IA64_ALIGNMENT = 16,
ARCH_ARM_ALIGNMENT = 4,
ARCH_ARMTHUMB_ALIGNMENT = 2,
ARCH_SPARC_ALIGNMENT = 4,
}
public static void X86Converter(byte[] data, UInt32 ip, ref UInt32 state) {
long i = 0;
long size = data.Length;
UInt32 pos = 0;
UInt32 mask = state & 7;
if (size < 5)
return;
size -= 4;
ip += 5;
for (;;)
{
i = pos;
for (; i < size; i++)
{
if ((data[i] & 0xFE) == 0xE8)
{
break;
}
}
UInt32 d = (UInt32)(i) - pos;
pos = (UInt32)i;
if (i >= size)
{
state = (d > 2 ? 0 : mask >> (int)d);
return;
}
if (d > 2)
{
mask = 0;
}
else
{
mask >>= (int)d;
if (mask != 0 && (mask > 4 || mask == 3 || (((((data[(UInt32)(mask >> 1) + 1])) + 1) & 0xFE) == 0) ))
{
mask = (mask >> 1) | 4;
pos++;
continue;
}
}
if ((((data[i + 4]) + 1) & 0xFE) == 0)
{
UInt32 inst = ((UInt32)data[i + 4] << 24) | ((UInt32)data[i + 3] << 16) | ((UInt32)data[i + 2] << 8) | ((UInt32)data[i + 1]);
UInt32 cur = ip + (UInt32)pos;
pos += 5;
inst -= cur;
if (mask != 0)
{
UInt32 sh = (mask & 6) << 2;
if (((((((Byte)(inst >> (int)sh))) + 1) & 0xFE) == 0))
{
inst ^= (((UInt32)0x100 << (int)sh) - 1);
inst -= cur;
}
mask = 0;
}
data[i + 1] = (Byte)inst;
data[i + 2] = (Byte)(inst >> 8);
data[i + 3] = (Byte)(inst >> 16);
data[i + 4] = (Byte)(0 - ((inst >> 24) & 1));
}
else
{
mask = (mask >> 1) | 4;
pos++;
}
}
}
public static void PowerPCConverter(byte[] data, UInt32 ip)
{
long i = 0;
long size = data.Length;
size &= ~(UInt32)3;
ip -= 4;
for (;; ) // infinite loop
{
for (;; ) // infinite loop
{
if (i >= size)
return;
i += 4;
if ((data[i - 4] & 0xFC) == 0x48 && (data[i - 1] & 3) == 1)
break;
}
{
UInt32 inst = BitConverter.ToUInt32(data, (int)i - 4);
if (BitConverter.IsLittleEndian)
{
inst = Utility.SwapUINT32(inst);
}
inst -= (UInt32)(ip + i);
inst &= 0x03FFFFFF;
inst |= 0x48000000;
Utility.SetBigUInt32(ref data, inst, (i - 4));
}
}
}
public static void ARMConverter(byte[] data, UInt32 ip)
{
long i = 0;
long size = data.Length;
size &= ~(UInt32)3;
ip += 4;
for (;;) // infinite loop
{
for (;;) // infinite loop
{
if (i >= size)
{
return;
}
i += 4;
if (data[i - 1] == 0xEB)
break;
}
UInt32 inst = BitConverter.ToUInt32(data, (int)i - 4);
inst <<= 2;
inst -= (UInt32)(ip + i);
inst >>= 2;
inst &= 0x00FFFFFF;
inst |= 0xEB000000;
Utility.SetLittleUInt32(ref data, inst, i - 4);
}
}
public static void ARMTConverter(byte[] data, UInt32 ip)
{
long i = 0;
long size = data.Length;
size &= ~(UInt32)1;
long lim = size - 4;
for (;;)
{
UInt32 b1;
for (;;)
{
UInt32 b3;
if (i > lim)
return;
b1 = data[i + 1];
b3 = data[i + 3];
i += 2;
b1 ^= 8;
if ((b3 & b1) >= 0xF8)
break;
}
UInt32 inst = ((UInt32)b1 << 19)
+ (((UInt32)data[i + 1] & 0x7) << 8)
+ (((UInt32)data[i - 2] << 11))
+ (data[i]);
i += 2;
UInt32 cur = ((UInt32)(ip + i)) >> 1;
inst -= cur;
data[i - 4] = (Byte)(inst >> 11);
data[i - 3] = (Byte)(0xF0 | ((inst >> 19) & 0x7));
data[i - 2] = (Byte)inst;
data[i - 1] = (Byte)(0xF8 | (inst >> 8));
}
}
public static void IA64Converter(byte[] data, UInt32 ip)
{
UInt32 i = 0;
long size = data.Length;
if (size < 16)
throw new InvalidDataException("Unexpected data size");
size -= 16;
do
{
UInt32 m = ((UInt32)0x334B0000 >> (data[i] & 0x1E)) & 3;
if (m != 0)
{
m++;
do
{
UInt32 iterator = (UInt32)( (i + (m * 5) - 8));
if (((data[iterator + 3] >> (int)m) & 15) == 5
&& (((data[iterator - 1] | ((UInt32)data[iterator] << 8)) >> (int)m) & 0x70) == 0)
{
UInt32 raw = BitConverter.ToUInt32(data, (int)iterator);
UInt32 inst = raw >> (int)m;
inst = (inst & 0xFFFFF) | ((inst & (1 << 23)) >> 3);
inst <<= 4;
inst -= (ip + (UInt32)i);
inst >>= 4;
inst &= 0x1FFFFF;
inst += 0x700000;
inst &= 0x8FFFFF;
raw &= ~((UInt32)0x8FFFFF << (int)m);
raw |= (inst << (int)m);
Utility.SetLittleUInt32(ref data, raw, iterator);
}
}
while (++m <= 4);
}
i += 16;
}
while (i <= size);
return;
}
public static void SPARCConverter(byte[] data, UInt32 ip)
{
long i = 0;
long size = data.Length;
size &= ~(UInt32)3;
ip -= 4;
for (;;) // infinite loop
{
for (;;) // infinite loop
{
if (i >= size)
return;
i += 4;
if ((data[i - 4] == 0x40 && (data[i - 3] & 0xC0) == 0) ||
(data[i - 4] == 0x7F && (data[i - 3] >= 0xC0)))
break;
}
UInt32 inst = BitConverter.ToUInt32(data, (int)i - 4);
if (BitConverter.IsLittleEndian)
{
inst = Utility.SwapUINT32(inst);
}
inst <<= 2;
inst -= (UInt32)(ip + i);
inst &= 0x01FFFFFF;
inst -= (UInt32)1 << 24;
inst ^= 0xFF000000;
inst >>= 2;
inst |= 0x40000000;
Utility.SetBigUInt32(ref data, inst, (i - 4));
}
}
}
}

View File

@@ -1,4 +1,4 @@
using System;
using System;
using System.IO;
using System.Security.Cryptography;
using System.Text;
@@ -20,11 +20,6 @@ namespace SharpCompress.Compressors.LZMA
public AesDecoderStream(Stream input, byte[] info, IPasswordProvider pass, long limit)
{
if (pass.CryptoGetTextPassword() == null)
{
throw new SharpCompress.Common.CryptographicException("Encrypted 7Zip archive has no password specified.");
}
mStream = input;
mLimit = limit;
@@ -148,8 +143,8 @@ namespace SharpCompress.Compressors.LZMA
if ((bt & 0xC0) == 0)
{
salt = Array.Empty<byte>();
iv = Array.Empty<byte>();
salt = new byte[0];
iv = new byte[0];
return;
}

View File

@@ -103,7 +103,7 @@ namespace SharpCompress.Compressors.LZMA.LZ
{
if (historySize > K_MAX_VAL_FOR_NORMALIZE - 256)
{
throw new ArgumentOutOfRangeException(nameof(historySize));
throw new Exception();
}
_cutValue = 16 + (matchMaxLen >> 1);
@@ -423,4 +423,4 @@ namespace SharpCompress.Compressors.LZMA.LZ
_cutValue = cutValue;
}
}
}
}

View File

@@ -1649,7 +1649,7 @@ namespace SharpCompress.Compressors.LZMA
{
for (int m = 0; m < K_MATCH_FINDER_I_DS.Length; m++)
{
if (string.Equals(s, K_MATCH_FINDER_I_DS[m], StringComparison.OrdinalIgnoreCase))
if (s == K_MATCH_FINDER_I_DS[m])
{
return m;
}
@@ -1696,7 +1696,7 @@ namespace SharpCompress.Compressors.LZMA
throw new InvalidParamException();
}
EMatchFinderType matchFinderIndexPrev = _matchFinderType;
int m = FindMatchFinder(((string)prop));
int m = FindMatchFinder(((string)prop).ToUpper());
if (m < 0)
{
throw new InvalidParamException();

View File

@@ -1,4 +1,4 @@
using System;
using System;
using System.Diagnostics;
using System.IO;
@@ -21,17 +21,16 @@ namespace SharpCompress.Compressors.LZMA.Utilites
protected override void Dispose(bool disposing)
{
//Nanook - is not equal here - _mCurrentCrc is yet to be negated
//if (_mCurrentCrc != _mExpectedCrc)
//{
// throw new InvalidOperationException();
//}
if (_mCurrentCrc != _mExpectedCrc)
{
throw new InvalidOperationException();
}
try
{
if (disposing && !_mClosed)
{
_mClosed = true;
_mCurrentCrc = Crc.Finish(_mCurrentCrc); //now becomes equal
_mCurrentCrc = Crc.Finish(_mCurrentCrc);
#if DEBUG
if (_mCurrentCrc == _mExpectedCrc)
{
@@ -55,10 +54,6 @@ namespace SharpCompress.Compressors.LZMA.Utilites
}
Debug.WriteLine("entropy: " + (int)(entropy * 100) + "%");
#endif
if (_mCurrentCrc != _mExpectedCrc) //moved test to here
{
throw new InvalidOperationException();
}
}
}
finally
@@ -107,4 +102,4 @@ namespace SharpCompress.Compressors.LZMA.Utilites
_mCurrentCrc = Crc.Update(_mCurrentCrc, buffer, offset, count);
}
}
}
}

View File

@@ -50,7 +50,7 @@ namespace SharpCompress.Compressors.LZMA.Utilites
Debugger.Break();
}
throw new InvalidOperationException("Assertion failed.");
throw new Exception("Assertion failed.");
}
}
@@ -89,4 +89,4 @@ namespace SharpCompress.Compressors.LZMA.Utilites
}
}
}
}
}

View File

@@ -83,7 +83,7 @@ namespace SharpCompress.Compressors.PPMd.H
internal void AriDecNormalize()
{
// while ((low ^ (low + range)) < TOP || range < BOT && ((range = -low & (BOT - 1)) != 0 ? true : true))
// while ((low ^ (low + range)) < TOP || range < BOT && ((range = -low & (BOT - 1)) != 0 ? true : true))
// {
// code = ((code << 8) | unpackRead.getChar()&0xff)&uintMask;
// range = (range << 8)&uintMask;
@@ -118,7 +118,7 @@ namespace SharpCompress.Compressors.PPMd.H
buffer.Append(_range);
buffer.Append("\n subrange=");
buffer.Append(SubRange);
buffer.Append(']');
buffer.Append("]");
return buffer.ToString();
}
}
@@ -150,8 +150,8 @@ namespace SharpCompress.Compressors.PPMd.H
buffer.Append(_highCount);
buffer.Append("\n scale=");
buffer.Append(_scale);
buffer.Append(']');
buffer.Append("]");
return buffer.ToString();
}
}
}
}

View File

@@ -1,4 +1,4 @@
#nullable disable
#nullable disable
using System;
using System.IO;
@@ -23,7 +23,6 @@ namespace SharpCompress.Compressors.Rar
private int outCount;
private int outTotal;
private bool isDisposed;
private long _position;
public RarStream(IRarUnpack unpack, FileHeader fileHeader, Stream readStream)
{
@@ -33,7 +32,6 @@ namespace SharpCompress.Compressors.Rar
fetch = true;
unpack.DoUnpack(fileHeader, readStream, this);
fetch = false;
_position = 0;
}
protected override void Dispose(bool disposing)
@@ -58,8 +56,7 @@ namespace SharpCompress.Compressors.Rar
public override long Length => fileHeader.UncompressedSize;
//commented out code always returned the length of the file
public override long Position { get => _position; /* fileHeader.UncompressedSize - unpack.DestSize;*/ set => throw new NotSupportedException(); }
public override long Position { get => fileHeader.UncompressedSize - unpack.DestSize; set => throw new NotSupportedException(); }
public override int Read(byte[] buffer, int offset, int count)
{
@@ -83,7 +80,6 @@ namespace SharpCompress.Compressors.Rar
unpack.DoUnpack();
fetch = false;
}
_position += (long)outTotal;
return outTotal;
}
@@ -133,4 +129,4 @@ namespace SharpCompress.Compressors.Rar
}
}
}
}
}

View File

@@ -1,68 +0,0 @@
/*
* ArmFilter.cs -- XZ converter ARM executable
* <Contribution by Louis-Michel Bergeron, on behalf of aDolus Technolog Inc., 2022>
* @TODO Properties offset
*/
using System;
using System.Collections.Generic;
using System.IO;
using System.Linq;
using System.Text;
using System.Threading.Tasks;
using SharpCompress.Compressors.Filters;
namespace SharpCompress.Compressors.Xz.Filters
{
internal class ArmFilter : BlockFilter
{
public override bool AllowAsLast => false;
public override bool AllowAsNonLast => true;
public override bool ChangesDataSize => false;
private UInt32 _ip = 0;
//private UInt32 _offset = 0;
public override void Init(byte[] properties)
{
if (properties.Length != 0 && properties.Length != 4)
{
throw new InvalidDataException("ARM properties unexpected length");
}
if (properties.Length == 4)
{
// Even XZ doesn't support it.
throw new InvalidDataException("ARM properties offset is not supported");
//_offset = BitConverter.ToUInt32(properties, 0);
//
//if (_offset % (UInt32)BranchExec.Alignment.ARCH_ARM_ALIGNMENT != 0)
//{
// throw new InvalidDataException("Filter offset does not match alignment");
//}
}
}
public override void ValidateFilter()
{
}
public override int Read(byte[] buffer, int offset, int count)
{
int bytesRead = BaseStream.Read(buffer, offset, count);
BranchExecFilter.ARMConverter(buffer, _ip);
_ip += (UInt32)bytesRead;
return bytesRead;
}
public override void SetBaseStream(Stream stream)
{
BaseStream = stream;
}
}
}

View File

@@ -1,69 +0,0 @@
/*
* ArmFThumbFilter.cs -- XZ converter ARMThumb executable
* <Contribution by Louis-Michel Bergeron, on behalf of aDolus Technolog Inc., 2022>
* @TODO Properties offset
*/
using System;
using System.Collections.Generic;
using System.IO;
using System.Linq;
using System.Text;
using System.Threading.Tasks;
using SharpCompress.Compressors.Filters;
namespace SharpCompress.Compressors.Xz.Filters
{
internal class ArmThumbFilter : BlockFilter
{
public override bool AllowAsLast => false;
public override bool AllowAsNonLast => true;
public override bool ChangesDataSize => false;
private UInt32 _ip = 0;
//private UInt32 _offset = 0;
public override void Init(byte[] properties)
{
if (properties.Length != 0 && properties.Length != 4)
{
throw new InvalidDataException("ARM Thumb properties unexpected length");
}
if (properties.Length == 4)
{
// Even XZ doesn't support it.
throw new InvalidDataException("ARM Thumb properties offset is not supported");
//_offset = BitConverter.ToUInt32(properties, 0);
//
//if (_offset % (UInt32)BranchExec.Alignment.ARCH_ARMTHUMB_ALIGNMENT != 0)
//{
// throw new InvalidDataException("Filter offset does not match alignment");
//}
}
}
public override void ValidateFilter()
{
}
public override int Read(byte[] buffer, int offset, int count)
{
int bytesRead = BaseStream.Read(buffer, offset, count);
BranchExecFilter.ARMTConverter(buffer, _ip);
_ip += (UInt32)bytesRead;
return bytesRead;
}
public override void SetBaseStream(Stream stream)
{
BaseStream = stream;
}
}
}

View File

@@ -1,4 +1,4 @@
using System;
using System;
using System.Collections.Generic;
using System.IO;
@@ -20,12 +20,6 @@ namespace SharpCompress.Compressors.Xz.Filters
private static readonly Dictionary<FilterTypes, Type> FilterMap = new Dictionary<FilterTypes, Type>
{
{FilterTypes.ARCH_x86_FILTER, typeof(X86Filter) },
{FilterTypes.ARCH_PowerPC_FILTER, typeof(PowerPCFilter) },
{FilterTypes.ARCH_IA64_FILTER, typeof(IA64Filter) },
{FilterTypes.ARCH_ARM_FILTER, typeof(ArmFilter) },
{FilterTypes.ARCH_ARMTHUMB_FILTER, typeof(ArmThumbFilter) },
{FilterTypes.ARCH_SPARC_FILTER, typeof(SparcFilter) },
{FilterTypes.LZMA2, typeof(Lzma2Filter) }
};

View File

@@ -1,67 +0,0 @@
/*
* IA64Filter.cs -- XZ converter IA64 executable
* <Contribution by Louis-Michel Bergeron, on behalf of aDolus Technolog Inc., 2022>
* @TODO Properties offset
*/
using System;
using System.Collections.Generic;
using System.IO;
using System.Linq;
using System.Text;
using System.Threading.Tasks;
using SharpCompress.Compressors.Filters;
namespace SharpCompress.Compressors.Xz.Filters
{
internal class IA64Filter : BlockFilter
{
public override bool AllowAsLast => false;
public override bool AllowAsNonLast => true;
public override bool ChangesDataSize => false;
private UInt32 _ip = 0;
//private UInt32 _offset = 0;
public override void Init(byte[] properties)
{
if (properties.Length != 0 && properties.Length != 4)
{
throw new InvalidDataException("IA64 properties unexpected length");
}
if (properties.Length == 4)
{
// Even XZ doesn't support it.
throw new InvalidDataException("IA64 properties offset is not supported");
//_offset = BitConverter.ToUInt32(properties, 0);
//
//if (_offset % (UInt32)BranchExec.Alignment.ARCH_IA64_ALIGNMENT != 0)
//{
// throw new InvalidDataException("Filter offset does not match alignment");
//}
}
}
public override void ValidateFilter()
{
}
public override int Read(byte[] buffer, int offset, int count)
{
int bytesRead = BaseStream.Read(buffer, offset, count);
BranchExecFilter.IA64Converter(buffer, _ip);
_ip += (UInt32)bytesRead;
return bytesRead;
}
public override void SetBaseStream(Stream stream)
{
BaseStream = stream;
}
}
}

View File

@@ -1,66 +0,0 @@
/*
* PowerPCFilter.cs -- XZ converter PowerPC executable
* <Contribution by Louis-Michel Bergeron, on behalf of aDolus Technolog Inc., 2022>
* @TODO Properties offset
*/
using System;
using System.Collections.Generic;
using System.IO;
using System.Linq;
using System.Text;
using System.Threading.Tasks;
using SharpCompress.Compressors.Filters;
namespace SharpCompress.Compressors.Xz.Filters
{
internal class PowerPCFilter : BlockFilter
{
public override bool AllowAsLast => false;
public override bool AllowAsNonLast => true;
public override bool ChangesDataSize => false;
private UInt32 _ip = 0;
//private UInt32 _offset = 0;
public override void Init(byte[] properties)
{
if (properties.Length != 0 && properties.Length != 4)
{
throw new InvalidDataException("PPC properties unexpected length");
}
if (properties.Length == 4)
{
// Even XZ doesn't support it.
throw new InvalidDataException("PPC properties offset is not supported");
//_offset = BitConverter.ToUInt32(properties, 0);
//
//if (_offset % (UInt32)BranchExec.Alignment.ARCH_PowerPC_ALIGNMENT != 0)
//{
// throw new InvalidDataException("Filter offset does not match alignment");
//}
}
}
public override void ValidateFilter() { }
public override int Read(byte[] buffer, int offset, int count)
{
int bytesRead = BaseStream.Read(buffer, offset, count);
BranchExecFilter.PowerPCConverter(buffer, _ip);
_ip += (UInt32)bytesRead;
return bytesRead;
}
public override void SetBaseStream(Stream stream)
{
BaseStream = stream;
}
}
}

View File

@@ -1,65 +0,0 @@
/*
* SparcFilter.cs -- XZ converter SPARC executable
* <Contribution by Louis-Michel Bergeron, on behalf of aDolus Technolog Inc., 2022>
* @TODO Properties offset
*/
using System;
using System.Collections.Generic;
using System.IO;
using System.Linq;
using System.Text;
using System.Threading.Tasks;
using SharpCompress.Compressors.Filters;
namespace SharpCompress.Compressors.Xz.Filters
{
internal class SparcFilter : BlockFilter
{
public override bool AllowAsLast => false;
public override bool AllowAsNonLast => true;
public override bool ChangesDataSize => false;
private UInt32 _ip = 0;
//private UInt32 _offset = 0;
public override void Init(byte[] properties) {
if (properties.Length != 0 && properties.Length != 4)
{
throw new InvalidDataException("SPARC properties unexpected length");
}
if (properties.Length == 4)
{
// Even XZ doesn't support it.
throw new InvalidDataException("SPARC properties offset is not supported");
//_offset = BitConverter.ToUInt32(properties, 0);
//
//if (_offset % (UInt32)BranchExec.Alignment.ARCH_SPARC_ALIGNMENT != 0)
//{
// throw new InvalidDataException("Filter offset does not match alignment");
//}
}
}
public override void ValidateFilter() {
}
public override int Read(byte[] buffer, int offset, int count){
int bytesRead = BaseStream.Read(buffer, offset, count);
BranchExecFilter.SPARCConverter(buffer, _ip);
_ip += (UInt32)bytesRead;
return bytesRead;
}
public override void SetBaseStream(Stream stream) {
BaseStream = stream;
}
}
}

View File

@@ -1,70 +0,0 @@
/*
* X86Filter.cs -- XZ converter x86 executable
* <Contribution by Louis-Michel Bergeron, on behalf of aDolus Technolog Inc., 2022>
* @TODO Properties offset
*/
using System;
using System.Collections.Generic;
using System.IO;
using System.Linq;
using System.Text;
using System.Threading.Tasks;
using SharpCompress.Compressors.Filters;
namespace SharpCompress.Compressors.Xz.Filters
{
internal class X86Filter : BlockFilter
{
public override bool AllowAsLast => false;
public override bool AllowAsNonLast => true;
public override bool ChangesDataSize => false;
private UInt32 _ip = 0;
private UInt32 _state = 0;
//private UInt32 _offset = 0;
public override void Init(byte[] properties)
{
if (properties.Length != 0 && properties.Length != 4)
{
throw new InvalidDataException("X86 properties unexpected length");
}
if (properties.Length == 4)
{
// Even XZ doesn't support it.
throw new InvalidDataException("X86 properties offset is not supported");
//_offset = BitConverter.ToUInt32(properties, 0);
//
//if (_offset % (UInt32)BranchExec.Alignment.ARCH_x86_ALIGNMENT != 0)
//{
// throw new InvalidDataException("Filter offset does not match alignment");
//}
}
}
public override void ValidateFilter()
{
}
public override int Read(byte[] buffer, int offset, int count)
{
int bytesRead = BaseStream.Read(buffer, offset, count);
BranchExecFilter.X86Converter(buffer, _ip, ref _state);
_ip += (UInt32)bytesRead;
return bytesRead;
}
public override void SetBaseStream(Stream stream)
{
BaseStream = stream;
}
}
}

View File

@@ -9,7 +9,7 @@ namespace SharpCompress.Compressors.Xz
{
if (MaxBytes <= 0)
{
throw new ArgumentOutOfRangeException(nameof(MaxBytes));
throw new ArgumentOutOfRangeException();
}
if (MaxBytes > 9)

View File

@@ -0,0 +1,201 @@
using System;
using System.Buffers;
using System.IO;
using System.Threading;
using System.Threading.Tasks;
using ZstdSharp.Unsafe;
namespace ZstdSharp
{
public class CompressionStream : Stream
{
private readonly Stream innerStream;
private readonly byte[] outputBuffer;
private Compressor compressor;
private ZSTD_outBuffer_s output;
public CompressionStream(Stream stream, int level = Compressor.DefaultCompressionLevel,
int bufferSize = 0)
{
if (stream == null)
throw new ArgumentNullException(nameof(stream));
if (!stream.CanWrite)
throw new ArgumentException("Stream is not writable", nameof(stream));
if (bufferSize < 0)
throw new ArgumentOutOfRangeException(nameof(bufferSize));
innerStream = stream;
compressor = new Compressor(level);
var outputBufferSize =
bufferSize > 0 ? bufferSize : (int) Methods.ZSTD_CStreamOutSize().EnsureZstdSuccess();
outputBuffer = ArrayPool<byte>.Shared.Rent(outputBufferSize);
output = new ZSTD_outBuffer_s {pos = 0, size = (nuint) outputBufferSize};
}
public void SetParameter(ZSTD_cParameter parameter, int value)
{
EnsureNotDisposed();
compressor.SetParameter(parameter, value);
}
public int GetParameter(ZSTD_cParameter parameter)
{
EnsureNotDisposed();
return compressor.GetParameter(parameter);
}
public void LoadDictionary(byte[] dict)
{
EnsureNotDisposed();
compressor.LoadDictionary(dict);
}
~CompressionStream() => Dispose(false);
#if !NETSTANDARD2_0 && !NET461
public override async ValueTask DisposeAsync()
#else
public async ValueTask DisposeAsync()
#endif
{
if (compressor == null)
return;
try
{
await FlushAsync().ConfigureAwait(false);
}
finally
{
ReleaseUnmanagedResources();
GC.SuppressFinalize(this);
}
}
protected override void Dispose(bool disposing)
{
if (compressor == null)
return;
try
{
if (disposing)
Flush();
}
finally
{
ReleaseUnmanagedResources();
}
}
private void ReleaseUnmanagedResources()
{
compressor.Dispose();
ArrayPool<byte>.Shared.Return(outputBuffer);
}
public override void Flush()
=> WriteInternal(null, true);
public override async Task FlushAsync(CancellationToken cancellationToken)
=> await WriteInternalAsync(null, true, cancellationToken).ConfigureAwait(false);
public override void Write(byte[] buffer, int offset, int count)
=> Write(new ReadOnlySpan<byte>(buffer, offset, count));
#if !NETSTANDARD2_0 && !NET461
public override void Write(ReadOnlySpan<byte> buffer)
=> WriteInternal(buffer, false);
#else
public void Write(ReadOnlySpan<byte> buffer)
=> WriteInternal(buffer, false);
#endif
private void WriteInternal(ReadOnlySpan<byte> buffer, bool lastChunk)
{
EnsureNotDisposed();
var input = new ZSTD_inBuffer_s {pos = 0, size = buffer != null ? (nuint) buffer.Length : 0};
nuint remaining;
do
{
output.pos = 0;
remaining = CompressStream(ref input, buffer,
lastChunk ? ZSTD_EndDirective.ZSTD_e_end : ZSTD_EndDirective.ZSTD_e_continue);
var written = (int) output.pos;
if (written > 0)
innerStream.Write(outputBuffer, 0, written);
} while (lastChunk ? remaining > 0 : input.pos < input.size);
}
private async ValueTask WriteInternalAsync(ReadOnlyMemory<byte>? buffer, bool lastChunk,
CancellationToken cancellationToken = default)
{
EnsureNotDisposed();
var input = new ZSTD_inBuffer_s { pos = 0, size = buffer.HasValue ? (nuint)buffer.Value.Length : 0 };
nuint remaining;
do
{
output.pos = 0;
remaining = CompressStream(ref input, buffer.HasValue ? buffer.Value.Span : null,
lastChunk ? ZSTD_EndDirective.ZSTD_e_end : ZSTD_EndDirective.ZSTD_e_continue);
var written = (int) output.pos;
if (written > 0)
await innerStream.WriteAsync(outputBuffer, 0, written, cancellationToken).ConfigureAwait(false);
} while (lastChunk ? remaining > 0 : input.pos < input.size);
}
public override Task WriteAsync(byte[] buffer, int offset, int count, CancellationToken cancellationToken)
=> WriteAsync(new ReadOnlyMemory<byte>(buffer, offset, count), cancellationToken).AsTask();
#if !NETSTANDARD2_0 && !NET461
public override async ValueTask WriteAsync(ReadOnlyMemory<byte> buffer,
CancellationToken cancellationToken = default)
=> await WriteInternalAsync(buffer, false, cancellationToken).ConfigureAwait(false);
#else
public async ValueTask WriteAsync(ReadOnlyMemory<byte> buffer,
CancellationToken cancellationToken = default)
=> await WriteInternalAsync(buffer, false, cancellationToken).ConfigureAwait(false);
#endif
internal unsafe nuint CompressStream(ref ZSTD_inBuffer_s input, ReadOnlySpan<byte> inputBuffer,
ZSTD_EndDirective directive)
{
fixed (byte* inputBufferPtr = inputBuffer)
fixed (byte* outputBufferPtr = outputBuffer)
{
input.src = inputBufferPtr;
output.dst = outputBufferPtr;
return compressor.CompressStream(ref input, ref output, directive).EnsureZstdSuccess();
}
}
public override bool CanRead => false;
public override bool CanSeek => false;
public override bool CanWrite => true;
public override long Length => throw new NotSupportedException();
public override long Position
{
get => throw new NotSupportedException();
set => throw new NotSupportedException();
}
public override long Seek(long offset, SeekOrigin origin) => throw new NotSupportedException();
public override void SetLength(long value) => throw new NotSupportedException();
public override int Read(byte[] buffer, int offset, int count) => throw new NotSupportedException();
private void EnsureNotDisposed()
{
if (compressor == null)
throw new ObjectDisposedException(nameof(CompressionStream));
}
}
}

View File

@@ -0,0 +1,164 @@
using System;
using ZstdSharp.Unsafe;
namespace ZstdSharp
{
public unsafe class Compressor : IDisposable
{
public static int MinCompressionLevel => Methods.ZSTD_minCLevel();
public static int MaxCompressionLevel => Methods.ZSTD_maxCLevel();
public const int DefaultCompressionLevel = 0;
private int level = DefaultCompressionLevel;
private ZSTD_CCtx_s* cctx;
public int Level
{
get => level;
set
{
if (level != value)
{
level = value;
SetParameter(ZSTD_cParameter.ZSTD_c_compressionLevel, value);
}
}
}
public void SetParameter(ZSTD_cParameter parameter, int value)
{
EnsureNotDisposed();
Methods.ZSTD_CCtx_setParameter(cctx, parameter, value).EnsureZstdSuccess();
}
public int GetParameter(ZSTD_cParameter parameter)
{
EnsureNotDisposed();
int value;
Methods.ZSTD_CCtx_getParameter(cctx, parameter, &value).EnsureZstdSuccess();
return value;
}
public void LoadDictionary(byte[] dict)
{
EnsureNotDisposed();
if (dict == null)
{
Methods.ZSTD_CCtx_loadDictionary(cctx, null, 0).EnsureZstdSuccess();
}
else
{
fixed (byte* dictPtr = dict)
Methods.ZSTD_CCtx_loadDictionary(cctx, dictPtr, (nuint) dict.Length).EnsureZstdSuccess();
}
}
public Compressor(int level = DefaultCompressionLevel)
{
cctx = Methods.ZSTD_createCCtx();
if (cctx == null)
throw new ZstdException(ZSTD_ErrorCode.ZSTD_error_GENERIC, "Failed to create cctx");
Level = level;
}
~Compressor()
{
ReleaseUnmanagedResources();
}
public static int GetCompressBound(int length)
=> (int) Methods.ZSTD_compressBound((nuint) length);
public static ulong GetCompressBoundLong(ulong length)
=> Methods.ZSTD_compressBound((nuint) length);
public Span<byte> Wrap(ReadOnlySpan<byte> src)
{
var dest = new byte[GetCompressBound(src.Length)];
var length = Wrap(src, dest);
return new Span<byte>(dest, 0, length);
}
public int Wrap(byte[] src, byte[] dest, int offset)
=> Wrap(src, new Span<byte>(dest, offset, dest.Length - offset));
public int Wrap(ReadOnlySpan<byte> src, Span<byte> dest)
{
EnsureNotDisposed();
fixed (byte* srcPtr = src)
fixed (byte* destPtr = dest)
return (int) Methods
.ZSTD_compress2(cctx, destPtr, (nuint) dest.Length, srcPtr, (nuint) src.Length)
.EnsureZstdSuccess();
}
public int Wrap(ArraySegment<byte> src, ArraySegment<byte> dest)
=> Wrap((ReadOnlySpan<byte>) src, dest);
public int Wrap(byte[] src, int srcOffset, int srcLength, byte[] dst, int dstOffset, int dstLength)
=> Wrap(new ReadOnlySpan<byte>(src, srcOffset, srcLength), new Span<byte>(dst, dstOffset, dstLength));
public bool TryWrap(byte[] src, byte[] dest, int offset, out int written)
=> TryWrap(src, new Span<byte>(dest, offset, dest.Length - offset), out written);
public bool TryWrap(ReadOnlySpan<byte> src, Span<byte> dest, out int written)
{
EnsureNotDisposed();
fixed (byte* srcPtr = src)
fixed (byte* destPtr = dest)
{
var returnValue =
Methods.ZSTD_compress2(cctx, destPtr, (nuint) dest.Length, srcPtr, (nuint) src.Length);
if (returnValue == unchecked(0 - (nuint)ZSTD_ErrorCode.ZSTD_error_dstSize_tooSmall))
{
written = default;
return false;
}
returnValue.EnsureZstdSuccess();
written = (int) returnValue;
return true;
}
}
public bool TryWrap(ArraySegment<byte> src, ArraySegment<byte> dest, out int written)
=> TryWrap((ReadOnlySpan<byte>)src, dest, out written);
public bool TryWrap(byte[] src, int srcOffset, int srcLength, byte[] dst, int dstOffset, int dstLength, out int written)
=> TryWrap(new ReadOnlySpan<byte>(src, srcOffset, srcLength), new Span<byte>(dst, dstOffset, dstLength), out written);
private void ReleaseUnmanagedResources()
{
if (cctx != null)
{
Methods.ZSTD_freeCCtx(cctx);
cctx = null;
}
}
public void Dispose()
{
ReleaseUnmanagedResources();
GC.SuppressFinalize(this);
}
private void EnsureNotDisposed()
{
if (cctx == null)
throw new ObjectDisposedException(nameof(Compressor));
}
internal nuint CompressStream(ref ZSTD_inBuffer_s input, ref ZSTD_outBuffer_s output, ZSTD_EndDirective directive)
{
fixed (ZSTD_inBuffer_s* inputPtr = &input)
fixed (ZSTD_outBuffer_s* outputPtr = &output)
{
return Methods.ZSTD_compressStream2(cctx, outputPtr, inputPtr, directive).EnsureZstdSuccess();
}
}
}
}

View File

@@ -0,0 +1,9 @@
namespace ZstdSharp
{
internal class Constants
{
//NOTE: https://docs.microsoft.com/en-us/dotnet/framework/configure-apps/file-schema/runtime/gcallowverylargeobjects-element#remarks
//NOTE: https://github.com/dotnet/runtime/blob/v5.0.0-rtm.20519.4/src/libraries/System.Private.CoreLib/src/System/Array.cs#L27
public const ulong MaxByteArrayLength = 0x7FFFFFC7;
}
}

View File

@@ -0,0 +1,190 @@
using System;
using System.Buffers;
using System.IO;
using System.Threading;
using System.Threading.Tasks;
using ZstdSharp.Unsafe;
namespace ZstdSharp
{
public class DecompressionStream : Stream
{
private readonly Stream innerStream;
private readonly byte[] inputBuffer;
private readonly int inputBufferSize;
private Decompressor decompressor;
private ZSTD_inBuffer_s input;
private nuint lastDecompressResult = 0;
public DecompressionStream(Stream stream, int bufferSize = 0)
{
if (stream == null)
throw new ArgumentNullException(nameof(stream));
if (!stream.CanRead)
throw new ArgumentException("Stream is not readable", nameof(stream));
if (bufferSize < 0)
throw new ArgumentOutOfRangeException(nameof(bufferSize));
innerStream = stream;
decompressor = new Decompressor();
inputBufferSize = bufferSize > 0 ? bufferSize : (int) Methods.ZSTD_CStreamInSize().EnsureZstdSuccess();
inputBuffer = ArrayPool<byte>.Shared.Rent(inputBufferSize);
input = new ZSTD_inBuffer_s {pos = (nuint) inputBufferSize, size = (nuint) inputBufferSize};
}
public void SetParameter(ZSTD_dParameter parameter, int value)
{
EnsureNotDisposed();
decompressor.SetParameter(parameter, value);
}
public int GetParameter(ZSTD_dParameter parameter)
{
EnsureNotDisposed();
return decompressor.GetParameter(parameter);
}
public void LoadDictionary(byte[] dict)
{
EnsureNotDisposed();
decompressor.LoadDictionary(dict);
}
~DecompressionStream() => Dispose(false);
protected override void Dispose(bool disposing)
{
if (decompressor == null)
return;
decompressor.Dispose();
ArrayPool<byte>.Shared.Return(inputBuffer);
}
public override int Read(byte[] buffer, int offset, int count)
=> Read(new Span<byte>(buffer, offset, count));
#if !NETSTANDARD2_0 && !NETFRAMEWORK
public override int Read(Span<byte> buffer)
#else
public int Read(Span<byte> buffer)
#endif
{
EnsureNotDisposed();
var output = new ZSTD_outBuffer_s {pos = 0, size = (nuint) buffer.Length};
while (output.pos < output.size)
{
if (input.pos >= input.size)
{
int bytesRead;
if ((bytesRead = innerStream.Read(inputBuffer, 0, inputBufferSize)) == 0)
{
if (lastDecompressResult != 0)
throw new EndOfStreamException("Premature end of stream");
break;
}
input.size = (nuint) bytesRead;
input.pos = 0;
}
lastDecompressResult = DecompressStream(ref output, buffer);
}
return (int) output.pos;
}
public override Task<int> ReadAsync(byte[] buffer, int offset, int count, CancellationToken cancellationToken)
=> ReadAsync(new Memory<byte>(buffer, offset, count), cancellationToken).AsTask();
#if !NETSTANDARD2_0 && !NET461
public override async ValueTask<int> ReadAsync(Memory<byte> buffer,
CancellationToken cancellationToken = default)
#else
public async ValueTask<int> ReadAsync(Memory<byte> buffer,
CancellationToken cancellationToken = default)
#endif
{
EnsureNotDisposed();
var output = new ZSTD_outBuffer_s { pos = 0, size = (nuint)buffer.Length};
while (output.pos < output.size)
{
if (input.pos >= input.size)
{
int bytesRead;
if ((bytesRead = await innerStream.ReadAsync(inputBuffer, 0, inputBufferSize, cancellationToken)
.ConfigureAwait(false)) == 0)
{
if (lastDecompressResult != 0)
throw new EndOfStreamException("Premature end of stream");
break;
}
input.size = (nuint) bytesRead;
input.pos = 0;
}
lastDecompressResult = DecompressStream(ref output, buffer.Span);
}
return (int) output.pos;
}
private unsafe nuint DecompressStream(ref ZSTD_outBuffer_s output, Span<byte> outputBuffer)
{
fixed (byte* inputBufferPtr = inputBuffer)
fixed (byte* outputBufferPtr = outputBuffer)
{
input.src = inputBufferPtr;
output.dst = outputBufferPtr;
return decompressor.DecompressStream(ref input, ref output);
}
}
public override bool CanRead => true;
public override bool CanSeek => false;
public override bool CanWrite => false;
public override long Length => throw new NotSupportedException();
public override long Position
{
get => throw new NotSupportedException();
set => throw new NotSupportedException();
}
public override void Flush() => throw new NotSupportedException();
public override long Seek(long offset, SeekOrigin origin) => throw new NotSupportedException();
public override void SetLength(long value) => throw new NotSupportedException();
public override void Write(byte[] buffer, int offset, int count) => throw new NotSupportedException();
private void EnsureNotDisposed()
{
if (decompressor == null)
throw new ObjectDisposedException(nameof(DecompressionStream));
}
#if NETSTANDARD2_0 || NET461
public virtual ValueTask DisposeAsync()
{
try
{
Dispose();
return default;
}
catch (Exception exc)
{
return new ValueTask(Task.FromException(exc));
}
}
#endif
}
}

View File

@@ -0,0 +1,149 @@
using System;
using ZstdSharp.Unsafe;
namespace ZstdSharp
{
public unsafe class Decompressor : IDisposable
{
private ZSTD_DCtx_s* dctx;
public Decompressor()
{
dctx = Methods.ZSTD_createDCtx();
if (dctx == null)
throw new ZstdException(ZSTD_ErrorCode.ZSTD_error_GENERIC, "Failed to create dctx");
}
~Decompressor()
{
ReleaseUnmanagedResources();
}
public void SetParameter(ZSTD_dParameter parameter, int value)
{
EnsureNotDisposed();
Methods.ZSTD_DCtx_setParameter(dctx, parameter, value).EnsureZstdSuccess();
}
public int GetParameter(ZSTD_dParameter parameter)
{
EnsureNotDisposed();
int value;
Methods.ZSTD_DCtx_getParameter(dctx, parameter, &value).EnsureZstdSuccess();
return value;
}
public void LoadDictionary(byte[] dict)
{
EnsureNotDisposed();
if (dict == null)
{
Methods.ZSTD_DCtx_loadDictionary(dctx, null, 0).EnsureZstdSuccess();
}
else
{
fixed (byte* dictPtr = dict)
Methods.ZSTD_DCtx_loadDictionary(dctx, dictPtr, (nuint) dict.Length).EnsureZstdSuccess();
}
}
public static ulong GetDecompressedSize(ReadOnlySpan<byte> src)
{
fixed (byte* srcPtr = src)
return Methods.ZSTD_decompressBound(srcPtr, (nuint) src.Length).EnsureContentSizeOk();
}
public static ulong GetDecompressedSize(ArraySegment<byte> src)
=> GetDecompressedSize((ReadOnlySpan<byte>) src);
public static ulong GetDecompressedSize(byte[] src, int srcOffset, int srcLength)
=> GetDecompressedSize(new ReadOnlySpan<byte>(src, srcOffset, srcLength));
public Span<byte> Unwrap(ReadOnlySpan<byte> src, int maxDecompressedSize = int.MaxValue)
{
var expectedDstSize = GetDecompressedSize(src);
if (expectedDstSize > (ulong) maxDecompressedSize)
throw new ZstdException(ZSTD_ErrorCode.ZSTD_error_dstSize_tooSmall,
$"Decompressed content size {expectedDstSize} is greater than {nameof(maxDecompressedSize)} {maxDecompressedSize}");
if (expectedDstSize > Constants.MaxByteArrayLength)
throw new ZstdException(ZSTD_ErrorCode.ZSTD_error_dstSize_tooSmall,
$"Decompressed content size {expectedDstSize} is greater than max possible byte array size {Constants.MaxByteArrayLength}");
var dest = new byte[expectedDstSize];
var length = Unwrap(src, dest);
return new Span<byte>(dest, 0, length);
}
public int Unwrap(byte[] src, byte[] dest, int offset)
=> Unwrap(src, new Span<byte>(dest, offset, dest.Length - offset));
public int Unwrap(ReadOnlySpan<byte> src, Span<byte> dest)
{
EnsureNotDisposed();
fixed (byte* srcPtr = src)
fixed (byte* destPtr = dest)
return (int) Methods
.ZSTD_decompressDCtx(dctx, destPtr, (nuint) dest.Length, srcPtr, (nuint) src.Length)
.EnsureZstdSuccess();
}
public int Unwrap(byte[] src, int srcOffset, int srcLength, byte[] dst, int dstOffset, int dstLength)
=> Unwrap(new ReadOnlySpan<byte>(src, srcOffset, srcLength), new Span<byte>(dst, dstOffset, dstLength));
public bool TryUnwrap(byte[] src, byte[] dest, int offset, out int written)
=> TryUnwrap(src, new Span<byte>(dest, offset, dest.Length - offset), out written);
public bool TryUnwrap(ReadOnlySpan<byte> src, Span<byte> dest, out int written)
{
EnsureNotDisposed();
fixed (byte* srcPtr = src)
fixed (byte* destPtr = dest)
{
var returnValue =
Methods.ZSTD_decompressDCtx(dctx, destPtr, (nuint) dest.Length, srcPtr, (nuint) src.Length);
if (returnValue == unchecked(0 - (nuint)ZSTD_ErrorCode.ZSTD_error_dstSize_tooSmall))
{
written = default;
return false;
}
returnValue.EnsureZstdSuccess();
written = (int) returnValue;
return true;
}
}
public bool TryUnwrap(byte[] src, int srcOffset, int srcLength, byte[] dst, int dstOffset, int dstLength, out int written)
=> TryUnwrap(new ReadOnlySpan<byte>(src, srcOffset, srcLength), new Span<byte>(dst, dstOffset, dstLength), out written);
private void ReleaseUnmanagedResources()
{
if (dctx != null)
{
Methods.ZSTD_freeDCtx(dctx);
dctx = null;
}
}
public void Dispose()
{
ReleaseUnmanagedResources();
GC.SuppressFinalize(this);
}
private void EnsureNotDisposed()
{
if (dctx == null)
throw new ObjectDisposedException(nameof(Decompressor));
}
internal nuint DecompressStream(ref ZSTD_inBuffer_s input, ref ZSTD_outBuffer_s output)
{
fixed (ZSTD_inBuffer_s* inputPtr = &input)
fixed (ZSTD_outBuffer_s* outputPtr = &output)
{
return Methods.ZSTD_decompressStream(dctx, outputPtr, inputPtr).EnsureZstdSuccess();
}
}
}
}

View File

@@ -0,0 +1,39 @@
using System;
using System.Collections.Generic;
using System.IO;
using System.Linq;
using ZstdSharp.Unsafe;
namespace ZstdSharp
{
public static unsafe class DictBuilder
{
public static byte[] TrainFromBuffer(IEnumerable<byte[]> samples, int dictCapacity = DefaultDictCapacity)
{
var ms = new MemoryStream();
var samplesSizes = samples.Select(sample =>
{
ms.Write(sample, 0, sample.Length);
return (nuint) sample.Length;
}).ToArray();
var dictBuffer = new byte[dictCapacity];
fixed (byte* dictBufferPtr = dictBuffer)
fixed (byte* samplesBufferPtr = ms.GetBuffer())
fixed (nuint* samplesSizesPtr = samplesSizes)
{
var dictSize = (int) Methods
.ZDICT_trainFromBuffer(dictBufferPtr, (nuint) dictCapacity, samplesBufferPtr, samplesSizesPtr,
(uint) samplesSizes.Length)
.EnsureZdictSuccess();
if (dictCapacity != dictSize)
Array.Resize(ref dictBuffer, dictSize);
return dictBuffer;
}
}
public const int DefaultDictCapacity = 112640; // Used by zstd utility by default
}
}

View File

@@ -0,0 +1,201 @@
// Licensed to the .NET Foundation under one or more agreements.
// The .NET Foundation licenses this file to you under the MIT license.
#if !NETCOREAPP3_0_OR_GREATER
using System.Runtime.CompilerServices;
using static ZstdSharp.UnsafeHelper;
// Some routines inspired by the Stanford Bit Twiddling Hacks by Sean Eron Anderson:
// http://graphics.stanford.edu/~seander/bithacks.html
namespace System.Numerics
{
/// <summary>
/// Utility methods for intrinsic bit-twiddling operations.
/// The methods use hardware intrinsics when available on the underlying platform,
/// otherwise they use optimized software fallbacks.
/// </summary>
public static unsafe class BitOperations
{
// hack: should be public because of inline
public static readonly byte* TrailingZeroCountDeBruijn = GetArrayPointer(new byte[]
{
00, 01, 28, 02, 29, 14, 24, 03,
30, 22, 20, 15, 25, 17, 04, 08,
31, 27, 13, 23, 21, 19, 16, 07,
26, 12, 18, 06, 11, 05, 10, 09
});
// hack: should be public because of inline
public static readonly byte* Log2DeBruijn = GetArrayPointer(new byte[]
{
00, 09, 01, 10, 13, 21, 02, 29,
11, 14, 16, 18, 22, 25, 03, 30,
08, 12, 20, 28, 15, 17, 24, 07,
19, 27, 23, 06, 26, 05, 04, 31
});
/// <summary>
/// Returns the integer (floor) log of the specified value, base 2.
/// Note that by convention, input value 0 returns 0 since log(0) is undefined.
/// </summary>
/// <param name="value">The value.</param>
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public static int Log2(uint value)
{
// The 0->0 contract is fulfilled by setting the LSB to 1.
// Log(1) is 0, and setting the LSB for values > 1 does not change the log2 result.
value |= 1;
// value lzcnt actual expected
// ..0001 31 31-31 0
// ..0010 30 31-30 1
// 0010.. 2 31-2 29
// 0100.. 1 31-1 30
// 1000.. 0 31-0 31
// Fallback contract is 0->0
// No AggressiveInlining due to large method size
// Has conventional contract 0->0 (Log(0) is undefined)
// Fill trailing zeros with ones, eg 00010010 becomes 00011111
value |= value >> 01;
value |= value >> 02;
value |= value >> 04;
value |= value >> 08;
value |= value >> 16;
// uint.MaxValue >> 27 is always in range [0 - 31] so we use Unsafe.AddByteOffset to avoid bounds check
return Log2DeBruijn[
// Using deBruijn sequence, k=2, n=5 (2^5=32) : 0b_0000_0111_1100_0100_1010_1100_1101_1101u
(int)((value * 0x07C4ACDDu) >> 27)];
}
/// <summary>
/// Returns the integer (floor) log of the specified value, base 2.
/// Note that by convention, input value 0 returns 0 since log(0) is undefined.
/// </summary>
/// <param name="value">The value.</param>
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public static int Log2(ulong value)
{
value |= 1;
uint hi = (uint)(value >> 32);
if (hi == 0)
{
return Log2((uint)value);
}
return 32 + Log2(hi);
}
/// <summary>
/// Count the number of trailing zero bits in an integer value.
/// Similar in behavior to the x86 instruction TZCNT.
/// </summary>
/// <param name="value">The value.</param>
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public static int TrailingZeroCount(int value)
=> TrailingZeroCount((uint)value);
/// <summary>
/// Count the number of trailing zero bits in an integer value.
/// Similar in behavior to the x86 instruction TZCNT.
/// </summary>
/// <param name="value">The value.</param>
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public static int TrailingZeroCount(uint value)
{
// Unguarded fallback contract is 0->0, BSF contract is 0->undefined
if (value == 0)
{
return 32;
}
// uint.MaxValue >> 27 is always in range [0 - 31] so we use Unsafe.AddByteOffset to avoid bounds check
return TrailingZeroCountDeBruijn[
// Using deBruijn sequence, k=2, n=5 (2^5=32) : 0b_0000_0111_0111_1100_1011_0101_0011_0001u
(int)(((value & (uint)-(int)value) * 0x077CB531u) >> 27)]; // Multi-cast mitigates redundant conv.u8
}
/// <summary>
/// Count the number of trailing zero bits in a mask.
/// Similar in behavior to the x86 instruction TZCNT.
/// </summary>
/// <param name="value">The value.</param>
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public static int TrailingZeroCount(long value)
=> TrailingZeroCount((ulong)value);
/// <summary>
/// Count the number of trailing zero bits in a mask.
/// Similar in behavior to the x86 instruction TZCNT.
/// </summary>
/// <param name="value">The value.</param>
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public static int TrailingZeroCount(ulong value)
{
uint lo = (uint)value;
if (lo == 0)
{
return 32 + TrailingZeroCount((uint)(value >> 32));
}
return TrailingZeroCount(lo);
}
/// <summary>
/// Rotates the specified value left by the specified number of bits.
/// Similar in behavior to the x86 instruction ROL.
/// </summary>
/// <param name="value">The value to rotate.</param>
/// <param name="offset">The number of bits to rotate by.
/// Any value outside the range [0..31] is treated as congruent mod 32.</param>
/// <returns>The rotated value.</returns>
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public static uint RotateLeft(uint value, int offset)
=> (value << offset) | (value >> (32 - offset));
/// <summary>
/// Rotates the specified value left by the specified number of bits.
/// Similar in behavior to the x86 instruction ROL.
/// </summary>
/// <param name="value">The value to rotate.</param>
/// <param name="offset">The number of bits to rotate by.
/// Any value outside the range [0..63] is treated as congruent mod 64.</param>
/// <returns>The rotated value.</returns>
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public static ulong RotateLeft(ulong value, int offset)
=> (value << offset) | (value >> (64 - offset));
/// <summary>
/// Rotates the specified value right by the specified number of bits.
/// Similar in behavior to the x86 instruction ROR.
/// </summary>
/// <param name="value">The value to rotate.</param>
/// <param name="offset">The number of bits to rotate by.
/// Any value outside the range [0..31] is treated as congruent mod 32.</param>
/// <returns>The rotated value.</returns>
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public static uint RotateRight(uint value, int offset)
=> (value >> offset) | (value << (32 - offset));
/// <summary>
/// Rotates the specified value right by the specified number of bits.
/// Similar in behavior to the x86 instruction ROR.
/// </summary>
/// <param name="value">The value to rotate.</param>
/// <param name="offset">The number of bits to rotate by.
/// Any value outside the range [0..63] is treated as congruent mod 64.</param>
/// <returns>The rotated value.</returns>
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public static ulong RotateRight(ulong value, int offset)
=> (value >> offset) | (value << (64 - offset));
}
}
#endif

View File

@@ -0,0 +1,25 @@
// Licensed to the .NET Foundation under one or more agreements.
// The .NET Foundation licenses this file to you under the MIT license.
#if !NET5_0_OR_GREATER
namespace System.Runtime.CompilerServices
{
/// <summary>
/// Used to indicate to the compiler that the <c>.locals init</c> flag should not be set in method headers.
/// </summary>
/// <remarks>Internal copy of the .NET 5 attribute.</remarks>
[AttributeUsage(
AttributeTargets.Module |
AttributeTargets.Class |
AttributeTargets.Struct |
AttributeTargets.Interface |
AttributeTargets.Constructor |
AttributeTargets.Method |
AttributeTargets.Property |
AttributeTargets.Event,
Inherited = false)]
internal sealed class SkipLocalsInitAttribute : Attribute
{
}
}
#endif

View File

@@ -0,0 +1,54 @@
// Licensed to the .NET Foundation under one or more agreements.
// The .NET Foundation licenses this file to you under the MIT license.
#if !NETCOREAPP3_0_OR_GREATER
using System.Runtime.CompilerServices;
namespace System.Runtime.Intrinsics
{
public static class Vector128
{
internal const int Size = 16;
public static unsafe Vector128<byte> Create(byte value)
{
byte* pResult = stackalloc byte[16]
{
value,
value,
value,
value,
value,
value,
value,
value,
value,
value,
value,
value,
value,
value,
value,
value,
};
return Unsafe.AsRef<Vector128<byte>>(pResult);
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public static Vector128<U> As<T, U>(this Vector128<T> vector)
where T : struct
where U : struct =>
Unsafe.As<Vector128<T>, Vector128<U>>(ref vector);
public static T GetElement<T>(this Vector128<T> vector, int index)
where T : struct
{
ref T e0 = ref Unsafe.As<Vector128<T>, T>(ref vector);
return Unsafe.Add(ref e0, index);
}
}
}
#endif

View File

@@ -0,0 +1,36 @@
// Licensed to the .NET Foundation under one or more agreements.
// The .NET Foundation licenses this file to you under the MIT license.
#if !NETCOREAPP3_0_OR_GREATER
using System.Runtime.CompilerServices;
using System.Runtime.InteropServices;
namespace System.Runtime.Intrinsics
{
[StructLayout(LayoutKind.Sequential, Size = Vector128.Size)]
public readonly struct Vector128<T> : IEquatable<Vector128<T>>
where T : struct
{
private readonly ulong _00;
private readonly ulong _01;
public static int Count => Vector128.Size / Unsafe.SizeOf<T>();
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public bool Equals(Vector128<T> other)
{
for (int i = 0; i < Count; i++)
{
if (!((IEquatable<T>)(this.GetElement(i))).Equals(other.GetElement(i)))
{
return false;
}
}
return true;
}
}
}
#endif

View File

@@ -0,0 +1,43 @@
using ZstdSharp.Unsafe;
namespace ZstdSharp
{
public static unsafe class ThrowHelper
{
private const ulong ZSTD_CONTENTSIZE_UNKNOWN = unchecked(0UL - 1);
private const ulong ZSTD_CONTENTSIZE_ERROR = unchecked(0UL - 2);
public static nuint EnsureZstdSuccess(this nuint returnValue)
{
if (Methods.ZSTD_isError(returnValue) != 0)
ThrowException(returnValue, Methods.ZSTD_getErrorName(returnValue));
return returnValue;
}
public static nuint EnsureZdictSuccess(this nuint returnValue)
{
if (Methods.ZDICT_isError(returnValue) != 0)
ThrowException(returnValue, Methods.ZDICT_getErrorName(returnValue));
return returnValue;
}
public static ulong EnsureContentSizeOk(this ulong returnValue)
{
if (returnValue == ZSTD_CONTENTSIZE_UNKNOWN)
throw new ZstdException(ZSTD_ErrorCode.ZSTD_error_GENERIC, "Decompressed content size is not specified");
if (returnValue == ZSTD_CONTENTSIZE_ERROR)
throw new ZstdException(ZSTD_ErrorCode.ZSTD_error_GENERIC, "Decompressed content size cannot be determined (e.g. invalid magic number, srcSize too small)");
return returnValue;
}
private static void ThrowException(nuint returnValue, string message)
{
var code = 0 - returnValue;
throw new ZstdException((ZSTD_ErrorCode) code, message);
}
}
}

View File

@@ -0,0 +1,368 @@
using System;
using static ZstdSharp.UnsafeHelper;
namespace ZstdSharp.Unsafe
{
public static unsafe partial class Methods
{
static uint* rtbTable = GetArrayPointer(new uint[8] {
0,
473195,
504333,
520860,
550000,
700000,
750000,
830000,
});
static byte* LL_Code = GetArrayPointer(new byte[64]
{
0,
1,
2,
3,
4,
5,
6,
7,
8,
9,
10,
11,
12,
13,
14,
15,
16,
16,
17,
17,
18,
18,
19,
19,
20,
20,
20,
20,
21,
21,
21,
21,
22,
22,
22,
22,
22,
22,
22,
22,
23,
23,
23,
23,
23,
23,
23,
23,
24,
24,
24,
24,
24,
24,
24,
24,
24,
24,
24,
24,
24,
24,
24,
24,
});
static byte* ML_Code = GetArrayPointer(new byte[128]
{
0,
1,
2,
3,
4,
5,
6,
7,
8,
9,
10,
11,
12,
13,
14,
15,
16,
17,
18,
19,
20,
21,
22,
23,
24,
25,
26,
27,
28,
29,
30,
31,
32,
32,
33,
33,
34,
34,
35,
35,
36,
36,
36,
36,
37,
37,
37,
37,
38,
38,
38,
38,
38,
38,
38,
38,
39,
39,
39,
39,
39,
39,
39,
39,
40,
40,
40,
40,
40,
40,
40,
40,
40,
40,
40,
40,
40,
40,
40,
40,
41,
41,
41,
41,
41,
41,
41,
41,
41,
41,
41,
41,
41,
41,
41,
41,
42,
42,
42,
42,
42,
42,
42,
42,
42,
42,
42,
42,
42,
42,
42,
42,
42,
42,
42,
42,
42,
42,
42,
42,
42,
42,
42,
42,
42,
42,
42,
42,
});
static ulong* srcSizeTiers = GetArrayPointer(new ulong[4]
{
16 * (1 << 10),
128 * (1 << 10),
256 * (1 << 10),
(unchecked(0UL - 1)),
});
static ZSTD_blockCompressor?[][] blockCompressor = new ZSTD_blockCompressor?[4][]
{
new ZSTD_blockCompressor[10]
{
ZSTD_compressBlock_fast,
ZSTD_compressBlock_fast,
ZSTD_compressBlock_doubleFast,
ZSTD_compressBlock_greedy,
ZSTD_compressBlock_lazy,
ZSTD_compressBlock_lazy2,
ZSTD_compressBlock_btlazy2,
ZSTD_compressBlock_btopt,
ZSTD_compressBlock_btultra,
ZSTD_compressBlock_btultra2,
},
new ZSTD_blockCompressor[10]
{
ZSTD_compressBlock_fast_extDict,
ZSTD_compressBlock_fast_extDict,
ZSTD_compressBlock_doubleFast_extDict,
ZSTD_compressBlock_greedy_extDict,
ZSTD_compressBlock_lazy_extDict,
ZSTD_compressBlock_lazy2_extDict,
ZSTD_compressBlock_btlazy2_extDict,
ZSTD_compressBlock_btopt_extDict,
ZSTD_compressBlock_btultra_extDict,
ZSTD_compressBlock_btultra_extDict,
},
new ZSTD_blockCompressor[10]
{
ZSTD_compressBlock_fast_dictMatchState,
ZSTD_compressBlock_fast_dictMatchState,
ZSTD_compressBlock_doubleFast_dictMatchState,
ZSTD_compressBlock_greedy_dictMatchState,
ZSTD_compressBlock_lazy_dictMatchState,
ZSTD_compressBlock_lazy2_dictMatchState,
ZSTD_compressBlock_btlazy2_dictMatchState,
ZSTD_compressBlock_btopt_dictMatchState,
ZSTD_compressBlock_btultra_dictMatchState,
ZSTD_compressBlock_btultra_dictMatchState,
},
new ZSTD_blockCompressor?[10]
{
null,
null,
null,
ZSTD_compressBlock_greedy_dedicatedDictSearch,
ZSTD_compressBlock_lazy_dedicatedDictSearch,
ZSTD_compressBlock_lazy2_dedicatedDictSearch,
null,
null,
null,
null,
},
};
static ZSTD_blockCompressor[][] rowBasedBlockCompressors = new ZSTD_blockCompressor[4][]
{
new ZSTD_blockCompressor[3]
{
ZSTD_compressBlock_greedy_row,
ZSTD_compressBlock_lazy_row,
ZSTD_compressBlock_lazy2_row,
},
new ZSTD_blockCompressor[3]
{
ZSTD_compressBlock_greedy_extDict_row,
ZSTD_compressBlock_lazy_extDict_row,
ZSTD_compressBlock_lazy2_extDict_row,
},
new ZSTD_blockCompressor[3]
{
ZSTD_compressBlock_greedy_dictMatchState_row,
ZSTD_compressBlock_lazy_dictMatchState_row,
ZSTD_compressBlock_lazy2_dictMatchState_row,
},
new ZSTD_blockCompressor[3]
{
ZSTD_compressBlock_greedy_dedicatedDictSearch_row,
ZSTD_compressBlock_lazy_dedicatedDictSearch_row,
ZSTD_compressBlock_lazy2_dedicatedDictSearch_row,
},
};
static searchMax_f?[][] searchFuncs = new searchMax_f?[4][]
{
new searchMax_f[3]
{
ZSTD_HcFindBestMatch_selectMLS,
ZSTD_BtFindBestMatch_selectMLS,
ZSTD_RowFindBestMatch_selectRowLog,
},
new searchMax_f?[3]
{
null,
null,
null,
},
new searchMax_f[3]
{
ZSTD_HcFindBestMatch_dictMatchState_selectMLS,
ZSTD_BtFindBestMatch_dictMatchState_selectMLS,
ZSTD_RowFindBestMatch_dictMatchState_selectRowLog,
},
new searchMax_f?[3]
{
ZSTD_HcFindBestMatch_dedicatedDictSearch_selectMLS,
null,
ZSTD_RowFindBestMatch_dedicatedDictSearch_selectRowLog,
},
};
static searchMax_f[] searchFuncsExtGeneric = new searchMax_f[3]
{
ZSTD_HcFindBestMatch_extDict_selectMLS,
ZSTD_BtFindBestMatch_extDict_selectMLS,
ZSTD_RowFindBestMatch_extDict_selectRowLog,
};
static decompressionAlgo[] decompress = new decompressionAlgo[2]
{
HUF_decompress4X1,
HUF_decompress4X2,
};
static uint* dec32table = GetArrayPointer(new uint[8]
{
0,
1,
2,
1,
4,
4,
4,
4,
});
static int* dec64table = GetArrayPointer(new int[8]
{
8,
8,
8,
7,
8,
9,
10,
11,
});
private static byte* emptyString = GetArrayPointer(new byte[] {0});
}
}

View File

@@ -0,0 +1,24 @@
using System;
namespace ZstdSharp.Unsafe
{
/*-******************************************
* bitStream encoding API (write forward)
********************************************/
/* bitStream can mix input from multiple sources.
* A critical property of these streams is that they encode and decode in **reverse** direction.
* So the first bit sequence you add will be the last to be read, like a LIFO stack.
*/
public unsafe partial struct BIT_CStream_t
{
public nuint bitContainer;
public uint bitPos;
public sbyte* startPtr;
public sbyte* ptr;
public sbyte* endPtr;
}
}

View File

@@ -0,0 +1,12 @@
using System;
namespace ZstdSharp.Unsafe
{
public enum BIT_DStream_status
{
BIT_DStream_unfinished = 0,
BIT_DStream_endOfBuffer = 1,
BIT_DStream_completed = 2,
BIT_DStream_overflow = 3,
}
}

View File

@@ -0,0 +1,20 @@
using System;
namespace ZstdSharp.Unsafe
{
/*-********************************************
* bitStream decoding API (read backward)
**********************************************/
public unsafe partial struct BIT_DStream_t
{
public nuint bitContainer;
public uint bitsConsumed;
public sbyte* ptr;
public sbyte* start;
public sbyte* limitPtr;
}
}

View File

@@ -0,0 +1,420 @@
using System;
using System.Numerics;
using System.Runtime.CompilerServices;
using static ZstdSharp.UnsafeHelper;
namespace ZstdSharp.Unsafe
{
public static unsafe partial class Methods
{
/*-**************************************************************
* Internal functions
****************************************************************/
[MethodImpl(MethodImplOptions.AggressiveInlining)]
[InlineMethod.Inline]
private static uint BIT_highbit32(uint val)
{
assert(val != 0);
{
return (uint) BitOperations.Log2(val);
}
}
public static uint* BIT_mask = GetArrayPointer(new uint[32]
{
0,
1,
3,
7,
0xF,
0x1F,
0x3F,
0x7F,
0xFF,
0x1FF,
0x3FF,
0x7FF,
0xFFF,
0x1FFF,
0x3FFF,
0x7FFF,
0xFFFF,
0x1FFFF,
0x3FFFF,
0x7FFFF,
0xFFFFF,
0x1FFFFF,
0x3FFFFF,
0x7FFFFF,
0xFFFFFF,
0x1FFFFFF,
0x3FFFFFF,
0x7FFFFFF,
0xFFFFFFF,
0x1FFFFFFF,
0x3FFFFFFF,
0x7FFFFFFF,
});
/*-**************************************************************
* bitStream encoding
****************************************************************/
/*! BIT_initCStream() :
* `dstCapacity` must be > sizeof(size_t)
* @return : 0 if success,
* otherwise an error code (can be tested using ERR_isError()) */
[MethodImpl(MethodImplOptions.AggressiveInlining)]
private static nuint BIT_initCStream(BIT_CStream_t* bitC, void* startPtr, nuint dstCapacity)
{
bitC->bitContainer = 0;
bitC->bitPos = 0;
bitC->startPtr = (sbyte*)(startPtr);
bitC->ptr = bitC->startPtr;
bitC->endPtr = bitC->startPtr + dstCapacity - (nuint)(sizeof(nuint));
if (dstCapacity <= (nuint)(sizeof(nuint)))
{
return (unchecked((nuint)(-(int)ZSTD_ErrorCode.ZSTD_error_dstSize_tooSmall)));
}
return 0;
}
/*! BIT_addBits() :
* can add up to 31 bits into `bitC`.
* Note : does not check for register overflow ! */
[MethodImpl(MethodImplOptions.AggressiveInlining)]
private static void BIT_addBits(BIT_CStream_t* bitC, nuint value, uint nbBits)
{
assert(nbBits < ((nuint)(sizeof(uint) * 32) / (nuint)(sizeof(uint))));
assert(nbBits + bitC->bitPos < (nuint)(sizeof(nuint)) * 8);
bitC->bitContainer |= (value & BIT_mask[nbBits]) << (int)bitC->bitPos;
bitC->bitPos += nbBits;
}
/*! BIT_addBitsFast() :
* works only if `value` is _clean_,
* meaning all high bits above nbBits are 0 */
[MethodImpl(MethodImplOptions.AggressiveInlining)]
private static void BIT_addBitsFast(BIT_CStream_t* bitC, nuint value, uint nbBits)
{
assert((value >> (int)nbBits) == 0);
assert(nbBits + bitC->bitPos < (nuint)(sizeof(nuint)) * 8);
bitC->bitContainer |= value << (int)bitC->bitPos;
bitC->bitPos += nbBits;
}
/*! BIT_flushBitsFast() :
* assumption : bitContainer has not overflowed
* unsafe version; does not check buffer overflow */
[MethodImpl(MethodImplOptions.AggressiveInlining)]
private static void BIT_flushBitsFast(BIT_CStream_t* bitC)
{
nuint nbBytes = bitC->bitPos >> 3;
assert(bitC->bitPos < (nuint)(sizeof(nuint)) * 8);
assert(bitC->ptr <= bitC->endPtr);
MEM_writeLEST((void*)bitC->ptr, bitC->bitContainer);
bitC->ptr += nbBytes;
bitC->bitPos &= 7;
bitC->bitContainer >>= (int)(nbBytes * 8);
}
/*! BIT_flushBits() :
* assumption : bitContainer has not overflowed
* safe version; check for buffer overflow, and prevents it.
* note : does not signal buffer overflow.
* overflow will be revealed later on using BIT_closeCStream() */
[MethodImpl(MethodImplOptions.AggressiveInlining)]
private static void BIT_flushBits(BIT_CStream_t* bitC)
{
nuint nbBytes = bitC->bitPos >> 3;
assert(bitC->bitPos < (nuint)(sizeof(nuint)) * 8);
assert(bitC->ptr <= bitC->endPtr);
MEM_writeLEST((void*)bitC->ptr, bitC->bitContainer);
bitC->ptr += nbBytes;
if (bitC->ptr > bitC->endPtr)
{
bitC->ptr = bitC->endPtr;
}
bitC->bitPos &= 7;
bitC->bitContainer >>= (int)(nbBytes * 8);
}
/*! BIT_closeCStream() :
* @return : size of CStream, in bytes,
* or 0 if it could not fit into dstBuffer */
[MethodImpl(MethodImplOptions.AggressiveInlining)]
private static nuint BIT_closeCStream(BIT_CStream_t* bitC)
{
BIT_addBitsFast(bitC, 1, 1);
BIT_flushBits(bitC);
if (bitC->ptr >= bitC->endPtr)
{
return 0;
}
return (nuint)((bitC->ptr - bitC->startPtr) + (((bitC->bitPos > 0)) ? 1 : 0));
}
/*-********************************************************
* bitStream decoding
**********************************************************/
/*! BIT_initDStream() :
* Initialize a BIT_DStream_t.
* `bitD` : a pointer to an already allocated BIT_DStream_t structure.
* `srcSize` must be the *exact* size of the bitStream, in bytes.
* @return : size of stream (== srcSize), or an errorCode if a problem is detected
*/
[MethodImpl(MethodImplOptions.AggressiveInlining)]
private static nuint BIT_initDStream(BIT_DStream_t* bitD, void* srcBuffer, nuint srcSize)
{
if (srcSize < 1)
{
memset((void*)(bitD), (0), ((nuint)(sizeof(BIT_DStream_t))));
return (unchecked((nuint)(-(int)ZSTD_ErrorCode.ZSTD_error_srcSize_wrong)));
}
bitD->start = (sbyte*)(srcBuffer);
bitD->limitPtr = bitD->start + (nuint)(sizeof(nuint));
if (srcSize >= (nuint)(sizeof(nuint)))
{
bitD->ptr = (sbyte*)(srcBuffer) + srcSize - (nuint)(sizeof(nuint));
bitD->bitContainer = MEM_readLEST((void*)bitD->ptr);
{
byte lastByte = ((byte*)(srcBuffer))[srcSize - 1];
bitD->bitsConsumed = lastByte != 0 ? 8 - BIT_highbit32(lastByte) : 0;
if (lastByte == 0)
{
return (unchecked((nuint)(-(int)ZSTD_ErrorCode.ZSTD_error_GENERIC)));
}
}
}
else
{
bitD->ptr = bitD->start;
bitD->bitContainer = *(byte*)(bitD->start);
switch (srcSize)
{
case 7:
{
bitD->bitContainer += (nuint)(((byte*)(srcBuffer))[6]) << (int)((nuint)(sizeof(nuint)) * 8 - 16);
}
goto case 6;
case 6:
{
bitD->bitContainer += (nuint)(((byte*)(srcBuffer))[5]) << (int)((nuint)(sizeof(nuint)) * 8 - 24);
}
goto case 5;
case 5:
{
bitD->bitContainer += (nuint)(((byte*)(srcBuffer))[4]) << (int)((nuint)(sizeof(nuint)) * 8 - 32);
}
goto case 4;
case 4:
{
bitD->bitContainer += (nuint)(((byte*)(srcBuffer))[3]) << 24;
}
goto case 3;
case 3:
{
bitD->bitContainer += (nuint)(((byte*)(srcBuffer))[2]) << 16;
}
goto case 2;
case 2:
{
bitD->bitContainer += (nuint)(((byte*)(srcBuffer))[1]) << 8;
}
goto default;
default:
{
break;
}
}
{
byte lastByte = ((byte*)(srcBuffer))[srcSize - 1];
bitD->bitsConsumed = lastByte != 0 ? 8 - BIT_highbit32(lastByte) : 0;
if (lastByte == 0)
{
return (unchecked((nuint)(-(int)ZSTD_ErrorCode.ZSTD_error_corruption_detected)));
}
}
bitD->bitsConsumed += (uint)((nuint)(sizeof(nuint)) - srcSize) * 8;
}
return srcSize;
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
private static nuint BIT_getUpperBits(nuint bitContainer, uint start)
{
return bitContainer >> (int)start;
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
private static nuint BIT_getMiddleBits(nuint bitContainer, uint start, uint nbBits)
{
uint regMask = (uint)((nuint)(sizeof(nuint)) * 8 - 1);
assert(nbBits < ((nuint)(sizeof(uint) * 32) / (nuint)(sizeof(uint))));
return (bitContainer >> (int)(start & regMask)) & BIT_mask[nbBits];
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
private static nuint BIT_getLowerBits(nuint bitContainer, uint nbBits)
{
assert(nbBits < ((nuint)(sizeof(uint) * 32) / (nuint)(sizeof(uint))));
return bitContainer & BIT_mask[nbBits];
}
/*! BIT_lookBits() :
* Provides next n bits from local register.
* local register is not modified.
* On 32-bits, maxNbBits==24.
* On 64-bits, maxNbBits==56.
* @return : value extracted */
[MethodImpl(MethodImplOptions.AggressiveInlining)]
private static nuint BIT_lookBits(BIT_DStream_t* bitD, uint nbBits)
{
return BIT_getMiddleBits(bitD->bitContainer, (uint)(((nuint)(sizeof(nuint)) * 8) - bitD->bitsConsumed - nbBits), nbBits);
}
/*! BIT_lookBitsFast() :
* unsafe version; only works if nbBits >= 1 */
[MethodImpl(MethodImplOptions.AggressiveInlining)]
private static nuint BIT_lookBitsFast(BIT_DStream_t* bitD, uint nbBits)
{
uint regMask = (uint)((nuint)(sizeof(nuint)) * 8 - 1);
assert(nbBits >= 1);
return (bitD->bitContainer << (int)(bitD->bitsConsumed & regMask)) >> (int)(((regMask + 1) - nbBits) & regMask);
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
private static void BIT_skipBits(BIT_DStream_t* bitD, uint nbBits)
{
bitD->bitsConsumed += nbBits;
}
/*! BIT_readBits() :
* Read (consume) next n bits from local register and update.
* Pay attention to not read more than nbBits contained into local register.
* @return : extracted value. */
[MethodImpl(MethodImplOptions.AggressiveInlining)]
private static nuint BIT_readBits(BIT_DStream_t* bitD, uint nbBits)
{
nuint value = BIT_lookBits(bitD, nbBits);
BIT_skipBits(bitD, nbBits);
return value;
}
/*! BIT_readBitsFast() :
* unsafe version; only works only if nbBits >= 1 */
[MethodImpl(MethodImplOptions.AggressiveInlining)]
private static nuint BIT_readBitsFast(BIT_DStream_t* bitD, uint nbBits)
{
nuint value = BIT_lookBitsFast(bitD, nbBits);
assert(nbBits >= 1);
BIT_skipBits(bitD, nbBits);
return value;
}
/*! BIT_reloadDStreamFast() :
* Similar to BIT_reloadDStream(), but with two differences:
* 1. bitsConsumed <= sizeof(bitD->bitContainer)*8 must hold!
* 2. Returns BIT_DStream_overflow when bitD->ptr < bitD->limitPtr, at this
* point you must use BIT_reloadDStream() to reload.
*/
[MethodImpl(MethodImplOptions.AggressiveInlining)]
private static BIT_DStream_status BIT_reloadDStreamFast(BIT_DStream_t* bitD)
{
if ((bitD->ptr < bitD->limitPtr))
{
return BIT_DStream_status.BIT_DStream_overflow;
}
assert(bitD->bitsConsumed <= (nuint)(sizeof(nuint)) * 8);
bitD->ptr -= bitD->bitsConsumed >> 3;
bitD->bitsConsumed &= 7;
bitD->bitContainer = MEM_readLEST((void*)bitD->ptr);
return BIT_DStream_status.BIT_DStream_unfinished;
}
/*! BIT_reloadDStream() :
* Refill `bitD` from buffer previously set in BIT_initDStream() .
* This function is safe, it guarantees it will not read beyond src buffer.
* @return : status of `BIT_DStream_t` internal register.
* when status == BIT_DStream_unfinished, internal register is filled with at least 25 or 57 bits */
[MethodImpl(MethodImplOptions.AggressiveInlining)]
private static BIT_DStream_status BIT_reloadDStream(BIT_DStream_t* bitD)
{
if (bitD->bitsConsumed > ((nuint)(sizeof(nuint)) * 8))
{
return BIT_DStream_status.BIT_DStream_overflow;
}
if (bitD->ptr >= bitD->limitPtr)
{
return BIT_reloadDStreamFast(bitD);
}
if (bitD->ptr == bitD->start)
{
if (bitD->bitsConsumed < (nuint)(sizeof(nuint)) * 8)
{
return BIT_DStream_status.BIT_DStream_endOfBuffer;
}
return BIT_DStream_status.BIT_DStream_completed;
}
{
uint nbBytes = bitD->bitsConsumed >> 3;
BIT_DStream_status result = BIT_DStream_status.BIT_DStream_unfinished;
if (bitD->ptr - nbBytes < bitD->start)
{
nbBytes = (uint)(bitD->ptr - bitD->start);
result = BIT_DStream_status.BIT_DStream_endOfBuffer;
}
bitD->ptr -= nbBytes;
bitD->bitsConsumed -= nbBytes * 8;
bitD->bitContainer = MEM_readLEST((void*)bitD->ptr);
return result;
}
}
/*! BIT_endOfDStream() :
* @return : 1 if DStream has _exactly_ reached its end (all bits consumed).
*/
[MethodImpl(MethodImplOptions.AggressiveInlining)]
private static uint BIT_endOfDStream(BIT_DStream_t* DStream)
{
return ((((DStream->ptr == DStream->start) && (DStream->bitsConsumed == (nuint)(sizeof(nuint)) * 8))) ? 1U : 0U);
}
}
}

View File

@@ -0,0 +1,29 @@
using System;
namespace ZstdSharp.Unsafe
{
/**
* COVER_best_t is used for two purposes:
* 1. Synchronizing threads.
* 2. Saving the best parameters and dictionary.
*
* All of the methods except COVER_best_init() are thread safe if zstd is
* compiled with multithreaded support.
*/
public unsafe partial struct COVER_best_s
{
public int mutex;
public int cond;
public nuint liveJobs;
public void* dict;
public nuint dictSize;
public ZDICT_cover_params_t parameters;
public nuint compressedSize;
}
}

View File

@@ -0,0 +1,16 @@
using System;
namespace ZstdSharp.Unsafe
{
/**
* Struct used for the dictionary selection function.
*/
public unsafe partial struct COVER_dictSelection
{
public byte* dictContent;
public nuint dictSize;
public nuint totalCompressedSize;
}
}

View File

@@ -0,0 +1,14 @@
using System;
namespace ZstdSharp.Unsafe
{
/**
*Number of epochs and size of each epoch.
*/
public partial struct COVER_epoch_info_t
{
public uint num;
public uint size;
}
}

View File

@@ -0,0 +1,16 @@
using System;
namespace ZstdSharp.Unsafe
{
/**
* A segment is a range in the source as well as the score of the segment.
*/
public partial struct COVER_segment_t
{
public uint begin;
public uint end;
public uint score;
}
}

View File

@@ -0,0 +1,411 @@
using System;
using static ZstdSharp.UnsafeHelper;
namespace ZstdSharp.Unsafe
{
public static unsafe partial class Methods
{
public static int g_displayLevel = 2;
/**
* Returns the sum of the sample sizes.
*/
public static nuint COVER_sum(nuint* samplesSizes, uint nbSamples)
{
nuint sum = 0;
uint i;
for (i = 0; i < nbSamples; ++i)
{
sum += samplesSizes[i];
}
return sum;
}
/**
* Warns the user when their corpus is too small.
*/
public static void COVER_warnOnSmallCorpus(nuint maxDictSize, nuint nbDmers, int displayLevel)
{
double ratio = (double)(nbDmers) / maxDictSize;
if (ratio >= 10)
{
return;
}
if (displayLevel >= 1)
{
;
}
}
/**
* Computes the number of epochs and the size of each epoch.
* We will make sure that each epoch gets at least 10 * k bytes.
*
* The COVER algorithms divide the data up into epochs of equal size and
* select one segment from each epoch.
*
* @param maxDictSize The maximum allowed dictionary size.
* @param nbDmers The number of dmers we are training on.
* @param k The parameter k (segment size).
* @param passes The target number of passes over the dmer corpus.
* More passes means a better dictionary.
*/
public static COVER_epoch_info_t COVER_computeEpochs(uint maxDictSize, uint nbDmers, uint k, uint passes)
{
uint minEpochSize = k * 10;
COVER_epoch_info_t epochs;
epochs.num = (uint)((1) > (maxDictSize / k / passes) ? (1) : (maxDictSize / k / passes));
epochs.size = nbDmers / epochs.num;
if (epochs.size >= minEpochSize)
{
assert(epochs.size * epochs.num <= nbDmers);
return epochs;
}
epochs.size = ((minEpochSize) < (nbDmers) ? (minEpochSize) : (nbDmers));
epochs.num = nbDmers / epochs.size;
assert(epochs.size * epochs.num <= nbDmers);
return epochs;
}
/**
* Checks total compressed size of a dictionary
*/
public static nuint COVER_checkTotalCompressedSize(ZDICT_cover_params_t parameters, nuint* samplesSizes, byte* samples, nuint* offsets, nuint nbTrainSamples, nuint nbSamples, byte* dict, nuint dictBufferCapacity)
{
nuint totalCompressedSize = (unchecked((nuint)(-(int)ZSTD_ErrorCode.ZSTD_error_GENERIC)));
ZSTD_CCtx_s* cctx;
ZSTD_CDict_s* cdict;
void* dst;
nuint dstCapacity;
nuint i;
{
nuint maxSampleSize = 0;
i = parameters.splitPoint < 1.0 ? nbTrainSamples : 0;
for (; i < nbSamples; ++i)
{
maxSampleSize = ((samplesSizes[i]) > (maxSampleSize) ? (samplesSizes[i]) : (maxSampleSize));
}
dstCapacity = ZSTD_compressBound(maxSampleSize);
dst = malloc(dstCapacity);
}
cctx = ZSTD_createCCtx();
cdict = ZSTD_createCDict((void*)dict, dictBufferCapacity, parameters.zParams.compressionLevel);
if (dst == null || cctx == null || cdict == null)
{
goto _compressCleanup;
}
totalCompressedSize = dictBufferCapacity;
i = parameters.splitPoint < 1.0 ? nbTrainSamples : 0;
for (; i < nbSamples; ++i)
{
nuint size = ZSTD_compress_usingCDict(cctx, dst, dstCapacity, (void*)(samples + offsets[i]), samplesSizes[i], cdict);
if ((ERR_isError(size)) != 0)
{
totalCompressedSize = size;
goto _compressCleanup;
}
totalCompressedSize += size;
}
_compressCleanup:
ZSTD_freeCCtx(cctx);
ZSTD_freeCDict(cdict);
if (dst != null)
{
free(dst);
}
return totalCompressedSize;
}
/**
* Initialize the `COVER_best_t`.
*/
public static void COVER_best_init(COVER_best_s* best)
{
if (best == null)
{
return;
}
best->liveJobs = 0;
best->dict = null;
best->dictSize = 0;
best->compressedSize = unchecked((nuint)(-1));
memset((void*)&best->parameters, 0, (nuint)(sizeof(ZDICT_cover_params_t)));
}
/**
* Wait until liveJobs == 0.
*/
public static void COVER_best_wait(COVER_best_s* best)
{
if (best == null)
{
return;
}
while (best->liveJobs != 0)
{
}
}
/**
* Call COVER_best_wait() and then destroy the COVER_best_t.
*/
public static void COVER_best_destroy(COVER_best_s* best)
{
if (best == null)
{
return;
}
COVER_best_wait(best);
if (best->dict != null)
{
free(best->dict);
}
}
/**
* Called when a thread is about to be launched.
* Increments liveJobs.
*/
public static void COVER_best_start(COVER_best_s* best)
{
if (best == null)
{
return;
}
++best->liveJobs;
}
/**
* Called when a thread finishes executing, both on error or success.
* Decrements liveJobs and signals any waiting threads if liveJobs == 0.
* If this dictionary is the best so far save it and its parameters.
*/
public static void COVER_best_finish(COVER_best_s* best, ZDICT_cover_params_t parameters, COVER_dictSelection selection)
{
void* dict = (void*)selection.dictContent;
nuint compressedSize = selection.totalCompressedSize;
nuint dictSize = selection.dictSize;
if (best == null)
{
return;
}
{
nuint liveJobs;
--best->liveJobs;
liveJobs = best->liveJobs;
if (compressedSize < best->compressedSize)
{
if (best->dict == null || best->dictSize < dictSize)
{
if (best->dict != null)
{
free(best->dict);
}
best->dict = malloc(dictSize);
if (best->dict == null)
{
best->compressedSize = (unchecked((nuint)(-(int)ZSTD_ErrorCode.ZSTD_error_GENERIC)));
best->dictSize = 0;
return;
}
}
if (dict != null)
{
memcpy(best->dict, dict, dictSize);
best->dictSize = dictSize;
best->parameters = parameters;
best->compressedSize = compressedSize;
}
}
if (liveJobs == 0)
{
}
}
}
/**
* Error function for COVER_selectDict function. Returns a struct where
* return.totalCompressedSize is a ZSTD error.
*/
public static COVER_dictSelection COVER_dictSelectionError(nuint error)
{
COVER_dictSelection selection = new COVER_dictSelection
{
dictContent = null,
dictSize = 0,
totalCompressedSize = error,
};
return selection;
}
/**
* Error function for COVER_selectDict function. Checks if the return
* value is an error.
*/
public static uint COVER_dictSelectionIsError(COVER_dictSelection selection)
{
return ((((ERR_isError(selection.totalCompressedSize)) != 0 || selection.dictContent == null)) ? 1U : 0U);
}
/**
* Always call after selectDict is called to free up used memory from
* newly created dictionary.
*/
public static void COVER_dictSelectionFree(COVER_dictSelection selection)
{
free((void*)selection.dictContent);
}
/**
* Called to finalize the dictionary and select one based on whether or not
* the shrink-dict flag was enabled. If enabled the dictionary used is the
* smallest dictionary within a specified regression of the compressed size
* from the largest dictionary.
*/
public static COVER_dictSelection COVER_selectDict(byte* customDictContent, nuint dictBufferCapacity, nuint dictContentSize, byte* samplesBuffer, nuint* samplesSizes, uint nbFinalizeSamples, nuint nbCheckSamples, nuint nbSamples, ZDICT_cover_params_t @params, nuint* offsets, nuint totalCompressedSize)
{
nuint largestDict = 0;
nuint largestCompressed = 0;
byte* customDictContentEnd = customDictContent + dictContentSize;
byte* largestDictbuffer = (byte*)(malloc(dictBufferCapacity));
byte* candidateDictBuffer = (byte*)(malloc(dictBufferCapacity));
double regressionTolerance = ((double)(@params.shrinkDictMaxRegression) / 100.0) + 1.00;
if (largestDictbuffer == null || candidateDictBuffer == null)
{
free((void*)largestDictbuffer);
free((void*)candidateDictBuffer);
return COVER_dictSelectionError(dictContentSize);
}
memcpy((void*)largestDictbuffer, (void*)customDictContent, dictContentSize);
dictContentSize = ZDICT_finalizeDictionary((void*)largestDictbuffer, dictBufferCapacity, (void*)customDictContent, dictContentSize, (void*)samplesBuffer, samplesSizes, nbFinalizeSamples, @params.zParams);
if ((ZDICT_isError(dictContentSize)) != 0)
{
free((void*)largestDictbuffer);
free((void*)candidateDictBuffer);
return COVER_dictSelectionError(dictContentSize);
}
totalCompressedSize = COVER_checkTotalCompressedSize(@params, samplesSizes, samplesBuffer, offsets, nbCheckSamples, nbSamples, largestDictbuffer, dictContentSize);
if ((ERR_isError(totalCompressedSize)) != 0)
{
free((void*)largestDictbuffer);
free((void*)candidateDictBuffer);
return COVER_dictSelectionError(totalCompressedSize);
}
if (@params.shrinkDict == 0)
{
COVER_dictSelection selection = new COVER_dictSelection
{
dictContent = largestDictbuffer,
dictSize = dictContentSize,
totalCompressedSize = totalCompressedSize,
};
free((void*)candidateDictBuffer);
return selection;
}
largestDict = dictContentSize;
largestCompressed = totalCompressedSize;
dictContentSize = 256;
while (dictContentSize < largestDict)
{
memcpy((void*)candidateDictBuffer, (void*)largestDictbuffer, largestDict);
dictContentSize = ZDICT_finalizeDictionary((void*)candidateDictBuffer, dictBufferCapacity, (void*)(customDictContentEnd - dictContentSize), dictContentSize, (void*)samplesBuffer, samplesSizes, nbFinalizeSamples, @params.zParams);
if ((ZDICT_isError(dictContentSize)) != 0)
{
free((void*)largestDictbuffer);
free((void*)candidateDictBuffer);
return COVER_dictSelectionError(dictContentSize);
}
totalCompressedSize = COVER_checkTotalCompressedSize(@params, samplesSizes, samplesBuffer, offsets, nbCheckSamples, nbSamples, candidateDictBuffer, dictContentSize);
if ((ERR_isError(totalCompressedSize)) != 0)
{
free((void*)largestDictbuffer);
free((void*)candidateDictBuffer);
return COVER_dictSelectionError(totalCompressedSize);
}
if (totalCompressedSize <= largestCompressed * regressionTolerance)
{
COVER_dictSelection selection = new COVER_dictSelection
{
dictContent = candidateDictBuffer,
dictSize = dictContentSize,
totalCompressedSize = totalCompressedSize,
};
free((void*)largestDictbuffer);
return selection;
}
dictContentSize *= 2;
}
dictContentSize = largestDict;
totalCompressedSize = largestCompressed;
{
COVER_dictSelection selection = new COVER_dictSelection
{
dictContent = largestDictbuffer,
dictSize = dictContentSize,
totalCompressedSize = totalCompressedSize,
};
free((void*)candidateDictBuffer);
return selection;
}
}
}
}

View File

@@ -0,0 +1,18 @@
using System;
namespace ZstdSharp.Unsafe
{
/*-***************************/
/* generic DTableDesc */
/*-***************************/
public partial struct DTableDesc
{
public byte maxTableLog;
public byte tableType;
public byte tableLog;
public byte reserved;
}
}

View File

@@ -0,0 +1,16 @@
using System;
namespace ZstdSharp.Unsafe
{
public unsafe partial struct EStats_ress_t
{
/* dictionary */
public ZSTD_CDict_s* dict;
/* working context */
public ZSTD_CCtx_s* zc;
/* must be ZSTD_BLOCKSIZE_MAX allocated */
public void* workPlace;
}
}

View File

@@ -0,0 +1,425 @@
using System;
using System.Numerics;
using System.Runtime.CompilerServices;
using static ZstdSharp.UnsafeHelper;
namespace ZstdSharp.Unsafe
{
public static unsafe partial class Methods
{
/*=== Version ===*/
public static uint FSE_versionNumber()
{
return (uint)((0 * 100 * 100 + 9 * 100 + 0));
}
/*=== Error Management ===*/
public static uint FSE_isError(nuint code)
{
return ERR_isError(code);
}
public static string FSE_getErrorName(nuint code)
{
return ERR_getErrorName(code);
}
/* Error Management */
public static uint HUF_isError(nuint code)
{
return ERR_isError(code);
}
public static string HUF_getErrorName(nuint code)
{
return ERR_getErrorName(code);
}
/*-**************************************************************
* FSE NCount encoding-decoding
****************************************************************/
[InlineMethod.Inline]
private static uint FSE_ctz(uint val)
{
assert(val != 0);
{
return (uint) BitOperations.TrailingZeroCount(val);
}
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
private static nuint FSE_readNCount_body(short* normalizedCounter, uint* maxSVPtr, uint* tableLogPtr, void* headerBuffer, nuint hbSize)
{
byte* istart = (byte*)(headerBuffer);
byte* iend = istart + hbSize;
byte* ip = istart;
int nbBits;
int remaining;
int threshold;
uint bitStream;
int bitCount;
uint charnum = 0;
uint maxSV1 = *maxSVPtr + 1;
int previous0 = 0;
if (hbSize < 8)
{
sbyte* buffer = stackalloc sbyte[8];
memset(buffer, 0, sizeof(sbyte) * 8);
memcpy((void*)(buffer), (headerBuffer), (hbSize));
{
nuint countSize = FSE_readNCount(normalizedCounter, maxSVPtr, tableLogPtr, (void*)buffer, (nuint)(8));
if ((FSE_isError(countSize)) != 0)
{
return countSize;
}
if (countSize > hbSize)
{
return (unchecked((nuint)(-(int)ZSTD_ErrorCode.ZSTD_error_corruption_detected)));
}
return countSize;
}
}
assert(hbSize >= 8);
memset((void*)(normalizedCounter), (0), ((*maxSVPtr + 1) * (nuint)(sizeof(short))));
bitStream = MEM_readLE32((void*)ip);
nbBits = (int)((bitStream & 0xF) + 5);
if (nbBits > 15)
{
return (unchecked((nuint)(-(int)ZSTD_ErrorCode.ZSTD_error_tableLog_tooLarge)));
}
bitStream >>= 4;
bitCount = 4;
*tableLogPtr = (uint)nbBits;
remaining = (1 << nbBits) + 1;
threshold = 1 << nbBits;
nbBits++;
for (;;)
{
if (previous0 != 0)
{
int repeats = (int)(FSE_ctz(~bitStream | 0x80000000) >> 1);
while (repeats >= 12)
{
charnum += (uint)(3 * 12);
if ((ip <= iend - 7))
{
ip += 3;
}
else
{
bitCount -= (int)(8 * (iend - 7 - ip));
bitCount &= 31;
ip = iend - 4;
}
bitStream = MEM_readLE32((void*)ip) >> bitCount;
repeats = (int)(FSE_ctz(~bitStream | 0x80000000) >> 1);
}
charnum += (uint)(3 * repeats);
bitStream >>= 2 * repeats;
bitCount += 2 * repeats;
assert((bitStream & 3) < 3);
charnum += bitStream & 3;
bitCount += 2;
if (charnum >= maxSV1)
{
break;
}
if ((ip <= iend - 7) || (ip + (bitCount >> 3) <= iend - 4))
{
assert((bitCount >> 3) <= 3);
ip += bitCount >> 3;
bitCount &= 7;
}
else
{
bitCount -= (int)(8 * (iend - 4 - ip));
bitCount &= 31;
ip = iend - 4;
}
bitStream = MEM_readLE32((void*)ip) >> bitCount;
}
{
int max = (2 * threshold - 1) - remaining;
int count;
if ((bitStream & (uint)((threshold - 1))) < (uint)(max))
{
count = (int)(bitStream & (uint)((threshold - 1)));
bitCount += nbBits - 1;
}
else
{
count = (int)(bitStream & (uint)((2 * threshold - 1)));
if (count >= threshold)
{
count -= max;
}
bitCount += nbBits;
}
count--;
if (count >= 0)
{
remaining -= count;
}
else
{
assert(count == -1);
remaining += count;
}
normalizedCounter[charnum++] = (short)(count);
previous0 = (count == 0 ? 1 : 0);
assert(threshold > 1);
if (remaining < threshold)
{
if (remaining <= 1)
{
break;
}
nbBits = (int)(BIT_highbit32((uint)remaining) + 1);
threshold = 1 << (nbBits - 1);
}
if (charnum >= maxSV1)
{
break;
}
if ((ip <= iend - 7) || (ip + (bitCount >> 3) <= iend - 4))
{
ip += bitCount >> 3;
bitCount &= 7;
}
else
{
bitCount -= (int)(8 * (iend - 4 - ip));
bitCount &= 31;
ip = iend - 4;
}
bitStream = MEM_readLE32((void*)ip) >> bitCount;
}
}
if (remaining != 1)
{
return (unchecked((nuint)(-(int)ZSTD_ErrorCode.ZSTD_error_corruption_detected)));
}
if (charnum > maxSV1)
{
return (unchecked((nuint)(-(int)ZSTD_ErrorCode.ZSTD_error_maxSymbolValue_tooSmall)));
}
if (bitCount > 32)
{
return (unchecked((nuint)(-(int)ZSTD_ErrorCode.ZSTD_error_corruption_detected)));
}
*maxSVPtr = charnum - 1;
ip += (bitCount + 7) >> 3;
return (nuint)(ip - istart);
}
/* Avoids the FORCE_INLINE of the _body() function. */
private static nuint FSE_readNCount_body_default(short* normalizedCounter, uint* maxSVPtr, uint* tableLogPtr, void* headerBuffer, nuint hbSize)
{
return FSE_readNCount_body(normalizedCounter, maxSVPtr, tableLogPtr, headerBuffer, hbSize);
}
private static nuint FSE_readNCount_body_bmi2(short* normalizedCounter, uint* maxSVPtr, uint* tableLogPtr, void* headerBuffer, nuint hbSize)
{
return FSE_readNCount_body(normalizedCounter, maxSVPtr, tableLogPtr, headerBuffer, hbSize);
}
/*! FSE_readNCount_bmi2():
* Same as FSE_readNCount() but pass bmi2=1 when your CPU supports BMI2 and 0 otherwise.
*/
public static nuint FSE_readNCount_bmi2(short* normalizedCounter, uint* maxSVPtr, uint* tableLogPtr, void* headerBuffer, nuint hbSize, int bmi2)
{
if (bmi2 != 0)
{
return FSE_readNCount_body_bmi2(normalizedCounter, maxSVPtr, tableLogPtr, headerBuffer, hbSize);
}
return FSE_readNCount_body_default(normalizedCounter, maxSVPtr, tableLogPtr, headerBuffer, hbSize);
}
/*! FSE_readNCount():
Read compactly saved 'normalizedCounter' from 'rBuffer'.
@return : size read from 'rBuffer',
or an errorCode, which can be tested using FSE_isError().
maxSymbolValuePtr[0] and tableLogPtr[0] will also be updated with their respective values */
public static nuint FSE_readNCount(short* normalizedCounter, uint* maxSVPtr, uint* tableLogPtr, void* headerBuffer, nuint hbSize)
{
return FSE_readNCount_bmi2(normalizedCounter, maxSVPtr, tableLogPtr, headerBuffer, hbSize, 0);
}
/*! HUF_readStats() :
Read compact Huffman tree, saved by HUF_writeCTable().
`huffWeight` is destination buffer.
`rankStats` is assumed to be a table of at least HUF_TABLELOG_MAX U32.
@return : size read from `src` , or an error Code .
Note : Needed by HUF_readCTable() and HUF_readDTableX?() .
*/
public static nuint HUF_readStats(byte* huffWeight, nuint hwSize, uint* rankStats, uint* nbSymbolsPtr, uint* tableLogPtr, void* src, nuint srcSize)
{
uint* wksp = stackalloc uint[218];
return HUF_readStats_wksp(huffWeight, hwSize, rankStats, nbSymbolsPtr, tableLogPtr, src, srcSize, (void*)wksp, (nuint)(sizeof(uint) * 218), 0);
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
private static nuint HUF_readStats_body(byte* huffWeight, nuint hwSize, uint* rankStats, uint* nbSymbolsPtr, uint* tableLogPtr, void* src, nuint srcSize, void* workSpace, nuint wkspSize, int bmi2)
{
uint weightTotal;
byte* ip = (byte*)(src);
nuint iSize;
nuint oSize;
if (srcSize == 0)
{
return (unchecked((nuint)(-(int)ZSTD_ErrorCode.ZSTD_error_srcSize_wrong)));
}
iSize = ip[0];
if (iSize >= 128)
{
oSize = iSize - 127;
iSize = ((oSize + 1) / 2);
if (iSize + 1 > srcSize)
{
return (unchecked((nuint)(-(int)ZSTD_ErrorCode.ZSTD_error_srcSize_wrong)));
}
if (oSize >= hwSize)
{
return (unchecked((nuint)(-(int)ZSTD_ErrorCode.ZSTD_error_corruption_detected)));
}
ip += 1;
{
uint n;
for (n = 0; n < oSize; n += 2)
{
huffWeight[n] = (byte)(ip[n / 2] >> 4);
huffWeight[n + 1] = (byte)(ip[n / 2] & 15);
}
}
}
else
{
if (iSize + 1 > srcSize)
{
return (unchecked((nuint)(-(int)ZSTD_ErrorCode.ZSTD_error_srcSize_wrong)));
}
oSize = FSE_decompress_wksp_bmi2((void*)huffWeight, hwSize - 1, (void*)(ip + 1), iSize, 6, workSpace, wkspSize, bmi2);
if ((FSE_isError(oSize)) != 0)
{
return oSize;
}
}
memset((void*)(rankStats), (0), ((uint)((12 + 1)) * (nuint)(sizeof(uint))));
weightTotal = 0;
{
uint n;
for (n = 0; n < oSize; n++)
{
if (huffWeight[n] >= 12)
{
return (unchecked((nuint)(-(int)ZSTD_ErrorCode.ZSTD_error_corruption_detected)));
}
rankStats[huffWeight[n]]++;
weightTotal += (uint)((1 << (int)(huffWeight[n])) >> 1);
}
}
if (weightTotal == 0)
{
return (unchecked((nuint)(-(int)ZSTD_ErrorCode.ZSTD_error_corruption_detected)));
}
{
uint tableLog = BIT_highbit32(weightTotal) + 1;
if (tableLog > 12)
{
return (unchecked((nuint)(-(int)ZSTD_ErrorCode.ZSTD_error_corruption_detected)));
}
*tableLogPtr = tableLog;
{
uint total = (uint)(1 << (int)tableLog);
uint rest = total - weightTotal;
uint verif = (uint)(1 << (int)(BIT_highbit32(rest)));
uint lastWeight = BIT_highbit32(rest) + 1;
if (verif != rest)
{
return (unchecked((nuint)(-(int)ZSTD_ErrorCode.ZSTD_error_corruption_detected)));
}
huffWeight[oSize] = (byte)(lastWeight);
rankStats[lastWeight]++;
}
}
if ((rankStats[1] < 2) || (rankStats[1] & 1) != 0)
{
return (unchecked((nuint)(-(int)ZSTD_ErrorCode.ZSTD_error_corruption_detected)));
}
*nbSymbolsPtr = (uint)(oSize + 1);
return iSize + 1;
}
/* Avoids the FORCE_INLINE of the _body() function. */
private static nuint HUF_readStats_body_default(byte* huffWeight, nuint hwSize, uint* rankStats, uint* nbSymbolsPtr, uint* tableLogPtr, void* src, nuint srcSize, void* workSpace, nuint wkspSize)
{
return HUF_readStats_body(huffWeight, hwSize, rankStats, nbSymbolsPtr, tableLogPtr, src, srcSize, workSpace, wkspSize, 0);
}
private static nuint HUF_readStats_body_bmi2(byte* huffWeight, nuint hwSize, uint* rankStats, uint* nbSymbolsPtr, uint* tableLogPtr, void* src, nuint srcSize, void* workSpace, nuint wkspSize)
{
return HUF_readStats_body(huffWeight, hwSize, rankStats, nbSymbolsPtr, tableLogPtr, src, srcSize, workSpace, wkspSize, 1);
}
public static nuint HUF_readStats_wksp(byte* huffWeight, nuint hwSize, uint* rankStats, uint* nbSymbolsPtr, uint* tableLogPtr, void* src, nuint srcSize, void* workSpace, nuint wkspSize, int bmi2)
{
if (bmi2 != 0)
{
return HUF_readStats_body_bmi2(huffWeight, hwSize, rankStats, nbSymbolsPtr, tableLogPtr, src, srcSize, workSpace, wkspSize);
}
return HUF_readStats_body_default(huffWeight, hwSize, rankStats, nbSymbolsPtr, tableLogPtr, src, srcSize, workSpace, wkspSize);
}
}
}

View File

@@ -0,0 +1,184 @@
using System;
using System.Runtime.CompilerServices;
using static ZstdSharp.UnsafeHelper;
namespace ZstdSharp.Unsafe
{
public static unsafe partial class Methods
{
[MethodImpl(MethodImplOptions.AggressiveInlining)]
private static uint ERR_isError(nuint code)
{
return (((code > (unchecked((nuint)(-(int)ZSTD_ErrorCode.ZSTD_error_maxCode))))) ? 1U : 0U);
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
private static ZSTD_ErrorCode ERR_getErrorCode(nuint code)
{
if ((ERR_isError(code)) == 0)
{
return (ZSTD_ErrorCode)(0);
}
return (ZSTD_ErrorCode)(0 - code);
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
private static string ERR_getErrorName(nuint code)
{
return ERR_getErrorString(ERR_getErrorCode(code));
}
/*-****************************************
* Error Strings
******************************************/
public static string ERR_getErrorString(ZSTD_ErrorCode code)
{
var notErrorCode = "Unspecified error code";
switch (code)
{
case ZSTD_ErrorCode.ZSTD_error_no_error:
{
return "No error detected";
}
case ZSTD_ErrorCode.ZSTD_error_GENERIC:
{
return "Error (generic)";
}
case ZSTD_ErrorCode.ZSTD_error_prefix_unknown:
{
return "Unknown frame descriptor";
}
case ZSTD_ErrorCode.ZSTD_error_version_unsupported:
{
return "Version not supported";
}
case ZSTD_ErrorCode.ZSTD_error_frameParameter_unsupported:
{
return "Unsupported frame parameter";
}
case ZSTD_ErrorCode.ZSTD_error_frameParameter_windowTooLarge:
{
return "Frame requires too much memory for decoding";
}
case ZSTD_ErrorCode.ZSTD_error_corruption_detected:
{
return "Corrupted block detected";
}
case ZSTD_ErrorCode.ZSTD_error_checksum_wrong:
{
return "Restored data doesn't match checksum";
}
case ZSTD_ErrorCode.ZSTD_error_parameter_unsupported:
{
return "Unsupported parameter";
}
case ZSTD_ErrorCode.ZSTD_error_parameter_outOfBound:
{
return "Parameter is out of bound";
}
case ZSTD_ErrorCode.ZSTD_error_init_missing:
{
return "Context should be init first";
}
case ZSTD_ErrorCode.ZSTD_error_memory_allocation:
{
return "Allocation error : not enough memory";
}
case ZSTD_ErrorCode.ZSTD_error_workSpace_tooSmall:
{
return "workSpace buffer is not large enough";
}
case ZSTD_ErrorCode.ZSTD_error_stage_wrong:
{
return "Operation not authorized at current processing stage";
}
case ZSTD_ErrorCode.ZSTD_error_tableLog_tooLarge:
{
return "tableLog requires too much memory : unsupported";
}
case ZSTD_ErrorCode.ZSTD_error_maxSymbolValue_tooLarge:
{
return "Unsupported max Symbol Value : too large";
}
case ZSTD_ErrorCode.ZSTD_error_maxSymbolValue_tooSmall:
{
return "Specified maxSymbolValue is too small";
}
case ZSTD_ErrorCode.ZSTD_error_dictionary_corrupted:
{
return "Dictionary is corrupted";
}
case ZSTD_ErrorCode.ZSTD_error_dictionary_wrong:
{
return "Dictionary mismatch";
}
case ZSTD_ErrorCode.ZSTD_error_dictionaryCreation_failed:
{
return "Cannot create Dictionary from provided samples";
}
case ZSTD_ErrorCode.ZSTD_error_dstSize_tooSmall:
{
return "Destination buffer is too small";
}
case ZSTD_ErrorCode.ZSTD_error_srcSize_wrong:
{
return "Src size is incorrect";
}
case ZSTD_ErrorCode.ZSTD_error_dstBuffer_null:
{
return "Operation on NULL destination buffer";
}
case ZSTD_ErrorCode.ZSTD_error_frameIndex_tooLarge:
{
return "Frame index is too large";
}
case ZSTD_ErrorCode.ZSTD_error_seekableIO:
{
return "An I/O error occurred when reading/seeking";
}
case ZSTD_ErrorCode.ZSTD_error_dstBuffer_wrong:
{
return "Destination buffer is wrong";
}
case ZSTD_ErrorCode.ZSTD_error_srcBuffer_wrong:
{
return "Source buffer is wrong";
}
case ZSTD_ErrorCode.ZSTD_error_maxCode:
default:
{
return notErrorCode;
}
}
}
}
}

View File

@@ -0,0 +1,16 @@
using System;
namespace ZstdSharp.Unsafe
{
/*-*************************************
* Acceleration
***************************************/
public partial struct FASTCOVER_accel_t
{
/* Percentage of training samples used for ZDICT_finalizeDictionary */
public uint finalize;
/* Number of dmer skipped between each dmer counted in computeFrequency */
public uint skip;
}
}

View File

@@ -0,0 +1,32 @@
using System;
namespace ZstdSharp.Unsafe
{
/*-*************************************
* Context
***************************************/
public unsafe partial struct FASTCOVER_ctx_t
{
public byte* samples;
public nuint* offsets;
public nuint* samplesSizes;
public nuint nbSamples;
public nuint nbTrainSamples;
public nuint nbTestSamples;
public nuint nbDmers;
public uint* freqs;
public uint d;
public uint f;
public FASTCOVER_accel_t accelParams;
}
}

View File

@@ -0,0 +1,18 @@
using System;
namespace ZstdSharp.Unsafe
{
/**
* Parameters for FASTCOVER_tryParameters().
*/
public unsafe partial struct FASTCOVER_tryParameters_data_s
{
public FASTCOVER_ctx_t* ctx;
public COVER_best_s* best;
public nuint dictBufferCapacity;
public ZDICT_cover_params_t parameters;
}
}

View File

@@ -0,0 +1,22 @@
using System;
namespace ZstdSharp.Unsafe
{
/* *****************************************
* FSE symbol compression API
*******************************************/
/*!
This API consists of small unitary functions, which highly benefit from being inlined.
Hence their body are included in next section.
*/
public unsafe partial struct FSE_CState_t
{
public nint value;
public void* stateTable;
public void* symbolTT;
public uint stateLog;
}
}

View File

@@ -0,0 +1,15 @@
using System;
namespace ZstdSharp.Unsafe
{
/* *****************************************
* FSE symbol decompression API
*******************************************/
public unsafe partial struct FSE_DState_t
{
public nuint state;
/* precise table may vary, depending on U16 */
public void* table;
}
}

View File

@@ -0,0 +1,12 @@
using System;
namespace ZstdSharp.Unsafe
{
/* ====== Decompression ====== */
public partial struct FSE_DTableHeader
{
public ushort tableLog;
public ushort fastMode;
}
}

View File

@@ -0,0 +1,12 @@
using System;
namespace ZstdSharp.Unsafe
{
public unsafe partial struct FSE_DecompressWksp
{
public fixed short ncount[256];
/* Dynamically sized */
public fixed uint dtable[1];
}
}

View File

@@ -0,0 +1,13 @@
using System;
namespace ZstdSharp.Unsafe
{
public partial struct FSE_decode_t
{
public ushort newState;
public byte symbol;
public byte nbBits;
}
}

View File

@@ -0,0 +1,11 @@
using System;
namespace ZstdSharp.Unsafe
{
public enum FSE_repeat
{
FSE_repeat_none,
FSE_repeat_check,
FSE_repeat_valid,
}
}

View File

@@ -0,0 +1,14 @@
using System;
namespace ZstdSharp.Unsafe
{
/* *****************************************
* Implementation of inlined functions
*******************************************/
public partial struct FSE_symbolCompressionTransform
{
public int deltaFindState;
public uint deltaNbBits;
}
}

View File

@@ -0,0 +1,715 @@
using System;
using static ZstdSharp.UnsafeHelper;
namespace ZstdSharp.Unsafe
{
public static unsafe partial class Methods
{
/*-*************************************
* Hash Functions
***************************************/
/**
* Hash the d-byte value pointed to by p and mod 2^f into the frequency vector
*/
private static nuint FASTCOVER_hashPtrToIndex(void* p, uint f, uint d)
{
if (d == 6)
{
return ZSTD_hash6Ptr(p, f);
}
return ZSTD_hash8Ptr(p, f);
}
public static FASTCOVER_accel_t* FASTCOVER_defaultAccelParameters = GetArrayPointer(new FASTCOVER_accel_t[11]
{
new FASTCOVER_accel_t
{
finalize = 100,
skip = 0,
},
new FASTCOVER_accel_t
{
finalize = 100,
skip = 0,
},
new FASTCOVER_accel_t
{
finalize = 50,
skip = 1,
},
new FASTCOVER_accel_t
{
finalize = 34,
skip = 2,
},
new FASTCOVER_accel_t
{
finalize = 25,
skip = 3,
},
new FASTCOVER_accel_t
{
finalize = 20,
skip = 4,
},
new FASTCOVER_accel_t
{
finalize = 17,
skip = 5,
},
new FASTCOVER_accel_t
{
finalize = 14,
skip = 6,
},
new FASTCOVER_accel_t
{
finalize = 13,
skip = 7,
},
new FASTCOVER_accel_t
{
finalize = 11,
skip = 8,
},
new FASTCOVER_accel_t
{
finalize = 10,
skip = 9,
},
});
/*-*************************************
* Helper functions
***************************************/
/**
* Selects the best segment in an epoch.
* Segments of are scored according to the function:
*
* Let F(d) be the frequency of all dmers with hash value d.
* Let S_i be hash value of the dmer at position i of segment S which has length k.
*
* Score(S) = F(S_1) + F(S_2) + ... + F(S_{k-d+1})
*
* Once the dmer with hash value d is in the dictionary we set F(d) = 0.
*/
private static COVER_segment_t FASTCOVER_selectSegment(FASTCOVER_ctx_t* ctx, uint* freqs, uint begin, uint end, ZDICT_cover_params_t parameters, ushort* segmentFreqs)
{
uint k = parameters.k;
uint d = parameters.d;
uint f = ctx->f;
uint dmersInK = k - d + 1;
COVER_segment_t bestSegment = new COVER_segment_t
{
begin = 0,
end = 0,
score = 0,
};
COVER_segment_t activeSegment;
activeSegment.begin = begin;
activeSegment.end = begin;
activeSegment.score = 0;
while (activeSegment.end < end)
{
nuint idx = FASTCOVER_hashPtrToIndex((void*)(ctx->samples + activeSegment.end), f, d);
if (segmentFreqs[idx] == 0)
{
activeSegment.score += freqs[idx];
}
activeSegment.end += 1;
segmentFreqs[idx] += 1;
if (activeSegment.end - activeSegment.begin == dmersInK + 1)
{
nuint delIndex = FASTCOVER_hashPtrToIndex((void*)(ctx->samples + activeSegment.begin), f, d);
segmentFreqs[delIndex] -= 1;
if (segmentFreqs[delIndex] == 0)
{
activeSegment.score -= freqs[delIndex];
}
activeSegment.begin += 1;
}
if (activeSegment.score > bestSegment.score)
{
bestSegment = activeSegment;
}
}
while (activeSegment.begin < end)
{
nuint delIndex = FASTCOVER_hashPtrToIndex((void*)(ctx->samples + activeSegment.begin), f, d);
segmentFreqs[delIndex] -= 1;
activeSegment.begin += 1;
}
{
uint pos;
for (pos = bestSegment.begin; pos != bestSegment.end; ++pos)
{
nuint i = FASTCOVER_hashPtrToIndex((void*)(ctx->samples + pos), f, d);
freqs[i] = 0;
}
}
return bestSegment;
}
private static int FASTCOVER_checkParameters(ZDICT_cover_params_t parameters, nuint maxDictSize, uint f, uint accel)
{
if (parameters.d == 0 || parameters.k == 0)
{
return 0;
}
if (parameters.d != 6 && parameters.d != 8)
{
return 0;
}
if (parameters.k > maxDictSize)
{
return 0;
}
if (parameters.d > parameters.k)
{
return 0;
}
if (f > 31 || f == 0)
{
return 0;
}
if (parameters.splitPoint <= 0 || parameters.splitPoint > 1)
{
return 0;
}
if (accel > 10 || accel == 0)
{
return 0;
}
return 1;
}
/**
* Clean up a context initialized with `FASTCOVER_ctx_init()`.
*/
private static void FASTCOVER_ctx_destroy(FASTCOVER_ctx_t* ctx)
{
if (ctx == null)
{
return;
}
free((void*)ctx->freqs);
ctx->freqs = null;
free((void*)ctx->offsets);
ctx->offsets = null;
}
/**
* Calculate for frequency of hash value of each dmer in ctx->samples
*/
private static void FASTCOVER_computeFrequency(uint* freqs, FASTCOVER_ctx_t* ctx)
{
uint f = ctx->f;
uint d = ctx->d;
uint skip = ctx->accelParams.skip;
uint readLength = ((d) > (8) ? (d) : (8));
nuint i;
assert(ctx->nbTrainSamples >= 5);
assert(ctx->nbTrainSamples <= ctx->nbSamples);
for (i = 0; i < ctx->nbTrainSamples; i++)
{
nuint start = ctx->offsets[i];
nuint currSampleEnd = ctx->offsets[i + 1];
while (start + readLength <= currSampleEnd)
{
nuint dmerIndex = FASTCOVER_hashPtrToIndex((void*)(ctx->samples + start), f, d);
freqs[dmerIndex]++;
start = start + skip + 1;
}
}
}
/**
* Prepare a context for dictionary building.
* The context is only dependent on the parameter `d` and can used multiple
* times.
* Returns 0 on success or error code on error.
* The context must be destroyed with `FASTCOVER_ctx_destroy()`.
*/
private static nuint FASTCOVER_ctx_init(FASTCOVER_ctx_t* ctx, void* samplesBuffer, nuint* samplesSizes, uint nbSamples, uint d, double splitPoint, uint f, FASTCOVER_accel_t accelParams)
{
byte* samples = (byte*)(samplesBuffer);
nuint totalSamplesSize = COVER_sum(samplesSizes, nbSamples);
uint nbTrainSamples = splitPoint < 1.0 ? (uint)((double)(nbSamples) * splitPoint) : nbSamples;
uint nbTestSamples = splitPoint < 1.0 ? nbSamples - nbTrainSamples : nbSamples;
nuint trainingSamplesSize = splitPoint < 1.0 ? COVER_sum(samplesSizes, nbTrainSamples) : totalSamplesSize;
nuint testSamplesSize = splitPoint < 1.0 ? COVER_sum(samplesSizes + nbTrainSamples, nbTestSamples) : totalSamplesSize;
if (totalSamplesSize < ((d) > ((nuint)(sizeof(ulong))) ? (d) : ((nuint)(sizeof(ulong)))) || totalSamplesSize >= (nuint)((nuint)(sizeof(nuint)) == 8 ? (unchecked((uint)(-1))) : ((uint)(1) * (1U << 30))))
{
return (unchecked((nuint)(-(int)ZSTD_ErrorCode.ZSTD_error_srcSize_wrong)));
}
if (nbTrainSamples < 5)
{
return (unchecked((nuint)(-(int)ZSTD_ErrorCode.ZSTD_error_srcSize_wrong)));
}
if (nbTestSamples < 1)
{
return (unchecked((nuint)(-(int)ZSTD_ErrorCode.ZSTD_error_srcSize_wrong)));
}
memset((void*)ctx, 0, (nuint)(sizeof(FASTCOVER_ctx_t)));
ctx->samples = samples;
ctx->samplesSizes = samplesSizes;
ctx->nbSamples = nbSamples;
ctx->nbTrainSamples = nbTrainSamples;
ctx->nbTestSamples = nbTestSamples;
ctx->nbDmers = trainingSamplesSize - ((d) > ((nuint)(sizeof(ulong))) ? (d) : ((nuint)(sizeof(ulong)))) + 1;
ctx->d = d;
ctx->f = f;
ctx->accelParams = accelParams;
ctx->offsets = (nuint*)(calloc((nbSamples + 1), (nuint)(sizeof(nuint))));
if (ctx->offsets == null)
{
FASTCOVER_ctx_destroy(ctx);
return (unchecked((nuint)(-(int)ZSTD_ErrorCode.ZSTD_error_memory_allocation)));
}
{
uint i;
ctx->offsets[0] = 0;
assert(nbSamples >= 5);
for (i = 1; i <= nbSamples; ++i)
{
ctx->offsets[i] = ctx->offsets[i - 1] + samplesSizes[i - 1];
}
}
ctx->freqs = (uint*)(calloc((nuint)((ulong)(1) << (int)f), (nuint)(sizeof(uint))));
if (ctx->freqs == null)
{
FASTCOVER_ctx_destroy(ctx);
return (unchecked((nuint)(-(int)ZSTD_ErrorCode.ZSTD_error_memory_allocation)));
}
FASTCOVER_computeFrequency(ctx->freqs, ctx);
return 0;
}
/**
* Given the prepared context build the dictionary.
*/
private static nuint FASTCOVER_buildDictionary(FASTCOVER_ctx_t* ctx, uint* freqs, void* dictBuffer, nuint dictBufferCapacity, ZDICT_cover_params_t parameters, ushort* segmentFreqs)
{
byte* dict = (byte*)(dictBuffer);
nuint tail = dictBufferCapacity;
COVER_epoch_info_t epochs = COVER_computeEpochs((uint)(dictBufferCapacity), (uint)(ctx->nbDmers), parameters.k, 1);
nuint maxZeroScoreRun = 10;
nuint zeroScoreRun = 0;
nuint epoch;
for (epoch = 0; tail > 0; epoch = (nuint)((epoch + 1) % epochs.num))
{
uint epochBegin = (uint)(epoch * epochs.size);
uint epochEnd = epochBegin + epochs.size;
nuint segmentSize;
COVER_segment_t segment = FASTCOVER_selectSegment(ctx, freqs, epochBegin, epochEnd, parameters, segmentFreqs);
if (segment.score == 0)
{
if (++zeroScoreRun >= maxZeroScoreRun)
{
break;
}
continue;
}
zeroScoreRun = 0;
segmentSize = ((segment.end - segment.begin + parameters.d - 1) < (tail) ? (segment.end - segment.begin + parameters.d - 1) : (tail));
if (segmentSize < parameters.d)
{
break;
}
tail -= segmentSize;
memcpy((void*)(dict + tail), (void*)(ctx->samples + segment.begin), segmentSize);
}
return tail;
}
/**
* Tries a set of parameters and updates the COVER_best_t with the results.
* This function is thread safe if zstd is compiled with multithreaded support.
* It takes its parameters as an *OWNING* opaque pointer to support threading.
*/
private static void FASTCOVER_tryParameters(void* opaque)
{
FASTCOVER_tryParameters_data_s* data = (FASTCOVER_tryParameters_data_s*)(opaque);
FASTCOVER_ctx_t* ctx = data->ctx;
ZDICT_cover_params_t parameters = data->parameters;
nuint dictBufferCapacity = data->dictBufferCapacity;
nuint totalCompressedSize = (unchecked((nuint)(-(int)ZSTD_ErrorCode.ZSTD_error_GENERIC)));
ushort* segmentFreqs = (ushort*)(calloc((nuint)((ulong)(1) << (int)ctx->f), (nuint)(2)));
byte* dict = (byte*)(malloc(dictBufferCapacity));
COVER_dictSelection selection = COVER_dictSelectionError((unchecked((nuint)(-(int)ZSTD_ErrorCode.ZSTD_error_GENERIC))));
uint* freqs = (uint*)(malloc((nuint)(((ulong)(1) << (int)ctx->f) * (nuint)(4))));
if (segmentFreqs == null || dict == null || freqs == null)
{
goto _cleanup;
}
memcpy((void*)freqs, (void*)ctx->freqs, (nuint)(((ulong)(1) << (int)ctx->f) * (nuint)(sizeof(uint))));
{
nuint tail = FASTCOVER_buildDictionary(ctx, freqs, (void*)dict, dictBufferCapacity, parameters, segmentFreqs);
uint nbFinalizeSamples = (uint)(ctx->nbTrainSamples * ctx->accelParams.finalize / 100);
selection = COVER_selectDict(dict + tail, dictBufferCapacity, dictBufferCapacity - tail, ctx->samples, ctx->samplesSizes, nbFinalizeSamples, ctx->nbTrainSamples, ctx->nbSamples, parameters, ctx->offsets, totalCompressedSize);
if ((COVER_dictSelectionIsError(selection)) != 0)
{
goto _cleanup;
}
}
_cleanup:
free((void*)dict);
COVER_best_finish(data->best, parameters, selection);
free((void*)data);
free((void*)segmentFreqs);
COVER_dictSelectionFree(selection);
free((void*)freqs);
}
private static void FASTCOVER_convertToCoverParams(ZDICT_fastCover_params_t fastCoverParams, ZDICT_cover_params_t* coverParams)
{
coverParams->k = fastCoverParams.k;
coverParams->d = fastCoverParams.d;
coverParams->steps = fastCoverParams.steps;
coverParams->nbThreads = fastCoverParams.nbThreads;
coverParams->splitPoint = fastCoverParams.splitPoint;
coverParams->zParams = fastCoverParams.zParams;
coverParams->shrinkDict = fastCoverParams.shrinkDict;
}
private static void FASTCOVER_convertToFastCoverParams(ZDICT_cover_params_t coverParams, ZDICT_fastCover_params_t* fastCoverParams, uint f, uint accel)
{
fastCoverParams->k = coverParams.k;
fastCoverParams->d = coverParams.d;
fastCoverParams->steps = coverParams.steps;
fastCoverParams->nbThreads = coverParams.nbThreads;
fastCoverParams->splitPoint = coverParams.splitPoint;
fastCoverParams->f = f;
fastCoverParams->accel = accel;
fastCoverParams->zParams = coverParams.zParams;
fastCoverParams->shrinkDict = coverParams.shrinkDict;
}
/*! ZDICT_trainFromBuffer_fastCover():
* Train a dictionary from an array of samples using a modified version of COVER algorithm.
* Samples must be stored concatenated in a single flat buffer `samplesBuffer`,
* supplied with an array of sizes `samplesSizes`, providing the size of each sample, in order.
* d and k are required.
* All other parameters are optional, will use default values if not provided
* The resulting dictionary will be saved into `dictBuffer`.
* @return: size of dictionary stored into `dictBuffer` (<= `dictBufferCapacity`)
* or an error code, which can be tested with ZDICT_isError().
* See ZDICT_trainFromBuffer() for details on failure modes.
* Note: ZDICT_trainFromBuffer_fastCover() requires 6 * 2^f bytes of memory.
* Tips: In general, a reasonable dictionary has a size of ~ 100 KB.
* It's possible to select smaller or larger size, just by specifying `dictBufferCapacity`.
* In general, it's recommended to provide a few thousands samples, though this can vary a lot.
* It's recommended that total size of all samples be about ~x100 times the target size of dictionary.
*/
public static nuint ZDICT_trainFromBuffer_fastCover(void* dictBuffer, nuint dictBufferCapacity, void* samplesBuffer, nuint* samplesSizes, uint nbSamples, ZDICT_fastCover_params_t parameters)
{
byte* dict = (byte*)(dictBuffer);
FASTCOVER_ctx_t ctx;
ZDICT_cover_params_t coverParams;
FASTCOVER_accel_t accelParams;
g_displayLevel = (int)parameters.zParams.notificationLevel;
parameters.splitPoint = 1.0;
parameters.f = (uint)(parameters.f == 0 ? 20 : parameters.f);
parameters.accel = (uint)(parameters.accel == 0 ? 1 : parameters.accel);
memset((void*)&coverParams, 0, (nuint)(sizeof(ZDICT_cover_params_t)));
FASTCOVER_convertToCoverParams(parameters, &coverParams);
if ((FASTCOVER_checkParameters(coverParams, dictBufferCapacity, parameters.f, parameters.accel)) == 0)
{
return (unchecked((nuint)(-(int)ZSTD_ErrorCode.ZSTD_error_parameter_outOfBound)));
}
if (nbSamples == 0)
{
return (unchecked((nuint)(-(int)ZSTD_ErrorCode.ZSTD_error_srcSize_wrong)));
}
if (dictBufferCapacity < 256)
{
return (unchecked((nuint)(-(int)ZSTD_ErrorCode.ZSTD_error_dstSize_tooSmall)));
}
accelParams = FASTCOVER_defaultAccelParameters[parameters.accel];
{
nuint initVal = FASTCOVER_ctx_init(&ctx, samplesBuffer, samplesSizes, nbSamples, coverParams.d, parameters.splitPoint, parameters.f, accelParams);
if ((ERR_isError(initVal)) != 0)
{
return initVal;
}
}
COVER_warnOnSmallCorpus(dictBufferCapacity, ctx.nbDmers, g_displayLevel);
{
ushort* segmentFreqs = (ushort*)(calloc((nuint)((ulong)(1) << (int)parameters.f), (nuint)(2)));
nuint tail = FASTCOVER_buildDictionary(&ctx, ctx.freqs, dictBuffer, dictBufferCapacity, coverParams, segmentFreqs);
uint nbFinalizeSamples = (uint)(ctx.nbTrainSamples * ctx.accelParams.finalize / 100);
nuint dictionarySize = ZDICT_finalizeDictionary((void*)dict, dictBufferCapacity, (void*)(dict + tail), dictBufferCapacity - tail, samplesBuffer, samplesSizes, nbFinalizeSamples, coverParams.zParams);
if ((ERR_isError(dictionarySize)) == 0)
{
;
}
FASTCOVER_ctx_destroy(&ctx);
free((void*)segmentFreqs);
return dictionarySize;
}
}
/*! ZDICT_optimizeTrainFromBuffer_fastCover():
* The same requirements as above hold for all the parameters except `parameters`.
* This function tries many parameter combinations (specifically, k and d combinations)
* and picks the best parameters. `*parameters` is filled with the best parameters found,
* dictionary constructed with those parameters is stored in `dictBuffer`.
* All of the parameters d, k, steps, f, and accel are optional.
* If d is non-zero then we don't check multiple values of d, otherwise we check d = {6, 8}.
* if steps is zero it defaults to its default value.
* If k is non-zero then we don't check multiple values of k, otherwise we check steps values in [50, 2000].
* If f is zero, default value of 20 is used.
* If accel is zero, default value of 1 is used.
*
* @return: size of dictionary stored into `dictBuffer` (<= `dictBufferCapacity`)
* or an error code, which can be tested with ZDICT_isError().
* On success `*parameters` contains the parameters selected.
* See ZDICT_trainFromBuffer() for details on failure modes.
* Note: ZDICT_optimizeTrainFromBuffer_fastCover() requires about 6 * 2^f bytes of memory for each thread.
*/
public static nuint ZDICT_optimizeTrainFromBuffer_fastCover(void* dictBuffer, nuint dictBufferCapacity, void* samplesBuffer, nuint* samplesSizes, uint nbSamples, ZDICT_fastCover_params_t* parameters)
{
ZDICT_cover_params_t coverParams;
FASTCOVER_accel_t accelParams;
uint nbThreads = parameters->nbThreads;
double splitPoint = parameters->splitPoint <= 0.0 ? 0.75 : parameters->splitPoint;
uint kMinD = (uint)(parameters->d == 0 ? 6 : parameters->d);
uint kMaxD = (uint)(parameters->d == 0 ? 8 : parameters->d);
uint kMinK = (uint)(parameters->k == 0 ? 50 : parameters->k);
uint kMaxK = (uint)(parameters->k == 0 ? 2000 : parameters->k);
uint kSteps = (uint)(parameters->steps == 0 ? 40 : parameters->steps);
uint kStepSize = (((kMaxK - kMinK) / kSteps) > (1) ? ((kMaxK - kMinK) / kSteps) : (1));
uint kIterations = (1 + (kMaxD - kMinD) / 2) * (1 + (kMaxK - kMinK) / kStepSize);
uint f = (uint)(parameters->f == 0 ? 20 : parameters->f);
uint accel = (uint)(parameters->accel == 0 ? 1 : parameters->accel);
uint shrinkDict = 0;
int displayLevel = (int)parameters->zParams.notificationLevel;
uint iteration = 1;
uint d;
uint k;
COVER_best_s best;
int warned = 0;
if (splitPoint <= 0 || splitPoint > 1)
{
if (displayLevel >= 1)
{
;
}
return (unchecked((nuint)(-(int)ZSTD_ErrorCode.ZSTD_error_parameter_outOfBound)));
}
if (accel == 0 || accel > 10)
{
if (displayLevel >= 1)
{
;
}
return (unchecked((nuint)(-(int)ZSTD_ErrorCode.ZSTD_error_parameter_outOfBound)));
}
if (kMinK < kMaxD || kMaxK < kMinK)
{
if (displayLevel >= 1)
{
;
}
return (unchecked((nuint)(-(int)ZSTD_ErrorCode.ZSTD_error_parameter_outOfBound)));
}
if (nbSamples == 0)
{
if (displayLevel >= 1)
{
;
}
return (unchecked((nuint)(-(int)ZSTD_ErrorCode.ZSTD_error_srcSize_wrong)));
}
if (dictBufferCapacity < 256)
{
if (displayLevel >= 1)
{
;
}
return (unchecked((nuint)(-(int)ZSTD_ErrorCode.ZSTD_error_dstSize_tooSmall)));
}
if (nbThreads > 1)
{
throw new NotImplementedException("Multiple threads are not supported");
}
COVER_best_init(&best);
memset((void*)&coverParams, 0, (nuint)(sizeof(ZDICT_cover_params_t)));
FASTCOVER_convertToCoverParams(*parameters, &coverParams);
accelParams = FASTCOVER_defaultAccelParameters[accel];
g_displayLevel = displayLevel == 0 ? 0 : displayLevel - 1;
if (displayLevel >= 2)
{
;
}
for (d = kMinD; d <= kMaxD; d += 2)
{
FASTCOVER_ctx_t ctx;
if (displayLevel >= 3)
{
;
}
{
nuint initVal = FASTCOVER_ctx_init(&ctx, samplesBuffer, samplesSizes, nbSamples, d, splitPoint, f, accelParams);
if ((ERR_isError(initVal)) != 0)
{
if (displayLevel >= 1)
{
;
}
COVER_best_destroy(&best);
return initVal;
}
}
if (warned == 0)
{
COVER_warnOnSmallCorpus(dictBufferCapacity, ctx.nbDmers, displayLevel);
warned = 1;
}
for (k = kMinK; k <= kMaxK; k += kStepSize)
{
FASTCOVER_tryParameters_data_s* data = (FASTCOVER_tryParameters_data_s*)(malloc((nuint)(sizeof(FASTCOVER_tryParameters_data_s))));
if (displayLevel >= 3)
{
;
}
if (data == null)
{
if (displayLevel >= 1)
{
;
}
COVER_best_destroy(&best);
FASTCOVER_ctx_destroy(&ctx);
return (unchecked((nuint)(-(int)ZSTD_ErrorCode.ZSTD_error_memory_allocation)));
}
data->ctx = &ctx;
data->best = &best;
data->dictBufferCapacity = dictBufferCapacity;
data->parameters = coverParams;
data->parameters.k = k;
data->parameters.d = d;
data->parameters.splitPoint = splitPoint;
data->parameters.steps = kSteps;
data->parameters.shrinkDict = shrinkDict;
data->parameters.zParams.notificationLevel = (uint)g_displayLevel;
if ((FASTCOVER_checkParameters(data->parameters, dictBufferCapacity, data->ctx->f, accel)) == 0)
{
free((void*)data);
continue;
}
COVER_best_start(&best);
FASTCOVER_tryParameters((void*)data);
++iteration;
}
COVER_best_wait(&best);
FASTCOVER_ctx_destroy(&ctx);
}
if (displayLevel >= 2)
{
;
}
{
nuint dictSize = best.dictSize;
if ((ERR_isError(best.compressedSize)) != 0)
{
nuint compressedSize = best.compressedSize;
COVER_best_destroy(&best);
return compressedSize;
}
FASTCOVER_convertToFastCoverParams(best.parameters, parameters, f, accel);
memcpy(dictBuffer, best.dict, dictSize);
COVER_best_destroy(&best);
return dictSize;
}
}
}
}

View File

@@ -0,0 +1,158 @@
using System;
using System.Runtime.CompilerServices;
using static ZstdSharp.UnsafeHelper;
namespace ZstdSharp.Unsafe
{
public static unsafe partial class Methods
{
[MethodImpl(MethodImplOptions.AggressiveInlining)]
private static void FSE_initCState(FSE_CState_t* statePtr, uint* ct)
{
void* ptr = (void*)ct;
ushort* u16ptr = (ushort*)(ptr);
uint tableLog = MEM_read16(ptr);
statePtr->value = (nint)(1) << (int)tableLog;
statePtr->stateTable = u16ptr + 2;
statePtr->symbolTT = ct + 1 + (tableLog != 0 ? (1 << (int)(tableLog - 1)) : 1);
statePtr->stateLog = tableLog;
}
/*! FSE_initCState2() :
* Same as FSE_initCState(), but the first symbol to include (which will be the last to be read)
* uses the smallest state value possible, saving the cost of this symbol */
[MethodImpl(MethodImplOptions.AggressiveInlining)]
private static void FSE_initCState2(FSE_CState_t* statePtr, uint* ct, uint symbol)
{
FSE_initCState(statePtr, ct);
{
FSE_symbolCompressionTransform symbolTT = ((FSE_symbolCompressionTransform*)(statePtr->symbolTT))[symbol];
ushort* stateTable = (ushort*)(statePtr->stateTable);
uint nbBitsOut = (uint)((symbolTT.deltaNbBits + (uint)((1 << 15))) >> 16);
statePtr->value = (nint)((nbBitsOut << 16) - symbolTT.deltaNbBits);
statePtr->value = (nint)(stateTable[(statePtr->value >> (int)nbBitsOut) + symbolTT.deltaFindState]);
}
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
private static void FSE_encodeSymbol(BIT_CStream_t* bitC, FSE_CState_t* statePtr, uint symbol)
{
FSE_symbolCompressionTransform symbolTT = ((FSE_symbolCompressionTransform*)(statePtr->symbolTT))[symbol];
ushort* stateTable = (ushort*)(statePtr->stateTable);
uint nbBitsOut = (uint)(((nuint)statePtr->value + symbolTT.deltaNbBits) >> 16);
BIT_addBits(bitC, (nuint)statePtr->value, nbBitsOut);
statePtr->value = (nint)(stateTable[(statePtr->value >> (int)nbBitsOut) + symbolTT.deltaFindState]);
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
private static void FSE_flushCState(BIT_CStream_t* bitC, FSE_CState_t* statePtr)
{
BIT_addBits(bitC, (nuint)statePtr->value, statePtr->stateLog);
BIT_flushBits(bitC);
}
/* FSE_getMaxNbBits() :
* Approximate maximum cost of a symbol, in bits.
* Fractional get rounded up (i.e : a symbol with a normalized frequency of 3 gives the same result as a frequency of 2)
* note 1 : assume symbolValue is valid (<= maxSymbolValue)
* note 2 : if freq[symbolValue]==0, @return a fake cost of tableLog+1 bits */
[MethodImpl(MethodImplOptions.AggressiveInlining)]
private static uint FSE_getMaxNbBits(void* symbolTTPtr, uint symbolValue)
{
FSE_symbolCompressionTransform* symbolTT = (FSE_symbolCompressionTransform*)(symbolTTPtr);
return (symbolTT[symbolValue].deltaNbBits + (uint)(((1 << 16) - 1))) >> 16;
}
/* FSE_bitCost() :
* Approximate symbol cost, as fractional value, using fixed-point format (accuracyLog fractional bits)
* note 1 : assume symbolValue is valid (<= maxSymbolValue)
* note 2 : if freq[symbolValue]==0, @return a fake cost of tableLog+1 bits */
[MethodImpl(MethodImplOptions.AggressiveInlining)]
private static uint FSE_bitCost(void* symbolTTPtr, uint tableLog, uint symbolValue, uint accuracyLog)
{
FSE_symbolCompressionTransform* symbolTT = (FSE_symbolCompressionTransform*)(symbolTTPtr);
uint minNbBits = symbolTT[symbolValue].deltaNbBits >> 16;
uint threshold = (minNbBits + 1) << 16;
assert(tableLog < 16);
assert(accuracyLog < 31 - tableLog);
{
uint tableSize = (uint)(1 << (int)tableLog);
uint deltaFromThreshold = threshold - (symbolTT[symbolValue].deltaNbBits + tableSize);
uint normalizedDeltaFromThreshold = (deltaFromThreshold << (int)accuracyLog) >> (int)tableLog;
uint bitMultiplier = (uint)(1 << (int)accuracyLog);
assert(symbolTT[symbolValue].deltaNbBits + tableSize <= threshold);
assert(normalizedDeltaFromThreshold <= bitMultiplier);
return (minNbBits + 1) * bitMultiplier - normalizedDeltaFromThreshold;
}
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
private static void FSE_initDState(FSE_DState_t* DStatePtr, BIT_DStream_t* bitD, uint* dt)
{
void* ptr = (void*)dt;
FSE_DTableHeader* DTableH = (FSE_DTableHeader*)(ptr);
DStatePtr->state = BIT_readBits(bitD, DTableH->tableLog);
BIT_reloadDStream(bitD);
DStatePtr->table = dt + 1;
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
private static byte FSE_peekSymbol(FSE_DState_t* DStatePtr)
{
FSE_decode_t DInfo = ((FSE_decode_t*)(DStatePtr->table))[DStatePtr->state];
return DInfo.symbol;
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
private static void FSE_updateState(FSE_DState_t* DStatePtr, BIT_DStream_t* bitD)
{
FSE_decode_t DInfo = ((FSE_decode_t*)(DStatePtr->table))[DStatePtr->state];
uint nbBits = DInfo.nbBits;
nuint lowBits = BIT_readBits(bitD, nbBits);
DStatePtr->state = DInfo.newState + lowBits;
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
private static byte FSE_decodeSymbol(FSE_DState_t* DStatePtr, BIT_DStream_t* bitD)
{
FSE_decode_t DInfo = ((FSE_decode_t*)(DStatePtr->table))[DStatePtr->state];
uint nbBits = DInfo.nbBits;
byte symbol = DInfo.symbol;
nuint lowBits = BIT_readBits(bitD, nbBits);
DStatePtr->state = DInfo.newState + lowBits;
return symbol;
}
/*! FSE_decodeSymbolFast() :
unsafe, only works if no symbol has a probability > 50% */
[MethodImpl(MethodImplOptions.AggressiveInlining)]
private static byte FSE_decodeSymbolFast(FSE_DState_t* DStatePtr, BIT_DStream_t* bitD)
{
FSE_decode_t DInfo = ((FSE_decode_t*)(DStatePtr->table))[DStatePtr->state];
uint nbBits = DInfo.nbBits;
byte symbol = DInfo.symbol;
nuint lowBits = BIT_readBitsFast(bitD, nbBits);
DStatePtr->state = DInfo.newState + lowBits;
return symbol;
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
private static uint FSE_endOfDState(FSE_DState_t* DStatePtr)
{
return ((DStatePtr->state == 0) ? 1U : 0U);
}
}
}

View File

@@ -0,0 +1,960 @@
using System;
using static ZstdSharp.UnsafeHelper;
namespace ZstdSharp.Unsafe
{
public static unsafe partial class Methods
{
/* FSE_buildCTable_wksp() :
* Same as FSE_buildCTable(), but using an externally allocated scratch buffer (`workSpace`).
* wkspSize should be sized to handle worst case situation, which is `1<<max_tableLog * sizeof(FSE_FUNCTION_TYPE)`
* workSpace must also be properly aligned with FSE_FUNCTION_TYPE requirements
*/
public static nuint FSE_buildCTable_wksp(uint* ct, short* normalizedCounter, uint maxSymbolValue, uint tableLog, void* workSpace, nuint wkspSize)
{
uint tableSize = (uint)(1 << (int)tableLog);
uint tableMask = tableSize - 1;
void* ptr = (void*)ct;
ushort* tableU16 = ((ushort*)(ptr)) + 2;
void* FSCT = (void*)(((uint*)(ptr)) + 1 + (tableLog != 0 ? tableSize >> 1 : 1));
FSE_symbolCompressionTransform* symbolTT = (FSE_symbolCompressionTransform*)(FSCT);
uint step = (((tableSize) >> 1) + ((tableSize) >> 3) + 3);
uint* cumul = (uint*)(workSpace);
byte* tableSymbol = (byte*)(cumul + (maxSymbolValue + 2));
uint highThreshold = tableSize - 1;
if (((nuint)(workSpace) & 3) != 0)
{
return (unchecked((nuint)(-(int)ZSTD_ErrorCode.ZSTD_error_GENERIC)));
}
if (((nuint)(sizeof(uint)) * (maxSymbolValue + 2 + (1UL << (int)(tableLog - 2)))) > wkspSize)
{
return (unchecked((nuint)(-(int)ZSTD_ErrorCode.ZSTD_error_tableLog_tooLarge)));
}
tableU16[-2] = (ushort)(tableLog);
tableU16[-1] = (ushort)(maxSymbolValue);
assert(tableLog < 16);
{
uint u;
cumul[0] = 0;
for (u = 1; u <= maxSymbolValue + 1; u++)
{
if (normalizedCounter[u - 1] == -1)
{
cumul[u] = cumul[u - 1] + 1;
tableSymbol[highThreshold--] = (byte)(u - 1);
}
else
{
cumul[u] = cumul[u - 1] + (ushort)(normalizedCounter[u - 1]);
}
}
cumul[maxSymbolValue + 1] = tableSize + 1;
}
{
uint position = 0;
uint symbol;
for (symbol = 0; symbol <= maxSymbolValue; symbol++)
{
int nbOccurrences;
int freq = normalizedCounter[symbol];
for (nbOccurrences = 0; nbOccurrences < freq; nbOccurrences++)
{
tableSymbol[position] = (byte)(symbol);
position = (position + step) & tableMask;
while (position > highThreshold)
{
position = (position + step) & tableMask;
}
}
}
assert(position == 0);
}
{
uint u;
for (u = 0; u < tableSize; u++)
{
byte s = tableSymbol[u];
tableU16[cumul[s]++] = (ushort)(tableSize + u);
}
}
{
uint total = 0;
uint s;
for (s = 0; s <= maxSymbolValue; s++)
{
switch (normalizedCounter[s])
{
case 0:
{
symbolTT[s].deltaNbBits = ((tableLog + 1) << 16) - (uint)((1 << (int)tableLog));
}
break;
case -1:
case 1:
{
symbolTT[s].deltaNbBits = (tableLog << 16) - (uint)((1 << (int)tableLog));
}
symbolTT[s].deltaFindState = (int)(total - 1);
total++;
break;
default:
{
uint maxBitsOut = tableLog - BIT_highbit32((uint)(normalizedCounter[s] - 1));
uint minStatePlus = (uint)(normalizedCounter[s] << (int)maxBitsOut);
symbolTT[s].deltaNbBits = (maxBitsOut << 16) - minStatePlus;
symbolTT[s].deltaFindState = (int)(total - (ushort)(normalizedCounter[s]));
total += (uint)(normalizedCounter[s]);
}
break;
}
}
}
return 0;
}
/*! FSE_buildCTable():
Builds `ct`, which must be already allocated, using FSE_createCTable().
@return : 0, or an errorCode, which can be tested using FSE_isError() */
public static nuint FSE_buildCTable(uint* ct, short* normalizedCounter, uint maxSymbolValue, uint tableLog)
{
byte* tableSymbol = stackalloc byte[4096];
return FSE_buildCTable_wksp(ct, normalizedCounter, maxSymbolValue, tableLog, (void*)tableSymbol, (nuint)(sizeof(byte) * 4096));
}
/*-**************************************************************
* FSE NCount encoding
****************************************************************/
public static nuint FSE_NCountWriteBound(uint maxSymbolValue, uint tableLog)
{
nuint maxHeaderSize = (((maxSymbolValue + 1) * tableLog) >> 3) + 3;
return maxSymbolValue != 0 ? maxHeaderSize : 512;
}
private static nuint FSE_writeNCount_generic(void* header, nuint headerBufferSize, short* normalizedCounter, uint maxSymbolValue, uint tableLog, uint writeIsSafe)
{
byte* ostart = (byte*)(header);
byte* @out = ostart;
byte* oend = ostart + headerBufferSize;
int nbBits;
int tableSize = 1 << (int)tableLog;
int remaining;
int threshold;
uint bitStream = 0;
int bitCount = 0;
uint symbol = 0;
uint alphabetSize = maxSymbolValue + 1;
int previousIs0 = 0;
bitStream += (tableLog - 5) << bitCount;
bitCount += 4;
remaining = tableSize + 1;
threshold = tableSize;
nbBits = (int)(tableLog + 1);
while ((symbol < alphabetSize) && (remaining > 1))
{
if (previousIs0 != 0)
{
uint start = symbol;
while ((symbol < alphabetSize) && (normalizedCounter[symbol]) == 0)
{
symbol++;
}
if (symbol == alphabetSize)
{
break;
}
while (symbol >= start + 24)
{
start += 24;
bitStream += 0xFFFFU << bitCount;
if (writeIsSafe == 0 && (@out > oend - 2))
{
return (unchecked((nuint)(-(int)ZSTD_ErrorCode.ZSTD_error_dstSize_tooSmall)));
}
@out[0] = (byte)(bitStream);
@out[1] = (byte)(bitStream >> 8);
@out += 2;
bitStream >>= 16;
}
while (symbol >= start + 3)
{
start += 3;
bitStream += (uint)(3 << bitCount);
bitCount += 2;
}
bitStream += (symbol - start) << bitCount;
bitCount += 2;
if (bitCount > 16)
{
if (writeIsSafe == 0 && (@out > oend - 2))
{
return (unchecked((nuint)(-(int)ZSTD_ErrorCode.ZSTD_error_dstSize_tooSmall)));
}
@out[0] = (byte)(bitStream);
@out[1] = (byte)(bitStream >> 8);
@out += 2;
bitStream >>= 16;
bitCount -= 16;
}
}
{
int count = normalizedCounter[symbol++];
int max = (2 * threshold - 1) - remaining;
remaining -= count < 0 ? -count : count;
count++;
if (count >= threshold)
{
count += max;
}
bitStream += (uint)(count << bitCount);
bitCount += nbBits;
bitCount -= ((count < max) ? 1 : 0);
previousIs0 = ((count == 1) ? 1 : 0);
if (remaining < 1)
{
return (unchecked((nuint)(-(int)ZSTD_ErrorCode.ZSTD_error_GENERIC)));
}
while (remaining < threshold)
{
nbBits--;
threshold >>= 1;
}
}
if (bitCount > 16)
{
if (writeIsSafe == 0 && (@out > oend - 2))
{
return (unchecked((nuint)(-(int)ZSTD_ErrorCode.ZSTD_error_dstSize_tooSmall)));
}
@out[0] = (byte)(bitStream);
@out[1] = (byte)(bitStream >> 8);
@out += 2;
bitStream >>= 16;
bitCount -= 16;
}
}
if (remaining != 1)
{
return (unchecked((nuint)(-(int)ZSTD_ErrorCode.ZSTD_error_GENERIC)));
}
assert(symbol <= alphabetSize);
if (writeIsSafe == 0 && (@out > oend - 2))
{
return (unchecked((nuint)(-(int)ZSTD_ErrorCode.ZSTD_error_dstSize_tooSmall)));
}
@out[0] = (byte)(bitStream);
@out[1] = (byte)(bitStream >> 8);
@out += (bitCount + 7) / 8;
return (nuint)((@out - ostart));
}
/*! FSE_writeNCount():
Compactly save 'normalizedCounter' into 'buffer'.
@return : size of the compressed table,
or an errorCode, which can be tested using FSE_isError(). */
public static nuint FSE_writeNCount(void* buffer, nuint bufferSize, short* normalizedCounter, uint maxSymbolValue, uint tableLog)
{
if (tableLog > (uint)((14 - 2)))
{
return (unchecked((nuint)(-(int)ZSTD_ErrorCode.ZSTD_error_tableLog_tooLarge)));
}
if (tableLog < 5)
{
return (unchecked((nuint)(-(int)ZSTD_ErrorCode.ZSTD_error_GENERIC)));
}
if (bufferSize < FSE_NCountWriteBound(maxSymbolValue, tableLog))
{
return FSE_writeNCount_generic(buffer, bufferSize, normalizedCounter, maxSymbolValue, tableLog, 0);
}
return FSE_writeNCount_generic(buffer, bufferSize, normalizedCounter, maxSymbolValue, tableLog, 1);
}
/*-**************************************************************
* FSE Compression Code
****************************************************************/
public static uint* FSE_createCTable(uint maxSymbolValue, uint tableLog)
{
nuint size;
if (tableLog > 15)
{
tableLog = 15;
}
size = ((uint)(1 + (1 << (int)((tableLog) - 1))) + (((maxSymbolValue) + 1) * 2)) * (nuint)(sizeof(uint));
return (uint*)(malloc(size));
}
public static void FSE_freeCTable(uint* ct)
{
free((void*)(ct));
}
/* provides the minimum logSize to safely represent a distribution */
[InlineMethod.Inline]
private static uint FSE_minTableLog(nuint srcSize, uint maxSymbolValue)
{
uint minBitsSrc = BIT_highbit32((uint)(srcSize)) + 1;
uint minBitsSymbols = BIT_highbit32(maxSymbolValue) + 2;
uint minBits = minBitsSrc < minBitsSymbols ? minBitsSrc : minBitsSymbols;
assert(srcSize > 1);
return minBits;
}
/* *****************************************
* FSE advanced API
***************************************** */
public static uint FSE_optimalTableLog_internal(uint maxTableLog, nuint srcSize, uint maxSymbolValue, uint minus)
{
uint maxBitsSrc = BIT_highbit32((uint)(srcSize - 1)) - minus;
uint tableLog = maxTableLog;
uint minBits = FSE_minTableLog(srcSize, maxSymbolValue);
assert(srcSize > 1);
if (tableLog == 0)
{
tableLog = (uint)((13 - 2));
}
if (maxBitsSrc < tableLog)
{
tableLog = maxBitsSrc;
}
if (minBits > tableLog)
{
tableLog = minBits;
}
if (tableLog < 5)
{
tableLog = 5;
}
if (tableLog > (uint)((14 - 2)))
{
tableLog = (uint)((14 - 2));
}
return tableLog;
}
/*! FSE_optimalTableLog():
dynamically downsize 'tableLog' when conditions are met.
It saves CPU time, by using smaller tables, while preserving or even improving compression ratio.
@return : recommended tableLog (necessarily <= 'maxTableLog') */
public static uint FSE_optimalTableLog(uint maxTableLog, nuint srcSize, uint maxSymbolValue)
{
return FSE_optimalTableLog_internal(maxTableLog, srcSize, maxSymbolValue, 2);
}
/* Secondary normalization method.
To be used when primary method fails. */
private static nuint FSE_normalizeM2(short* norm, uint tableLog, uint* count, nuint total, uint maxSymbolValue, short lowProbCount)
{
short NOT_YET_ASSIGNED = (short)-2;
uint s;
uint distributed = 0;
uint ToDistribute;
uint lowThreshold = (uint)(total >> (int)tableLog);
uint lowOne = (uint)((total * 3) >> (int)(tableLog + 1));
for (s = 0; s <= maxSymbolValue; s++)
{
if (count[s] == 0)
{
norm[s] = 0;
continue;
}
if (count[s] <= lowThreshold)
{
norm[s] = lowProbCount;
distributed++;
total -= count[s];
continue;
}
if (count[s] <= lowOne)
{
norm[s] = 1;
distributed++;
total -= count[s];
continue;
}
norm[s] = NOT_YET_ASSIGNED;
}
ToDistribute = (uint)((1 << (int)tableLog)) - distributed;
if (ToDistribute == 0)
{
return 0;
}
if ((total / ToDistribute) > lowOne)
{
lowOne = (uint)((total * 3) / (ToDistribute * 2));
for (s = 0; s <= maxSymbolValue; s++)
{
if ((norm[s] == NOT_YET_ASSIGNED) && (count[s] <= lowOne))
{
norm[s] = 1;
distributed++;
total -= count[s];
continue;
}
}
ToDistribute = (uint)((1 << (int)tableLog)) - distributed;
}
if (distributed == maxSymbolValue + 1)
{
uint maxV = 0, maxC = 0;
for (s = 0; s <= maxSymbolValue; s++)
{
if (count[s] > maxC)
{
maxV = s;
maxC = count[s];
}
}
norm[maxV] += (short)(short)(ToDistribute);
return 0;
}
if (total == 0)
{
for (s = 0; ToDistribute > 0; s = (s + 1) % (maxSymbolValue + 1))
{
if (norm[s] > 0)
{
ToDistribute--;
norm[s]++;
}
}
return 0;
}
{
ulong vStepLog = 62 - tableLog;
ulong mid = (1UL << (int)(vStepLog - 1)) - 1;
ulong rStep = (((((ulong)(1) << (int)vStepLog) * ToDistribute) + mid) / ((uint)(total)));
ulong tmpTotal = mid;
for (s = 0; s <= maxSymbolValue; s++)
{
if (norm[s] == NOT_YET_ASSIGNED)
{
ulong end = tmpTotal + (count[s] * rStep);
uint sStart = (uint)(tmpTotal >> (int)vStepLog);
uint sEnd = (uint)(end >> (int)vStepLog);
uint weight = sEnd - sStart;
if (weight < 1)
{
return (unchecked((nuint)(-(int)ZSTD_ErrorCode.ZSTD_error_GENERIC)));
}
norm[s] = (short)(weight);
tmpTotal = end;
}
}
}
return 0;
}
/*! FSE_normalizeCount():
normalize counts so that sum(count[]) == Power_of_2 (2^tableLog)
'normalizedCounter' is a table of short, of minimum size (maxSymbolValue+1).
useLowProbCount is a boolean parameter which trades off compressed size for
faster header decoding. When it is set to 1, the compressed data will be slightly
smaller. And when it is set to 0, FSE_readNCount() and FSE_buildDTable() will be
faster. If you are compressing a small amount of data (< 2 KB) then useLowProbCount=0
is a good default, since header deserialization makes a big speed difference.
Otherwise, useLowProbCount=1 is a good default, since the speed difference is small.
@return : tableLog,
or an errorCode, which can be tested using FSE_isError() */
public static nuint FSE_normalizeCount(short* normalizedCounter, uint tableLog, uint* count, nuint total, uint maxSymbolValue, uint useLowProbCount)
{
if (tableLog == 0)
{
tableLog = (uint)((13 - 2));
}
if (tableLog < 5)
{
return (unchecked((nuint)(-(int)ZSTD_ErrorCode.ZSTD_error_GENERIC)));
}
if (tableLog > (uint)((14 - 2)))
{
return (unchecked((nuint)(-(int)ZSTD_ErrorCode.ZSTD_error_tableLog_tooLarge)));
}
if (tableLog < FSE_minTableLog(total, maxSymbolValue))
{
return (unchecked((nuint)(-(int)ZSTD_ErrorCode.ZSTD_error_GENERIC)));
}
{
short lowProbCount = (short)(useLowProbCount != 0 ? -1 : 1);
ulong scale = 62 - tableLog;
ulong step = (((ulong)(1) << 62) / ((uint)(total)));
ulong vStep = 1UL << (int)(scale - 20);
int stillToDistribute = 1 << (int)tableLog;
uint s;
uint largest = 0;
short largestP = 0;
uint lowThreshold = (uint)(total >> (int)tableLog);
for (s = 0; s <= maxSymbolValue; s++)
{
if (count[s] == total)
{
return 0;
}
if (count[s] == 0)
{
normalizedCounter[s] = 0;
continue;
}
if (count[s] <= lowThreshold)
{
normalizedCounter[s] = lowProbCount;
stillToDistribute--;
}
else
{
short proba = (short)((count[s] * step) >> (int)scale);
if (proba < 8)
{
ulong restToBeat = vStep * rtbTable[proba];
proba += (short)((((count[s] * step) - ((ulong)(proba) << (int)scale) > restToBeat) ? 1 : 0));
}
if (proba > largestP)
{
largestP = proba;
largest = s;
}
normalizedCounter[s] = proba;
stillToDistribute -= proba;
}
}
if (-stillToDistribute >= (normalizedCounter[largest] >> 1))
{
nuint errorCode = FSE_normalizeM2(normalizedCounter, tableLog, count, total, maxSymbolValue, lowProbCount);
if ((ERR_isError(errorCode)) != 0)
{
return errorCode;
}
}
else
{
normalizedCounter[largest] += (short)(short)(stillToDistribute);
}
}
return tableLog;
}
/* fake FSE_CTable, for raw (uncompressed) input */
public static nuint FSE_buildCTable_raw(uint* ct, uint nbBits)
{
uint tableSize = (uint)(1 << (int)nbBits);
uint tableMask = tableSize - 1;
uint maxSymbolValue = tableMask;
void* ptr = (void*)ct;
ushort* tableU16 = ((ushort*)(ptr)) + 2;
void* FSCT = (void*)(((uint*)(ptr)) + 1 + (tableSize >> 1));
FSE_symbolCompressionTransform* symbolTT = (FSE_symbolCompressionTransform*)(FSCT);
uint s;
if (nbBits < 1)
{
return (unchecked((nuint)(-(int)ZSTD_ErrorCode.ZSTD_error_GENERIC)));
}
tableU16[-2] = (ushort)(nbBits);
tableU16[-1] = (ushort)(maxSymbolValue);
for (s = 0; s < tableSize; s++)
{
tableU16[s] = (ushort)(tableSize + s);
}
{
uint deltaNbBits = (nbBits << 16) - (uint)((1 << (int)nbBits));
for (s = 0; s <= maxSymbolValue; s++)
{
symbolTT[s].deltaNbBits = deltaNbBits;
symbolTT[s].deltaFindState = (int)(s - 1);
}
}
return 0;
}
/* fake FSE_CTable, for rle input (always same symbol) */
public static nuint FSE_buildCTable_rle(uint* ct, byte symbolValue)
{
void* ptr = (void*)ct;
ushort* tableU16 = ((ushort*)(ptr)) + 2;
void* FSCTptr = (void*)((uint*)(ptr) + 2);
FSE_symbolCompressionTransform* symbolTT = (FSE_symbolCompressionTransform*)(FSCTptr);
tableU16[-2] = (ushort)(0);
tableU16[-1] = (ushort)(symbolValue);
tableU16[0] = 0;
tableU16[1] = 0;
symbolTT[symbolValue].deltaNbBits = 0;
symbolTT[symbolValue].deltaFindState = 0;
return 0;
}
private static nuint FSE_compress_usingCTable_generic(void* dst, nuint dstSize, void* src, nuint srcSize, uint* ct, uint fast)
{
byte* istart = (byte*)(src);
byte* iend = istart + srcSize;
byte* ip = iend;
BIT_CStream_t bitC;
FSE_CState_t CState1, CState2;
if (srcSize <= 2)
{
return 0;
}
{
nuint initError = BIT_initCStream(&bitC, dst, dstSize);
if ((ERR_isError(initError)) != 0)
{
return 0;
}
}
if ((srcSize & 1) != 0)
{
FSE_initCState2(&CState1, ct, *--ip);
FSE_initCState2(&CState2, ct, *--ip);
FSE_encodeSymbol(&bitC, &CState1, *--ip);
if (fast != 0)
{
BIT_flushBitsFast(&bitC);
}
else
{
BIT_flushBits(&bitC);
}
}
else
{
FSE_initCState2(&CState2, ct, *--ip);
FSE_initCState2(&CState1, ct, *--ip);
}
srcSize -= 2;
if (((nuint)(sizeof(nuint)) * 8 > (uint)((14 - 2) * 4 + 7)) && (srcSize & 2) != 0)
{
FSE_encodeSymbol(&bitC, &CState2, *--ip);
FSE_encodeSymbol(&bitC, &CState1, *--ip);
if (fast != 0)
{
BIT_flushBitsFast(&bitC);
}
else
{
BIT_flushBits(&bitC);
}
}
while (ip > istart)
{
FSE_encodeSymbol(&bitC, &CState2, *--ip);
if ((nuint)(sizeof(nuint)) * 8 < (uint)((14 - 2) * 2 + 7))
{
if (fast != 0)
{
BIT_flushBitsFast(&bitC);
}
else
{
BIT_flushBits(&bitC);
}
}
FSE_encodeSymbol(&bitC, &CState1, *--ip);
if ((nuint)(sizeof(nuint)) * 8 > (uint)((14 - 2) * 4 + 7))
{
FSE_encodeSymbol(&bitC, &CState2, *--ip);
FSE_encodeSymbol(&bitC, &CState1, *--ip);
}
if (fast != 0)
{
BIT_flushBitsFast(&bitC);
}
else
{
BIT_flushBits(&bitC);
}
}
FSE_flushCState(&bitC, &CState2);
FSE_flushCState(&bitC, &CState1);
return BIT_closeCStream(&bitC);
}
/*! FSE_compress_usingCTable():
Compress `src` using `ct` into `dst` which must be already allocated.
@return : size of compressed data (<= `dstCapacity`),
or 0 if compressed data could not fit into `dst`,
or an errorCode, which can be tested using FSE_isError() */
public static nuint FSE_compress_usingCTable(void* dst, nuint dstSize, void* src, nuint srcSize, uint* ct)
{
uint fast = (((dstSize >= ((srcSize) + ((srcSize) >> 7) + 4 + (nuint)(sizeof(nuint))))) ? 1U : 0U);
if (fast != 0)
{
return FSE_compress_usingCTable_generic(dst, dstSize, src, srcSize, ct, 1);
}
else
{
return FSE_compress_usingCTable_generic(dst, dstSize, src, srcSize, ct, 0);
}
}
/*-*****************************************
* Tool functions
******************************************/
public static nuint FSE_compressBound(nuint size)
{
return (512 + ((size) + ((size) >> 7) + 4 + (nuint)(sizeof(nuint))));
}
/* FSE_compress_wksp() :
* Same as FSE_compress2(), but using an externally allocated scratch buffer (`workSpace`).
* `wkspSize` size must be `(1<<tableLog)`.
*/
public static nuint FSE_compress_wksp(void* dst, nuint dstSize, void* src, nuint srcSize, uint maxSymbolValue, uint tableLog, void* workSpace, nuint wkspSize)
{
byte* ostart = (byte*)(dst);
byte* op = ostart;
byte* oend = ostart + dstSize;
uint* count = stackalloc uint[256];
short* norm = stackalloc short[256];
uint* CTable = (uint*)(workSpace);
nuint CTableSize = ((uint)(1 + (1 << (int)((tableLog) - 1))) + (((maxSymbolValue) + 1) * 2));
void* scratchBuffer = (void*)(CTable + CTableSize);
nuint scratchBufferSize = wkspSize - (CTableSize * (nuint)(4));
if (wkspSize < (((uint)(1 + (1 << (int)((tableLog) - 1))) + (((maxSymbolValue) + 1) * 2)) + (uint)(((tableLog > 12) ? (1 << (int)(tableLog - 2)) : 1024))))
{
return (unchecked((nuint)(-(int)ZSTD_ErrorCode.ZSTD_error_tableLog_tooLarge)));
}
if (srcSize <= 1)
{
return 0;
}
if (maxSymbolValue == 0)
{
maxSymbolValue = 255;
}
if (tableLog == 0)
{
tableLog = (uint)((13 - 2));
}
{
nuint maxCount = HIST_count_wksp((uint*)count, &maxSymbolValue, src, srcSize, scratchBuffer, scratchBufferSize);
if ((ERR_isError(maxCount)) != 0)
{
return maxCount;
}
if (maxCount == srcSize)
{
return 1;
}
if (maxCount == 1)
{
return 0;
}
if (maxCount < (srcSize >> 7))
{
return 0;
}
}
tableLog = FSE_optimalTableLog(tableLog, srcSize, maxSymbolValue);
{
nuint _var_err__ = FSE_normalizeCount((short*)norm, tableLog, (uint*)count, srcSize, maxSymbolValue, ((srcSize >= 2048) ? 1U : 0U));
if ((ERR_isError(_var_err__)) != 0)
{
return _var_err__;
}
}
{
nuint nc_err = FSE_writeNCount((void*)op, (nuint)(oend - op), (short*)norm, maxSymbolValue, tableLog);
if ((ERR_isError(nc_err)) != 0)
{
return nc_err;
}
op += nc_err;
}
{
nuint _var_err__ = FSE_buildCTable_wksp(CTable, (short*)norm, maxSymbolValue, tableLog, scratchBuffer, scratchBufferSize);
if ((ERR_isError(_var_err__)) != 0)
{
return _var_err__;
}
}
{
nuint cSize = FSE_compress_usingCTable((void*)op, (nuint)(oend - op), src, srcSize, CTable);
if ((ERR_isError(cSize)) != 0)
{
return cSize;
}
if (cSize == 0)
{
return 0;
}
op += cSize;
}
if ((nuint)(op - ostart) >= srcSize - 1)
{
return 0;
}
return (nuint)(op - ostart);
}
/*-*****************************************
* FSE advanced functions
******************************************/
/*! FSE_compress2() :
Same as FSE_compress(), but allows the selection of 'maxSymbolValue' and 'tableLog'
Both parameters can be defined as '0' to mean : use default value
@return : size of compressed data
Special values : if return == 0, srcData is not compressible => Nothing is stored within cSrc !!!
if return == 1, srcData is a single byte symbol * srcSize times. Use RLE compression.
if FSE_isError(return), it's an error code.
*/
public static nuint FSE_compress2(void* dst, nuint dstCapacity, void* src, nuint srcSize, uint maxSymbolValue, uint tableLog)
{
fseWkspMax_t scratchBuffer;
if (tableLog > (uint)((14 - 2)))
{
return (unchecked((nuint)(-(int)ZSTD_ErrorCode.ZSTD_error_tableLog_tooLarge)));
}
return FSE_compress_wksp(dst, dstCapacity, src, srcSize, maxSymbolValue, tableLog, (void*)&scratchBuffer, (nuint)(sizeof(fseWkspMax_t)));
}
/*-****************************************
* FSE simple functions
******************************************/
/*! FSE_compress() :
Compress content of buffer 'src', of size 'srcSize', into destination buffer 'dst'.
'dst' buffer must be already allocated. Compression runs faster is dstCapacity >= FSE_compressBound(srcSize).
@return : size of compressed data (<= dstCapacity).
Special values : if return == 0, srcData is not compressible => Nothing is stored within dst !!!
if return == 1, srcData is a single byte symbol * srcSize times. Use RLE compression instead.
if FSE_isError(return), compression failed (more details using FSE_getErrorName())
*/
public static nuint FSE_compress(void* dst, nuint dstCapacity, void* src, nuint srcSize)
{
return FSE_compress2(dst, dstCapacity, src, srcSize, 255, (uint)((13 - 2)));
}
}
}

View File

@@ -0,0 +1,446 @@
using System;
using System.Runtime.CompilerServices;
using static ZstdSharp.UnsafeHelper;
namespace ZstdSharp.Unsafe
{
public static unsafe partial class Methods
{
/* Function templates */
public static uint* FSE_createDTable(uint tableLog)
{
if (tableLog > 15)
{
tableLog = 15;
}
return (uint*)(malloc((uint)((1 + (1 << (int)(tableLog)))) * (nuint)(sizeof(uint))));
}
public static void FSE_freeDTable(uint* dt)
{
free((void*)(dt));
}
private static nuint FSE_buildDTable_internal(uint* dt, short* normalizedCounter, uint maxSymbolValue, uint tableLog, void* workSpace, nuint wkspSize)
{
void* tdPtr = (void*)(dt + 1);
FSE_decode_t* tableDecode = (FSE_decode_t*)(tdPtr);
ushort* symbolNext = (ushort*)(workSpace);
byte* spread = (byte*)(symbolNext + maxSymbolValue + 1);
uint maxSV1 = maxSymbolValue + 1;
uint tableSize = (uint)(1 << (int)tableLog);
uint highThreshold = tableSize - 1;
if (((nuint)(sizeof(short)) * (maxSymbolValue + 1) + (1UL << (int)tableLog) + 8) > wkspSize)
{
return (unchecked((nuint)(-(int)ZSTD_ErrorCode.ZSTD_error_maxSymbolValue_tooLarge)));
}
if (maxSymbolValue > 255)
{
return (unchecked((nuint)(-(int)ZSTD_ErrorCode.ZSTD_error_maxSymbolValue_tooLarge)));
}
if (tableLog > (uint)((14 - 2)))
{
return (unchecked((nuint)(-(int)ZSTD_ErrorCode.ZSTD_error_tableLog_tooLarge)));
}
{
FSE_DTableHeader DTableH;
DTableH.tableLog = (ushort)(tableLog);
DTableH.fastMode = 1;
{
short largeLimit = (short)(1 << (int)(tableLog - 1));
uint s;
for (s = 0; s < maxSV1; s++)
{
if (normalizedCounter[s] == -1)
{
tableDecode[highThreshold--].symbol = (byte)(s);
symbolNext[s] = 1;
}
else
{
if (normalizedCounter[s] >= largeLimit)
{
DTableH.fastMode = 0;
}
symbolNext[s] = (ushort)(normalizedCounter[s]);
}
}
}
memcpy((void*)(dt), (void*)(&DTableH), ((nuint)(sizeof(FSE_DTableHeader))));
}
if (highThreshold == tableSize - 1)
{
nuint tableMask = tableSize - 1;
nuint step = (((tableSize) >> 1) + ((tableSize) >> 3) + 3);
{
ulong add = 0x0101010101010101UL;
nuint pos = 0;
ulong sv = 0;
uint s;
for (s = 0; s < maxSV1; ++s , sv += add)
{
int i;
int n = normalizedCounter[s];
MEM_write64((void*)(spread + pos), sv);
for (i = 8; i < n; i += 8)
{
MEM_write64((void*)(spread + pos + i), sv);
}
pos += (nuint)n;
}
}
{
nuint position = 0;
nuint s;
nuint unroll = 2;
assert(tableSize % unroll == 0);
for (s = 0; s < (nuint)(tableSize); s += unroll)
{
nuint u;
for (u = 0; u < unroll; ++u)
{
nuint uPosition = (position + (u * step)) & tableMask;
tableDecode[uPosition].symbol = spread[s + u];
}
position = (position + (unroll * step)) & tableMask;
}
assert(position == 0);
}
}
else
{
uint tableMask = tableSize - 1;
uint step = (((tableSize) >> 1) + ((tableSize) >> 3) + 3);
uint s, position = 0;
for (s = 0; s < maxSV1; s++)
{
int i;
for (i = 0; i < normalizedCounter[s]; i++)
{
tableDecode[position].symbol = (byte)(s);
position = (position + step) & tableMask;
while (position > highThreshold)
{
position = (position + step) & tableMask;
}
}
}
if (position != 0)
{
return (unchecked((nuint)(-(int)ZSTD_ErrorCode.ZSTD_error_GENERIC)));
}
}
{
uint u;
for (u = 0; u < tableSize; u++)
{
byte symbol = (byte)(tableDecode[u].symbol);
uint nextState = symbolNext[symbol]++;
tableDecode[u].nbBits = (byte)(tableLog - BIT_highbit32(nextState));
tableDecode[u].newState = (ushort)((nextState << (int)(tableDecode[u].nbBits)) - tableSize);
}
}
return 0;
}
public static nuint FSE_buildDTable_wksp(uint* dt, short* normalizedCounter, uint maxSymbolValue, uint tableLog, void* workSpace, nuint wkspSize)
{
return FSE_buildDTable_internal(dt, normalizedCounter, maxSymbolValue, tableLog, workSpace, wkspSize);
}
/*-*******************************************************
* Decompression (Byte symbols)
*********************************************************/
public static nuint FSE_buildDTable_rle(uint* dt, byte symbolValue)
{
void* ptr = (void*)dt;
FSE_DTableHeader* DTableH = (FSE_DTableHeader*)(ptr);
void* dPtr = (void*)(dt + 1);
FSE_decode_t* cell = (FSE_decode_t*)(dPtr);
DTableH->tableLog = 0;
DTableH->fastMode = 0;
cell->newState = 0;
cell->symbol = symbolValue;
cell->nbBits = 0;
return 0;
}
public static nuint FSE_buildDTable_raw(uint* dt, uint nbBits)
{
void* ptr = (void*)dt;
FSE_DTableHeader* DTableH = (FSE_DTableHeader*)(ptr);
void* dPtr = (void*)(dt + 1);
FSE_decode_t* dinfo = (FSE_decode_t*)(dPtr);
uint tableSize = (uint)(1 << (int)nbBits);
uint tableMask = tableSize - 1;
uint maxSV1 = tableMask + 1;
uint s;
if (nbBits < 1)
{
return (unchecked((nuint)(-(int)ZSTD_ErrorCode.ZSTD_error_GENERIC)));
}
DTableH->tableLog = (ushort)(nbBits);
DTableH->fastMode = 1;
for (s = 0; s < maxSV1; s++)
{
dinfo[s].newState = 0;
dinfo[s].symbol = (byte)(s);
dinfo[s].nbBits = (byte)(nbBits);
}
return 0;
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
private static nuint FSE_decompress_usingDTable_generic(void* dst, nuint maxDstSize, void* cSrc, nuint cSrcSize, uint* dt, uint fast)
{
byte* ostart = (byte*)(dst);
byte* op = ostart;
byte* omax = op + maxDstSize;
byte* olimit = omax - 3;
BIT_DStream_t bitD;
FSE_DState_t state1;
FSE_DState_t state2;
{
nuint _var_err__ = BIT_initDStream(&bitD, cSrc, cSrcSize);
if ((ERR_isError(_var_err__)) != 0)
{
return _var_err__;
}
}
FSE_initDState(&state1, &bitD, dt);
FSE_initDState(&state2, &bitD, dt);
for (; ((BIT_reloadDStream(&bitD) == BIT_DStream_status.BIT_DStream_unfinished) && (op < olimit)); op += 4)
{
op[0] = fast != 0 ? FSE_decodeSymbolFast(&state1, &bitD) : FSE_decodeSymbol(&state1, &bitD);
if ((uint)((14 - 2) * 2 + 7) > (nuint)(sizeof(nuint)) * 8)
{
BIT_reloadDStream(&bitD);
}
op[1] = fast != 0 ? FSE_decodeSymbolFast(&state2, &bitD) : FSE_decodeSymbol(&state2, &bitD);
if ((uint)((14 - 2) * 4 + 7) > (nuint)(sizeof(nuint)) * 8)
{
if (BIT_reloadDStream(&bitD) > BIT_DStream_status.BIT_DStream_unfinished)
{
op += 2;
break;
}
}
op[2] = fast != 0 ? FSE_decodeSymbolFast(&state1, &bitD) : FSE_decodeSymbol(&state1, &bitD);
if ((uint)((14 - 2) * 2 + 7) > (nuint)(sizeof(nuint)) * 8)
{
BIT_reloadDStream(&bitD);
}
op[3] = fast != 0 ? FSE_decodeSymbolFast(&state2, &bitD) : FSE_decodeSymbol(&state2, &bitD);
}
while (1 != 0)
{
if (op > (omax - 2))
{
return (unchecked((nuint)(-(int)ZSTD_ErrorCode.ZSTD_error_dstSize_tooSmall)));
}
*op++ = fast != 0 ? FSE_decodeSymbolFast(&state1, &bitD) : FSE_decodeSymbol(&state1, &bitD);
if (BIT_reloadDStream(&bitD) == BIT_DStream_status.BIT_DStream_overflow)
{
*op++ = fast != 0 ? FSE_decodeSymbolFast(&state2, &bitD) : FSE_decodeSymbol(&state2, &bitD);
break;
}
if (op > (omax - 2))
{
return (unchecked((nuint)(-(int)ZSTD_ErrorCode.ZSTD_error_dstSize_tooSmall)));
}
*op++ = fast != 0 ? FSE_decodeSymbolFast(&state2, &bitD) : FSE_decodeSymbol(&state2, &bitD);
if (BIT_reloadDStream(&bitD) == BIT_DStream_status.BIT_DStream_overflow)
{
*op++ = fast != 0 ? FSE_decodeSymbolFast(&state1, &bitD) : FSE_decodeSymbol(&state1, &bitD);
break;
}
}
return (nuint)(op - ostart);
}
/*! FSE_decompress_usingDTable():
Decompress compressed source `cSrc` of size `cSrcSize` using `dt`
into `dst` which must be already allocated.
@return : size of regenerated data (necessarily <= `dstCapacity`),
or an errorCode, which can be tested using FSE_isError() */
public static nuint FSE_decompress_usingDTable(void* dst, nuint originalSize, void* cSrc, nuint cSrcSize, uint* dt)
{
void* ptr = (void*)dt;
FSE_DTableHeader* DTableH = (FSE_DTableHeader*)(ptr);
uint fastMode = DTableH->fastMode;
if (fastMode != 0)
{
return FSE_decompress_usingDTable_generic(dst, originalSize, cSrc, cSrcSize, dt, 1);
}
return FSE_decompress_usingDTable_generic(dst, originalSize, cSrc, cSrcSize, dt, 0);
}
public static nuint FSE_decompress_wksp(void* dst, nuint dstCapacity, void* cSrc, nuint cSrcSize, uint maxLog, void* workSpace, nuint wkspSize)
{
return FSE_decompress_wksp_bmi2(dst, dstCapacity, cSrc, cSrcSize, maxLog, workSpace, wkspSize, 0);
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
private static nuint FSE_decompress_wksp_body(void* dst, nuint dstCapacity, void* cSrc, nuint cSrcSize, uint maxLog, void* workSpace, nuint wkspSize, int bmi2)
{
byte* istart = (byte*)(cSrc);
byte* ip = istart;
uint tableLog;
uint maxSymbolValue = 255;
FSE_DecompressWksp* wksp = (FSE_DecompressWksp*)(workSpace);
if (wkspSize < (nuint)(sizeof(FSE_DecompressWksp)))
{
return (unchecked((nuint)(-(int)ZSTD_ErrorCode.ZSTD_error_GENERIC)));
}
{
nuint NCountLength = FSE_readNCount_bmi2((short*)wksp->ncount, &maxSymbolValue, &tableLog, (void*)istart, cSrcSize, bmi2);
if ((ERR_isError(NCountLength)) != 0)
{
return NCountLength;
}
if (tableLog > maxLog)
{
return (unchecked((nuint)(-(int)ZSTD_ErrorCode.ZSTD_error_tableLog_tooLarge)));
}
assert(NCountLength <= cSrcSize);
ip += NCountLength;
cSrcSize -= NCountLength;
}
if ((((uint)((1 + (1 << (int)(tableLog)))) + ((((nuint)(sizeof(short)) * (maxSymbolValue + 1) + (1UL << (int)tableLog) + 8) + (nuint)(sizeof(uint)) - 1) / (nuint)(sizeof(uint))) + (uint)((255 + 1) / 2) + 1) * (nuint)(sizeof(uint))) > wkspSize)
{
return (unchecked((nuint)(-(int)ZSTD_ErrorCode.ZSTD_error_tableLog_tooLarge)));
}
workSpace = wksp->dtable + (1 + (1 << (int)(tableLog)));
wkspSize -= (nuint)(sizeof(FSE_DecompressWksp)) + ((uint)((1 + (1 << (int)(tableLog)))) * (nuint)(sizeof(uint)));
{
nuint _var_err__ = FSE_buildDTable_internal((uint*)wksp->dtable, (short*)wksp->ncount, maxSymbolValue, tableLog, workSpace, wkspSize);
if ((ERR_isError(_var_err__)) != 0)
{
return _var_err__;
}
}
{
void* ptr = (void*)wksp->dtable;
FSE_DTableHeader* DTableH = (FSE_DTableHeader*)(ptr);
uint fastMode = DTableH->fastMode;
if (fastMode != 0)
{
return FSE_decompress_usingDTable_generic(dst, dstCapacity, (void*)ip, cSrcSize, (uint*)wksp->dtable, 1);
}
return FSE_decompress_usingDTable_generic(dst, dstCapacity, (void*)ip, cSrcSize, (uint*)wksp->dtable, 0);
}
}
/* Avoids the FORCE_INLINE of the _body() function. */
private static nuint FSE_decompress_wksp_body_default(void* dst, nuint dstCapacity, void* cSrc, nuint cSrcSize, uint maxLog, void* workSpace, nuint wkspSize)
{
return FSE_decompress_wksp_body(dst, dstCapacity, cSrc, cSrcSize, maxLog, workSpace, wkspSize, 0);
}
private static nuint FSE_decompress_wksp_body_bmi2(void* dst, nuint dstCapacity, void* cSrc, nuint cSrcSize, uint maxLog, void* workSpace, nuint wkspSize)
{
return FSE_decompress_wksp_body(dst, dstCapacity, cSrc, cSrcSize, maxLog, workSpace, wkspSize, 1);
}
public static nuint FSE_decompress_wksp_bmi2(void* dst, nuint dstCapacity, void* cSrc, nuint cSrcSize, uint maxLog, void* workSpace, nuint wkspSize, int bmi2)
{
if (bmi2 != 0)
{
return FSE_decompress_wksp_body_bmi2(dst, dstCapacity, cSrc, cSrcSize, maxLog, workSpace, wkspSize);
}
return FSE_decompress_wksp_body_default(dst, dstCapacity, cSrc, cSrcSize, maxLog, workSpace, wkspSize);
}
/*! FSE_buildDTable():
Builds 'dt', which must be already allocated, using FSE_createDTable().
return : 0, or an errorCode, which can be tested using FSE_isError() */
public static nuint FSE_buildDTable(uint* dt, short* normalizedCounter, uint maxSymbolValue, uint tableLog)
{
uint* wksp = stackalloc uint[8322];
return FSE_buildDTable_wksp(dt, normalizedCounter, maxSymbolValue, tableLog, (void*)wksp, (nuint)(sizeof(uint) * 8322));
}
/*! FSE_decompress():
Decompress FSE data from buffer 'cSrc', of size 'cSrcSize',
into already allocated destination buffer 'dst', of size 'dstCapacity'.
@return : size of regenerated data (<= maxDstSize),
or an error code, which can be tested using FSE_isError() .
** Important ** : FSE_decompress() does not decompress non-compressible nor RLE data !!!
Why ? : making this distinction requires a header.
Header management is intentionally delegated to the user layer, which can better manage special cases.
*/
public static nuint FSE_decompress(void* dst, nuint dstCapacity, void* cSrc, nuint cSrcSize)
{
uint* wksp = stackalloc uint[5380];
return FSE_decompress_wksp(dst, dstCapacity, cSrc, cSrcSize, (uint)((14 - 2)), (void*)wksp, (nuint)(sizeof(uint) * 5380));
}
}
}

View File

@@ -0,0 +1,10 @@
using System;
namespace ZstdSharp.Unsafe
{
public enum HIST_checkInput_e
{
trustInput,
checkMaxSymbolValue,
}
}

View File

@@ -0,0 +1,13 @@
using System;
namespace ZstdSharp.Unsafe
{
/* static allocation of HUF's Compression Table */
/* this is a private definition, just exposed for allocation and strict aliasing purpose. never EVER access its members directly */
public partial struct HUF_CElt_s
{
public ushort val;
public byte nbBits;
}
}

View File

@@ -0,0 +1,15 @@
using System;
namespace ZstdSharp.Unsafe
{
public unsafe partial struct HUF_CompressWeightsWksp
{
public fixed uint CTable[59];
public fixed uint scratchBuffer[30];
public fixed uint count[13];
public fixed short norm[13];
}
}

View File

@@ -0,0 +1,16 @@
using System;
namespace ZstdSharp.Unsafe
{
/*-***************************/
/* single-symbol decoding */
/*-***************************/
public partial struct HUF_DEltX1
{
/* single-symbol decoding */
public byte @byte;
/* single-symbol decoding */
public byte nbBits;
}
}

View File

@@ -0,0 +1,19 @@
using System;
namespace ZstdSharp.Unsafe
{
/* *************************/
/* double-symbols decoding */
/* *************************/
public partial struct HUF_DEltX2
{
/* double-symbols decoding */
public ushort sequence;
/* double-symbols decoding */
public byte nbBits;
/* double-symbols decoding */
public byte length;
}
}

View File

@@ -0,0 +1,17 @@
using System;
namespace ZstdSharp.Unsafe
{
public unsafe partial struct HUF_ReadDTableX1_Workspace
{
public fixed uint rankVal[16];
public fixed uint rankStart[16];
public fixed uint statsWksp[218];
public fixed byte symbols[256];
public fixed byte huffWeight[256];
}
}

View File

@@ -0,0 +1,389 @@
using InlineIL;
using System;
using System.Runtime.CompilerServices;
using System.Runtime.InteropServices;
using static InlineIL.IL.Emit;
namespace ZstdSharp.Unsafe
{
public unsafe partial struct HUF_ReadDTableX2_Workspace
{
public _rankVal_e__FixedBuffer rankVal;
public fixed uint rankStats[13];
public fixed uint rankStart0[14];
public _sortedSymbol_e__FixedBuffer sortedSymbol;
public fixed byte weightList[256];
public fixed uint calleeWksp[218];
public unsafe partial struct _rankVal_e__FixedBuffer
{
public rankValCol_t e0;
public rankValCol_t e1;
public rankValCol_t e2;
public rankValCol_t e3;
public rankValCol_t e4;
public rankValCol_t e5;
public rankValCol_t e6;
public rankValCol_t e7;
public rankValCol_t e8;
public rankValCol_t e9;
public rankValCol_t e10;
public rankValCol_t e11;
public ref rankValCol_t this[int index]
{
[MethodImpl(MethodImplOptions.AggressiveInlining)]
[InlineMethod.Inline]
get => ref *(this + (uint)index);
}
public ref rankValCol_t this[uint index]
{
[MethodImpl(MethodImplOptions.AggressiveInlining)]
[InlineMethod.Inline]
get => ref *(this + index);
}
public ref rankValCol_t this[nuint index]
{
[MethodImpl(MethodImplOptions.AggressiveInlining)]
[InlineMethod.Inline]
get => ref *(this + (uint)index);
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
[InlineMethod.Inline]
public static implicit operator rankValCol_t*(in _rankVal_e__FixedBuffer t)
{
Ldarg_0();
Ldflda(new FieldRef(typeof(_rankVal_e__FixedBuffer), nameof(e0)));
return IL.ReturnPointer<rankValCol_t>();
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
[InlineMethod.Inline]
public static rankValCol_t* operator +(in _rankVal_e__FixedBuffer t, uint index)
{
Ldarg_0();
Ldflda(new FieldRef(typeof(_rankVal_e__FixedBuffer), nameof(e0)));
Ldarg_1();
Conv_I();
Sizeof<rankValCol_t>();
Conv_I();
Mul();
Add();
return IL.ReturnPointer<rankValCol_t>();
}
}
public unsafe partial struct _sortedSymbol_e__FixedBuffer
{
public sortedSymbol_t e0;
public sortedSymbol_t e1;
public sortedSymbol_t e2;
public sortedSymbol_t e3;
public sortedSymbol_t e4;
public sortedSymbol_t e5;
public sortedSymbol_t e6;
public sortedSymbol_t e7;
public sortedSymbol_t e8;
public sortedSymbol_t e9;
public sortedSymbol_t e10;
public sortedSymbol_t e11;
public sortedSymbol_t e12;
public sortedSymbol_t e13;
public sortedSymbol_t e14;
public sortedSymbol_t e15;
public sortedSymbol_t e16;
public sortedSymbol_t e17;
public sortedSymbol_t e18;
public sortedSymbol_t e19;
public sortedSymbol_t e20;
public sortedSymbol_t e21;
public sortedSymbol_t e22;
public sortedSymbol_t e23;
public sortedSymbol_t e24;
public sortedSymbol_t e25;
public sortedSymbol_t e26;
public sortedSymbol_t e27;
public sortedSymbol_t e28;
public sortedSymbol_t e29;
public sortedSymbol_t e30;
public sortedSymbol_t e31;
public sortedSymbol_t e32;
public sortedSymbol_t e33;
public sortedSymbol_t e34;
public sortedSymbol_t e35;
public sortedSymbol_t e36;
public sortedSymbol_t e37;
public sortedSymbol_t e38;
public sortedSymbol_t e39;
public sortedSymbol_t e40;
public sortedSymbol_t e41;
public sortedSymbol_t e42;
public sortedSymbol_t e43;
public sortedSymbol_t e44;
public sortedSymbol_t e45;
public sortedSymbol_t e46;
public sortedSymbol_t e47;
public sortedSymbol_t e48;
public sortedSymbol_t e49;
public sortedSymbol_t e50;
public sortedSymbol_t e51;
public sortedSymbol_t e52;
public sortedSymbol_t e53;
public sortedSymbol_t e54;
public sortedSymbol_t e55;
public sortedSymbol_t e56;
public sortedSymbol_t e57;
public sortedSymbol_t e58;
public sortedSymbol_t e59;
public sortedSymbol_t e60;
public sortedSymbol_t e61;
public sortedSymbol_t e62;
public sortedSymbol_t e63;
public sortedSymbol_t e64;
public sortedSymbol_t e65;
public sortedSymbol_t e66;
public sortedSymbol_t e67;
public sortedSymbol_t e68;
public sortedSymbol_t e69;
public sortedSymbol_t e70;
public sortedSymbol_t e71;
public sortedSymbol_t e72;
public sortedSymbol_t e73;
public sortedSymbol_t e74;
public sortedSymbol_t e75;
public sortedSymbol_t e76;
public sortedSymbol_t e77;
public sortedSymbol_t e78;
public sortedSymbol_t e79;
public sortedSymbol_t e80;
public sortedSymbol_t e81;
public sortedSymbol_t e82;
public sortedSymbol_t e83;
public sortedSymbol_t e84;
public sortedSymbol_t e85;
public sortedSymbol_t e86;
public sortedSymbol_t e87;
public sortedSymbol_t e88;
public sortedSymbol_t e89;
public sortedSymbol_t e90;
public sortedSymbol_t e91;
public sortedSymbol_t e92;
public sortedSymbol_t e93;
public sortedSymbol_t e94;
public sortedSymbol_t e95;
public sortedSymbol_t e96;
public sortedSymbol_t e97;
public sortedSymbol_t e98;
public sortedSymbol_t e99;
public sortedSymbol_t e100;
public sortedSymbol_t e101;
public sortedSymbol_t e102;
public sortedSymbol_t e103;
public sortedSymbol_t e104;
public sortedSymbol_t e105;
public sortedSymbol_t e106;
public sortedSymbol_t e107;
public sortedSymbol_t e108;
public sortedSymbol_t e109;
public sortedSymbol_t e110;
public sortedSymbol_t e111;
public sortedSymbol_t e112;
public sortedSymbol_t e113;
public sortedSymbol_t e114;
public sortedSymbol_t e115;
public sortedSymbol_t e116;
public sortedSymbol_t e117;
public sortedSymbol_t e118;
public sortedSymbol_t e119;
public sortedSymbol_t e120;
public sortedSymbol_t e121;
public sortedSymbol_t e122;
public sortedSymbol_t e123;
public sortedSymbol_t e124;
public sortedSymbol_t e125;
public sortedSymbol_t e126;
public sortedSymbol_t e127;
public sortedSymbol_t e128;
public sortedSymbol_t e129;
public sortedSymbol_t e130;
public sortedSymbol_t e131;
public sortedSymbol_t e132;
public sortedSymbol_t e133;
public sortedSymbol_t e134;
public sortedSymbol_t e135;
public sortedSymbol_t e136;
public sortedSymbol_t e137;
public sortedSymbol_t e138;
public sortedSymbol_t e139;
public sortedSymbol_t e140;
public sortedSymbol_t e141;
public sortedSymbol_t e142;
public sortedSymbol_t e143;
public sortedSymbol_t e144;
public sortedSymbol_t e145;
public sortedSymbol_t e146;
public sortedSymbol_t e147;
public sortedSymbol_t e148;
public sortedSymbol_t e149;
public sortedSymbol_t e150;
public sortedSymbol_t e151;
public sortedSymbol_t e152;
public sortedSymbol_t e153;
public sortedSymbol_t e154;
public sortedSymbol_t e155;
public sortedSymbol_t e156;
public sortedSymbol_t e157;
public sortedSymbol_t e158;
public sortedSymbol_t e159;
public sortedSymbol_t e160;
public sortedSymbol_t e161;
public sortedSymbol_t e162;
public sortedSymbol_t e163;
public sortedSymbol_t e164;
public sortedSymbol_t e165;
public sortedSymbol_t e166;
public sortedSymbol_t e167;
public sortedSymbol_t e168;
public sortedSymbol_t e169;
public sortedSymbol_t e170;
public sortedSymbol_t e171;
public sortedSymbol_t e172;
public sortedSymbol_t e173;
public sortedSymbol_t e174;
public sortedSymbol_t e175;
public sortedSymbol_t e176;
public sortedSymbol_t e177;
public sortedSymbol_t e178;
public sortedSymbol_t e179;
public sortedSymbol_t e180;
public sortedSymbol_t e181;
public sortedSymbol_t e182;
public sortedSymbol_t e183;
public sortedSymbol_t e184;
public sortedSymbol_t e185;
public sortedSymbol_t e186;
public sortedSymbol_t e187;
public sortedSymbol_t e188;
public sortedSymbol_t e189;
public sortedSymbol_t e190;
public sortedSymbol_t e191;
public sortedSymbol_t e192;
public sortedSymbol_t e193;
public sortedSymbol_t e194;
public sortedSymbol_t e195;
public sortedSymbol_t e196;
public sortedSymbol_t e197;
public sortedSymbol_t e198;
public sortedSymbol_t e199;
public sortedSymbol_t e200;
public sortedSymbol_t e201;
public sortedSymbol_t e202;
public sortedSymbol_t e203;
public sortedSymbol_t e204;
public sortedSymbol_t e205;
public sortedSymbol_t e206;
public sortedSymbol_t e207;
public sortedSymbol_t e208;
public sortedSymbol_t e209;
public sortedSymbol_t e210;
public sortedSymbol_t e211;
public sortedSymbol_t e212;
public sortedSymbol_t e213;
public sortedSymbol_t e214;
public sortedSymbol_t e215;
public sortedSymbol_t e216;
public sortedSymbol_t e217;
public sortedSymbol_t e218;
public sortedSymbol_t e219;
public sortedSymbol_t e220;
public sortedSymbol_t e221;
public sortedSymbol_t e222;
public sortedSymbol_t e223;
public sortedSymbol_t e224;
public sortedSymbol_t e225;
public sortedSymbol_t e226;
public sortedSymbol_t e227;
public sortedSymbol_t e228;
public sortedSymbol_t e229;
public sortedSymbol_t e230;
public sortedSymbol_t e231;
public sortedSymbol_t e232;
public sortedSymbol_t e233;
public sortedSymbol_t e234;
public sortedSymbol_t e235;
public sortedSymbol_t e236;
public sortedSymbol_t e237;
public sortedSymbol_t e238;
public sortedSymbol_t e239;
public sortedSymbol_t e240;
public sortedSymbol_t e241;
public sortedSymbol_t e242;
public sortedSymbol_t e243;
public sortedSymbol_t e244;
public sortedSymbol_t e245;
public sortedSymbol_t e246;
public sortedSymbol_t e247;
public sortedSymbol_t e248;
public sortedSymbol_t e249;
public sortedSymbol_t e250;
public sortedSymbol_t e251;
public sortedSymbol_t e252;
public sortedSymbol_t e253;
public sortedSymbol_t e254;
public sortedSymbol_t e255;
public ref sortedSymbol_t this[int index]
{
[MethodImpl(MethodImplOptions.AggressiveInlining)]
[InlineMethod.Inline]
get => ref *(this + (uint)index);
}
public ref sortedSymbol_t this[uint index]
{
[MethodImpl(MethodImplOptions.AggressiveInlining)]
[InlineMethod.Inline]
get => ref *(this + index);
}
public ref sortedSymbol_t this[nuint index]
{
[MethodImpl(MethodImplOptions.AggressiveInlining)]
[InlineMethod.Inline]
get => ref *(this + (uint)index);
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
[InlineMethod.Inline]
public static implicit operator sortedSymbol_t*(in _sortedSymbol_e__FixedBuffer t)
{
Ldarg_0();
Ldflda(new FieldRef(typeof(_sortedSymbol_e__FixedBuffer), nameof(e0)));
return IL.ReturnPointer<sortedSymbol_t>();
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
[InlineMethod.Inline]
public static sortedSymbol_t* operator +(in _sortedSymbol_e__FixedBuffer t, uint index)
{
Ldarg_0();
Ldflda(new FieldRef(typeof(_sortedSymbol_e__FixedBuffer), nameof(e0)));
Ldarg_1();
Conv_I();
Sizeof<sortedSymbol_t>();
Conv_I();
Mul();
Add();
return IL.ReturnPointer<sortedSymbol_t>();
}
}
}
}

View File

@@ -0,0 +1,14 @@
using System;
namespace ZstdSharp.Unsafe
{
public unsafe partial struct HUF_WriteCTableWksp
{
public HUF_CompressWeightsWksp wksp;
/* precomputed conversion table */
public fixed byte bitsToWeight[13];
public fixed byte huffWeight[255];
}
}

Some files were not shown because too many files have changed in this diff Show More