V1.10.x and stealth add v2 features

.rspec

@@ -0,0 +1 @@
+--require spec_helper

.rubocop.yml

@@ -88,12 +88,18 @@ Style/IfInsideElse:
 Style/IfUnlessModifier:
   Enabled: false
 
+Style/InverseMethods:
+  Enabled: false
+
 Style/NestedTernaryOperator:
   Enabled: false
 
 Style/PreferredHashMethods:
   Enabled: false
 
+Style/Proc:
+  Enabled: false
+
 Style/NumericPredicate:
   Enabled: false
 
@@ -129,6 +135,9 @@ Style/SymbolProc: # old Ruby versions can't do this
 Style/TrailingCommaInHashLiteral:
   Enabled: false
 
+Style/TrailingCommaInArrayLiteral:
+  Enabled: false
+
 Style/TrailingUnderscoreVariable:
   Enabled: false
 
@@ -138,6 +147,9 @@ Style/TrivialAccessors:
 # Style/UnlessModifier:
 #   Enabled: false
 
+Style/WordArray:
+  Enabled: false
+
 Style/ZeroLengthPredicate:
   Enabled: false
 

CHANGELOG.md

@@ -1,6 +1,39 @@
 
 # SmarterCSV 1.x Change Log
 
+## T.B.D.
+
+  * code refactor
+
+  * NEW BEHAVIOR:
+    - hidden `:v2_mode` options (incomplete!)
+    - pre-processing for v2 options
+    - implemented v2 `:header_transformations` (DO NOT USE YET!)
+      + -> check if all v1 transformations are correctly done
+        How are we going to 
+        * disambiguate headers?
+
+
+        * do key_mapping? -> seems to work
+          - remove_unmapped_keys  ? 
+          - silence missing keys ... a missing mapped key should raise an exception, except when silenced
+          - required_keys needs to be a header-validation
+
+
+        * keep original headers? -> :none
+        * do strings_as_* ? -> either :keys_as_symbols, :keys_as_strings
+        * remove quote_chars? -> included in keys_as_*
+        * strip whitespace? -> included in keys_as_*
+
+        TODO:
+
+        - add tests for header_validations
+
+        - modify options to handle v1 and v2 options
+        - add v1 defaults in v2 processing
+        - add tests for all options processing
+        - 100% backwards compatibility when working in v1 mode
+
 ## 1.10.2 (2024-02-11)
   * improve error message for missing keys
 

lib/smarter_csv.rb

@@ -1,5 +1,7 @@
 # frozen_string_literal: true
 
+require 'set'
+
 require "smarter_csv/version"
 require "smarter_csv/file_io"
 require "smarter_csv/options_processing"

lib/smarter_csv/hash_transformations.rb

@@ -2,7 +2,16 @@
 
 module SmarterCSV
   class << self
+    # this is processing the headers from the input file
     def hash_transformations(hash, options)
+      if options[:v2_mode]
+        hash_transformations_v2(hash, options)
+      else
+        hash_transformations_v1(hash, options)
+      end
+    end
+
+    def hash_transformations_v1(hash, options)
       # there may be unmapped keys, or keys purposedly mapped to nil or an empty key..
       # make sure we delete any key/value pairs from the hash, which the user wanted to delete:
       remove_empty_values = options[:remove_empty_values] == true
@@ -33,46 +42,117 @@ def hash_transformations(hash, options)
       end
     end
 
-    # def hash_transformations(hash, options)
-    #   # there may be unmapped keys, or keys purposedly mapped to nil or an empty key..
-    #   # make sure we delete any key/value pairs from the hash, which the user wanted to delete:
-    #   hash.delete(nil)
-    #   hash.delete('')
-    #   hash.delete(:"")
-
-    #   if options[:remove_empty_values] == true
-    #     hash.delete_if{|_k, v| has_rails ? v.blank? : blank?(v)}
-    #   end
-
-    #   hash.delete_if{|_k, v| !v.nil? && v =~ /^(0+|0+\.0+)$/} if options[:remove_zero_values] # values are Strings
-    #   hash.delete_if{|_k, v| v =~ options[:remove_values_matching]} if options[:remove_values_matching]
-
-    #   if options[:convert_values_to_numeric]
-    #     hash.each do |k, v|
-    #       # deal with the :only / :except options to :convert_values_to_numeric
-    #       next if limit_execution_for_only_or_except(options, :convert_values_to_numeric, k)
-
-    #       # convert if it's a numeric value:
-    #       case v
-    #       when /^[+-]?\d+\.\d+$/
-    #         hash[k] = v.to_f
-    #       when /^[+-]?\d+$/
-    #         hash[k] = v.to_i
-    #       end
-    #     end
-    #   end
-
-    #   if options[:value_converters]
-    #     hash.each do |k, v|
-    #       converter = options[:value_converters][k]
-    #       next unless converter
-
-    #       hash[k] = converter.convert(v)
-    #     end
-    #   end
-
-    #   hash
-    # end
+    def hash_transformations_v2(hash, options)
+      return hash if options[:hash_transformations].nil? || options[:hash_transformations].empty?
+
+      # do the header transformations the user requested:
+      if options[:hash_transformations]
+        options[:hash_transformations].each do |transformation|
+          if transformation.respond_to?(:call) # this is used when a user-provided Proc is passed in
+            hash = transformation.call(hash, options)
+          else
+            case transformation
+            when Symbol # this is used for pre-defined transformations that are defined in the SmarterCSV module
+              hash = public_send(transformation, hash, options)
+            when Hash # this is called for hash arguments, e.g. hash_transformations
+              trans, args = transformation.first # .first treats the hash first element as an array
+              hash = apply_transformation(trans, hash, args, options)
+            when Array # this can be used for passing additional arguments in array form (e.g. into a Proc)
+              trans, *args = transformation
+              hash = apply_transformation(trans, hash, args, options)
+            else
+              raise SmarterCSV::IncorrectOption, "Invalid transformation type: #{transformation.class}"
+            end
+          end
+        end
+      end
+
+      hash
+    end
+
+    #
+    # To handle v1-backward-compatible behavior, it is faster to roll all behavior into one method
+    #
+    def v1_backwards_compatibility(hash, options)
+      hash.each_with_object({}) do |(k, v), new_hash|
+        next if k.nil? || k == '' || k == :"" # remove_empty_keys
+        next if has_rails ? v.blank? : blank?(v) # remove_empty_values
+
+        # convert_values_to_numeric:
+        # deal with the :only / :except options to :convert_values_to_numeric
+        unless limit_execution_for_only_or_except(options, :convert_values_to_numeric, k)
+          if v =~ /^[+-]?\d+\.\d+$/
+            v = v.to_f
+          elsif v =~ /^[+-]?\d+$/
+            v = v.to_i
+          end
+        end
+
+        new_hash[k] = v
+      end
+    end
+
+    #
+    # Building Blocks in case you want to build your own flow:
+    #
+
+    def value_converters(hash, _options)
+      #
+      # TO BE IMPLEMENTED
+      #
+    end
+
+    def strip_spaces(hash, _options)
+      hash.each_key {|key| hash[key].strip! unless hash[key].nil? } # &. syntax was introduced in Ruby 2.3 - need to stay backwards compatible
+    end
+
+    def remove_blank_values(hash, _options)
+      hash.each_key {|key| hash.delete(key) if hash[key].nil? || hash[key].is_a?(String) && hash[key] !~ /[^[:space:]]/ }
+    end
+
+    def remove_zero_values(hash, _options)
+      hash.each_key {|key| hash.delete(key) if hash[key].is_a?(Numeric) && hash[key].zero? }
+    end
+
+    def remove_empty_keys(hash, _options)
+      hash.reject!{|key, _v| key.nil? || key.empty?}
+    end
+
+    def convert_values_to_numeric(hash, _options)
+      hash.each_key do |k|
+        case hash[k]
+        when /^[+-]?\d+\.\d+$/
+          hash[k] = hash[k].to_f
+        when /^[+-]?\d+$/
+          hash[k] = hash[k].to_i
+        end
+      end
+    end
+
+    def convert_values_to_numeric_unless_leading_zeroes(hash, _options)
+      hash.each_key do |k|
+        case hash[k]
+        when /^[+-]?[1-9]\d*\.\d+$/
+          hash[k] = hash[k].to_f
+        when /^[+-]?[1-9]\d*$/
+          hash[k] = hash[k].to_i
+        end
+      end
+    end
+
+    # IMPORTANT NOTE:
+    # this can lead to cases where a nil or empty value gets converted into 0 or 0.0,
+    # and can then not be properly removed!
+    #
+    # you should first try to use convert_values_to_numeric or convert_values_to_numeric_unless_leading_zeroes
+    #
+    def convert_to_integer(hash, _options)
+      hash.each_key {|key| hash[key] = hash[key].to_i }
+    end
+
+    def convert_to_float(hash, _options)
+      hash.each_key {|key| hash[key] = hash[key].to_f }
+    end
 
     protected
 

lib/smarter_csv/header_transformations.rb

@@ -2,8 +2,18 @@
 
 module SmarterCSV
   class << self
-    # transform the headers that were in the file:
+    # this is processing the headers from the input file
     def header_transformations(header_array, options)
+      if options[:v2_mode]
+        header_transformations_v2(header_array, options)
+      else
+        header_transformations_v1(header_array, options)
+      end
+    end
+
+    # ---- V1.x Version: transform the headers that were in the file: ------------------------------------------
+    #
+    def header_transformations_v1(header_array, options)
       header_array.map!{|x| x.gsub(%r/#{options[:quote_char]}/, '')}
       header_array.map!{|x| x.strip} if options[:strip_whitespace]
 
@@ -57,7 +67,99 @@ def remap_headers(headers, options)
           header
         end
       end
+
       headers
     end
+
+    # ---- V2.x Version: transform the headers that were in the file: ------------------------------------------
+    #
+    def header_transformations_v2(header_array, options)
+      return header_array if options[:header_transformations].nil? || options[:header_transformations].empty?
+
+      # do the header transformations the user requested:
+      if options[:header_transformations]
+        options[:header_transformations].each do |transformation|
+          if transformation.respond_to?(:call) # this is used when a user-provided Proc is passed in
+            header_array = transformation.call(header_array, options)
+          else
+            case transformation
+            when Symbol # this is used for pre-defined transformations that are defined in the SmarterCSV module
+              header_array = public_send(transformation, header_array, options)
+            when Hash # this is called for hash arguments, e.g. header_transformations
+              trans, args = transformation.first # .first treats the hash first element as an array
+              header_array = apply_transformation(trans, header_array, args, options)
+            when Array # this can be used for passing additional arguments in array form (e.g. into a Proc)
+              trans, *args = transformation
+              header_array = apply_transformation(trans, header_array, args, options)
+            else
+              raise SmarterCSV::IncorrectOption, "Invalid transformation type: #{transformation.class}"
+            end
+          end
+        end
+      end
+
+      header_array
+    end
+
+    def apply_transformation(transformation, header_array, args, options)
+      if transformation.respond_to?(:call)
+        # If transformation is a callable object (like a Proc)
+        transformation.call(header_array, args, options)
+      else
+        # If transformation is a symbol (method name)
+        public_send(transformation, header_array, args, options)
+      end
+    end
+
+    # pre-defined v2 header transformations:
+
+    # these are some pre-defined header transformations which can be used
+    # all these take the headers array as the input
+    #
+    # the computed options can be accessed via @options
+
+    def keys_as_symbols(headers, options)
+      headers.map do |header|
+        header.strip.downcase.gsub(%r{#{options[:quote_char]}}, '').gsub(/(\s|-)+/, '_').to_sym
+      end
+    end
+
+    def keys_as_strings(headers, options)
+      headers.map do |header|
+        header.strip.gsub(%r{#{options[:quote_char]}}, '').downcase.gsub(/(\s|-)+/, '_')
+      end
+    end
+
+    def downcase_headers(headers, _options)
+      headers.map do |header|
+        header.strip.downcase!
+      end
+    end
+
+    def key_mapping(headers, mapping = {}, options)
+      raise(SmarterCSV::IncorrectOption, "ERROR: incorrect format for key_mapping! Expecting hash with from -> to mappings") if mapping.empty? || !mapping.is_a?(Hash)
+
+      headers_set = headers.to_set
+      mapping_keys_set = mapping.keys.to_set
+      silence_keys_set = (options[:silence_missing_keys] || []).to_set
+
+      # Check for missing keys
+      missing_keys = mapping_keys_set - headers_set - silence_keys_set
+      raise SmarterCSV::KeyMappingError, "ERROR: cannot map headers: #{missing_keys.to_a.join(', ')}" if missing_keys.any? && !options[:silence_missing_keys]
+
+      # Apply key mapping, retaining nils for explicitly mapped headers
+      headers.map do |header|
+        if mapping.key?(header)
+          # Maps the key according to the mapping, including nil mapping
+          mapping[header]
+        elsif options[:remove_unmapped_keys]
+          # Remove headers not specified in the mapping
+          nil
+        else
+          # Keep the original header if not specified in the mapping
+          header
+        end
+      end
+    end
   end
 end