Merge pull request #538 from vrtdev/feature/support-all-uri-schemas-in-checksum-url

Feature/support all uri schemas in checksum url

Author: bastelfreak

README.md

@@ -480,7 +480,8 @@ archive { '/tmp/staging/master.zip':
 * `ensure`: whether archive file should be present/absent (default: present)
 * `path`: namevar, archive file fully qualified file path.
 * `filename`: archive file name (derived from path).
-* `source`: archive file source, supports http|https|ftp|file|s3|gs uri.
+* `source`: archive file source,
+   supports puppet|http|https|ftp|file|s3|gs|abspath uri.
 * `headers`: array of headers to pass source, like an authentication token
 * `username`: username to download source file.
 * `password`: password to download source file.
@@ -489,7 +490,8 @@ archive { '/tmp/staging/master.zip':
 * `checksum_type`: archive file checksum type (none|md5|sha1|sha2|sha256|sha384|
   sha512). (default: none)
 * `checksum`: archive file checksum (match checksum_type)
-* `checksum_url`: archive file checksum source (instead of specify checksum)
+* `checksum_url`: archive file checksum source (instead of specify checksum),
+   supports puppet|http|https|ftp|file|s3|gs|abspath uri.
 * `checksum_verify`: whether checksum will be verified (true|false). (default: true)
 * `extract`: whether archive will be extracted after download (true|false).
   (default: false)

REFERENCE.md

@@ -953,7 +953,8 @@ Default value: `none`
 
 ##### <a name="-archive--checksum_url"></a>`checksum_url`
 
-archive file checksum source (instead of specifying checksum)
+archive file checksum source (instead of specifying checksum),
+supports puppet|http|https|ftp|file|s3|gs|abspath uri.
 
 ##### <a name="-archive--checksum_verify"></a>`checksum_verify`
 
@@ -991,7 +992,8 @@ archive file checksum type (none|md5|sha1|sha2|sha256|sha384|sha512)
 
 ##### <a name="-archive--digest_url"></a>`digest_url`
 
-archive file checksum source (instead of specifying checksum)
+archive file checksum source (instead of specifying checksum),
+supports puppet|http|https|ftp|file|s3|gs|abspath uri.
 (this parameter is for camptocamp/archive compatibility)
 
 ##### <a name="-archive--download_options"></a>`download_options`
@@ -1061,7 +1063,7 @@ proxy type (none|ftp|http|https)
 
 ##### <a name="-archive--source"></a>`source`
 
-archive file source, supports puppet|http|https|ftp|file|s3|gs uri.
+archive file source, supports puppet|http|https|ftp|file|s3|gs|abspath uri.
 
 ##### <a name="-archive--target"></a>`target`
 
@@ -1074,7 +1076,7 @@ used.
 
 ##### <a name="-archive--url"></a>`url`
 
-archive file source, supports http|https|ftp|file uri.
+archive file source, supports puppet|http|https|ftp|file|s3|gs|abspath uri.
 (for camptocamp/archive compatibility)
 
 ##### <a name="-archive--user"></a>`user`

lib/puppet/provider/archive/curl.rb

@@ -4,12 +4,14 @@
 require 'tempfile'
 
 Puppet::Type.type(:archive).provide(:curl, parent: :ruby) do
+  desc 'Curl-based implementation of the Archive resource type for http(s)/ftp-based urls.'
   commands curl: 'curl'
   defaultfor feature: :posix
 
-  def curl_params(params)
+  def curl_params(location, params)
     params_ordered = []
     params_ordered += optional_switch(resource[:headers], ['--header', '%s']) if resource[:headers]
+    params_ordered += [location]
     params_ordered += params
 
     if resource[:username]
@@ -18,7 +20,7 @@ def curl_params(params)
         account = [resource[:username], resource[:password]].compact.join(':')
         params_ordered += optional_switch(account, ['--user', '%s'])
       else
-        create_netrcfile
+        create_netrcfile(location)
         params_ordered += ['--netrc-file', @netrc_file.path]
       end
     end
@@ -30,9 +32,9 @@ def curl_params(params)
     params_ordered
   end
 
-  def create_netrcfile
+  def create_netrcfile(location)
     @netrc_file = Tempfile.new('.puppet_archive_curl')
-    machine = URI.parse(resource[:source]).host
+    machine = URI.parse(location).host
     @netrc_file.write("machine #{machine}\nlogin #{resource[:username]}\npassword #{resource[:password]}\n")
     @netrc_file.close
   end
@@ -44,10 +46,10 @@ def delete_netrcfile
     @netrc_file = nil
   end
 
-  def download(filepath)
+  def download(location, filepath)
     params = curl_params(
+      location,
       [
-        resource[:source],
         '-o',
         filepath,
         '-fsSLg',
@@ -62,21 +64,4 @@ def download(filepath)
       delete_netrcfile
     end
   end
-
-  def remote_checksum
-    params = curl_params(
-      [
-        resource[:checksum_url],
-        '-fsSLg',
-        '--max-redirs',
-        5
-      ]
-    )
-
-    begin
-      curl(params)[%r{\b[\da-f]{32,128}\b}i]
-    ensure
-      delete_netrcfile
-    end
-  end
 end

lib/puppet/provider/archive/ruby.rb

@@ -59,6 +59,7 @@
 #
 
 Puppet::Type.type(:archive).provide(:ruby) do
+  desc 'Archive resource type for Puppet.'
   optional_commands aws: 'aws'
   optional_commands gsutil: 'gsutil'
   defaultfor feature: :microsoft_windows
@@ -112,15 +113,12 @@ def checksum
   end
 
   def remote_checksum
-    PuppetX::Bodeco::Util.content(
-      resource[:checksum_url],
-      username: resource[:username],
-      password: resource[:password],
-      cookie: resource[:cookie],
-      proxy_server: resource[:proxy_server],
-      proxy_type: resource[:proxy_type],
-      insecure: resource[:allow_insecure]
-    )[%r{\b[\da-f]{32,128}\b}i]
+    temp_path = download_file(resource[:checksum_url])
+    raise(Puppet::Error, 'Unable to create temporary checksum file.') if temp_path.nil?
+
+    File.read(temp_path)[%r{\b[\da-f]{32,128}\b}i]
+  ensure
+    FileUtils.rm_f(temp_path) if temp_path && File.exist?(temp_path)
   end
 
   # Private: See if local archive checksum matches.
@@ -160,46 +158,48 @@ def extracted?
   end
 
   def transfer_download(archive_filepath)
-    raise Puppet::Error, "Temporary directory #{resource[:temp_dir]} doesn't exist" if resource[:temp_dir] && !File.directory?(resource[:temp_dir])
+    raise(Puppet::Error, 'Unable to fetch archive, the source parameter is nil.') if resource[:source].nil?
 
-    tempfile = Tempfile.new(tempfile_name, resource[:temp_dir])
-
-    temppath = tempfile.path
-    tempfile.close!
-
-    case resource[:source]
-    when %r{^(puppet)}
-      puppet_download(temppath)
-    when %r{^(http|ftp)}
-      download(temppath)
-    when %r{^file}
-      uri = URI(resource[:source])
-      FileUtils.copy(Puppet::Util.uri_to_path(uri), temppath)
-    when %r{^s3}
-      s3_download(temppath)
-    when %r{^gs}
-      gs_download(temppath)
-    when nil
-      raise(Puppet::Error, 'Unable to fetch archive, the source parameter is nil.')
-    else
-      raise(Puppet::Error, "Source file: #{resource[:source]} does not exists.") unless File.exist?(resource[:source])
-
-      FileUtils.copy(resource[:source], temppath)
-    end
+    temp_path = download_file(resource[:source])
+    raise(Puppet::Error, 'Unable to create temporary file.') if temp_path.nil?
 
     # conditionally verify checksum:
     if resource[:checksum_verify] == :true && resource[:checksum_type] != :none
-      archive = PuppetX::Bodeco::Archive.new(temppath)
+      archive = PuppetX::Bodeco::Archive.new(temp_path)
       actual_checksum = archive.checksum(resource[:checksum_type])
       if actual_checksum != checksum
         destroy
         raise(Puppet::Error, "Download file checksum mismatch (expected: #{checksum} actual: #{actual_checksum})")
       end
     end
 
-    move_file_in_place(temppath, archive_filepath)
+    move_file_in_place(temp_path, archive_filepath)
   ensure
-    FileUtils.rm_f(temppath) if File.exist?(temppath)
+    FileUtils.rm_f(temp_path) if temp_path && File.exist?(temp_path)
+  end
+
+  def download_file(location)
+    raise Puppet::Error, "Temporary directory #{resource[:temp_dir]} doesn't exist" if resource[:temp_dir] && !File.directory?(resource[:temp_dir])
+
+    Dir::Tmpname.create(tempfile_name, resource[:temp_dir]) do |temp_path|
+      case location
+      when %r{^(puppet)}
+        puppet_download(location, temp_path)
+      when %r{^(http|ftp)}
+        download(location, temp_path)
+      when %r{^file}
+        uri = URI(location)
+        FileUtils.copy(Puppet::Util.uri_to_path(uri), temp_path)
+      when %r{^s3}
+        s3_download(location, temp_path)
+      when %r{^gs}
+        gs_download(location, temp_path)
+      else
+        raise(Puppet::Error, "Source file: #{location} does not exists.") unless File.exist?(location)
+
+        FileUtils.copy(location, temp_path)
+      end
+    end
   end
 
   def move_file_in_place(from, to)
@@ -208,9 +208,9 @@ def move_file_in_place(from, to)
     FileUtils.mv(from, to)
   end
 
-  def download(filepath)
+  def download(location, filepath)
     PuppetX::Bodeco::Util.download(
-      resource[:source],
+      location,
       filepath,
       username: resource[:username],
       password: resource[:password],
@@ -221,29 +221,29 @@ def download(filepath)
     )
   end
 
-  def puppet_download(filepath)
+  def puppet_download(location, filepath)
     PuppetX::Bodeco::Util.puppet_download(
-      resource[:source],
+      location,
       filepath
     )
   end
 
-  def s3_download(path)
+  def s3_download(location, path)
     params = [
       's3',
       'cp',
-      resource[:source],
+      location,
       path
     ]
     params += resource[:download_options] if resource[:download_options]
 
     aws(params)
   end
 
-  def gs_download(path)
+  def gs_download(location, path)
     params = [
       'cp',
-      resource[:source],
+      location,
       path
     ]
     params += resource[:download_options] if resource[:download_options]

lib/puppet/provider/archive/wget.rb

@@ -1,6 +1,7 @@
 # frozen_string_literal: true
 
 Puppet::Type.type(:archive).provide(:wget, parent: :ruby) do
+  desc 'Wget-based implementation of the Archive resource type for http(s)/ftp-based urls.'
   commands wget: 'wget'
 
   def wget_params(params)
@@ -16,31 +17,18 @@ def wget_params(params)
     params
   end
 
-  def download(filepath)
-    params = wget_params(
+  def download(location, filepath)
+    command = wget_params(
       [
-        Shellwords.shellescape(resource[:source]),
+        'wget',
+        Shellwords.shellescape(location),
         '-O',
         filepath,
         '--max-redirect=5'
       ]
     )
 
     # NOTE: Do NOT use wget(params) until https://tickets.puppetlabs.com/browse/PUP-6066 is resolved.
-    command = "wget #{params.join(' ')}"
     Puppet::Util::Execution.execute(command)
   end
-
-  def remote_checksum
-    params = wget_params(
-      [
-        '-qO-',
-        Shellwords.shellescape(resource[:checksum_url]),
-        '--max-redirect=5'
-      ]
-    )
-
-    command = "wget #{params.join(' ')}"
-    Puppet::Util::Execution.execute(command)[%r{\b[\da-f]{32,128}\b}i]
-  end
 end

lib/puppet/type/archive.rb

@@ -118,17 +118,17 @@ def should_to_s(value)
   end
 
   newparam(:source) do
-    desc 'archive file source, supports puppet|http|https|ftp|file|s3|gs uri.'
+    desc 'archive file source, supports puppet|http|https|ftp|file|s3|gs|abspath uri.'
     validate do |value|
       raise ArgumentError, "invalid source url: #{value}" unless value =~ %r{puppet|http|https|ftp|file|s3|gs} || Puppet::Util.absolute_path?(value)
     end
   end
 
   newparam(:url) do
-    desc 'archive file source, supports http|https|ftp|file uri.
+    desc 'archive file source, supports puppet|http|https|ftp|file|s3|gs|abspath uri.
     (for camptocamp/archive compatibility)'
     validate do |value|
-      raise ArgumentError, "invalid source url: #{value}" unless value =~ %r{http|https|file|ftp}
+      raise ArgumentError, "invalid source url: #{value}" unless value =~ %r{puppet|http|https|ftp|file|s3|gs} || Puppet::Util.absolute_path?(value)
     end
     munge do |val|
       resource[:source] = val
@@ -166,11 +166,19 @@ def should_to_s(value)
   end
 
   newparam(:checksum_url) do
-    desc 'archive file checksum source (instead of specifying checksum)'
+    desc 'archive file checksum source (instead of specifying checksum),
+    supports puppet|http|https|ftp|file|s3|gs|abspath uri.'
+    validate do |value|
+      raise ArgumentError, "invalid checksum url: #{value}" unless value =~ %r{puppet|http|https|ftp|file|s3|gs} || Puppet::Util.absolute_path?(value)
+    end
   end
   newparam(:digest_url) do
-    desc 'archive file checksum source (instead of specifying checksum)
+    desc 'archive file checksum source (instead of specifying checksum),
+    supports puppet|http|https|ftp|file|s3|gs|abspath uri.
     (this parameter is for camptocamp/archive compatibility)'
+    validate do |value|
+      raise ArgumentError, "invalid digest url: #{value}" unless value =~ %r{puppet|http|https|ftp|file|s3|gs} || Puppet::Util.absolute_path?(value)
+    end
     munge do |val|
       resource[:checksum_url] = val
     end

spec/acceptance/authentication_spec.rb

@@ -6,7 +6,7 @@
 context 'authenticated download' do
   let(:source) do
     parser = URI::RFC2396_Parser.new
-    parser.escape("http://httpbin.org/basic-auth/user/#{password}")
+    parser.escape("http://httpbin.io/basic-auth/user/#{password}")
   end
   let(:pp) do
     <<-EOS
@@ -48,7 +48,7 @@
 
           describe file('/tmp/testfile') do
             it { is_expected.to be_file }
-            its(:content_as_json) { is_expected.to include('authenticated' => true, 'user' => 'user') }
+            its(:content_as_json) { is_expected.to include('authorized' => true, 'user' => 'user') }
           end
         end
       end