diff --git a/gems/aws-sdk-core/CHANGELOG.md b/gems/aws-sdk-core/CHANGELOG.md index 2c9adb6e8ae..73e349c8856 100644 --- a/gems/aws-sdk-core/CHANGELOG.md +++ b/gems/aws-sdk-core/CHANGELOG.md @@ -1,6 +1,8 @@ Unreleased Changes ------------------ +* Feature - Add support for `AWS_CONTAINER_CREDENTIALS_FULL_URI` and `AWS_CONTAINER_AUTHORIZATION_TOKEN` environment variables to `ECSCredentials`. + 3.170.1 (2023-03-17) ------------------ diff --git a/gems/aws-sdk-core/lib/aws-sdk-core/credential_provider_chain.rb b/gems/aws-sdk-core/lib/aws-sdk-core/credential_provider_chain.rb index 0f7f5570448..7574659c937 100644 --- a/gems/aws-sdk-core/lib/aws-sdk-core/credential_provider_chain.rb +++ b/gems/aws-sdk-core/lib/aws-sdk-core/credential_provider_chain.rb @@ -161,7 +161,8 @@ def assume_role_web_identity_credentials(options) def instance_profile_credentials(options) profile_name = determine_profile_name(options) - if ENV['AWS_CONTAINER_CREDENTIALS_RELATIVE_URI'] + if ENV['AWS_CONTAINER_CREDENTIALS_RELATIVE_URI'] || + ENV['AWS_CONTAINER_CREDENTIALS_FULL_URI'] ECSCredentials.new(options) else InstanceProfileCredentials.new(options.merge(profile: profile_name)) diff --git a/gems/aws-sdk-core/lib/aws-sdk-core/ecs_credentials.rb b/gems/aws-sdk-core/lib/aws-sdk-core/ecs_credentials.rb index 9dd1c79e7ff..f0f5d36e765 100644 --- a/gems/aws-sdk-core/lib/aws-sdk-core/ecs_credentials.rb +++ b/gems/aws-sdk-core/lib/aws-sdk-core/ecs_credentials.rb @@ -2,6 +2,7 @@ require 'time' require 'net/http' +require 'resolv' module Aws # An auto-refreshing credential provider that loads credentials from @@ -10,7 +11,6 @@ module Aws # ecs_credentials = Aws::ECSCredentials.new(retries: 3) # ec2 = Aws::EC2::Client.new(credentials: ecs_credentials) class ECSCredentials - include CredentialProvider include RefreshingCredentials @@ -29,16 +29,22 @@ class Non200Response < RuntimeError; end Errno::ENETUNREACH, SocketError, Timeout::Error, - Non200Response, - ] + Non200Response + ].freeze # @param [Hash] options # @option options [Integer] :retries (5) Number of times to retry # when retrieving credentials. - # @option options [String] :ip_address ('169.254.170.2') - # @option options [Integer] :port (80) + # @option options [String] :ip_address ('169.254.170.2') This value is + # ignored if `endpoint` is set and `credential_path` is not set. + # @option options [Integer] :port (80) This value is ignored if `endpoint` + # is set and `credential_path` is not set. # @option options [String] :credential_path By default, the value of the # AWS_CONTAINER_CREDENTIALS_RELATIVE_URI environment variable. + # @option options [String] :endpoint The ECS credential endpoint. + # By default, this is the value of the AWS_CONTAINER_CREDENTIALS_FULL_URI + # environment variable. This value is ignored if `credential_path` or + # ENV['AWS_CONTAINER_CREDENTIALS_RELATIVE_URI'] is set. # @option options [Float] :http_open_timeout (5) # @option options [Float] :http_read_timeout (5) # @option options [Numeric, Proc] :delay By default, failures are retried @@ -52,17 +58,15 @@ class Non200Response < RuntimeError; end # credentials are refreshed. `before_refresh` is called # with an instance of this object when # AWS credentials are required and need to be refreshed. - def initialize options = {} + def initialize(options = {}) + credential_path = options[:credential_path] || + ENV['AWS_CONTAINER_CREDENTIALS_RELATIVE_URI'] + endpoint = options[:endpoint] || + ENV['AWS_CONTAINER_CREDENTIALS_FULL_URI'] + initialize_uri(options, credential_path, endpoint) + @authorization_token = ENV['AWS_CONTAINER_AUTHORIZATION_TOKEN'] + @retries = options[:retries] || 5 - @ip_address = options[:ip_address] || '169.254.170.2' - @port = options[:port] || 80 - @credential_path = options[:credential_path] - @credential_path ||= ENV['AWS_CONTAINER_CREDENTIALS_RELATIVE_URI'] - unless @credential_path - raise ArgumentError.new( - "Cannot instantiate an ECS Credential Provider without a credential path." - ) - end @http_open_timeout = options[:http_open_timeout] || 5 @http_read_timeout = options[:http_read_timeout] || 5 @http_debug_output = options[:http_debug_output] @@ -77,11 +81,69 @@ def initialize options = {} private + def initialize_uri(options, credential_path, endpoint) + if credential_path + initialize_relative_uri(options, credential_path) + # Use FULL_URI/endpoint only if RELATIVE_URI/path is not set + elsif endpoint + initialize_full_uri(endpoint) + else + raise ArgumentError, + 'Cannot instantiate an ECS Credential Provider '\ + 'without a credential path or endpoint.' + end + end + + def initialize_relative_uri(options, path) + @host = options[:ip_address] || '169.254.170.2' + @port = options[:port] || 80 + @scheme = 'http' + @credential_path = path + end + + def initialize_full_uri(endpoint) + uri = URI.parse(endpoint) + validate_full_uri!(uri) + @host = uri.host + @port = uri.port + @scheme = uri.scheme + @credential_path = uri.path + end + + # Validate that the full URI is using a loopback address if scheme is http. + def validate_full_uri!(full_uri) + return unless full_uri.scheme == 'http' + + begin + return if ip_loopback?(IPAddr.new(full_uri.host)) + rescue IPAddr::InvalidAddressError + addresses = Resolv.getaddresses(full_uri.host) + return if addresses.all? { |addr| ip_loopback?(IPAddr.new(addr)) } + end + + raise ArgumentError, + 'AWS_CONTAINER_CREDENTIALS_FULL_URI must use a loopback '\ + 'address when using the http scheme.' + end + + # loopback? method is available in Ruby 2.5+ + # Replicate the logic here. + def ip_loopback?(ip_address) + case ip_address.family + when Socket::AF_INET + ip_address & 0xff000000 == 0x7f000000 + when Socket::AF_INET6 + ip_address == 1 + else + false + end + end + def backoff(backoff) case backoff when Proc then backoff - when Numeric then lambda { |_| sleep(backoff) } - else lambda { |num_failures| Kernel.sleep(1.2 ** num_failures) } + when Numeric then ->(_) { sleep(backoff) } + else ->(num_failures) { Kernel.sleep(1.2**num_failures) } end end @@ -89,68 +151,64 @@ def refresh # Retry loading credentials up to 3 times is the instance metadata # service is responding but is returning invalid JSON documents # in response to the GET profile credentials call. - begin - retry_errors([Aws::Json::ParseError, StandardError], max_retries: 3) do - c = Aws::Json.load(get_credentials.to_s) - @credentials = Credentials.new( - c['AccessKeyId'], - c['SecretAccessKey'], - c['Token'] - ) - @expiration = c['Expiration'] ? Time.iso8601(c['Expiration']) : nil - end - rescue Aws::Json::ParseError - raise Aws::Errors::MetadataParserError.new + + retry_errors([Aws::Json::ParseError, StandardError], max_retries: 3) do + c = Aws::Json.load(get_credentials.to_s) + @credentials = Credentials.new( + c['AccessKeyId'], + c['SecretAccessKey'], + c['Token'] + ) + @expiration = c['Expiration'] ? Time.iso8601(c['Expiration']) : nil end + rescue Aws::Json::ParseError + raise Aws::Errors::MetadataParserError end def get_credentials # Retry loading credentials a configurable number of times if # the instance metadata service is not responding. - begin - retry_errors(NETWORK_ERRORS, max_retries: @retries) do - open_connection do |conn| - http_get(conn, @credential_path) - end + + retry_errors(NETWORK_ERRORS, max_retries: @retries) do + open_connection do |conn| + http_get(conn, @credential_path) end - rescue - '{}' end + rescue StandardError + '{}' end def open_connection - http = Net::HTTP.new(@ip_address, @port, nil) + http = Net::HTTP.new(@host, @port, nil) http.open_timeout = @http_open_timeout http.read_timeout = @http_read_timeout http.set_debug_output(@http_debug_output) if @http_debug_output + http.use_ssl = @scheme == 'https' http.start yield(http).tap { http.finish } end def http_get(connection, path) - response = connection.request(Net::HTTP::Get.new(path)) - if response.code.to_i == 200 - response.body - else - raise Non200Response - end + request = Net::HTTP::Get.new(path) + request['Authorization'] = @authorization_token if @authorization_token + response = connection.request(request) + raise Non200Response unless response.code.to_i == 200 + + response.body end - def retry_errors(error_classes, options = {}, &block) + def retry_errors(error_classes, options = {}) max_retries = options[:max_retries] retries = 0 begin yield - rescue *error_classes => _error - if retries < max_retries - @backoff.call(retries) - retries += 1 - retry - else - raise - end + rescue *error_classes => _e + raise unless retries < max_retries + + @backoff.call(retries) + retries += 1 + retry end end - end end diff --git a/gems/aws-sdk-core/spec/aws/credential_provider_chain_spec.rb b/gems/aws-sdk-core/spec/aws/credential_provider_chain_spec.rb index 19e0e976992..164abe62037 100644 --- a/gems/aws-sdk-core/spec/aws/credential_provider_chain_spec.rb +++ b/gems/aws-sdk-core/spec/aws/credential_provider_chain_spec.rb @@ -123,6 +123,14 @@ def validate_credentials(expected_creds) expect(credentials).to be(mock_ecs_creds) end + it 'hydrates credentials from ECS when AWS_CONTAINER_CREDENTIALS_FULL_URI is set' do + ENV['AWS_CONTAINER_CREDENTIALS_FULL_URI'] = 'test_uri' + mock_ecs_creds = double('ECSCredentials') + expect(ECSCredentials).to receive(:new).and_return(mock_ecs_creds) + expect(mock_ecs_creds).to receive(:set?).and_return(true) + expect(credentials).to be(mock_ecs_creds) + end + describe 'with config set to nil' do let(:config) { nil } diff --git a/gems/aws-sdk-core/spec/aws/ecs_credentials_spec.rb b/gems/aws-sdk-core/spec/aws/ecs_credentials_spec.rb index 2bacbf63498..f26d01d406b 100644 --- a/gems/aws-sdk-core/spec/aws/ecs_credentials_spec.rb +++ b/gems/aws-sdk-core/spec/aws/ecs_credentials_spec.rb @@ -1,67 +1,61 @@ # frozen_string_literal: true -require 'spec_helper' +require_relative '../spec_helper' module Aws describe ECSCredentials do - let(:path) { '/latest/credentials?id=foobarbaz' } - describe 'without instance metadata service present' do - + context 'without ECS credential service present' do [ Errno::EHOSTUNREACH, Errno::ECONNREFUSED, SocketError, - Timeout::Error, + Timeout::Error ].each do |error_class| it "returns no credentials for #{error_class}" do stub_request(:get, "http://169.254.170.2#{path}").to_raise(error_class) - expect(ECSCredentials.new(credential_path: path, backoff:0).set?).to be(false) + expect(ECSCredentials.new(credential_path: path, backoff: 0).set?).to be(false) end end - end - describe 'with ECS credential service present' do - - before(:each) do - stub_const('ENV', { - "AWS_CONTAINER_CREDENTIALS_RELATIVE_URI" => path - }) + context 'with ECS credential service present' do + before do + ENV['AWS_CONTAINER_CREDENTIALS_RELATIVE_URI'] = path end let(:expiration) { Time.now.utc + 3600 } let(:expiration2) { expiration + 3600 } - let(:resp) { <<-JSON.strip } -{ - "RoleArn" : "arn:aws:iam::123456789012:role/BarFooRole", - "AccessKeyId" : "akid", - "SecretAccessKey" : "secret", - "Token" : "session-token", - "Expiration" : "#{expiration.strftime('%Y-%m-%dT%H:%M:%SZ')}" -} + let(:resp) { <<~JSON.strip } + { + "RoleArn" : "arn:aws:iam::123456789012:role/BarFooRole", + "AccessKeyId" : "akid", + "SecretAccessKey" : "secret", + "Token" : "session-token", + "Expiration" : "#{expiration.strftime('%Y-%m-%dT%H:%M:%SZ')}" + } JSON - let(:resp2) { <<-JSON.strip } -{ - "RoleArn" : "arn:aws:iam::123456789012:role/BarFooRole", - "AccessKeyId" : "akid-2", - "SecretAccessKey" : "secret-2", - "Token" : "session-token-2", - "Expiration" : "#{(expiration2).strftime('%Y-%m-%dT%H:%M:%SZ')}" -} + let(:resp2) { <<~JSON.strip } + { + "RoleArn" : "arn:aws:iam::123456789012:role/BarFooRole", + "AccessKeyId" : "akid-2", + "SecretAccessKey" : "secret-2", + "Token" : "session-token-2", + "Expiration" : "#{expiration2.strftime('%Y-%m-%dT%H:%M:%SZ')}" + } JSON before(:each) do - stub_request(:get, "http://169.254.170.2#{path}"). - to_return(:status => 200, :body => resp). - to_return(:status => 200, :body => resp2) + stub_request(:get, "http://169.254.170.2#{path}") + .to_return(status: 200, body: resp) + .to_return(status: 200, body: resp2) end it 'populates credentials from the instance profile' do - c = ECSCredentials.new(backoff:0) + c = ECSCredentials.new(backoff: 0) expect(c.credentials.access_key_id).to eq('akid') expect(c.credentials.secret_access_key).to eq('secret') expect(c.credentials.session_token).to eq('session-token') @@ -78,9 +72,9 @@ module Aws end it 'retries if the first load fails' do - stub_request(:get, "http://169.254.170.2#{path}"). - to_return(:status => 200, :body => resp2) - c = ECSCredentials.new(backoff:0) + stub_request(:get, "http://169.254.170.2#{path}") + .to_return(status: 200, body: resp2) + c = ECSCredentials.new(backoff: 0) expect(c.credentials.access_key_id).to eq('akid-2') expect(c.credentials.secret_access_key).to eq('secret-2') expect(c.credentials.session_token).to eq('session-token-2') @@ -88,12 +82,12 @@ module Aws end it 'retries if get profile response is invalid JSON' do - stub_request(:get, "http://169.254.170.2#{path}"). - to_return(:status => 200, :body => ' '). - to_return(:status => 200, :body => ''). - to_return(:status => 200, :body => '{'). - to_return(:status => 200, :body => resp2) - c = ECSCredentials.new(backoff:0) + stub_request(:get, "http://169.254.170.2#{path}") + .to_return(status: 200, body: ' ') + .to_return(status: 200, body: '') + .to_return(status: 200, body: '{') + .to_return(status: 200, body: resp2) + c = ECSCredentials.new(backoff: 0) expect(c.credentials.access_key_id).to eq('akid-2') expect(c.credentials.secret_access_key).to eq('secret-2') expect(c.credentials.session_token).to eq('session-token-2') @@ -101,32 +95,41 @@ module Aws end it 'retries invalid JSON exactly 3 times' do - stub_request(:get, "http://169.254.170.2#{path}"). - to_return(:status => 200, :body => ''). - to_return(:status => 200, :body => ' '). - to_return(:status => 200, :body => '{'). - to_return(:status => 200, :body => ' ') - expect { - ECSCredentials.new(backoff:0) - }.to raise_error( + stub_request(:get, "http://169.254.170.2#{path}") + .to_return(status: 200, body: '') + .to_return(status: 200, body: ' ') + .to_return(status: 200, body: '{') + .to_return(status: 200, body: ' ') + expect do + ECSCredentials.new(backoff: 0) + end.to raise_error( Aws::Errors::MetadataParserError, 'Failed to parse metadata service response.' ) end it 'retries errors parsing expiration time 3 times' do - stub_request(:get, "http://169.254.170.2#{path}"). - to_return(:status => 200, :body => '{ "Expiration": "Expiration" }'). - to_return(:status => 200, :body => '{ "Expiration": "Expiration" }'). - to_return(:status => 200, :body => '{ "Expiration": "Expiration" }'). - to_return(:status => 200, :body => '{ "Expiration": "Expiration" }') - expect { - ECSCredentials.new(backoff:0) - }.to raise_error(ArgumentError) + stub_request(:get, "http://169.254.170.2#{path}") + .to_return(status: 200, body: '{ "Expiration": "Expiration" }') + .to_return(status: 200, body: '{ "Expiration": "Expiration" }') + .to_return(status: 200, body: '{ "Expiration": "Expiration" }') + .to_return(status: 200, body: '{ "Expiration": "Expiration" }') + expect do + ECSCredentials.new(backoff: 0) + end.to raise_error(ArgumentError) end - describe 'auto refreshing' do + it "ignores ENV['AWS_CONTAINER_CREDENTIALS_FULL_URI'] if also set" do + # is not stubbed and should not be used + ENV['AWS_CONTAINER_CREDENTIALS_FULL_URI'] = 'https://amazon.com:1234/path' + c = ECSCredentials.new(backoff: 0) + expect(c.credentials.access_key_id).to eq('akid') + expect(c.credentials.secret_access_key).to eq('secret') + expect(c.credentials.session_token).to eq('session-token') + expect(c.expiration.to_s).to eq(expiration.to_s) + end + context 'auto refreshing' do # expire in 4 minutes let(:expiration) { Time.now.utc + 299 } @@ -137,18 +140,16 @@ module Aws expect(c.credentials.session_token).to eq('session-token-2') expect(c.expiration.to_s).to eq(expiration2.to_s) end - end - describe 'failure cases' do - + context 'failure cases' do let(:resp) { '{}' } it 'given an empty response, entry credentials are returned' do # This handles the case when the service response but returns # a JSON document without credentials (error cases) - stub_request(:get, "http://169.254.170.2#{path}"). - to_return(:status => 200, :body => resp) + stub_request(:get, "http://169.254.170.2#{path}") + .to_return(status: 200, body: resp) c = ECSCredentials.new expect(c.set?).to be(false) expect(c.credentials.access_key_id).to be(nil) @@ -157,36 +158,122 @@ module Aws expect(c.expiration).to be(nil) end + it 'raises if credential path or endpoint is not set' do + ENV['AWS_CONTAINER_CREDENTIALS_RELATIVE_URI'] = nil + expect do + ECSCredentials.new + end.to raise_error(ArgumentError, /without a credential path/) + end end - end - describe '#retries' do - + context 'retries' do before(:each) do - stub_const('ENV', { - "AWS_CONTAINER_CREDENTIALS_RELATIVE_URI" => path - }) + ENV['AWS_CONTAINER_CREDENTIALS_RELATIVE_URI'] = path end it 'defaults to 0' do stub_request(:get, "http://169.254.170.2#{path}").to_raise(SocketError) - expect(ECSCredentials.new(backoff:0).retries).to be(5) + expect(ECSCredentials.new(backoff: 0).retries).to be(5) end it 'keeps trying "retries" times, with exponential backoff' do - expected_request = stub_request(:get, "http://169.254.170.2#{path}"). - to_raise(Errno::ECONNREFUSED) + expected_request = stub_request(:get, "http://169.254.170.2#{path}") + .to_raise(Errno::ECONNREFUSED) expect(Kernel).to receive(:sleep).with(1) expect(Kernel).to receive(:sleep).with(2) expect(Kernel).to receive(:sleep).with(4) ECSCredentials.new( - backoff: lambda{|n| Kernel.sleep(2 ** n ) }, - retries:3 + backoff: ->(n) { Kernel.sleep(2**n) }, + retries: 3 ) - assert_requested(expected_request, times:4) + assert_requested(expected_request, times: 4) end + end + context 'AWS_CONTAINER_CREDENTIALS_FULL_URI' do + let(:full_uri) { 'https://amazon.com:1234/path' } + let(:loopback_uri) { 'http://localhost/path' } + let(:loopback_ip) { 'http://127.0.0.1/path' } + let(:loopback_ipv6) { 'http://[::1]/path' } + let(:expiration) { Time.now.utc + 3600 } + + let(:resp) { <<~JSON.strip } + { + "RoleArn" : "arn:aws:iam::123456789012:role/BarFooRole", + "AccessKeyId" : "akid-full", + "SecretAccessKey" : "secret-full", + "Token" : "session-token-full", + "Expiration" : "#{expiration.strftime('%Y-%m-%dT%H:%M:%SZ')}" + } + JSON + + before(:each) do + ENV['AWS_CONTAINER_CREDENTIALS_FULL_URI'] = full_uri + stub_request(:get, full_uri).to_return(status: 200, body: resp) + end + + it 'raises for an http URI that is not a loopback' do + ENV['AWS_CONTAINER_CREDENTIALS_FULL_URI'] = 'http://amazon.com/path' + expect(Resolv).to receive(:getaddresses).and_return( + %w[205.251.242.103 52.94.236.248 54.239.28.85] + ) + expect do + ECSCredentials.new(backoff: 0) + end.to raise_error(ArgumentError, /loopback/) + end + + it 'raises for an http IP that is not a loopback' do + ENV['AWS_CONTAINER_CREDENTIALS_FULL_URI'] = 'http://205.251.242.103/path' + expect do + ECSCredentials.new(backoff: 0) + end.to raise_error(ArgumentError, /loopback/) + end + + it 'uses the full uri if https' do + c = ECSCredentials.new(backoff: 0) + expect(c.credentials.access_key_id).to eq('akid-full') + expect(c.credentials.secret_access_key).to eq('secret-full') + expect(c.credentials.session_token).to eq('session-token-full') + expect(c.expiration.to_s).to eq(expiration.to_s) + end + + it 'uses an http URI if it is a loopback' do + ENV['AWS_CONTAINER_CREDENTIALS_FULL_URI'] = loopback_uri + stub_request(:get, loopback_uri).to_return(status: 200, body: resp) + c = ECSCredentials.new(backoff: 0) + expect(c.credentials.access_key_id).to eq('akid-full') + expect(c.credentials.secret_access_key).to eq('secret-full') + expect(c.credentials.session_token).to eq('session-token-full') + expect(c.expiration.to_s).to eq(expiration.to_s) + end + + it 'uses an http IP if it is a loopback' do + ENV['AWS_CONTAINER_CREDENTIALS_FULL_URI'] = loopback_ip + stub_request(:get, loopback_ip).to_return(status: 200, body: resp) + c = ECSCredentials.new(backoff: 0) + expect(c.credentials.access_key_id).to eq('akid-full') + expect(c.credentials.secret_access_key).to eq('secret-full') + expect(c.credentials.session_token).to eq('session-token-full') + expect(c.expiration.to_s).to eq(expiration.to_s) + end + + it 'uses an http IPv6 if it is a loopback' do + ENV['AWS_CONTAINER_CREDENTIALS_FULL_URI'] = loopback_ipv6 + stub_request(:get, loopback_ipv6).to_return(status: 200, body: resp) + c = ECSCredentials.new(backoff: 0) + expect(c.credentials.access_key_id).to eq('akid-full') + expect(c.credentials.secret_access_key).to eq('secret-full') + expect(c.credentials.session_token).to eq('session-token-full') + expect(c.expiration.to_s).to eq(expiration.to_s) + end + + it 'uses an authorization token if provided' do + ENV['AWS_CONTAINER_AUTHORIZATION_TOKEN'] = 'token' + ECSCredentials.new(backoff: 0) + expect(WebMock).to have_requested(:get, full_uri) + .with(headers: { 'Authorization' => 'token' }) + end end end end