Skip to content

Commit

Permalink
Remove remote paramfile functionality
Browse files Browse the repository at this point in the history 
The only special behavior now is if a param
starts with file:// or fileb://.

We can evaluate if we want some option to local paramfile
functionality, but for now the simplified version is that
there's no http/https auto-fetching, and file/fileb is always
on.

CLIv2: aws#3587
  • Loading branch information
@jamesls
jamesls committed Sep 19, 2018
1 parent 904ab77 commit b8f2d1a
Show file tree
Hide file tree
Showing 5 changed files with 10 additions and 409 deletions.
154 changes: 2 additions & 152 deletions awscli/paramfile.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -14,9 +14,6 @@
import os
import copy

from botocore.awsrequest import AWSRequest
from botocore.httpsession import URLLib3Session
from botocore.exceptions import ProfileNotFound
from awscli.compat import six

from awscli.compat import compat_open
Expand All @@ -25,150 +22,23 @@

logger = logging.getLogger(__name__)

# These are special cased arguments that do _not_ get the
# special param file processing. This is typically because it
# refers to an actual URI of some sort and we don't want to actually
# download the content (i.e TemplateURL in cloudformation).
PARAMFILE_DISABLED = set([
'api-gateway.put-integration.uri',
'appstream.create-stack.redirect-url',
'appstream.update-stack.redirect-url',
'cloudformation.create-stack.template-url',
'cloudformation.update-stack.template-url',
'cloudformation.create-stack-set.template-url',
'cloudformation.update-stack-set.template-url',
'cloudformation.create-change-set.template-url',
'cloudformation.validate-template.template-url',
'cloudformation.estimate-template-cost.template-url',
'cloudformation.get-template-summary.template-url',

'cloudformation.create-stack.stack-policy-url',
'cloudformation.update-stack.stack-policy-url',
'cloudformation.set-stack-policy.stack-policy-url',
# aws cloudformation package --template-file
'custom.package.template-file',
# aws cloudformation deploy --template-file
'custom.deploy.template-file',

'cloudformation.update-stack.stack-policy-during-update-url',
# We will want to change the event name to ``s3`` as opposed to
# custom in the near future along with ``s3`` to ``s3api``.
'custom.cp.website-redirect',
'custom.mv.website-redirect',
'custom.sync.website-redirect',

'guardduty.create-ip-set.location',
'guardduty.update-ip-set.location',
'guardduty.create-threat-intel-set.location',
'guardduty.update-threat-intel-set.location',
'comprehend.detect-dominant-language.text',
'comprehend.batch-detect-dominant-language.text-list',
'comprehend.detect-entities.text',
'comprehend.batch-detect-entities.text-list',
'comprehend.detect-key-phrases.text',
'comprehend.batch-detect-key-phrases.text-list',
'comprehend.detect-sentiment.text',
'comprehend.batch-detect-sentiment.text-list',

'iam.create-open-id-connect-provider.url',

'machine-learning.predict.predict-endpoint',

'mediatailor.put-playback-configuration.ad-decision-server-url',
'mediatailor.put-playback-configuration.slate-ad-url',
'mediatailor.put-playback-configuration.video-content-source-url',

'rds.copy-db-cluster-snapshot.pre-signed-url',
'rds.create-db-cluster.pre-signed-url',
'rds.copy-db-snapshot.pre-signed-url',
'rds.create-db-instance-read-replica.pre-signed-url',

'serverlessapplicationrepository.create-application.home-page-url',
'serverlessapplicationrepository.create-application.license-url',
'serverlessapplicationrepository.create-application.readme-url',
'serverlessapplicationrepository.create-application.source-code-url',
'serverlessapplicationrepository.create-application.template-url',
'serverlessapplicationrepository.create-application-version.source-code-url',
'serverlessapplicationrepository.create-application-version.template-url',
'serverlessapplicationrepository.update-application.home-page-url',
'serverlessapplicationrepository.update-application.readme-url',

'sqs.add-permission.queue-url',
'sqs.change-message-visibility.queue-url',
'sqs.change-message-visibility-batch.queue-url',
'sqs.delete-message.queue-url',
'sqs.delete-message-batch.queue-url',
'sqs.delete-queue.queue-url',
'sqs.get-queue-attributes.queue-url',
'sqs.list-dead-letter-source-queues.queue-url',
'sqs.receive-message.queue-url',
'sqs.remove-permission.queue-url',
'sqs.send-message.queue-url',
'sqs.send-message-batch.queue-url',
'sqs.set-queue-attributes.queue-url',
'sqs.purge-queue.queue-url',
'sqs.list-queue-tags.queue-url',
'sqs.tag-queue.queue-url',
'sqs.untag-queue.queue-url',

's3.copy-object.website-redirect-location',
's3.create-multipart-upload.website-redirect-location',
's3.put-object.website-redirect-location',

# Double check that this has been renamed!
'sns.subscribe.notification-endpoint',

'iot.create-job.document-source',
'translate.translate-text.text',

'workdocs.create-notification-subscription.notification-endpoint'
])


class ResourceLoadingError(Exception):
pass


def register_uri_param_handler(session, **kwargs):
prefix_map = copy.deepcopy(LOCAL_PREFIX_MAP)
try:
fetch_url = session.get_scoped_config().get(
'cli_follow_urlparam', 'true') == 'true'
except ProfileNotFound:
# If a --profile is provided that does not exist, loading
# a value from get_scoped_config will crash the CLI.
# This function can be called as the first handler for
# the session-initialized event, which happens before a
# profile can be created, even if the command would have
# successfully created a profile. Instead of crashing here
# on a ProfileNotFound the CLI should just use 'none'.
fetch_url = True

if fetch_url:
prefix_map.update(REMOTE_PREFIX_MAP)

handler = URIArgumentHandler(prefix_map)
session.register('load-cli-arg', handler)


class URIArgumentHandler(object):
def __init__(self, prefixes=None):
if prefixes is None:
prefixes = copy.deepcopy(LOCAL_PREFIX_MAP)
prefixes.update(REMOTE_PREFIX_MAP)
def __init__(self, prefixes):
self._prefixes = prefixes

def __call__(self, event_name, param, value, **kwargs):
"""Handler that supports param values from URIs."""
cli_argument = param
qualified_param_name = '.'.join(event_name.split('.')[1:])
if qualified_param_name in PARAMFILE_DISABLED or \
getattr(cli_argument, 'no_paramfile', None):
return
else:
return self._check_for_uri_param(cli_argument, value)

def _check_for_uri_param(self, param, value):
"""Handler that supports param values from local files."""
if isinstance(value, list) and len(value) == 1:
value = value[0]
try:
Expand Down Expand Up @@ -224,27 +94,7 @@ def get_file(prefix, path, mode):
path, e))


def get_uri(prefix, uri):
try:
session = URLLib3Session()
r = session.send(AWSRequest('GET', uri).prepare())
if r.status_code == 200:
return r.text
else:
raise ResourceLoadingError(
"received non 200 status code of %s" % (
r.status_code))
except Exception as e:
raise ResourceLoadingError('Unable to retrieve %s: %s' % (uri, e))


LOCAL_PREFIX_MAP = {
'file://': (get_file, {'mode': 'r'}),
'fileb://': (get_file, {'mode': 'rb'}),
}


REMOTE_PREFIX_MAP = {
'http://': (get_uri, {}),
'https://': (get_uri, {}),
}
138 changes: 4 additions & 134 deletions tests/functional/test_paramfile.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -20,20 +20,14 @@
logger = logging.getLogger(__name__)


class FakeResponse(object):
def __init__(self, content):
self.status_code = 200
self.text = content


class BaseTestCLIFollowParamURL(BaseAWSCommandParamsTest):
class BaseTestCLIFollowParamFile(BaseAWSCommandParamsTest):
def setUp(self):
super(BaseTestCLIFollowParamURL, self).setUp()
super(BaseTestCLIFollowParamFile, self).setUp()
self.files = FileCreator()
self.prefix = 'lambda get-function --function-name'

def tearDown(self):
super(BaseTestCLIFollowParamURL, self).tearDown()
super(BaseTestCLIFollowParamFile, self).tearDown()
self.files.remove_all()

def assert_param_expansion_is_correct(self, provided_param, expected_param):
Expand All @@ -48,137 +42,13 @@ def assert_param_expansion_is_correct(self, provided_param, expected_param):
expected_rc=ANY)


class TestCLIFollowParamURLDefault(BaseTestCLIFollowParamURL):
class TestCLIFollowParamFileDefault(BaseTestCLIFollowParamFile):
def test_does_not_prefixes_when_none_in_param(self):
self.assert_param_expansion_is_correct(
provided_param='foobar',
expected_param='foobar'
)

@patch('awscli.paramfile.URLLib3Session.send')
def test_does_use_http_prefix(self, mock_send):
content = 'http_content'
mock_send.return_value = FakeResponse(content=content)
param = 'http://foobar.com'
self.assert_param_expansion_is_correct(
provided_param=param,
expected_param=content
)

@patch('awscli.paramfile.URLLib3Session.send')
def test_does_use_https_prefix(self, mock_send):
content = 'https_content'
mock_send.return_value = FakeResponse(content=content)
param = 'https://foobar.com'
self.assert_param_expansion_is_correct(
provided_param=param,
expected_param=content
)

def test_does_use_file_prefix(self):
path = self.files.create_file('foobar.txt', 'file content')
param = 'file://%s' % path
self.assert_param_expansion_is_correct(
provided_param=param,
expected_param='file content'
)

def test_does_use_fileb_prefix(self):
# The command will fail with error code 255 since bytes type does
# not work for this parameter, however we still record the raw
# parameter that we passed, which is all this test is concerend about.
path = self.files.create_file('foobar.txt', b'file content', mode='wb')
param = 'fileb://%s' % path
self.assert_param_expansion_is_correct(
provided_param=param,
expected_param=b'file content'
)


class TestCLIFollowParamURLDisabled(BaseTestCLIFollowParamURL):
def setUp(self):
super(TestCLIFollowParamURLDisabled, self).setUp()
self.environ['AWS_CONFIG_FILE'] = self.files.create_file(
'config',
'[default]\ncli_follow_urlparam = false\n')
self.driver = create_clidriver()

def test_does_not_prefixes_when_none_in_param(self):
param = 'foobar'
self.assert_param_expansion_is_correct(
provided_param=param,
expected_param=param
)

@patch('awscli.paramfile.URLLib3Session.send')
def test_does_not_use_http_prefix(self, mock_send):
param = 'http://foobar'
self.assert_param_expansion_is_correct(
provided_param=param,
expected_param=param
)
mock_send.assert_not_called()

@patch('awscli.paramfile.URLLib3Session.send')
def test_does_not_use_https_prefix(self, mock_send):
param = 'https://foobar'
self.assert_param_expansion_is_correct(
provided_param=param,
expected_param=param
)
mock_send.assert_not_called()

def test_does_use_file_prefix(self):
path = self.files.create_file('foobar.txt', 'file content')
param = 'file://%s' % path
self.assert_param_expansion_is_correct(
provided_param=param,
expected_param='file content'
)

def test_does_use_fileb_prefix(self):
# The command will fail with error code 255 since bytes type does
# not work for this parameter, however we still record the raw
# parameter that we passed, which is all this test is concerend about.
path = self.files.create_file('foobar.txt', b'file content', mode='wb')
param = 'fileb://%s' % path
self.assert_param_expansion_is_correct(
provided_param=param,
expected_param=b'file content'
)


class TestCLIFollowParamURLEnabled(BaseTestCLIFollowParamURL):
def setUp(self):
super(TestCLIFollowParamURLEnabled, self).setUp()
self.environ['AWS_CONFIG_FILE'] = self.files.create_file(
'config',
'[default]\ncli_follow_urlparam = true\n')
self.driver = create_clidriver()

def test_does_not_prefixes_when_none_in_param(self):
self.assert_param_expansion_is_correct('foobar', 'foobar')

@patch('awscli.paramfile.URLLib3Session.send')
def test_does_use_http_prefix(self, mock_send):
content = 'http_content'
mock_send.return_value = FakeResponse(content=content)
param = 'http://foobar.com'
self.assert_param_expansion_is_correct(
provided_param=param,
expected_param=content
)

@patch('awscli.paramfile.URLLib3Session.send')
def test_does_use_https_prefix(self, mock_send):
content = 'https_content'
mock_send.return_value = FakeResponse(content=content)
param = 'https://foobar.com'
self.assert_param_expansion_is_correct(
provided_param=param,
expected_param=content
)

def test_does_use_file_prefix(self):
path = self.files.create_file('foobar.txt', 'file content')
param = 'file://%s' % path
Expand Down
24 changes: 2 additions & 22 deletions tests/unit/test_argprocess.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -27,7 +27,7 @@
from awscli.argprocess import ParamShorthandDocGen
from awscli.argprocess import ParamError
from awscli.argprocess import ParamUnknownKeyError
from awscli.paramfile import URIArgumentHandler
from awscli.paramfile import URIArgumentHandler, LOCAL_PREFIX_MAP
from awscli.arguments import CustomArgument, CLIArgument
from awscli.arguments import ListArgument, BooleanArgument
from awscli.arguments import create_argument_model_from_schema
Expand Down Expand Up @@ -70,7 +70,7 @@ def create_argument(self, argument_model, argument_name=None):
class TestURIParams(BaseArgProcessTest):
def setUp(self):
super(TestURIParams, self).setUp()
self.uri_param = URIArgumentHandler()
self.uri_param = URIArgumentHandler(LOCAL_PREFIX_MAP.copy())

def test_uri_param(self):
p = self.get_param_model('ec2.DescribeInstances.Filters')
Expand All @@ -83,26 +83,6 @@ def test_uri_param(self):
result = self.uri_param('event-name', p, 'file://%s' % f.name)
self.assertEqual(result, json_argument)

def test_uri_param_no_paramfile_false(self):
p = self.get_param_model('ec2.DescribeInstances.Filters')
p.no_paramfile = False
with temporary_file('r+') as f:
json_argument = json.dumps([{"Name": "instance-id", "Values": ["i-1234"]}])
f.write(json_argument)
f.flush()
result = self.uri_param('event-name', p, 'file://%s' % f.name)
self.assertEqual(result, json_argument)

def test_uri_param_no_paramfile_true(self):
p = self.get_param_model('ec2.DescribeInstances.Filters')
p.no_paramfile = True
with temporary_file('r+') as f:
json_argument = json.dumps([{"Name": "instance-id", "Values": ["i-1234"]}])
f.write(json_argument)
f.flush()
result = self.uri_param('event-name', p, 'file://%s' % f.name)
self.assertEqual(result, None)


class TestArgShapeDetection(BaseArgProcessTest):

Expand Down
Loading

0 comments on commit b8f2d1a

Please sign in to comment.