Merge pull request #723 from rocky4570/awslambda-mods

lambderize the moto lambda

moto/awslambda/models.py

@@ -3,7 +3,15 @@ from __future__ import unicode_literals
 import base64
 import datetime
 import hashlib
+import io
 import json
+import sys
+import zipfile
+
+try:
+    from StringIO import StringIO
+except:
+    from io import StringIO
 
 import boto.awslambda
 from moto.core import BaseBackend
@@ -34,9 +42,18 @@ class LambdaFunction(object):
         self.version = '$LATEST'
         self.last_modified = datetime.datetime.utcnow().strftime('%Y-%m-%d %H:%M:%S')
         if 'ZipFile' in self.code:
-            code = base64.b64decode(self.code['ZipFile'])
+            # more hackery to handle unicode/bytes/str in python3 and python2 - argh!
-            self.code_size = len(code)
+            try:
-            self.code_sha_256 = hashlib.sha256(code).hexdigest()
+                to_unzip_code = base64.b64decode(bytes(self.code['ZipFile'], 'utf-8'))
+            except Exception:
+                to_unzip_code = base64.b64decode(self.code['ZipFile'])
+
+            zbuffer = io.BytesIO()
+            zbuffer.write(to_unzip_code)
+            zip_file = zipfile.ZipFile(zbuffer, 'r', zipfile.ZIP_DEFLATED)
+            self.code = zip_file.read("".join(zip_file.namelist()))
+            self.code_size = len(to_unzip_code)
+            self.code_sha_256 = hashlib.sha256(to_unzip_code).hexdigest()
         else:
             # validate s3 bucket
             try:
@@ -93,15 +110,56 @@ class LambdaFunction(object):
             "Configuration": self.get_configuration(),
         }
 
+    def convert(self, s):
+        try:
+            return str(s, encoding='utf8')
+        except:
+            return s
+
+    def is_json(self, test_str):
+        try:
+            response = json.loads(test_str)
+        except:
+            response = test_str
+        return response
+
+    def _invoke_lambda(self, code, event={}, context={}):
+        # TO DO: context not yet implemented
+        try:
+            mycode = "\n".join(['import json',
+                                self.convert(self.code),
+                                self.convert('print(lambda_handler(%s, %s))' % (self.is_json(self.convert(event)), context))])
+            #print("moto_lambda_debug: ", mycode)
+        except Exception as ex:
+            print("Exception %s", ex)
+
+        try:
+            codeOut = StringIO()
+            codeErr = StringIO()
+            sys.stdout = codeOut
+            sys.stderr = codeErr
+            exec(mycode)
+            exec_err = codeErr.getvalue()
+            exec_out = codeOut.getvalue()
+            result = "\n".join([exec_out, self.convert(exec_err)])
+        except Exception as ex:
+            result = '%s\n\n\nException %s' % (mycode, ex)
+        finally:
+            codeErr.close()
+            codeOut.close()
+            sys.stdout = sys.__stdout__
+            sys.stderr = sys.__stderr__
+        return self.convert(result)
+
     def invoke(self, request, headers):
         payload = dict()
 
         # Get the invocation type:
+        r = self._invoke_lambda(code=self.code, event=request.body)
         if request.headers.get("x-amz-invocation-type") == "RequestResponse":
-            encoded = base64.b64encode("Some log file output...".encode('utf-8'))
+            encoded = base64.b64encode(r.encode('utf-8'))
             headers["x-amz-log-result"] = encoded.decode('utf-8')
-
+            payload['result'] = headers["x-amz-log-result"]
-            payload["result"] = "Good"
 
         return json.dumps(payload, indent=4)
 
@@ -154,3 +212,7 @@ class LambdaBackend(BaseBackend):
 lambda_backends = {}
 for region in boto.awslambda.regions():
     lambda_backends[region.name] = LambdaBackend()
+
+# Handle us forgotten regions, unless Lambda truly only runs out of US and EU?????
+for region in ['ap-southeast-2']:
+    lambda_backends[region] = LambdaBackend()

moto/awslambda/responses.py

@@ -43,7 +43,7 @@ class LambdaResponse(BaseResponse):
         if lambda_backend.has_function(function_name):
             fn = lambda_backend.get_function(function_name)
             payload = fn.invoke(request, headers)
-            return 200, headers, payload
+            return 202, headers, payload
         else:
             return 404, headers, "{}"
 

tests/test_awslambda/test_lambda.py

@@ -1,51 +1,66 @@
 from __future__ import unicode_literals
 
+import base64
 import botocore.client
 import boto3
 import hashlib
 import io
+import json
 import zipfile
 import sure  # noqa
 
 from freezegun import freeze_time
-from moto import mock_lambda, mock_s3
+from moto import mock_lambda, mock_s3, mock_ec2
 
 
-def get_test_zip_file():
+def _process_lamda(pfunc):
     zip_output = io.BytesIO()
-    zip_file = zipfile.ZipFile(zip_output, 'w')
+    zip_file = zipfile.ZipFile(zip_output, 'w', zipfile.ZIP_DEFLATED)
-    zip_file.writestr('lambda_function.py', b'''\
+    zip_file.writestr('lambda_function.zip', pfunc)
-def handler(event, context):
-    return "hello world"
-''')
     zip_file.close()
     zip_output.seek(0)
     return zip_output.read()
 
 
-@mock_lambda
+def get_test_zip_file1():
-def test_list_functions():
+    pfunc = """
-    conn = boto3.client('lambda', 'us-west-2')
+def lambda_handler(event, context):
+    return (event, context)
+"""
+    return _process_lamda(pfunc)
 
-    result = conn.list_functions()
 
-    result['Functions'].should.have.length_of(0)
+def get_test_zip_file2():
+    pfunc = """
+def lambda_handler(event, context):
+    volume_id = event.get('volume_id')
+    print('get volume details for %s' % volume_id)
+    import boto3
+    ec2 = boto3.resource('ec2', region_name='us-west-2')
+    vol = ec2.Volume(volume_id)
+    print('Volume - %s  state=%s, size=%s' % (volume_id, vol.state, vol.size))
+"""
+    return _process_lamda(pfunc)
 
 
 @mock_lambda
 @mock_s3
-@freeze_time('2015-01-01 00:00:00')
+def test_list_functions():
-def test_invoke_function():
     conn = boto3.client('lambda', 'us-west-2')
+    result = conn.list_functions()
+    result['Functions'].should.have.length_of(0)
 
-    zip_content = get_test_zip_file()
+@mock_lambda
+@freeze_time('2015-01-01 00:00:00')
+def test_invoke_event_function():
+    conn = boto3.client('lambda', 'us-west-2')
     conn.create_function(
         FunctionName='testFunction',
         Runtime='python2.7',
         Role='test-iam-role',
         Handler='lambda_function.handler',
         Code={
-            'ZipFile': zip_content,
+            'ZipFile': get_test_zip_file1(),
         },
         Description='test lambda function',
         Timeout=3,
@@ -53,8 +68,8 @@ def test_invoke_function():
         Publish=True,
     )
 
-    success_result = conn.invoke(FunctionName='testFunction', InvocationType='Event', Payload='{}')
+    success_result = conn.invoke(FunctionName='testFunction', InvocationType='Event', Payload=json.dumps({'msg': 'Mostly Harmless'}))
-    success_result["StatusCode"].should.equal(200)
+    success_result["StatusCode"].should.equal(202)
 
     conn.invoke.when.called_with(
         FunctionName='notAFunction',
@@ -62,11 +77,63 @@ def test_invoke_function():
         Payload='{}'
     ).should.throw(botocore.client.ClientError)
 
-    success_result = conn.invoke(FunctionName='testFunction', InvocationType='RequestResponse', Payload='{}')
+
-    success_result["StatusCode"].should.equal(200)
+@mock_lambda
+@freeze_time('2015-01-01 00:00:00')
+def test_invoke_requestresponse_function():
+    conn = boto3.client('lambda', 'us-west-2')
+    conn.create_function(
+        FunctionName='testFunction',
+        Runtime='python2.7',
+        Role='test-iam-role',
+        Handler='lambda_function.handler',
+        Code={
+            'ZipFile': get_test_zip_file1(),
+        },
+        Description='test lambda function',
+        Timeout=3,
+        MemorySize=128,
+        Publish=True,
+    )
+
+    success_result = conn.invoke(FunctionName='testFunction', InvocationType='RequestResponse',
+                                 Payload=json.dumps({'msg': 'So long and thanks for all the fish'}))
+    success_result["StatusCode"].should.equal(202)
+
+    #nasty hack - hope someone has better solution dealing with unicode tests working for Py2 and Py3.
+    base64.b64decode(success_result["LogResult"]).decode('utf-8').replace("u'", "'").should.equal("({'msg': 'So long and thanks for all the fish'}, {})\n\n")
+
+@mock_ec2
+@mock_lambda
+@freeze_time('2015-01-01 00:00:00')
+def test_invoke_function_get_ec2_volume():
+    conn = boto3.resource("ec2", "us-west-2")
+    vol = conn.create_volume(Size=99, AvailabilityZone='us-west-2')
+    vol = conn.Volume(vol.id)
+
+    conn = boto3.client('lambda', 'us-west-2')
+    conn.create_function(
+        FunctionName='testFunction',
+        Runtime='python2.7',
+        Role='test-iam-role',
+        Handler='lambda_function.handler',
+        Code={
+            'ZipFile': get_test_zip_file2(),
+        },
+        Description='test lambda function',
+        Timeout=3,
+        MemorySize=128,
+        Publish=True,
+    )
+
+    import json
+    success_result = conn.invoke(FunctionName='testFunction', InvocationType='RequestResponse', Payload=json.dumps({'volume_id': vol.id}))
+    success_result["StatusCode"].should.equal(202)
 
     import base64
-    base64.b64decode(success_result["LogResult"]).decode('utf-8').should.equal("Some log file output...")
+    msg = 'get volume details for %s\nVolume - %s  state=%s, size=%s\nNone\n\n' % (vol.id, vol.id, vol.state, vol.size)
+    # yet again hacky solution to allow code to run tests for python2 and python3 - pls someone fix :(
+    base64.b64decode(success_result["LogResult"]).decode('utf-8').replace("u'", "'").should.equal(msg)
 
 
 @mock_lambda
@@ -100,8 +167,8 @@ def test_create_based_on_s3_with_missing_bucket():
 def test_create_function_from_aws_bucket():
     s3_conn = boto3.client('s3', 'us-west-2')
     s3_conn.create_bucket(Bucket='test-bucket')
+    zip_content = get_test_zip_file2()
 
-    zip_content = get_test_zip_file()
     s3_conn.put_object(Bucket='test-bucket', Key='test.zip', Body=zip_content)
     conn = boto3.client('lambda', 'us-west-2')
 
@@ -124,6 +191,7 @@ def test_create_function_from_aws_bucket():
         },
     )
     result['ResponseMetadata'].pop('HTTPHeaders', None)  # this is hard to match against, so remove it
+    result['ResponseMetadata'].pop('RetryAttempts', None)  # Botocore inserts retry attempts not seen in Python27
     result.should.equal({
         'FunctionName': 'testFunction',
         'FunctionArn': 'arn:aws:lambda:123456789012:function:testFunction',
@@ -142,7 +210,6 @@ def test_create_function_from_aws_bucket():
             "SubnetIds": ["subnet-123abc"],
             "VpcId": "vpc-123abc"
         },
-
         'ResponseMetadata': {'HTTPStatusCode': 201},
     })
 
@@ -151,8 +218,7 @@ def test_create_function_from_aws_bucket():
 @freeze_time('2015-01-01 00:00:00')
 def test_create_function_from_zipfile():
     conn = boto3.client('lambda', 'us-west-2')
-
+    zip_content = get_test_zip_file1()
-    zip_content = get_test_zip_file()
     result = conn.create_function(
         FunctionName='testFunction',
         Runtime='python2.7',
@@ -167,6 +233,8 @@ def test_create_function_from_zipfile():
         Publish=True,
     )
     result['ResponseMetadata'].pop('HTTPHeaders', None)  # this is hard to match against, so remove it
+    result['ResponseMetadata'].pop('RetryAttempts', None)  # Botocore inserts retry attempts not seen in Python27
+
     result.should.equal({
         'FunctionName': 'testFunction',
         'FunctionArn': 'arn:aws:lambda:123456789012:function:testFunction',
@@ -196,7 +264,7 @@ def test_get_function():
     s3_conn = boto3.client('s3', 'us-west-2')
     s3_conn.create_bucket(Bucket='test-bucket')
 
-    zip_content = get_test_zip_file()
+    zip_content = get_test_zip_file1()
     s3_conn.put_object(Bucket='test-bucket', Key='test.zip', Body=zip_content)
     conn = boto3.client('lambda', 'us-west-2')
 
@@ -217,6 +285,7 @@ def test_get_function():
 
     result = conn.get_function(FunctionName='testFunction')
     result['ResponseMetadata'].pop('HTTPHeaders', None)  # this is hard to match against, so remove it
+    result['ResponseMetadata'].pop('RetryAttempts', None)  # Botocore inserts retry attempts not seen in Python27
 
     result.should.equal({
         "Code": {
@@ -245,14 +314,13 @@ def test_get_function():
     })
 
 
-
 @mock_lambda
 @mock_s3
 def test_delete_function():
     s3_conn = boto3.client('s3', 'us-west-2')
     s3_conn.create_bucket(Bucket='test-bucket')
 
-    zip_content = get_test_zip_file()
+    zip_content = get_test_zip_file2()
     s3_conn.put_object(Bucket='test-bucket', Key='test.zip', Body=zip_content)
     conn = boto3.client('lambda', 'us-west-2')
 
@@ -273,6 +341,8 @@ def test_delete_function():
 
     success_result = conn.delete_function(FunctionName='testFunction')
     success_result['ResponseMetadata'].pop('HTTPHeaders', None)  # this is hard to match against, so remove it
+    success_result['ResponseMetadata'].pop('RetryAttempts', None)  # Botocore inserts retry attempts not seen in Python27
+
     success_result.should.equal({'ResponseMetadata': {'HTTPStatusCode': 204}})
 
     conn.delete_function.when.called_with(FunctionName='testFunctionThatDoesntExist').should.throw(botocore.client.ClientError)
@@ -289,7 +359,7 @@ def test_list_create_list_get_delete_list():
     s3_conn = boto3.client('s3', 'us-west-2')
     s3_conn.create_bucket(Bucket='test-bucket')
 
-    zip_content = get_test_zip_file()
+    zip_content = get_test_zip_file2()
     s3_conn.put_object(Bucket='test-bucket', Key='test.zip', Body=zip_content)
     conn = boto3.client('lambda', 'us-west-2')
 
@@ -338,6 +408,8 @@ def test_list_create_list_get_delete_list():
 
     func = conn.get_function(FunctionName='testFunction')
     func['ResponseMetadata'].pop('HTTPHeaders', None)  # this is hard to match against, so remove it
+    func['ResponseMetadata'].pop('RetryAttempts', None)  # Botocore inserts retry attempts not seen in Python27
+
     func.should.equal(expected_function_result)
     conn.delete_function(FunctionName='testFunction')
 

tests/test_cloudformation/test_cloudformation_stack_integration.py

@@ -1,6 +1,7 @@
 from __future__ import unicode_literals
 import json
 
+import base64
 import boto
 import boto.cloudformation
 import boto.datapipeline
@@ -1724,10 +1725,29 @@ def test_datapipeline():
     stack_resources.should.have.length_of(1)
     stack_resources[0].physical_resource_id.should.equal(data_pipelines['pipelineIdList'][0]['id'])
 
+def _process_lamda(pfunc):
+    import io
+    import zipfile
+    zip_output = io.BytesIO()
+    zip_file = zipfile.ZipFile(zip_output, 'w', zipfile.ZIP_DEFLATED)
+    zip_file.writestr('lambda_function.zip', pfunc)
+    zip_file.close()
+    zip_output.seek(0)
+    return zip_output.read()
+
+
+def get_test_zip_file1():
+    pfunc = """
+def lambda_handler(event, context):
+    return (event, context)
+"""
+    return _process_lamda(pfunc)
+
 
 @mock_cloudformation
 @mock_lambda
 def test_lambda_function():
+    # switch this to python as backend lambda only supports python execution.
     conn = boto3.client('lambda', 'us-east-1')
     template = {
         "AWSTemplateFormatVersion": "2010-09-09",
@@ -1736,22 +1756,15 @@ def test_lambda_function():
                 "Type": "AWS::Lambda::Function",
                 "Properties": {
                     "Code": {
-                        "ZipFile": {"Fn::Join": [
+                        "ZipFile": base64.b64encode(get_test_zip_file1()).decode('utf-8')
-                            "\n",
-                            """
-                            exports.handler = function(event, context) {
-                                context.succeed();
-                            }
-                            """.splitlines()
-                        ]}
                     },
-                    "Handler": "index.handler",
+                    "Handler": "lambda_function.handler",
                     "Description": "Test function",
                     "MemorySize": 128,
                     "Role": "test-role",
-                    "Runtime": "nodejs",
+                    "Runtime": "python2.7"
+                }
             }
-            },
         }
     }
 
@@ -1765,10 +1778,10 @@ def test_lambda_function():
     result = conn.list_functions()
     result['Functions'].should.have.length_of(1)
     result['Functions'][0]['Description'].should.equal('Test function')
-    result['Functions'][0]['Handler'].should.equal('index.handler')
+    result['Functions'][0]['Handler'].should.equal('lambda_function.handler')
     result['Functions'][0]['MemorySize'].should.equal(128)
     result['Functions'][0]['Role'].should.equal('test-role')
-    result['Functions'][0]['Runtime'].should.equal('nodejs')
+    result['Functions'][0]['Runtime'].should.equal('python2.7')
 
 
 @mock_cloudformation