diff --git a/moto/awslambda/models.py b/moto/awslambda/models.py
index 9fc41c11e..f291f3beb 100644
--- a/moto/awslambda/models.py
+++ b/moto/awslambda/models.py
@@ -30,7 +30,7 @@ from moto.s3.models import s3_backend
 from moto.logs.models import logs_backends
 from moto.s3.exceptions import MissingBucket, MissingKey
 from moto import settings
-from .utils import make_function_arn
+from .utils import make_function_arn, make_function_ver_arn
 
 logger = logging.getLogger(__name__)
 
@@ -215,12 +215,12 @@ class LambdaFunction(BaseModel):
                 self.code_size = key.size
                 self.code_sha_256 = hashlib.sha256(key.value).hexdigest()
 
-        self.function_arn = make_function_arn(self.region, ACCOUNT_ID, self.function_name, version)
+        self.function_arn = make_function_arn(self.region, ACCOUNT_ID, self.function_name)
 
         self.tags = dict()
 
     def set_version(self, version):
-        self.function_arn = make_function_arn(self.region, ACCOUNT_ID, self.function_name, version)
+        self.function_arn = make_function_ver_arn(self.region, ACCOUNT_ID, self.function_name, version)
         self.version = version
         self.last_modified = datetime.datetime.utcnow().strftime('%Y-%m-%d %H:%M:%S')
 
@@ -503,7 +503,10 @@ class LambdaStorage(object):
     def list_versions_by_function(self, name):
         if name not in self._functions:
             return None
-        return [self._functions[name]['latest']]
+
+        latest = copy.copy(self._functions[name]['latest'])
+        latest.function_arn += ':$LATEST'
+        return [latest] + self._functions[name]['versions']
 
     def get_arn(self, arn):
         return self._arns.get(arn, None)
@@ -535,6 +538,7 @@ class LambdaStorage(object):
         fn.set_version(new_version)
 
         self._functions[name]['versions'].append(fn)
+        self._arns[fn.function_arn] = fn
         return fn
 
     def del_function(self, name, qualifier=None):
@@ -604,6 +608,9 @@ class LambdaBackend(BaseBackend):
 
         self._lambdas.put_function(fn)
 
+        if spec.get('Publish'):
+            ver = self.publish_function(function_name)
+            fn.version = ver.version
         return fn
 
     def publish_function(self, function_name):
diff --git a/moto/awslambda/responses.py b/moto/awslambda/responses.py
index 1c43ef84b..c29c9acd9 100644
--- a/moto/awslambda/responses.py
+++ b/moto/awslambda/responses.py
@@ -150,7 +150,7 @@ class LambdaResponse(BaseResponse):
 
         for fn in self.lambda_backend.list_functions():
             json_data = fn.get_configuration()
-
+            json_data['Version'] = '$LATEST'
             result['Functions'].append(json_data)
 
         return 200, {}, json.dumps(result)
@@ -204,7 +204,10 @@ class LambdaResponse(BaseResponse):
 
         if fn:
             code = fn.get_code()
-
+            if qualifier is None or qualifier == '$LATEST':
+                code['Configuration']['Version'] = '$LATEST'
+            if qualifier == '$LATEST':
+                code['Configuration']['FunctionArn'] += ':$LATEST'
             return 200, {}, json.dumps(code)
         else:
             return 404, {}, "{}"
diff --git a/moto/awslambda/utils.py b/moto/awslambda/utils.py
index 88146d34f..82027cb2f 100644
--- a/moto/awslambda/utils.py
+++ b/moto/awslambda/utils.py
@@ -3,8 +3,13 @@ from collections import namedtuple
 ARN = namedtuple('ARN', ['region', 'account', 'function_name', 'version'])
 
 
-def make_function_arn(region, account, name, version='1'):
-    return 'arn:aws:lambda:{0}:{1}:function:{2}:{3}'.format(region, account, name, version)
+def make_function_arn(region, account, name):
+    return 'arn:aws:lambda:{0}:{1}:function:{2}'.format(region, account, name)
+
+
+def make_function_ver_arn(region, account, name, version='1'):
+    arn = make_function_arn(region, account, name)
+    return '{0}:{1}'.format(arn, version)
 
 
 def split_function_arn(arn):
diff --git a/tests/test_awslambda/test_lambda.py b/tests/test_awslambda/test_lambda.py
index 479aaaa8a..9ef6fdb0d 100644
--- a/tests/test_awslambda/test_lambda.py
+++ b/tests/test_awslambda/test_lambda.py
@@ -282,7 +282,7 @@ def test_create_function_from_aws_bucket():
     result.pop('LastModified')
     result.should.equal({
         'FunctionName': 'testFunction',
-        'FunctionArn': 'arn:aws:lambda:{}:123456789012:function:testFunction:$LATEST'.format(_lambda_region),
+        'FunctionArn': 'arn:aws:lambda:{}:123456789012:function:testFunction'.format(_lambda_region),
         'Runtime': 'python2.7',
         'Role': 'test-iam-role',
         'Handler': 'lambda_function.lambda_handler',
@@ -291,7 +291,7 @@ def test_create_function_from_aws_bucket():
         'Description': 'test lambda function',
         'Timeout': 3,
         'MemorySize': 128,
-        'Version': '$LATEST',
+        'Version': '1',
         'VpcConfig': {
             "SecurityGroupIds": ["sg-123abc"],
             "SubnetIds": ["subnet-123abc"],
@@ -327,7 +327,7 @@ def test_create_function_from_zipfile():
 
     result.should.equal({
         'FunctionName': 'testFunction',
-        'FunctionArn': 'arn:aws:lambda:{}:123456789012:function:testFunction:$LATEST'.format(_lambda_region),
+        'FunctionArn': 'arn:aws:lambda:{}:123456789012:function:testFunction'.format(_lambda_region),
         'Runtime': 'python2.7',
         'Role': 'test-iam-role',
         'Handler': 'lambda_function.lambda_handler',
@@ -336,7 +336,7 @@ def test_create_function_from_zipfile():
         'Timeout': 3,
         'MemorySize': 128,
         'CodeSha256': hashlib.sha256(zip_content).hexdigest(),
-        'Version': '$LATEST',
+        'Version': '1',
         'VpcConfig': {
             "SecurityGroupIds": [],
             "SubnetIds": [],
@@ -398,6 +398,8 @@ def test_get_function():
     # Test get function with
     result = conn.get_function(FunctionName='testFunction', Qualifier='$LATEST')
     result['Configuration']['Version'].should.equal('$LATEST')
+    result['Configuration']['FunctionArn'].should.equal('arn:aws:lambda:us-west-2:123456789012:function:testFunction:$LATEST')
+
 
     # Test get function when can't find function name
     with assert_raises(ClientError):
@@ -464,7 +466,7 @@ def test_publish():
         Description='test lambda function',
         Timeout=3,
         MemorySize=128,
-        Publish=True,
+        Publish=False,
     )
 
     function_list = conn.list_functions()
@@ -485,7 +487,7 @@ def test_publish():
 
     function_list = conn.list_functions()
     function_list['Functions'].should.have.length_of(1)
-    function_list['Functions'][0]['FunctionArn'].should.contain('testFunction:$LATEST')
+    function_list['Functions'][0]['FunctionArn'].should.contain('testFunction')
 
 
 @mock_lambda
@@ -528,7 +530,7 @@ def test_list_create_list_get_delete_list():
             "CodeSha256": hashlib.sha256(zip_content).hexdigest(),
             "CodeSize": len(zip_content),
             "Description": "test lambda function",
-            "FunctionArn": 'arn:aws:lambda:{}:123456789012:function:testFunction:$LATEST'.format(_lambda_region),
+            "FunctionArn": 'arn:aws:lambda:{}:123456789012:function:testFunction'.format(_lambda_region),
             "FunctionName": "testFunction",
             "Handler": "lambda_function.lambda_handler",
             "MemorySize": 128,
@@ -701,7 +703,7 @@ def test_invoke_async_function():
     )
 
     success_result = conn.invoke_async(
-        FunctionName='testFunction', 
+        FunctionName='testFunction',
         InvokeArgs=json.dumps({'test': 'event'})
         )
 
@@ -741,7 +743,7 @@ def test_get_function_created_with_zipfile():
             "CodeSha256": hashlib.sha256(zip_content).hexdigest(),
             "CodeSize": len(zip_content),
             "Description": "test lambda function",
-            "FunctionArn": 'arn:aws:lambda:{}:123456789012:function:testFunction:$LATEST'.format(_lambda_region),
+            "FunctionArn": 'arn:aws:lambda:{}:123456789012:function:testFunction'.format(_lambda_region),
             "FunctionName": "testFunction",
             "Handler": "lambda_function.handler",
             "MemorySize": 128,
@@ -842,7 +844,7 @@ def test_list_versions_by_function():
     conn.create_function(
         FunctionName='testFunction',
         Runtime='python2.7',
-        Role='test-iam-role',
+        Role='arn:aws:iam::123456789012:role/test-iam-role',
         Handler='lambda_function.lambda_handler',
         Code={
             'S3Bucket': 'test-bucket',
@@ -857,8 +859,28 @@ def test_list_versions_by_function():
     res = conn.publish_version(FunctionName='testFunction')
     assert res['ResponseMetadata']['HTTPStatusCode'] == 201
     versions = conn.list_versions_by_function(FunctionName='testFunction')
-
+    assert len(versions['Versions']) == 3
     assert versions['Versions'][0]['FunctionArn'] == 'arn:aws:lambda:us-west-2:123456789012:function:testFunction:$LATEST'
+    assert versions['Versions'][1]['FunctionArn'] == 'arn:aws:lambda:us-west-2:123456789012:function:testFunction:1'
+    assert versions['Versions'][2]['FunctionArn'] == 'arn:aws:lambda:us-west-2:123456789012:function:testFunction:2'
+
+    conn.create_function(
+        FunctionName='testFunction_2',
+        Runtime='python2.7',
+        Role='arn:aws:iam::123456789012:role/test-iam-role',
+        Handler='lambda_function.lambda_handler',
+        Code={
+            'S3Bucket': 'test-bucket',
+            'S3Key': 'test.zip',
+        },
+        Description='test lambda function',
+        Timeout=3,
+        MemorySize=128,
+        Publish=False,
+    )
+    versions = conn.list_versions_by_function(FunctionName='testFunction_2')
+    assert len(versions['Versions']) == 1
+    assert versions['Versions'][0]['FunctionArn'] == 'arn:aws:lambda:us-west-2:123456789012:function:testFunction_2:$LATEST'
 
 
 @mock_lambda