Closed
Description
What is the problem?
We're starting to use PythonFunction from @aws-cdk/aws-lambda-python-alpha with poetry (a pyproject.toml and poetry.lock file in the directory specified by entry), and it works pretty well, thanks.
However, the exported requirements.txt includes hashes for external dependencies but not path deps, and if there's any hashes in the file, then pip requires a hash for every dep:
--require-hashes Require a hash to check each requirement
against, for repeatable installs. This option is
implied when any package in a requirements file
has a --hash option.
This means that if a Poetry package depends on both a path and pypi package, PythonFunction won't be able to bundle the requirements.
This is arguably a poetry and/or pip issue, but poetry export does provide a --without-hashes argument to emit the hashes, which could be added to:
Reproduction Steps
import * as cdk from 'aws-cdk-lib';
import * as lambdaPython from '@aws-cdk/aws-lambda-python-alpha';
export const app = new cdk.App();
const stack = new cdk.Stack(app, 'Stack', {});
new lambdaPython.PythonFunction(stack, 'Func', {
entry: 'parent',
index: 'parent/__init__.py',
runtime: cdk.aws_lambda.Runtime.PYTHON_3_9,
});Directory tree of parent:
parent
├── child
│ ├── child
│ │ └── __init__.py
│ └── pyproject.toml
├── parent
│ └── __init__.py
├── poetry.lock
└── pyproject.toml
parent/pyproject.toml:
[tool.poetry]
name = "parent"
version = "0.1.0"
description = ""
authors = []
[tool.poetry.dependencies]
python = "^3.9"
child = { path = "child" }
boto3 = "^1.21.12"
[build-system]
requires = ["poetry-core>=1.0.0"]
build-backend = "poetry.core.masonry.api"parent/poetry.lock:
[[package]]
name = "boto3"
version = "1.21.12"
description = "The AWS SDK for Python"
category = "main"
optional = false
python-versions = ">= 3.6"
[package.dependencies]
botocore = ">=1.24.12,<1.25.0"
jmespath = ">=0.7.1,<1.0.0"
s3transfer = ">=0.5.0,<0.6.0"
[package.extras]
crt = ["botocore[crt] (>=1.21.0,<2.0a0)"]
[[package]]
name = "botocore"
version = "1.24.12"
description = "Low-level, data-driven core of boto 3."
category = "main"
optional = false
python-versions = ">= 3.6"
[package.dependencies]
jmespath = ">=0.7.1,<1.0.0"
python-dateutil = ">=2.1,<3.0.0"
urllib3 = ">=1.25.4,<1.27"
[package.extras]
crt = ["awscrt (==0.12.5)"]
[[package]]
name = "child"
version = "0.1.0"
description = ""
category = "main"
optional = false
python-versions = "*"
develop = false
[package.source]
type = "directory"
url = "child"
[[package]]
name = "jmespath"
version = "0.10.0"
description = "JSON Matching Expressions"
category = "main"
optional = false
python-versions = ">=2.6, !=3.0.*, !=3.1.*, !=3.2.*"
[[package]]
name = "python-dateutil"
version = "2.8.2"
description = "Extensions to the standard Python datetime module"
category = "main"
optional = false
python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,>=2.7"
[package.dependencies]
six = ">=1.5"
[[package]]
name = "s3transfer"
version = "0.5.2"
description = "An Amazon S3 Transfer Manager"
category = "main"
optional = false
python-versions = ">= 3.6"
[package.dependencies]
botocore = ">=1.12.36,<2.0a.0"
[package.extras]
crt = ["botocore[crt] (>=1.20.29,<2.0a.0)"]
[[package]]
name = "six"
version = "1.16.0"
description = "Python 2 and 3 compatibility utilities"
category = "main"
optional = false
python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*"
[[package]]
name = "urllib3"
version = "1.26.8"
description = "HTTP library with thread-safe connection pooling, file post, and more."
category = "main"
optional = false
python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*, <4"
[package.extras]
brotli = ["brotlipy (>=0.6.0)"]
secure = ["pyOpenSSL (>=0.14)", "cryptography (>=1.3.4)", "idna (>=2.0.0)", "certifi", "ipaddress"]
socks = ["PySocks (>=1.5.6,!=1.5.7,<2.0)"]
[metadata]
lock-version = "1.1"
python-versions = "^3.9"
content-hash = "8ae88458c4b21dd26abae2f44d834b102dcde4f235634e676d62eddbc563e93f"
[metadata.files]
boto3 = [
{file = "boto3-1.21.12-py3-none-any.whl", hash = "sha256:75709628320cea8ce137975dc33b75213c2e4f6e7cd09e55290de7245e2c79e2"},
{file = "boto3-1.21.12.tar.gz", hash = "sha256:c92ec20a670721b5a1bc013b305a84db2b7f9c716653b3056ce7e2fbd2a180ef"},
]
botocore = [
{file = "botocore-1.24.12-py3-none-any.whl", hash = "sha256:0cd7395311a3fef4aad8df8f511b4f7d221c24ae30934bd5c03458b0fc096d0c"},
{file = "botocore-1.24.12.tar.gz", hash = "sha256:0174999a04b0a2e42457106093ace9b36fa94772a442d9bcf60750263d1d073e"},
]
child = []
jmespath = [
{file = "jmespath-0.10.0-py2.py3-none-any.whl", hash = "sha256:cdf6525904cc597730141d61b36f2e4b8ecc257c420fa2f4549bac2c2d0cb72f"},
{file = "jmespath-0.10.0.tar.gz", hash = "sha256:b85d0567b8666149a93172712e68920734333c0ce7e89b78b3e987f71e5ed4f9"},
]
python-dateutil = [
{file = "python-dateutil-2.8.2.tar.gz", hash = "sha256:0123cacc1627ae19ddf3c27a5de5bd67ee4586fbdd6440d9748f8abb483d3e86"},
{file = "python_dateutil-2.8.2-py2.py3-none-any.whl", hash = "sha256:961d03dc3453ebbc59dbdea9e4e11c5651520a876d0f4db161e8674aae935da9"},
]
s3transfer = [
{file = "s3transfer-0.5.2-py3-none-any.whl", hash = "sha256:7a6f4c4d1fdb9a2b640244008e142cbc2cd3ae34b386584ef044dd0f27101971"},
{file = "s3transfer-0.5.2.tar.gz", hash = "sha256:95c58c194ce657a5f4fb0b9e60a84968c808888aed628cd98ab8771fe1db98ed"},
]
six = [
{file = "six-1.16.0-py2.py3-none-any.whl", hash = "sha256:8abb2f1d86890a2dfb989f9a77cfcfd3e47c2a354b01111771326f8aa26e0254"},
{file = "six-1.16.0.tar.gz", hash = "sha256:1e61c37477a1626458e36f7b1d82aa5c9b094fa4802892072e49de9c60c4c926"},
]
urllib3 = [
{file = "urllib3-1.26.8-py2.py3-none-any.whl", hash = "sha256:000ca7f471a233c2251c6c7023ee85305721bfdf18621ebff4fd17a8653427ed"},
{file = "urllib3-1.26.8.tar.gz", hash = "sha256:0e7c33d9a63e7ddfcb86780aac87befc2fbddf46c58dbb487e0855f7ceec283c"},
]parent/child/pyproject.toml
[tool.poetry]
name = "child"
version = "0.1.0"
description = ""
authors = []
[build-system]
requires = ["poetry-core>=1.0.0"]
build-backend = "poetry.core.masonry.api"What did you expect to happen?
The bundling should work when there's both pypi/external and path dependencies, similar to if there's only pypi deps.
What actually happened?
The bundling computes a requirements file like the following:
boto3==1.21.12; python_version >= "3.6" \
--hash=sha256:75709628320cea8ce137975dc33b75213c2e4f6e7cd09e55290de7245e2c79e2 \
--hash=sha256:c92ec20a670721b5a1bc013b305a84db2b7f9c716653b3056ce7e2fbd2a180ef
botocore==1.24.12; python_version >= "3.6" \
--hash=sha256:0cd7395311a3fef4aad8df8f511b4f7d221c24ae30934bd5c03458b0fc096d0c \
--hash=sha256:0174999a04b0a2e42457106093ace9b36fa94772a442d9bcf60750263d1d073e
child @ file:///asset-input/child
jmespath==0.10.0; python_version >= "3.6" and python_full_version < "3.0.0" or python_full_version >= "3.3.0" and python_version >= "3.6" \
--hash=sha256:cdf6525904cc597730141d61b36f2e4b8ecc257c420fa2f4549bac2c2d0cb72f \
--hash=sha256:b85d0567b8666149a93172712e68920734333c0ce7e89b78b3e987f71e5ed4f9
python-dateutil==2.8.2; python_version >= "3.6" and python_full_version < "3.0.0" or python_full_version >= "3.3.0" and python_version >= "3.6" \
--hash=sha256:0123cacc1627ae19ddf3c27a5de5bd67ee4586fbdd6440d9748f8abb483d3e86 \
--hash=sha256:961d03dc3453ebbc59dbdea9e4e11c5651520a876d0f4db161e8674aae935da9
s3transfer==0.5.2; python_version >= "3.6" \
--hash=sha256:7a6f4c4d1fdb9a2b640244008e142cbc2cd3ae34b386584ef044dd0f27101971 \
--hash=sha256:95c58c194ce657a5f4fb0b9e60a84968c808888aed628cd98ab8771fe1db98ed
six==1.16.0; python_version >= "3.6" and python_full_version < "3.0.0" or python_full_version >= "3.3.0" and python_version >= "3.6" \
--hash=sha256:8abb2f1d86890a2dfb989f9a77cfcfd3e47c2a354b01111771326f8aa26e0254 \
--hash=sha256:1e61c37477a1626458e36f7b1d82aa5c9b094fa4802892072e49de9c60c4c926
urllib3==1.26.8; python_version >= "3.6" and python_full_version < "3.0.0" or python_full_version >= "3.5.0" and python_version < "4" and python_version >= "3.6" \
--hash=sha256:000ca7f471a233c2251c6c7023ee85305721bfdf18621ebff4fd17a8653427ed \
--hash=sha256:0e7c33d9a63e7ddfcb86780aac87befc2fbddf46c58dbb487e0855f7ceec283c
Note how child has no --hash argument. The bundling then fails with output including:
Processing ./child
ERROR: Can't verify hashes for these file:// requirements because they point to directories:
child@ file:///asset-input/child from file:///asset-input/child (from -r requirements.txt (line 7))
CDK CLI Version
2.14.0 (build 762d71b)
Framework Version
No response
Node.js Version
v14.17.5
OS
macOS
Language
Typescript
Language Version
TypeScript 4.1.6
Other information
No response
Activity