Closed
Description
Describe the bug
The problem arises while trying to create a redshift table with many columns (in the order of the hundreds) using the Table construct.
During the deployment of the resource (by the means of a CodeBuild process issuing cdk deploy -c env=test), I obtain the following error during the cloudformation CREATE action:
DpSlsMarketFrForecastAndMonitoringRedshiftTables | 3/6 | 7:40:50 AM | CREATE_FAILED | Custom::RedshiftDatabaseQuery | DpSlsMarketFrForecastMonitoringTable_example_table/Resource/Resource/Default (DpSlsMarketFrForecastMonitoringTableexampletable0E5B78B0) Received response status [FAILED] from custom resource. Message returned: Error: 2024-08-12T07:40:48.077Z df13b1ab-f383-46d6-8d17-079f0f966428 Task timed out after 60.07 seconds
Logs: /aws/lambda/DpSlsMarketFrForecastAndM-QueryRedshiftDatabase3de-k79HUXaOlRym
at invokeUserFunction (/var/task/framework.js:2:6)
at process.processTicksAndRejections (node:internal/process/task_queues:95:5)
at async onEvent (/var/task/framework.js:1:369)
at async Runtime.handler (/var/task/cfn-response.js:1:1826) (RequestId: ced8d7fe-26fa-4189-a931-b92c26d08b66)
The timeout error is related to a lambda generated automatically by the framework. Such timeout is by default set to 1 minute and I did not find a way to change it programmatically at runtime. Here's the code of the lambda in framework.js which raises the Exception:
"use strict";const cfnResponse=require("./cfn-response"),consts=require("./consts"),outbound_1=require("./outbound"),util_1=require("./util");async function onEvent(cfnRequest){const sanitizedRequest={...cfnRequest,ResponseURL:"..."};(0,util_1.log)("onEventHandler",sanitizedRequest),cfnRequest.ResourceProperties=cfnRequest.ResourceProperties||{};const onEventResult=await invokeUserFunction(consts.USER_ON_EVENT_FUNCTION_ARN_ENV,sanitizedRequest,cfnRequest.ResponseURL);onEventResult?.NoEcho?(0,util_1.log)("redacted onEvent returned:",cfnResponse.redactDataFromPayload(onEventResult)):(0,util_1.log)("onEvent returned:",onEventResult);const resourceEvent=createResponseEvent(cfnRequest,onEventResult);if(onEventResult?.NoEcho?(0,util_1.log)("readacted event:",cfnResponse.redactDataFromPayload(resourceEvent)):(0,util_1.log)("event:",resourceEvent),!process.env[consts.USER_IS_COMPLETE_FUNCTION_ARN_ENV])return cfnResponse.submitResponse("SUCCESS",resourceEvent,{noEcho:resourceEvent.NoEcho});const waiter={stateMachineArn:(0,util_1.getEnv)(consts.WAITER_STATE_MACHINE_ARN_ENV),name:resourceEvent.RequestId,input:JSON.stringify(resourceEvent)};(0,util_1.log)("starting waiter",{stateMachineArn:(0,util_1.getEnv)(consts.WAITER_STATE_MACHINE_ARN_ENV),name:resourceEvent.RequestId}),await(0,outbound_1.startExecution)(waiter)}async function isComplete(event){const sanitizedRequest={...event,ResponseURL:"..."};event?.NoEcho?(0,util_1.log)("redacted isComplete request",cfnResponse.redactDataFromPayload(sanitizedRequest)):(0,util_1.log)("isComplete",sanitizedRequest);const isCompleteResult=await invokeUserFunction(consts.USER_IS_COMPLETE_FUNCTION_ARN_ENV,sanitizedRequest,event.ResponseURL);if(event?.NoEcho?(0,util_1.log)("redacted user isComplete returned:",cfnResponse.redactDataFromPayload(isCompleteResult)):(0,util_1.log)("user isComplete returned:",isCompleteResult),!isCompleteResult.IsComplete)throw isCompleteResult.Data&&Object.keys(isCompleteResult.Data).length>0?new Error('"Data" is not allowed if "IsComplete" is "False"'):new cfnResponse.Retry(JSON.stringify(event));const response={...event,...isCompleteResult,Data:{...event.Data,...isCompleteResult.Data}};await cfnResponse.submitResponse("SUCCESS",response,{noEcho:event.NoEcho})}async function onTimeout(timeoutEvent){(0,util_1.log)("timeoutHandler",timeoutEvent);const isCompleteRequest=JSON.parse(JSON.parse(timeoutEvent.Cause).errorMessage);await cfnResponse.submitResponse("FAILED",isCompleteRequest,{reason:"Operation timed out"})}async function invokeUserFunction(functionArnEnv,sanitizedPayload,responseUrl){const functionArn=(0,util_1.getEnv)(functionArnEnv);(0,util_1.log)(`executing user function ${functionArn} with payload`,sanitizedPayload);const resp=await(0,outbound_1.invokeFunction)({FunctionName:functionArn,Payload:JSON.stringify({...sanitizedPayload,ResponseURL:responseUrl})});(0,util_1.log)("user function response:",resp,typeof resp);const jsonPayload=(0,util_1.parseJsonPayload)(resp.Payload);if(resp.FunctionError){(0,util_1.log)("user function threw an error:",resp.FunctionError);const errorMessage=jsonPayload.errorMessage||"error",arn=functionArn.split(":"),functionName=arn[arn.length-1],message=[errorMessage,"",`Logs: /aws/lambda/${functionName}`,""].join(`
`),e=new Error(message);throw jsonPayload.trace&&(e.stack=[message,...jsonPayload.trace.slice(1)].join(`
`)),e}return jsonPayload}function createResponseEvent(cfnRequest,onEventResult){onEventResult=onEventResult||{};const physicalResourceId=onEventResult.PhysicalResourceId||defaultPhysicalResourceId(cfnRequest);if(cfnRequest.RequestType==="Delete"&&physicalResourceId!==cfnRequest.PhysicalResourceId)throw new Error(`DELETE: cannot change the physical resource ID from "${cfnRequest.PhysicalResourceId}" to "${onEventResult.PhysicalResourceId}" during deletion`);return cfnRequest.RequestType==="Update"&&physicalResourceId!==cfnRequest.PhysicalResourceId&&(0,util_1.log)(`UPDATE: changing physical resource ID from "${cfnRequest.PhysicalResourceId}" to "${onEventResult.PhysicalResourceId}"`),{...cfnRequest,...onEventResult,PhysicalResourceId:physicalResourceId}}function defaultPhysicalResourceId(req){switch(req.RequestType){case"Create":return req.RequestId;case"Update":case"Delete":return req.PhysicalResourceId;default:throw new Error(`Invalid "RequestType" in request "${JSON.stringify(req)}"`)}}module.exports={[consts.FRAMEWORK_ON_EVENT_HANDLER_NAME]:cfnResponse.safeHandler(onEvent),[consts.FRAMEWORK_IS_COMPLETE_HANDLER_NAME]:cfnResponse.safeHandler(isComplete),[consts.FRAMEWORK_ON_TIMEOUT_HANDLER_NAME]:onTimeout};
Moreover, the table creation in redshfit is completed correctly (it looks like the CREATE TABLE instruction is issued and not aborted when the lambda fails), leaving the table untied to the cloudformation stack.
Here's an example of the code I am using:
import aws_cdk as cdk
from aws_cdk import Stack
from aws_cdk.aws_redshift_alpha import Cluster, Column, Table, TableDistStyle, TableSortStyle
from constructs import Construct
class RedshiftCdkStack(Stack):
def __init__(self, scope: Construct, construct_id: str, **kwargs) -> None:
## ...
## some stuff here and there
## ...
cluster = Cluster.from_cluster_attributes(
self,
"ImportedCluster",
cluster_name=env_config["clusterName"],
cluster_endpoint_address=env_config["clusterEndpointAddress"],
cluster_endpoint_port=env_config["clusterPort"],
)
table_columns = [
Column(
name="column_1",
data_type="integer",
dist_key=False,
comment="My first column"
)
]
dist_style = TableDistStyle["KEY"]
removal_policy = cdk.RemovalPolicy["DESTROY"]
sort_style = TableSortStyle["AUTO"]
Table(
self,
"MyTableName",
cluster=cluster,
database_name="my_db",
admin_user=my_secret,
table_columns=table_columns,
dist_style=dist_style,
removal_policy=removal_policy,
sort_style=sort_style,
table_name="my_schema.my_table",
table_comment="My first table"
)
Regression Issue
- Select this option if this issue appears to be a regression.
Last Known Working CDK Version
No response
Expected Behavior
I expect the table to be created successfully in redshift and the table to be part of the cloudformation stack.
Current Behavior
The table is created successfully in redshift but the table is not part of the cloudformation stack.
Reproduction Steps
import aws_cdk as cdk
from aws_cdk import Stack
from aws_cdk.aws_redshift_alpha import Cluster, Column, Table, TableDistStyle, TableSortStyle
from constructs import Construct
class RedshiftCdkStack(Stack):
def __init__(self, scope: Construct, construct_id: str, **kwargs) -> None:
cluster = Cluster.from_cluster_attributes(
self,
"ImportedCluster",
cluster_name=env_config["clusterName"],
cluster_endpoint_address=env_config["clusterEndpointAddress"],
cluster_endpoint_port=env_config["clusterPort"],
)
table_columns = [
Column(
name=f"column_{i}",
data_type="integer",
dist_key=False,
comment=f"My #{i} column"
)
for i in range(1000)
]
dist_style = TableDistStyle["KEY"]
removal_policy = cdk.RemovalPolicy["DESTROY"]
sort_style = TableSortStyle["AUTO"]
Table(
self,
"MyTableName",
cluster=cluster,
database_name="my_db",
admin_user=my_secret,
table_columns=table_columns,
dist_style=dist_style,
removal_policy=removal_policy,
sort_style=sort_style,
table_name="my_schema.my_table",
table_comment="My first table"
)
########
# main #
########
app = cdk.App()
RedshiftCdkStack(app, "MyRedshiftTablesStack",
env=cdk.Environment(account=os.getenv('AWS_DEFAULT_ACCOUNT'), region='eu-west-1'),
)
app.synth()
Possible Solution
I think it is sufficient to increase the timeout for the lambda function
Additional Information/Context
No response
CDK CLI Version
2.150.0
Framework Version
No response
Node.js Version
18
OS
Ubuntu
Language
Python
Language Version
No response
Other information
I am using an EC2 instance with the following runtime provided in codebuild
aws/codebuild/standard:7.0 (Ubuntu standard:7.0)