Solving the AWS Lambda timeout limitation with Amazon ECS Fargate

plugins:
  - serverless-plugin-lambda-dead-letter

SQSFailureQueue:
  Type: AWS::SQS::Queue
  Properties:
    QueueName: sqs-failure-queue
    MessageRetentionPeriod: "1209600"
    VisibilityTimeout: "60"

  secondaryHandler:
    handler: path/to/immortal-handlerDlq.handler
    name: immortal-handlerDql
    environment:
      REGION: eu-west-1
      # ECS task to execute handler without timeout limitations
      RUNNER: runner
      # Required for runTask API call, preliminary created on AWS.
      SUBNET: subnet-xxxxxxxx
      ...
    events:
      - sqs:
          arn:
            Fn::GetAtt:
              - SQSFailureQueue
              - Arn

functions:
  primaryHandler:
    handler: path/to/immortal-handler.handler
    name: immortal-handler
    deadLetter:
      targetArn:
        GetResourceArn: SQSFailureQueue

  iamRoleStatements:
    ...
    # Allow running AWS Fargate containers when AWS Lambda timeout cannot be bypassed.
    - Effect: 'Allow'
      Action:
        - ecs:RunTask
      Resource: '*'
    - Effect: Allow
      Action:
        - iam:PassRole
      Resource:
        Fn::Join:
          - ''
          - - 'arn:aws:iam::'
            - Ref: 'AWS::AccountId'
            - ':role/ecsTaskExecutionRole'
    # ecsTaskExecutionRole is the name of the default role created by ECS,
    ...
    # Allow queueing messages to the DLQ https://docs.aws.amazon.com/lambda/latest/dg/dlq.html
    - Effect: 'Allow'
      Action:
        - sqs:SendMessage
      Resource: '*'

import AWS from 'aws-sdk';

export const handler = async (event, context, callback) => {
  // Coming from environment variables setup in your `serverless.yaml` file
  // See https://serverless.com/framework/docs/providers/aws/guide/variables/#referencing-environment-variables
  const { RUNNER, SUBNET } = process.env;
  const ecs = new AWS.ECS();

  // Some other logic specific to your case here ...

  try {
    const runParams = {
      taskDefinition: RUNNER,
      launchType: 'FARGATE',
      networkConfiguration: {
        awsvpcConfiguration: {
          assignPublicIp: 'ENABLED',
          subnets: [SUBNET],
        },
      },
      overrides: {
        containerOverrides: [
          {
            environment: [
              {
                name: 'AWS_LAMBDA_FUNCTION_EVENT',
                value: JSON.stringify(event),
              },
              // Help the runner locate the handler to run inside a container.
              {
                name: 'AWS_LAMBDA_HANDLER_PATH',
                // Matches path/to/immortal-handler.handler
                value: 'path/to/immortal-handler.js',
              },
              {
                name: 'AWS_LAMBDA_FUNCTION_CONTEXT',
                value: JSON.stringify(context),
              },
              ...
            ],
            name: RUNNER,
          },
        ],
      },
    };

    const result = await ecs.runTask(runParams).promise();
    // Returns null
    return callback(null, result);
  } catch (e) {
    return callback(e);
  }
};

export default handler;

FROM node:8

# Environment variables acting as parameters of the runner.
ENV AWS_LAMBDA_FUNCTION_EVENT {}
ENV AWS_LAMBDA_FUNCTION_CONTEXT {}

# Create app directory
WORKDIR /usr/src/app

COPY . .

RUN npm install

# Parameters are named arguments extracted later with yargs or something else working on argv.
CMD ["sh", "-c", "./runner.js --event ${AWS_LAMBDA_FUNCTION_EVENT} --context ${AWS_LAMBDA_FUNCTION_CONTEXT}"]

#!/usr/bin/env node

const path = require("path");
const https = require("https");

const AWS = require("aws-sdk");
const promisePipe = require("promisepipe");
const unzip = require("unzip");
const { argv } = require("yargs");

const runner = async () => {
  const { REGION, AWS_LAMBDA_HANDLER_PATH } = process.env;
  // Required as strings by the API: https://docs.aws.amazon.com/AmazonECS/latest/APIReference/API_TaskOverride.html?shortFooter=true
  const { event: e, context: c } = argv;
  const context = JSON.parse(c);
  const event = JSON.parse(e);

  if (!context.functionName) {
    return console.error("Required function name not found in context.");
  }

  const invoking = context.functionName;
  // Assumes dead letter queue lambda functions are suffixed with Dlq
  const originalInvoking = invoking.substring(0, invoking.indexOf("Dlq"));

  try {
    const lambda = new AWS.Lambda({ region: REGION });

    const lambdaInfo = await lambda
      .getFunction({ FunctionName: originalInvoking })
      .promise();

    const sourceCodeSignedUrl = lambdaInfo.Code.Location;

    return https.get(sourceCodeSignedUrl, async (res) => {
      // Download source from cloud and extract it at the current directory at the same time.
      await promisePipe(res, unzip.Extract({ path: __dirname }));

      const pathToHandler = path.resolve(
        `${__dirname}/${AWS_LAMBDA_HANDLER_PATH}`,
      );
      const handler = require(pathToHandler);
      // Run the taget handler's code.
      // handler.handler is the default export of webpack's bundle file
      const result = await handler.handler(event, context);
      return console.log(result);
    });
  } catch (err) {
    return console.error(err.message);
  }
};

runner();

$ chmod +x runner.js

$ docker build -t runner .

$ docker tag runner:latest {accountId}.dkr.ecr.eu-central-1.amazonaws.com/runner:latest

$ docker push {accountId}.dkr.ecr.eu-central-1.amazonaws.com/runner:latest

$ aws ecr get-login --no-include-email | source /dev/stdin