diff --git a/LAMBDA_OPTIMIZATION_GUIDE.md b/LAMBDA_OPTIMIZATION_GUIDE.md new file mode 100644 index 0000000..f26b0a8 --- /dev/null +++ b/LAMBDA_OPTIMIZATION_GUIDE.md @@ -0,0 +1,490 @@ +# AWS Lambda Bundle Size Optimization Guide + +## Overview + +This guide documents how to optimize AWS Lambda function bundle sizes when using: +- **Serverless Framework v4** (with built-in esbuild) +- **Prisma ORM** (with driver adapters) +- **NestJS** or any Node.js framework + +## Problem + +Lambda functions can become bloated (25+ MB) due to: +1. **Prisma binary engines** (~50MB uncompressed) +2. **AWS SDK v3** being bundled (~5-10MB) +3. **Dependencies copied to node_modules** instead of being bundled + +## Solution Architecture + +``` +┌─────────────────────────────────────────────────────────┐ +│ Lambda Function │ +│ ┌─────────────────────────────────────────────────┐ │ +│ │ Your Code (bundled by esbuild) ~50-500 KB │ │ +│ └─────────────────────────────────────────────────┘ │ +│ │ │ +│ ▼ │ +│ ┌─────────────────────────────────────────────────┐ │ +│ │ Prisma Layer (shared) ~15 MB │ │ +│ │ - @prisma/client │ │ +│ │ - @prisma/adapter-pg │ │ +│ │ - .prisma/client (generated) │ │ +│ │ - pg driver │ │ +│ └─────────────────────────────────────────────────┘ │ +│ │ │ +│ ▼ │ +│ ┌─────────────────────────────────────────────────┐ │ +│ │ AWS Lambda Runtime │ │ +│ │ - AWS SDK v3 (built-in for Node.js 18+) │ │ +│ └─────────────────────────────────────────────────┘ │ +└─────────────────────────────────────────────────────────┘ +``` + +--- + +## Step-by-Step Setup + +### 1. Project Structure + +``` +your-project/ +├── serverless.yml +├── package.json +├── prisma/ +│ └── schema.prisma +├── layers/ +│ └── prisma/ +│ └── nodejs/ +│ └── package.json +├── src/ +│ └── ... your code +└── build-prisma-layer.ps1 (or .sh for Linux/Mac) +``` + +### 2. Prisma Schema Configuration + +**`prisma/schema.prisma`** +```prisma +generator client { + provider = "prisma-client-js" + // For Prisma 7+ with driver adapters, no binary targets needed + // The WASM-based query engine is used automatically +} + +datasource db { + provider = "postgresql" + url = env("DATABASE_URL") +} +``` + +> **Note**: Prisma 7+ uses WASM-based query compiler instead of binary engines when using driver adapters, which is much smaller. + +### 3. Layer Package.json + +**`layers/prisma/nodejs/package.json`** +```json +{ + "name": "prisma-layer", + "version": "1.0.0", + "description": "Lambda layer for Prisma with pg driver adapter", + "dependencies": { + "@prisma/client": "^7.0.0", + "@prisma/adapter-pg": "^7.0.0", + "pg": "^8.13.0" + } +} +``` + +### 4. Serverless Configuration + +**`serverless.yml`** +```yaml +service: your-service-name + +provider: + name: aws + runtime: nodejs22.x + region: your-region + memorySize: 512 + # Apply Prisma layer to ALL functions + layers: + - !Ref PrismaLambdaLayer + environment: + DATABASE_URL: ${env:DATABASE_URL} + # ... other env vars + +# esbuild configuration (Serverless v4 built-in) +build: + esbuild: + bundle: true + minify: true + sourcemap: false + target: node20 + platform: node + # Mark packages as external (not bundled into JS) + external: + - '@prisma/client' + - '@prisma/adapter-pg' + - '.prisma/client' + - '.prisma' + - 'pg' + - '@aws-sdk/*' + - '@smithy/*' + - '@aws-crypto/*' + # Exclude from npm install in zip (CRITICAL!) + exclude: + - 'aws-sdk' + - '@aws-sdk/*' + - '@smithy/*' + - '@aws-crypto/*' + - '@prisma/client' + - '@prisma/adapter-pg' + - '.prisma' + - '.prisma/client' + - 'pg' + - 'pg-*' + - 'postgres-*' + - 'pgpass' + - 'split2' + - 'xtend' + +# Define the Prisma layer +layers: + prisma: + path: layers/prisma + name: ${self:service}-prisma-layer-${sls:stage} + description: Prisma client with pg driver adapter + compatibleRuntimes: + - nodejs22.x + retain: false + +# Package configuration +package: + individually: true + patterns: + - '!node_modules/**' + - '!node_modules/@prisma/**' + - '!node_modules/.prisma/**' + - '!**/*.test.js' + - '!**/*.spec.js' + - '!**/test/**' + - '!**/__tests__/**' + - '!package-lock.json' + - '!yarn.lock' + - '!README.md' + - '!.git/**' + +functions: + myFunction: + handler: src/handlers/myHandler.handler + events: + - httpApi: + path: /my-endpoint + method: get + +plugins: + - serverless-offline +``` + +### 5. Build Script for Prisma Layer + +**Windows (PowerShell) - `build-prisma-layer.ps1`** +```powershell +# Build Prisma Lambda Layer +$layerPath = "layers\prisma\nodejs" + +Write-Host "Building Prisma Lambda Layer..." -ForegroundColor Cyan + +# 1. Clean existing node_modules in layer +Write-Host "Cleaning layer node_modules..." +if (Test-Path "$layerPath\node_modules") { + Remove-Item -Recurse -Force "$layerPath\node_modules" +} + +# 2. Install dependencies in layer +Write-Host "Installing layer dependencies..." +Push-Location $layerPath +npm install --omit=dev +Pop-Location + +# 3. Generate Prisma client +Write-Host "Generating Prisma client..." +npx prisma generate + +# 4. Copy .prisma/client to layer +Write-Host "Copying generated Prisma client to layer..." +$sourcePrisma = "node_modules\.prisma" +$destPrisma = "$layerPath\node_modules\.prisma" + +if (Test-Path $sourcePrisma) { + if (Test-Path $destPrisma) { + Remove-Item -Recurse -Force $destPrisma + } + Copy-Item -Recurse $sourcePrisma $destPrisma + Write-Host "Copied .prisma/client successfully!" -ForegroundColor Green +} else { + Write-Host "ERROR: .prisma folder not found. Run 'npx prisma generate' first." -ForegroundColor Red + exit 1 +} + +# 5. Show layer size +$layerSize = (Get-ChildItem "$layerPath\node_modules" -Recurse | Measure-Object -Property Length -Sum).Sum / 1MB +Write-Host "`nTotal layer size: $([math]::Round($layerSize, 2)) MB" -ForegroundColor Yellow +Write-Host "Prisma layer built successfully!" -ForegroundColor Green +``` + +**Linux/Mac (Bash) - `build-prisma-layer.sh`** +```bash +#!/bin/bash +set -e + +LAYER_PATH="layers/prisma/nodejs" + +echo "Building Prisma Lambda Layer..." + +# 1. Clean existing node_modules in layer +echo "Cleaning layer node_modules..." +rm -rf "$LAYER_PATH/node_modules" + +# 2. Install dependencies in layer +echo "Installing layer dependencies..." +cd "$LAYER_PATH" +npm install --omit=dev +cd - + +# 3. Generate Prisma client +echo "Generating Prisma client..." +npx prisma generate + +# 4. Copy .prisma/client to layer +echo "Copying generated Prisma client to layer..." +if [ -d "node_modules/.prisma" ]; then + rm -rf "$LAYER_PATH/node_modules/.prisma" + cp -r "node_modules/.prisma" "$LAYER_PATH/node_modules/.prisma" + echo "Copied .prisma/client successfully!" +else + echo "ERROR: .prisma folder not found. Run 'npx prisma generate' first." + exit 1 +fi + +# 5. Show layer size +LAYER_SIZE=$(du -sm "$LAYER_PATH/node_modules" | cut -f1) +echo "Total layer size: ${LAYER_SIZE} MB" +echo "Prisma layer built successfully!" +``` + +### 6. Prisma Client Usage + +**`src/common/database/prisma.client.ts`** +```typescript +import { PrismaClient } from '@prisma/client'; +import { PrismaPg } from '@prisma/adapter-pg'; +import { Pool } from 'pg'; + +// Connection pool for serverless +const pool = new Pool({ + connectionString: process.env.DATABASE_URL, + max: 5, // Limit connections in Lambda +}); + +const adapter = new PrismaPg(pool); + +// Single instance for Lambda warm starts +let prisma: PrismaClient; + +export function getPrismaClient(): PrismaClient { + if (!prisma) { + prisma = new PrismaClient({ + adapter, + log: process.env.NODE_ENV === 'development' ? ['query', 'error', 'warn'] : ['error'], + }); + } + return prisma; +} + +export { prisma }; +``` + +--- + +## Deployment Workflow + +```bash +# 1. Build the Prisma layer (run after schema changes) +./build-prisma-layer.ps1 # Windows +# or +./build-prisma-layer.sh # Linux/Mac + +# 2. Deploy +npx serverless deploy --stage=dev + +# 3. Deploy single function (faster, uses existing layer) +npx serverless deploy function -f myFunction --stage=dev +``` + +--- + +## Key Configuration Explained + +### esbuild `external` vs `exclude` + +| Property | Purpose | +|----------|---------| +| `external` | Tells esbuild NOT to bundle these into the JS file. They become `require()` calls. | +| `exclude` | Tells Serverless NOT to `npm install` these packages into the function zip. | + +**Both are required!** +- `external` alone = esbuild doesn't bundle, but Serverless still installs to node_modules +- `exclude` alone = Serverless doesn't install, but esbuild bundles the code + +### Layer Reference + +```yaml +# This creates a CloudFormation reference to the layer defined in the same stack +layers: + - !Ref PrismaLambdaLayer +``` + +The `PrismaLambdaLayer` name comes from the layer key (`prisma`) converted to PascalCase + `LambdaLayer`. + +### Why Exclude pg-* packages? + +When `pg` is external, its dependencies still get installed: +- `pg-connection-string` +- `pg-pool` +- `pg-protocol` +- `pg-types` +- etc. + +These must all be in the `exclude` list to prevent duplication. + +--- + +## Expected Results + +| Function Type | Before | After | +|---------------|--------|-------| +| Simple handlers | 25+ MB | **50-100 kB** | +| With validation (zod/yup) | 25+ MB | **300-500 kB** | +| With S3 uploads | 30+ MB | **1-2 MB** | + +--- + +## Troubleshooting + +### 1. "Cannot find module '@prisma/client'" + +**Cause**: Layer doesn't have the generated `.prisma/client` + +**Fix**: Run `build-prisma-layer.ps1` to regenerate the layer + +### 2. Function size still large + +**Debug**: Extract and inspect the zip: +```powershell +Expand-Archive ".serverless\build\your-function.zip" -DestinationPath "extracted" +Get-ChildItem "extracted\node_modules" -Directory +``` + +If you see `@prisma` or `pg` folders, the `exclude` config isn't working. + +### 3. "Cannot resolve CloudFormation reference" + +**Cause**: Using `${cf:...}` reference before first deploy + +**Fix**: Use `!Ref PrismaLambdaLayer` instead (works on first deploy) + +### 4. Cold starts still slow + +Consider: +- **Provisioned Concurrency**: Pre-warm instances +- **Reduce memory**: Sometimes lower memory = same speed, lower cost +- **Connection pooling**: Use tools like PgBouncer for RDS + +--- + +## Additional Optimizations + +### 1. Remove duplicate validation libraries + +Pick ONE of: `zod`, `yup`, or `class-validator`. Don't use all three. + +### 2. Tree-shake NestJS + +If not using full NestJS, import only what you need: +```typescript +// Instead of +import { Controller, Get } from '@nestjs/common'; + +// For Lambda handlers, you might not need NestJS at all +``` + +### 3. Use AWS SDK v3 selectively + +```yaml +external: + - '@aws-sdk/*' # Exclude all +``` + +Then in code: +```typescript +// AWS SDK v3 is available in Lambda runtime (Node.js 18+) +import { S3Client } from '@aws-sdk/client-s3'; +``` + +--- + +## Quick Reference + +```yaml +# Minimal serverless.yml for Prisma + Lambda optimization +build: + esbuild: + bundle: true + minify: true + external: + - '@prisma/client' + - '@prisma/adapter-pg' + - '.prisma/client' + - '.prisma' + - 'pg' + - '@aws-sdk/*' + exclude: + - '@prisma/client' + - '@prisma/adapter-pg' + - '.prisma' + - '.prisma/client' + - 'pg' + - 'pg-*' + - 'postgres-*' + - 'pgpass' + - 'split2' + - 'xtend' + - '@aws-sdk/*' + +layers: + prisma: + path: layers/prisma + name: ${self:service}-prisma-${sls:stage} + compatibleRuntimes: + - nodejs22.x + +provider: + layers: + - !Ref PrismaLambdaLayer +``` + +--- + +## Version Compatibility + +| Tool | Tested Version | +|------|----------------| +| Serverless Framework | v4.x | +| Prisma | v7.x | +| Node.js | 20.x, 22.x | +| AWS Lambda Runtime | nodejs20.x, nodejs22.x | + +--- + +*Last updated: December 2025* diff --git a/build-prisma-layer.ps1 b/build-prisma-layer.ps1 new file mode 100644 index 0000000..1872c13 --- /dev/null +++ b/build-prisma-layer.ps1 @@ -0,0 +1,51 @@ +# Build Prisma Lambda Layer +# Run this script before deploying to ensure the layer has the generated client + +$layerPath = "layers\prisma\nodejs" + +Write-Host "Building Prisma Lambda Layer..." -ForegroundColor Cyan + +# 1. Clean existing node_modules in layer +Write-Host "Cleaning layer node_modules..." +if (Test-Path "$layerPath\node_modules") { + Remove-Item -Recurse -Force "$layerPath\node_modules" +} + +# 2. Install dependencies in layer +Write-Host "Installing layer dependencies..." +Push-Location $layerPath +npm install --omit=dev +Pop-Location + +# 3. Generate Prisma client into the layer +Write-Host "Generating Prisma client into layer..." +# Set the output directory for Prisma client +$env:PRISMA_GENERATE_DATAPROXY = "false" + +# Generate client - this creates .prisma/client +npx prisma generate + +# 4. Copy .prisma/client to layer +Write-Host "Copying generated Prisma client to layer..." +$sourcePrisma = "node_modules\.prisma" +$destPrisma = "$layerPath\node_modules\.prisma" + +if (Test-Path $sourcePrisma) { + if (Test-Path $destPrisma) { + Remove-Item -Recurse -Force $destPrisma + } + Copy-Item -Recurse $sourcePrisma $destPrisma + Write-Host "Copied .prisma/client successfully!" -ForegroundColor Green +} else { + Write-Host "ERROR: .prisma folder not found. Run 'npx prisma generate' first." -ForegroundColor Red + exit 1 +} + +# 5. Show layer size +Write-Host "`nLayer contents:" +Get-ChildItem "$layerPath\node_modules" -Directory | Select-Object Name +$layerSize = (Get-ChildItem "$layerPath\node_modules" -Recurse | Measure-Object -Property Length -Sum).Sum / 1MB +Write-Host "`nTotal layer size: $([math]::Round($layerSize, 2)) MB" -ForegroundColor Yellow + +Write-Host "`nPrisma layer built successfully!" -ForegroundColor Green +Write-Host "Run 'serverless deploy' to deploy with the updated layer." diff --git a/serverless.yml b/serverless.yml index 098b89b..f3fa230 100644 --- a/serverless.yml +++ b/serverless.yml @@ -7,9 +7,9 @@ provider: versionFunctions: false memorySize: 512 # Apply Prisma layer to all functions - # Use the published layer version ARN (works for full deploy and `deploy function`) + # Reference the layer defined in this stack using CloudFormation Ref layers: - - ${cf:${self:service}-${sls:stage}.PrismaLambdaLayerQualifiedArn} + - !Ref PrismaLambdaLayer apiGateway: binaryMediaTypes: - '*/*' @@ -68,14 +68,32 @@ build: sourcemap: false target: node20 platform: node + # Mark as external so they're not bundled into the JS external: - # These are provided by the Prisma layer + - '@prisma/client' + - '@prisma/adapter-pg' + - '.prisma/client' + - '.prisma' + - 'pg' + - '@aws-sdk/*' + - '@smithy/*' + - '@aws-crypto/*' + # Exclude prevents npm install of these packages in the zip + exclude: + - 'aws-sdk' + - '@aws-sdk/*' + - '@smithy/*' + - '@aws-crypto/*' - '@prisma/client' - '@prisma/adapter-pg' - '.prisma' + - '.prisma/client' - 'pg' - exclude: - - 'aws-sdk' + - 'pg-*' + - 'postgres-*' + - 'pgpass' + - 'split2' + - 'xtend' # Define layers layers: @@ -91,6 +109,8 @@ package: individually: true patterns: - '!node_modules/**' + - '!node_modules/@prisma/**' + - '!node_modules/.prisma/**' - '!**/*.test.js' - '!**/*.spec.js' - '!**/test/**' diff --git a/serverless/functions/host.yml b/serverless/functions/host.yml index 97615bf..117b18c 100644 --- a/serverless/functions/host.yml +++ b/serverless/functions/host.yml @@ -253,28 +253,6 @@ submitCompanyDetails: - 'src/modules/host/handlers/addCompanyDetails.*' - 'src/modules/host/services/**' - 'src/common/**' - - 'node_modules/@prisma/client/**' - - 'node_modules/.prisma/client/libquery_engine-rhel-openssl-3.0.x.so.node' - # Only include specific AWS SDK modules needed for S3 - - 'node_modules/@aws-sdk/client-s3/**' - - 'node_modules/@aws-sdk/s3-request-presigner/**' - - 'node_modules/@aws-sdk/types/**' - - 'node_modules/@aws-sdk/middleware-logger/**' - - 'node_modules/@aws-sdk/util-utf8-node/**' - - 'node_modules/@aws-sdk/util-utf8-browser/**' - - 'node_modules/@smithy/**' - - 'node_modules/tslib/**' - # Remove these large/unnecessary packages: - - 'node_modules/fast-xml-parser/**' # Remove if not used - - 'node_modules/lambda-multipart-parser/**' # You're using busboy directly - - 'node_modules/busboy/**' - # Remove these AWS utility packages (included in main SDK): - - 'node_modules/@aws-crypto/**' - # - 'node_modules/uuid/**' # AWS SDK includes its own - # - 'node_modules/@aws/util-uri-escape/**' - # - 'node_modules/@aws/util-middleware/**' - - 'node_modules/@aws/smithy-client/**' - # - 'node_modules/@aws/lambda-invoke-store/**' events: - httpApi: path: /host/Host_Admin/onboarding/add-company-details @@ -291,24 +269,6 @@ submitPQQ_Answer: - ${file(./serverless/patterns/base.yml):pattern2} - ${file(./serverless/patterns/base.yml):pattern3} - ${file(./serverless/patterns/base.yml):pattern4} - - 'node_modules/@aws-sdk/client-s3/**' - - 'node_modules/@aws-sdk/s3-request-presigner/**' - - 'node_modules/@aws-sdk/types/**' - - 'node_modules/@aws-sdk/middleware-logger/**' - - 'node_modules/@aws-sdk/util-utf8-node/**' - - 'node_modules/@aws-sdk/util-utf8-browser/**' - - 'node_modules/@smithy/**' - - 'node_modules/tslib/**' - # Remove these large/unnecessary packages: - - 'node_modules/fast-xml-parser/**' # Remove if not used - - 'node_modules/lambda-multipart-parser/**' # You're using busboy directly - - 'node_modules/busboy/**' - # Remove these AWS utility packages (included in main SDK): - - 'node_modules/@aws-crypto/**' - # - 'node_modules/uuid/**' # AWS SDK includes its own - # - 'node_modules/@aws/util-uri-escape/**' - # - 'node_modules/@aws/util-middleware/**' - - 'node_modules/@aws/smithy-client/**' events: - httpApi: path: /host/Activity_Hub/OnBoarding/submit-pqq-answer diff --git a/serverless/functions/minglaradmin.yml b/serverless/functions/minglaradmin.yml index bf68211..c660469 100644 --- a/serverless/functions/minglaradmin.yml +++ b/serverless/functions/minglaradmin.yml @@ -58,22 +58,6 @@ updateMinglarProfile: - ${file(./serverless/patterns/base.yml):pattern2} - ${file(./serverless/patterns/base.yml):pattern3} - ${file(./serverless/patterns/base.yml):pattern4} - - ${file(./serverless/patterns/aws-s3.yml):pattern1} - - ${file(./serverless/patterns/aws-s3.yml):pattern2} - - ${file(./serverless/patterns/aws-s3.yml):pattern3} - - ${file(./serverless/patterns/aws-s3.yml):pattern4} - - ${file(./serverless/patterns/aws-s3.yml):pattern5} - - ${file(./serverless/patterns/aws-s3.yml):pattern6} - - ${file(./serverless/patterns/aws-s3.yml):pattern7} - - ${file(./serverless/patterns/aws-s3.yml):pattern8} - - ${file(./serverless/patterns/aws-s3.yml):pattern9} - - ${file(./serverless/patterns/aws-s3.yml):pattern10} - - ${file(./serverless/patterns/aws-s3.yml):pattern11} - - ${file(./serverless/patterns/aws-s3.yml):pattern12} - - ${file(./serverless/patterns/aws-s3.yml):pattern13} - - ${file(./serverless/patterns/aws-s3.yml):pattern14} - - ${file(./serverless/patterns/aws-s3.yml):pattern15} - - ${file(./serverless/patterns/aws-s3.yml):pattern16} events: - httpApi: path: /minglaradmin/update-profile diff --git a/serverless/patterns/base.yml b/serverless/patterns/base.yml index 1fce4e7..96eac12 100644 --- a/serverless/patterns/base.yml +++ b/serverless/patterns/base.yml @@ -1,6 +1,9 @@ # Base packaging patterns shared across all functions pattern1: 'src/common/**' pattern2: 'common/**' -pattern3: 'node_modules/@prisma/client/**' -pattern4: 'node_modules/.prisma/client/libquery_engine-rhel-openssl-3.0.x.so.node' +# REMOVED: Prisma is now provided by Lambda layer - DO NOT include in function packages +# pattern3: 'node_modules/@prisma/client/**' +# pattern4: 'node_modules/.prisma/client/libquery_engine-rhel-openssl-3.0.x.so.node' +pattern3: '!node_modules/@prisma/**' +pattern4: '!node_modules/.prisma/**' pattern5: '!node_modules/.prisma/client/libquery_engine*' \ No newline at end of file