How to build a layer for lambda - johnzheng1975/devops_way GitHub Wiki

apiVersion: v1
kind: Pod
metadata:
  creationTimestamp: null
  labels:
    run: awslinux
  name: awslinux
spec:
  containers:
  - image: lambci/lambda:build-python3.8
    env:
    - name: ODBCINI
      value: /opt/odbc.ini
    - name: ODBCSYSINI
      value: /opt/
    name: awslinux
    resources:
      limits:
        cpu: 500m
        memory: 1024M
      requests:
        cpu: 500m
        memory: 1024M
    command: [ "/bin/bash", "-c", "--" ]
    args: [ "while true; do sleep 30; done;" ]
  dnsPolicy: ClusterFirst
  restartPolicy: Always

k exec -ti -n hpbp-xxx-portal   awslinux  -- bash
    3  cat /etc/os-release 
    4  curl ftp://ftp.unixodbc.org/pub/unixODBC/unixODBC-2.3.7.tar.gz -O
    5  tar xzvf unixODBC-2.3.7.tar.gz
    6  cd unixODBC-2.3.7
    7  ./configure --sysconfdir=/opt --disable-gui --disable-drivers --enable-iconv --with-iconv-char-enc=UTF8 --with-iconv-ucode-enc=UTF16LE --prefix=/opt
    8  make
    9  make install
   10  cp include/*.h /usr/include/
   11  cd ..
   12  rm -rf unixODBC-2.3.7 unixODBC-2.3.7.tar.gz
   13  curl https://packages.microsoft.com/config/rhel/6/prod.repo > /etc/yum.repos.d/mssql-release.repo
   14  yum install -y e2fsprogs.x86_64 0:1.43.5-2.43.amzn1 fuse-libs.x86_64 0:2.9.4-1.18.amzn1 libss.x86_64 0:1.43.5-2.43.amzn1 openssl
   15  ACCEPT_EULA=Y yum install -y msodbcsql17 --disablerepo=amzn*
   16  export CFLAGS="-I/opt/include"
   17  export LDFLAGS="-L/opt/lib"
   18  cd /opt
   19  cp -r /opt/microsoft/msodbcsql17/ .
   20  rm -rf /opt/microsoft/
   21  mkdir /opt/python/
   22  cd /opt/python/
   23  pip install pyodbc -t .
   24  cd /opt
   25  cat <<EOF > odbcinst.ini
   26  [ODBC Driver 17 for SQL Server]
   27  Description=Microsoft ODBC Driver 17 for SQL Server
   28  Driver=/opt/msodbcsql17/lib64/libmsodbcsql-17.5.so.2.1
   29  UsageCount=1
   30  EOF
   31  cat <<EOF > odbc.ini
   32  [ODBC Driver 17 for SQL Server]
   33  Driver = ODBC Driver 17 for SQL Server
   34  Description = My ODBC Driver 17 for SQL Server
   35  Trace = No
   36  EOF
   37  cd /opt
   38  zip -r9 ~/pyodbc-layer.zip .
   40  aws configure
   42  aws s3 ls
   46  aws s3 cp  pyodbc-layer.zip s3://s3-migration-test-v1/pyodbc-layer.zip
   50  cat /opt/odbcinst.ini 
   53  cd /opt/
   54  ls
   55  rm odbcinst.ini
   56  cat odbc.ini 
   57  ls
   58  pwd
   59  cat <<EOF > odbcinst.ini
   60  [ODBC Driver 17 for SQL Server]
   61  Description=Microsoft ODBC Driver 17 for SQL Server
   62  Driver=/opt/msodbcsql17/lib64/libmsodbcsql-17.7.so.2.1
   63  UsageCount=1
   64  EOF
   65  ls
   66  rm ~/pyodbc-layer.zip 
   67  ls
   68  cd /opt
   69  zip -r9 ~/pyodbc-layer.zip .
   70  unzip pyodbc-layer.zip -d /var/opt/
   74  aws s3 cp  pyodbc-layer.zip s3://s3-migration-test-v1/pyodbc-layer.zip

# use https://github.com/lambci/docker-lambda to simulate a lambda environment
docker run -it --rm \
  --entrypoint bash \
  -e ODBCINI=/opt/odbc.ini \
  -e ODBCSYSINI=/opt/ \
  -v $PWD:/export \
  lambci/lambda:build-python3.8

# download and install unixODBC
# http://www.unixodbc.org/download.html
curl ftp://ftp.unixodbc.org/pub/unixODBC/unixODBC-2.3.7.tar.gz -O
tar xzvf unixODBC-2.3.7.tar.gz
cd unixODBC-2.3.7

./configure --sysconfdir=/opt --disable-gui --disable-drivers --enable-iconv --with-iconv-char-enc=UTF8 --with-iconv-ucode-enc=UTF16LE --prefix=/opt
make
make install
cp include/*.h /usr/include/

cd ..
rm -rf unixODBC-2.3.7 unixODBC-2.3.7.tar.gz

# download and install ODBC driver for MSSQL 17
# https://docs.microsoft.com/en-us/sql/connect/odbc/linux-mac/installing-the-microsoft-odbc-driver-for-sql-server?view=sql-server-2017
curl https://packages.microsoft.com/config/rhel/6/prod.repo > /etc/yum.repos.d/mssql-release.repo
yum install -y e2fsprogs.x86_64 0:1.43.5-2.43.amzn1 fuse-libs.x86_64 0:2.9.4-1.18.amzn1 libss.x86_64 0:1.43.5-2.43.amzn1 openssl
ACCEPT_EULA=Y yum install -y msodbcsql17 --disablerepo=amzn*
export CFLAGS="-I/opt/include"
export LDFLAGS="-L/opt/lib"

cd /opt
cp -r /opt/microsoft/msodbcsql17/ .
rm -rf /opt/microsoft/

# install pyodbc for use with python.
# Notice the folder structure to support python 3.7 runtime 
# https://docs.aws.amazon.com/lambda/latest/dg/configuration-layers.html#configuration-layers-path
mkdir /opt/python/
cd /opt/python/
pip install pyodbc -t .

cd /opt
cat <<EOF > odbcinst.ini
[ODBC Driver 17 for SQL Server]
Description=Microsoft ODBC Driver 17 for SQL Server
Driver=/opt/msodbcsql17/lib64/libmsodbcsql-17.5.so.2.1
UsageCount=1
EOF

cat <<EOF > odbc.ini
[ODBC Driver 17 for SQL Server]
Driver = ODBC Driver 17 for SQL Server
Description = My ODBC Driver 17 for SQL Server
Trace = No
EOF

# package the content in a zip file to use as a lambda layer
cd /opt
zip -r9 ~/pyodbc-layer.zip .

# to test it locally:
# unzip the content of your layer to your local environment, to /var/opt/ for example:
unzip pyodbc-layer.zip -d /var/opt/

# In your local environment, have your lambda function handy at /var/task/lambda_function.py

import pyodbc
server = 'myserver'
database =  'mydb'
username = 'myuser'
password = 'mypwd'

def lambda_handler(event, context):
    # TODO implement
    cnxn = pyodbc.connect('DRIVER={ODBC Driver 17 for SQL Server};SERVER='+server+';DATABASE='+database+';UID='+username+';PWD='+ password)
    cursor = cnxn.cursor()
    cursor.execute("select @@version;")
    row = cursor.fetchone()
    while row:
        print(row[0])
        row = cursor.fetchone()

# Time to test
docker run --rm -v /var/task:/var/task -v /var/opt:/opt lambci/lambda:python3.7 lambda_function.lambda_handler '{"some": "event"}'

# useful links
https://medium.com/devopslinks/aws-lambda-microsoft-sql-server-how-to-66c5f9d275ed
https://stackoverflow.com/questions/47682991/aws-lambda-function-to-connect-to-sql-server-with-python
https://gist.github.com/carlochess/658a98589709f46dbb3d20502e48556b