270 Nginx as load balancer and reverse proxy in Docker - chempkovsky/CS82ANGULAR Wiki

Notes

Steps required to accomplish the task

Notes

  • current Hand-on-lab will be base on the articles
  • we are going to
    • create three WebApi apps
      • with only http connection (https will be disabled)
      • with Bearer token protection
    • configure Nginx
      • only with ssl enabled
      • which redirects an HTTPS requests to an HTTP-only WebApi applications
        • it means Nginx will work as a reverse proxy
      • which works as a load balancer for WebApi applications

Steps required to accomplish the task

Create solution

  • choose any drive you like and create NginxBalancer folder
    • we selected E:\-drive
mkdir NginxBalancer
cd NginxBalancer
  • create solution
dotnet new sln -n NginxBalancerSolution

Create three WebApi projects

dotnet new webapi -o WebApplication1
dotnet sln NginxBalancerSolution.sln add WebApplication1/WebApplication1.csproj
dotnet add WebApplication1/WebApplication1.csproj package Microsoft.AspNetCore.Authentication.JwtBearer

dotnet new webapi -o WebApplication2
dotnet sln NginxBalancerSolution.sln add WebApplication2/WebApplication2.csproj
dotnet add WebApplication2/WebApplication2.csproj package Microsoft.AspNetCore.Authentication.JwtBearer

dotnet new webapi -o WebApplication3
dotnet sln NginxBalancerSolution.sln add WebApplication3/WebApplication3.csproj
dotnet add WebApplication3/WebApplication3.csproj package Microsoft.AspNetCore.Authentication.JwtBearer

Modify appsettings json

  • for each created project modify appsettings.json as follows
Click to show the file
{
  "Logging": {
    "LogLevel": {
      "Default": "Information",
      "Microsoft.AspNetCore": "Warning"
    }
  },
  "AllowedHosts": "*",
  "JWT": {
    "ValidAudience": "NginxBalancerAudience",
    "ValidIssuer": "NginxBalancerIssuer",
    "Secret": "[email protected]@56"
  }
}

Modify Program file

  • for each created project modify Program.cs as follows
Click to show the file
using Microsoft.AspNetCore.Authentication.JwtBearer;
using Microsoft.IdentityModel.Tokens;
using System.Text;

var builder = WebApplication.CreateBuilder(args);

ConfigurationManager configuration = builder.Configuration;

#region authentification
builder.Services.AddAuthentication(options => {
    options.DefaultAuthenticateScheme = JwtBearerDefaults.AuthenticationScheme;
    options.DefaultChallengeScheme = JwtBearerDefaults.AuthenticationScheme;
    options.DefaultScheme = JwtBearerDefaults.AuthenticationScheme;
}).AddJwtBearer(options => {
    options.SaveToken = true;
    options.RequireHttpsMetadata = true;
    options.TokenValidationParameters = new TokenValidationParameters()
    {
        ValidateIssuer = true,
        ValidateAudience = true,
        ValidAudience = configuration["JWT:ValidAudience"],
        ValidIssuer = configuration["JWT:ValidIssuer"],
        IssuerSigningKey = new SymmetricSecurityKey(Encoding.UTF8.GetBytes(configuration["JWT:Secret"]))
    };
});
builder.Services.AddHttpContextAccessor();
#endregion


// Add services to the container.

builder.Services.AddControllers();
// Learn more about configuring Swagger/OpenAPI at https://aka.ms/aspnetcore/swashbuckle
builder.Services.AddEndpointsApiExplorer();
builder.Services.AddSwaggerGen();

var app = builder.Build();

app.UseForwardedHeaders();

// Configure the HTTP request pipeline.
if (app.Environment.IsDevelopment())
{
    app.UseSwagger();
    app.UseSwaggerUI();
}

// app.UseHttpsRedirection();

app.UseAuthentication();
app.UseAuthorization();

app.MapControllers();

app.Run();

Modify WeatherForecastController

  • For the WebApplication1 project
    • replace [Route("[controller]")] with [Route("app1/[controller]")]
    • replace [HttpGet(Name = "GetWeatherForecast")] with [HttpGet] [Route("Weather")]
Click to show the code
[ApiController]
[Route("app1/[controller]")]
public class WeatherForecastController : ControllerBase
{
...
    [HttpGet]
    [Route("Weather")]
    public IEnumerable<WeatherForecast> Get()
    {
    ...
    }

}
  • For the WebApplication2 project
    • replace [Route("[controller]")] with [Route("app1/[controller]")]
    • replace [HttpGet(Name = "GetWeatherForecast")] with [HttpGet] [Route("Weather")]
Click to show the code
[ApiController]
[Route("app2/[controller]")]
public class WeatherForecastController : ControllerBase
{
...
    [HttpGet]
    [Route("Weather")]
    public IEnumerable<WeatherForecast> Get()
    {
    ...
    }

}
  • For the WebApplication3 project
    • replace [Route("[controller]")] with [Route("app1/[controller]")]
    • replace [HttpGet(Name = "GetWeatherForecast")] with [HttpGet] [Route("Weather")]
Click to show the code
[ApiController]
[Route("app3/[controller]")]
public class WeatherForecastController : ControllerBase
{
...
    [HttpGet]
    [Route("Weather")]
    public IEnumerable<WeatherForecast> Get()
    {
    ...
    }
}

Add token method to the Controller of WebApplication1

  • modify constructor in the WeatherForecastController.cs file of the WebApplication1-project
  • add WebApi-method to WeatherForecastController.cs of the WebApplication1-project
Click to show the code
using Microsoft.AspNetCore.Mvc;
using Microsoft.IdentityModel.Tokens;
using System.IdentityModel.Tokens.Jwt;
using System.Security.Claims;
using System.Text;

namespace WebApplication1.Controllers;

[ApiController]
[Route("app1/[controller]")]
public class WeatherForecastController : ControllerBase
{
    private static readonly string[] Summaries = new[]
    {
        "Freezing", "Bracing", "Chilly", "Cool", "Mild", "Warm", "Balmy", "Hot", "Sweltering", "Scorching"
    };

    private readonly ILogger<WeatherForecastController> _logger;

    private readonly IConfiguration _configuration;


    public WeatherForecastController(ILogger<WeatherForecastController> logger, IConfiguration configuration)
    {
        _logger = logger;
        _configuration = configuration;
    }


    [HttpGet]
    [Route("Weather")]
    public IEnumerable<WeatherForecast> Get()
    {
        return Enumerable.Range(1, 5).Select(index => new WeatherForecast
        {
            Date = DateTime.Now.AddDays(index),
            TemperatureC = Random.Shared.Next(-20, 55),
            Summary = Summaries[Random.Shared.Next(Summaries.Length)]
        })
        .ToArray();
    }

    [HttpGet]
    [Route("token")]
    public IActionResult GetToken()
    {

        var authClaims = new List<Claim>
                {
                    new Claim(ClaimTypes.Name, "userUserName"),
                    new Claim(JwtRegisteredClaimNames.Jti, Guid.NewGuid().ToString()),
                };

        authClaims.Add(new Claim(ClaimTypes.Role, "AdminRole"));
        authClaims.Add(new Claim(ClaimTypes.Role, "GuestRole"));

        var token = GetToken(authClaims);

        return Ok(new
        {
            token_type = "Bearer",
            user_name = "userUserName",
            access_token = new JwtSecurityTokenHandler().WriteToken(token),
            expiration = token.ValidTo
        });
    }

    private JwtSecurityToken GetToken(List<Claim> authClaims)
    {
        var authSigningKey = new SymmetricSecurityKey(Encoding.UTF8.GetBytes(_configuration["JWT:Secret"]));

        var token = new JwtSecurityToken(
            issuer: _configuration["JWT:ValidIssuer"],
            audience: _configuration["JWT:ValidAudience"],
            expires: DateTime.Now.AddHours(8),
            claims: authClaims,
            signingCredentials: new SigningCredentials(authSigningKey, SecurityAlgorithms.HmacSha256)
            );

        return token;
    }

}
- the method we added return `Bearer`-token with two roles - `AdminRole` - `GuestRole`

Protect WebApi method of the WebApplication2 project with AdminRole

  • in the WebApplication2 project
    • add [Authorize(Roles = "AdminRole")] attribute for the Get() method of the WeatherForecastController
Click to show the code
[ApiController]
[Route("app1/[controller]")]
public class WeatherForecastController : ControllerBase
{
    ...
    [HttpGet]
    [Route("Weather")]
    [Authorize(Roles = "AdminRole")]
    public IEnumerable<WeatherForecast> Get()
    {
    ...
    ]
}

Docker file for each project

  • with Visual Studio 2022 right-click each project
    • select Add/Docker support menu item
      • select linux
      • click Ok button

Create Docker Images

  • make solution folder active
    • in our case it is e:\NginxBalancer
  • in the command prompt run the commands:
docker build . -f ./WebApplication1/Dockerfile -t webapplication1
docker build . -f ./WebApplication2/Dockerfile -t webapplication2
docker build . -f ./WebApplication3/Dockerfile -t webapplication3

Create nginx config file

  • in the solution folder create nginxdefault.conf file with the content as follows
Click to show the file
server {

    listen                443 ssl;
    server_name           localhost;
    ssl_certificate       /run/secrets/site.crt;
    ssl_certificate_key   /run/secrets/site.key;

    resolver 127.0.0.11 valid=1s;
    set $webapplication01  http://webapplication01:80;
    set $webapplication02  http://webapplication02:80;
    set $webapplication03  http://webapplication03:80;

    location /app2/ {
        proxy_pass         $webapplication02;
        proxy_http_version 1.1;
        proxy_set_header   Upgrade $http_upgrade;
        proxy_set_header   Connection keep-alive;
        proxy_set_header   Host $host;
        proxy_cache_bypass $http_upgrade;
        proxy_set_header   X-Forwarded-For $proxy_add_x_forwarded_for;
        proxy_set_header   X-Forwarded-Proto $scheme;
    }
    location /app3/ {
        proxy_pass         $webapplication03;
        proxy_http_version 1.1;
        proxy_set_header   Upgrade $http_upgrade;
        proxy_set_header   Connection keep-alive;
        proxy_set_header   Host $host;
        proxy_cache_bypass $http_upgrade;
        proxy_set_header   X-Forwarded-For $proxy_add_x_forwarded_for;
        proxy_set_header   X-Forwarded-Proto $scheme;
    }
    location / {
        proxy_pass         $webapplication01;
        proxy_http_version 1.1;
        proxy_set_header   Upgrade $http_upgrade;
        proxy_set_header   Connection keep-alive;
        proxy_set_header   Host $host;
        proxy_cache_bypass $http_upgrade;
        proxy_set_header   X-Forwarded-For $proxy_add_x_forwarded_for;
        proxy_set_header   X-Forwarded-Proto $scheme;
    }

    #error_page  404              /404.html;

    # redirect server error pages to the static page /50x.html
    #
    error_page   500 502 503 504  /50x.html;
}
  • Nginx.conf declares
    • listen on port 443
    • use ssl connection
    • use site.crt and site.key files for ssl connection
    • three entities which is defined with a resolver
      • $webapplication01
      • $webapplication02
      • $webapplication03
    • three locations
    • /
    • /app2/
    • /app3/
  • resolver is defined by IP=127.0.0.11, which is a Docker Dns-server

Create certificate

openssl genrsa -out "root-ca.key" 4096

openssl req -new -key "root-ca.key" -out "root-ca.csr" -sha256 -subj '/C=US/ST=CA/L=San Francisco/O=Docker/CN=Swarm Secret Example CA'

  • create root-ca.cnf file with the content
[root_ca]
basicConstraints = critical,CA:TRUE,pathlen:1
keyUsage = critical, nonRepudiation, cRLSign, keyCertSign
subjectKeyIdentifier=hash
  • run three commands
openssl x509 -req -days 3650 -in "root-ca.csr" -signkey "root-ca.key" -sha256 -out "root-ca.crt" -extfile "root-ca.cnf" -extensions root_ca

openssl genrsa -out "site.key" 4096

openssl req -new -key "site.key" -out "site.csr" -sha256 -subj '/C=US/ST=CA/L=San Francisco/O=Docker/CN=localhost'
  • create site.cnf file with the content
[server]
authorityKeyIdentifier=keyid,issuer
basicConstraints = critical,CA:FALSE
extendedKeyUsage=serverAuth
keyUsage = critical, digitalSignature, keyEncipherment
subjectAltName = DNS:localhost, IP:127.0.0.1
subjectKeyIdentifier=hash
  • run the command
openssl x509 -req -days 750 -in "site.csr" -sha256 -CA "root-ca.crt" -CAkey "root-ca.key" -CAcreateserial -out "site.crt" -extfile "site.cnf" -extensions server
  • here is a result
E:\NginxBalancer>dir
 Volume in drive E is Local Drive
 Volume Serial Number is 6041-FF43

 Directory of E:\NginxBalancer

06/23/2022  05:24 PM    <DIR>          .
06/23/2022  05:24 PM    <DIR>          ..
06/23/2022  04:34 PM               340 .dockerignore
06/23/2022  05:19 PM               221 Dockerfile
06/23/2022  03:45 PM             2,020 NginxBalancerSolution.sln
06/23/2022  05:25 PM             1,822 nginxdefault.conf
06/23/2022  05:08 PM               142 root-ca.cnf
06/23/2022  05:11 PM             2,009 root-ca.crt
06/23/2022  05:09 PM             1,695 root-ca.csr
06/23/2022  05:05 PM             3,243 root-ca.key
06/23/2022  05:14 PM                41 root-ca.srl
06/23/2022  05:13 PM               241 site.cnf
06/23/2022  05:14 PM             2,094 site.crt
06/23/2022  05:12 PM             1,675 site.csr
06/23/2022  05:11 PM             3,243 site.key
06/23/2022  04:34 PM    <DIR>          WebApplication1
06/23/2022  04:45 PM    <DIR>          WebApplication2
06/23/2022  04:46 PM    <DIR>          WebApplication3
              13 File(s)         18,786 bytes
               5 Dir(s)  269,358,137,344 bytes free

E:\NginxBalancer>

Create Docker file for Nginx

  • make solution folder active
    • in our case it is e:\NginxBalancer
  • create Dockerfile-file with the content
Click to show the file
FROM nginx:latest as nginxrslt
COPY nginxdefault.conf /etc/nginx/conf.d/default.conf
COPY root-ca.crt /run/secrets/site.crt
COPY root-ca.key /run/secrets/site.key
CMD ["/bin/sh",  "-c",  "exec nginx -g 'daemon off;'"]

Create Image for Nginx

  • make solution folder active
    • in our case it is e:\NginxBalancer
  • run the command
docker build . -f Dockerfile -t nginxlb

Create Docker compose file

  • make solution folder active
    • in our case it is e:\NginxBalancer
  • create docker-compose.yml-file with the content
Click to show the file
services:
  webapplication01:
    image: "webapplication1:latest"
    deploy:
      replicas: 2    
    expose:
      - "80"
  webapplication02:
    image: "webapplication2:latest"
    deploy:
      replicas: 2    
    expose:
      - "80"
  webapplication03:
    image: "webapplication3:latest"
    deploy:
      replicas: 2    
    expose:
      - "80"
  nginxlb:
    image: "nginxlb:latest"
    ports:
      - 91:443
    depends_on:
      - webapplication01
      - webapplication02
      - webapplication03      

Run Docker compose

  • make solution folder active
    • in our case it is e:\NginxBalancer
  • run the command
docker-compose -f "docker-compose.yml" up -d
  • to stop app
    • run the command
docker-compose -f "docker-compose.yml" down

Call Webapi methods

Proxy

  • go to URL=https://localhost:91/app1/WeatherForecast/token
    • the Browser show Your connection is not private-message
      • click Advances button
      • click proceed
  • the response will be similar to the following
{"token_type":"Bearer","user_name":"userUserName","access_token":"eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJodHRwOi8vc2NoZW1hcy54bWxzb2FwLm9yZy93cy8yMDA1LzA1L2lkZW50aXR5L2NsYWltcy9uYW1lIjoidXNlclVzZXJOYW1lIiwianRpIjoiMzliMzFmNzktNWQxNS00YTRkLWIyNmYtZjcyNWIzNzc4MDczIiwiaHR0cDovL3NjaGVtYXMubWljcm9zb2Z0LmNvbS93cy8yMDA4LzA2L2lkZW50aXR5L2NsYWltcy9yb2xlIjpbIkFkbWluUm9sZSIsIkd1ZXN0Um9sZSJdLCJleHAiOjE2NTYwMjQxMDYsImlzcyI6Ik5naW54QmFsYW5jZXJJc3N1ZXIiLCJhdWQiOiJOZ2lueEJhbGFuY2VyQXVkaWVuY2UifQ.qA2k3BJYzk4qpAbP4rpIhYmkwpRQtLvIAEkRvK6mRMc","expiration":"2022-06-23T22:41:46Z"}
  • We need to test if the service of the WebApplication2 project has role-based protection
    • the url for the method of the second project is as follows
https://localhost:91/app2/WeatherForecast/Weather
  • We installed Yet Another REST Client extensions. Using it we are going to call protected method
  • The first call was without Authentication:
    • we took 404-status code
Click to show the picture

project structure

  • The second call was with Authentication in the form:
Bearer eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJodHRwOi8vc2NoZW1hcy54bWxzb2FwLm9yZy93cy8yMDA1LzA1L2lkZW50aXR5L2NsYWltcy9uYW1lIjoidXNlclVzZXJOYW1lIiwianRpIjoiMzliMzFmNzktNWQxNS00YTRkLWIyNmYtZjcyNWIzNzc4MDczIiwiaHR0cDovL3NjaGVtYXMubWljcm9zb2Z0LmNvbS93cy8yMDA4LzA2L2lkZW50aXR5L2NsYWltcy9yb2xlIjpbIkFkbWluUm9sZSIsIkd1ZXN0Um9sZSJdLCJleHAiOjE2NTYwMjQxMDYsImlzcyI6Ik5naW54QmFsYW5jZXJJc3N1ZXIiLCJhdWQiOiJOZ2lueEJhbGFuY2VyQXVkaWVuY2UifQ.qA2k3BJYzk4qpAbP4rpIhYmkwpRQtLvIAEkRvK6mRMc
Click to show the picture

project structure

Balancer

  • stop app
docker-compose -f "docker-compose.yml" down
  • In the WebApplication3 project
    • we add _logger.LogInformation method call as follwos
    [HttpGet]
    [Route("Weather")]
    public IEnumerable<WeatherForecast> Get()
    {
        _logger.LogInformation("Before App3 Get called at {DT}", DateTime.UtcNow.ToLongTimeString());
        return Enumerable.Range(1, 5).Select(index => new WeatherForecast
        {
            Date = DateTime.Now.AddDays(index),
            TemperatureC = Random.Shared.Next(-20, 55),
            Summary = Summaries[Random.Shared.Next(Summaries.Length)]
        })
        .ToArray();
    }

rebuild the docker image

docker build . -f ./WebApplication3/Dockerfile -t webapplication3
  • start app with the command without -d flag
docker-compose -f "docker-compose.yml" up
  • in the browser run goto url=https://localhost:91/app3/WeatherForecast/Weather
  • we refreshed several times with 2s interval
  • here is a result
nginxbalancer-webapplication03-1  | info: WebApplication3.Controllers.WeatherForecastController[0]
nginxbalancer-webapplication03-1  |       Before App3 Get called at 15:53:48
nginxbalancer-nginxlb-1           | 192.168.48.1 - - [23/Jun/2022:15:53:48 +0000] "GET /app3/WeatherForecast/Weather HTTP/1.1" 200 508 "-" "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/102.0.0.0 Safari/537.36" "-"
nginxbalancer-webapplication03-2  | info: WebApplication3.Controllers.WeatherForecastController[0]
nginxbalancer-webapplication03-2  |       Before App3 Get called at 15:53:51
nginxbalancer-nginxlb-1           | 192.168.48.1 - - [23/Jun/2022:15:53:51 +0000] "GET /app3/WeatherForecast/Weather HTTP/1.1" 200 520 "-" "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/102.0.0.0 Safari/537.36" "-"
nginxbalancer-webapplication03-1  | info: WebApplication3.Controllers.WeatherForecastController[0]
nginxbalancer-webapplication03-1  |       Before App3 Get called at 15:53:53
nginxbalancer-nginxlb-1           | 192.168.48.1 - - [23/Jun/2022:15:53:53 +0000] "GET /app3/WeatherForecast/Weather HTTP/1.1" 200 522 "-" "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/102.0.0.0 Safari/537.36" "-"
nginxbalancer-webapplication03-1  | info: WebApplication3.Controllers.WeatherForecastController[0]
nginxbalancer-webapplication03-1  |       Before App3 Get called at 15:53:59
nginxbalancer-nginxlb-1           | 192.168.48.1 - - [23/Jun/2022:15:53:59 +0000] "GET /app3/WeatherForecast/Weather HTTP/1.1" 200 512 "-" "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/102.0.0.0 Safari/537.36" "-"
nginxbalancer-webapplication03-2  | info: WebApplication3.Controllers.WeatherForecastController[0]
nginxbalancer-webapplication03-2  |       Before App3 Get called at 15:54:00
nginxbalancer-nginxlb-1           | 192.168.48.1 - - [23/Jun/2022:15:54:00 +0000] "GET /app3/WeatherForecast/Weather HTTP/1.1" 200 511 "-" "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/102.0.0.0 Safari/537.36" "-"
⚠️ **GitHub.com Fallback** ⚠️