Styjun

I have an nginx reverse proxy sitting in front of an aspnetcore application that authenticates against AzureAD, When I run in a swarm with a single replica all works fine, however when I have more than 1 replica I get the following Error

Exception: An error was encountered while handling the remote login.
Microsoft.AspNetCore.Authentication.RemoteAuthenticationHandler<TOptions>.HandleRequestAsync()
Microsoft.AspNetCore.Authentication.AuthenticationMiddleware.Invoke(HttpContext context)
Microsoft.AspNetCore.StaticFiles.StaticFileMiddleware.Invoke(HttpContext context)
Microsoft.AspNetCore.Builder.Extensions.MapWhenMiddleware.Invoke(HttpContext context)
Microsoft.AspNetCore.Diagnostics.DeveloperExceptionPageMiddleware.Invoke(HttpContext context)

Below is my Startup.cs

using System.IO;
using Microsoft.AspNetCore.Authentication;
using Microsoft.AspNetCore.Authentication.AzureAD.UI;
using Microsoft.AspNetCore.Authorization;
using Microsoft.AspNetCore.Builder;
using Microsoft.AspNetCore.Hosting;
using Microsoft.AspNetCore.Http;
using Microsoft.AspNetCore.Mvc;
using Microsoft.AspNetCore.Mvc.Authorization;
using Microsoft.Extensions.Configuration;
using Microsoft.Extensions.DependencyInjection;
using DataAccess.Models;
using Newtonsoft.Json.Serialization;
using Microsoft.AspNetCore.HttpOverrides;
using System.Globalization;
using Microsoft.Net.Http.Headers;
using System.Data.SqlClient;
using Microsoft.Extensions.Diagnostics.HealthChecks;
using Microsoft.AspNetCore.Diagnostics.HealthChecks;
using Newtonsoft.Json;
using System.Linq;

namespace www 
 public class Startup 
 public Startup(IConfiguration configuration) 

 CultureInfo.DefaultThreadCurrentCulture = new CultureInfo("en-US");

 Configuration = configuration;

 var builder = new ConfigurationBuilder()
 .AddJsonFile("appsettings.json")
 .AddEnvironmentVariables()
 ;
 Configuration = builder.Build();

 TradingContext.ConnectionString = $"Server=Configuration.GetSection("SQL_SERVER").Value;Database=Trading;User ID=Configuration.GetSection("SQL_USER").Value;Password=Configuration.GetSection("SQL_PASSWORD").Value;MultipleActiveResultSets=true;";

 

 public IConfiguration Configuration get; 

 public void ConfigureServices(IServiceCollection services) 

 services.AddHealthChecks()
 .AddCheck("SqlServer", () => 
 try 
 using (var db = new TradingContext()) 
 var x = db.InstrumentRoots.Count();
 
 
 catch 
 return HealthCheckResult.Unhealthy();
 

 return HealthCheckResult.Healthy();
 
 );

 services.AddMvc().AddJsonOptions(options => 
 var resolver = options.SerializerSettings.ContractResolver;
 if (resolver != null) 
 var res = (DefaultContractResolver)resolver;
 res.NamingStrategy = null;
 
 );

 services.Configure<ForwardedHeadersOptions>(options => 
 options.ForwardedHeaders =
 ForwardedHeaders.XForwardedFor );

 services.Configure<EmailSettings>(Configuration.GetSection("EmailSettings"));
 services.AddTransient<IEmailSender, AuthMessageSender>();

 services.Configure<CookiePolicyOptions>(options => 
 options.CheckConsentNeeded = context => true;
 options.MinimumSameSitePolicy = Microsoft.AspNetCore.Http.SameSiteMode.None;
 );

 services.AddDbContext<TradingContext>();

 services.AddAuthentication(AzureADDefaults.AuthenticationScheme)
 .AddAzureAD(options => Configuration.Bind("AzureAd", options));

 services.AddMvc(options => 
 var policy = new AuthorizationPolicyBuilder()
 .RequireAuthenticatedUser()
 .Build();
 options.Filters.Add(new AuthorizeFilter(policy));
 )
 .AddRazorPagesOptions(options => 
 options.Conventions.AddPageRoute("/Dashboard/Index", "");
 )
 .SetCompatibilityVersion(CompatibilityVersion.Version_2_2);
 

 public void Configure(IApplicationBuilder app, IHostingEnvironment env) 
 if (env.IsDevelopment()) 
 app.UseDeveloperExceptionPage();
 
 else 
 app.UseExceptionHandler("/Error");
 

 var options = new HealthCheckOptions();
 options.ResponseWriter = async (c, r) => 
 c.Response.ContentType = "application/json";

 var result = JsonConvert.SerializeObject(new 
 status = r.Status.ToString(),
 errors = r.Entries.Select(e => new key = e.Key, value = e.Value.Status.ToString() )
 );
 await c.Response.WriteAsync(result);
 ;
 app.UseHealthChecks("/healthcheck", options);

 app.UseCookiePolicy();

 app.UseStaticFiles(new StaticFileOptions 
 OnPrepareResponse = ctx =>
 
 const int durationInSeconds = 60 * 60 * 24;
 ctx.Context.Response.Headers[HeaderNames.CacheControl] =
 "public,max-age=" + durationInSeconds;
 
 );

 var fordwardedHeaderOptions = new ForwardedHeadersOptions ForwardedHeaders.XForwardedProto
 ;
 fordwardedHeaderOptions.KnownNetworks.Clear();
 fordwardedHeaderOptions.KnownProxies.Clear();

 app.UseForwardedHeaders(fordwardedHeaderOptions);

 app.UseAuthentication();

 app.UseMvc();
 
 

Below is my nginx.config

events 



http 
 proxy_buffer_size 128k;
 proxy_buffers 4 256k;
 proxy_busy_buffers_size 256k;
 large_client_header_buffers 4 16k;
 gzip on;

 server 
 listen 80;
 listen [::]:80;
 server_name xxxx;

 location / 
 rewrite ^ https://$host$request_uri? permanent;
 

 #for certbot challenges (renewal process)
 location ~ /.well-known/acme-challenge 
 allow all;
 root /data/letsencrypt;
 
 

 #https://xxxx
 server 
 server_name xxxx;
 listen 443 ssl http2;

 server_tokens off;

 ssl_buffer_size 8k;
 ssl_dhparam /etc/ssl/certs/dhparam-2048.pem;

 ssl_protocols TLSv1.3 TLSv1.2;
 ssl_prefer_server_ciphers on;
 ssl_ciphers ECDH+AESGCM:ECDH+AES256:ECDH+AES128:DH+3DES:!ADH:!AECDH:!MD5;

 ssl_ecdh_curve secp384r1;
 ssl_session_tickets off;

 # OCSP stapling
 ssl_stapling on;
 ssl_stapling_verify on;
 resolver 8.8.8.8 8.8.4.4;

 ssl_certificate /etc/letsencrypt/live/xxxxx/fullchain.pem;
 ssl_certificate_key /etc/letsencrypt/live/xxxxx/privkey.pem;

 location / 
 proxy_pass http://www:5000;
 proxy_http_version 1.1;
 proxy_set_header Upgrade $http_upgrade;
 proxy_set_header Connection keep-alive;
 proxy_set_header Host $http_host;
 proxy_cache_bypass $http_upgrade;
 proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
 proxy_set_header X-Forwarded-Proto $scheme;
 fastcgi_buffers 16 16k;
 fastcgi_buffer_size 32k;

 #security headers
 add_header Strict-Transport-Security "max-age=31536000; includeSubDomains; preload";
 add_header X-XSS-Protection "1; mode=block" always;
 add_header X-Content-Type-Options "nosniff" always;
 add_header X-Frame-Options "DENY" always;
 #CSP
 #add_header Content-Security-Policy "frame-src 'self'; default-src 'self'; script-src 'self' 'unsafe-inline' https://maxcdn.bootstrapcdn.com https://ajax.googleapis.com; img-src 'self'; style-src 'self' https://maxcdn.bootstrapcdn.com; font-src 'self' data: https://maxcdn.bootstrapcdn.com; form-action 'self'; upgrade-insecure-requests;" always;
 add_header Referrer-Policy "strict-origin-when-cross-origin" always;
 
 

Any Idea what I am missing?