Support nginx LB with docker-compose example (#10434)

docs/orchestration/docker-compose/README.md

@@ -11,7 +11,7 @@ With Compose, you use a Compose file to configure MinIO services. Then, using a
 
 ## 2. Run Distributed MinIO on Docker Compose
 
-To deploy Distributed MinIO on Docker Compose, please download [docker-compose.yaml](https://github.com/minio/minio/blob/master/docs/orchestration/docker-compose/docker-compose.yaml?raw=true) to your current working directory. Note that Docker Compose pulls the MinIO Docker image, so there is no need to explicitly download MinIO binary. Then run one of the below commands
+To deploy Distributed MinIO on Docker Compose, please download [docker-compose.yaml](https://github.com/minio/minio/blob/master/docs/orchestration/docker-compose/docker-compose.yaml?raw=true) and [nginx.conf](https://github.com/minio/minio/blob/master/docs/orchestration/docker-compose/nginx.conf?raw=true) to your current working directory. Note that Docker Compose pulls the MinIO Docker image, so there is no need to explicitly download MinIO binary. Then run one of the below commands
 
 ### GNU/Linux and macOS
 
@@ -27,7 +27,7 @@ docker-compose.exe pull
 docker-compose.exe up
 ```
 
-Each instance is now accessible on the host at ports 9001 through 9004, proceed to access the Web browser at http://127.0.0.1:9001/
+Distributed instances are now accessible on the host at ports 9000, proceed to access the Web browser at http://127.0.0.1:9000/. Here 4 MinIO server instances are reverse proxied through Nginx load balancing.
 
 ### Notes
 
@@ -36,12 +36,10 @@ Each instance is now accessible on the host at ports 9001 through 9004, proceed
 * There are 4 minio distributed instances created by default. You can add more MinIO services (up to total 16) to your MinIO Compose deployment. To add a service
   * Replicate a service definition and change the name of the new service appropriately.
   * Update the command section in each service.
-  * Update the port number to exposed for the new service. Also, make sure the port assigned for the new service is not already being used on the host.
+  * Add a new MinIO server instance to the upstream directive in the Nginx configuration file.
 
   Read more about distributed MinIO [here](https://docs.min.io/docs/distributed-minio-quickstart-guide).
 
-* MinIO services in the Docker compose file expose ports 9001 to 9004. This allows multiple services to run on a host.
-
 ### Explore Further
 - [Overview of Docker Compose](https://docs.docker.com/compose/overview/)
 - [MinIO Docker Quickstart Guide](https://docs.min.io/docs/minio-docker-quickstart-guide)

docs/orchestration/docker-compose/docker-compose.yaml

@@ -1,16 +1,16 @@
 version: '3.7'
 
-# starts 4 docker containers running minio server instances. Each
+# starts 4 docker containers running minio server instances.
-# minio server's web interface will be accessible on the host at port
+# using nginx reverse proxy, load balancing, you can access
-# 9001 through 9004.
+# it through port 9000.
 services:
   minio1:
     image: minio/minio:RELEASE.2020-09-08T23-05-18Z
     volumes:
       - data1-1:/data1
       - data1-2:/data2
-    ports:
+    expose:
-      - "9001:9000"
+      - "9000"
     environment:
       MINIO_ACCESS_KEY: minio
       MINIO_SECRET_KEY: minio123
@@ -26,8 +26,8 @@ services:
     volumes:
       - data2-1:/data1
       - data2-2:/data2
-    ports:
+    expose:
-      - "9002:9000"
+      - "9000"
     environment:
       MINIO_ACCESS_KEY: minio
       MINIO_SECRET_KEY: minio123
@@ -43,8 +43,8 @@ services:
     volumes:
       - data3-1:/data1
       - data3-2:/data2
-    ports:
+    expose:
-      - "9003:9000"
+      - "9000"
     environment:
       MINIO_ACCESS_KEY: minio
       MINIO_SECRET_KEY: minio123
@@ -60,8 +60,8 @@ services:
     volumes:
       - data4-1:/data1
       - data4-2:/data2
-    ports:
+    expose:
-      - "9004:9000"
+      - "9000"
     environment:
       MINIO_ACCESS_KEY: minio
       MINIO_SECRET_KEY: minio123
@@ -72,6 +72,18 @@ services:
       timeout: 20s
       retries: 3
 
+  nginx:
+    image: nginx:1.19.2-alpine
+    volumes:
+      - ./nginx.conf:/etc/nginx/nginx.conf:ro
+    ports:
+      - "9000:9000"
+    depends_on:
+      - minio1
+      - minio2
+      - minio3
+      - minio4
+
 ## By default this config uses default local driver,
 ## For custom volumes replace with volume driver configuration.
 volumes:

docs/orchestration/docker-compose/nginx.conf

@@ -0,0 +1,68 @@
+
+user  nginx;
+worker_processes  auto;
+
+error_log  /var/log/nginx/error.log warn;
+pid        /var/run/nginx.pid;
+
+
+events {
+    worker_connections  1024;
+}
+
+
+http {
+    include       /etc/nginx/mime.types;
+    default_type  application/octet-stream;
+
+    log_format  main  '$remote_addr - $remote_user [$time_local] "$request" '
+                      '$status $body_bytes_sent "$http_referer" '
+                      '"$http_user_agent" "$http_x_forwarded_for"';
+
+    access_log  /var/log/nginx/access.log  main;
+
+    sendfile        on;
+    #tcp_nopush     on;
+
+    keepalive_timeout  65;
+
+    #gzip  on;
+
+    # include /etc/nginx/conf.d/*.conf;
+
+    upstream minio {
+        server minio1:9000;
+        server minio2:9000;
+        server minio3:9000;
+        server minio4:9000;
+    }
+
+    server {
+        listen       9000;
+        listen  [::]:9000;
+        server_name  localhost;
+
+         # To allow special characters in headers
+         ignore_invalid_headers off;
+         # Allow any size file to be uploaded.
+         # Set to a value such as 1000m; to restrict file size to a specific value
+         client_max_body_size 0;
+         # To disable buffering
+         proxy_buffering off;
+
+        location / {
+            proxy_set_header Host $http_host;
+            proxy_set_header X-Real-IP $remote_addr;
+            proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
+            proxy_set_header X-Forwarded-Proto $scheme;
+
+            proxy_connect_timeout 300;
+            # Default is HTTP/1, keepalive is only enabled in HTTP/1.1
+            proxy_http_version 1.1;
+            proxy_set_header Connection "";
+            chunked_transfer_encoding off;
+
+            proxy_pass http://minio;
+        }
+    }
+}