docker-ise backend

.gitignore

@@ -1,9 +1,9 @@
 *.csv
 .idea/
-config.json
+backend/api/backend/static/
-backend/static/
 *.gexf
-whitelist.txt
+backend/api/backend/whitelist.txt
+data/
 
 # Byte-compiled / optimized / DLL files
 __pycache__/
@@ -90,6 +90,7 @@ celerybeat-schedule
 
 # Environments
 .env
+.env*
 .venv
 env/
 venv/

backend/api/Dockerfile

@@ -0,0 +1,12 @@
+FROM python:3
+ENV PYTHONUNBUFFERED 1
+
+RUN apt-get update && \
+    apt-get install -qqy --no-install-recommends \
+        postgresql-client-9.6=9.6.10-0+deb9u1
+
+RUN mkdir /code
+WORKDIR /code
+COPY requirements.txt /code/
+RUN pip install -r requirements.txt
+COPY . /code/

backend/api/backend/settings/base.py

@@ -14,35 +14,11 @@ import os
 import json
 from django.core.exceptions import ImproperlyConfigured
 
-with open(os.environ.get('FEDIVERSE_CONFIG')) as f:
+SECRET_KEY = os.getenv("SECRET_KEY")
-    configs = json.loads(f.read())
-
-
-def get_config(setting):
-    try:
-        val = configs[setting]
-        if val == 'True':
-            val = True
-        elif val == 'False':
-            val = False
-        return val
-    except KeyError:
-        error_msg = "ImproperlyConfigured: Set {0} environment variable".format(setting)
-        raise ImproperlyConfigured(error_msg)
-
-
-SECRET_KEY = get_config("SECRET_KEY")
 
 # Build paths inside the project like this: os.path.join(BASE_DIR, ...)
 BASE_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
 
-
-# Quick-start development settings - unsuitable for production
-# See https://docs.djangoproject.com/en/2.1/howto/deployment/checklist/
-
-ALLOWED_HOSTS = []
-
-
 # Application definition
 
 INSTALLED_APPS = [
@@ -76,7 +52,7 @@ ROOT_URLCONF = 'backend.urls'
 TEMPLATES = [
     {
         'BACKEND': 'django.template.backends.django.DjangoTemplates',
-        'DIRS': [os.path.join(BASE_DIR, '../frontend/build')],
+        'DIRS': [os.path.join(BASE_DIR, '../../frontend/build')],
         'APP_DIRS': True,
         'OPTIONS': {
             'context_processors': [
@@ -97,10 +73,12 @@ WSGI_APPLICATION = 'backend.wsgi.application'
 
 DATABASES = {
     'default': {
-        'ENGINE': get_config("DATABASE_ENGINE"),
+        'ENGINE': 'django.db.backends.postgresql',
-        'NAME': get_config("DATABASE_NAME"),
+        'NAME': os.getenv("POSTGRES_DB"),
-        'USER': get_config("DATABASE_USER"),
+        'USER': os.getenv("POSTGRES_USER"),
-        'PASSWORD': get_config("DATABASE_PASSWORD"),
+        'PASSWORD': os.getenv("POSTGRES_PASSWORD"),
+        'HOST': 'db',
+        'PORT': 5432,
     }
 }
 
@@ -143,7 +121,7 @@ USE_TZ = False
 
 STATIC_URL = '/static/'
 STATICFILES_DIRS = [
-    os.path.join(BASE_DIR, '../frontend/build/static')
+    os.path.join(BASE_DIR, '../../frontend/build/static')
 ]
 STATIC_ROOT = os.path.join(BASE_DIR, 'static')
 

backend/api/backend/settings/dev.py

@@ -2,7 +2,7 @@ from .base import *
 
 DEBUG = True
 
-ALLOWED_HOSTS += ['localhost']
+ALLOWED_HOSTS = ['localhost']
 
 CORS_ORIGIN_WHITELIST = [
     'localhost:3000',

backend/api/backend/settings/production.py

@@ -1,6 +1,8 @@
 from .base import *
 
-ALLOWED_HOSTS += ['fediverse.space']
+DEBUG = False
+
+ALLOWED_HOSTS = ['fediverse.space']
 
 CORS_ORIGIN_WHITELIST = [
     'fediverse.space',

backend/api/backend/wsgi.py

@@ -8,9 +8,6 @@ https://docs.djangoproject.com/en/2.1/howto/deployment/wsgi/
 """
 
 import os
-
 from django.core.wsgi import get_wsgi_application
 
-os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'backend.settings.production')
-
 application = get_wsgi_application()

backend/api/manage.py

@@ -3,9 +3,9 @@ import os
 import sys
 
 if __name__ == '__main__':
-    os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'backend.settings')
     try:
         from django.core.management import execute_from_command_line
+
     except ImportError as exc:
         raise ImportError(
             "Couldn't import Django. Are you sure it's installed and "

backend/api/requirements.txt

@@ -10,6 +10,7 @@ django-silk==3.0.1
 djangorestframework==3.8.2
 future==0.16.0
 gprof2dot==2016.10.13
+gunicorn==19.9.0
 idna==2.7
 Jinja2==2.10
 MarkupSafe==1.0

backend/api/scripts/docker-entrypoint.sh

@@ -0,0 +1,26 @@
+#! /bin/bash
+
+SLEEP_SECONDS=3
+
+>&2 echo "Checking Postgres status..."
+
+# https://docs.docker.com/compose/startup-order/
+export PGPASSWORD=$POSTGRES_PASSWORD
+until psql -h db -U "$POSTGRES_USER" -p 5432 -d "$POSTGRES_DB" -c '\q'
+do
+  >&2 echo "Postgres is unavailable - sleeping"
+  sleep $SLEEP_SECONDS
+done
+>&2 echo "Postgres is up"
+
+python manage.py migrate --noinput
+
+
+if [[ $ENVIRONMENT == "development" ]]
+then
+  >&2 echo "Running Django server on port 8000 for development"
+  python manage.py runserver 0.0.0.0:8000
+else
+  >&2 echo "Running gunicorn server"
+  gunicorn backend.wsgi -c /config/gunicorn.conf.py
+fi

config.json.template

@@ -1,7 +0,0 @@
-{
-    "SECRET_KEY": "df4p)6h(#ktn*#ckh2m^^(-)w6_9mn$b%^!+$02vnb&e3sz&!r",
-    "DATABASE_ENGINE": "django.db.backends.sqlite",
-    "DATABASE_NAME": "os.path.join(BASE_DIR, 'db.sqlite3')",
-    "DATABASE_USER": "",
-    "DATABASE_PASSWORD": ""
-}

config/gunicorn.conf.py

@@ -0,0 +1,196 @@
+#
+# Server socket
+#
+#   bind - The socket to bind.
+#
+#       A string of the form: 'HOST', 'HOST:PORT', 'unix:PATH'.
+#       An IP is a valid HOST.
+#
+#   backlog - The number of pending connections. This refers
+#       to the number of clients that can be waiting to be
+#       served. Exceeding this number results in the client
+#       getting an error when attempting to connect. It should
+#       only affect servers under significant load.
+#
+#       Must be a positive integer. Generally set in the 64-2048
+#       range.
+#
+
+bind = ['unix:/var/gunicorn/.sock', ':8000']
+
+#
+# Worker processes
+#
+#   workers - The number of worker processes that this server
+#       should keep alive for handling requests.
+#
+#       A positive integer generally in the 2-4 x $(NUM_CORES)
+#       range. You'll want to vary this a bit to find the best
+#       for your particular application's work load.
+#
+#   worker_class - The type of workers to use. The default
+#       sync class should handle most 'normal' types of work
+#       loads. You'll want to read
+#       http://docs.gunicorn.org/en/latest/design.html#choosing-a-worker-type
+#       for information on when you might want to choose one
+#       of the other worker classes.
+#
+#       A string referring to a Python path to a subclass of
+#       gunicorn.workers.base.Worker. The default provided values
+#       can be seen at
+#       http://docs.gunicorn.org/en/latest/settings.html#worker-class
+#
+#   worker_connections - For the eventlet and gevent worker classes
+#       this limits the maximum number of simultaneous clients that
+#       a single process can handle.
+#
+#       A positive integer generally set to around 1000.
+#
+#   timeout - If a worker does not notify the master process in this
+#       number of seconds it is killed and a new worker is spawned
+#       to replace it.
+#
+#       Generally set to thirty seconds. Only set this noticeably
+#       higher if you're sure of the repercussions for sync workers.
+#       For the non sync workers it just means that the worker
+#       process is still communicating and is not tied to the length
+#       of time required to handle a single request.
+#
+#   keepalive - The number of seconds to wait for the next request
+#       on a Keep-Alive HTTP connection.
+#
+#       A positive integer. Generally set in the 1-5 seconds range.
+#
+
+# try:
+#     # fail 'successfully' if either of these modules aren't installed
+#     from gevent import monkey
+#     from psycogreen.gevent import patch_psycopg
+
+
+#     # setting this inside the 'try' ensures that we only
+#     # activate the gevent worker pool if we have gevent installed
+#     worker_class = 'gevent'
+#     workers = 4
+#     # this ensures forked processes are patched with gevent/gevent-psycopg2
+#     def do_post_fork(server, worker):
+#         monkey.patch_all()
+#         patch_psycopg()
+
+#         # you should see this text in your gunicorn logs if it was successful
+#         worker.log.info("Made Psycopg2 Green")
+
+#     post_fork = do_post_fork
+# except ImportError:
+#     pass
+
+workers = 4
+# worker_connections = 1000
+# timeout = 30
+# keepalive = 2
+
+#
+#   spew - Install a trace function that spews every line of Python
+#       that is executed when running the server. This is the
+#       nuclear option.
+#
+#       True or False
+#
+
+spew = False
+
+#
+# Server mechanics
+#
+#   daemon - Detach the main Gunicorn process from the controlling
+#       terminal with a standard fork/fork sequence.
+#
+#       True or False
+#
+#   pidfile - The path to a pid file to write
+#
+#       A path string or None to not write a pid file.
+#
+#   user - Switch worker processes to run as this user.
+#
+#       A valid user id (as an integer) or the name of a user that
+#       can be retrieved with a call to pwd.getpwnam(value) or None
+#       to not change the worker process user.
+#
+#   group - Switch worker process to run as this group.
+#
+#       A valid group id (as an integer) or the name of a user that
+#       can be retrieved with a call to pwd.getgrnam(value) or None
+#       to change the worker processes group.
+#
+#   umask - A mask for file permissions written by Gunicorn. Note that
+#       this affects unix socket permissions.
+#
+#       A valid value for the os.umask(mode) call or a string
+#       compatible with int(value, 0) (0 means Python guesses
+#       the base, so values like "0", "0xFF", "0022" are valid
+#       for decimal, hex, and octal representations)
+#
+#   tmp_upload_dir - A directory to store temporary request data when
+#       requests are read. This will most likely be disappearing soon.
+#
+#       A path to a directory where the process owner can write. Or
+#       None to signal that Python should choose one on its own.
+#
+
+daemon = False
+pidfile = '/var/gunicorn/.pid'
+umask = 0
+user = None
+group = None
+tmp_upload_dir = None
+
+#
+#   Logging
+#
+#   logfile - The path to a log file to write to.
+#
+#       A path string. "-" means log to stdout.
+#
+#   loglevel - The granularity of log output
+#
+#       A string of "debug", "info", "warning", "error", "critical"
+#
+
+errorlog = '-'
+loglevel = 'warning'
+accesslog = '-'
+access_log_format = '%(h)s %(t)s %(m)s %(U)s %(q)s %(H)s %(s)s %(B)s %(f)s %(a)s %(L)s'
+
+#
+# Process naming
+#
+#   proc_name - A base to use with setproctitle to change the way
+#       that Gunicorn processes are reported in the system process
+#       table. This affects things like 'ps' and 'top'. If you're
+#       going to be running more than one instance of Gunicorn you'll
+#       probably want to set a name to tell them apart. This requires
+#       that you install the setproctitle module.
+#
+#       A string or None to choose a default of something like 'gunicorn'.
+#
+
+proc_name = None
+
+#
+# Server hooks
+#
+#   post_fork - Called just after a worker has been forked.
+#
+#       A callable that takes a server and worker instance
+#       as arguments.
+#
+#   pre_fork - Called just prior to forking the worker subprocess.
+#
+#       A callable that accepts the same arguments as after_fork
+#
+#   pre_exec - Called just prior to forking off a secondary
+#       master process during things like config reloading.
+#
+#       A callable that takes a server instance as the sole argument.
+#

config/nginx/conf.d/local.conf

@@ -0,0 +1,20 @@
+upstream app {
+    server unix:/var/gunicorn/.sock;
+}
+
+server {
+    listen 80 default_server;
+    server_name localhost;
+
+    location / {
+        allow all;
+        proxy_http_version 1.1;
+        proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
+        proxy_set_header Host $http_host;
+        proxy_set_header X-Cluster-Client-Ip $remote_addr;
+        proxy_pass http://app;
+    }
+
+    # Note that nginx doesn't serve any static files.
+}
+

docker-compose.production.yml

@@ -0,0 +1,38 @@
+version: '3'
+services:
+  db:
+    restart: always
+    networks:
+      - database_network
+  web:
+    restart: always
+    volumes:
+      - gunicorn-socket:/var/gunicorn
+      - ./config/gunicorn.conf.py:/config/gunicorn.conf.py
+    networks:
+      - database_network
+      - nginx_network
+    environment:
+      - ENVIRONMENT=production
+      - DJANGO_SETTINGS_MODULE=backend.settings.production
+  nginx:
+    image: nginx:1.14
+    # build:
+    #   context: .
+    #   dockerfile: Dockerfile.nginx
+    ports:
+      - "80:80"
+    volumes:
+      - gunicorn-socket:/var/gunicorn
+      - ./config/nginx/conf.d:/etc/nginx/conf.d
+    networks:
+      - nginx_network
+    depends_on:
+      - web
+networks:
+  database_network:
+    driver: bridge
+  nginx_network:
+    driver: bridge
+volumes:
+  gunicorn-socket:

docker-compose.yml

@@ -0,0 +1,31 @@
+version: '3'
+services:
+  db:
+    image: postgres
+    environment:
+      # Set these in .env
+      - POSTGRES_USER
+      - POSTGRES_PASSWORD
+      - POSTGRES_DB
+    ports:
+      - "5432:5432"
+    volumes:
+      - pgdata:/var/lib/postgresql/data
+  web:
+    environment:
+      # Set these in .env
+      - SECRET_KEY
+      - POSTGRES_USER
+      - POSTGRES_PASSWORD
+      - POSTGRES_DB
+      - DJANGO_SETTINGS_MODULE
+    build: ./backend/api
+    command: bash scripts/docker-entrypoint.sh
+    volumes:
+      - ./backend/api:/code
+    ports:
+      - "8000:8000"
+    depends_on:
+      - db
+volumes:
+  pgdata:

example.env

@@ -0,0 +1,5 @@
+SECRET_KEY=a-long-secret-key
+POSTGRES_USER=postgres
+POSTGRES_PASSWORD=postgres
+POSTGRES_DB=fediverse
+DJANGO_SETTINGS_MODULE=backend.settings.dev