Add project

Docker/Dockerfile

@@ -0,0 +1,15 @@
+FROM python:3.6-alpine
+
+MAINTAINER Anna Sudnitsyna
+
+WORKDIR /usr/src/app
+COPY requirements.txt ./
+RUN pip install --no-cache-dir -r requirements.txt
+RUN pip install gunicorn
+
+COPY . .
+
+ADD Docker/entrypoint.sh /
+
+EXPOSE 8000
+ENTRYPOINT /entrypoint.sh

Docker/config/mydjango.conf

@@ -0,0 +1,17 @@
+upstream web {
+  ip_hash;
+  server web:8000;
+}
+
+# portal
+server {
+  location / {
+        proxy_pass http://web/;
+    }
+  location /static {
+        alias /src/static;
+    }
+
+  listen 8000;
+  server_name localhost;
+}

Docker/docker-compose.yml

@@ -0,0 +1,35 @@
+version: '2'
+services:
+  nginx:
+    image: nginx:latest
+    container_name: ng02
+    ports:
+     - "80:8000"
+    volumes:
+      - static-files:/src/static
+      - ./config:/etc/nginx/conf.d
+    depends_on:
+      - web
+
+  redis:
+      image: redis
+      container_name: rd02
+      ports:
+      - "6379:6379"
+
+  web:
+    build:
+      context: ../
+      dockerfile: Docker/Dockerfile
+    container_name: dg02
+    volumes:
+      - static-files:/usr/src/app/static
+    expose:
+      - "8000"
+    depends_on:
+      - redis
+
+      #    command: bash -c "python3 manage.py collectstatic && gunicorn deplodock.wsgi --bind 0.0.0.0:8000"
+
+volumes:
+    static-files:

Docker/entrypoint.sh

@@ -0,0 +1,17 @@
+#!/bin/sh
+
+collect_static() {
+    python3 manage.py collectstatic --noinput
+}
+
+run_gunicorn(){
+    gunicorn mysite.wsgi --bind 0.0.0.0:8000
+ }
+
+run_celery() {
+    celery -A mysite worker -l info -B
+}
+
+collect_static && run_celery&
+
+run_gunicorn

Docker/systemd/celery.service

@@ -0,0 +1,7 @@
+[Unit]
+Description=Celery daemon for Django Project
+
+[Service]
+WorkingDirectory=/usr/src/app
+ExecStart=/usr/bin/celery -A mysite worker -l info -B
+Restart=always

Docker/systemd/django.service

@@ -0,0 +1,7 @@
+[Unit]
+Description=Gunicorn daemon for Django Project
+
+[Service]
+WorkingDirectory=/usr/src/app
+ExecStart=/usr/bin/gunicorn mysite.wsgi --bind 0.0.0.0:8000
+Restart=always

README.md

@@ -1,2 +1,24 @@
 # cb_parser
 
+Application gets xml from https://www.cbr-xml-daily.ru/daily_utf8.xml once a day according to schedule and save data to redis.
+Data is available at the main page.
+
+### Deploy:
+
+##### Docker:
+```sh
+docker-compose up --build -d [--force-recreate]
+```
+(App started on localhost:80)
+
+##### Ansible:
+
+Update hosts file and run:
+```sh
+ansible-playbook -i hosts deploy.yaml
+```
+
+Tested on:
+- Ubuntu 18.04 x64
+- Ubuntu 18.10 x64
+- Ubuntu 19.04 x64

ansible/deploy.yaml

@@ -0,0 +1,27 @@
+---
+- hosts: all
+  gather_facts: no
+  tasks:
+#    Set up environment
+    - name: Running apt update
+      apt: update_cache=yes
+    - name: Enable ru_RU.UTF-8
+      lineinfile: dest=/etc/locale.gen line='ru_RU.UTF-8 UTF-8'
+    - name: Gen locale
+      command: locale-gen
+    - name: Installing required packages
+      apt: name={{item}} state=present
+      with_items:
+       - python3-pip
+       - docker.io
+    - name: install compose
+      pip:
+        name: docker-compose
+        executable: pip3
+#    Deploy project
+    - name: copy files
+      synchronize: src=/home/anya/PycharmProjects/cb_parser_celery/mysite dest=/src
+    - name: run the service defined in my_project's docker-compose.yml
+      docker_service:
+        build: yes
+        project_src: /src/mysite/Docker

ansible/hosts

@@ -0,0 +1,5 @@
+[py3-hosts]
+root ansible_ssh_host=188.166.64.76 ansible_ssh_user=root privatekeyfile=~/.ssh/id_rsa_do
+
+[py3-hosts:vars]
+ansible_python_interpreter=/usr/bin/python3

ansible/system.yaml

@@ -0,0 +1,8 @@
+---
+  - hosts: all
+    gather_facts: no
+    tasks:
+    - name: Read SSH public key
+      slurp: src=~/.ssh/id_rsa_do.pub
+      register: public_key
+#  debug: msg="{{ public_key['content'] | b64decode }}"

manage.py

@@ -0,0 +1,15 @@
+#!/usr/bin/env python
+import os
+import sys
+
+if __name__ == "__main__":
+    os.environ.setdefault("DJANGO_SETTINGS_MODULE", "mysite.settings")
+    try:
+        from django.core.management import execute_from_command_line
+    except ImportError as exc:
+        raise ImportError(
+            "Couldn't import Django. Are you sure it's installed and "
+            "available on your PYTHONPATH environment variable? Did you "
+            "forget to activate a virtual environment?"
+        ) from exc
+    execute_from_command_line(sys.argv)

mysite/__init__.py

@@ -0,0 +1,7 @@
+from __future__ import absolute_import, unicode_literals
+
+# This will make sure the app is always imported when
+# Django starts so that shared_task will use this app.
+from .celery import app as celery_app
+
+__all__ = ('celery_app',)

mysite/celery.py

@@ -0,0 +1,32 @@
+from __future__ import absolute_import, unicode_literals
+
+import os
+
+from celery import Celery
+from celery.schedules import crontab
+
+# set the default Django settings module for the 'celery' program.
+os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'mysite.settings')
+
+app = Celery('mysite')
+
+# Using a string here means the worker doesn't have to serialize
+# the configuration object to child processes.
+# - namespace='CELERY' means all celery-related configuration keys
+#   should have a `CELERY_` prefix.
+app.config_from_object('django.conf:settings', namespace='CELERY')
+
+# Load task modules from all registered Django app configs.
+app.autodiscover_tasks()
+
+
+@app.task(bind=True)
+def debug_task(self):
+    print('Request: {0!r}'.format(self.request))
+
+app.conf.beat_schedule = {
+    'get_rates': {
+        'task': 'rates.tasks.parse_cb',
+        'schedule': crontab(hour=0, minute=0,),  # run daily at midnight
+    },
+}

mysite/settings.py

@@ -0,0 +1,130 @@
+"""
+Django settings for mysite project.
+
+Generated by 'django-admin startproject' using Django 2.0.
+
+For more information on this file, see
+https://docs.djangoproject.com/en/2.0/topics/settings/
+
+For the full list of settings and their values, see
+https://docs.djangoproject.com/en/2.0/ref/settings/
+"""
+
+import os
+
+# Build paths inside the project like this: os.path.join(BASE_DIR, ...)
+BASE_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
+
+
+# Quick-start development settings - unsuitable for production
+# See https://docs.djangoproject.com/en/2.0/howto/deployment/checklist/
+
+# SECURITY WARNING: keep the secret key used in production secret!
+SECRET_KEY = 't4-k79wmngkt*1!z0b!75yk&+1@!*wu)s(o0$wx+qt&jf*7hp@'
+
+# SECURITY WARNING: don't run with debug turned on in production!
+DEBUG = False
+
+ALLOWED_HOSTS = ['web', '127.0.0.1']
+
+
+# Application definition
+
+INSTALLED_APPS = [
+    'django.contrib.admin',
+    'django.contrib.auth',
+    'django.contrib.contenttypes',
+    'django.contrib.sessions',
+    'django.contrib.messages',
+    'django.contrib.staticfiles',
+    'rates'
+]
+
+MIDDLEWARE = [
+    'django.middleware.security.SecurityMiddleware',
+    'django.contrib.sessions.middleware.SessionMiddleware',
+    'django.middleware.common.CommonMiddleware',
+    'django.middleware.csrf.CsrfViewMiddleware',
+    'django.contrib.auth.middleware.AuthenticationMiddleware',
+    'django.contrib.messages.middleware.MessageMiddleware',
+    'django.middleware.clickjacking.XFrameOptionsMiddleware',
+]
+
+ROOT_URLCONF = 'mysite.urls'
+
+TEMPLATES = [
+    {
+        'BACKEND': 'django.template.backends.django.DjangoTemplates',
+        'DIRS': [],
+        'APP_DIRS': True,
+        'OPTIONS': {
+            'context_processors': [
+                'django.template.context_processors.debug',
+                'django.template.context_processors.request',
+                'django.contrib.auth.context_processors.auth',
+                'django.contrib.messages.context_processors.messages',
+            ],
+        },
+    },
+]
+
+WSGI_APPLICATION = 'mysite.wsgi.application'
+
+
+# Database
+# https://docs.djangoproject.com/en/2.0/ref/settings/#databases
+
+DATABASES = {
+    'default': {
+        'ENGINE': 'django.db.backends.sqlite3',
+        'NAME': os.path.join(BASE_DIR, 'db.sqlite3'),
+    }
+}
+
+
+# Password validation
+# https://docs.djangoproject.com/en/2.0/ref/settings/#auth-password-validators
+
+AUTH_PASSWORD_VALIDATORS = [
+    {
+        'NAME': 'django.contrib.auth.password_validation.UserAttributeSimilarityValidator',
+    },
+    {
+        'NAME': 'django.contrib.auth.password_validation.MinimumLengthValidator',
+    },
+    {
+        'NAME': 'django.contrib.auth.password_validation.CommonPasswordValidator',
+    },
+    {
+        'NAME': 'django.contrib.auth.password_validation.NumericPasswordValidator',
+    },
+]
+
+
+# Internationalization
+# https://docs.djangoproject.com/en/2.0/topics/i18n/
+
+LANGUAGE_CODE = 'en-us'
+
+TIME_ZONE = 'UTC'
+
+USE_I18N = True
+
+USE_L10N = True
+
+USE_TZ = True
+
+
+# Static files (CSS, JavaScript, Images)
+# https://docs.djangoproject.com/en/2.0/howto/static-files/
+
+PROJECT_ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
+STATIC_URL = '/static/'
+STATIC_ROOT = os.path.join(PROJECT_ROOT, 'static')
+
+
+CELERY_BROKER_URL = 'redis://redis:6379'
+CELERY_RESULT_BACKEND = 'redis://redis:6379'
+CELERY_ACCEPT_CONTENT = ['application/json']
+CELERY_TASK_SERIALIZER = 'json'
+CELERY_RESULT_SERIALIZER = 'json'

mysite/urls.py

@@ -0,0 +1,23 @@
+"""mysite URL Configuration
+
+The `urlpatterns` list routes URLs to views. For more information please see:
+    https://docs.djangoproject.com/en/2.0/topics/http/urls/
+Examples:
+Function views
+    1. Add an import:  from my_app import views
+    2. Add a URL to urlpatterns:  path('', views.home, name='home')
+Class-based views
+    1. Add an import:  from other_app.views import Home
+    2. Add a URL to urlpatterns:  path('', Home.as_view(), name='home')
+Including another URLconf
+    1. Import the include() function: from django.urls import include, path
+    2. Add a URL to urlpatterns:  path('blog/', include('blog.urls'))
+"""
+from django.contrib import admin
+from django.urls import path
+from rates import views
+
+urlpatterns = [
+    path('admin/', admin.site.urls),
+    path('', views.index, name="rates")
+]

mysite/wsgi.py

@@ -0,0 +1,16 @@
+"""
+WSGI config for mysite project.
+
+It exposes the WSGI callable as a module-level variable named ``application``.
+
+For more information on this file, see
+https://docs.djangoproject.com/en/2.0/howto/deployment/wsgi/
+"""
+
+import os
+
+from django.core.wsgi import get_wsgi_application
+
+os.environ.setdefault("DJANGO_SETTINGS_MODULE", "mysite.settings")
+
+application = get_wsgi_application()

rates/admin.py

@@ -0,0 +1,3 @@
+from django.contrib import admin
+
+# Register your models here.

rates/apps.py

@@ -0,0 +1,5 @@
+from django.apps import AppConfig
+
+
+class RatesConfig(AppConfig):
+    name = 'rates'

rates/models.py

@@ -0,0 +1,3 @@
+from django.db import models
+
+# Create your models here.

rates/tasks.py

@@ -0,0 +1,20 @@
+import xml.etree.ElementTree as ET
+from datetime import datetime
+
+import redis
+import requests
+from celery import task
+from celery.utils.log import get_task_logger
+
+logger = get_task_logger(__name__)
+
+@task
+def parse_cb():
+    url = 'https://www.cbr-xml-daily.ru/daily_utf8.xml'
+    res = requests.get(url)
+    root = ET.fromstring(res.content.decode())
+    result = {child.find("CharCode").text: child.find("Value").text for child in root}
+    redis_ = redis.StrictRedis(host='redis', port=6379, db=0)
+    redis_.set('result', str(result))
+    redis_.set('time', str(datetime.now()))
+    logger.info(result)

rates/tests.py

@@ -0,0 +1,11 @@
+"""Tests for cb parser application.
+Run: py.test rates/tests.py """
+import re
+import requests
+
+
+def test_smoke():
+    """Use regexp to check web page content."""
+    res = requests.get("http://localhost/")
+    pattern = r"^data: {('[A-Z]{3}': '\d+,\d+',?\s?)+}<br><br>updated on \d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2}.\d{6}$"
+    assert re.match(re.compile(pattern), res.content.decode("utf-8"))

rates/views.py

@@ -0,0 +1,14 @@
+from django.http import HttpResponse
+import redis
+from .tasks import parse_cb
+
+
+def index(request):
+    r = redis.StrictRedis(host="redis", port=6379, db=0)
+    try:
+        data = r.get("result").decode()
+        last_time = r.get("time").decode()
+    except AttributeError:
+        parse_cb.delay()
+        return HttpResponse("Service is not available.")
+    return HttpResponse("data: {}<br><br>updated on {}".format(data, last_time))

requirements.txt

@@ -0,0 +1,4 @@
+celery==4.2.1
+Django==2.0
+redis
+requests==2.19.1