diff --git a/Docker/Dockerfile b/Docker/Dockerfile new file mode 100644 index 0000000..13eb148 --- /dev/null +++ b/Docker/Dockerfile @@ -0,0 +1,15 @@ +FROM python:3.6-alpine + +MAINTAINER Anna Sudnitsyna + +WORKDIR /usr/src/app +COPY requirements.txt ./ +RUN pip install --no-cache-dir -r requirements.txt +RUN pip install gunicorn + +COPY . . + +ADD Docker/entrypoint.sh / + +EXPOSE 8000 +ENTRYPOINT /entrypoint.sh diff --git a/Docker/config/mydjango.conf b/Docker/config/mydjango.conf new file mode 100644 index 0000000..8e19165 --- /dev/null +++ b/Docker/config/mydjango.conf @@ -0,0 +1,17 @@ +upstream web { + ip_hash; + server web:8000; +} + +# portal +server { + location / { + proxy_pass http://web/; + } + location /static { + alias /src/static; + } + + listen 8000; + server_name localhost; +} \ No newline at end of file diff --git a/Docker/docker-compose.yml b/Docker/docker-compose.yml new file mode 100644 index 0000000..3411a7c --- /dev/null +++ b/Docker/docker-compose.yml @@ -0,0 +1,35 @@ +version: '2' +services: + nginx: + image: nginx:latest + container_name: ng02 + ports: + - "80:8000" + volumes: + - static-files:/src/static + - ./config:/etc/nginx/conf.d + depends_on: + - web + + redis: + image: redis + container_name: rd02 + ports: + - "6379:6379" + + web: + build: + context: ../ + dockerfile: Docker/Dockerfile + container_name: dg02 + volumes: + - static-files:/usr/src/app/static + expose: + - "8000" + depends_on: + - redis + + # command: bash -c "python3 manage.py collectstatic && gunicorn deplodock.wsgi --bind 0.0.0.0:8000" + +volumes: + static-files: \ No newline at end of file diff --git a/Docker/entrypoint.sh b/Docker/entrypoint.sh new file mode 100755 index 0000000..4e89853 --- /dev/null +++ b/Docker/entrypoint.sh @@ -0,0 +1,17 @@ +#!/bin/sh + +collect_static() { + python3 manage.py collectstatic --noinput +} + +run_gunicorn(){ + gunicorn mysite.wsgi --bind 0.0.0.0:8000 + } + +run_celery() { + celery -A mysite worker -l info -B +} + +collect_static && run_celery& + +run_gunicorn \ No newline at end of file diff --git a/Docker/systemd/celery.service b/Docker/systemd/celery.service new file mode 100644 index 0000000..0dd3cce --- /dev/null +++ b/Docker/systemd/celery.service @@ -0,0 +1,7 @@ +[Unit] +Description=Celery daemon for Django Project + +[Service] +WorkingDirectory=/usr/src/app +ExecStart=/usr/bin/celery -A mysite worker -l info -B +Restart=always \ No newline at end of file diff --git a/Docker/systemd/django.service b/Docker/systemd/django.service new file mode 100644 index 0000000..704a39c --- /dev/null +++ b/Docker/systemd/django.service @@ -0,0 +1,7 @@ +[Unit] +Description=Gunicorn daemon for Django Project + +[Service] +WorkingDirectory=/usr/src/app +ExecStart=/usr/bin/gunicorn mysite.wsgi --bind 0.0.0.0:8000 +Restart=always diff --git a/PUBLISH b/PUBLISH new file mode 100644 index 0000000..e69de29 diff --git a/README.md b/README.md index b83ad91..354e6be 100644 --- a/README.md +++ b/README.md @@ -1,2 +1,24 @@ # cb_parser +Application gets xml from https://www.cbr-xml-daily.ru/daily_utf8.xml once a day according to schedule and save data to redis. +Data is available at the main page. + +### Deploy: + +##### Docker: +```sh +docker-compose up --build -d [--force-recreate] +``` +(App started on localhost:80) + +##### Ansible: + +Update hosts file and run: +```sh +ansible-playbook -i hosts deploy.yaml +``` + +Tested on: +- Ubuntu 18.04 x64 +- Ubuntu 18.10 x64 +- Ubuntu 19.04 x64 diff --git a/ansible/deploy.yaml b/ansible/deploy.yaml new file mode 100644 index 0000000..75b41e3 --- /dev/null +++ b/ansible/deploy.yaml @@ -0,0 +1,27 @@ +--- +- hosts: all + gather_facts: no + tasks: +# Set up environment + - name: Running apt update + apt: update_cache=yes + - name: Enable ru_RU.UTF-8 + lineinfile: dest=/etc/locale.gen line='ru_RU.UTF-8 UTF-8' + - name: Gen locale + command: locale-gen + - name: Installing required packages + apt: name={{item}} state=present + with_items: + - python3-pip + - docker.io + - name: install compose + pip: + name: docker-compose + executable: pip3 +# Deploy project + - name: copy files + synchronize: src=/home/anya/PycharmProjects/cb_parser_celery/mysite dest=/src + - name: run the service defined in my_project's docker-compose.yml + docker_service: + build: yes + project_src: /src/mysite/Docker diff --git a/ansible/hosts b/ansible/hosts new file mode 100644 index 0000000..45ae915 --- /dev/null +++ b/ansible/hosts @@ -0,0 +1,5 @@ +[py3-hosts] +root ansible_ssh_host=188.166.64.76 ansible_ssh_user=root privatekeyfile=~/.ssh/id_rsa_do + +[py3-hosts:vars] +ansible_python_interpreter=/usr/bin/python3 \ No newline at end of file diff --git a/ansible/system.yaml b/ansible/system.yaml new file mode 100644 index 0000000..ca20d70 --- /dev/null +++ b/ansible/system.yaml @@ -0,0 +1,8 @@ +--- + - hosts: all + gather_facts: no + tasks: + - name: Read SSH public key + slurp: src=~/.ssh/id_rsa_do.pub + register: public_key +# debug: msg="{{ public_key['content'] | b64decode }}" \ No newline at end of file diff --git a/db.sqlite3 b/db.sqlite3 new file mode 100644 index 0000000..883ebe5 Binary files /dev/null and b/db.sqlite3 differ diff --git a/manage.py b/manage.py new file mode 100755 index 0000000..66ed3a9 --- /dev/null +++ b/manage.py @@ -0,0 +1,15 @@ +#!/usr/bin/env python +import os +import sys + +if __name__ == "__main__": + os.environ.setdefault("DJANGO_SETTINGS_MODULE", "mysite.settings") + try: + from django.core.management import execute_from_command_line + except ImportError as exc: + raise ImportError( + "Couldn't import Django. Are you sure it's installed and " + "available on your PYTHONPATH environment variable? Did you " + "forget to activate a virtual environment?" + ) from exc + execute_from_command_line(sys.argv) diff --git a/mysite/__init__.py b/mysite/__init__.py new file mode 100644 index 0000000..d128d39 --- /dev/null +++ b/mysite/__init__.py @@ -0,0 +1,7 @@ +from __future__ import absolute_import, unicode_literals + +# This will make sure the app is always imported when +# Django starts so that shared_task will use this app. +from .celery import app as celery_app + +__all__ = ('celery_app',) \ No newline at end of file diff --git a/mysite/celery.py b/mysite/celery.py new file mode 100644 index 0000000..6e8872f --- /dev/null +++ b/mysite/celery.py @@ -0,0 +1,32 @@ +from __future__ import absolute_import, unicode_literals + +import os + +from celery import Celery +from celery.schedules import crontab + +# set the default Django settings module for the 'celery' program. +os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'mysite.settings') + +app = Celery('mysite') + +# Using a string here means the worker doesn't have to serialize +# the configuration object to child processes. +# - namespace='CELERY' means all celery-related configuration keys +# should have a `CELERY_` prefix. +app.config_from_object('django.conf:settings', namespace='CELERY') + +# Load task modules from all registered Django app configs. +app.autodiscover_tasks() + + +@app.task(bind=True) +def debug_task(self): + print('Request: {0!r}'.format(self.request)) + +app.conf.beat_schedule = { + 'get_rates': { + 'task': 'rates.tasks.parse_cb', + 'schedule': crontab(hour=0, minute=0,), # run daily at midnight + }, +} \ No newline at end of file diff --git a/mysite/settings.py b/mysite/settings.py new file mode 100644 index 0000000..372da5f --- /dev/null +++ b/mysite/settings.py @@ -0,0 +1,130 @@ +""" +Django settings for mysite project. + +Generated by 'django-admin startproject' using Django 2.0. + +For more information on this file, see +https://docs.djangoproject.com/en/2.0/topics/settings/ + +For the full list of settings and their values, see +https://docs.djangoproject.com/en/2.0/ref/settings/ +""" + +import os + +# Build paths inside the project like this: os.path.join(BASE_DIR, ...) +BASE_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) + + +# Quick-start development settings - unsuitable for production +# See https://docs.djangoproject.com/en/2.0/howto/deployment/checklist/ + +# SECURITY WARNING: keep the secret key used in production secret! +SECRET_KEY = 't4-k79wmngkt*1!z0b!75yk&+1@!*wu)s(o0$wx+qt&jf*7hp@' + +# SECURITY WARNING: don't run with debug turned on in production! +DEBUG = False + +ALLOWED_HOSTS = ['web', '127.0.0.1'] + + +# Application definition + +INSTALLED_APPS = [ + 'django.contrib.admin', + 'django.contrib.auth', + 'django.contrib.contenttypes', + 'django.contrib.sessions', + 'django.contrib.messages', + 'django.contrib.staticfiles', + 'rates' +] + +MIDDLEWARE = [ + 'django.middleware.security.SecurityMiddleware', + 'django.contrib.sessions.middleware.SessionMiddleware', + 'django.middleware.common.CommonMiddleware', + 'django.middleware.csrf.CsrfViewMiddleware', + 'django.contrib.auth.middleware.AuthenticationMiddleware', + 'django.contrib.messages.middleware.MessageMiddleware', + 'django.middleware.clickjacking.XFrameOptionsMiddleware', +] + +ROOT_URLCONF = 'mysite.urls' + +TEMPLATES = [ + { + 'BACKEND': 'django.template.backends.django.DjangoTemplates', + 'DIRS': [], + 'APP_DIRS': True, + 'OPTIONS': { + 'context_processors': [ + 'django.template.context_processors.debug', + 'django.template.context_processors.request', + 'django.contrib.auth.context_processors.auth', + 'django.contrib.messages.context_processors.messages', + ], + }, + }, +] + +WSGI_APPLICATION = 'mysite.wsgi.application' + + +# Database +# https://docs.djangoproject.com/en/2.0/ref/settings/#databases + +DATABASES = { + 'default': { + 'ENGINE': 'django.db.backends.sqlite3', + 'NAME': os.path.join(BASE_DIR, 'db.sqlite3'), + } +} + + +# Password validation +# https://docs.djangoproject.com/en/2.0/ref/settings/#auth-password-validators + +AUTH_PASSWORD_VALIDATORS = [ + { + 'NAME': 'django.contrib.auth.password_validation.UserAttributeSimilarityValidator', + }, + { + 'NAME': 'django.contrib.auth.password_validation.MinimumLengthValidator', + }, + { + 'NAME': 'django.contrib.auth.password_validation.CommonPasswordValidator', + }, + { + 'NAME': 'django.contrib.auth.password_validation.NumericPasswordValidator', + }, +] + + +# Internationalization +# https://docs.djangoproject.com/en/2.0/topics/i18n/ + +LANGUAGE_CODE = 'en-us' + +TIME_ZONE = 'UTC' + +USE_I18N = True + +USE_L10N = True + +USE_TZ = True + + +# Static files (CSS, JavaScript, Images) +# https://docs.djangoproject.com/en/2.0/howto/static-files/ + +PROJECT_ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) +STATIC_URL = '/static/' +STATIC_ROOT = os.path.join(PROJECT_ROOT, 'static') + + +CELERY_BROKER_URL = 'redis://redis:6379' +CELERY_RESULT_BACKEND = 'redis://redis:6379' +CELERY_ACCEPT_CONTENT = ['application/json'] +CELERY_TASK_SERIALIZER = 'json' +CELERY_RESULT_SERIALIZER = 'json' diff --git a/mysite/urls.py b/mysite/urls.py new file mode 100644 index 0000000..14e51e1 --- /dev/null +++ b/mysite/urls.py @@ -0,0 +1,23 @@ +"""mysite URL Configuration + +The `urlpatterns` list routes URLs to views. For more information please see: + https://docs.djangoproject.com/en/2.0/topics/http/urls/ +Examples: +Function views + 1. Add an import: from my_app import views + 2. Add a URL to urlpatterns: path('', views.home, name='home') +Class-based views + 1. Add an import: from other_app.views import Home + 2. Add a URL to urlpatterns: path('', Home.as_view(), name='home') +Including another URLconf + 1. Import the include() function: from django.urls import include, path + 2. Add a URL to urlpatterns: path('blog/', include('blog.urls')) +""" +from django.contrib import admin +from django.urls import path +from rates import views + +urlpatterns = [ + path('admin/', admin.site.urls), + path('', views.index, name="rates") +] diff --git a/mysite/wsgi.py b/mysite/wsgi.py new file mode 100644 index 0000000..ffbfe4f --- /dev/null +++ b/mysite/wsgi.py @@ -0,0 +1,16 @@ +""" +WSGI config for mysite project. + +It exposes the WSGI callable as a module-level variable named ``application``. + +For more information on this file, see +https://docs.djangoproject.com/en/2.0/howto/deployment/wsgi/ +""" + +import os + +from django.core.wsgi import get_wsgi_application + +os.environ.setdefault("DJANGO_SETTINGS_MODULE", "mysite.settings") + +application = get_wsgi_application() diff --git a/rates/__init__.py b/rates/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/rates/admin.py b/rates/admin.py new file mode 100644 index 0000000..8c38f3f --- /dev/null +++ b/rates/admin.py @@ -0,0 +1,3 @@ +from django.contrib import admin + +# Register your models here. diff --git a/rates/apps.py b/rates/apps.py new file mode 100644 index 0000000..50c55c5 --- /dev/null +++ b/rates/apps.py @@ -0,0 +1,5 @@ +from django.apps import AppConfig + + +class RatesConfig(AppConfig): + name = 'rates' diff --git a/rates/migrations/__init__.py b/rates/migrations/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/rates/models.py b/rates/models.py new file mode 100644 index 0000000..71a8362 --- /dev/null +++ b/rates/models.py @@ -0,0 +1,3 @@ +from django.db import models + +# Create your models here. diff --git a/rates/tasks.py b/rates/tasks.py new file mode 100644 index 0000000..008bc08 --- /dev/null +++ b/rates/tasks.py @@ -0,0 +1,20 @@ +import xml.etree.ElementTree as ET +from datetime import datetime + +import redis +import requests +from celery import task +from celery.utils.log import get_task_logger + +logger = get_task_logger(__name__) + +@task +def parse_cb(): + url = 'https://www.cbr-xml-daily.ru/daily_utf8.xml' + res = requests.get(url) + root = ET.fromstring(res.content.decode()) + result = {child.find("CharCode").text: child.find("Value").text for child in root} + redis_ = redis.StrictRedis(host='redis', port=6379, db=0) + redis_.set('result', str(result)) + redis_.set('time', str(datetime.now())) + logger.info(result) diff --git a/rates/tests.py b/rates/tests.py new file mode 100644 index 0000000..e2c5f9c --- /dev/null +++ b/rates/tests.py @@ -0,0 +1,11 @@ +"""Tests for cb parser application. +Run: py.test rates/tests.py """ +import re +import requests + + +def test_smoke(): + """Use regexp to check web page content.""" + res = requests.get("http://localhost/") + pattern = r"^data: {('[A-Z]{3}': '\d+,\d+',?\s?)+}

updated on \d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2}.\d{6}$" + assert re.match(re.compile(pattern), res.content.decode("utf-8")) diff --git a/rates/views.py b/rates/views.py new file mode 100644 index 0000000..f441664 --- /dev/null +++ b/rates/views.py @@ -0,0 +1,14 @@ +from django.http import HttpResponse +import redis +from .tasks import parse_cb + + +def index(request): + r = redis.StrictRedis(host="redis", port=6379, db=0) + try: + data = r.get("result").decode() + last_time = r.get("time").decode() + except AttributeError: + parse_cb.delay() + return HttpResponse("Service is not available.") + return HttpResponse("data: {}

updated on {}".format(data, last_time)) diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 0000000..0b3b09b --- /dev/null +++ b/requirements.txt @@ -0,0 +1,4 @@ +celery==4.2.1 +Django==2.0 +redis +requests==2.19.1