Add project

This commit is contained in:
Anna Sudnitsina 2018-09-12 22:29:12 +03:00
parent 00844f4106
commit 6e49f1e097
27 changed files with 443 additions and 0 deletions

15
Docker/Dockerfile Normal file
View File

@ -0,0 +1,15 @@
FROM python:3.6-alpine
MAINTAINER Anna Sudnitsyna
WORKDIR /usr/src/app
COPY requirements.txt ./
RUN pip install --no-cache-dir -r requirements.txt
RUN pip install gunicorn
COPY . .
ADD Docker/entrypoint.sh /
EXPOSE 8000
ENTRYPOINT /entrypoint.sh

View File

@ -0,0 +1,17 @@
upstream web {
ip_hash;
server web:8000;
}
# portal
server {
location / {
proxy_pass http://web/;
}
location /static {
alias /src/static;
}
listen 8000;
server_name localhost;
}

35
Docker/docker-compose.yml Normal file
View File

@ -0,0 +1,35 @@
version: '2'
services:
nginx:
image: nginx:latest
container_name: ng02
ports:
- "80:8000"
volumes:
- static-files:/src/static
- ./config:/etc/nginx/conf.d
depends_on:
- web
redis:
image: redis
container_name: rd02
ports:
- "6379:6379"
web:
build:
context: ../
dockerfile: Docker/Dockerfile
container_name: dg02
volumes:
- static-files:/usr/src/app/static
expose:
- "8000"
depends_on:
- redis
# command: bash -c "python3 manage.py collectstatic && gunicorn deplodock.wsgi --bind 0.0.0.0:8000"
volumes:
static-files:

17
Docker/entrypoint.sh Executable file
View File

@ -0,0 +1,17 @@
#!/bin/sh
collect_static() {
python3 manage.py collectstatic --noinput
}
run_gunicorn(){
gunicorn mysite.wsgi --bind 0.0.0.0:8000
}
run_celery() {
celery -A mysite worker -l info -B
}
collect_static && run_celery&
run_gunicorn

View File

@ -0,0 +1,7 @@
[Unit]
Description=Celery daemon for Django Project
[Service]
WorkingDirectory=/usr/src/app
ExecStart=/usr/bin/celery -A mysite worker -l info -B
Restart=always

View File

@ -0,0 +1,7 @@
[Unit]
Description=Gunicorn daemon for Django Project
[Service]
WorkingDirectory=/usr/src/app
ExecStart=/usr/bin/gunicorn mysite.wsgi --bind 0.0.0.0:8000
Restart=always

0
PUBLISH Normal file
View File

View File

@ -1,2 +1,24 @@
# cb_parser # cb_parser
Application gets xml from https://www.cbr-xml-daily.ru/daily_utf8.xml once a day according to schedule and save data to redis.
Data is available at the main page.
### Deploy:
##### Docker:
```sh
docker-compose up --build -d [--force-recreate]
```
(App started on localhost:80)
##### Ansible:
Update hosts file and run:
```sh
ansible-playbook -i hosts deploy.yaml
```
Tested on:
- Ubuntu 18.04 x64
- Ubuntu 18.10 x64
- Ubuntu 19.04 x64

27
ansible/deploy.yaml Normal file
View File

@ -0,0 +1,27 @@
---
- hosts: all
gather_facts: no
tasks:
# Set up environment
- name: Running apt update
apt: update_cache=yes
- name: Enable ru_RU.UTF-8
lineinfile: dest=/etc/locale.gen line='ru_RU.UTF-8 UTF-8'
- name: Gen locale
command: locale-gen
- name: Installing required packages
apt: name={{item}} state=present
with_items:
- python3-pip
- docker.io
- name: install compose
pip:
name: docker-compose
executable: pip3
# Deploy project
- name: copy files
synchronize: src=/home/anya/PycharmProjects/cb_parser_celery/mysite dest=/src
- name: run the service defined in my_project's docker-compose.yml
docker_service:
build: yes
project_src: /src/mysite/Docker

5
ansible/hosts Normal file
View File

@ -0,0 +1,5 @@
[py3-hosts]
root ansible_ssh_host=188.166.64.76 ansible_ssh_user=root privatekeyfile=~/.ssh/id_rsa_do
[py3-hosts:vars]
ansible_python_interpreter=/usr/bin/python3

8
ansible/system.yaml Normal file
View File

@ -0,0 +1,8 @@
---
- hosts: all
gather_facts: no
tasks:
- name: Read SSH public key
slurp: src=~/.ssh/id_rsa_do.pub
register: public_key
# debug: msg="{{ public_key['content'] | b64decode }}"

BIN
db.sqlite3 Normal file

Binary file not shown.

15
manage.py Executable file
View File

@ -0,0 +1,15 @@
#!/usr/bin/env python
import os
import sys
if __name__ == "__main__":
os.environ.setdefault("DJANGO_SETTINGS_MODULE", "mysite.settings")
try:
from django.core.management import execute_from_command_line
except ImportError as exc:
raise ImportError(
"Couldn't import Django. Are you sure it's installed and "
"available on your PYTHONPATH environment variable? Did you "
"forget to activate a virtual environment?"
) from exc
execute_from_command_line(sys.argv)

7
mysite/__init__.py Normal file
View File

@ -0,0 +1,7 @@
from __future__ import absolute_import, unicode_literals
# This will make sure the app is always imported when
# Django starts so that shared_task will use this app.
from .celery import app as celery_app
__all__ = ('celery_app',)

32
mysite/celery.py Normal file
View File

@ -0,0 +1,32 @@
from __future__ import absolute_import, unicode_literals
import os
from celery import Celery
from celery.schedules import crontab
# set the default Django settings module for the 'celery' program.
os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'mysite.settings')
app = Celery('mysite')
# Using a string here means the worker doesn't have to serialize
# the configuration object to child processes.
# - namespace='CELERY' means all celery-related configuration keys
# should have a `CELERY_` prefix.
app.config_from_object('django.conf:settings', namespace='CELERY')
# Load task modules from all registered Django app configs.
app.autodiscover_tasks()
@app.task(bind=True)
def debug_task(self):
print('Request: {0!r}'.format(self.request))
app.conf.beat_schedule = {
'get_rates': {
'task': 'rates.tasks.parse_cb',
'schedule': crontab(hour=0, minute=0,), # run daily at midnight
},
}

130
mysite/settings.py Normal file
View File

@ -0,0 +1,130 @@
"""
Django settings for mysite project.
Generated by 'django-admin startproject' using Django 2.0.
For more information on this file, see
https://docs.djangoproject.com/en/2.0/topics/settings/
For the full list of settings and their values, see
https://docs.djangoproject.com/en/2.0/ref/settings/
"""
import os
# Build paths inside the project like this: os.path.join(BASE_DIR, ...)
BASE_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
# Quick-start development settings - unsuitable for production
# See https://docs.djangoproject.com/en/2.0/howto/deployment/checklist/
# SECURITY WARNING: keep the secret key used in production secret!
SECRET_KEY = 't4-k79wmngkt*1!z0b!75yk&+1@!*wu)s(o0$wx+qt&jf*7hp@'
# SECURITY WARNING: don't run with debug turned on in production!
DEBUG = False
ALLOWED_HOSTS = ['web', '127.0.0.1']
# Application definition
INSTALLED_APPS = [
'django.contrib.admin',
'django.contrib.auth',
'django.contrib.contenttypes',
'django.contrib.sessions',
'django.contrib.messages',
'django.contrib.staticfiles',
'rates'
]
MIDDLEWARE = [
'django.middleware.security.SecurityMiddleware',
'django.contrib.sessions.middleware.SessionMiddleware',
'django.middleware.common.CommonMiddleware',
'django.middleware.csrf.CsrfViewMiddleware',
'django.contrib.auth.middleware.AuthenticationMiddleware',
'django.contrib.messages.middleware.MessageMiddleware',
'django.middleware.clickjacking.XFrameOptionsMiddleware',
]
ROOT_URLCONF = 'mysite.urls'
TEMPLATES = [
{
'BACKEND': 'django.template.backends.django.DjangoTemplates',
'DIRS': [],
'APP_DIRS': True,
'OPTIONS': {
'context_processors': [
'django.template.context_processors.debug',
'django.template.context_processors.request',
'django.contrib.auth.context_processors.auth',
'django.contrib.messages.context_processors.messages',
],
},
},
]
WSGI_APPLICATION = 'mysite.wsgi.application'
# Database
# https://docs.djangoproject.com/en/2.0/ref/settings/#databases
DATABASES = {
'default': {
'ENGINE': 'django.db.backends.sqlite3',
'NAME': os.path.join(BASE_DIR, 'db.sqlite3'),
}
}
# Password validation
# https://docs.djangoproject.com/en/2.0/ref/settings/#auth-password-validators
AUTH_PASSWORD_VALIDATORS = [
{
'NAME': 'django.contrib.auth.password_validation.UserAttributeSimilarityValidator',
},
{
'NAME': 'django.contrib.auth.password_validation.MinimumLengthValidator',
},
{
'NAME': 'django.contrib.auth.password_validation.CommonPasswordValidator',
},
{
'NAME': 'django.contrib.auth.password_validation.NumericPasswordValidator',
},
]
# Internationalization
# https://docs.djangoproject.com/en/2.0/topics/i18n/
LANGUAGE_CODE = 'en-us'
TIME_ZONE = 'UTC'
USE_I18N = True
USE_L10N = True
USE_TZ = True
# Static files (CSS, JavaScript, Images)
# https://docs.djangoproject.com/en/2.0/howto/static-files/
PROJECT_ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
STATIC_URL = '/static/'
STATIC_ROOT = os.path.join(PROJECT_ROOT, 'static')
CELERY_BROKER_URL = 'redis://redis:6379'
CELERY_RESULT_BACKEND = 'redis://redis:6379'
CELERY_ACCEPT_CONTENT = ['application/json']
CELERY_TASK_SERIALIZER = 'json'
CELERY_RESULT_SERIALIZER = 'json'

23
mysite/urls.py Normal file
View File

@ -0,0 +1,23 @@
"""mysite URL Configuration
The `urlpatterns` list routes URLs to views. For more information please see:
https://docs.djangoproject.com/en/2.0/topics/http/urls/
Examples:
Function views
1. Add an import: from my_app import views
2. Add a URL to urlpatterns: path('', views.home, name='home')
Class-based views
1. Add an import: from other_app.views import Home
2. Add a URL to urlpatterns: path('', Home.as_view(), name='home')
Including another URLconf
1. Import the include() function: from django.urls import include, path
2. Add a URL to urlpatterns: path('blog/', include('blog.urls'))
"""
from django.contrib import admin
from django.urls import path
from rates import views
urlpatterns = [
path('admin/', admin.site.urls),
path('', views.index, name="rates")
]

16
mysite/wsgi.py Normal file
View File

@ -0,0 +1,16 @@
"""
WSGI config for mysite project.
It exposes the WSGI callable as a module-level variable named ``application``.
For more information on this file, see
https://docs.djangoproject.com/en/2.0/howto/deployment/wsgi/
"""
import os
from django.core.wsgi import get_wsgi_application
os.environ.setdefault("DJANGO_SETTINGS_MODULE", "mysite.settings")
application = get_wsgi_application()

0
rates/__init__.py Normal file
View File

3
rates/admin.py Normal file
View File

@ -0,0 +1,3 @@
from django.contrib import admin
# Register your models here.

5
rates/apps.py Normal file
View File

@ -0,0 +1,5 @@
from django.apps import AppConfig
class RatesConfig(AppConfig):
name = 'rates'

View File

3
rates/models.py Normal file
View File

@ -0,0 +1,3 @@
from django.db import models
# Create your models here.

20
rates/tasks.py Normal file
View File

@ -0,0 +1,20 @@
import xml.etree.ElementTree as ET
from datetime import datetime
import redis
import requests
from celery import task
from celery.utils.log import get_task_logger
logger = get_task_logger(__name__)
@task
def parse_cb():
url = 'https://www.cbr-xml-daily.ru/daily_utf8.xml'
res = requests.get(url)
root = ET.fromstring(res.content.decode())
result = {child.find("CharCode").text: child.find("Value").text for child in root}
redis_ = redis.StrictRedis(host='redis', port=6379, db=0)
redis_.set('result', str(result))
redis_.set('time', str(datetime.now()))
logger.info(result)

11
rates/tests.py Normal file
View File

@ -0,0 +1,11 @@
"""Tests for cb parser application.
Run: py.test rates/tests.py """
import re
import requests
def test_smoke():
"""Use regexp to check web page content."""
res = requests.get("http://localhost/")
pattern = r"^data: {('[A-Z]{3}': '\d+,\d+',?\s?)+}<br><br>updated on \d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2}.\d{6}$"
assert re.match(re.compile(pattern), res.content.decode("utf-8"))

14
rates/views.py Normal file
View File

@ -0,0 +1,14 @@
from django.http import HttpResponse
import redis
from .tasks import parse_cb
def index(request):
r = redis.StrictRedis(host="redis", port=6379, db=0)
try:
data = r.get("result").decode()
last_time = r.get("time").decode()
except AttributeError:
parse_cb.delay()
return HttpResponse("Service is not available.")
return HttpResponse("data: {}<br><br>updated on {}".format(data, last_time))

4
requirements.txt Normal file
View File

@ -0,0 +1,4 @@
celery==4.2.1
Django==2.0
redis
requests==2.19.1