From c9b924a1e366ca4d669c49caab9387d323e62e0b Mon Sep 17 00:00:00 2001 From: William Peebles Date: Tue, 21 Mar 2023 18:12:48 -0400 Subject: [PATCH 01/16] update dependencies --- app/requirements.txt | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/app/requirements.txt b/app/requirements.txt index bd4eaad..3e68a82 100644 --- a/app/requirements.txt +++ b/app/requirements.txt @@ -6,31 +6,31 @@ gunicorn==20.1.0 # Flask Framework click==8.1.3 -Flask==2.2.2 +Flask==2.2.3 itsdangerous==2.1.2 Jinja2==3.1.2 -MarkupSafe==2.1.1 -Werkzeug==2.2.2 +MarkupSafe==2.1.2 +Werkzeug==2.2.3 # Flask Packages Flask-Login==0.6.2 -Flask-Migrate==3.1.0 +Flask-Migrate==4.0.4 Flask-Script==2.0.6 -Flask-SQLAlchemy==3.0.2 -Flask-WTF==1.0.1 +Flask-SQLAlchemy==3.0.3 +Flask-WTF==1.1.1 Flask-User==1.0.2.2 # WTForms Extensions WTForms-SQLAlchemy==0.3.0 # APScheduler automated scheduler -APScheduler==3.9.1.post1 +APScheduler==3.10.1 # Python Time packages -pytz==2022.6 +pytz==2022.7.1 # Automated tests -pytest==7.2.0 +pytest==7.2.2 pytest-cov==4.0.0 # MySQL Package -- 2.49.1 From d9391c73e9e9e3bfa3240b51dabe355217a8943d Mon Sep 17 00:00:00 2001 From: William Peebles Date: Tue, 21 Mar 2023 18:15:01 -0400 Subject: [PATCH 02/16] change version to 1.1.0 --- app/__version__ | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/__version__ b/app/__version__ index 7f20734..1cc5f65 100644 --- a/app/__version__ +++ b/app/__version__ @@ -1 +1 @@ -1.0.1 \ No newline at end of file +1.1.0 \ No newline at end of file -- 2.49.1 From b68e48b1cfe44dd35ae05f0d9286fbf7fa032de7 Mon Sep 17 00:00:00 2001 From: William Peebles Date: Tue, 21 Mar 2023 19:17:35 -0400 Subject: [PATCH 03/16] create logging service --- app/log.py | 0 1 file changed, 0 insertions(+), 0 deletions(-) create mode 100644 app/log.py diff --git a/app/log.py b/app/log.py new file mode 100644 index 0000000..e69de29 -- 2.49.1 From c21cb0354608014e7f7429cc727f4fcf93c45cb1 Mon Sep 17 00:00:00 2001 From: William Peebles Date: Tue, 21 Mar 2023 19:18:08 -0400 Subject: [PATCH 04/16] add logging to apps --- app/app.py | 31 +++++++++++++++++++++++++++---- app/create_events.py | 5 ++++- app/log.py | 12 ++++++++++++ 3 files changed, 43 insertions(+), 5 deletions(-) diff --git a/app/app.py b/app/app.py index 79351ef..7c459b9 100644 --- a/app/app.py +++ b/app/app.py @@ -7,6 +7,10 @@ from misc import datetime, date, time, currDay, prevDay, ZoneInfo, currVersion, from db import (db, Period, Task, Event, User) from forms import (TaskForm, EventForm, PeriodForm, SignupForm, LoginForm, SettingsForm) from create_events import createEvents +from log import appLogger + +# init logger +logger = appLogger('app') basedir = os.path.abspath(os.path.dirname(__file__)) @@ -25,10 +29,6 @@ db.init_app(app) migrate = Migrate() migrate.init_app(app, db) -# Schedule creation of events every hour -#with app.app_context(): -# scheduleCreateEvents(app, db, currDay, Period, Event, createEvents) - # Authentication stuff login_manager = LoginManager() @@ -78,6 +78,7 @@ def index(): # Authentication routes @app.route('/login', methods=['GET', 'POST']) def login(): + sourceIP = request.environ.get('HTTP_X_REAL_IP', request.remote_addr) form = LoginForm() if form.validate_on_submit(): userName = form.userName.data @@ -87,13 +88,16 @@ def login(): user = User.query.filter_by(userName=userName).first() if not user or not check_password_hash(user.password, password): flash('Credentials incorrect! Please try again') + logger.info(f'User \'{userName}\' logon FAILED (bad password) from {sourceIP}') return redirect(url_for('login')) login_user(user, remember=remember) + logger.info(f'User \'{userName}\' logged in successfully from {sourceIP}') return redirect(url_for('events')) return render_template('login.html', form=form) @app.route('/createaccount', methods=['GET', 'POST']) def createAccount(): + sourceIP = request.environ.get('HTTP_X_REAL_IP', request.remote_addr) if signup_enabled == True: form = SignupForm() if form.validate_on_submit(): @@ -105,6 +109,7 @@ def createAccount(): password=generate_password_hash(form.password.data, method='sha256')) db.session.add(new_user) db.session.commit() + logger.info(f'New user \'{new_user.userName}\' created from {sourceIP}') return redirect(url_for('login')) return render_template('createAccount.html', form=form) @@ -113,12 +118,15 @@ def createAccount(): @app.route('/logout') @login_required def logout(): + sourceIP = request.environ.get('HTTP_X_REAL_IP', request.remote_addr) + logger.info(f'User \'{current_user.userName}\' logged out from {sourceIP} ') logout_user() return redirect(url_for('index')) @app.route('/settings', methods=('GET', 'POST')) @login_required def settings(): + sourceIP = request.environ.get('HTTP_X_REAL_IP', request.remote_addr) user = User.query.get_or_404(current_user.id) form = SettingsForm(obj=user) if form.validate_on_submit(): @@ -127,6 +135,7 @@ def settings(): if form.password.data != '': user.password = generate_password_hash(form.password.data, method='sha256') db.session.commit() + logger.info(f'User \'{current_user.userName}\' settings changed from {sourceIP}') return render_template('settings.html', form=form) # Periods routes @@ -139,6 +148,7 @@ def periods(): @app.route('/period/new', methods=('GET', 'POST')) @login_required def newPeriod(): + sourceIP = request.environ.get('HTTP_X_REAL_IP', request.remote_addr) form = PeriodForm() if form.validate_on_submit(): period = Period(periodTime=form.periodTime.data, @@ -146,6 +156,7 @@ def newPeriod(): ) db.session.add(period) db.session.commit() + logger.info(f'New period added by \'{current_user.userName}\' from {sourceIP}') # Run createEvents upon adding new period createEvents(db, currDay, Period, Event) return redirect(f'/period/edit/{period.period}') @@ -155,21 +166,25 @@ def newPeriod(): @app.route('/period/edit/', methods=('GET', 'POST')) @login_required def editPeriod(periodNum): + sourceIP = request.environ.get('HTTP_X_REAL_IP', request.remote_addr) period = Period.query.get_or_404(periodNum) form = PeriodForm(obj=period) if form.validate_on_submit(): period.periodTime = form.periodTime.data period.weekendSchedule = form.weekendSchedule.data db.session.commit() + logger.info(f'Period {periodNum} edited by \'{current_user.userName}\' from {sourceIP}') return redirect(f'/period/edit/{periodNum}') return render_template('editPeriod.html', period=period, form=form, datetime=datetime) @app.post('/period/delete/') @login_required def delete_period(periodNum): + sourceIP = request.environ.get('HTTP_X_REAL_IP', request.remote_addr) period = Period.query.get_or_404(periodNum) db.session.delete(period) db.session.commit() + logger.info(f'Period {periodNum} deleted by \'{current_user.userName}\' from {sourceIP}') return redirect('/periods') @@ -185,6 +200,7 @@ def events(): @app.route('/event/edit//', methods=('GET', 'POST')) @login_required def editEvent(event_id): + sourceIP = request.environ.get('HTTP_X_REAL_IP', request.remote_addr) event = Event.query.get_or_404(event_id) form = EventForm(obj=event) if form.validate_on_submit(): @@ -193,6 +209,7 @@ def editEvent(event_id): else: event.task_id = None db.session.commit() + logger.info(f'Event {event_id} edited by \'{current_user.userName}\' from {sourceIP}') return redirect('/events') return render_template('editEvent.html', event=event, form=form, datetime=datetime) @@ -212,6 +229,7 @@ def task(task_id): @app.route('/task/new', methods=('GET', 'POST')) @login_required def newTask(): + sourceIP = request.environ.get('HTTP_X_REAL_IP', request.remote_addr) form = TaskForm() if form.validate_on_submit(): task = Task(title=form.title.data, @@ -219,24 +237,29 @@ def newTask(): created_timestamp=int(time.time())) db.session.add(task) db.session.commit() + logger.info(f'New task added by \'{current_user.userName}\' from {sourceIP}') return redirect(f'/task/{task.id}') return render_template('newtask.html', form=form) @app.route('/task//edit', methods=('GET', 'POST')) @login_required def editTask(task_id): + sourceIP = request.environ.get('HTTP_X_REAL_IP', request.remote_addr) task = Task.query.get_or_404(task_id) form = TaskForm(obj=task) if form.validate_on_submit(): task.title=form.title.data task.description=form.description.data db.session.commit() + logger.info(f'Task {task_id} edited by \'{current_user.userName}\' from {sourceIP}') return redirect(f'/task/{task_id}') return render_template('edittask.html', task=task, form=form) @app.post('/task//delete') @login_required def delete_task(task_id): + sourceIP = request.environ.get('HTTP_X_REAL_IP', request.remote_addr) task = Task.query.get_or_404(task_id) db.session.delete(task) db.session.commit() + logger.info(f'Task {task_id} deleted by \'{current_user.userName}\' from {sourceIP}') return redirect('/tasks') \ No newline at end of file diff --git a/app/create_events.py b/app/create_events.py index 2033c0d..05b89bf 100644 --- a/app/create_events.py +++ b/app/create_events.py @@ -1,3 +1,6 @@ +from log import appLogger + +logger = appLogger('app') def createEvents(db, currDay, Period, Event): periods = Period.query.all() for period in periods: @@ -10,4 +13,4 @@ def createEvents(db, currDay, Period, Event): db.session.add(event) db.session.commit() - print("createEvents script ran successfully") \ No newline at end of file + logger.info('createEvents script ran successfully') \ No newline at end of file diff --git a/app/log.py b/app/log.py index e69de29..746749a 100644 --- a/app/log.py +++ b/app/log.py @@ -0,0 +1,12 @@ +import logging +import sys + +def appLogger(name): + formatter = logging.Formatter(fmt='%(asctime)s %(levelname)-8s %(message)s', + datefmt='%Y-%m-%d %H:%M:%S') + screen_handler = logging.StreamHandler(stream=sys.stdout) + screen_handler.setFormatter(formatter) + logger = logging.getLogger(name) + logger.setLevel(logging.DEBUG) + logger.addHandler(screen_handler) + return logger \ No newline at end of file -- 2.49.1 From 437c093adfffb7bea057e013c17ac72b95e5e2b0 Mon Sep 17 00:00:00 2001 From: William Peebles Date: Tue, 21 Mar 2023 22:44:51 -0400 Subject: [PATCH 05/16] logging bugfix --- app/log.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/app/log.py b/app/log.py index 746749a..94ab670 100644 --- a/app/log.py +++ b/app/log.py @@ -8,5 +8,7 @@ def appLogger(name): screen_handler.setFormatter(formatter) logger = logging.getLogger(name) logger.setLevel(logging.DEBUG) + if (logger.hasHandlers()): + logger.handlers.clear() logger.addHandler(screen_handler) return logger \ No newline at end of file -- 2.49.1 From a58cc0ffcb2ef6327a23efd5298d8274d98a9161 Mon Sep 17 00:00:00 2001 From: William Peebles Date: Wed, 22 Mar 2023 21:59:09 -0400 Subject: [PATCH 06/16] make links clickable in task descriptions --- app/app.py | 7 ++++--- app/templates/events.html | 12 +++++++++++- app/templates/task.html | 12 +++++++++++- app/templates/tasks.html | 12 +++++++++++- 4 files changed, 37 insertions(+), 6 deletions(-) diff --git a/app/app.py b/app/app.py index 7c459b9..ab2eb6a 100644 --- a/app/app.py +++ b/app/app.py @@ -1,4 +1,5 @@ import os +import re from flask import Flask, render_template, redirect, url_for, request, flash from flask_migrate import Migrate from werkzeug.security import generate_password_hash, check_password_hash @@ -195,7 +196,7 @@ def events(): events = Event.query.all() periods = Period.query.all() - return render_template('events.html', events=events, periods=periods, datetime=datetime, date=date, ZoneInfo=ZoneInfo) + return render_template('events.html', events=events, periods=periods, datetime=datetime, date=date, ZoneInfo=ZoneInfo, re=re) @app.route('/event/edit//', methods=('GET', 'POST')) @login_required @@ -218,13 +219,13 @@ def editEvent(event_id): @login_required def tasks(): tasks = Task.query.all() - return render_template('tasks.html', str=str, tasks=tasks, datetime=datetime, date=date) + return render_template('tasks.html', str=str, tasks=tasks, datetime=datetime, date=date, re=re) @app.route('/task//') @login_required def task(task_id): task = Task.query.get_or_404(task_id) - return render_template('task.html', str=str, task=task, datetime=datetime, date=date) + return render_template('task.html', str=str, task=task, datetime=datetime, date=date, re=re) @app.route('/task/new', methods=('GET', 'POST')) @login_required diff --git a/app/templates/events.html b/app/templates/events.html index f0dea65..366845c 100644 --- a/app/templates/events.html +++ b/app/templates/events.html @@ -21,7 +21,17 @@
{% if event.tasks.description != None %} -

{{ event.tasks.description }}

+ {% set description = event.tasks.description %} + {% set url_regex = '(https?://[^\s]+)' %} + + {% set match = re.search(url_regex, description) %} + {% if match %} + {% set url = match.group(0) %} + {% set rest = description[match.end(0):] %} +

{{ description[:match.start(0)] }}{{ url }}{{ rest }}

+ {% else %} +

{{ description }}

+ {% endif %} {% endif %}
{% else %} diff --git a/app/templates/task.html b/app/templates/task.html index 886659b..071c3c7 100644 --- a/app/templates/task.html +++ b/app/templates/task.html @@ -9,7 +9,17 @@

{% if task.description != None %} -

{{ task.description }}

+ {% set description = task.description %} + {% set url_regex = '(https?://[^\s]+)' %} + + {% set match = re.search(url_regex, description) %} + {% if match %} + {% set url = match.group(0) %} + {% set rest = description[match.end(0):] %} +

{{ description[:match.start(0)] }}{{ url }}{{ rest }}

+ {% else %} +

{{ description }}

+ {% endif %} {% endif %}
diff --git a/app/templates/tasks.html b/app/templates/tasks.html index e7f0db8..9e39ed7 100644 --- a/app/templates/tasks.html +++ b/app/templates/tasks.html @@ -12,7 +12,17 @@ {% if task.description != None %} - {{ task.description }} + {% set description = task.description %} + {% set url_regex = '(https?://[^\s]+)' %} + + {% set match = re.search(url_regex, description) %} + {% if match %} + {% set url = match.group(0) %} + {% set rest = description[match.end(0):] %} +

{{ description[:match.start(0)] }}{{ url }}{{ rest }}

+ {% else %} +

{{ description }}

+ {% endif %} {% endif %}
-- 2.49.1 From f4106f20451c8d973542125532c0026ff6f42b74 Mon Sep 17 00:00:00 2001 From: William Peebles Date: Wed, 22 Mar 2023 22:14:42 -0400 Subject: [PATCH 07/16] update base docker image from alpine 3.16/python3.11.0 to alpine 3.17/python3.11.2 --- Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index 95e5a63..1bbcd72 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,4 +1,4 @@ -FROM python:3.11.0-alpine3.16 +FROM python:3.11.2-alpine3.17 COPY ./app /app WORKDIR /app RUN apk add gcc musl-dev mariadb-connector-c-dev -- 2.49.1 From 55161f0330bfd45491c151390093c91616993603 Mon Sep 17 00:00:00 2001 From: William Peebles Date: Thu, 23 Mar 2023 18:56:59 -0400 Subject: [PATCH 08/16] add podStatus --- app/app.py | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/app/app.py b/app/app.py index ab2eb6a..73dc8d7 100644 --- a/app/app.py +++ b/app/app.py @@ -71,6 +71,16 @@ def forbidden(e): def ISEerror(e): return render_template('errors/500.html', NODE_NAME=NODE_NAME, POD_NAME=POD_NAME), 500 +# Pod Status route +@app.route('/podStatus') +def podStatus(): + blankPage = "" + try: + db.engine.connect().close() + return blankPage, 200 + except: + return blankPage, 500 + # Index route @app.route('/') def index(): -- 2.49.1 From 3a3fd2da9aa425a6857b05f8b1c8476446e3330e Mon Sep 17 00:00:00 2001 From: William Peebles Date: Thu, 23 Mar 2023 19:53:38 -0400 Subject: [PATCH 09/16] 'created' time to appear in user's defined tz --- app/templates/task.html | 1 + app/templates/tasks.html | 1 + 2 files changed, 2 insertions(+) diff --git a/app/templates/task.html b/app/templates/task.html index 071c3c7..d9246a5 100644 --- a/app/templates/task.html +++ b/app/templates/task.html @@ -1,6 +1,7 @@ {% extends 'base.html' %} {% set createdTime = datetime.fromtimestamp(task.created_timestamp) %} {% set createdTime = datetime.strptime(str(createdTime), '%Y-%m-%d %H:%M:%S') %} +{% set createdTime = createdTime.astimezone(ZoneInfo(current_user.timezone)) %} {% block content %}

{% block title %} {{ task.title }} {% endblock %}

 diff --git a/app/templates/tasks.html b/app/templates/tasks.html index 9e39ed7..f403efd 100644 --- a/app/templates/tasks.html +++ b/app/templates/tasks.html @@ -6,6 +6,7 @@ {% for task in tasks %} {% set createdTime = datetime.fromtimestamp(task.created_timestamp) %} {% set createdTime = datetime.strptime(str(createdTime), '%Y-%m-%d %H:%M:%S') %} + {% set createdTime = createdTime.astimezone(ZoneInfo(current_user.timezone)) %}

{{ task.title }}

-- 2.49.1 From 20af991841c949ecfea8f190eb6845ab2c609b1e Mon Sep 17 00:00:00 2001 From: William Peebles Date: Thu, 23 Mar 2023 20:57:37 -0400 Subject: [PATCH 10/16] add table detection logic --- app/app.py | 15 ++++++++++++++- 1 file changed, 14 insertions(+), 1 deletion(-) diff --git a/app/app.py b/app/app.py index 73dc8d7..0b5c3a4 100644 --- a/app/app.py +++ b/app/app.py @@ -84,7 +84,20 @@ def podStatus(): # Index route @app.route('/') def index(): - return redirect('/events') + # Check for empty database, go to setup page if not setup + global tablesSetup + required_tables = ['period', 'task', 'event', 'user'] + for table in required_tables: + if table in db.metadata.tables: + tablesSetup = True + else: + tablesSetup = False + break + if not tablesSetup: + return "" + # Otherwise, redirect to /events + else: + return redirect('/events') # Authentication routes @app.route('/login', methods=['GET', 'POST']) -- 2.49.1 From 99dc76b25a92cf02cf5ecaa5bba32e8b11e89cdd Mon Sep 17 00:00:00 2001 From: William Peebles Date: Thu, 23 Mar 2023 22:19:03 -0400 Subject: [PATCH 11/16] check for empty database before begin --- app/app.py | 19 ++++++++++--------- 1 file changed, 10 insertions(+), 9 deletions(-) diff --git a/app/app.py b/app/app.py index 0b5c3a4..785adb7 100644 --- a/app/app.py +++ b/app/app.py @@ -6,6 +6,7 @@ from werkzeug.security import generate_password_hash, check_password_hash from flask_login import (LoginManager, login_user, login_required, logout_user, current_user) from misc import datetime, date, time, currDay, prevDay, ZoneInfo, currVersion, currCommit from db import (db, Period, Task, Event, User) +from sqlalchemy import inspect from forms import (TaskForm, EventForm, PeriodForm, SignupForm, LoginForm, SettingsForm) from create_events import createEvents from log import appLogger @@ -82,19 +83,19 @@ def podStatus(): return blankPage, 500 # Index route -@app.route('/') +@app.route('/', methods=['GET', 'POST']) def index(): # Check for empty database, go to setup page if not setup global tablesSetup required_tables = ['period', 'task', 'event', 'user'] - for table in required_tables: - if table in db.metadata.tables: - tablesSetup = True - else: - tablesSetup = False - break + inspector = inspect(db.engine) + tables = inspector.get_table_names() + tablesSetup = all(table in tables for table in required_tables) if not tablesSetup: - return "" + # Initial setup page, should only appear if database is ready but not set up + db.create_all() + logger.info('DB initialized on first run') + return redirect(url_for('createAccount')) # Otherwise, redirect to /events else: return redirect('/events') @@ -138,7 +139,7 @@ def createAccount(): return redirect(url_for('login')) return render_template('createAccount.html', form=form) else: - return 'Account creation is currently disabled' + return 'Account creation is currently disabled.
Set env var SIGNUP_ENABLED=YES to enable account creation' @app.route('/logout') @login_required def logout(): -- 2.49.1 From 85753c5afe371caf99f0b69da19324c85c134b17 Mon Sep 17 00:00:00 2001 From: William Peebles Date: Thu, 13 Apr 2023 02:11:59 -0400 Subject: [PATCH 12/16] rewrite worker to run on celery and redis --- app/app.py | 5 +++-- app/requirements.txt | 4 ++++ app/worker.py | 35 +++++++++++++++++++++-------------- 3 files changed, 28 insertions(+), 16 deletions(-) diff --git a/app/app.py b/app/app.py index 785adb7..4893c55 100644 --- a/app/app.py +++ b/app/app.py @@ -8,9 +8,10 @@ from misc import datetime, date, time, currDay, prevDay, ZoneInfo, currVersion, from db import (db, Period, Task, Event, User) from sqlalchemy import inspect from forms import (TaskForm, EventForm, PeriodForm, SignupForm, LoginForm, SettingsForm) -from create_events import createEvents from log import appLogger +from worker import runCreateEvents + # init logger logger = appLogger('app') @@ -183,7 +184,7 @@ def newPeriod(): db.session.commit() logger.info(f'New period added by \'{current_user.userName}\' from {sourceIP}') # Run createEvents upon adding new period - createEvents(db, currDay, Period, Event) + runCreateEvents() return redirect(f'/period/edit/{period.period}') return render_template('newPeriod.html', form=form) diff --git a/app/requirements.txt b/app/requirements.txt index 3e68a82..95eab1e 100644 --- a/app/requirements.txt +++ b/app/requirements.txt @@ -26,6 +26,10 @@ WTForms-SQLAlchemy==0.3.0 # APScheduler automated scheduler APScheduler==3.10.1 +# Celery Task Queue +celery[redis]==5.2.7 +redis==4.5.4 + # Python Time packages pytz==2022.7.1 diff --git a/app/worker.py b/app/worker.py index 288a29e..ae9541d 100644 --- a/app/worker.py +++ b/app/worker.py @@ -1,9 +1,10 @@ +from celery import Celery +from celery.schedules import crontab +from create_events import createEvents import os from flask import Flask from misc import currDay, datetime, time, timedelta from db import db, Period, Event -from create_events import createEvents -from apscheduler.schedulers.background import BlockingScheduler basedir = os.path.abspath(os.path.dirname(__file__)) @@ -18,18 +19,24 @@ app.config['SQLALCHEMY_DATABASE_URI'] =\ app.config['SQLALCHEMY_TRACK_MODIFICATIONS'] = False db.init_app(app) -# Define function to run create_events with app context -def run_create_events(): +redis_url = 'redis://' + \ + os.environ['REDIS_HOST'] + \ + ':' + os.environ['REDIS_PORT'] + \ + '/' + os.environ['REDIS_DBNUM'] + +celerymsg = Celery('tasks', backend=redis_url, broker=redis_url) + +# Task definitions +@celerymsg.task +def runCreateEvents(): with app.app_context(): createEvents(db, currDay, Period, Event) -# Call createEvents on initial launch of the script -run_create_events() - - -# Set up scheduler to run function at 59th minute of every hour -scheduler = BlockingScheduler() -scheduler.add_job(run_create_events, 'cron', minute=59) - -# Start scheduler -scheduler.start() \ No newline at end of file +# Scheduled tasks +celerymsg.conf.beat_schedule = { + 'hourly-createevents': { + 'task': 'worker.runCreateEvents', + # Run hourly + 'schedule': crontab(hour="*"), + } +} \ No newline at end of file -- 2.49.1 From e46b1ef4b99c1f490da0e5d381935d95d9aa53bc Mon Sep 17 00:00:00 2001 From: William Peebles Date: Thu, 13 Apr 2023 23:56:02 -0400 Subject: [PATCH 13/16] edit docker-compose for celery --- docker-compose.yaml | 16 ++++++++++++++-- 1 file changed, 14 insertions(+), 2 deletions(-) diff --git a/docker-compose.yaml b/docker-compose.yaml index c811c2a..600d3dc 100644 --- a/docker-compose.yaml +++ b/docker-compose.yaml @@ -9,31 +9,43 @@ services: - MYSQL_HOST=db - MYSQL_PORT=3306 - MYSQL_DB=bellscheduler + - REDIS_HOST=redis + - REDIS_PORT=6379 + - REDIS_DBNUM=0 - SECRET_KEY=notasecuresecretkeyonlyuseforlocaldevelopment - NODE_NAME=local - POD_NAME=local - FLASK_ENV=development - FLASK_DEBUG=1 - PYTHONUNBUFFERED=1 + - SIGNUP_ENABLED=YES ports: - 127.0.0.1:80:80 worker: image: container-registry.infra.dubyatp.xyz/bellscheduler/app:latest-testing restart: always - entrypoint: python3 - command: "-m worker" + entrypoint: celery + command: "-A worker.celerymsg worker --loglevel=DEBUG -B" environment: - MYSQL_USER=root - MYSQL_PASSWORD=notasecuresecretkeyonlyuseforlocaldevelopment - MYSQL_HOST=db - MYSQL_PORT=3306 - MYSQL_DB=bellscheduler + - REDIS_HOST=redis + - REDIS_PORT=6379 + - REDIS_DBNUM=0 - SECRET_KEY=notasecuresecretkeyonlyuseforlocaldevelopment - NODE_NAME=local - POD_NAME=local - FLASK_ENV=development - FLASK_DEBUG=1 - PYTHONUNBUFFERED=1 + redis: + image: redis:7.0.10-alpine3.17 + restart: always + ports: + - 127.0.0.1:6379:6379 db: image: mariadb:10.7.8-focal restart: always -- 2.49.1 From 8d51a752758f333cba067ad4c49ec1c754201471 Mon Sep 17 00:00:00 2001 From: William Peebles Date: Sun, 16 Apr 2023 10:54:43 -0400 Subject: [PATCH 14/16] bugfix, have hourly run occur at the 59th minute --- app/worker.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/worker.py b/app/worker.py index ae9541d..f9c54b0 100644 --- a/app/worker.py +++ b/app/worker.py @@ -37,6 +37,6 @@ celerymsg.conf.beat_schedule = { 'hourly-createevents': { 'task': 'worker.runCreateEvents', # Run hourly - 'schedule': crontab(hour="*"), + 'schedule': crontab(hour="*", minute="59"), } } \ No newline at end of file -- 2.49.1 From 723421160fe15f1e4e09c446cfc0404d0070fd30 Mon Sep 17 00:00:00 2001 From: William Peebles Date: Sun, 16 Apr 2023 15:24:25 -0400 Subject: [PATCH 15/16] create cleanupEvents function --- app/cleanup_events.py | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) create mode 100644 app/cleanup_events.py diff --git a/app/cleanup_events.py b/app/cleanup_events.py new file mode 100644 index 0000000..69feb19 --- /dev/null +++ b/app/cleanup_events.py @@ -0,0 +1,16 @@ +from datetime import datetime, timedelta + +def cleanupEvents(db, Event): + # calculate the date one month ago + one_month_ago = datetime.now() - timedelta(days=30) + + # get events older than one month + old_events = db.session.query(Event).filter( + db.func.STR_TO_DATE(Event.scheduled_date, '%m-%d-%Y') < one_month_ago.date() + ).all() + + # delete old events + for event in old_events: + db.session.delete(event) + + db.session.commit() \ No newline at end of file -- 2.49.1 From c4e1a7ab75bccfb3c14cfd33b7c1791ba85bbeb7 Mon Sep 17 00:00:00 2001 From: William Peebles Date: Sun, 16 Apr 2023 15:24:51 -0400 Subject: [PATCH 16/16] add cleanupEvents to worker --- app/worker.py | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/app/worker.py b/app/worker.py index f9c54b0..6e1e617 100644 --- a/app/worker.py +++ b/app/worker.py @@ -1,6 +1,7 @@ from celery import Celery from celery.schedules import crontab from create_events import createEvents +from cleanup_events import cleanupEvents import os from flask import Flask from misc import currDay, datetime, time, timedelta @@ -32,11 +33,20 @@ def runCreateEvents(): with app.app_context(): createEvents(db, currDay, Period, Event) +def runCleanupEvents(): + with app.app_context(): + cleanupEvents(db, Event) + # Scheduled tasks celerymsg.conf.beat_schedule = { 'hourly-createevents': { 'task': 'worker.runCreateEvents', # Run hourly 'schedule': crontab(hour="*", minute="59"), + }, + 'monthly-cleanupevents': { + 'task': 'worker.runCleanupEvents', + # Run monthly + 'schedule': crontab(day_of_month="29") } } \ No newline at end of file -- 2.49.1