From af610ab49194f12783fdfb2cc676c4877b22ed7c Mon Sep 17 00:00:00 2001 From: clarkzjw Date: Thu, 23 Feb 2023 22:24:05 -0800 Subject: deploy: add Dockerfile and docker-compose.yaml --- .dockerignore | 3 +++ Dockerfile | 11 ++++++++++- bot.py | 2 +- config.ini.example | 7 +++++++ config.py | 11 ++++++----- contrib/docker-compose.yaml | 13 +++++++++++++ dbstore/peewee_store.py | 2 +- requirements.txt | 8 -------- 8 files changed, 41 insertions(+), 16 deletions(-) create mode 100644 .dockerignore create mode 100644 contrib/docker-compose.yaml diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 0000000..8a229dd --- /dev/null +++ b/.dockerignore @@ -0,0 +1,3 @@ +.venv/ +*.db +*.ini diff --git a/Dockerfile b/Dockerfile index 2d13273..6e97ed6 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1 +1,10 @@ -FROM python:3 +FROM python:3.10 + +WORKDIR /usr/src/app + +COPY requirements.txt ./ +RUN pip install --no-cache-dir -r requirements.txt + +COPY . . + +CMD [ "python", "./bot.py" ] diff --git a/bot.py b/bot.py index 1d59f9e..b44f9df 100644 --- a/bot.py +++ b/bot.py @@ -229,7 +229,7 @@ async def main() -> None: app=starlette_app, port=BOT_PORT, use_colors=False, - host="100.93.242.2", + host="0.0.0.0", ) ) diff --git a/config.ini.example b/config.ini.example index e3a9121..d52a4d1 100644 --- a/config.ini.example +++ b/config.ini.example @@ -2,3 +2,10 @@ BOT_TOKEN = FOURSQUARE_API_KEY = ENCRYPT_KEY = + +[API] +TELEGRAM_WEBHOOK_URL = "/checkinbot/webhook" +HEALTHCHECK_URL = "/checkinbot/healthcheck" +FEDI_LOGIN_CALLBACK_URL = "/checkinbot/fedi_login_callback" +BOT_DOMAIN = "https://" +BOT_PORT = 8080 diff --git a/config.py b/config.py index 0fae9b1..fe69bd9 100644 --- a/config.py +++ b/config.py @@ -10,6 +10,12 @@ BOT_TOKEN = config["DEFAULT"]["BOT_TOKEN"] FSQ_API_KEY = config["DEFAULT"]["FOURSQUARE_API_KEY"] ENCRYPT_KEY = config["DEFAULT"]["ENCRYPT_KEY"] +TELEGRAM_WEBHOOK_URL = config["API"]["TELEGRAM_WEBHOOK_URL"] +HEALTHCHECK_URL = config["API"]["HEALTHCHECK_URL"] +FEDI_LOGIN_CALLBACK_URL = config["API"]["FEDI_LOGIN_CALLBACK_URL"] +BOT_DOMAIN = config["API"]["BOT_DOMAIN"] +BOT_PORT = int(config["API"]["BOT_PORT"]) + MEDIA_GROUP_TIMEOUT = 3 FEDI_LOGIN, WAIT_VISIBILITY, WAIT_LOCATION, LOCATION_SEARCH_KEYWORD, LOCATION_CONFIRMATION, ADD_MEDIA, ADD_COMMENT = range(7) @@ -41,9 +47,4 @@ class MsgDict(TypedDict): KEY_TOOT_STATUS_ID = "toot_status_id" KEY_TOOT_STATUS_CONTENT = "toot_status_content" -TELEGRAM_WEBHOOK_URL = "/checkinbot/webhook" -HEALTHCHECK_URL = "/checkinbot/healthcheck" -FEDI_LOGIN_CALLBACK_URL = "/checkinbot/fedi_login_callback" -BOT_DOMAIN = "https://zjw.social" -BOT_PORT = 30010 BOT_SCOPE = ['read:accounts', 'write:media', 'write:statuses'] diff --git a/contrib/docker-compose.yaml b/contrib/docker-compose.yaml new file mode 100644 index 0000000..7ba8c2d --- /dev/null +++ b/contrib/docker-compose.yaml @@ -0,0 +1,13 @@ +version: '3' +services: + checkin.bot: + image: "clarkzjw/checkin.bot:latest" + restart: always + volumes: + - "./config.ini:/usr/src/app/config.ini" + - sqlite3:/usr/src/app/database/ + ports: + - "8080:8080" + +volumes: + sqlite3: diff --git a/dbstore/peewee_store.py b/dbstore/peewee_store.py index c8d9c22..d05c642 100644 --- a/dbstore/peewee_store.py +++ b/dbstore/peewee_store.py @@ -5,7 +5,7 @@ TOOT_VISIBILITY_UNLISTED = "unlisted" TOOT_VISIBILITY_PRIVATE = "private" -db = SqliteDatabase("checkinbot.db") +db = SqliteDatabase("database/checkinbot.db") db.connect(reuse_if_open=True) diff --git a/requirements.txt b/requirements.txt index d1e0036..39887b9 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,12 +1,10 @@ anyio==3.6.2 APScheduler==3.10.0 -asgiref==3.6.0 blurhash==1.1.4 certifi==2022.12.7 cffi==1.15.1 charset-normalizer==3.0.1 click==8.1.3 -crypto==1.4.1 cryptography==39.0.1 decorator==5.1.1 greenlet==2.0.2 @@ -18,24 +16,18 @@ httpx==0.23.3 hyperframe==6.0.1 idna==3.4 Mastodon.py @ git+https://cgit.jinwei.me/clarkzjw/mastodon.py@503d58d226c52901532067d3dcb5a7814699f7fb -Naked==0.1.32 peewee==3.15.4 -Pillow==9.4.0 pycparser==2.21 -pycrypto==2.6.1 python-dateutil==2.8.2 python-magic==0.4.27 python-telegram-bot==20.1 pytz==2022.7.1 pytz-deprecation-shim==0.1.0.post0 -PyYAML==6.0 requests==2.28.2 rfc3986==1.5.0 -shellescape==3.8.1 six==1.16.0 sniffio==1.3.0 SQLAlchemy==2.0.4 -sqlparse==0.4.3 starlette==0.25.0 typing_extensions==4.5.0 tzdata==2022.7 -- cgit v1.2.3 From b097f3c13ddfa51551bce9ca9344c796e5cea9cb Mon Sep 17 00:00:00 2001 From: clarkzjw Date: Thu, 23 Feb 2023 23:42:50 -0800 Subject: deploy: add krakend api gateway --- bot.py | 8 ++-- contrib/docker-compose.yaml | 11 +++++- contrib/krakend.json | 90 +++++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 104 insertions(+), 5 deletions(-) create mode 100644 contrib/krakend.json diff --git a/bot.py b/bot.py index b44f9df..bcfab8f 100644 --- a/bot.py +++ b/bot.py @@ -8,7 +8,7 @@ from config import BOT_TOKEN, TELEGRAM_WEBHOOK_URL, HEALTHCHECK_URL, FEDI_LOGIN_ import uvicorn from starlette.applications import Starlette from starlette.requests import Request -from starlette.responses import PlainTextResponse, Response +from starlette.responses import PlainTextResponse, Response, JSONResponse from starlette.routing import Route from telegram import Update from telegram.ext import ( @@ -188,12 +188,12 @@ async def main() -> None: await application.bot.set_webhook(url=f"{BOT_DOMAIN}{TELEGRAM_WEBHOOK_URL}") # Set up webserver - async def telegram_webhook(request: Request) -> Response: + async def telegram_webhook(request: Request) -> JSONResponse: """Handle incoming Telegram updates by putting them into the `update_queue`""" await application.update_queue.put( Update.de_json(data=await request.json(), bot=application.bot) ) - return Response() + return JSONResponse({'OK': 200}) async def fedi_oauth_login_callback(request: Request) -> PlainTextResponse: """ @@ -221,7 +221,7 @@ async def main() -> None: routes=[ Route(TELEGRAM_WEBHOOK_URL, telegram_webhook, methods=["POST"]), Route(HEALTHCHECK_URL, healthcheck, methods=["GET"]), - Route(FEDI_LOGIN_CALLBACK_URL, fedi_oauth_login_callback, methods=["POST", "GET"]), + Route(FEDI_LOGIN_CALLBACK_URL, fedi_oauth_login_callback, methods=["GET"]), ] ) webserver = uvicorn.Server( diff --git a/contrib/docker-compose.yaml b/contrib/docker-compose.yaml index 7ba8c2d..3ca7981 100644 --- a/contrib/docker-compose.yaml +++ b/contrib/docker-compose.yaml @@ -1,6 +1,6 @@ version: '3' services: - checkin.bot: + checkinbot: image: "clarkzjw/checkin.bot:latest" restart: always volumes: @@ -9,5 +9,14 @@ services: ports: - "8080:8080" + krakend: + image: "devopsfaith/krakend:latest" + restart: always + ports: + - "8081:8080" + - "9091:9091" + volumes: + - "./krakend.json:/etc/krakend/krakend.json" + volumes: sqlite3: diff --git a/contrib/krakend.json b/contrib/krakend.json new file mode 100644 index 0000000..1f89340 --- /dev/null +++ b/contrib/krakend.json @@ -0,0 +1,90 @@ +{ + "$schema": "https://www.krakend.io/schema/v3.json", + "version": 3, + "name": "Checkin.bot API", + "extra_config": { + "telemetry/opencensus": { + "sample_rate": 100, + "reporting_period": 0, + "exporters": { + "prometheus": { + "port": 9091, + "namespace": "krakend", + "tag_host": false, + "tag_path": true, + "tag_method": true, + "tag_statuscode": false + } + } + }, + "telemetry/metrics": { + "collection_time": "60s", + "proxy_disabled": false, + "router_disabled": false, + "backend_disabled": false, + "endpoint_disabled": false, + "listen_address": ":8090" + } + }, + "timeout": "3000ms", + "cache_ttl": "300s", + "output_encoding": "json", + "debug_endpoint": true, + "endpoints": [ + { + "endpoint": "/checkinbot/webhook", + "method": "POST", + "output_encoding": "json", + "backend": [ + { + "url_pattern": "/checkinbot/webhook", + "encoding": "json", + "sd": "static", + "method": "POST", + "host": [ + "http://checkinbot:8080" + ], + "disable_host_sanitize": false + } + ] + }, + { + "endpoint": "/checkinbot/healthcheck", + "method": "GET", + "output_encoding": "string", + "backend": [ + { + "url_pattern": "/checkinbot/healthcheck", + "encoding": "string", + "sd": "static", + "method": "GET", + "host": [ + "http://checkinbot:8080" + ], + "disable_host_sanitize": false + } + ] + }, + { + "endpoint": "/checkinbot/fedi_login_callback", + "method": "GET", + "output_encoding": "string", + "backend": [ + { + "url_pattern": "/checkinbot/fedi_login_callback", + "encoding": "string", + "sd": "static", + "method": "GET", + "disable_host_sanitize": false, + "host": [ + "http://checkinbot:8080" + ] + } + ], + "input_query_strings": [ + "code", + "state" + ] + } + ] +} \ No newline at end of file -- cgit v1.2.3 From 391abe42adf3bf065d9543ada068b696b00efdcd Mon Sep 17 00:00:00 2001 From: clarkzjw Date: Fri, 24 Feb 2023 00:49:12 -0800 Subject: deploy: add krakend monitoring using influxdb and grafana --- contrib/docker-compose.yaml | 27 ++++++++++++++++++++++++++- contrib/influx/initdb.sh | 16 ++++++++++++++++ contrib/krakend.json | 20 +++++++------------- 3 files changed, 49 insertions(+), 14 deletions(-) create mode 100755 contrib/influx/initdb.sh diff --git a/contrib/docker-compose.yaml b/contrib/docker-compose.yaml index 3ca7981..8aeaff8 100644 --- a/contrib/docker-compose.yaml +++ b/contrib/docker-compose.yaml @@ -10,7 +10,7 @@ services: - "8080:8080" krakend: - image: "devopsfaith/krakend:latest" + image: "devopsfaith/krakend:2.2" restart: always ports: - "8081:8080" @@ -18,5 +18,30 @@ services: volumes: - "./krakend.json:/etc/krakend/krakend.json" + influx: + image: influxdb:2.6.1 + environment: + - "DOCKER_INFLUXDB_INIT_MODE=setup" + - "DOCKER_INFLUXDB_INIT_USERNAME=krakend" + - "DOCKER_INFLUXDB_INIT_PASSWORD=password" + - "DOCKER_INFLUXDB_INIT_ORG=my-org" + - "DOCKER_INFLUXDB_INIT_BUCKET=krakend" + - "DOCKER_INFLUXDB_INIT_RETENTION=1w" + - "DOCKER_INFLUXDB_INIT_ADMIN_TOKEN=my-super-secret-auth-token" + ports: + - "8086:8086" + volumes: + - "./influx/initdb.sh:/docker-entrypoint-initdb.d/initdb.sh" + + grafana: + image: "grafana/grafana:9.3.6" + restart: always + ports: + - "3000:3000" + volumes: + - grafana:/var/lib/grafana + volumes: sqlite3: + grafana: + influxdb: diff --git a/contrib/influx/initdb.sh b/contrib/influx/initdb.sh new file mode 100755 index 0000000..b1d07af --- /dev/null +++ b/contrib/influx/initdb.sh @@ -0,0 +1,16 @@ +#!/bin/bash +# The following script creates the authentication for Grafana +# and is meant to be added to InfluxDB docker-compose.yml file. +# image: influxdb +# volumes: +# - "./config/influx:/docker-entrypoint-initdb.d" +set -e + +# Retrieve the ID from the bucket created during setup +BUCKET_ID=$(influx bucket list | grep "$DOCKER_INFLUXDB_INIT_BUCKET" | awk '{print $1}') + +influx v1 auth create \ + --username ${DOCKER_INFLUXDB_INIT_USERNAME} \ + --password ${DOCKER_INFLUXDB_INIT_PASSWORD} \ + --write-bucket ${BUCKET_ID} \ + --org ${DOCKER_INFLUXDB_INIT_ORG} diff --git a/contrib/krakend.json b/contrib/krakend.json index 1f89340..2ed23d8 100644 --- a/contrib/krakend.json +++ b/contrib/krakend.json @@ -3,19 +3,13 @@ "version": 3, "name": "Checkin.bot API", "extra_config": { - "telemetry/opencensus": { - "sample_rate": 100, - "reporting_period": 0, - "exporters": { - "prometheus": { - "port": 9091, - "namespace": "krakend", - "tag_host": false, - "tag_path": true, - "tag_method": true, - "tag_statuscode": false - } - } + "telemetry/influx": { + "address": "http://influx:8086", + "ttl": "25s", + "buffer_size": 100, + "db": "krakend", + "username": "krakend", + "password": "password" }, "telemetry/metrics": { "collection_time": "60s", -- cgit v1.2.3