From 57e1a8a8992c1b4e6012d970635caeb42a5e17ac Mon Sep 17 00:00:00 2001 From: strNophix Date: Sat, 6 Aug 2022 19:05:26 +0200 Subject: [PATCH] Initial commit --- .gitignore | 160 ++++++++++++++++++++++++++++++++++++++++ .pre-commit-config.yaml | 17 +++++ Makefile | 4 + README.md | 1 + lurker/bot.py | 107 +++++++++++++++++++++++++++ requirements.txt | 2 + requirements_dev.txt | 2 + tables.sql | 33 +++++++++ 8 files changed, 326 insertions(+) create mode 100644 .gitignore create mode 100644 .pre-commit-config.yaml create mode 100644 Makefile create mode 100644 README.md create mode 100644 lurker/bot.py create mode 100644 requirements.txt create mode 100644 requirements_dev.txt create mode 100644 tables.sql diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..68bc17f --- /dev/null +++ b/.gitignore @@ -0,0 +1,160 @@ +# Byte-compiled / optimized / DLL files +__pycache__/ +*.py[cod] +*$py.class + +# C extensions +*.so + +# Distribution / packaging +.Python +build/ +develop-eggs/ +dist/ +downloads/ +eggs/ +.eggs/ +lib/ +lib64/ +parts/ +sdist/ +var/ +wheels/ +share/python-wheels/ +*.egg-info/ +.installed.cfg +*.egg +MANIFEST + +# PyInstaller +# Usually these files are written by a python script from a template +# before PyInstaller builds the exe, so as to inject date/other infos into it. +*.manifest +*.spec + +# Installer logs +pip-log.txt +pip-delete-this-directory.txt + +# Unit test / coverage reports +htmlcov/ +.tox/ +.nox/ +.coverage +.coverage.* +.cache +nosetests.xml +coverage.xml +*.cover +*.py,cover +.hypothesis/ +.pytest_cache/ +cover/ + +# Translations +*.mo +*.pot + +# Django stuff: +*.log +local_settings.py +db.sqlite3 +db.sqlite3-journal + +# Flask stuff: +instance/ +.webassets-cache + +# Scrapy stuff: +.scrapy + +# Sphinx documentation +docs/_build/ + +# PyBuilder +.pybuilder/ +target/ + +# Jupyter Notebook +.ipynb_checkpoints + +# IPython +profile_default/ +ipython_config.py + +# pyenv +# For a library or package, you might want to ignore these files since the code is +# intended to run in multiple environments; otherwise, check them in: +# .python-version + +# pipenv +# According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control. +# However, in case of collaboration, if having platform-specific dependencies or dependencies +# having no cross-platform support, pipenv may install dependencies that don't work, or not +# install all needed dependencies. +#Pipfile.lock + +# poetry +# Similar to Pipfile.lock, it is generally recommended to include poetry.lock in version control. +# This is especially recommended for binary packages to ensure reproducibility, and is more +# commonly ignored for libraries. +# https://python-poetry.org/docs/basic-usage/#commit-your-poetrylock-file-to-version-control +#poetry.lock + +# pdm +# Similar to Pipfile.lock, it is generally recommended to include pdm.lock in version control. +#pdm.lock +# pdm stores project-wide configurations in .pdm.toml, but it is recommended to not include it +# in version control. +# https://pdm.fming.dev/#use-with-ide +.pdm.toml + +# PEP 582; used by e.g. github.com/David-OConnor/pyflow and github.com/pdm-project/pdm +__pypackages__/ + +# Celery stuff +celerybeat-schedule +celerybeat.pid + +# SageMath parsed files +*.sage.py + +# Environments +.env +.venv +env/ +venv/ +ENV/ +env.bak/ +venv.bak/ + +# Spyder project settings +.spyderproject +.spyproject + +# Rope project settings +.ropeproject + +# mkdocs documentation +/site + +# mypy +.mypy_cache/ +.dmypy.json +dmypy.json + +# Pyre type checker +.pyre/ + +# pytype static type analyzer +.pytype/ + +# Cython debug symbols +cython_debug/ + +# PyCharm +# JetBrains specific template is maintained in a separate JetBrains.gitignore that can +# be found at https://github.com/github/gitignore/blob/main/Global/JetBrains.gitignore +# and can be added to the global gitignore or merged into this file. For a more nuclear +# option (not recommended) you can uncomment the following to ignore the entire idea folder. +#.idea/ diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml new file mode 100644 index 0000000..fe1b354 --- /dev/null +++ b/.pre-commit-config.yaml @@ -0,0 +1,17 @@ +repos: +- repo: https://github.com/pre-commit/pre-commit-hooks + rev: v2.3.0 + hooks: + - id: check-yaml + - id: end-of-file-fixer + - id: trailing-whitespace +- repo: https://github.com/psf/black + rev: 22.3.0 + hooks: + - id: black +- repo: https://github.com/asottile/reorder_python_imports + rev: v3.1.0 + hooks: + - id: reorder-python-imports +default_language_version: + python: python3.10 diff --git a/Makefile b/Makefile new file mode 100644 index 0000000..1732433 --- /dev/null +++ b/Makefile @@ -0,0 +1,4 @@ +.PHONY: lint + +lint/black: + black lurker/ diff --git a/README.md b/README.md new file mode 100644 index 0000000..6686565 --- /dev/null +++ b/README.md @@ -0,0 +1 @@ +# lurker.py diff --git a/lurker/bot.py b/lurker/bot.py new file mode 100644 index 0000000..50fd2c0 --- /dev/null +++ b/lurker/bot.py @@ -0,0 +1,107 @@ +import sys +import typing +from datetime import datetime + +import aiohttp +import discord +from loguru import logger + +T = typing.TypeVar("T") +FlushFunc = typing.Callable[[list[T]], typing.Coroutine[typing.Any, typing.Any, bool]] + +# API_URL = "http://dyn.hillcraft.net:8085/discord_trust_and_safety_is_a_complete_and_utter_joke/batch" +# API_TOKEN = "i8id8xGrFWB67Gjm7KZkU2nAH897oEaU4ytLVT5HZasXFRXoPv" +API_URL = "https://requestbin.io/168xm7b1" +API_TOKEN = ":)" + + +class SpotifyRecord(typing.NamedTuple): + user_id: int + track_id: str + artist: str + album: str + title: str + album_cover_url: str + duration: int + created_at: str + + @staticmethod + def from_activity(user_id: int, activity: discord.Spotify) -> "SpotifyRecord": + return SpotifyRecord( + user_id=user_id, + track_id=str(activity.track_id), + artist=str(activity.artist), + album=str(activity.album), + title=str(activity.title), + album_cover_url=str(activity.album_cover_url), + duration=int(activity.duration.total_seconds()), + created_at=activity.created_at.strftime("%Y-%m-%d %H:%M:%S"), + ) + + +def find_instance( + objs: typing.Iterable[typing.Any], cls: typing.Any +) -> typing.Any | None: + for obj in objs: + if isinstance(obj, cls): + return obj + return None + + +async def submit_records(batch: typing.List[SpotifyRecord]) -> bool: + async with aiohttp.ClientSession() as session: + async with session.post( + API_URL, + headers={"content-type": "application/json", "Authorization": API_TOKEN}, + json={"spotify": batch}, + ) as resp: + return resp.status == 200 + + +class RecordBatchStore(typing.Generic[T]): + def __init__(self, on_flush: FlushFunc[T], batch_size: int = 50) -> None: + self._store: list[T] = [] + self.batch_size = batch_size + self.on_flush = on_flush + + async def append(self, record: T): + self._store.append(record) + if len(self._store) >= self.batch_size: + success = await self.on_flush(self._store) + if success: + self._store = [] + + +class Bot(discord.Client): + last_insert: dict[int, typing.Any] = {} + track_store: RecordBatchStore[SpotifyRecord] = RecordBatchStore(submit_records, 1) + + async def on_ready(self): + logger.info(f"Logged in as {self.user}") + + async def on_member_update(self, _: discord.Member, after: discord.Member): + user_id: int = after._user.id + spotify = find_instance(after.activities, discord.Spotify) + if not spotify: + return + + spotify: discord.Spotify + if self.last_insert.get(user_id) == spotify.created_at: + return + + self.last_insert[user_id] = spotify.created_at + record = SpotifyRecord.from_activity(user_id, spotify) + await self.track_store.append(record) + + +def main(argv: typing.Sequence[str]): + if len(argv) == 0: + print("Usage: lurker.py ") + return 1 + + Bot().run(argv[0]) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main(sys.argv[1:])) diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 0000000..e72dd88 --- /dev/null +++ b/requirements.txt @@ -0,0 +1,2 @@ +discord.py-self +loguru diff --git a/requirements_dev.txt b/requirements_dev.txt new file mode 100644 index 0000000..81fc38c --- /dev/null +++ b/requirements_dev.txt @@ -0,0 +1,2 @@ +black +pre-commit diff --git a/tables.sql b/tables.sql new file mode 100644 index 0000000..9f12fda --- /dev/null +++ b/tables.sql @@ -0,0 +1,33 @@ +CREATE OR REPLACE TABLE test.spotify_albums ( + album_id BIGINT UNSIGNED auto_increment NOT NULL, + artist varchar(100) NULL, + name varchar(100) NULL, + cover_url varchar(100) NULL, + CONSTRAINT spotify_albums_PK PRIMARY KEY (album_id) +) +ENGINE=InnoDB +DEFAULT CHARSET=utf8mb4 +COLLATE=utf8mb4_general_ci; + +CREATE TABLE test.spotify_tracks ( + track_id varchar(100) NOT NULL, + album_id BIGINT UNSIGNED NOT NULL, + title varchar(100) NULL, + duration INT NULL, + CONSTRAINT spotify_tracks_PK PRIMARY KEY (track_id), + CONSTRAINT spotify_tracks_FK FOREIGN KEY (album_id) REFERENCES test.spotify_albums(album_id) +) +ENGINE=InnoDB +DEFAULT CHARSET=utf8mb4 +COLLATE=utf8mb4_general_ci; + +CREATE TABLE test.spotify_plays ( + user_id BIGINT UNSIGNED NOT NULL, + track_id varchar(100) NOT NULL, + created_at TIMESTAMP NOT NULL, + CONSTRAINT spotify_plays_PK PRIMARY KEY (user_id,created_at), + CONSTRAINT spotify_plays_FK FOREIGN KEY (track_id) REFERENCES test.spotify_tracks(track_id) +) +ENGINE=InnoDB +DEFAULT CHARSET=utf8mb4 +COLLATE=utf8mb4_general_ci;