diff --git a/README.md b/README.md index 4c6b42c..bbb374d 100644 --- a/README.md +++ b/README.md @@ -7,65 +7,23 @@ ## 数据库准备 -本服务的部分模块依赖外部数据源 `jianshu` 和 `jpep` 数据库,您需事先下载并进行导入。 +本服务的部分模块依赖外部数据源 `jianshu` 与 `jpep` 数据库,您需事先下载并进行导入。 -创建用户: +进入 `sql` 目录: -```sql -CREATE ROLE jtools LOGIN PASSWORD 'jtools'; -``` - -创建数据库: - -```sql -CREATE DATABASE jtools WITH OWNER = jtools; -CREATE DATABASE logs; -``` - -创建扩展: - -(`jianshu` 数据库) - -```sql -CREATE EXTENSION IF NOT EXISTS pg_trgm; -``` - -为用户授权: - -(`logs` 数据库) - -```sql -GRANT CREATE ON SCHEMA public TO jtools; -``` - -(`jianshu` 数据库) - -```sql -GRANT SELECT ON TABLE article_earning_ranking_records TO jtools; -GRANT SELECT ON TABLE lottery_win_records TO jtools; -GRANT SELECT ON TABLE users TO jtools; -``` - -(`jpep` 数据库) - -```sql -GRANT SELECT ON TABLE ftn_macket_records TO jtools; -GRANT SELECT ON TABLE ftn_orders TO jtools; +```shell +cd sql ``` -创建索引: +如果您需要修改数据库用户名和密码,请修改 `sql` 目录下的 `0.sql` 和每个子目录下的 `0.sql` 文件。 -(`jianshu` 数据库) +您需要一个具有创建用户和数据库权限的用户(一般是超级用户)来完成数据库准备。 -```sql -CREATE INDEX CONCURRENTLY IF NOT EXISTS idx_article_earning_ranking_records_ranking ON article_earning_ranking_records (ranking); +每个目录中的 SQL 脚本均应按照编号顺序执行。 -CREATE INDEX CONCURRENTLY IF NOT EXISTS idx_lottery_win_records_time ON lottery_win_records (time); -CREATE INDEX CONCURRENTLY IF NOT EXISTS idx_lottery_win_records_user_slug ON lottery_win_records (user_slug); -CREATE INDEX CONCURRENTLY IF NOT EXISTS idx_lottery_win_records_award_name ON lottery_win_records (award_name); +首先,执行 `sql` 目录下的脚本。 -CREATE INDEX CONCURRENTLY IF NOT EXISTS idx_users_name ON users USING gin (name gin_trgm_ops); -``` +依次切换到与 `sql` 下的子目录(数据库目录)名称相同的数据库中,先执行每个表目录中的 SQL 脚本,再执行数据库目录下的 SQL 脚本。 ## 配置 @@ -82,7 +40,7 @@ cp config.example.toml config.toml - jpep_postgres.host 填写 `postgres` - uvicorn.host 填写 `0.0.0.0` -同时,您需要填写正确的 `postgres.user` 和 `postgres.password`。 +同时,您需要填写正确的 `{db_name}_postgres.user` 和 `{db_name}_postgres.password`。 `word_split_access_key` 为具有 [NLP 服务](https://ai.aliyun.com/nlp) 使用权限的阿里云用户 Access Key,您可在 [RAM 访问控制](https://ram.console.aliyun.com) 中创建用户,并为其赋予 `AliyunNLPReadOnlyAccess` 权限。 diff --git a/backend/main.py b/backend/main.py index 64dfd43..e5bb212 100644 --- a/backend/main.py +++ b/backend/main.py @@ -3,31 +3,14 @@ from uvicorn import run as uvicorn_run -from models.debug_project_record import DebugProjectRecord -from models.jianshu.lottery_win_record import LotteryWinRecord -from models.jianshu.user import User -from models.tech_stack import TechStack -from models.tool import Tool +from models import init_db from utils.config import CONFIG -from utils.db import jianshu_pool, jpep_pool, jtools_pool from utils.log import logger logging.getLogger("httpx").setLevel(logging.CRITICAL) logging.getLogger("httpcore").setLevel(logging.CRITICAL) -async def init_db() -> None: - await jianshu_pool.prepare() - await jpep_pool.prepare() - await jtools_pool.prepare() - - await DebugProjectRecord.init() - await LotteryWinRecord.init() - await TechStack.init() - await User.init() - await Tool.init() - - if __name__ == "__main__": asyncio_run(init_db()) logger.debug("初始化数据库成功") diff --git a/backend/models/__init__.py b/backend/models/__init__.py new file mode 100644 index 0000000..8f418bb --- /dev/null +++ b/backend/models/__init__.py @@ -0,0 +1,7 @@ +from utils.db import jianshu_pool, jpep_pool, jtools_pool + + +async def init_db() -> None: + await jianshu_pool.prepare() + await jpep_pool.prepare() + await jtools_pool.prepare() diff --git a/backend/models/debug_project_record.py b/backend/models/debug_project_record.py index 68a6ea9..3d91194 100644 --- a/backend/models/debug_project_record.py +++ b/backend/models/debug_project_record.py @@ -17,24 +17,6 @@ class DebugProjectRecord(Table, frozen=True): user_slug: NonEmptyStr reward: PositiveInt - @classmethod - async def _create_table(cls) -> None: - async with jtools_pool.get_conn() as conn: - await conn.execute( - """ - CREATE TABLE IF NOT EXISTS debug_project_records ( - id SMALLSERIAL CONSTRAINT pk_debug_project_records_id PRIMARY KEY, - date DATE NOT NULL, - type TEXT NOT NULL, - module TEXT NOT NULL, - description TEXT NOT NULL, - user_name TEXT NOT NULL, - user_slug VARCHAR(12) NOT NULL, - reward SMALLINT NOT NULL - ); - """ - ) - @classmethod async def iter(cls) -> AsyncGenerator["DebugProjectRecord", None]: async with jtools_pool.get_conn() as conn: diff --git a/backend/models/jianshu/user.py b/backend/models/jianshu/user.py index f2e7f75..28df6bd 100644 --- a/backend/models/jianshu/user.py +++ b/backend/models/jianshu/user.py @@ -80,7 +80,11 @@ async def get_by_name(cls, name: str) -> Optional["User"]: async def iter_similar_names(cls, name: str, limit: int) -> AsyncGenerator[str]: async with jianshu_pool.get_conn() as conn: cursor = await conn.execute( - "SELECT name FROM users WHERE name LIKE %s LIMIT %s;", + "SELECT users.name, COUNT(*) FROM users " + "JOIN article_earning_ranking_records ON users.slug = " + "article_earning_ranking_records.author_slug " + "WHERE users.name LIKE %s GROUP BY users.name " + "ORDER BY count DESC LIMIT %s;", (f"{name}%", limit), ) diff --git a/backend/models/tech_stack.py b/backend/models/tech_stack.py index 95672ea..e71b77f 100644 --- a/backend/models/tech_stack.py +++ b/backend/models/tech_stack.py @@ -2,7 +2,7 @@ from enum import Enum from typing import Optional -from sshared.postgres import Table, create_enum +from sshared.postgres import Table from sshared.strict_struct import NonEmptyStr from utils.db import jtools_pool @@ -27,32 +27,6 @@ class TechStack(Table, frozen=True): description: NonEmptyStr url: NonEmptyStr - @classmethod - async def _create_enum(cls) -> None: - async with jtools_pool.get_conn() as conn: - await create_enum( - conn=conn, name="enum_tech_stacks_type", enum_class=TypeEnum - ) - await create_enum( - conn=conn, name="enum_tech_stacks_scope", enum_class=ScopeEnum - ) - - @classmethod - async def _create_table(cls) -> None: - async with jtools_pool.get_conn() as conn: - await conn.execute( - """ - CREATE TABLE IF NOT EXISTS tech_stacks ( - name TEXT NOT NULL CONSTRAINT pk_tech_stacks_name PRIMARY KEY, - type enum_tech_stacks_type NOT NULL, - scope enum_tech_stacks_scope NOT NULL, - is_self_developed BOOLEAN NOT NULL, - description TEXT NOT NULL, - url TEXT NOT NULL - ); - """ - ) - @classmethod async def iter( cls, scope: Optional[ScopeEnum] = None diff --git a/backend/models/tool.py b/backend/models/tool.py index 4f211d0..6da576a 100644 --- a/backend/models/tool.py +++ b/backend/models/tool.py @@ -2,7 +2,7 @@ from typing import Optional from psycopg.types.json import Jsonb -from sshared.postgres import Table, create_enum +from sshared.postgres import Table from sshared.strict_struct import NonEmptyStr from utils.db import jtools_pool @@ -26,32 +26,6 @@ class Tool(Table, frozen=True): data_count_table: Optional[NonEmptyStr] data_source: Optional[dict[str, str]] - @classmethod - async def _create_enum(cls) -> None: - async with jtools_pool.get_conn() as conn: - await create_enum( - conn=conn, name="enum_tools_status", enum_class=StatusEnum - ) - - @classmethod - async def _create_table(cls) -> None: - async with jtools_pool.get_conn() as conn: - await conn.execute( - """ - CREATE TABLE IF NOT EXISTS tools ( - slug TEXT CONSTRAINT pk_tools_slug PRIMARY KEY, - status enum_tools_status NOT NULL, - status_description TEXT, - data_update_freq TEXT NOT NULL, - last_update_time_table TEXT, - last_update_time_order_by TEXT, - last_update_time_target_field TEXT, - data_count_table TEXT, - data_source JSONB - ); - """ - ) - @classmethod async def init(cls) -> None: await super().init() diff --git a/backend/pyproject.toml b/backend/pyproject.toml index 3e1c880..3c201d0 100644 --- a/backend/pyproject.toml +++ b/backend/pyproject.toml @@ -1,6 +1,6 @@ [project] name = "jtools" -version = "3.18.0" +version = "3.19.0" description = "探索未知" license = {file = "LICENSE"} authors = [ diff --git a/backend/requirements-dev.txt b/backend/requirements-dev.txt index bb9ce68..169814a 100644 --- a/backend/requirements-dev.txt +++ b/backend/requirements-dev.txt @@ -4,7 +4,7 @@ click==8.1.7 colorama==0.4.6 ; platform_system == 'Windows' dnspython==2.7.0 exceptiongroup==1.2.2 ; python_full_version < '3.11' -faker==30.8.2 +faker==32.1.0 h11==0.14.0 h2==4.1.0 hpack==4.0.0 @@ -37,7 +37,7 @@ six==1.16.0 sniffio==1.3.1 sshared==0.18.0 sspeedup==0.25.1 -tomli==2.0.2 +tomli==2.1.0 typing-extensions==4.12.2 tzdata==2024.2 ; sys_platform == 'win32' uvicorn==0.32.0 diff --git a/backend/requirements.txt b/backend/requirements.txt index b6207a4..297de66 100644 --- a/backend/requirements.txt +++ b/backend/requirements.txt @@ -4,7 +4,7 @@ click==8.1.7 colorama==0.4.6 ; platform_system == 'Windows' dnspython==2.7.0 exceptiongroup==1.2.2 ; python_full_version < '3.11' -faker==30.8.2 +faker==32.1.0 h11==0.14.0 h2==4.1.0 hpack==4.0.0 @@ -34,7 +34,7 @@ six==1.16.0 sniffio==1.3.1 sshared==0.18.0 sspeedup==0.25.1 -tomli==2.0.2 +tomli==2.1.0 typing-extensions==4.12.2 tzdata==2024.2 ; sys_platform == 'win32' uvicorn==0.32.0 diff --git a/backend/sql/0.sql b/backend/sql/0.sql new file mode 100644 index 0000000..1a2af25 --- /dev/null +++ b/backend/sql/0.sql @@ -0,0 +1,10 @@ +-- date: 2024-11-13 +-- description: 初始化 + +CREATE ROLE jtools LOGIN PASSWORD 'jtools'; + +CREATE DATABASE jtools WITH OWNER = jtools; + +GRANT CONNECT ON DATABASE jtools TO jtools; +GRANT CONNECT ON DATABASE jianshu TO jtools; +GRANT CONNECT ON DATABASE jpep TO jtools; \ No newline at end of file diff --git a/backend/sql/jianshu/0.sql b/backend/sql/jianshu/0.sql new file mode 100644 index 0000000..90a46ad --- /dev/null +++ b/backend/sql/jianshu/0.sql @@ -0,0 +1,14 @@ +-- date: 2024-11-13 +-- description: 初始化 + +GRANT SELECT ON TABLE article_earning_ranking_records TO jtools; +GRANT SELECT ON TABLE lottery_win_records TO jtools; +GRANT SELECT ON TABLE users TO jtools; + +CREATE EXTENSION pg_trgm; + +CREATE INDEX CONCURRENTLY idx_article_earning_ranking_records_ranking ON article_earning_ranking_records (ranking); +CREATE INDEX CONCURRENTLY idx_lottery_win_records_time ON lottery_win_records (time); +CREATE INDEX CONCURRENTLY idx_lottery_win_records_user_slug ON lottery_win_records (user_slug); +CREATE INDEX CONCURRENTLY idx_lottery_win_records_award_name ON lottery_win_records (award_name); +CREATE INDEX CONCURRENTLY idx_users_name ON users USING gin (name gin_trgm_ops); diff --git a/backend/sql/jianshu/1.sql b/backend/sql/jianshu/1.sql new file mode 100644 index 0000000..9dc9c27 --- /dev/null +++ b/backend/sql/jianshu/1.sql @@ -0,0 +1,4 @@ +-- date: 2024-11-14 +-- description: 添加 article_earning_ranking_records author_slug 索引 + +CREATE INDEX CONCURRENTLY idx_article_earning_ranking_records_author_slug ON article_earning_ranking_records (author_slug); \ No newline at end of file diff --git a/backend/sql/jpep/0.sql b/backend/sql/jpep/0.sql new file mode 100644 index 0000000..d2b4df9 --- /dev/null +++ b/backend/sql/jpep/0.sql @@ -0,0 +1,5 @@ +-- date: 2024-11-13 +-- description: 初始化 + +GRANT SELECT ON TABLE ftn_macket_records TO jtools; +GRANT SELECT ON TABLE ftn_orders TO jtools; \ No newline at end of file diff --git a/backend/sql/jtools/0.sql b/backend/sql/jtools/0.sql new file mode 100644 index 0000000..ba11f12 --- /dev/null +++ b/backend/sql/jtools/0.sql @@ -0,0 +1,6 @@ +-- date: 2024-11-13 +-- description: 初始化 + +GRANT SELECT ON TABLE debug_project_records TO jtools; +GRANT SELECT ON TABLE tech_stacks TO jtools; +GRANT SELECT, INSERT ON TABLE tools TO jtools; \ No newline at end of file diff --git a/backend/sql/jtools/debug_project_records/0.sql b/backend/sql/jtools/debug_project_records/0.sql new file mode 100644 index 0000000..496e521 --- /dev/null +++ b/backend/sql/jtools/debug_project_records/0.sql @@ -0,0 +1,13 @@ +-- date: 2024-11-13 +-- description: 初始化 + +CREATE TABLE debug_project_records ( + id SMALLSERIAL CONSTRAINT pk_debug_project_records_id PRIMARY KEY, + date DATE NOT NULL, + type TEXT NOT NULL, + module TEXT NOT NULL, + description TEXT NOT NULL, + user_name TEXT NOT NULL, + user_slug VARCHAR(12) NOT NULL, + reward SMALLINT NOT NULL +); \ No newline at end of file diff --git a/backend/sql/jtools/tech_stacks/0.sql b/backend/sql/jtools/tech_stacks/0.sql new file mode 100644 index 0000000..adac2f5 --- /dev/null +++ b/backend/sql/jtools/tech_stacks/0.sql @@ -0,0 +1,14 @@ +-- date: 2024-11-13 +-- description: 初始化 + +CREATE TYPE enum_tech_stacks_type AS ENUM ('LIBRARY', 'EXTERNAL_SERVICE'); +CREATE TYPE enum_tech_stacks_scope AS ENUM ('FRONTEND', 'BACKEND', 'TOOLCHAIN'); + +CREATE TABLE tech_stacks ( + name TEXT NOT NULL CONSTRAINT pk_tech_stacks_name PRIMARY KEY, + type enum_tech_stacks_type NOT NULL, + scope enum_tech_stacks_scope NOT NULL, + is_self_developed BOOLEAN NOT NULL, + description TEXT NOT NULL, + url TEXT NOT NULL +); \ No newline at end of file diff --git a/backend/sql/jtools/tools/0.sql b/backend/sql/jtools/tools/0.sql new file mode 100644 index 0000000..2188817 --- /dev/null +++ b/backend/sql/jtools/tools/0.sql @@ -0,0 +1,16 @@ +-- date: 2024-11-13 +-- description: 初始化 + +CREATE TYPE enum_tools_status AS ENUM ('NORMAL', 'DOWNGRADED', 'UNAVAILABLE'); + +CREATE TABLE tools ( + slug TEXT CONSTRAINT pk_tools_slug PRIMARY KEY, + status enum_tools_status NOT NULL, + status_description TEXT, + data_update_freq TEXT NOT NULL, + last_update_time_table TEXT, + last_update_time_order_by TEXT, + last_update_time_target_field TEXT, + data_count_table TEXT, + data_source JSONB +); \ No newline at end of file diff --git a/backend/uv.lock b/backend/uv.lock index a7ef11a..3ea03de 100644 --- a/backend/uv.lock +++ b/backend/uv.lock @@ -66,15 +66,15 @@ wheels = [ [[package]] name = "faker" -version = "30.8.2" +version = "32.1.0" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "python-dateutil" }, { name = "typing-extensions" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/c1/df/7574c0d13f0bbab725e52bec4b00783aaa14163fe9093dde11a928a4c638/faker-30.8.2.tar.gz", hash = "sha256:aa31b52cdae3673d6a78b4857c7bcdc0e98f201a5cb77d7827fa9e6b5876da94", size = 1808329 } +sdist = { url = "https://files.pythonhosted.org/packages/1c/2a/dd2c8f55d69013d0eee30ec4c998250fb7da957f5fe860ed077b3df1725b/faker-32.1.0.tar.gz", hash = "sha256:aac536ba04e6b7beb2332c67df78485fc29c1880ff723beac6d1efd45e2f10f5", size = 1850193 } wheels = [ - { url = "https://files.pythonhosted.org/packages/64/82/f7d0c0a4ab512fd1572a315eec903d50a578c75d5aa894cf3f5cc04025e5/Faker-30.8.2-py3-none-any.whl", hash = "sha256:4a82b2908cd19f3bba1a4da2060cc4eb18a40410ccdf9350d071d79dc92fe3ce", size = 1846458 }, + { url = "https://files.pythonhosted.org/packages/7e/fa/4a82dea32d6262a96e6841cdd4a45c11ac09eecdff018e745565410ac70e/Faker-32.1.0-py3-none-any.whl", hash = "sha256:c77522577863c264bdc9dad3a2a750ad3f7ee43ff8185072e482992288898814", size = 1889123 }, ] [[package]] @@ -231,7 +231,7 @@ wheels = [ [[package]] name = "jtools" -version = "3.18.0" +version = "3.19.0" source = { virtual = "." } dependencies = [ { name = "httptools" }, @@ -805,11 +805,11 @@ api-litestar = [ [[package]] name = "tomli" -version = "2.0.2" +version = "2.1.0" source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/35/b9/de2a5c0144d7d75a57ff355c0c24054f965b2dc3036456ae03a51ea6264b/tomli-2.0.2.tar.gz", hash = "sha256:d46d457a85337051c36524bc5349dd91b1877838e2979ac5ced3e710ed8a60ed", size = 16096 } +sdist = { url = "https://files.pythonhosted.org/packages/1e/e4/1b6cbcc82d8832dd0ce34767d5c560df8a3547ad8cbc427f34601415930a/tomli-2.1.0.tar.gz", hash = "sha256:3f646cae2aec94e17d04973e4249548320197cfabdf130015d023de4b74d8ab8", size = 16622 } wheels = [ - { url = "https://files.pythonhosted.org/packages/cf/db/ce8eda256fa131af12e0a76d481711abe4681b6923c27efb9a255c9e4594/tomli-2.0.2-py3-none-any.whl", hash = "sha256:2ebe24485c53d303f690b0ec092806a085f07af5a5aa1464f3931eec36caaa38", size = 13237 }, + { url = "https://files.pythonhosted.org/packages/de/f7/4da0ffe1892122c9ea096c57f64c2753ae5dd3ce85488802d11b0992cc6d/tomli-2.1.0-py3-none-any.whl", hash = "sha256:a5c57c3d1c56f5ccdf89f6523458f60ef716e210fc47c4cfb188c5ba473e0391", size = 13750 }, ] [[package]] diff --git a/backend/version.py b/backend/version.py index ed2ab66..9f92b45 100644 --- a/backend/version.py +++ b/backend/version.py @@ -1 +1 @@ -VERSION = "3.18.0" +VERSION = "3.19.0" diff --git a/docker-compose.yml b/docker-compose.yml index e4fa31c..e2015d4 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -5,7 +5,7 @@ networks: services: frontend: - image: jtools-frontend:3.18.0 + image: jtools-frontend:3.19.0 container_name: jtools-frontend build: dockerfile: Dockerfile.frontend @@ -22,7 +22,7 @@ services: delay: 5s max_attempts: 3 backend: - image: jtools-backend:3.18.0 + image: jtools-backend:3.19.0 container_name: jtools-backend build: dockerfile: Dockerfile.backend diff --git a/frontend/bun.lockb b/frontend/bun.lockb index 83b136b..1c27c53 100644 Binary files a/frontend/bun.lockb and b/frontend/bun.lockb differ diff --git a/frontend/package.json b/frontend/package.json index 897d05d..db107c0 100644 --- a/frontend/package.json +++ b/frontend/package.json @@ -1,7 +1,7 @@ { "name": "jtools", "private": true, - "version": "3.18.0", + "version": "3.19.0", "type": "module", "scripts": { "dev": "vite", @@ -14,7 +14,7 @@ "biome-ci": "biome ci ." }, "dependencies": { - "@mantine/hooks": "^7.13.5", + "@mantine/hooks": "^7.14.0", "@preact/preset-vite": "^2.9.1", "@preact/signals": "^1.3.0", "@sscreator/ui": "^0.27.0", @@ -29,7 +29,7 @@ "resize-observer": "^1.0.4", "swr": "^2.2.5", "unocss": "=0.61.6", - "vite": "^5.4.10", + "vite": "^5.4.11", "vite-plugin-compression": "^0.5.1", "vite-plugin-pwa": "^0.20.5", "wouter-preact": "^3.3.5"