diff --git a/migrate/README.md b/migrate/README.md index a2e3a42..93ae243 100644 --- a/migrate/README.md +++ b/migrate/README.md @@ -1,2 +1,44 @@ # BirdXplorer Migrations + +## Alembic によるマイグレーション + +### 前提条件 + +- `birdxplorer_common` のインストールが完了していること +- `birdxplorer_migration` のインストールが完了していること +- `.env` ファイルに `BX_STORAGE_SETTINGS__PASSWORD` が設定されていること + - DB のデフォルトホスト名 `db` が解決できない場合は `BX_STORAGE_SETTINGS__HOST` も設定すること + +### 現在のマイグレーションを適用する + +データベースのスキーマを最新の状態にします。 + +`alembic.ini` があるディレクトリの場合: + +```bash +alembic upgrade head +``` + +`alembic.ini` が別の場所にある場合: + +```bash +alembic upgrade head -c /path/to/alembic.ini +``` + +### 新しいマイグレーションを作成する + +データベースのスキーマを変更した場合、以下の手順で新しいマイグレーションを作成します。 + +`alembic.ini` があるディレクトリの場合: + +```bash +alembic revision --autogenerate -m "マイグレーションの説明" +``` + +`alembic.ini` が別の場所にある場合: + +```bash +alembic revision --autogenerate -m "マイグレーションの説明" -c /path/to/alembic.ini +``` + diff --git a/migrate/alembic.ini b/migrate/alembic.ini new file mode 100644 index 0000000..3c064a5 --- /dev/null +++ b/migrate/alembic.ini @@ -0,0 +1,36 @@ +[alembic] +script_location = %(here)s/migration + +[loggers] +keys = root,sqlalchemy,alembic + +[handlers] +keys = console + +[formatters] +keys = generic + +[logger_root] +level = WARNING +handlers = console +qualname = + +[logger_sqlalchemy] +level = WARNING +handlers = +qualname = sqlalchemy.engine + +[logger_alembic] +level = INFO +handlers = +qualname = alembic + +[handler_console] +class = StreamHandler +args = (sys.stderr,) +level = NOTSET +formatter = generic + +[formatter_generic] +format = %(levelname)-5.5s [%(name)s] %(message)s +datefmt = %H:%M:%S diff --git a/migrate/birdxplorer_migration/__init__.py b/migrate/birdxplorer_migration/__init__.py index f102a9c..3dc1f76 100644 --- a/migrate/birdxplorer_migration/__init__.py +++ b/migrate/birdxplorer_migration/__init__.py @@ -1 +1 @@ -__version__ = "0.0.1" +__version__ = "0.1.0" diff --git a/migrate/migration/env.py b/migrate/migration/env.py new file mode 100644 index 0000000..3107f51 --- /dev/null +++ b/migrate/migration/env.py @@ -0,0 +1,80 @@ +import os +from logging.config import fileConfig + +from alembic import context +from dotenv import load_dotenv + +from birdxplorer_common.settings import GlobalSettings +from birdxplorer_common.storage import Base, gen_storage + +# this is the Alembic Config object, which provides +# access to the values within the .ini file in use. +config = context.config + +# Interpret the config file for Python logging. +# This line sets up loggers basically. +if config.config_file_name is not None: + fileConfig(config.config_file_name) + +work_dir = os.path.dirname(os.path.abspath(__file__)) +env_file = os.path.join(os.path.dirname(os.path.dirname(work_dir)), ".env") +load_dotenv(env_file) +bx_settings = GlobalSettings() + +# add your model's MetaData object here +# for 'autogenerate' support +# from myapp import mymodel +# target_metadata = mymodel.Base.metadata +target_metadata = Base.metadata + +# other values from the config, defined by the needs of env.py, +# can be acquired: +# my_important_option = config.get_main_option("my_important_option") +# ... etc. + + +def run_migrations_offline() -> None: + """Run migrations in 'offline' mode. + + This configures the context with just a URL + and not an Engine, though an Engine is acceptable + here as well. By skipping the Engine creation + we don't even need a DBAPI to be available. + + Calls to context.execute() here emit the given string to the + script output. + + """ + url = bx_settings.storage_settings.sqlalchemy_database_url + context.configure( + url=url, + target_metadata=target_metadata, + literal_binds=True, + dialect_opts={"paramstyle": "named"}, + ) + + with context.begin_transaction(): + context.run_migrations() + + +def run_migrations_online() -> None: + """Run migrations in 'online' mode. + + In this scenario we need to create an Engine + and associate a connection with the context. + + """ + storage = gen_storage(settings=bx_settings) + connectable = storage.engine + + with connectable.connect() as connection: + context.configure(connection=connection, target_metadata=target_metadata) + + with context.begin_transaction(): + context.run_migrations() + + +if context.is_offline_mode(): + run_migrations_offline() +else: + run_migrations_online() diff --git a/migrate/migration/script.py.mako b/migrate/migration/script.py.mako new file mode 100644 index 0000000..fbc4b07 --- /dev/null +++ b/migrate/migration/script.py.mako @@ -0,0 +1,26 @@ +"""${message} + +Revision ID: ${up_revision} +Revises: ${down_revision | comma,n} +Create Date: ${create_date} + +""" +from typing import Sequence, Union + +from alembic import op +import sqlalchemy as sa +${imports if imports else ""} + +# revision identifiers, used by Alembic. +revision: str = ${repr(up_revision)} +down_revision: Union[str, None] = ${repr(down_revision)} +branch_labels: Union[str, Sequence[str], None] = ${repr(branch_labels)} +depends_on: Union[str, Sequence[str], None] = ${repr(depends_on)} + + +def upgrade() -> None: + ${upgrades if upgrades else "pass"} + + +def downgrade() -> None: + ${downgrades if downgrades else "pass"} diff --git a/migrate/migration/versions/1f33aeb4bebf_notes_and_posts.py b/migrate/migration/versions/1f33aeb4bebf_notes_and_posts.py new file mode 100644 index 0000000..8248a89 --- /dev/null +++ b/migrate/migration/versions/1f33aeb4bebf_notes_and_posts.py @@ -0,0 +1,271 @@ +"""notes and posts + +Revision ID: 1f33aeb4bebf +Revises: +Create Date: 2025-02-22 16:16:44.777479 + +""" + +from typing import Sequence, Union + +import sqlalchemy as sa +from alembic import op + +# revision identifiers, used by Alembic. +revision: str = "1f33aeb4bebf" +down_revision: Union[str, None] = None +branch_labels: Union[str, Sequence[str], None] = None +depends_on: Union[str, Sequence[str], None] = None + + +def upgrade() -> None: + # ### commands auto generated by Alembic - please adjust! ### + op.create_table( + "links", + sa.Column("link_id", sa.Uuid(), nullable=False), + sa.Column("url", sa.String(), nullable=False), + sa.PrimaryKeyConstraint("link_id"), + ) + op.create_index(op.f("ix_links_url"), "links", ["url"], unique=False) + op.create_table( + "media", + sa.Column("media_key", sa.String(), nullable=False), + sa.Column("type", sa.Enum("photo", "video", "animated_gif", native_enum=False), nullable=False), + sa.Column("url", sa.String(), nullable=False), + sa.Column("width", sa.DECIMAL(), nullable=False), + sa.Column("height", sa.DECIMAL(), nullable=False), + sa.PrimaryKeyConstraint("media_key"), + ) + op.create_table( + "notes", + sa.Column("note_id", sa.String(), nullable=False), + sa.Column("post_id", sa.String(), nullable=False), + sa.Column("language", sa.String(), nullable=False), + sa.Column("summary", sa.String(), nullable=False), + sa.Column("current_status", sa.String(), nullable=True), + sa.Column("created_at", sa.DECIMAL(), nullable=False), + sa.PrimaryKeyConstraint("note_id"), + ) + op.create_table( + "row_users", + sa.Column("user_id", sa.String(), nullable=False), + sa.Column("name", sa.String(), nullable=False), + sa.Column("user_name", sa.String(), nullable=False), + sa.Column("description", sa.String(), nullable=False), + sa.Column("profile_image_url", sa.String(), nullable=False), + sa.Column("followers_count", sa.DECIMAL(), nullable=False), + sa.Column("following_count", sa.DECIMAL(), nullable=False), + sa.Column("tweet_count", sa.DECIMAL(), nullable=False), + sa.Column("verified", sa.Boolean(), nullable=False), + sa.Column("verified_type", sa.String(), nullable=False), + sa.Column("location", sa.String(), nullable=False), + sa.Column("url", sa.String(), nullable=False), + sa.PrimaryKeyConstraint("user_id"), + ) + op.create_table( + "topics", + sa.Column("topic_id", sa.Integer(), nullable=False), + sa.Column("label", sa.JSON(), nullable=False), + sa.PrimaryKeyConstraint("topic_id"), + ) + op.create_table( + "x_users", + sa.Column("user_id", sa.String(), nullable=False), + sa.Column("name", sa.String(), nullable=False), + sa.Column("profile_image", sa.String(), nullable=False), + sa.Column("followers_count", sa.DECIMAL(), nullable=False), + sa.Column("following_count", sa.DECIMAL(), nullable=False), + sa.PrimaryKeyConstraint("user_id"), + ) + op.create_table( + "note_topic", + sa.Column("note_id", sa.String(), nullable=False), + sa.Column("topic_id", sa.Integer(), nullable=False), + sa.ForeignKeyConstraint( + ["note_id"], + ["notes.note_id"], + ), + sa.ForeignKeyConstraint( + ["topic_id"], + ["topics.topic_id"], + ), + sa.PrimaryKeyConstraint("note_id", "topic_id"), + ) + op.create_table( + "posts", + sa.Column("post_id", sa.String(), nullable=False), + sa.Column("user_id", sa.String(), nullable=False), + sa.Column("text", sa.String(), nullable=False), + sa.Column("created_at", sa.DECIMAL(), nullable=False), + sa.Column("like_count", sa.DECIMAL(), nullable=False), + sa.Column("repost_count", sa.DECIMAL(), nullable=False), + sa.Column("impression_count", sa.DECIMAL(), nullable=False), + sa.ForeignKeyConstraint( + ["user_id"], + ["x_users.user_id"], + ), + sa.PrimaryKeyConstraint("post_id"), + ) + op.create_table( + "row_posts", + sa.Column("post_id", sa.String(), nullable=False), + sa.Column("author_id", sa.String(), nullable=False), + sa.Column("text", sa.String(), nullable=False), + sa.Column("created_at", sa.DECIMAL(), nullable=False), + sa.Column("like_count", sa.DECIMAL(), nullable=False), + sa.Column("repost_count", sa.DECIMAL(), nullable=False), + sa.Column("bookmark_count", sa.DECIMAL(), nullable=False), + sa.Column("impression_count", sa.DECIMAL(), nullable=False), + sa.Column("quote_count", sa.DECIMAL(), nullable=False), + sa.Column("reply_count", sa.DECIMAL(), nullable=False), + sa.Column("lang", sa.String(), nullable=False), + sa.ForeignKeyConstraint( + ["author_id"], + ["row_users.user_id"], + ), + sa.PrimaryKeyConstraint("post_id"), + ) + op.create_table( + "post_link", + sa.Column("post_id", sa.String(), nullable=False), + sa.Column("link_id", sa.Uuid(), nullable=False), + sa.ForeignKeyConstraint( + ["link_id"], + ["links.link_id"], + ), + sa.ForeignKeyConstraint( + ["post_id"], + ["posts.post_id"], + ), + sa.PrimaryKeyConstraint("post_id", "link_id"), + ) + op.create_table( + "post_media", + sa.Column("post_id", sa.String(), nullable=False), + sa.Column("media_key", sa.String(), nullable=False), + sa.ForeignKeyConstraint( + ["media_key"], + ["media.media_key"], + ), + sa.ForeignKeyConstraint( + ["post_id"], + ["posts.post_id"], + ), + sa.PrimaryKeyConstraint("post_id", "media_key"), + ) + op.create_table( + "row_notes", + sa.Column("note_id", sa.String(), nullable=False), + sa.Column("note_author_participant_id", sa.String(), nullable=False), + sa.Column("created_at_millis", sa.DECIMAL(), nullable=False), + sa.Column("tweet_id", sa.String(), nullable=False), + sa.Column("believable", sa.CHAR(), nullable=False), + sa.Column("misleading_other", sa.CHAR(), nullable=False), + sa.Column("misleading_factual_error", sa.CHAR(), nullable=False), + sa.Column("misleading_manipulated_media", sa.CHAR(), nullable=False), + sa.Column("misleading_outdated_information", sa.CHAR(), nullable=False), + sa.Column("misleading_missing_important_context", sa.CHAR(), nullable=False), + sa.Column("misleading_unverified_claim_as_fact", sa.CHAR(), nullable=False), + sa.Column("misleading_satire", sa.CHAR(), nullable=False), + sa.Column("not_misleading_other", sa.CHAR(), nullable=False), + sa.Column("not_misleading_factually_correct", sa.CHAR(), nullable=False), + sa.Column("not_misleading_outdated_but_not_when_written", sa.CHAR(), nullable=False), + sa.Column("not_misleading_clearly_satire", sa.CHAR(), nullable=False), + sa.Column("not_misleading_personal_opinion", sa.CHAR(), nullable=False), + sa.Column("trustworthy_sources", sa.CHAR(), nullable=False), + sa.Column("is_media_note", sa.CHAR(), nullable=False), + sa.Column( + "classification", + sa.Enum("not_misleading", "misinformed_or_potentially_misleading", name="notesclassification"), + nullable=False, + ), + sa.Column("harmful", sa.Enum("little_harm", "considerable_harm", "empty", name="notesharmful"), nullable=False), + sa.Column("validation_difficulty", sa.String(), nullable=False), + sa.Column("summary", sa.String(), nullable=False), + sa.Column("row_post_id", sa.String(), nullable=True), + sa.ForeignKeyConstraint( + ["row_post_id"], + ["row_posts.post_id"], + ), + sa.PrimaryKeyConstraint("note_id"), + ) + op.create_table( + "row_post_embed_urls", + sa.Column("post_id", sa.String(), nullable=False), + sa.Column("url", sa.String(), nullable=False), + sa.Column("expanded_url", sa.String(), nullable=False), + sa.Column("unwound_url", sa.String(), nullable=False), + sa.ForeignKeyConstraint( + ["post_id"], + ["row_posts.post_id"], + ), + sa.PrimaryKeyConstraint("post_id", "url"), + ) + op.create_table( + "row_post_media", + sa.Column("media_key", sa.String(), nullable=False), + sa.Column("url", sa.String(), nullable=False), + sa.Column("type", sa.Enum("photo", "video", "animated_gif", native_enum=False), nullable=False), + sa.Column("width", sa.DECIMAL(), nullable=False), + sa.Column("height", sa.DECIMAL(), nullable=False), + sa.Column("post_id", sa.String(), nullable=False), + sa.ForeignKeyConstraint( + ["post_id"], + ["row_posts.post_id"], + ), + sa.PrimaryKeyConstraint("media_key"), + sa.UniqueConstraint("media_key"), + ) + op.create_table( + "row_note_status", + sa.Column("note_id", sa.String(), nullable=False), + sa.Column("note_author_participant_id", sa.String(), nullable=False), + sa.Column("created_at_millis", sa.DECIMAL(), nullable=False), + sa.Column("timestamp_millis_of_first_non_n_m_r_status", sa.DECIMAL(), nullable=True), + sa.Column("first_non_n_m_r_status", sa.String(), nullable=True), + sa.Column("timestamp_millis_of_current_status", sa.DECIMAL(), nullable=True), + sa.Column("current_status", sa.String(), nullable=True), + sa.Column("timestamp_millis_of_latest_non_n_m_r_status", sa.DECIMAL(), nullable=True), + sa.Column("most_recent_non_n_m_r_status", sa.String(), nullable=True), + sa.Column("timestamp_millis_of_status_lock", sa.DECIMAL(), nullable=True), + sa.Column("locked_status", sa.String(), nullable=True), + sa.Column("timestamp_millis_of_retro_lock", sa.DECIMAL(), nullable=True), + sa.Column("current_core_status", sa.String(), nullable=True), + sa.Column("current_expansion_status", sa.String(), nullable=True), + sa.Column("current_group_status", sa.String(), nullable=True), + sa.Column("current_decided_by", sa.String(), nullable=True), + sa.Column("current_modeling_group", sa.Integer(), nullable=True), + sa.Column("timestamp_millis_of_most_recent_status_change", sa.DECIMAL(), nullable=True), + sa.Column("timestamp_millis_of_nmr_due_to_min_stable_crh_time", sa.DECIMAL(), nullable=True), + sa.Column("current_multi_group_status", sa.String(), nullable=True), + sa.Column("current_modeling_multi_group", sa.Integer(), nullable=True), + sa.Column("timestamp_minute_of_final_scoring_output", sa.DECIMAL(), nullable=True), + sa.Column("timestamp_millis_of_first_nmr_due_to_min_stable_crh_time", sa.DECIMAL(), nullable=True), + sa.ForeignKeyConstraint( + ["note_id"], + ["row_notes.note_id"], + ), + sa.PrimaryKeyConstraint("note_id"), + ) + # ### end Alembic commands ### + + +def downgrade() -> None: + # ### commands auto generated by Alembic - please adjust! ### + op.drop_table("row_note_status") + op.drop_table("row_post_media") + op.drop_table("row_post_embed_urls") + op.drop_table("row_notes") + op.drop_table("post_media") + op.drop_table("post_link") + op.drop_table("row_posts") + op.drop_table("posts") + op.drop_table("note_topic") + op.drop_table("x_users") + op.drop_table("topics") + op.drop_table("row_users") + op.drop_table("notes") + op.drop_table("media") + op.drop_index(op.f("ix_links_url"), table_name="links") + op.drop_table("links") + # ### end Alembic commands ### diff --git a/migrate/pyproject.toml b/migrate/pyproject.toml index f11c627..7fdea46 100644 --- a/migrate/pyproject.toml +++ b/migrate/pyproject.toml @@ -25,9 +25,9 @@ classifiers = [ ] dependencies = [ - "birdxplorer_common @ git+https://github.com/codeforjapan/BirdXplorer.git@feature/issue-53-divide-python-packages#subdirectory=common", "sqlalchemy", "python-dotenv", + "alembic", ] [project.urls] @@ -59,7 +59,8 @@ dev=[ "httpx", ] prod=[ - "psycopg2" + "psycopg2", + "birdxplorer_common @ git+https://github.com/codeforjapan/BirdXplorer.git@main#subdirectory=common", ] [tool.black] @@ -94,11 +95,10 @@ legacy_tox_ini = """ VIRTUALENV_PIP = 24.0 deps = -e .[dev] + -e ../common commands = - black birdxplorer_api tests - isort birdxplorer_api tests - pytest - pflake8 birdxplorer_api/ tests/ - mypy birdxplorer_api --strict - mypy tests --strict + black migration + isort migration + pflake8 migration + mypy migration --strict """