Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

db: Make 'text' field in 'comments' table NOT NULL and handling data migration #1019

Merged
merged 1 commit into from
May 5, 2024
Merged

db: Make 'text' field in 'comments' table NOT NULL and handling data migration #1019

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 2 additions & 0 deletions CHANGES.rst
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -45,6 +45,7 @@ Bugfixes & Improvements
- Fix newline character handling in data-isso-* i18n strings (`#992`_, pkvach)
- Add link logging for management of new comments in Stdout (`#1016`_, pkvach)
- Change logging to include datetime and loglevel (`#1023`_, ix5)
- Make 'text' field in 'comments' table NOT NULL and handling data migration (`#1019`_, pkvach)

.. _#951: https://github.com/posativ/isso/pull/951
.. _#967: https://github.com/posativ/isso/pull/967
Expand All @@ -56,6 +57,7 @@ Bugfixes & Improvements
.. _#992: https://github.com/isso-comments/isso/pull/992
.. _#1016: https://github.com/isso-comments/isso/pull/1016
.. _#1023: https://github.com/isso-comments/isso/pull/1023
.. _#1019: https://github.com/isso-comments/isso/pull/1019

0.13.1.dev0 (2023-02-05)
------------------------
Expand Down
116 changes: 99 additions & 17 deletions isso/db/__init__.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -22,7 +22,7 @@ class SQLite3:
a trigger for automated orphan removal.
"""

MAX_VERSION = 3
MAX_VERSION = 5

def __init__(self, path, conf):

Expand Down Expand Up @@ -68,7 +68,7 @@ def migrate(self, to):
if self.version >= to:
return

logger.info("migrate database from version %i to %i", self.version, to)
logger.info("Migrating database from version %i to %i", self.version, to)

# re-initialize voters blob due a bug in the bloomfilter signature
# which added older commenter's ip addresses to the current voters blob
Expand All @@ -78,20 +78,36 @@ def migrate(self, to):
bf = memoryview(Bloomfilter(iterable=["127.0.0.0"]).array)

with sqlite3.connect(self.path) as con:
con.execute('UPDATE comments SET voters=?', (bf, ))
con.execute('PRAGMA user_version = 1')
logger.info("%i rows changed", con.total_changes)
con.execute("BEGIN TRANSACTION")
try:
con.execute('UPDATE comments SET voters=?', (bf, ))
con.execute('PRAGMA user_version = 1')
con.execute("COMMIT")
logger.info("Migrating DB version 0 to 1 by re-initializing voters blob, %i rows changed",
con.total_changes)
except sqlite3.Error as e:
con.execute("ROLLBACK")
logger.error("Migrating DB version 0 to 1 failed: %s", e)
raise RuntimeError("Migrating DB version 0 to 1 failed: %s" % e)

# move [general] session-key to database
if self.version == 1:

with sqlite3.connect(self.path) as con:
if self.conf.has_option("general", "session-key"):
con.execute('UPDATE preferences SET value=? WHERE key=?', (
self.conf.get("general", "session-key"), "session-key"))

con.execute('PRAGMA user_version = 2')
logger.info("%i rows changed", con.total_changes)
con.execute("BEGIN TRANSACTION")
try:
if self.conf.has_option("general", "session-key"):
con.execute('UPDATE preferences SET value=? WHERE key=?', (
self.conf.get("general", "session-key"), "session-key"))

con.execute('PRAGMA user_version = 2')
con.execute("COMMIT")
logger.info("Migrating DB version 1 to 2 by moving session-key to database, %i rows changed",
con.total_changes)
except sqlite3.Error as e:
con.execute("ROLLBACK")
logger.error("Migrating DB version 1 to 2 failed: %s", e)
raise RuntimeError("Migrating DB version 1 to 2 failed: %s" % e)

# limit max. nesting level to 1
if self.version == 2:
Expand All @@ -114,10 +130,76 @@ def first(rv):
ids.extend(rv)
flattened[id].update(set(rv))

for id in flattened.keys():
for n in flattened[id]:
con.execute(
"UPDATE comments SET parent=? WHERE id=?", (id, n))
con.execute("BEGIN TRANSACTION")
try:
for id in flattened.keys():
for n in flattened[id]:
con.execute(
"UPDATE comments SET parent=? WHERE id=?", (id, n))

con.execute('PRAGMA user_version = 3')
con.execute("COMMIT")
logger.info("Migrating DB version 2 to 3 by limiting nesting level to 1, %i rows changed",
con.total_changes)
except sqlite3.Error as e:
con.execute("ROLLBACK")
logger.error("Migrating DB version 2 to 3 failed: %s", e)
raise RuntimeError("Migrating DB version 2 to 3 failed: %s" % e)

# add notification field to comments (moved from Comments class to migration)
if self.version == 3:
with sqlite3.connect(self.path) as con:
self.migrate_to_version_4(con)
ix5 marked this conversation as resolved.
Show resolved Hide resolved

# "text" field in "comments" became NOT NULL
if self.version == 4:
with sqlite3.connect(self.path) as con:
con.execute("BEGIN TRANSACTION")
pkvach marked this conversation as resolved.
Show resolved Hide resolved
con.execute("UPDATE comments SET text = '' WHERE text IS NULL")

# create new table with NOT NULL constraint for "text" field
con.execute(Comments.create_table_query("comments_new"))

try:
# copy data from old table to new table
con.execute("""
INSERT INTO comments_new (
tid, id, parent, created, modified, mode, remote_addr, text, author, email, website, likes, dislikes, voters, notification
)
SELECT
tid, id, parent, created, modified, mode, remote_addr, text, author, email, website, likes, dislikes, voters, notification
FROM comments
""")

# swap tables and drop old table
con.execute("ALTER TABLE comments RENAME TO comments_backup_v4")
con.execute("ALTER TABLE comments_new RENAME TO comments")
con.execute("DROP TABLE comments_backup_v4")

con.execute('PRAGMA user_version = 5')
con.execute("COMMIT")
logger.info("Migrating DB version 4 to 5 by setting empty comments.text to '', %i rows changed",
con.total_changes)
except sqlite3.Error as e:
con.execute("ROLLBACK")
logger.error("Migrating DB version 4 to 5 failed: %s", e)
raise RuntimeError("Migrating DB version 4 to 5 failed: %s" % e)

def migrate_to_version_4(self, con):
# check if "notification" column exists in "comments" table
rv = con.execute("PRAGMA table_info(comments)").fetchall()
if any([row[1] == 'notification' for row in rv]):
logger.info("Migrating DB version 3 to 4 skipped, 'notification' field already exists in comments")
con.execute('PRAGMA user_version = 4')
return

con.execute('PRAGMA user_version = 3')
logger.info("%i rows changed", con.total_changes)
con.execute("BEGIN TRANSACTION")
try:
con.execute('ALTER TABLE comments ADD COLUMN notification INTEGER DEFAULT 0;')
con.execute('PRAGMA user_version = 4')
con.execute("COMMIT")
logger.info("Migrating DB version 3 to 4 by adding 'notification' field to comments")
except sqlite3.Error as e:
con.execute("ROLLBACK")
logger.error("Migrating DB version 3 to 4 failed: %s", e)
raise RuntimeError("Migrating DB version 3 to 4 failed: %s" % e)
36 changes: 25 additions & 11 deletions isso/db/comments.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -31,20 +31,34 @@ class Comments:
'remote_addr', 'text', 'author', 'email', 'website',
'likes', 'dislikes', 'voters', 'notification']

# This method is used in the migration script from version 4 to 5.
# You need to write a new migration if you change the database schema!
@staticmethod
def create_table_query(table_name):
return f'''
CREATE TABLE IF NOT EXISTS {table_name} (
tid REFERENCES threads(id),
id INTEGER PRIMARY KEY,
parent INTEGER,
created FLOAT NOT NULL,
modified FLOAT,
mode INTEGER,
remote_addr VARCHAR,
text VARCHAR NOT NULL,
author VARCHAR,
email VARCHAR,
website VARCHAR,
likes INTEGER DEFAULT 0,
dislikes INTEGER DEFAULT 0,
voters BLOB NOT NULL,
notification INTEGER DEFAULT 0
);
'''

def __init__(self, db):

self.db = db
self.db.execute([
'CREATE TABLE IF NOT EXISTS comments (',
' tid REFERENCES threads(id), id INTEGER PRIMARY KEY, parent INTEGER,',
' created FLOAT NOT NULL, modified FLOAT, mode INTEGER, remote_addr VARCHAR,',
' text VARCHAR NOT NULL, author VARCHAR, email VARCHAR, website VARCHAR,',
' likes INTEGER DEFAULT 0, dislikes INTEGER DEFAULT 0, voters BLOB NOT NULL,',
' notification INTEGER DEFAULT 0);'])
try:
self.db.execute(['ALTER TABLE comments ADD COLUMN notification INTEGER DEFAULT 0;'])
except Exception:
pass
self.db.execute(Comments.create_table_query("comments"))

def add(self, uri, c):
"""
Expand Down
Loading
Loading