Skip to content

Commit

Permalink
db: Make 'text' field in 'comments' table NOT NULL and handling data …
Browse files Browse the repository at this point in the history 
…migration

This update introduces a schema migration to version 4 for
the database, focusing on enhancing the 'comments' table.
This ensures that the 'text' field in the 'comments' table
will always have a value, which improves data consistency
and integrity.

See:
- #979
- #994
  • Loading branch information
@pkvach @ix5
pkvach authored and ix5 committed May 5, 2024
1 parent 6c6d5d2 commit c8602c0
Show file tree
Hide file tree
Showing 4 changed files with 294 additions and 32 deletions.
2 changes: 2 additions & 0 deletions CHANGES.rst
View file
Original file line number Diff line number Diff line change
Expand Up @@ -45,6 +45,7 @@ Bugfixes & Improvements
- Fix newline character handling in data-isso-* i18n strings (`#992`_, pkvach)
- Add link logging for management of new comments in Stdout (`#1016`_, pkvach)
- Change logging to include datetime and loglevel (`#1023`_, ix5)
- Make 'text' field in 'comments' table NOT NULL and handling data migration (`#1019`_, pkvach)

.. _#951: https://github.com/posativ/isso/pull/951
.. _#967: https://github.com/posativ/isso/pull/967
Expand All @@ -56,6 +57,7 @@ Bugfixes & Improvements
.. _#992: https://github.com/isso-comments/isso/pull/992
.. _#1016: https://github.com/isso-comments/isso/pull/1016
.. _#1023: https://github.com/isso-comments/isso/pull/1023
.. _#1019: https://github.com/isso-comments/isso/pull/1019

0.13.1.dev0 (2023-02-05)
------------------------
Expand Down
116 changes: 99 additions & 17 deletions isso/db/__init__.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -22,7 +22,7 @@ class SQLite3:
a trigger for automated orphan removal.
"""

MAX_VERSION = 3
MAX_VERSION = 5

def __init__(self, path, conf):

Expand Down Expand Up @@ -68,7 +68,7 @@ def migrate(self, to):
if self.version >= to:
return

logger.info("migrate database from version %i to %i", self.version, to)
logger.info("Migrating database from version %i to %i", self.version, to)

# re-initialize voters blob due a bug in the bloomfilter signature
# which added older commenter's ip addresses to the current voters blob
Expand All @@ -78,20 +78,36 @@ def migrate(self, to):
bf = memoryview(Bloomfilter(iterable=["127.0.0.0"]).array)

with sqlite3.connect(self.path) as con:
con.execute('UPDATE comments SET voters=?', (bf, ))
con.execute('PRAGMA user_version = 1')
logger.info("%i rows changed", con.total_changes)
con.execute("BEGIN TRANSACTION")
try:
con.execute('UPDATE comments SET voters=?', (bf, ))
con.execute('PRAGMA user_version = 1')
con.execute("COMMIT")
logger.info("Migrating DB version 0 to 1 by re-initializing voters blob, %i rows changed",
con.total_changes)
except sqlite3.Error as e:
con.execute("ROLLBACK")
logger.error("Migrating DB version 0 to 1 failed: %s", e)
raise RuntimeError("Migrating DB version 0 to 1 failed: %s" % e)

# move [general] session-key to database
if self.version == 1:

with sqlite3.connect(self.path) as con:
if self.conf.has_option("general", "session-key"):
con.execute('UPDATE preferences SET value=? WHERE key=?', (
self.conf.get("general", "session-key"), "session-key"))

con.execute('PRAGMA user_version = 2')
logger.info("%i rows changed", con.total_changes)
con.execute("BEGIN TRANSACTION")
try:
if self.conf.has_option("general", "session-key"):
con.execute('UPDATE preferences SET value=? WHERE key=?', (
self.conf.get("general", "session-key"), "session-key"))

con.execute('PRAGMA user_version = 2')
con.execute("COMMIT")
logger.info("Migrating DB version 1 to 2 by moving session-key to database, %i rows changed",
con.total_changes)
except sqlite3.Error as e:
con.execute("ROLLBACK")
logger.error("Migrating DB version 1 to 2 failed: %s", e)
raise RuntimeError("Migrating DB version 1 to 2 failed: %s" % e)

# limit max. nesting level to 1
if self.version == 2:
Expand All @@ -114,10 +130,76 @@ def first(rv):
ids.extend(rv)
flattened[id].update(set(rv))

for id in flattened.keys():
for n in flattened[id]:
con.execute(
"UPDATE comments SET parent=? WHERE id=?", (id, n))
con.execute("BEGIN TRANSACTION")
try:
for id in flattened.keys():
for n in flattened[id]:
con.execute(
"UPDATE comments SET parent=? WHERE id=?", (id, n))

con.execute('PRAGMA user_version = 3')
con.execute("COMMIT")
logger.info("Migrating DB version 2 to 3 by limiting nesting level to 1, %i rows changed",
con.total_changes)
except sqlite3.Error as e:
con.execute("ROLLBACK")
logger.error("Migrating DB version 2 to 3 failed: %s", e)
raise RuntimeError("Migrating DB version 2 to 3 failed: %s" % e)

# add notification field to comments (moved from Comments class to migration)
if self.version == 3:
with sqlite3.connect(self.path) as con:
self.migrate_to_version_4(con)

# "text" field in "comments" became NOT NULL
if self.version == 4:
with sqlite3.connect(self.path) as con:
con.execute("BEGIN TRANSACTION")
con.execute("UPDATE comments SET text = '' WHERE text IS NULL")

# create new table with NOT NULL constraint for "text" field
con.execute(Comments.create_table_query("comments_new"))

try:
# copy data from old table to new table
con.execute("""
INSERT INTO comments_new (
tid, id, parent, created, modified, mode, remote_addr, text, author, email, website, likes, dislikes, voters, notification
)
SELECT
tid, id, parent, created, modified, mode, remote_addr, text, author, email, website, likes, dislikes, voters, notification
FROM comments
""")

# swap tables and drop old table
con.execute("ALTER TABLE comments RENAME TO comments_backup_v4")
con.execute("ALTER TABLE comments_new RENAME TO comments")
con.execute("DROP TABLE comments_backup_v4")

con.execute('PRAGMA user_version = 5')
con.execute("COMMIT")
logger.info("Migrating DB version 4 to 5 by setting empty comments.text to '', %i rows changed",
con.total_changes)
except sqlite3.Error as e:
con.execute("ROLLBACK")
logger.error("Migrating DB version 4 to 5 failed: %s", e)
raise RuntimeError("Migrating DB version 4 to 5 failed: %s" % e)

def migrate_to_version_4(self, con):
# check if "notification" column exists in "comments" table
rv = con.execute("PRAGMA table_info(comments)").fetchall()
if any([row[1] == 'notification' for row in rv]):
logger.info("Migrating DB version 3 to 4 skipped, 'notification' field already exists in comments")
con.execute('PRAGMA user_version = 4')
return

con.execute('PRAGMA user_version = 3')
logger.info("%i rows changed", con.total_changes)
con.execute("BEGIN TRANSACTION")
try:
con.execute('ALTER TABLE comments ADD COLUMN notification INTEGER DEFAULT 0;')
con.execute('PRAGMA user_version = 4')
con.execute("COMMIT")
logger.info("Migrating DB version 3 to 4 by adding 'notification' field to comments")
except sqlite3.Error as e:
con.execute("ROLLBACK")
logger.error("Migrating DB version 3 to 4 failed: %s", e)
raise RuntimeError("Migrating DB version 3 to 4 failed: %s" % e)
36 changes: 25 additions & 11 deletions isso/db/comments.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -31,20 +31,34 @@ class Comments:
'remote_addr', 'text', 'author', 'email', 'website',
'likes', 'dislikes', 'voters', 'notification']

# This method is used in the migration script from version 4 to 5.
# You need to write a new migration if you change the database schema!
@staticmethod
def create_table_query(table_name):
return f'''
CREATE TABLE IF NOT EXISTS {table_name} (
tid REFERENCES threads(id),
id INTEGER PRIMARY KEY,
parent INTEGER,
created FLOAT NOT NULL,
modified FLOAT,
mode INTEGER,
remote_addr VARCHAR,
text VARCHAR NOT NULL,
author VARCHAR,
email VARCHAR,
website VARCHAR,
likes INTEGER DEFAULT 0,
dislikes INTEGER DEFAULT 0,
voters BLOB NOT NULL,
notification INTEGER DEFAULT 0
);
'''

def __init__(self, db):

self.db = db
self.db.execute([
'CREATE TABLE IF NOT EXISTS comments (',
' tid REFERENCES threads(id), id INTEGER PRIMARY KEY, parent INTEGER,',
' created FLOAT NOT NULL, modified FLOAT, mode INTEGER, remote_addr VARCHAR,',
' text VARCHAR NOT NULL, author VARCHAR, email VARCHAR, website VARCHAR,',
' likes INTEGER DEFAULT 0, dislikes INTEGER DEFAULT 0, voters BLOB NOT NULL,',
' notification INTEGER DEFAULT 0);'])
try:
self.db.execute(['ALTER TABLE comments ADD COLUMN notification INTEGER DEFAULT 0;'])
except Exception:
pass
self.db.execute(Comments.create_table_query("comments"))

def add(self, uri, c):
"""
Expand Down
Loading

0 comments on commit c8602c0

Please sign in to comment.