From 98a713aaa1426bb4598ac04daacb1bf6f907cd80 Mon Sep 17 00:00:00 2001 From: flow Date: Thu, 27 Jul 2023 22:34:47 +0800 Subject: [PATCH] add local cache for tags --- lib/main.dart | 11 +-- lib/repository/chat_room.dart | 51 ++++++++--- lib/repository/tags.dart | 165 +++++++++++++++++++--------------- 3 files changed, 135 insertions(+), 92 deletions(-) diff --git a/lib/main.dart b/lib/main.dart index e168c61..f724098 100644 --- a/lib/main.dart +++ b/lib/main.dart @@ -6,20 +6,18 @@ import 'package:moyubie/controller/message.dart'; import 'package:moyubie/controller/prompt.dart'; import 'package:moyubie/controller/settings.dart'; import 'package:moyubie/components/setting.dart'; +import 'package:moyubie/repository/tags.dart'; import 'package:get/get.dart'; import 'package:get_storage/get_storage.dart'; import 'package:moyubie/repository/tags.dart'; import 'package:moyubie/utils/tag_collector.dart'; import 'package:sqflite_common_ffi_web/sqflite_ffi_web.dart'; import 'package:sqflite_common_ffi/sqflite_ffi.dart'; -import 'package:moyubie/configs/translations.dart'; import 'package:flutter/foundation.dart' show kIsWeb; -import 'package:webview_flutter/webview_flutter.dart'; import 'dart:io' show Platform; import 'package:path/path.dart'; import 'package:firebase_core/firebase_core.dart'; import 'firebase_options.dart'; -import 'package:firebase_analytics/firebase_analytics.dart'; import 'components/chat_room.dart'; import 'controller/chat_room.dart'; @@ -86,7 +84,7 @@ class MyApp extends StatelessWidget { var shortestSide = MediaQuery.of(context).size.shortestSide; final ChatRoomType type = ChatRoomType.phone; final settingsCtl = SettingsController(); - final tagsRepo = TagsRepository.byConfig(settingsCtl); + final tagsRepo = TagsRepository(); Get.put(settingsCtl); Get.put(MessageController()); Get.put(PromptController()); @@ -111,7 +109,10 @@ class MyApp extends StatelessWidget { physics: const NeverScrollableScrollPhysics(), children: [ ChatRoom(restorationId: "chat_room", type: type), - NewsWindow(ty: type, key: newsWinKey,), + NewsWindow( + ty: type, + key: newsWinKey, + ), const SettingPage(), ], ), diff --git a/lib/repository/chat_room.dart b/lib/repository/chat_room.dart index 9254158..f608d6b 100644 --- a/lib/repository/chat_room.dart +++ b/lib/repository/chat_room.dart @@ -186,6 +186,11 @@ class ChatRoomRepository { // The user role of this chat room, could be 'host' or 'guest' static const String _columnChatRoomRole = 'role'; + static const _tableTags = "tags"; + static const _columnTagsName = "name"; + // UTC time zone. SQLite: Text, TiDB: DateTime + static const _columnTagsAddedAt = "added_at"; + static const String _columnMessageUuid = 'uuid'; static const String _columnMessageUserName = 'user_name'; // UTC time zone. SQLite: Text, TiDB: DateTime @@ -212,14 +217,16 @@ class ChatRoomRepository { return _instance!; } - Future _getDb() async { + Future getLocalDb() async { if (_database == null) { final String path = join(await getDatabasesPath(), 'moyubie.db'); _database = await openDatabase(path, version: 1, onCreate: (Database db, int version) async { print("on create!"); - await db.execute(''' - CREATE TABLE $_tableChatRoom ( + final batch = db.batch(); + + batch.execute(''' + CREATE TABLE IF NOT EXISTS $_tableChatRoom ( $_columnChatRoomUuid VARCHAR(36) PRIMARY KEY, $_columnChatRoomName TEXT, $_columnChatRoomCreateTime TEXT, @@ -227,6 +234,15 @@ class ChatRoomRepository { $_columnChatRoomRole TEXT ) '''); + batch.execute(''' + CREATE TABLE IF NOT EXISTS $_tableTags ( + $_columnTagsName TEXT, + $_columnTagsAddedAt TEXT, + "INDEX sand_of_time(added_at)" + ) + '''); + + await batch.commit(); }); } return _database!; @@ -291,6 +307,10 @@ class ChatRoomRepository { return "hose: ${myTiDBConn.host}, port: ${myTiDBConn.port}, userName: ${myTiDBConn.userName}, password: ${myTiDBConn.password}"; } + Future getMyRemoteDb() { + return getRemoteDb(myTiDBConn, true); + } + static Future getRemoteDb(TiDBConnection conn, bool isHost, {bool forceInit = false}) async { bool shouldInit = @@ -324,9 +344,7 @@ class ChatRoomRepository { await dbConn.execute("CREATE DATABASE IF NOT EXISTS moyubie;"); } await dbConn.execute("USE moyubie;"); - res = await dbConn.execute("SHOW TABLES LIKE 'chat_room';"); - if (res.rows.isEmpty) { - await dbConn.execute(''' + await dbConn.execute(''' CREATE TABLE IF NOT EXISTS $_tableChatRoom ( $_columnChatRoomUuid VARCHAR(36) PRIMARY KEY, $_columnChatRoomName TEXT, @@ -335,7 +353,12 @@ class ChatRoomRepository { $_columnChatRoomRole TEXT ) '''); - } + await dbConn.execute(''' + CREATE TABLE IF NOT EXISTS $_tableTags ( + $_columnTagsName TEXT, + $_columnTagsAddedAt DATETIME(6) + ) + '''); } } } catch (e) { @@ -346,7 +369,7 @@ class ChatRoomRepository { } Future> getChatRooms({String? roomId}) async { - final db = await _getDb(); + final db = await getLocalDb(); String? where = roomId == null ? null : "$_columnChatRoomUuid = '$roomId'"; final List> maps = await db.query(_tableChatRoom, where: where); @@ -385,7 +408,7 @@ class ChatRoomRepository { } Future upsertLocalChatRooms(List rooms) async { - final db = await _getDb(); + final db = await getLocalDb(); // await db.execute("DELETE FROM $_tableChatRoom;"); for (var room in rooms) { // TODO Remote this @@ -486,7 +509,7 @@ class ChatRoomRepository { // Use the user who is dedicated for this chat room to chat room.connectionToken = roomConn.toToken(); - final db = await _getDb(); + final db = await getLocalDb(); await db.insert( _tableChatRoom, room.toSQLMap(), @@ -505,7 +528,7 @@ class ChatRoomRepository { } Future updateChatRoom(ChatRoom chatRoom) async { - final db = await _getDb(); + final db = await getLocalDb(); await db.update( _tableChatRoom, chatRoom.toSQLMap(), @@ -529,7 +552,7 @@ class ChatRoomRepository { Future deleteChatRoom(ChatRoom room) async { final uuid = room.uuid; - final db = await _getDb(); + final db = await getLocalDb(); await db.transaction((txn) async { await txn.delete( _tableChatRoom, @@ -558,7 +581,7 @@ class ChatRoomRepository { Future> getMessagesByChatRoomUUid(ChatRoom room, {int limit = 500}) async { - final db = await _getDb(); + final db = await getLocalDb(); final List> maps = await db.query( '`msg_${room.uuid}`', orderBy: "julianday($_columnMessageCreateTime) desc", @@ -621,7 +644,7 @@ class ChatRoomRepository { } Future addMessageLocal(ChatRoom room, List messages) async { - final db = await _getDb(); + final db = await getLocalDb(); final batch = db.batch(); for (final m in messages) { batch.insert('`msg_${room.uuid}`', m.toSQLMap(), diff --git a/lib/repository/tags.dart b/lib/repository/tags.dart index 1aca072..b7b0460 100644 --- a/lib/repository/tags.dart +++ b/lib/repository/tags.dart @@ -1,97 +1,116 @@ import 'dart:developer'; +import 'package:flutter/widgets.dart'; +import 'package:sqflite/sqflite.dart'; import 'package:get/get.dart'; import 'package:get/get_rx/get_rx.dart'; import 'package:moyubie/controller/settings.dart'; import 'package:moyubie/utils/tidb.dart'; +import 'package:moyubie/repository/chat_room.dart'; import 'package:mysql_client/mysql_client.dart'; import 'package:stream_transform/stream_transform.dart'; -class _CachedConnection { - MySQLConnection? _last; +class Tag { + String name; + DateTime time; - _CachedConnection(Stream stream) { - stream.listen((event) { - if (_last != null && _last != event) { - log("Update connection. [old=$_last, new=$event]", - name: "moyubie::_CachedConnection"); - _last!.close(); - } - _last = event; - }, - onError: (err) => - log("ERROR [$err]", name: "moyubie::_CachedConnection"), - onDone: () => log("APP CLOSED", name: "moyubie::_CachedConnection"), - cancelOnError: false); - } + Tag({ + required this.name, + required this.time, + }); - MySQLConnection? get value => _last; + Map toSQLMap() { + return { + 'name': name, + 'added_at': time.toIso8601String(), + }; + } } +// TODO refactor TagsRepository and ChatRoomRepository class TagsRepository { - factory TagsRepository.byConfig(SettingsController ctl, - {bool forceInit = false}) { - final stream = ctl.serverlessCmd.stream.switchMap((p0) { - final (host, port, user, password, msgTable) = - parseTiDBConnectionText(p0); - if (host.isEmpty || port == 0 || user.isEmpty) { - return Stream.error(Exception("Invalid DB")); - } - return Stream.fromFuture(() async { - final conn = await MySQLConnection.createConnection( - host: host, port: port, userName: user, password: password); - await conn.connect(); - await prepareTables(conn); - return conn; - }()); - }); - return TagsRepository(_CachedConnection(stream)); - } + static const tableTags = "tags"; + static const columnTagsName = "name"; + // UTC time zone. SQLite: Text, TiDB: DateTime + static const columnTagsAddedAt = "added_at"; - final _CachedConnection _conn; - - TagsRepository(this._conn); + Future addNewTags(List tags) async { + var now = DateTime.now().toUtc(); + return await _addNewTags(tags.map((n) => Tag(name: n, time: now)).toList()); + } - static const _tagName = "name"; - static const _tagAddedAt = "added_at"; - static const _table = "tags"; - static const _db = "moyubie"; + // Note that we don't wait for database operations + Future _addNewTags(List tags) async { + final db = ChatRoomRepository().getLocalDb(); + db.then((db_) async { + final batch = db_.batch(); + for (final tag in tags) { + db_.insert( + tableTags, + tag.toSQLMap(), + conflictAlgorithm: ConflictAlgorithm.replace, + ); + } + await batch.commit(); + }); - static Future prepareTables(MySQLConnection conn) async { - await conn.execute("CREATE DATABASE IF NOT EXISTS $_db"); - await conn.execute("CREATE TABLE IF NOT EXISTS `$_db`.$_table(" - "$_tagName TEXT," - "$_tagAddedAt DATETIME," - "INDEX sand_of_time(added_at)" - ");"); - } + final remoteDB = ChatRoomRepository().getMyRemoteDb(); + remoteDB.then((remoteDB_) async { + if (remoteDB_ == null) { + return; + } - Future addNewTags(List tags) async { - final now = DateTime.now(); - if (_conn.value == null) { - throw Exception("The connection isn't ready for now..."); - } - final insert = await _conn.value!.prepare( - "INSERT INTO $_db.$_table($_tagName, $_tagAddedAt) VALUES (?, ?);"); - await Future.wait(tags.map((e) => insert.execute([e, now])), - eagerError: true); + for (final tag in tags) { + await remoteDB_.execute( + "INSERT IGNORE INTO moyubie.`$tableTags` VALUES (:name, :time)", + {"name": tag.name, "time": tag.time.toString()}); + } + }); } Future> fetchMostPopularTags(int limit) async { - if (_conn.value == null) { - throw Exception("The connection isn't ready for now..."); - } - final res = await _conn.value!.execute( - "SELECT t1.NAME AS $_tagName FROM " - "(SELECT COUNT($_tagName) AS CNT, $_tagName AS NAME FROM $_db.$_table GROUP BY $_tagName) AS t1" - " ORDER BY t1.CNT DESC LIMIT :limit; ", - {"limit": limit}); - final output = []; - for (final rs in res) { - for (final row in rs.rows) { - output.add(row.colByName(_tagName) as String); + // Firt return what we have in local + final db = await ChatRoomRepository().getLocalDb(); + final List> maps = await db.rawQuery(''' + SELECT t1.NAME AS $columnTagsName FROM + (SELECT COUNT($columnTagsName) AS CNT, $columnTagsName AS NAME FROM $tableTags GROUP BY $columnTagsName) AS t1 + ORDER BY t1.CNT DESC LIMIT $limit; + '''); + + final localTags = maps.map((e) { + String tag_ = e[columnTagsName]; + return tag_; + }).toList(); + + // Then start async task to synchronize remote to local + final remoteDB = ChatRoomRepository().getMyRemoteDb(); + remoteDB.then((remoteDB_) async { + if (remoteDB_ == null) { + return; } - } - return output; + final List> maps = await db.query(tableTags, + orderBy: "$columnTagsAddedAt DESC", limit: 1); + DateTime? last; + if (maps.isNotEmpty) { + last = DateTime.parse(maps.first[columnTagsAddedAt]); + } + var sql = last == null + ? "SELECT * FROM moyubie.`$tableTags` ORDER BY `$columnTagsAddedAt` DESC LIMIT 100" + : "SELECT * FROM moyubie.`$tableTags` WHERE `$columnTagsAddedAt` > '${last.toString()}' ORDER BY `$columnTagsAddedAt` DESC LIMIT 100"; + final res = await remoteDB_.execute(sql); + if (res.rows.isEmpty) { + return; + } + final remoteTags = res.rows.map((e) { + var map = e.assoc(); + return Tag( + name: map[columnTagsName]!, + time: DateTime.parse(map[columnTagsAddedAt]!)); + }).toList(); + + await _addNewTags(remoteTags); + }); + + return localTags; } }