Skip to content

Commit

Permalink
add local cache for tags
Browse files Browse the repository at this point in the history
  • Loading branch information
flowbehappy committed Jul 27, 2023
1 parent c4f761a commit 98a713a
Show file tree
Hide file tree
Showing 3 changed files with 135 additions and 92 deletions.
11 changes: 6 additions & 5 deletions lib/main.dart
Original file line number Diff line number Diff line change
Expand Up @@ -6,20 +6,18 @@ import 'package:moyubie/controller/message.dart';
import 'package:moyubie/controller/prompt.dart';
import 'package:moyubie/controller/settings.dart';
import 'package:moyubie/components/setting.dart';
import 'package:moyubie/repository/tags.dart';
import 'package:get/get.dart';
import 'package:get_storage/get_storage.dart';
import 'package:moyubie/repository/tags.dart';
import 'package:moyubie/utils/tag_collector.dart';
import 'package:sqflite_common_ffi_web/sqflite_ffi_web.dart';
import 'package:sqflite_common_ffi/sqflite_ffi.dart';
import 'package:moyubie/configs/translations.dart';
import 'package:flutter/foundation.dart' show kIsWeb;
import 'package:webview_flutter/webview_flutter.dart';
import 'dart:io' show Platform;
import 'package:path/path.dart';
import 'package:firebase_core/firebase_core.dart';
import 'firebase_options.dart';
import 'package:firebase_analytics/firebase_analytics.dart';

import 'components/chat_room.dart';
import 'controller/chat_room.dart';
Expand Down Expand Up @@ -86,7 +84,7 @@ class MyApp extends StatelessWidget {
var shortestSide = MediaQuery.of(context).size.shortestSide;
final ChatRoomType type = ChatRoomType.phone;
final settingsCtl = SettingsController();
final tagsRepo = TagsRepository.byConfig(settingsCtl);
final tagsRepo = TagsRepository();
Get.put(settingsCtl);
Get.put(MessageController());
Get.put(PromptController());
Expand All @@ -111,7 +109,10 @@ class MyApp extends StatelessWidget {
physics: const NeverScrollableScrollPhysics(),
children: [
ChatRoom(restorationId: "chat_room", type: type),
NewsWindow(ty: type, key: newsWinKey,),
NewsWindow(
ty: type,
key: newsWinKey,
),
const SettingPage(),
],
),
Expand Down
51 changes: 37 additions & 14 deletions lib/repository/chat_room.dart
Original file line number Diff line number Diff line change
Expand Up @@ -186,6 +186,11 @@ class ChatRoomRepository {
// The user role of this chat room, could be 'host' or 'guest'
static const String _columnChatRoomRole = 'role';

static const _tableTags = "tags";
static const _columnTagsName = "name";
// UTC time zone. SQLite: Text, TiDB: DateTime
static const _columnTagsAddedAt = "added_at";

static const String _columnMessageUuid = 'uuid';
static const String _columnMessageUserName = 'user_name';
// UTC time zone. SQLite: Text, TiDB: DateTime
Expand All @@ -212,21 +217,32 @@ class ChatRoomRepository {
return _instance!;
}

Future<Database> _getDb() async {
Future<Database> getLocalDb() async {
if (_database == null) {
final String path = join(await getDatabasesPath(), 'moyubie.db');
_database = await openDatabase(path, version: 1,
onCreate: (Database db, int version) async {
print("on create!");
await db.execute('''
CREATE TABLE $_tableChatRoom (
final batch = db.batch();

batch.execute('''
CREATE TABLE IF NOT EXISTS $_tableChatRoom (
$_columnChatRoomUuid VARCHAR(36) PRIMARY KEY,
$_columnChatRoomName TEXT,
$_columnChatRoomCreateTime TEXT,
$_columnChatRoomConnectionToken TEXT,
$_columnChatRoomRole TEXT
)
''');
batch.execute('''
CREATE TABLE IF NOT EXISTS $_tableTags (
$_columnTagsName TEXT,
$_columnTagsAddedAt TEXT,
"INDEX sand_of_time(added_at)"
)
''');

await batch.commit();
});
}
return _database!;
Expand Down Expand Up @@ -291,6 +307,10 @@ class ChatRoomRepository {
return "hose: ${myTiDBConn.host}, port: ${myTiDBConn.port}, userName: ${myTiDBConn.userName}, password: ${myTiDBConn.password}";
}

Future<MySQLConnection?> getMyRemoteDb() {
return getRemoteDb(myTiDBConn, true);
}

static Future<MySQLConnection?> getRemoteDb(TiDBConnection conn, bool isHost,
{bool forceInit = false}) async {
bool shouldInit =
Expand Down Expand Up @@ -324,9 +344,7 @@ class ChatRoomRepository {
await dbConn.execute("CREATE DATABASE IF NOT EXISTS moyubie;");
}
await dbConn.execute("USE moyubie;");
res = await dbConn.execute("SHOW TABLES LIKE 'chat_room';");
if (res.rows.isEmpty) {
await dbConn.execute('''
await dbConn.execute('''
CREATE TABLE IF NOT EXISTS $_tableChatRoom (
$_columnChatRoomUuid VARCHAR(36) PRIMARY KEY,
$_columnChatRoomName TEXT,
Expand All @@ -335,7 +353,12 @@ class ChatRoomRepository {
$_columnChatRoomRole TEXT
)
''');
}
await dbConn.execute('''
CREATE TABLE IF NOT EXISTS $_tableTags (
$_columnTagsName TEXT,
$_columnTagsAddedAt DATETIME(6)
)
''');
}
}
} catch (e) {
Expand All @@ -346,7 +369,7 @@ class ChatRoomRepository {
}

Future<List<ChatRoom>> getChatRooms({String? roomId}) async {
final db = await _getDb();
final db = await getLocalDb();
String? where = roomId == null ? null : "$_columnChatRoomUuid = '$roomId'";
final List<Map<String, dynamic>> maps =
await db.query(_tableChatRoom, where: where);
Expand Down Expand Up @@ -385,7 +408,7 @@ class ChatRoomRepository {
}

Future<void> upsertLocalChatRooms(List<ChatRoom> rooms) async {
final db = await _getDb();
final db = await getLocalDb();
// await db.execute("DELETE FROM $_tableChatRoom;");
for (var room in rooms) {
// TODO Remote this
Expand Down Expand Up @@ -486,7 +509,7 @@ class ChatRoomRepository {
// Use the user who is dedicated for this chat room to chat
room.connectionToken = roomConn.toToken();

final db = await _getDb();
final db = await getLocalDb();
await db.insert(
_tableChatRoom,
room.toSQLMap(),
Expand All @@ -505,7 +528,7 @@ class ChatRoomRepository {
}

Future<void> updateChatRoom(ChatRoom chatRoom) async {
final db = await _getDb();
final db = await getLocalDb();
await db.update(
_tableChatRoom,
chatRoom.toSQLMap(),
Expand All @@ -529,7 +552,7 @@ class ChatRoomRepository {

Future<void> deleteChatRoom(ChatRoom room) async {
final uuid = room.uuid;
final db = await _getDb();
final db = await getLocalDb();
await db.transaction((txn) async {
await txn.delete(
_tableChatRoom,
Expand Down Expand Up @@ -558,7 +581,7 @@ class ChatRoomRepository {

Future<List<Message>> getMessagesByChatRoomUUid(ChatRoom room,
{int limit = 500}) async {
final db = await _getDb();
final db = await getLocalDb();
final List<Map<String, dynamic>> maps = await db.query(
'`msg_${room.uuid}`',
orderBy: "julianday($_columnMessageCreateTime) desc",
Expand Down Expand Up @@ -621,7 +644,7 @@ class ChatRoomRepository {
}

Future<void> addMessageLocal(ChatRoom room, List<Message> messages) async {
final db = await _getDb();
final db = await getLocalDb();
final batch = db.batch();
for (final m in messages) {
batch.insert('`msg_${room.uuid}`', m.toSQLMap(),
Expand Down
165 changes: 92 additions & 73 deletions lib/repository/tags.dart
Original file line number Diff line number Diff line change
@@ -1,97 +1,116 @@
import 'dart:developer';

import 'package:flutter/widgets.dart';
import 'package:sqflite/sqflite.dart';
import 'package:get/get.dart';
import 'package:get/get_rx/get_rx.dart';
import 'package:moyubie/controller/settings.dart';
import 'package:moyubie/utils/tidb.dart';
import 'package:moyubie/repository/chat_room.dart';
import 'package:mysql_client/mysql_client.dart';
import 'package:stream_transform/stream_transform.dart';

class _CachedConnection {
MySQLConnection? _last;
class Tag {
String name;
DateTime time;

_CachedConnection(Stream<MySQLConnection> stream) {
stream.listen((event) {
if (_last != null && _last != event) {
log("Update connection. [old=$_last, new=$event]",
name: "moyubie::_CachedConnection");
_last!.close();
}
_last = event;
},
onError: (err) =>
log("ERROR [$err]", name: "moyubie::_CachedConnection"),
onDone: () => log("APP CLOSED", name: "moyubie::_CachedConnection"),
cancelOnError: false);
}
Tag({
required this.name,
required this.time,
});

MySQLConnection? get value => _last;
Map<String, dynamic> toSQLMap() {
return {
'name': name,
'added_at': time.toIso8601String(),
};
}
}

// TODO refactor TagsRepository and ChatRoomRepository
class TagsRepository {
factory TagsRepository.byConfig(SettingsController ctl,
{bool forceInit = false}) {
final stream = ctl.serverlessCmd.stream.switchMap((p0) {
final (host, port, user, password, msgTable) =
parseTiDBConnectionText(p0);
if (host.isEmpty || port == 0 || user.isEmpty) {
return Stream<MySQLConnection>.error(Exception("Invalid DB"));
}
return Stream.fromFuture(() async {
final conn = await MySQLConnection.createConnection(
host: host, port: port, userName: user, password: password);
await conn.connect();
await prepareTables(conn);
return conn;
}());
});
return TagsRepository(_CachedConnection(stream));
}
static const tableTags = "tags";
static const columnTagsName = "name";
// UTC time zone. SQLite: Text, TiDB: DateTime
static const columnTagsAddedAt = "added_at";

final _CachedConnection _conn;

TagsRepository(this._conn);
Future<void> addNewTags(List<String> tags) async {
var now = DateTime.now().toUtc();
return await _addNewTags(tags.map((n) => Tag(name: n, time: now)).toList());
}

static const _tagName = "name";
static const _tagAddedAt = "added_at";
static const _table = "tags";
static const _db = "moyubie";
// Note that we don't wait for database operations
Future<void> _addNewTags(List<Tag> tags) async {
final db = ChatRoomRepository().getLocalDb();
db.then((db_) async {
final batch = db_.batch();
for (final tag in tags) {
db_.insert(
tableTags,
tag.toSQLMap(),
conflictAlgorithm: ConflictAlgorithm.replace,
);
}
await batch.commit();
});

static Future<void> prepareTables(MySQLConnection conn) async {
await conn.execute("CREATE DATABASE IF NOT EXISTS $_db");
await conn.execute("CREATE TABLE IF NOT EXISTS `$_db`.$_table("
"$_tagName TEXT,"
"$_tagAddedAt DATETIME,"
"INDEX sand_of_time(added_at)"
");");
}
final remoteDB = ChatRoomRepository().getMyRemoteDb();
remoteDB.then((remoteDB_) async {
if (remoteDB_ == null) {
return;
}

Future<void> addNewTags(List<String> tags) async {
final now = DateTime.now();
if (_conn.value == null) {
throw Exception("The connection isn't ready for now...");
}
final insert = await _conn.value!.prepare(
"INSERT INTO $_db.$_table($_tagName, $_tagAddedAt) VALUES (?, ?);");
await Future.wait(tags.map((e) => insert.execute([e, now])),
eagerError: true);
for (final tag in tags) {
await remoteDB_.execute(
"INSERT IGNORE INTO moyubie.`$tableTags` VALUES (:name, :time)",
{"name": tag.name, "time": tag.time.toString()});
}
});
}

Future<List<String>> fetchMostPopularTags(int limit) async {
if (_conn.value == null) {
throw Exception("The connection isn't ready for now...");
}
final res = await _conn.value!.execute(
"SELECT t1.NAME AS $_tagName FROM "
"(SELECT COUNT($_tagName) AS CNT, $_tagName AS NAME FROM $_db.$_table GROUP BY $_tagName) AS t1"
" ORDER BY t1.CNT DESC LIMIT :limit; ",
{"limit": limit});
final output = <String>[];
for (final rs in res) {
for (final row in rs.rows) {
output.add(row.colByName(_tagName) as String);
// Firt return what we have in local
final db = await ChatRoomRepository().getLocalDb();
final List<Map<String, dynamic>> maps = await db.rawQuery('''
SELECT t1.NAME AS $columnTagsName FROM
(SELECT COUNT($columnTagsName) AS CNT, $columnTagsName AS NAME FROM $tableTags GROUP BY $columnTagsName) AS t1
ORDER BY t1.CNT DESC LIMIT $limit;
''');

final localTags = maps.map((e) {
String tag_ = e[columnTagsName];
return tag_;
}).toList();

// Then start async task to synchronize remote to local
final remoteDB = ChatRoomRepository().getMyRemoteDb();
remoteDB.then((remoteDB_) async {
if (remoteDB_ == null) {
return;
}
}
return output;
final List<Map<String, dynamic>> maps = await db.query(tableTags,
orderBy: "$columnTagsAddedAt DESC", limit: 1);
DateTime? last;
if (maps.isNotEmpty) {
last = DateTime.parse(maps.first[columnTagsAddedAt]);
}
var sql = last == null
? "SELECT * FROM moyubie.`$tableTags` ORDER BY `$columnTagsAddedAt` DESC LIMIT 100"
: "SELECT * FROM moyubie.`$tableTags` WHERE `$columnTagsAddedAt` > '${last.toString()}' ORDER BY `$columnTagsAddedAt` DESC LIMIT 100";
final res = await remoteDB_.execute(sql);
if (res.rows.isEmpty) {
return;
}
final remoteTags = res.rows.map((e) {
var map = e.assoc();
return Tag(
name: map[columnTagsName]!,
time: DateTime.parse(map[columnTagsAddedAt]!));
}).toList();

await _addNewTags(remoteTags);
});

return localTags;
}
}

0 comments on commit 98a713a

Please sign in to comment.