From 9a948f4653d05c0f4f27b3c7be2167542e3c0630 Mon Sep 17 00:00:00 2001 From: Marc Jamot Date: Tue, 29 Sep 2015 11:27:51 +0200 Subject: [PATCH] Added a backup tool to import/export store data --- api/restapi.py | 27 +++++++++ backuptool.py | 160 +++++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 187 insertions(+) create mode 100644 backuptool.py diff --git a/api/restapi.py b/api/restapi.py index a7c0b8c4..5a5067e8 100644 --- a/api/restapi.py +++ b/api/restapi.py @@ -1,4 +1,5 @@ __author__ = 'chris' +import backuptool import json import os from txrestapi.resource import APIResource @@ -601,3 +602,29 @@ def upload_image(self, request): request.write(json.dumps({"success": False, "reason": e.message}, indent=4)) request.finish() return server.NOT_DONE_YET + + @POST('^/api/v1/backup_files') + def backup_files(self, request): + """Archives OpenBazaar files in a single tar archive.""" + output = request.args["output"][0] + return backuptool.backupfiles(output) + + @POST('^/api/v1/export_database') + def export_database(self, request): + """Exports given tables to the OpenBazaar folder.""" + tables_and_columns = request.args["tables_and_columns"][0] + remove_previous = request.args["remove_previous"][0] + return backuptool.exportdatabase(tables_and_columns, remove_previous) + + @POST('^/api/v1/restore_files') + def restore_files(self, request): + """Restores files of given archive to OpenBazaar folder.""" + input_file = request.args["input_file"][0] + rem_db_files = request.args["rem_db_files"][0] + return backuptool.restorefiles(input_file, rem_db_files) + + @POST('^/api/v1/import_database') + def import_database(self, request): + """Imports given CSV file to the database.""" + remove_previous = request.args["remove_previous"][0] + return backuptool.importdatabase(remove_previous) diff --git a/backuptool.py b/backuptool.py new file mode 100644 index 00000000..75858d0f --- /dev/null +++ b/backuptool.py @@ -0,0 +1,160 @@ +"""Import and export data for the OpenBazaar server.""" +__author__ = 'marc' +from constants import DATA_FOLDER +import csv +import db.datastore as db +import errno +import os +import re +import shutil +import sqlite3 as lite +import tarfile +import time + +_TABLES = { + 'hashmap': ['hash', 'filepath'], + 'profile': ['id', 'serializedUserInfo'], + 'listings': ['id', 'serializedListings'], + 'keys': ['type', 'privkey', 'pubkey'], + 'followers': ['id', 'serializedFollowers'], + 'following': ['id', 'serializedFollowing'], + 'messages': [ + 'guid', 'handle', 'signed_pubkey', 'encryption_pubkey', 'subject', + 'message_type', 'message', 'timestamp', 'avatar_hash', 'signature', + 'outgoing' + ], + 'notifications': ['guid', 'handle', 'message', 'timestamp', 'avatar_hash'], + 'vendors': ['guid', 'ip', 'port', 'signedPubkey'], + 'moderators': [ + 'guid', 'signedPubkey', 'encryptionKey', 'encryptionSignature', + 'bitcoinKey', 'bitcoinSignature', 'handle' + ], + 'purchases': [ + 'id', 'title', 'timestamp', 'btc', 'address', 'status', 'thumbnail', + 'seller', 'proofSig' + ], + 'sales': [ + 'id', 'title', 'timestamp', 'btc', 'address', 'status', 'thumbnail', + 'seller' + ], + 'dht': ['keyword', 'id', 'value', 'birthday'] +} + +def _getdatabase(): + """Retrieves the OpenBazaar database file.""" + database = db.Database() + return database.DATABASE + +def silentremove(filename): + """Silently removes a file if it exists.""" + try: + os.remove(filename) + except OSError as err: + if err.errno != errno.ENOENT: # ENOENT: no such file or directory + raise + +def backupfiles(output=None): + """Archives OpenBazaar files in a single tar archive.""" + os.chdir(DATA_FOLDER) + + # Archive files + files = os.listdir(DATA_FOLDER) + if not output: + output = 'backup_{0}.tar.gz'.format(time.strftime('%Y-%m-%d')) + silentremove(output) + with tarfile.open(output, 'w:gz') as tar: + for fil in files: + tar.add(fil) + tar.close() + return True + + +def restorefiles(input_file, remove_previous_database_files=False): + """Restores files of given archive to OpenBazaar folder.""" + if not input_file: + return 'Input path is needed' + os.chdir(DATA_FOLDER) + + # Remove existing database files if any + if remove_previous_database_files and os.path.exists('backup'): + shutil.rmtree('backup') + + # Unarchive files + with tarfile.open(input_file, 'r:gz') as tar: + tar.extractall() + + return True + +def _exportdatabase_tocsv(tables_and_columns): + """Reads the database for all given tables and stores them as CSV files.""" + db_file = _getdatabase() + with lite.connect(db_file) as db_connection: + db_connection.text_factory = str + cursor = db_connection.cursor() + for table in tables_and_columns: + table_name = table[0] + table_columns = ', '.join(table[1]) + query = "SELECT {0} FROM {1}".format(table_columns, table_name) + data = cursor.execute(query) + file_name = 'table_{0}.csv'.format(table_name) + file_path = os.path.join('backup', file_name) + with open(file_path, 'wb') as fil: + writer = csv.writer(fil) + writer.writerow(table[1]) + writer.writerows(data) + return True + +def exportdatabase(table_list, remove_previous=False): + """Exports given tables to the OpenBazaar folder.""" + os.chdir(DATA_FOLDER) + + # Parse table list + table_list = table_list.replace(' ', '').split(',') + tables_and_columns = [] + for table in table_list: + if table in _TABLES: + tables_and_columns.append((table, _TABLES[table])) + else: + return 'ERROR, Table not found: {0}'.format(table) + + # Remove existing database files and re-make them + if remove_previous and os.path.exists('backup'): + shutil.rmtree('backup') + if not os.path.exists('backup'): + os.makedirs('backup') + return _exportdatabase_tocsv(tables_and_columns) + +def _importcsv_totable(file_name, delete_data_first=False): + """Imports given CSV file to the database.""" + table_name = re.search(r'table_(\w+).csv', file_name).group(1) + db_file = _getdatabase() + with lite.connect(db_file) as db_connection: + db_connection.text_factory = str + cursor = db_connection.cursor() + if delete_data_first: + cursor.execute('DELETE FROM {0}'.format(table_name)) + with open(file_name, 'rb') as fil: + reader = csv.reader(fil) + header = True + for row in reader: + if header: + header = False + columns = ', '.join(['?' for _ in row]) + insertsql = 'INSERT INTO {0} VALUES ({1})'.format( + table_name, columns) + rowlen = len(row) + else: + if len(row) == rowlen: + cursor.execute(insertsql, row) + +def importdatabase(delete_previous_data=False): + """Imports table files from the OpenBazaar folder.""" + os.chdir(DATA_FOLDER) + + # Restore database files to the database + result = True + if os.path.exists('backup'): + files = ['backup/{0}'.format(fil) for fil in os.listdir('backup')] + for fil in files: + result = result and _importcsv_totable(fil, delete_previous_data) + return result