This repository has been archived by the owner on May 16, 2019. It is now read-only.
-
Notifications
You must be signed in to change notification settings - Fork 173
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Added a backup tool to import/export store data
- Loading branch information
Showing
2 changed files
with
187 additions
and
0 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,160 @@ | ||
"""Import and export data for the OpenBazaar server.""" | ||
__author__ = 'marc' | ||
from constants import DATA_FOLDER | ||
import csv | ||
import db.datastore as db | ||
import errno | ||
import os | ||
import re | ||
import shutil | ||
import sqlite3 as lite | ||
import tarfile | ||
import time | ||
|
||
_TABLES = { | ||
'hashmap': ['hash', 'filepath'], | ||
'profile': ['id', 'serializedUserInfo'], | ||
'listings': ['id', 'serializedListings'], | ||
'keys': ['type', 'privkey', 'pubkey'], | ||
'followers': ['id', 'serializedFollowers'], | ||
'following': ['id', 'serializedFollowing'], | ||
'messages': [ | ||
'guid', 'handle', 'signed_pubkey', 'encryption_pubkey', 'subject', | ||
'message_type', 'message', 'timestamp', 'avatar_hash', 'signature', | ||
'outgoing' | ||
], | ||
'notifications': ['guid', 'handle', 'message', 'timestamp', 'avatar_hash'], | ||
'vendors': ['guid', 'ip', 'port', 'signedPubkey'], | ||
'moderators': [ | ||
'guid', 'signedPubkey', 'encryptionKey', 'encryptionSignature', | ||
'bitcoinKey', 'bitcoinSignature', 'handle' | ||
], | ||
'purchases': [ | ||
'id', 'title', 'timestamp', 'btc', 'address', 'status', 'thumbnail', | ||
'seller', 'proofSig' | ||
], | ||
'sales': [ | ||
'id', 'title', 'timestamp', 'btc', 'address', 'status', 'thumbnail', | ||
'seller' | ||
], | ||
'dht': ['keyword', 'id', 'value', 'birthday'] | ||
} | ||
|
||
def _getdatabase(): | ||
"""Retrieves the OpenBazaar database file.""" | ||
database = db.Database() | ||
return database.DATABASE | ||
|
||
def silentremove(filename): | ||
"""Silently removes a file if it exists.""" | ||
try: | ||
os.remove(filename) | ||
except OSError as err: | ||
if err.errno != errno.ENOENT: # ENOENT: no such file or directory | ||
raise | ||
|
||
def backupfiles(output=None): | ||
"""Archives OpenBazaar files in a single tar archive.""" | ||
os.chdir(DATA_FOLDER) | ||
|
||
# Archive files | ||
files = os.listdir(DATA_FOLDER) | ||
if not output: | ||
output = 'backup_{0}.tar.gz'.format(time.strftime('%Y-%m-%d')) | ||
silentremove(output) | ||
with tarfile.open(output, 'w:gz') as tar: | ||
for fil in files: | ||
tar.add(fil) | ||
tar.close() | ||
return True | ||
|
||
|
||
def restorefiles(input_file, remove_previous_database_files=False): | ||
"""Restores files of given archive to OpenBazaar folder.""" | ||
if not input_file: | ||
return 'Input path is needed' | ||
os.chdir(DATA_FOLDER) | ||
|
||
# Remove existing database files if any | ||
if remove_previous_database_files and os.path.exists('backup'): | ||
shutil.rmtree('backup') | ||
|
||
# Unarchive files | ||
with tarfile.open(input_file, 'r:gz') as tar: | ||
tar.extractall() | ||
|
||
return True | ||
|
||
def _exportdatabase_tocsv(tables_and_columns): | ||
"""Reads the database for all given tables and stores them as CSV files.""" | ||
db_file = _getdatabase() | ||
with lite.connect(db_file) as db_connection: | ||
db_connection.text_factory = str | ||
cursor = db_connection.cursor() | ||
for table in tables_and_columns: | ||
table_name = table[0] | ||
table_columns = ', '.join(table[1]) | ||
query = "SELECT {0} FROM {1}".format(table_columns, table_name) | ||
data = cursor.execute(query) | ||
file_name = 'table_{0}.csv'.format(table_name) | ||
file_path = os.path.join('backup', file_name) | ||
with open(file_path, 'wb') as fil: | ||
writer = csv.writer(fil) | ||
writer.writerow(table[1]) | ||
writer.writerows(data) | ||
return True | ||
|
||
def exportdatabase(table_list, remove_previous=False): | ||
"""Exports given tables to the OpenBazaar folder.""" | ||
os.chdir(DATA_FOLDER) | ||
|
||
# Parse table list | ||
table_list = table_list.replace(' ', '').split(',') | ||
tables_and_columns = [] | ||
for table in table_list: | ||
if table in _TABLES: | ||
tables_and_columns.append((table, _TABLES[table])) | ||
else: | ||
return 'ERROR, Table not found: {0}'.format(table) | ||
|
||
# Remove existing database files and re-make them | ||
if remove_previous and os.path.exists('backup'): | ||
shutil.rmtree('backup') | ||
if not os.path.exists('backup'): | ||
os.makedirs('backup') | ||
return _exportdatabase_tocsv(tables_and_columns) | ||
|
||
def _importcsv_totable(file_name, delete_data_first=False): | ||
"""Imports given CSV file to the database.""" | ||
table_name = re.search(r'table_(\w+).csv', file_name).group(1) | ||
db_file = _getdatabase() | ||
with lite.connect(db_file) as db_connection: | ||
db_connection.text_factory = str | ||
cursor = db_connection.cursor() | ||
if delete_data_first: | ||
cursor.execute('DELETE FROM {0}'.format(table_name)) | ||
with open(file_name, 'rb') as fil: | ||
reader = csv.reader(fil) | ||
header = True | ||
for row in reader: | ||
if header: | ||
header = False | ||
columns = ', '.join(['?' for _ in row]) | ||
insertsql = 'INSERT INTO {0} VALUES ({1})'.format( | ||
table_name, columns) | ||
rowlen = len(row) | ||
else: | ||
if len(row) == rowlen: | ||
cursor.execute(insertsql, row) | ||
|
||
def importdatabase(delete_previous_data=False): | ||
"""Imports table files from the OpenBazaar folder.""" | ||
os.chdir(DATA_FOLDER) | ||
|
||
# Restore database files to the database | ||
result = True | ||
if os.path.exists('backup'): | ||
files = ['backup/{0}'.format(fil) for fil in os.listdir('backup')] | ||
for fil in files: | ||
result = result and _importcsv_totable(fil, delete_previous_data) | ||
return result |