diff --git a/README.md b/README.md index fc21fa5..418881a 100644 --- a/README.md +++ b/README.md @@ -66,7 +66,7 @@ Custom data extractor is a command that can be invoked with a single parameter - ## Technical information -Record in librer is the result of a single scan operation and is shown as one of many top nodes in the main tree window. Contains a directory tree with collected custom data and CRC data. It is stored as a single .dat file in librer database directory. Its internal format is optimized for security, fast initial access and maximum compression (just check :)) Every section is a python data structure serialized by [pickle](https://docs.python.org/3/library/pickle.html) and compressed separately by [bzip2](https://docs.python.org/3/library/bz2.html#module-bz2) algorithm. The record file, once saved, is never modified afterward. It can only be deleted upon request. All record files are independent of each other. +Record in librer is the result of a single scan operation and is shown as one of many top nodes in the main tree window. Contains a directory tree with collected custom data and CRC data. It is stored as a single .dat file in librer database directory. Its internal format is optimized for security, fast initial access and maximum compression (just check :)) Every section is a python data structure serialized by [pickle](https://docs.python.org/3/library/pickle.html) and compressed separately by [Zstandard](https://pypi.org/project/zstandard/) algorithm. The record file, once saved, is never modified afterward. It can only be deleted upon request or exported. All record files are independent of each other. ###### Manual build (linux): ``` diff --git a/requirements.txt b/requirements.txt index d19d15b..a5501f1 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,5 +1,5 @@ pyinstaller==6.2.0 -nuitka==1.8.6 +nuitka==1.9.1 appdirs==1.4.4 send2trash==1.8.2 zstandard==0.22.0 diff --git a/src/core.py b/src/core.py index ec62abf..c7e2b70 100644 --- a/src/core.py +++ b/src/core.py @@ -26,8 +26,10 @@ # #################################################################################### +from zstandard import ZstdCompressor +from zstandard import ZstdDecompressor + from zipfile import ZipFile -from zipfile import ZIP_BZIP2 from os import scandir from os import stat @@ -36,6 +38,7 @@ from os.path import join as path_join from os.path import abspath from os.path import normpath +from os.path import basename from platform import system as platform_system from platform import release as platform_release @@ -48,6 +51,8 @@ from re import search from sys import getsizeof +from shutil import copy as shutil_copy + from hashlib import sha1 from collections import defaultdict @@ -139,7 +144,7 @@ def test_regexp(expr): #print(i, temp_tuple) ####################################################################### -data_format_version='1.0007' +data_format_version='1.0008' class LibrerRecordHeader : def __init__(self,label='',path=''): @@ -164,13 +169,13 @@ def __init__(self,label='',path=''): self.creation_os,self.creation_host = f'{platform_system()} {platform_release()}',platform_node() ####################################################################### -class LibrerRecord : +class LibrerRecord: def __init__(self,label,path,log): self.header = LibrerRecordHeader(label,path) - self.filenames = () + #self.filenames = () self.filestructure = () - self.custom_data = () + self.custom_data = [] self.log = log self.find_results = [] @@ -187,6 +192,8 @@ def __init__(self,label,path,log): self.FILE_SIZE = 0 self.file_path = '' + self.zipinfo={'header':'?','filestructure':'?','filenames':'?','customdata':'?'} + def get_time(self): return self.header.creation_time/1000 @@ -276,8 +283,12 @@ def scan_rec(self, path, scan_like_data,file_names_set,check_dev=True,dev_call=N self.log.error('stat error:%s', e ) #size -1 <=> error, dev,in ==0 is_bind = False - scan_like_data[entry_name] = [is_dir,is_file,is_symlink,is_bind,-1,0,None] + size=-1 + mtime=0 + has_files = False + scan_like_data[entry_name] = [size,is_dir,is_file,is_symlink,is_bind,has_files,mtime] else: + dict_entry={} is_bind=False if check_dev: if dev_call: @@ -289,31 +300,33 @@ def scan_rec(self, path, scan_like_data,file_names_set,check_dev=True,dev_call=N if is_dir: if is_symlink : - dict_entry = None - size_entry = 0 + has_files = False + size = 0 elif is_bind: - dict_entry = None - size_entry = 0 + has_files = False + size = 0 else: - dict_entry={} - size_entry = self_scan_rec(path_join_loc(path,entry_name),dict_entry,file_names_set,check_dev,dev) + size = self_scan_rec(path_join_loc(path,entry_name),dict_entry,file_names_set,check_dev,dev) + has_files = bool(size) - local_folder_size_with_subtree += size_entry + local_folder_size_with_subtree += size local_folder_folders_count += 1 else: if is_symlink : - dict_entry = None - size_entry = 0 + has_files = False + size = 0 else: - dict_entry = None - size_entry = int(stat_res.st_size) + has_files = False + size = int(stat_res.st_size) - local_folder_size += size_entry + local_folder_size += size local_folder_files_count += 1 - scan_like_data[entry_name]=[is_dir,is_file,is_symlink,is_bind,size_entry,mtime_ns,dict_entry] + temp_list_ref = scan_like_data[entry_name]=[size,is_dir,is_file,is_symlink,is_bind,has_files,mtime_ns] + if has_files: + temp_list_ref.append(dict_entry) self_header = self.header self_header.sum_size += local_folder_size @@ -330,10 +343,10 @@ def scan_rec(self, path, scan_like_data,file_names_set,check_dev=True,dev_call=N def get_file_name(self,nr): return self.FILE_NAMES[nr] - def scan(self,db_dir,cde_list,check_dev=True): + def scan(self,cde_list,check_dev=True): self.info_line = 'Scanning filesystem' self.abort_action=False - self.db_dir = db_dir + self.header.sum_size = 0 self.ext_statistics=defaultdict(int) @@ -364,38 +377,25 @@ def scan(self,db_dir,cde_list,check_dev=True): #for ext,stat in sorted(self.ext_statistics.items(),key = lambda x : x[1],reverse=True): # print(ext,stat) - #def data_group_test_rec(self,scan_like_data): - # list_flat = [] - # for entry_name,items_list in scan_like_data.items(): - - # list_flat.append(entry_name) - - # (is_dir,is_file,is_symlink,is_bind,size,mtime,sub_dict) = items_list[0:7] - - # if is_dir: - # if not is_symlink and not is_bind: - # list_flat.extend(self.data_group_test_rec(sub_dict)) - - # return list_flat - - def prepare_custom_data_pool_rec(self,scan_like_data,parent_path): scan_path = self.header.scan_path self_prepare_custom_data_pool_rec = self.prepare_custom_data_pool_rec - self_db_cde_list = self.cde_list + self_cde_list = self.cde_list self_custom_data_pool = self.custom_data_pool for entry_name,items_list in scan_like_data.items(): if self.abort_action: break try: - is_dir,is_file,is_symlink,is_bind,size,mtime,sub_dict = items_list + #is_dir,is_file,is_symlink,is_bind,has_files,size,mtime = items_list + size,is_dir,is_file,is_symlink,is_bind,has_files,mtime = items_list[0:7] subpath_list = parent_path.copy() + [entry_name] if not is_symlink and not is_bind: if is_dir: - self_prepare_custom_data_pool_rec(sub_dict,subpath_list) + if has_files: + self_prepare_custom_data_pool_rec(items_list[7],subpath_list) else: subpath=sep.join(subpath_list) ############################ @@ -405,7 +405,7 @@ def prepare_custom_data_pool_rec(self,scan_like_data,parent_path): matched = False rule_nr=-1 - for expressions,use_smin,smin_int,use_smax,smax_int,executable,timeout,crc in self_db_cde_list: + for expressions,use_smin,smin_int,use_smax,smax_int,executable,timeout,crc in self_cde_list: if self.abort_action: break if matched: @@ -435,14 +435,13 @@ def prepare_custom_data_pool_rec(self,scan_like_data,parent_path): except Exception as e: self.log.error('prepare_custom_data_pool_rec error::%s',e ) - print('prepare_custom_data_pool_rec',e,entry_name,is_dir,is_file,is_symlink,is_bind,size,mtime) + print('prepare_custom_data_pool_rec',e,entry_name,size,is_dir,is_file,is_symlink,is_bind,has_files,mtime) def extract_custom_data(self): self_header = self.header scan_path = self_header.scan_path self.info_line = f'custom data extraction ...' - #self_db = self.db self_header.files_cde_quant = 0 self_header.files_cde_size = 0 @@ -450,51 +449,56 @@ def extract_custom_data(self): self_header.files_cde_errors_quant = 0 self_header.files_cde_quant_sum = len(self.custom_data_pool) - self_db_cde_list = self.cde_list + self_cde_list = self.cde_list exe = Executor() custom_data_helper={} - custom_data_list=[] - custom_data_index=0 - for (list_ref,subpath,rule_nr) in self.custom_data_pool.values(): + cd_index=0 + self_custom_data_append = self.custom_data.append + self_calc_crc = self.calc_crc + exe_run = exe.run + + for (scan_like_list,subpath,rule_nr) in self.custom_data_pool.values(): if self.abort_action: break - expressions,use_smin,smin_int,use_smax,smax_int,executable,timeout,crc = self_db_cde_list[rule_nr] + expressions,use_smin,smin_int,use_smax,smax_int,executable,timeout,crc = self_cde_list[rule_nr] full_file_path = normpath(abspath(sep.join([scan_path,subpath]))).replace('/',sep) - size = list_ref[4] + size = scan_like_list[0] cde_run_list = executable + [full_file_path] if crc: self.info_line_current = f'{subpath} CRC calculation ({bytes_to_str(size)})' - crc_val = self.calc_crc(full_file_path,size) - #print(crc_val) + crc_val = self_calc_crc(full_file_path,size) self.info_line_current = f'{subpath} ({bytes_to_str(size)})' - cd_ok,output = exe.run(cde_run_list,timeout) - - new_list_ref_elem = [cd_ok,output] + cd_ok,output = exe_run(cde_run_list,timeout) if not cd_ok: self_header.files_cde_errors_quant +=1 - if crc: - new_list_ref_elem.append(crc_val) + new_elem={} + new_elem['cd_ok']=cd_ok + + if output not in custom_data_helper: + custom_data_helper[output]=cd_index + new_elem['cd_index']=cd_index + cd_index+=1 - new_list_ref_elem_tuple = tuple(new_list_ref_elem) + self_custom_data_append(output) + else: + new_elem['cd_index']=custom_data_helper[output] - if new_list_ref_elem_tuple not in custom_data_helper: - custom_data_helper[new_list_ref_elem_tuple]=custom_data_index - custom_data_index+=1 - custom_data_list.append(new_list_ref_elem_tuple) + if crc: + new_elem['crc_val']=crc_val - list_ref.append( custom_data_helper[new_list_ref_elem_tuple] ) + scan_like_list.append(new_elem) self_header.files_cde_quant += 1 self_header.files_cde_size += size @@ -503,8 +507,7 @@ def extract_custom_data(self): self.info_line_current = '' del self.custom_data_pool - self.custom_data = tuple(custom_data_list) - #print('self.custom_data:',len(self.custom_data),'vs',custom_data_index,'\n',self.custom_data) + del custom_data_helper exe.end() @@ -516,6 +519,7 @@ def tupelize_rec(self,scan_like_data): self_tupelize_rec = self.tupelize_rec + self_custom_data = self.custom_data sub_list = [] for entry_name,items_list in scan_like_data.items(): @@ -528,57 +532,57 @@ def tupelize_rec(self,scan_like_data): #print('entry_name_index:',entry_name_index) try: - len_items_list = len(items_list) + #len_items_list = len(items_list) #print('items_list:',items_list,'len_items_list:',len_items_list) - (is_dir,is_file,is_symlink,is_bind,size,mtime,sub_dict) = items_list[0:7] + #(is_dir,is_file,is_symlink,is_bind,has_files,size,mtime) = items_list[0:7] + (size,is_dir,is_file,is_symlink,is_bind,has_files,mtime) = items_list[0:7] - if len_items_list==7: - has_cd = False - has_files = True if bool(sub_dict) else False - - cd_ok=False - output = None - - elif len_items_list==8: - cd_nr = items_list[7] - cd = self.custom_data[cd_nr] - #print('cd:',cd) + elem_index = 7 + if has_files: + sub_dict = items_list[elem_index] + elem_index+=1 - cd_len = len(cd) - if cd_len==2: - cd_ok,output = cd - elif cd_len==3: - cd_ok,output,crc = cd + try: + info_dict = items_list[elem_index] + except: + has_cd = False + cd_ok = False + has_crc = False + else: + if 'cd_ok' in info_dict: + cd_ok = info_dict['cd_ok'] + cd_index = info_dict['cd_index'] + has_cd = True else: - print('lewizna crc:') - continue + cd_ok = False + has_cd = False - has_cd = True - - has_files = False - else: - print('lewizna:',items_list) - continue + if 'crc_val' in info_dict: + crc_val = info_dict['crc_val'] + has_crc = True + else: + has_crc = False - has_crc = False code_new = entry_LUT_encode_loc[ (is_dir,is_file,is_symlink,is_bind,has_cd,has_files,cd_ok,has_crc) ] sub_list_elem=[entry_name_index,code_new,size,mtime] - if has_cd: #only files - sub_list_elem.append( cd_nr ) - elif is_dir: - if not is_symlink and not is_bind: - sub_tuple = self_tupelize_rec(sub_dict) - sub_list_elem.append(sub_tuple) + if has_files: + sub_list_elem.append(self_tupelize_rec(sub_dict)) + else: + if has_cd: #only files + sub_list_elem.append( cd_index ) + if has_crc: #only files + sub_list_elem.append( crc_val ) + + sub_list.append( tuple(sub_list_elem) ) except Exception as e: self.log.error('tupelize_rec error::%s',e ) - print('tupelize_rec error:',e,' items_list:',items_list) + print('tupelize_rec error:',e,' entry_name:',entry_name,' items_list:',items_list) - sub_list.append( tuple(sub_list_elem) ) return tuple(sorted(sub_list,key = lambda x : x[1:4])) ############################################################# @@ -597,22 +601,8 @@ def pack_data(self): code = entry_LUT_encode[ (is_dir,is_file,is_symlink,is_bind,has_cd,has_files,cd_ok,has_crc) ] self.filestructure = ('',code,size,mtime,self.tupelize_rec(self.scan_data)) - #!!!!! - #del self.FILE_NAMES_helper - - #self.info_line = 'serializing filesystem structure' - #serialized_filestructure = dumps(self.filestructure) - - #self.info_line = 'compressing filesystem structure' - #self.pack_filestructure = bz2_compress(serialized_filestructure) - - #self.info_line = 'serializing custom data' - #serialized_custom_data = dumps(self.custom_data) - - #self.info_line = 'compressing custom data' - #self.pack_custom_data = bz2_compress(serialized_custom_data) - - self.scan_data={} + del self.FILE_NAMES_helper + del self.scan_data def find_items(self, size_min,size_max, @@ -620,7 +610,6 @@ def find_items(self, self.decompress_filestructure() - dont_kind_code = self.search_kind_code_tab['dont'] regexp_kind_code = self.search_kind_code_tab['regexp'] glob_kind_code = self.search_kind_code_tab['glob'] @@ -653,6 +642,7 @@ def find_items(self, rgf_group = (regexp_kind_code,glob_kind_code,fuzzy_kind_code) + self_custom_data = self.custom_data while search_list: if self.abort_action: break @@ -663,26 +653,40 @@ def find_items(self, if self.abort_action: break - data_entry_len = len(data_entry) - if data_entry_len==5: - name_nr,code,size,mtime,fifth_field = data_entry - elif data_entry_len==4: - name_nr,code,size,mtime = data_entry - else: - print('format error:',data_entry_len,data_entry[0]) - continue + self.files_search_progress +=1 + + name_nr,code,size,mtime = data_entry[0:4] + + #data_entry_len = len(data_entry) + #if data_entry_len==5: + #elif data_entry_len==4: + # name_nr,code,size,mtime = data_entry + #else: + # print('format error:',data_entry_len,data_entry[0]) + # continue + name = file_names_loc[name_nr] is_dir,is_file,is_symlink,is_bind,has_cd,has_files,cd_ok,has_crc = entry_LUT_decode_loc[code] - sub_data = fifth_field if is_dir and has_files else None + elem_index=4 + if has_files: + sub_data = data_entry[elem_index] + elem_index+=1 + else: + sub_data = None - self.files_search_progress +=1 + if has_cd: + cd_nr = data_entry[elem_index] + elem_index+=1 + + if has_crc: + crc = data_entry[elem_index] cd_search_kind_code_is_rgf = True if cd_search_kind_code in rgf_group else False + next_level = parent_path_components + [name] if is_dir : - next_level = parent_path_components + [name] if cd_search_kind_code==dont_kind_code and not use_size: #katalog moze spelniac kryteria naazwy pliku ale nie ma rozmiaru i custom data if name_func_to_call: @@ -718,18 +722,16 @@ def find_items(self, continue elif cd_search_kind_code_is_rgf: if has_cd and cd_ok: - cd_nr = fifth_field - cd_data = self.custom_data[cd_nr] - #cd_data = fifth_field + cd_data = self_custom_data[cd_nr] else: continue if cd_func_to_call: try: - cd_txt = cd_data + #cd_txt = cd_data #self.get_cd_text(cd_data) - if not cd_func_to_call(cd_txt): + if not cd_func_to_call(cd_data): continue except Exception as e: self.log.error('find_items_rec:%s',str(e) ) @@ -760,7 +762,7 @@ def find_items_sort(self,what,reverse): print('unknown sorting',what,mod) def prepare_info(self): - self.zipinfo={'header':(0,0),'filesystem':(0,0),'filenames':(0,0),'customdata':(0,0),'crc':(0,0)} + info_list = [] try: @@ -768,9 +770,10 @@ def prepare_info(self): except Exception as e: print('prepare_info stat error:%s' % e ) else: + zip_file_info = {'header':0,'filestructure':0,'filenames':0,'customdata':0} with ZipFile(self.file_path, "r") as zip_file: for info in zip_file.infolist(): - self.zipinfo[info.filename] = (info.compress_size,info.file_size) + zip_file_info[info.filename] = info.compress_size self_header = self.header @@ -789,56 +792,70 @@ def prepare_info(self): info_list.append(f'') info_list.append( ' compressed decompressed') - header_sizes = self.zipinfo['header'] - info_list.append(f'header :{bytes_to_str(header_sizes[0]).rjust(14)}{bytes_to_str(header_sizes[1]).rjust(14)}') - - filestructure_sizes = self.zipinfo['filestructure'] - info_list.append(f'filestructure :{bytes_to_str(filestructure_sizes[0]).rjust(14)}{bytes_to_str(filestructure_sizes[1]).rjust(14)}') - - filenames_sizes = self.zipinfo['filenames'] - info_list.append(f'file names :{bytes_to_str(filenames_sizes[0]).rjust(14)}{bytes_to_str(filenames_sizes[1]).rjust(14)}') - - custom_data_sizes = self.zipinfo['customdata'] - info_list.append(f'custom data :{bytes_to_str(custom_data_sizes[0]).rjust(14)}{bytes_to_str(custom_data_sizes[1]).rjust(14)}') + bytes_to_str_mod = lambda x : bytes_to_str(x) if type(x) == int else x - crc_data_sizes = self.zipinfo['crc'] - info_list.append(f'crc data :{bytes_to_str(crc_data_sizes[0]).rjust(14)}{bytes_to_str(crc_data_sizes[1]).rjust(14)}') + info_list.append(f'header :{bytes_to_str_mod(zip_file_info["header"]).rjust(14)}{bytes_to_str_mod(self.zipinfo["header"]).rjust(14)}') + info_list.append(f'filestructure :{bytes_to_str_mod(zip_file_info["filestructure"]).rjust(14)}{bytes_to_str_mod(self.zipinfo["filestructure"]).rjust(14)}') + info_list.append(f'file names :{bytes_to_str_mod(zip_file_info["filenames"]).rjust(14)}{bytes_to_str_mod(self.zipinfo["filenames"]).rjust(14)}') + info_list.append(f'custom data :{bytes_to_str_mod(zip_file_info["customdata"]).rjust(14)}{bytes_to_str_mod(self.zipinfo["customdata"]).rjust(14)}') self.txtinfo = '\n'.join(info_list) - def save(self) : - self.FILE_NAME = self.new_file_name() + def save(self,file_path=None) : + if file_path: + filename = basename(normpath(file_path)) + else: + filename = self.FILE_NAME = self.new_file_name() + self.file_path = file_path = sep.join([self.db_dir,filename]) - self.info_line = f'saving {self.FILE_NAME}' - self.file_path = file_path = sep.join([self.db_dir,self.FILE_NAME]) + self.info_line = f'saving {filename}' self.log.info('saving %s' % file_path) with ZipFile(file_path, "w") as zip_file: - zip_file.writestr('header',dumps(self.header),compress_type=ZIP_BZIP2, compresslevel=9) + cctx = ZstdCompressor(level=16,threads=-1) + + header_ser = dumps(self.header) + self.zipinfo['header'] = len(header_ser) + zip_file.writestr('header',cctx.compress(header_ser)) - self.info_line = f'saving {self.FILE_NAME} (File stucture)' - zip_file.writestr('filestructure',dumps(self.filestructure),compress_type=ZIP_BZIP2, compresslevel=9) + self.info_line = f'saving {filename} (File stucture)' + filestructure_ser = dumps(self.filestructure) + self.zipinfo['filestructure'] = len(filestructure_ser) + zip_file.writestr('filestructure',cctx.compress(filestructure_ser)) - self.info_line = f'saving {self.FILE_NAME} (File names)' - zip_file.writestr('filenames',dumps(self.FILE_NAMES),compress_type=ZIP_BZIP2, compresslevel=9) + self.info_line = f'saving {filename} (File names)' + filenames_ser = dumps(self.FILE_NAMES) + self.zipinfo['filenames'] = len(filenames_ser) + zip_file.writestr('filenames',cctx.compress(filenames_ser)) - self.info_line = f'saving {self.FILE_NAME} (Custom Data)' - zip_file.writestr('customdata',dumps(self.custom_data),compress_type=ZIP_BZIP2, compresslevel=9) + self.info_line = f'saving {filename} (Custom Data)' + custom_data_ser = dumps(self.custom_data) + self.zipinfo['customdata'] = len(custom_data_ser) + zip_file.writestr('customdata',cctx.compress(custom_data_ser)) self.prepare_info() self.info_line = '' - def load(self,db_dir,file_name): - self.log.info('loading %s' % file_name) + def load_wrap(self,db_dir,file_name): self.FILE_NAME = file_name + file_path = sep.join([db_dir,self.FILE_NAME]) + + return self.load(file_path) - self.file_path = file_path = sep.join([db_dir,self.FILE_NAME]) + def load(self,file_path): + self.file_path = file_path + file_name = basename(normpath(file_path)) + self.log.info('loading %s' % file_name) + + dctx = ZstdDecompressor() try: with ZipFile(file_path, "r") as zip_file: - self.header = loads( zip_file.read('header') ) + header_ser = dctx.decompress(zip_file.read('header')) + self.header = loads( header_ser ) + self.zipinfo['header'] = len(header_ser) self.prepare_info() @@ -856,12 +873,20 @@ def load(self,db_dir,file_name): decompressed_filestructure = False def decompress_filestructure(self): if not self.decompressed_filestructure: + dctx = ZstdDecompressor() with ZipFile(self.file_path, "r") as zip_file: - self.filestructure = loads( zip_file.read('filestructure') ) - self.FILE_NAMES = loads( zip_file.read('filenames') ) + filestructure_ser = dctx.decompress(zip_file.read('filestructure')) + self.filestructure = loads( filestructure_ser ) + self.zipinfo['filestructure'] = len(filestructure_ser) + + filenames_ser = dctx.decompress(zip_file.read('filenames')) + self.FILE_NAMES = loads(filenames_ser) + self.zipinfo['filenames'] = len(filenames_ser) self.decompressed_filestructure = True + + self.prepare_info() return True else: return False @@ -869,10 +894,17 @@ def decompress_filestructure(self): decompressed_custom_data = False def decompress_custom_data(self): if not self.decompressed_custom_data: + dctx = ZstdDecompressor() with ZipFile(self.file_path, "r") as zip_file: - self.custom_data = loads( zip_file.read('customdata') ) + + customdata_ser_comp = zip_file.read('customdata') + customdata_ser = dctx.decompress(customdata_ser_comp) + self.custom_data = loads( customdata_ser ) + self.zipinfo['customdata'] = len(customdata_ser) self.decompressed_custom_data = True + + self.prepare_info() return True else: return False @@ -913,6 +945,7 @@ def update_sorted(self): def create(self,label='',path=''): new_record = LibrerRecord(label,path,self.log) + new_record.db_dir = self.db_dir self.records.add(new_record) self.update_sorted() @@ -944,6 +977,29 @@ def read_records_pre(self): def abort(self): self.abort_action = True + def import_record(self,file_path): + new_record = self.create() + if new_record.load(file_path): #tylko dla testu + self.log.warning('import failed :%s',file_path) + self.records.remove(new_record) + return None + else: + new_file_tail = 'imported' + str(new_record.header.rid) + '.' + str(time()) + '.dat' + file_path_loc = sep.join([self.db_dir,new_file_tail]) + try: + shutil_copy(file_path,file_path_loc) + except Exception as e: + print(e) + return None + else: + new_record = self.create() + if new_record.load_wrap(self.db_dir,new_file_tail): + return None + else: + return new_record + + #self.records_to_show.append( (new_record,info_curr_quant,info_curr_size) ) + def read_records(self): self.log.info('read_records: %s',self.db_dir) self.records_to_show=[] @@ -963,7 +1019,7 @@ def read_records(self): info_curr_quant+=1 info_curr_size+=size - if new_record.load(self.db_dir,ename) : + if new_record.load_wrap(self.db_dir,ename) : self.log.warning('removing:%s',ename) self.records.remove(new_record) else: diff --git a/src/icons/cd_error_compr.png b/src/icons/cd_error_compr.png deleted file mode 100644 index 1193957..0000000 Binary files a/src/icons/cd_error_compr.png and /dev/null differ diff --git a/src/icons/cd_error_crc.png b/src/icons/cd_error_crc.png new file mode 100644 index 0000000..8568fa1 Binary files /dev/null and b/src/icons/cd_error_crc.png differ diff --git a/src/icons/cd_ok_compr.png b/src/icons/cd_ok_compr.png deleted file mode 100644 index a362d5b..0000000 Binary files a/src/icons/cd_ok_compr.png and /dev/null differ diff --git a/src/icons/cd_ok_crc.png b/src/icons/cd_ok_crc.png new file mode 100644 index 0000000..0143b08 Binary files /dev/null and b/src/icons/cd_ok_crc.png differ diff --git a/src/icons/crc.png b/src/icons/crc.png new file mode 100644 index 0000000..e3cdd66 Binary files /dev/null and b/src/icons/crc.png differ diff --git a/src/librer.py b/src/librer.py index 00e3570..71c2197 100644 --- a/src/librer.py +++ b/src/librer.py @@ -975,11 +975,27 @@ def get_license_dialog(self): return self.license_dialog + @restore_status_line + @block_actions_processing + @gui_block def record_export(self): - print('TODO record_export') + if self.current_record: + if record_file := asksaveasfilename(parent = self.main, initialfile = 'record.dat',defaultextension=".dat",filetypes=[("Dat Files","*.dat"),("All Files","*.*")]): + self.status('saving file "%s" ...' % str(record_file)) + self.current_record.save(record_file) + @restore_status_line + @block_actions_processing + @gui_block def record_import(self): - print('TODO record_import') + initialdir = self.last_dir if self.last_dir else self.cwd + if res:=askopenfilename(title='Select Record File',initialdir=initialdir,parent=self.main,filetypes=( ("dat Files","*.dat"),("All Files","*.*") ) ): + self.last_dir=dirname(res) + + self.status('importing record ...') + + if new_record := librer_core.import_record(res): + self.single_record_show(new_record) def __init__(self,cwd): self.cwd=cwd @@ -1025,10 +1041,10 @@ def __init__(self,cwd): self.ico_record = self_ico['record'] self.ico_cd_ok = self_ico['cd_ok'] + self.ico_cd_ok_crc = self_ico['cd_ok_crc'] self.ico_cd_error = self_ico['cd_error'] - - self.ico_cd_ok_compr = self_ico['cd_ok_compr'] - self.ico_cd_error_compr = self_ico['cd_error_compr'] + self.ico_cd_error_crc = self_ico['cd_error_crc'] + self.ico_crc = self_ico['crc'] self.ico_folder = self_ico['folder'] self.ico_folder_link = self_ico['folder_link'] @@ -1390,6 +1406,12 @@ def help_cascade_post(): self_main_after = self.main.after + wait_var_set = wait_var.set + wait_var_get = wait_var.get + self_main_wait_variable = self.main.wait_variable + + self_single_record_show = self.single_record_show + while read_thread_is_alive() or librer_core.records_to_show : self_progress_dialog_on_load_lab[2].configure(image=self.get_hg_ico()) @@ -1404,10 +1426,10 @@ def help_cascade_post(): self_progress_dialog_on_load_progr1var_set(100*size/records_size) self_progress_dialog_on_load_progr2var_set(100*quant/records_quant) - self.single_record_show(new_rec) + self_single_record_show(new_rec) else: - self_main_after(25,lambda : wait_var.set(not wait_var.get())) - self.main.wait_variable(wait_var) + self_main_after(25,lambda : wait_var_set(not wait_var_get())) + self_main_wait_variable(wait_var) if self.action_abort: librer_core.abort() @@ -1557,8 +1579,8 @@ def show_tooltips_tree(self,event): if has_cd: cd_nr = fifth_field - node_cd = '\nDouble click to open custom data for file.' - #if cd_data := record.custom_data[cd_nr][1]: + node_cd = ' Double click to open custom data for file.' + #if cd_data := record.custom_data[cd_nr]: # #print('a1',cd_nr,cd_data) # cd_txt = cd_data #record.get_cd_text(cd_data) @@ -1571,7 +1593,7 @@ def show_tooltips_tree(self,event): record_path = record.header.scan_path size = core_bytes_to_str(record.header.sum_size) time_info = strftime('%Y/%m/%d %H:%M:%S',localtime(record.get_time())) - self.tooltip_lab_configure(text=record.txtinfo + (f'\n\n================================================================\n{node_cd}' if node_cd else '') ) + self.tooltip_lab_configure(text=record.txtinfo + (f'\n\n=====================================================\n{node_cd}\n=====================================================' if node_cd else '') ) self.tooltip_deiconify() @@ -1708,10 +1730,10 @@ def find_next(self): def find_save_results(self): self.find_items() - if report_file := asksaveasfilename(parent = self.find_dialog.widget, initialfile = 'librer_search_report.txt',defaultextension=".txt",filetypes=[("All Files","*.*"),("Text Files","*.txt")]): - self.status('saving file "%s" ...' % str(report_file)) + if report_file_name := asksaveasfilename(parent = self.find_dialog.widget, initialfile = 'librer_search_report.txt',defaultextension=".txt",filetypes=[("All Files","*.*"),("Text Files","*.txt")]): + self.status('saving file "%s" ...' % str(report_file_name)) - with open(report_file,'w') as report_file: + with open(report_file_name,'w') as report_file: report_file_write = report_file.write #report_file_write('criteria: \n') @@ -1723,7 +1745,7 @@ def find_save_results(self): report_file_write('\n') - self.status('file saved: "%s"' % str(report_file)) + self.status('file saved: "%s"' % str(report_file_name)) def find_show_results(self): self.find_items() @@ -2671,7 +2693,7 @@ def scan(self): check_dev = self.cfg_get_bool(CFG_KEY_SINGLE_DEVICE) ############################# - scan_thread=Thread(target=lambda : new_record.scan(librer_core.db_dir,cde_list,check_dev),daemon=True) + scan_thread=Thread(target=lambda : new_record.scan(cde_list,check_dev),daemon=True) scan_thread.start() scan_thread_is_alive = scan_thread.is_alive @@ -2752,7 +2774,6 @@ def scan(self): self_progress_dialog_on_scan_progr1var_set = self_progress_dialog_on_scan_progr1var.set self_progress_dialog_on_scan_progr2var_set = self_progress_dialog_on_scan_progr2var.set - #new_record_db = new_record.db while cd_thread_is_alive(): change0 = self_progress_dialog_on_scan_update_lab_text(0,new_record.info_line) change3 = self_progress_dialog_on_scan_update_lab_text(3,'Extracted Custom Data: ' + local_core_bytes_to_str(new_record.header.files_cde_size_extracted) ) @@ -2794,19 +2815,41 @@ def scan(self): update_once=True - self.main.after(25,lambda : wait_var.set(not wait_var.get())) - self.main.wait_variable(wait_var) + self_main_after(25,lambda : wait_var_set(not wait_var_get())) + self_main_wait_variable(wait_var) cd_thread.join() - new_record.pack_data() - new_record.save() - self_progress_dialog_on_scan_update_lab_text(1,'') self_progress_dialog_on_scan_update_lab_image(2,self_ico_empty) self_progress_dialog_on_scan_update_lab_text(3,'') self_progress_dialog_on_scan_update_lab_text(4,'') + ################################## + pack_thread=Thread(target=lambda : new_record.pack_data(),daemon=True) + pack_thread.start() + pack_thread_is_alive = pack_thread.is_alive + while pack_thread_is_alive(): + change0 = self_progress_dialog_on_scan_update_lab_text(0,new_record.info_line) + self_progress_dialog_on_scan_update_lab_image(2,self.get_hg_ico()) + self_main_after(25,lambda : wait_var_set(not wait_var_get())) + self_main_wait_variable(wait_var) + pack_thread.join() + self_progress_dialog_on_scan_update_lab_image(2,self_ico_empty) + + ################################## + save_thread=Thread(target=lambda : new_record.save(),daemon=True) + save_thread.start() + save_thread_is_alive = save_thread.is_alive + while save_thread_is_alive(): + change0 = self_progress_dialog_on_scan_update_lab_text(0,new_record.info_line) + self_progress_dialog_on_scan_update_lab_image(2,self.get_hg_ico()) + self_main_after(25,lambda : wait_var_set(not wait_var_get())) + self_main_wait_variable(wait_var) + save_thread.join() + self_progress_dialog_on_scan_update_lab_image(2,self_ico_empty) + ################################## + self.single_record_show(new_record) self_progress_dialog_on_scan.hide(True) @@ -3021,83 +3064,43 @@ def open_item(self,item=None,to_the_bottom=False): if tree.tag_has(self.RECORD,item): self.access_filestructure(record) self.item_to_data[item] = record.filestructure - #if record.decompress_filestructure() and record.decompress_custom_data(): top_data_tuple = self.item_to_data[item] - len_top_data_tuple = len(top_data_tuple) - if len_top_data_tuple==4: - (top_entry_name_nr,top_code,top_size,top_mtime) = top_data_tuple - top_fifth_field=None - elif len_top_data_tuple==5: - (top_entry_name_nr,top_code,top_size,top_mtime,top_fifth_field) = top_data_tuple - else: - l_error(f'data top format incompatible:{top_data_tuple}') - print(f'data top format incompatible:{top_data_tuple}') - return - - #sub_dictionary,cd + (top_entry_name_nr,top_code,top_size,top_mtime) = top_data_tuple[0:4] top_is_dir,top_is_file,top_is_symlink,top_is_bind,top_has_cd,top_has_files,top_cd_ok,top_has_crc = entry_LUT_decode_loc[top_code] record_get_file_name = record.get_file_name #print('top_has_files:',item,top_entry_name,top_has_files,top_fifth_field) if top_has_files: - for data_tuple in top_fifth_field: - - #print('data_tuple:',data_tuple) + for data_tuple in top_data_tuple[4]: (entry_name_nr,code,size,mtime) = data_tuple[0:4] - #len_data_tuple = len(data_tuple) - #if len_data_tuple==4: - #elif len_data_tuple==5: - try: - fifth_field = data_tuple[4] - #(entry_name_nr,code,size,mtime,fifth_field) = data_tuple - except: - pass - entry_name = record_get_file_name(entry_name_nr) - #else: - # fifth_field = None - # l_error(f'data format incompatible:{data_tuple}') - # print(f'data format incompatible:{data_tuple}') - # continue - - #sub_dictionary,cd is_dir,is_file,is_symlink,is_bind,has_cd,has_files,cd_ok,has_crc = entry_LUT_decode_loc[code] sub_data_tuple = None - if has_cd: - #cd_data = fifth_field - - #cd_nr = fifth_field - #fla node'u rekordu - - #print('a2',cd_nr) - #try: - # cd_data = record.custom_data[cd_nr][1] - #except: - # cd_data = cd_nr - - sub_dictionary = False - elif has_files: + elem_index = 4 + if has_files: sub_dictionary = True - sub_data_tuple = fifth_field + sub_data_tuple = data_tuple[elem_index] + elem_index+=1 else: sub_dictionary = False - if is_dir: - image=self.ico_folder_error if size==-1 else self.ico_folder_link if is_symlink or is_bind else self.ico_folder - kind = self_DIR - else: - image= self.ico_cd_ok if cd_ok else self.ico_cd_error if has_cd else self.ico_empty - #((self.ico_cd_ok_compr if cd_ok else self.ico_cd_error_compr) if is_compressed else + if has_cd: + elem_index+=1 + + if has_crc: + pass + + kind = self_DIR if is_dir else self_FILE - kind = self_FILE + image = (self.ico_folder_error if size==-1 else self.ico_folder_link if is_symlink or is_bind else self.ico_folder) if is_dir else (self.ico_cd_ok if cd_ok else self.ico_cd_error) if has_cd and not has_crc else (self.ico_cd_ok_crc if cd_ok else self.ico_cd_error_crc) if has_cd and has_crc else self.ico_crc if has_crc else self.ico_empty if is_symlink or is_bind: tags=self_SYMLINK @@ -3108,10 +3111,10 @@ def open_item(self,item=None,to_the_bottom=False): values = (entry_name,'','0',entry_name,size,core_bytes_to_str(size),mtime,strftime('%Y/%m/%d %H:%M:%S',localtime(mtime//1000000000)),kind) sort_index = ( dir_code if is_dir else non_dir_code , sort_val_func(values[sort_index_local]) ) - new_items_values[ ( sort_index,values,entry_name,image,True if sub_dictionary else False) ] = (sub_dictionary,tags,data_tuple) + new_items_values[ ( sort_index,values,entry_name,image,True if has_files else False) ] = (has_files,tags,data_tuple) tree_insert = tree.insert - for (sort_index,values,entry_name,image,sub_dictionary_bool),(sub_dictionary,tags,data_tuple) in sorted(new_items_values.items(),key = lambda x : x[0][0],reverse=reverse) : + for (sort_index,values,entry_name,image,sub_dictionary_bool),(has_files,tags,data_tuple) in sorted(new_items_values.items(),key = lambda x : x[0][0],reverse=reverse) : new_item=tree_insert(item,'end',iid=None,values=values,open=False,text=entry_name,image=image,tags=tags) self.item_to_data[new_item] = data_tuple if sub_dictionary_bool: @@ -3128,17 +3131,11 @@ def open_item(self,item=None,to_the_bottom=False): @gui_block @logwrapper def single_record_show(self,record): - #record_db = record.db - - #self.status(f'loading {record_db.label} ...') - size=record.header.sum_size - #print('R size:',size) - #print(record.label) #('data','record','opened','path','size','size_h','ctime','ctime_h','kind') values = (record.header.label,record.header.label,0,record.header.scan_path,size,core.bytes_to_str(size),record.header.creation_time,strftime('%Y/%m/%d %H:%M:%S',localtime(record.get_time())),self.RECORD) - #print('insert:',values) + record_item=self.tree.insert('','end',iid=None,values=values,open=False,text=record.header.label,image=self.ico_record,tags=self.RECORD) self.tree.insert(record_item,'end',text='dummy') #dummy_sub_item @@ -3269,24 +3266,21 @@ def tree_action(self,item): # self.open_item(item) else: try: - #code,size,mtime,sub_dictionary - tuple_len = len(self.item_to_data[item]) - if tuple_len==5: - (entry_name,code,size,mtime,fifth_field) = self.item_to_data[item] + data_tuple = self.item_to_data[item] + (entry_name,code,size,mtime) = data_tuple[0:4] + + is_dir,is_file,is_symlink,is_bind,has_cd,has_files,cd_ok,has_crc = core.entry_LUT_decode[code] - is_dir,is_file,is_symlink,is_bind,has_cd,has_files,cd_ok,has_crc = core.entry_LUT_decode[code] + if has_cd: #wiec nie has_files + cd_index = data_tuple[4] - if has_cd : - cd_nr = fifth_field - #print('a3',cd_nr) - self.access_custom_data(record) + self.access_custom_data(record) - if cd_data := record.custom_data[cd_nr][1]: - cd_txt = cd_data - #record.get_cd_text(cd_data,is_compressed) + if cd_data := record.custom_data[cd_index]: + cd_txt = cd_data - self.get_text_info_dialog().show('Custom Data',cd_txt) - return + self.get_text_info_dialog().show('Custom Data',cd_txt) + return self.info_dialog_on_main.show('Information','No Custom data.') except Exception as e: