From 3e211a609a86fd8baaa9b30ffc32598aa6af54d5 Mon Sep 17 00:00:00 2001 From: Kameron Kenny <1267885+kkenny@users.noreply.github.com> Date: Sun, 6 Oct 2024 15:46:11 -0400 Subject: [PATCH] checksum stuff --- BitMover.ui | 7 +- BitMover_ui.py | 313 ++++++++++++++++++++++++++++------- _BitMover_MainWindow.py | 2 +- _ComparisonDialog.ui | 22 +-- _Window_comparison_dialog.py | 66 ++++---- _dialog_checksum_progress.py | 44 +++-- _dialog_find_files.py | 4 +- _dialog_import.py | 21 ++- _file_stuff.py | 11 +- _find_files.py | 5 +- _hashing.py | 36 ++-- _media_file.py | 4 +- _preview.py | 6 +- _thread_my_stuff.py | 6 + _verify_file_checksum.py | 61 +++---- scripts/gen_gui_py.sh | 8 +- 16 files changed, 411 insertions(+), 205 deletions(-) diff --git a/BitMover.ui b/BitMover.ui index 60cbb46..bffe7aa 100644 --- a/BitMover.ui +++ b/BitMover.ui @@ -14,7 +14,8 @@ BitMover - + + .. true @@ -95,7 +96,7 @@ 910 610 - 311 + 541 211 @@ -344,7 +345,7 @@ 41 - + diff --git a/BitMover_ui.py b/BitMover_ui.py index 97ae17a..e31a11d 100755 --- a/BitMover_ui.py +++ b/BitMover_ui.py @@ -11,14 +11,15 @@ from PyQt6.QtWidgets import QMainWindow, QApplication, QFileDialog from _BitMover_MainWindow import Ui_MainWindow from _configure import CONFIG_FILE, Configure from _find_files import FindFiles -from _find_files_dialog import FindProgress -from _import_dialog import DialogImport +from _dialog_find_files import FindProgress +from _dialog_import import DialogImport +from _dialog_checksum_progress import DialogChecksumProgress # from _media_import import MediaImporter from _preview import MediaPreview from _thread_my_stuff import Worker from _media_file import MediaFile -from _file_stuff import path_exists,is_dir,is_file,create_folder,cmp_files -from _hashing import hash_path +from _file_stuff import path_exists,is_file,create_folder,cmp_hashes +from _verify_file_checksum import FileHash basedir = os.path.dirname(__file__) @@ -31,23 +32,23 @@ class MainWindow(QMainWindow, Ui_MainWindow): self.setupUi(self) self.setWindowTitle("BitMover") self.setWindowIcon(QIcon(os.path.join(basedir,'assets', 'forklift.ico'))) - self.threadpool = QThreadPool() - self.config = None - self.src_dir = None - self.dst_dir = None - self.verify_checksum = None - self.cleanup_files = None - self.store_originals = None - self.file_types = None - self.source_path_hash = None - self.search_types = None - self.chunk_size = (16 * 1024) * 1 + self.threadpool = QThreadPool() + self.config = None + self.src_dir = None + self.dst_dir = None + self.verify_checksum = None + self.cleanup_files = None + self.store_originals = None + self.file_types = None + self.source_path_hash = None + self.search_types = None + self.chunk_size = (16 * 1024) * 1 self.load_config() - self.destination_original_path = None - self.path_file_source = None - self.path_file_destination = None - self.path_file_destination_original = None + self.path_file_source = None + self.path_file_destination = None + self.path_file_destination_original = None + self.checksum_fph = None # File Stuff self.total_files = 0 @@ -58,6 +59,7 @@ class MainWindow(QMainWindow, Ui_MainWindow): self.imp_dialog = DialogImport() self.find_files_dialog = FindProgress() + self.checksum_progress_dialog = DialogChecksumProgress() self.widgets_config() @@ -131,7 +133,7 @@ class MainWindow(QMainWindow, Ui_MainWindow): self.lineEdit_dst_dir.setText(self.dst_dir) def get_source_path_hash(self,f): - self.source_path_hash = hash_path(f) + self.source_path_hash = FileHash(f).path_hash return self.source_path_hash def verify_checksum_changed(self): @@ -407,8 +409,10 @@ class MainWindow(QMainWindow, Ui_MainWindow): current_file_progress_callback, imported_file_count_callback, found_file_callback, - total_file_count_callback - ): + total_file_count_callback, + checksum_file_callback, + checksum_progress_callback, + checksum_dialog_open_callback): """ Copy Files. """ count = int(0) for line in range(self.file_list.count()): @@ -423,29 +427,37 @@ class MainWindow(QMainWindow, Ui_MainWindow): self.files[sph]['name']) self.path_file_destination = path.join(self.dst_dir, self.files[sph]['name']) - self.destination_original_path = path.join(self.dst_dir, - self.files[sph]['folders']['destination_original']) - self.path_file_destination_original = path.join(self.dst_dir, - self.files[sph]['folders']['destination_original'], - self.files[sph]['name']) self.imp_dialog.set_importing_file(self.path_file_source) self.copy_a_file( sph, + self.path_file_destination, progress_callback, current_file_progress_callback, imported_file_count_callback, found_file_callback, - total_file_count_callback - ) + total_file_count_callback, + checksum_file_callback, + checksum_progress_callback, + checksum_dialog_open_callback) + + print(f'check_store_original({sph}): {self.check_store_original(sph)}') if self.check_store_original(sph) is True: + self.path_file_destination_original = path.join(self.dst_dir, + self.files[sph]['folders']['destination_original'], + self.files[sph]['name']) + self.copy_a_file(sph, + self.path_file_destination_original, progress_callback, current_file_progress_callback, imported_file_count_callback, found_file_callback, - total_file_count_callback) + total_file_count_callback, + checksum_file_callback, + checksum_progress_callback, + checksum_dialog_open_callback) count += 1 @@ -455,37 +467,48 @@ class MainWindow(QMainWindow, Ui_MainWindow): def copy_a_file(self, fph, + target, progress_callback, current_file_progress_callback, imported_file_count_callback, found_file_callback, - total_file_count_callback): + total_file_count_callback, + checksum_file_callback, + checksum_progress_callback, + checksum_dialog_open_callback): size = path.getsize(self.path_file_source) - create_folder(self.dst_dir) + target_dir = os.path.dirname(target) + create_folder(target_dir) - self.check_duplicate(fph) + dup_check = self.check_duplicate(fph,target,target_dir, + progress_callback, + current_file_progress_callback, + imported_file_count_callback, + found_file_callback, + total_file_count_callback, + checksum_file_callback, + checksum_progress_callback, + checksum_dialog_open_callback) + if dup_check is False: + if self.is_video(fph): + self.chunk_size = (1024 * 1024) * 5 + else: + self.chunk_size = (16 * 1024) * 1 - if self.is_video(fph): - self.chunk_size = (1024 * 1024) * 5 - else: - self.chunk_size = (16 * 1024) * 1 - - with open(self.path_file_source, 'rb') as fs: - with open(self.path_file_destination, 'wb') as fd: - while True: - chunk = fs.read(self.chunk_size) - if not chunk: - break - fd.write(chunk) - dst_size = path.getsize(self.path_file_destination) - current_file_progress_callback.emit(round((dst_size / size) * 100, 1)) + with open(self.path_file_source, 'rb') as fs: + with open(target, 'wb') as fd: + while True: + chunk = fs.read(self.chunk_size) + if not chunk: + break + fd.write(chunk) + dst_size = path.getsize(target) + current_file_progress_callback.emit(round((dst_size / size) * 100, 1)) def check_store_original(self,sph): if self.config['store_originals'] is True: if self.is_image(sph): - self.dst_dir = self.destination_original_path - self.path_file_destination = self.path_file_destination_original r = True else: r = False @@ -494,15 +517,103 @@ class MainWindow(QMainWindow, Ui_MainWindow): return r - def check_duplicate(self,sph): - if path_exists(self.path_file_destination): - check_match = cmp_files(self.path_file_source, self.path_file_destination) + def get_checksum(self,_pf, + progress_callback, + current_file_progress_callback, + imported_file_count_callback, + found_file_callback, + total_file_count_callback, + checksum_file_callback, + checksum_progress_callback, + checksum_dialog_open_callback): + + sph = FileHash(_pf).path_hash + self.path_file_source = os.path.join(self.files[sph]['folders']['source_path'], + self.files[sph]['name']) + self.path_file_destination = os.path.join(self.files[sph]['folders']['destination'], + self.files[sph]['name']) + if self.store_originals is True: + self.path_file_destination_original = os.path.join( + self.files[sph]['folders']['destination_original'], + self.files[sph]['name']) + + + def set_checksum(_pf): + print(f'set_checksum, _pf: {_pf}') + if path_exists(_pf) and is_file(_pf): + + + self.files[sph]['xx_checksum'][_pf] = FileHash(_pf).get_hash(_pf, + progress_callback, + current_file_progress_callback, + imported_file_count_callback, + found_file_callback, + total_file_count_callback, + checksum_file_callback, + checksum_progress_callback, + checksum_dialog_open_callback) + else: + print(f"set_checksum: {_pf} doesn't exist yet or isn't a real file.") + + if self.files[sph].get('xx_checksum') is None: + self.files[sph]['xx_checksum'] = {} + set_checksum(self.path_file_source) + set_checksum(self.path_file_destination) + if self.store_originals is True: + set_checksum(self.path_file_destination_original) + else: + if self.files[sph]['xx_checksum'].get(self.path_file_source) is None: + set_checksum(self.path_file_source) + if self.files[sph]['xx_checksum'].get(self.path_file_destination) is None: + set_checksum(self.path_file_destination) + if self.store_originals is True: + if self.files[sph]['xx_checksum'].get(self.path_file_destination_original) is None: + set_checksum(self.path_file_destination_original) + + # self.checksum_progress_dialog.close_dialog() + + def check_duplicate(self, + sph, + target, + target_dir, + progress_callback, + current_file_progress_callback, + imported_file_count_callback, + found_file_callback, + total_file_count_callback, + checksum_file_callback, + checksum_progress_callback, + checksum_dialog_open_callback): + if path_exists(target): + print(f'Path Exists: path_file_source: {self.path_file_source}\ntarget: {target}') + self.get_checksum(self.path_file_source, + progress_callback, + current_file_progress_callback, + imported_file_count_callback, + found_file_callback, + total_file_count_callback, + checksum_file_callback, + checksum_progress_callback, + checksum_dialog_open_callback) + + hash1 = self.files[sph]['xx_checksum'][self.path_file_source] + hash2 = self.files[sph]['xx_checksum'][target] + check_match = cmp_hashes(hash1,hash2) if check_match is False: print(f'\nFound duplicate for {self.path_file_source}, renaming destination with hash appended.') base, extension = path.splitext(self.files[sph]['name']) - f_xxhash = xx_hash(self.path_file_destination) - file_name_hash = base + '_' + f_xxhash + extension - rename(self.path_file_destination, path.join(self.dst_dir, file_name_hash)) + file_name_hash = base + '_' + hash2 + extension + rename(target, path.join(target_dir, file_name_hash)) + r = False # This is false because we move the original conflict out of the way. + else: + print(f"\n########################\nFound duplicate for: {self.path_file_source}\n" + f"at: {target}... \nHashes MATCH\n" + f"########################") + r = True + else: + # No duplicate + r = False + return r # END from _media_import.py @@ -533,17 +644,99 @@ class MainWindow(QMainWindow, Ui_MainWindow): self.thread_complete) worker.signals.finished.connect( self.worker_thread_done) + worker.signals.finished.connect( + self.compare_imported_checksums) + + worker.signals.checksum_dialog_open.connect( + self.checksum_progress_dialog.open_dialog) + worker.signals.checksum_file.connect( + self.checksum_progress_dialog.set_file) + worker.signals.checksum_progress.connect( + self.checksum_progress_dialog.set_progress) + worker.signals.finished.connect( + self.checksum_progress_dialog.close_dialog) # Execute self.threadpool.start(worker) - def verify_checksum(self): - # fh_match = FileHash() - print(f'verify_checksum,self.config: {self.config}') + def compare_imported_checksums(self): + # if self.config['verify_checksum'] is True: + # self.checksum_fph = fph + # Initialize Widgets + self.checksum_progress_dialog.set_progress(0) + worker = Worker(self.t_compare_imported_checksums) + worker.signals.started.connect( + self.worker_thread_started) + worker.signals.started.connect( + self.checksum_progress_dialog.open_dialog) + worker.signals.found_file.connect( + self.checksum_progress_dialog.set_file) + worker.signals.progress.connect( + self.checksum_progress_dialog.set_progress) + worker.signals.finished.connect( + self.thread_complete) + worker.signals.finished.connect( + self.worker_thread_done) + worker.signals.finished.connect( + self.checksum_progress_dialog.close_dialog) + + + worker.signals.checksum_dialog_open.connect( + self.checksum_progress_dialog.open_dialog) + worker.signals.checksum_file.connect( + self.checksum_progress_dialog.set_file) + worker.signals.checksum_progress.connect( + self.checksum_progress_dialog.set_progress) + worker.signals.finished.connect( + self.checksum_progress_dialog.close_dialog) + + # Execute + self.threadpool.start(worker) + + def t_compare_imported_checksums(self, + progress_callback, + current_file_progress_callback, + imported_file_count_callback, + found_file_callback, + total_file_count_callback, + checksum_file_callback, + checksum_progress_callback, + checksum_dialog_open_callback): + + for file in self.files: + _pf = os.path.join(self.files[file]['folders']['source_path'], + self.files[file]['name']) + print(f"source_path: {self.files[file]['folders']['source_path']}") + print(f"name: {self.files[file]['name']}") + print(f'_pf: {_pf}') + self.get_checksum(_pf, + progress_callback, + current_file_progress_callback, + imported_file_count_callback, + found_file_callback, + total_file_count_callback, + checksum_file_callback, + checksum_progress_callback, + checksum_dialog_open_callback) + i = 0 + c = {} + for checksum in self.files[file]['xx_checksum']: + c[i] = self.files[file]['xx_checksum'][checksum] + if i > 0: + p = i - 1 + if c[i] == c[p]: + self.files[file]['checksum_match'] = True + print(f'Checksums match for: {self.files[file]["name"]}') + else: + self.files[file]['checksum_match'] = False + print(f'Checksum failed for: {self.files[file]["name"]}') + i += 1 + +# Main App app = QApplication(sys.argv) - +# Main Window window = MainWindow() +# Show Window window.show() - app.exec() \ No newline at end of file diff --git a/_BitMover_MainWindow.py b/_BitMover_MainWindow.py index cea5097..a67e89d 100644 --- a/_BitMover_MainWindow.py +++ b/_BitMover_MainWindow.py @@ -50,7 +50,7 @@ class Ui_MainWindow(object): self.file_list.setGeometry(QtCore.QRect(20, 160, 871, 701)) self.file_list.setObjectName("file_list") self.gridLayoutWidget_2 = QtWidgets.QWidget(parent=self.centralwidget) - self.gridLayoutWidget_2.setGeometry(QtCore.QRect(910, 610, 311, 211)) + self.gridLayoutWidget_2.setGeometry(QtCore.QRect(910, 610, 541, 211)) self.gridLayoutWidget_2.setObjectName("gridLayoutWidget_2") self.grid_metadata = QtWidgets.QGridLayout(self.gridLayoutWidget_2) self.grid_metadata.setContentsMargins(0, 0, 0, 0) diff --git a/_ComparisonDialog.ui b/_ComparisonDialog.ui index 626a083..44a535d 100644 --- a/_ComparisonDialog.ui +++ b/_ComparisonDialog.ui @@ -62,35 +62,35 @@ - + - TextLabel + - + - TextLabel + - + - TextLabel + - + - TextLabel + - + 80 @@ -109,7 +109,7 @@ - + 80 @@ -129,7 +129,7 @@ - + 20 diff --git a/_Window_comparison_dialog.py b/_Window_comparison_dialog.py index bc859b6..87e0d3d 100644 --- a/_Window_comparison_dialog.py +++ b/_Window_comparison_dialog.py @@ -29,34 +29,38 @@ class Ui_FileComparisonDialog(object): self.gridLayout = QtWidgets.QGridLayout(self.gridLayoutWidget) self.gridLayout.setContentsMargins(0, 0, 0, 0) self.gridLayout.setObjectName("gridLayout") - self.label_4 = QtWidgets.QLabel(parent=self.gridLayoutWidget) - self.label_4.setObjectName("label_4") - self.gridLayout.addWidget(self.label_4, 2, 2, 1, 1) - self.label = QtWidgets.QLabel(parent=self.gridLayoutWidget) - self.label.setObjectName("label") - self.gridLayout.addWidget(self.label, 0, 1, 1, 1) - self.label_3 = QtWidgets.QLabel(parent=self.gridLayoutWidget) - self.label_3.setObjectName("label_3") - self.gridLayout.addWidget(self.label_3, 2, 1, 1, 1) - self.label_2 = QtWidgets.QLabel(parent=self.gridLayoutWidget) - self.label_2.setObjectName("label_2") - self.gridLayout.addWidget(self.label_2, 0, 2, 1, 1) - self.label_6 = QtWidgets.QLabel(parent=self.gridLayoutWidget) - self.label_6.setMaximumSize(QtCore.QSize(80, 16777215)) - self.label_6.setBaseSize(QtCore.QSize(0, 0)) - self.label_6.setObjectName("label_6") - self.gridLayout.addWidget(self.label_6, 0, 0, 1, 1) - self.label_7 = QtWidgets.QLabel(parent=self.gridLayoutWidget) - self.label_7.setMaximumSize(QtCore.QSize(80, 16777215)) - self.label_7.setBaseSize(QtCore.QSize(0, 0)) - self.label_7.setObjectName("label_7") - self.gridLayout.addWidget(self.label_7, 2, 0, 1, 1) - self.label_5 = QtWidgets.QLabel(parent=FileComparisonDialog) - self.label_5.setGeometry(QtCore.QRect(20, 10, 231, 61)) + self.l_hash_dest = QtWidgets.QLabel(parent=self.gridLayoutWidget) + self.l_hash_dest.setText("") + self.l_hash_dest.setObjectName("l_hash_dest") + self.gridLayout.addWidget(self.l_hash_dest, 2, 2, 1, 1) + self.l_path_file_source = QtWidgets.QLabel(parent=self.gridLayoutWidget) + self.l_path_file_source.setText("") + self.l_path_file_source.setObjectName("l_path_file_source") + self.gridLayout.addWidget(self.l_path_file_source, 0, 1, 1, 1) + self.l_path_file_dest = QtWidgets.QLabel(parent=self.gridLayoutWidget) + self.l_path_file_dest.setText("") + self.l_path_file_dest.setObjectName("l_path_file_dest") + self.gridLayout.addWidget(self.l_path_file_dest, 2, 1, 1, 1) + self.l_hash_source = QtWidgets.QLabel(parent=self.gridLayoutWidget) + self.l_hash_source.setText("") + self.l_hash_source.setObjectName("l_hash_source") + self.gridLayout.addWidget(self.l_hash_source, 0, 2, 1, 1) + self.label_title_source = QtWidgets.QLabel(parent=self.gridLayoutWidget) + self.label_title_source.setMaximumSize(QtCore.QSize(80, 16777215)) + self.label_title_source.setBaseSize(QtCore.QSize(0, 0)) + self.label_title_source.setObjectName("label_title_source") + self.gridLayout.addWidget(self.label_title_source, 0, 0, 1, 1) + self.label_title_dest = QtWidgets.QLabel(parent=self.gridLayoutWidget) + self.label_title_dest.setMaximumSize(QtCore.QSize(80, 16777215)) + self.label_title_dest.setBaseSize(QtCore.QSize(0, 0)) + self.label_title_dest.setObjectName("label_title_dest") + self.gridLayout.addWidget(self.label_title_dest, 2, 0, 1, 1) + self.label_title_comparing_files = QtWidgets.QLabel(parent=FileComparisonDialog) + self.label_title_comparing_files.setGeometry(QtCore.QRect(20, 10, 231, 61)) font = QtGui.QFont() font.setPointSize(28) - self.label_5.setFont(font) - self.label_5.setObjectName("label_5") + self.label_title_comparing_files.setFont(font) + self.label_title_comparing_files.setObjectName("label_title_comparing_files") self.retranslateUi(FileComparisonDialog) QtCore.QMetaObject.connectSlotsByName(FileComparisonDialog) @@ -64,10 +68,6 @@ class Ui_FileComparisonDialog(object): def retranslateUi(self, FileComparisonDialog): _translate = QtCore.QCoreApplication.translate FileComparisonDialog.setWindowTitle(_translate("FileComparisonDialog", "Dialog")) - self.label_4.setText(_translate("FileComparisonDialog", "TextLabel")) - self.label.setText(_translate("FileComparisonDialog", "TextLabel")) - self.label_3.setText(_translate("FileComparisonDialog", "TextLabel")) - self.label_2.setText(_translate("FileComparisonDialog", "TextLabel")) - self.label_6.setText(_translate("FileComparisonDialog", "Source")) - self.label_7.setText(_translate("FileComparisonDialog", "Destination")) - self.label_5.setText(_translate("FileComparisonDialog", "Comparing Files")) + self.label_title_source.setText(_translate("FileComparisonDialog", "Source")) + self.label_title_dest.setText(_translate("FileComparisonDialog", "Destination")) + self.label_title_comparing_files.setText(_translate("FileComparisonDialog", "Comparing Files")) diff --git a/_dialog_checksum_progress.py b/_dialog_checksum_progress.py index bcbd296..61fc55a 100644 --- a/_dialog_checksum_progress.py +++ b/_dialog_checksum_progress.py @@ -1,9 +1,9 @@ from PyQt6.QtWidgets import QDialog -from _Window_import_dialog import Ui_DialogImport +from _Window_checksum_progress_dialog import Ui_ChecksumProgressDialog -class DialogImport(QDialog, Ui_DialogImport): +class DialogChecksumProgress(QDialog, Ui_ChecksumProgressDialog): def __init__(self,*args,**kwargs): - super(DialogImport,self).__init__(*args,**kwargs) + super(DialogChecksumProgress,self).__init__(*args,**kwargs) self.setupUi(self) # self.import_dialog = UI_DialogImport() # self.ui_import_dialog = Ui_DialogImport() @@ -13,33 +13,27 @@ class DialogImport(QDialog, Ui_DialogImport): print(f'is_shown: {self.isVisible()}') return self.isVisible() - def open_import_dialog(self): - print(f'open_import_dialog: {self.is_shown()}') - if not self.is_shown(): - print('Inside if not self.is_shown') - print('showing window') - self.show() + def open_dialog(self,should_be_open): + if should_be_open is True: + if self.is_shown() is False: + print(f'open_checksum_dialog: {self.is_shown()}') + self.show() + else: + print('Checksum dialog already open.') + else: + self.close_dialog() - def close_import_dialog(self): + def close_dialog(self): print('close_import_dialog') if self.is_shown(): print('inside self.is_shown()') print('hiding window') self.hide() - def set_progress_importing(self, n): - print("%d%% done" % n) - self.progressBar_importing.setValue(int(n)) - self.lcd_import_progress.display(n) + def set_progress(self, n): + # print("%d%% done" % n) + self.progressBar_getting_checksum.setValue(int(n)) + # self.lcd_import_progress.display(n) - def set_progress_current_file(self, n): - print("%d%% done" % n) - self.progressBar_importing_2.setValue(int(n)) - self.lcd_current_file_progress.display(n) - - def set_importing_file(self,f): - self.l_importing_file_name.setText(f) - - def add_to_imported_list(self,n): - self.set_importing_file('') - self.list_imported.addItem(n) \ No newline at end of file + def set_file(self,f): + self.l_content_checksum_filename.setText(f) \ No newline at end of file diff --git a/_dialog_find_files.py b/_dialog_find_files.py index 93c7fc6..40eaa9b 100644 --- a/_dialog_find_files.py +++ b/_dialog_find_files.py @@ -1,5 +1,5 @@ from PyQt6.QtWidgets import QDialog -from _finding_files_dialog_Window import Ui_FindProgress +from _Window_finding_files_dialog import Ui_FindProgress class FindProgress(QDialog, Ui_FindProgress): def __init__(self,*args,**kwargs): @@ -26,6 +26,6 @@ class FindProgress(QDialog, Ui_FindProgress): self.hide() def set_progress_finding_files(self,n): - print("%d%% done" % n) + # print("%d%% done" % n) self.progressBar_importing.setValue(float(n)) self.lcd_import_progress.display(n) \ No newline at end of file diff --git a/_dialog_import.py b/_dialog_import.py index ab1c794..2cf5f0a 100644 --- a/_dialog_import.py +++ b/_dialog_import.py @@ -1,5 +1,5 @@ from PyQt6.QtWidgets import QDialog -from _import_dialog_Window import Ui_DialogImport +from _Window_import_dialog import Ui_DialogImport class DialogImport(QDialog, Ui_DialogImport): def __init__(self,*args,**kwargs): @@ -14,11 +14,16 @@ class DialogImport(QDialog, Ui_DialogImport): return self.isVisible() def open_import_dialog(self): - print(f'open_import_dialog: {self.is_shown()}') - if not self.is_shown(): - print('Inside if not self.is_shown') - print('showing window') - self.show() + if self.is_shown() is False: + print(f'open_import_dialog: {self.is_shown()}') + if not self.is_shown(): + print('Inside if not self.is_shown') + print('showing window') + self.show() + else: + print('Import Dialog already open.') + else: + self.close_import_dialog() def close_import_dialog(self): print('close_import_dialog') @@ -28,12 +33,12 @@ class DialogImport(QDialog, Ui_DialogImport): self.hide() def set_progress_importing(self, n): - print("%d%% done" % n) + # print("%d%% done" % n) self.progressBar_importing.setValue(int(n)) self.lcd_import_progress.display(n) def set_progress_current_file(self, n): - print("%d%% done" % n) + # print("%d%% done" % n) self.progressBar_importing_2.setValue(int(n)) self.lcd_current_file_progress.display(n) diff --git a/_file_stuff.py b/_file_stuff.py index af4fa0e..e103211 100644 --- a/_file_stuff.py +++ b/_file_stuff.py @@ -9,7 +9,8 @@ import yaml from tqdm import tqdm ### Local Imports -from _hashing import xx_hash +# from _hashing import xx_hash +from _verify_file_checksum import FileHash def check_log_dir(d): create_folder(d) @@ -38,10 +39,14 @@ def cmp_files(f1,f2): #TODO: Determine if path is actually a file #TODO: Determine if the hash has already been stored and use it if so - hash1 = xx_hash(f1) - hash2 = xx_hash(f2) + hash1 = FileHash(f1).file_hash + hash2 = FileHash(f2).file_hash return hash1 == hash2 +def cmp_hashes(h1,h2): + """ compare two files """ + return h1 == h2 + def path_access_read(path): """ make sure we can read from the path """ val = os.access(path, os.R_OK) diff --git a/_find_files.py b/_find_files.py index eca7c33..e89212c 100644 --- a/_find_files.py +++ b/_find_files.py @@ -29,7 +29,10 @@ class FindFiles: current_file_progress_callback, imported_file_count_callback, found_file_callback, - total_file_count_callback): + total_file_count_callback, + checksum_file_callback, + checksum_progress_callback, + checksum_dialog_open_callback): file_count = int(0) if len(self.search_types) > 0: for folder, subfolders, filename in os.walk(self.src_dir): diff --git a/_hashing.py b/_hashing.py index d19e2ce..40b0e60 100644 --- a/_hashing.py +++ b/_hashing.py @@ -8,28 +8,24 @@ import os import xxhash from tqdm import tqdm -def xx_hash(file): - """ calculates and returns file hash based on xxHash """ - size = os.path.getsize(file) - hasher = xxhash.xxh64() +# def xx_hash(file): +# """ calculates and returns file hash based on xxHash """ +# size = os.path.getsize(file) +# hasher = xxhash.xxh64() +# +# with open(file, 'rb') as f: +# with tqdm(total=size, +# unit='B', +# unit_scale=True, +# desc=f'Getting hash for {os.path.basename(file)}') as pbar: +# for chunk in iter(lambda: f.read(4096), b""): +# hasher.update(chunk) +# pbar.update(len(chunk)) +# file_hash = hasher.hexdigest() +# +# return file_hash - with open(file, 'rb') as f: - with tqdm(total=size, - unit='B', - unit_scale=True, - desc=f'Getting hash for {os.path.basename(file)}') as pbar: - for chunk in iter(lambda: f.read(4096), b""): - hasher.update(chunk) - pbar.update(len(chunk)) - file_hash = hasher.hexdigest() - return file_hash - -def hash_path(path): - """ hashes a string passed as a path """ - - hasher = xxhash.xxh64(path) - return hasher.hexdigest() def gen_xxhashes(f): """ Generate xxHashes """ diff --git a/_media_file.py b/_media_file.py index 0eebae4..9838f9a 100644 --- a/_media_file.py +++ b/_media_file.py @@ -2,7 +2,7 @@ import os.path import sys from _audio import AudioFile -from _hashing import hash_path +from _verify_file_checksum import FileHash from _video import VideoFile from _photo import PhotoFile from _file_stuff import (get_file_name, @@ -23,7 +23,7 @@ class MediaFile: self.store_originals = self.config['store_originals'] self.src_dir = os.path.dirname(self.path_file_name) self.base_dst_dir = self.config['folders']['destination']['base'] - self.source_path_hash = hash_path(self.path_file_name) + self.source_path_hash = FileHash(self.path_file_name).path_file self.file_name = get_file_name(self.path_file_name) self.dotted_file_ext = get_dotted_file_ext(self.file_name) self.file_ext = get_file_ext(self.file_name) diff --git a/_preview.py b/_preview.py index 48f2f8c..581ee17 100644 --- a/_preview.py +++ b/_preview.py @@ -1,10 +1,8 @@ #!/usr/bin/env python from PIL import Image - -from _media_file import MediaFile from _raw_photo import extract_jpg_thumb from _video import VideoFile -from _hashing import hash_path +from _verify_file_checksum import FileHash class MediaPreview: def __init__(self,path_file_name,media_files): @@ -12,7 +10,7 @@ class MediaPreview: self.path_file_name = path_file_name self.media_files_list = media_files # self.media_file = MediaFile(self.path_file_name) - self.source_path_hash = hash_path(self.path_file_name) + self.source_path_hash = FileHash(self.path_file_name).path_hash # print(f'_preview.py,MediaPreview:\n\tpath_file_name: {self.path_file_name}\n\thash: {self.source_path_hash}\n\tmedia_files_list: {self.media_files_list}\n') self.thumbnail = 'thumbnail.jpg' self.file_type = self.media_files_list[self.source_path_hash]['file_type'] diff --git a/_thread_my_stuff.py b/_thread_my_stuff.py index ff283da..afdbd53 100644 --- a/_thread_my_stuff.py +++ b/_thread_my_stuff.py @@ -31,6 +31,9 @@ class WorkerSignals(QObject): imported_file_count = pyqtSignal(int) found_file = pyqtSignal(str) total_file_count = pyqtSignal(int) + checksum_progress = pyqtSignal(int) + checksum_file = pyqtSignal(str) + checksum_dialog_open = pyqtSignal(bool) class Worker(QRunnable): """ @@ -60,6 +63,9 @@ class Worker(QRunnable): self.kwargs['imported_file_count_callback'] = self.signals.imported_file_count self.kwargs['found_file_callback'] = self.signals.found_file self.kwargs['total_file_count_callback'] = self.signals.total_file_count + self.kwargs['checksum_file_callback'] = self.signals.checksum_file + self.kwargs['checksum_progress_callback'] = self.signals.checksum_progress + self.kwargs['checksum_dialog_open_callback'] = self.signals.checksum_dialog_open @pyqtSlot() def run(self): diff --git a/_verify_file_checksum.py b/_verify_file_checksum.py index 31394d7..2f3779c 100644 --- a/_verify_file_checksum.py +++ b/_verify_file_checksum.py @@ -2,47 +2,52 @@ import os import xxhash class FileHash: - def __init__(self,files,*args,**kwargs): + def __init__(self, + path_file, + chunk_size=1024 * 1024, + *args, + **kwargs): super(FileHash,self).__init__(*args,**kwargs) - self.files = files - self.chunk_size = kwargs['chunk_size'] + self.path_file = path_file + self._path = os.path.dirname(self.path_file) + self.chunk_size = chunk_size + self.path_hasher = xxhash.xxh64(self._path) + # noinspection PyArgumentList + # self.file_hash = self.get_hash(self.path_file) + self.path_hash = self.hash_path() + + def get_hash(self,f, + progress_callback, + current_file_progress_callback, + imported_file_count_callback, + found_file_callback, + total_file_count_callback, + checksum_file_callback, + checksum_progress_callback, + checksum_dialog_open_callback): + + checksum_dialog_open_callback.emit(True) - @staticmethod - def xx_hash(self,f,progress_callback): size = os.path.getsize(f) hasher = xxhash.xxh64() - #todo: add callbacks chunk_count = 0 + + checksum_file_callback.emit(f) + with open(f, 'rb') as f: for chunk in iter(lambda: f.read(self.chunk_size),b""): hasher.update(chunk) chunk_count += 1 hashed_size = chunk_count * self.chunk_size - progress_callback.emit(round((hashed_size / size) * 100, 1)) + checksum_progress_callback.emit(round((hashed_size / size) * 100, 1)) file_hash = hasher.hexdigest() + checksum_dialog_open_callback.emit(False) + return file_hash - @staticmethod - def t_verify_checksum(self, - progress_callback, - import_progress_callback, - current_file_progress_callback, - imported_file_count_callback, - found_file, - total_file_count): + def hash_path(self): + """ hashes a string passed as a path """ - for file in self.files: - i = 0 - c = {} - for checksum in self.files[file]['xx_checksum']: - c[i] = self.files[file]['xx_checksum'][checksum] - if i > 0: - p = i - 1 - if c[i] == c[p]: - self.files[file]['checksum_match'] = True - else: - self.files[file]['checksum_match'] = False - i += 1 - return self.files \ No newline at end of file + return self.path_hasher.hexdigest() \ No newline at end of file diff --git a/scripts/gen_gui_py.sh b/scripts/gen_gui_py.sh index e861824..82352d9 100755 --- a/scripts/gen_gui_py.sh +++ b/scripts/gen_gui_py.sh @@ -1,5 +1,5 @@ pyuic6 BitMover.ui -o _BitMover_MainWindow.py -pyuic6 import_dialogue.ui -o _import_dialog_Window.py -pyuic6 _finding_files_dialog.ui -o _finding_files_dialog_Window.py -pyuic6 _ComparisonDialog.ui -o _comparison_dialog_Window.py -pyuic6 _checksum_progress_dialog.ui -o _checksum_progress_dialog_Window.py +pyuic6 import_dialogue.ui -o _Window_import_dialog.py +pyuic6 _finding_files_dialog.ui -o _Window_finding_files_dialog.py +pyuic6 _ComparisonDialog.ui -o _Window_comparison_dialog.py +pyuic6 _checksum_progress_dialog.ui -o _Window_checksum_progress_dialog.py