diff --git a/import_media.py b/import_media.py index ce135f5..80fbfbb 100644 --- a/import_media.py +++ b/import_media.py @@ -74,27 +74,64 @@ def get_capture_date(p, t): month = stamp.strftime("%m") day = stamp.strftime("%d") return year, month, day +def path_exists(path): + """ Does the path exist """ + return os.path.exists(path) + +def is_dir(path): + """ determine if the argument passed is a directory """ + p_exists = path_exists(path) + + if p_exists is True: + return os.path.isdir(path) + else: + return p_exists + +def path_access_read(path): + """ make sure we can read from the path """ + val = os.access(path, os.R_OK) + + if val is False: + print(f'Can not read from {path}') + + return val + +def path_access_write(path): + """ make sure we can write to the path """ + val = os.access(path, os.W_OK) + + if val is False: + print(f'Can not write to {path}') + + return val def create_folder(f): """ Function to create folder """ - try: + if path_exists(f) is False: os.makedirs(f) - except FileExistsError: - pass + elif is_dir(f) is False: + pass # this needs to turn into bailing out as there is a collision. -def copy_from_source(p,d,f): +def copy_from_source(source_path,dest_path,file_name): """ Copy file from source to destination """ - if os.path.exists(os.path.join(d, f)): - check_match = cmp_files(p, os.path.join(d, f)) + + file_exists = path_exists(os.path.join(dest_path,file_name)) + + if file_exists is True: + check_match = cmp_files(os.path.join(source_path,file_name), + os.path.join(dest_path, file_name)) if check_match is False: print(f'Found duplicate for {p}, renaming destination with md5 appended.') - base, extension = os.path.splitext(f) - file_name_hash = base + '_' + md5_hash(os.path.join(d, f)) + extension - os.rename(os.path.join(d, f), os.path.join(d, file_name_hash)) + base, extension = os.path.splitext(file_name) + md5 = md5_hash(os.path.join(dest_path, file_name)) + file_name_hash = base + '_' + md5 + extension + os.rename(os.path.join(dest_path, file_name), + os.path.join(dest_path, file_name_hash)) else: return - shutil.copy(p, d) + create_folder(dest_path) + shutil.copy(os.path.join(source_path,file_name), dest_path) def process_file(path, f_type, f_name, ext): """ gather information and add to dictionary """ @@ -136,21 +173,16 @@ def process_file(path, f_type, f_name, ext): if files[i]['type'] == 'image': files[i]['folders']['destination'] = files[i]['folders']['destination'] + '/PHOTO' - if config['store_originals'] is True: - files[i]['folders']['destination_original'] = files[i]['folders']['destination'] + \ - '/ORIGINALS' - if files[i]['extension'] in ('jpg', 'jpeg'): files[i]['folders']['destination'] = files[i]['folders']['destination'] + \ '/JPG' - if files[i]['folders']['destination_original']: - files[i]['folders']['destination_original'] = \ - files[i]['folders']['destination_original'] + '/JPG' + if config['store_originals'] is True: + files[i]['folders']['destination_original'] = files[i]['folders']['destination'] + '/ORIGINALS/JPG' else: files[i]['folders']['destination'] = files[i]['folders']['destination'] + '/RAW' - if files[i]['folders']['destination_original']: - files[i]['folders']['destination_original'] = \ - files[i]['folders']['destination_original'] + '/RAW' + + if config['store_originals'] is True: + files[i]['folders']['destination_original'] = files[i]['folders']['destination'] + '/ORIGINALS/RAW' elif files[i]['type'] == 'video': files[i]['folders']['destination'] = files[i]['folders']['destination'] + '/VIDEO' @@ -161,81 +193,95 @@ def process_file(path, f_type, f_name, ext): else: print('WARN: ', files[i]['type'], ' is not a known type and you never should have landed here.') - try: - files[i]['folders']['destination_original'] - except (NameError, KeyError): - files[i]['folders']['destination_original'] = False - -# copy_from_source(p, d, o, file) - def find_files(directory): """ find files to build a dictionary out of """ - for folder, subfolders, filename in tqdm(os.walk(directory), desc = 'Finding Files', ncols = 100): + for folder, subfolders, filename in os.walk(directory): for f_type in config['file_types']: - for ext in tqdm(config['file_types'][f_type], desc = 'Finding ' + f_type + ' Files', ncols = 100): + for ext in config['file_types'][f_type]: for file in tqdm(filename, desc = 'Finding ' + ext + ' Files', ncols = 100): if file.lower().endswith(ext): process_file(folder, f_type, file, ext) -find_files(config['folders']['source']['base']) - -#pprint(files) - -#for file in files: -# pprint(files[file]['folders']) - -# Copy File -for file in tqdm(files, desc = "Copying Files:", ncols = 100): - create_folder(files[file]['folders']['destination']) - - copy_from_source(os.path.join(files[file]['folders']['source_path'],files[file]['name']), - files[file]['folders']['destination'], - files[file]['name']) - - if files[file]['folders']['destination_original'] is not False: - create_folder(files[file]['folders']['destination_original']) - - copy_from_source(os.path.join(files[file]['folders']['destination'],files[file]['name']), - files[file]['folders']['destination_original'], - files[file]['name']) - -# validate checksum -for file in tqdm(files, desc = "Generating MD5 Hashes:", ncols = 100): - - #print(files[file]) - files[file]['md5_checksums'] = {} - for folder in files[file]['folders']: - #print(files[file]['folders'][folder]) - if files[file]['folders'][folder] is not False: - #print('folder is not false.') - k = os.path.join(files[file]['folders'][folder], files[file]['name']) - #print(k) - - files[file]['md5_checksums'][k] = md5_hash(k) - #print(files[file]['md5_checksums'][k]) - -for file in tqdm(files, desc = "Verifying Checksums:", ncols = 100): - i = 0 - c = {} - for checksum in files[file]['md5_checksums']: - c[i] = files[file]['md5_checksums'][checksum] - if i > 0: - P = i - 1 - if c[i] == c[P]: - files[file]['source_cleanable'] = True +def validate_config_dir_access(): + """ Validate we can op in the defined directories """ + check = path_access_write(config['folders']['destination']['base']) + if check is False: + return False + else: + check = path_access_read(config['folders']['source']['base']) + if check is False: + return False + else: + if config['store_backup'] is True: + check = path_access_write(config['folders']['backup']) + if check is False: + return False + else: + return True else: - files[file]['source_cleanable'] = False - print(f'FATAL: Checksum validation failed for: \ - {files[file]["name"]} \n{c[i]}\n is not equal to \n{c[P]}\n') - print('\n File Meta:\n') - pprint(files[file]) - i = i + 1 + return True -# cleanup sd -if config['cleanup_sd'] is True: - for file in tqdm(files, desc = "Cleaning Up SD:", ncols = 100): - if files[file]['source_cleanable'] is True: - os.remove(os.path.join(files[file]['folders']['source_path'],files[file]['name'])) +def copy_files(): + """ Copy Files. """ + for file in tqdm(files, desc = "Copying Files:", ncols = 100): + create_folder(files[file]['folders']['destination']) + + copy_from_source(files[file]['folders']['source_path'], + files[file]['folders']['destination'], + files[file]['name']) + + if config['store_originals'] is True: + if files[file]['type'] == 'image': + create_folder(files[file]['folders']['destination_original']) + + copy_from_source(files[file]['folders']['destination'], + files[file]['folders']['destination_original'], + files[file]['name']) + +def gen_hashes(): + """ Generate Hashes """ + for file in tqdm(files, desc = "Generating MD5 Hashes:", ncols = 100): + #print(files[file]) + files[file]['md5_checksums'] = {} + for folder in files[file]['folders']: + k = os.path.join(files[file]['folders'][folder], files[file]['name']) + files[file]['md5_checksums'][k] = md5_hash(k) + +def validate_checksums(): + """ Validate Checksums """ + for file in tqdm(files, desc = "Verifying Checksums:", ncols = 100): + i = 0 + c = {} + for checksum in files[file]['md5_checksums']: + c[i] = files[file]['md5_checksums'][checksum] + if i > 0: + P = i - 1 + if c[i] == c[P]: + files[file]['source_cleanable'] = True + else: + files[file]['source_cleanable'] = False + print(f'FATAL: Checksum validation failed for: \ + {files[file]["name"]} \n{c[i]}\n is not equal to \n{c[P]}\n') + print('\n File Meta:\n') + pprint(files[file]) + i = i + 1 + +def cleanup_sd(): + """ If we should cleanup the SD, nuke the copied files. """ + if config['cleanup_sd'] is True: + for file in tqdm(files, desc = "Cleaning Up SD:", ncols = 100): + if files[file]['source_cleanable'] is True: + os.remove(os.path.join(files[file]['folders']['source_path'],files[file]['name'])) #pprint(files) +go = validate_config_dir_access() +if go is True: + find_files(config['folders']['source']['base']) + copy_files() + gen_hashes() + validate_checksums() + cleanup_sd() +else: + print("There was a problem accessing one or more directories defined in the configuration.") + print('done.')