diff --git a/dedup.py b/dedup.py deleted file mode 100755 index abd02c0..0000000 --- a/dedup.py +++ /dev/null @@ -1,92 +0,0 @@ -#!/usr/bin/env python - -import argparse -import sys -import os - -from bitmover import copy_from_source -from file_stuff import create_folder, cmp_files -from lumberjack import timber - -log = timber(__name__) - -parser = argparse.ArgumentParser() -parser.add_argument("-f", "--folder", help = "folder with files to rename") -parser.add_argument("-d", "--dryrun", help = "dry run, no action") - -args = parser.parse_args() - -if args.folder: - FOLDER = args.folder -else: - print("you need to specify a folder. Dedup") - sys.exit() -if args.dryrun: - dry_run = True -else: - dry_run = False - -def get_file_size(f_name): - return os.path.getsize(f_name) - - -l = [] -files = os.listdir(FOLDER) -dup_folder = os.path.join(FOLDER,"__dups") -SIZE = 0 -MAX_SIZE = 0 -BIGGEST_FILE = '' - -create_folder(dup_folder) -f_list = {} -dictionary = {} -for x in files: - if os.path.isfile(os.path.join(FOLDER,x)): - if x.lower().endswith("jpg") or x.lower().endswith("jpeg"): - group = dictionary.get(x[:23],[]) - group.append(x) - dictionary[x[:23]] = group - -for g in dictionary: - f_list[g] = {'files': {}} - - for f in dictionary[g]: - p = os.path.join(FOLDER,f) - size = os.path.getsize(p) - f_list[g]['files'][f] = {} - f_list[g]['files'][f]['path'] = p - f_list[g]['files'][f]['size'] = size - -# print(f_list) - -for g in f_list: - MAX_SIZE = 0 - log.debug(g) - if len(f_list[g]['files']) > 1: - for f in f_list[g]['files']: - log.debug(f"{f_list[g]['files'][f]['path']}: {f_list[g]['files'][f]['size']}") - SIZE = f_list[g]['files'][f]['size'] - - if SIZE > MAX_SIZE: - MAX_SIZE = SIZE - BIGGEST_FILE = f_list[g]['files'][f]['path'] - log.debug(f'New Biggest File: {BIGGEST_FILE}, {MAX_SIZE*1024}KB') - f_list[g]['biggest_file'] = BIGGEST_FILE - else: - log.debug(f'Only 1 file in {g}') - -for g in f_list: - # log.debug(g) - if len(f_list[g]['files']) > 1: - for f in f_list[g]['files']: - if f_list[g]['biggest_file'] != f_list[g]['files'][f]['path']: - copy_from_source(FOLDER, dup_folder, os.path.basename(f_list[g]['files'][f]['path'])) - - file_match = cmp_files(f_list[g]['files'][f]['path'], - os.path.join(dup_folder, - os.path.basename(f_list[g]['files'][f]['path']))) - - if file_match is True: - os.remove(f_list[g]['files'][f]['path']) - else: - print(f"{f_list[g]['files'][f]['path']} does not match {os.path.join(dup_folder, os.path.basename(f_list[g]['files'][f]['path']))}") \ No newline at end of file diff --git a/dedup_wrapper.sh b/dedup_wrapper.sh deleted file mode 100755 index 8c21149..0000000 --- a/dedup_wrapper.sh +++ /dev/null @@ -1,78 +0,0 @@ -#!/usr/bin/env bash - - -root_dir="$1" - -if [[ $root_dir == '' ]]; then - echo "You must pass a directory as the first argument." - exit 1 -fi - -function dedup { - echo "renaming: $1" - ./rename.py -f "$1" - - echo "deduping: $1" - ./dedup.py -f "$1" -} - -for i in `ls -1 $root_dir`; do - echo $i - if [[ -d "${root_dir}/$i" ]]; then - echo "${root_dir}/$i is a dir" - if [[ $i == *JPG* || $i == *RAW* ]]; then - echo "${root_dir}/$i contains JPG/RAW" - dedup ${root_dir}/$i - else - echo "${root_dir}/$i does not contain JPG/RAW" - for s1 in `ls -1 ${root_dir}/$i`; do - echo "$s1" - if [[ -d "${root_dir}/${i}/$s1" ]]; then - echo "${root_dir}/${i}/$s1 is a dir" - if [[ $s1 == *JPG* || $s1 == *RAW* ]]; then - echo "${root_dir}/${i}/$s1 contains JPG/RAW" - dedup ${root_dir}/${i}/$s1 - else - echo "${root_dir}/${i}/$s1 does not contain JPG/RAW" - for s2 in `ls -1 ${root_dir}/${i}/$s1`; do - echo ${root_dir}/${i}/${s1}/$s2 - if [[ -d "${root_dir}/${i}/${s1}/$s2" ]]; then - echo "${root_dir}/${i}/${s1}/$s2 is a dir" - if [[ ${root_dir}/${i}/${s1}/$s2 == *JPG* || ${root_dir}/${i}/${s1}/$s2 == *RAW* ]]; then - echo "${root_dir}/${i}/${s1}/$s2 contains JPG/RAW" - dedup ${root_dir}/${i}/${s1}/$s2 - else - echo "${root_dir}/${i}/${s1}/$s2 does not contain JPG/RAW" - for s3 in `ls -1 ${root_dir}/${i}/${s1}/$s2`; do - echo ${root_dir}/${i}/${s1}/${s2}/$s3 - if [[ -d "${root_dir}/${i}/${s1}/${s2}/$s3" ]]; then - echo "${root_dir}/${i}/${s1}/${s2}/$s3 is a dir" - if [[ ${root_dir}/${i}/${s1}/${s2}/$s3 == *JPG* || ${root_dir}/${i}/${s1}/${s2}/$s3 == *RAW* ]]; then - echo "${root_dir}/${i}/${s1}/${s2}/$s3 contains JPG/RAW" - dedup ${root_dir}/${i}/${s1}/${s2}/$s3 - else - echo "${root_dir}/${i}/${s1}/${s2}/${s3} does not contain JPG/RAW" - for s4 in `ls -1 ${root_dir}/${i}/${s1}/${s2}/${s3}`; do - echo ${root_dir}/${i}/${s1}/${s2}/${s3}/$s4 - if [[ -d "${root_dir}/${i}/${s1}/${s2}/${s3}/$s4" ]]; then - echo "${root_dir}/${i}/${s1}/${s2}/${s3}/$s4 is a dir" - if [[ ${root_dir}/${i}/${s1}/${s2}/${s3}/$s4 == *JPG* || ${root_dir}/${i}/${s1}/${s2}/${s3}/$s4 == *RAW* ]]; then - echo "${root_dir}/${i}/${s1}/${s2}/${s3}/$s4 contains JPG/RAW" - dedup ${root_dir}/${i}/${s1}/${s2}/${s3}/$s4 - fi - fi - done - fi - fi - done - fi - fi - done - fi - fi - done - fi - fi -done - - diff --git a/import_media.py b/import_media.py deleted file mode 100755 index 11586dc..0000000 --- a/import_media.py +++ /dev/null @@ -1,110 +0,0 @@ -#!/usr/bin/env python - -""" -Import photos from SD card into folder with today's date + nickname -Use: import_media.py (--jpg|--raw|--both) -Add script to path - -TODO: -8. Optionally allow specification of a backup location on another disk - or NAS to ship a 3rd copy to -10. Every config option has an arg override -11. Optionally rename file if EVENT name was passed in --- STRETCH -- -12. Make a graphical interface -""" - -import argparse -import os -from tqdm import tqdm - -### Local Imports -from configure import CONFIG_FILE, Configure, files -from file_stuff import cleanup_sd, validate_config_dir_access, is_file -from hashing import gen_xxhashes, validate_xx_checksums -from bitmover import copy_files -from lumberjack import timber -from media import process_file - -c = Configure(CONFIG_FILE) -config = c.load_config() -log = timber(__name__) -log.info("Starting import_media") - -parser = argparse.ArgumentParser() -parser.add_argument("-e", "--event", help = "Event Name") -parser.add_argument("-s", "--source", help = "Source Directory to search for files") -parser.add_argument("-d", "--destination", help = "Destination Directory to put files") -parser.add_argument("-o", "--create-originals", help = "For images only, create an originals \ - folder for safe keeping") -parser.add_argument("-b", "--backup-destination", help = "Create a backup of everything at the \ - specified location") -parser.add_argument("-D", "--delete-source-files", help = "Delete files from SD after validating \ - checksum of copied files") -parser.add_argument("-v", "--verify", help = "[True|False] Verify the checksum of \ - the copied file") -parser.add_argument("-c", "--config", help = "Load the specified config file instead \ - of the default " + CONFIG_FILE) -parser.add_argument("-g", "--generate-config", help = "Generate config file based on options \ - passed from command arguments") - -args = parser.parse_args() - -if args.event: - EVENT = args.event -else: - EVENT = False -if args.source: - config['folders']['source']['base'] = args.source -if args.destination: - config['folders']['destination']['base'] = args.source -#if args.create-originals: -# pass -#if args.backup-destination: -# pass -#if args.delete-source-files: -# pass -#if args.config: -# pass -#if args.generate-config: -# pass - - -def find_files(directory): - """ find files to build a dictionary out of """ - log.debug(f'find_files({directory})') - os.system('clear') - for folder, subfolders, filename in os.walk(directory): - log.debug(f'{folder},{filename}') - for f_type in config['file_types']: - log.debug(f'Type: {f_type}') - for ext in config['file_types'][f_type]: - log.debug(f'Extension: {ext}') - os.system('clear') - for file in tqdm(filename, - desc = 'Finding ' + ext + ' Files in ' + folder): - log.debug(f'File: {file}') - if file.lower().endswith(ext): - current_file = os.path.join(folder,file) - log.debug(f'Current File: {current_file}') - if is_file(current_file): - log.debug(f'Is File: {current_file}') - log.debug(f'Call function: process_file({folder}, {file}, {EVENT}, {config})') - #process_file(folder, f_type, file, ext) - process_file(folder, file, EVENT, config) - else: - log.warn(f"Skipping {current_file} as it does not look like a real file.") - -GO = validate_config_dir_access(config) -if GO is True: - find_files(config['folders']['source']['base']) - copy_files(files,config) - gen_xxhashes(files) - validate_xx_checksums(files) - cleanup_sd(files,config) -else: - log.critical('There was a problem accessing one or more directories defined in the configuration.') - - -# dump_yaml(files, 'files_dict.yaml') -log.info('done.') diff --git a/rename.py b/rename.py deleted file mode 100755 index 8c7162a..0000000 --- a/rename.py +++ /dev/null @@ -1,116 +0,0 @@ -#!/usr/bin/env python - -import argparse -import sys -import os - -from bitmover import copy_from_source -from file_stuff import path_exists, cmp_files -from get_image_tag import get_image_date, get_exif_tag -from hashing import xx_hash - -parser = argparse.ArgumentParser() -parser.add_argument("-f", "--folder", help = "folder with files to rename") -parser.add_argument("-o", "--keeporiginalname", help = "keeps the original name attached to the file.") -parser.add_argument("-d", "--dryrun", help = "dry run, no action") - -args = parser.parse_args() - -if args.folder: - FOLDER = args.folder -else: - print("you need to specify a folder. Duh.") - sys.exit() -if args.dryrun: - dry_run = True -else: - dry_run = False - -if args.keeporiginalname: - keep_orig_name = True -else: - keep_orig_name = False - -def get_file_size(f): - return os.path.getsize(f) - -for file in os.listdir(FOLDER): - if file.lower().endswith("gif"): - copy_from_source(FOLDER,'/Volumes/VIDEO_ARRAY_01/Multimedia/gif',file) - file_match = cmp_files(os.path.join(FOLDER,file), - os.path.join('/Volumes/VIDEO_ARRAY_01/Multimedia/gif',file)) - - if file_match is True: - os.remove(os.path.join(FOLDER,file)) - - if file.lower().endswith("png"): - copy_from_source(FOLDER,'/Volumes/VIDEO_ARRAY_01/Multimedia/png',file) - - file_match = cmp_files(os.path.join(FOLDER, file), - os.path.join('/Volumes/VIDEO_ARRAY_01/Multimedia/gif', file)) - - if file_match is True: - os.remove(os.path.join(FOLDER, file)) - - if file.lower().endswith("heic"): - copy_from_source(FOLDER,'/Volumes/VIDEO_ARRAY_01/Multimedia/heic',file) - - file_match = cmp_files(os.path.join(FOLDER, file), - os.path.join('/Volumes/VIDEO_ARRAY_01/Multimedia/heic', file)) - - if file_match is True: - os.remove(os.path.join(FOLDER, file)) - - if file.lower().endswith("jpg") or \ - file.lower().endswith("jpeg") or \ - file.lower().endswith("nef") or \ - file.lower().endswith("rw2") or \ - file.lower().endswith("arw") or \ - file.lower().endswith("dng"): - - old_path = os.path.join(FOLDER,file) - - lowered_name = file.lower() - file_size = get_file_size(old_path) - file_extension = os.path.splitext(lowered_name)[1] - image_date = get_image_date(old_path) - image_date_year = image_date.strftime("%Y") - image_date_month = image_date.strftime("%m") - image_date_day = image_date.strftime("%d") - image_date_hour = image_date.strftime("%H") - image_date_minute = image_date.strftime("%M") - image_date_second = image_date.strftime("%S") - image_date_microsecond = image_date.strftime("%f") - image_date_subsecond = str(get_exif_tag(old_path,'EXIF SubSec')) - image_hash = xx_hash(old_path) - - if image_date_subsecond: - subsecond_desired_length = 6 - - if image_date_subsecond == 'None': - image_date_subsecond = subsecond_desired_length*str('0') - - l_image_date_subsecond = len(image_date_subsecond) - - if subsecond_desired_length > l_image_date_subsecond: - pad = subsecond_desired_length - l_image_date_subsecond - image_date_subsecond = image_date_subsecond + pad*str("0") - - new_name = f'{image_date_year}-{image_date_month}-{image_date_day}-{image_date_hour}{image_date_minute}{image_date_second}{image_date_subsecond}_{file_size}_{image_hash}' - else: - new_name = f'{image_date_year}-{image_date_month}-{image_date_day}-{image_date_hour}{image_date_minute}{image_date_second}000_{file_size}_{image_hash}' - - if keep_orig_name is True: - new_file_name = f'{new_name}-{lowered_name}' - else: - new_file_name = f'{new_name}{file_extension}' - - new_path = os.path.join(FOLDER,new_file_name) - - if path_exists(new_path): - print(f"{new_path} exists.. skipping.") - elif dry_run is True: - print(f'Dry run: {old_path} becomes {new_path}') - else: - print(f'Renaming {old_path} to: {new_path}') - os.rename(old_path,new_path) \ No newline at end of file