cleanup
This commit is contained in:
Kameron Kenny
parent
e8861abbdc
commit
eb9b007703
92
dedup.py
92
dedup.py
|
|
@ -1,92 +0,0 @@
|
||||||
#!/usr/bin/env python
|
|
||||||
|
|
||||||
import argparse
|
|
||||||
import sys
|
|
||||||
import os
|
|
||||||
|
|
||||||
from bitmover import copy_from_source
|
|
||||||
from file_stuff import create_folder, cmp_files
|
|
||||||
from lumberjack import timber
|
|
||||||
|
|
||||||
log = timber(__name__)
|
|
||||||
|
|
||||||
parser = argparse.ArgumentParser()
|
|
||||||
parser.add_argument("-f", "--folder", help = "folder with files to rename")
|
|
||||||
parser.add_argument("-d", "--dryrun", help = "dry run, no action")
|
|
||||||
|
|
||||||
args = parser.parse_args()
|
|
||||||
|
|
||||||
if args.folder:
|
|
||||||
FOLDER = args.folder
|
|
||||||
else:
|
|
||||||
print("you need to specify a folder. Dedup")
|
|
||||||
sys.exit()
|
|
||||||
if args.dryrun:
|
|
||||||
dry_run = True
|
|
||||||
else:
|
|
||||||
dry_run = False
|
|
||||||
|
|
||||||
def get_file_size(f_name):
|
|
||||||
return os.path.getsize(f_name)
|
|
||||||
|
|
||||||
|
|
||||||
l = []
|
|
||||||
files = os.listdir(FOLDER)
|
|
||||||
dup_folder = os.path.join(FOLDER,"__dups")
|
|
||||||
SIZE = 0
|
|
||||||
MAX_SIZE = 0
|
|
||||||
BIGGEST_FILE = ''
|
|
||||||
|
|
||||||
create_folder(dup_folder)
|
|
||||||
f_list = {}
|
|
||||||
dictionary = {}
|
|
||||||
for x in files:
|
|
||||||
if os.path.isfile(os.path.join(FOLDER,x)):
|
|
||||||
if x.lower().endswith("jpg") or x.lower().endswith("jpeg"):
|
|
||||||
group = dictionary.get(x[:23],[])
|
|
||||||
group.append(x)
|
|
||||||
dictionary[x[:23]] = group
|
|
||||||
|
|
||||||
for g in dictionary:
|
|
||||||
f_list[g] = {'files': {}}
|
|
||||||
|
|
||||||
for f in dictionary[g]:
|
|
||||||
p = os.path.join(FOLDER,f)
|
|
||||||
size = os.path.getsize(p)
|
|
||||||
f_list[g]['files'][f] = {}
|
|
||||||
f_list[g]['files'][f]['path'] = p
|
|
||||||
f_list[g]['files'][f]['size'] = size
|
|
||||||
|
|
||||||
# print(f_list)
|
|
||||||
|
|
||||||
for g in f_list:
|
|
||||||
MAX_SIZE = 0
|
|
||||||
log.debug(g)
|
|
||||||
if len(f_list[g]['files']) > 1:
|
|
||||||
for f in f_list[g]['files']:
|
|
||||||
log.debug(f"{f_list[g]['files'][f]['path']}: {f_list[g]['files'][f]['size']}")
|
|
||||||
SIZE = f_list[g]['files'][f]['size']
|
|
||||||
|
|
||||||
if SIZE > MAX_SIZE:
|
|
||||||
MAX_SIZE = SIZE
|
|
||||||
BIGGEST_FILE = f_list[g]['files'][f]['path']
|
|
||||||
log.debug(f'New Biggest File: {BIGGEST_FILE}, {MAX_SIZE*1024}KB')
|
|
||||||
f_list[g]['biggest_file'] = BIGGEST_FILE
|
|
||||||
else:
|
|
||||||
log.debug(f'Only 1 file in {g}')
|
|
||||||
|
|
||||||
for g in f_list:
|
|
||||||
# log.debug(g)
|
|
||||||
if len(f_list[g]['files']) > 1:
|
|
||||||
for f in f_list[g]['files']:
|
|
||||||
if f_list[g]['biggest_file'] != f_list[g]['files'][f]['path']:
|
|
||||||
copy_from_source(FOLDER, dup_folder, os.path.basename(f_list[g]['files'][f]['path']))
|
|
||||||
|
|
||||||
file_match = cmp_files(f_list[g]['files'][f]['path'],
|
|
||||||
os.path.join(dup_folder,
|
|
||||||
os.path.basename(f_list[g]['files'][f]['path'])))
|
|
||||||
|
|
||||||
if file_match is True:
|
|
||||||
os.remove(f_list[g]['files'][f]['path'])
|
|
||||||
else:
|
|
||||||
print(f"{f_list[g]['files'][f]['path']} does not match {os.path.join(dup_folder, os.path.basename(f_list[g]['files'][f]['path']))}")
|
|
||||||
|
|
@ -1,78 +0,0 @@
|
||||||
#!/usr/bin/env bash
|
|
||||||
|
|
||||||
|
|
||||||
root_dir="$1"
|
|
||||||
|
|
||||||
if [[ $root_dir == '' ]]; then
|
|
||||||
echo "You must pass a directory as the first argument."
|
|
||||||
exit 1
|
|
||||||
fi
|
|
||||||
|
|
||||||
function dedup {
|
|
||||||
echo "renaming: $1"
|
|
||||||
./rename.py -f "$1"
|
|
||||||
|
|
||||||
echo "deduping: $1"
|
|
||||||
./dedup.py -f "$1"
|
|
||||||
}
|
|
||||||
|
|
||||||
for i in `ls -1 $root_dir`; do
|
|
||||||
echo $i
|
|
||||||
if [[ -d "${root_dir}/$i" ]]; then
|
|
||||||
echo "${root_dir}/$i is a dir"
|
|
||||||
if [[ $i == *JPG* || $i == *RAW* ]]; then
|
|
||||||
echo "${root_dir}/$i contains JPG/RAW"
|
|
||||||
dedup ${root_dir}/$i
|
|
||||||
else
|
|
||||||
echo "${root_dir}/$i does not contain JPG/RAW"
|
|
||||||
for s1 in `ls -1 ${root_dir}/$i`; do
|
|
||||||
echo "$s1"
|
|
||||||
if [[ -d "${root_dir}/${i}/$s1" ]]; then
|
|
||||||
echo "${root_dir}/${i}/$s1 is a dir"
|
|
||||||
if [[ $s1 == *JPG* || $s1 == *RAW* ]]; then
|
|
||||||
echo "${root_dir}/${i}/$s1 contains JPG/RAW"
|
|
||||||
dedup ${root_dir}/${i}/$s1
|
|
||||||
else
|
|
||||||
echo "${root_dir}/${i}/$s1 does not contain JPG/RAW"
|
|
||||||
for s2 in `ls -1 ${root_dir}/${i}/$s1`; do
|
|
||||||
echo ${root_dir}/${i}/${s1}/$s2
|
|
||||||
if [[ -d "${root_dir}/${i}/${s1}/$s2" ]]; then
|
|
||||||
echo "${root_dir}/${i}/${s1}/$s2 is a dir"
|
|
||||||
if [[ ${root_dir}/${i}/${s1}/$s2 == *JPG* || ${root_dir}/${i}/${s1}/$s2 == *RAW* ]]; then
|
|
||||||
echo "${root_dir}/${i}/${s1}/$s2 contains JPG/RAW"
|
|
||||||
dedup ${root_dir}/${i}/${s1}/$s2
|
|
||||||
else
|
|
||||||
echo "${root_dir}/${i}/${s1}/$s2 does not contain JPG/RAW"
|
|
||||||
for s3 in `ls -1 ${root_dir}/${i}/${s1}/$s2`; do
|
|
||||||
echo ${root_dir}/${i}/${s1}/${s2}/$s3
|
|
||||||
if [[ -d "${root_dir}/${i}/${s1}/${s2}/$s3" ]]; then
|
|
||||||
echo "${root_dir}/${i}/${s1}/${s2}/$s3 is a dir"
|
|
||||||
if [[ ${root_dir}/${i}/${s1}/${s2}/$s3 == *JPG* || ${root_dir}/${i}/${s1}/${s2}/$s3 == *RAW* ]]; then
|
|
||||||
echo "${root_dir}/${i}/${s1}/${s2}/$s3 contains JPG/RAW"
|
|
||||||
dedup ${root_dir}/${i}/${s1}/${s2}/$s3
|
|
||||||
else
|
|
||||||
echo "${root_dir}/${i}/${s1}/${s2}/${s3} does not contain JPG/RAW"
|
|
||||||
for s4 in `ls -1 ${root_dir}/${i}/${s1}/${s2}/${s3}`; do
|
|
||||||
echo ${root_dir}/${i}/${s1}/${s2}/${s3}/$s4
|
|
||||||
if [[ -d "${root_dir}/${i}/${s1}/${s2}/${s3}/$s4" ]]; then
|
|
||||||
echo "${root_dir}/${i}/${s1}/${s2}/${s3}/$s4 is a dir"
|
|
||||||
if [[ ${root_dir}/${i}/${s1}/${s2}/${s3}/$s4 == *JPG* || ${root_dir}/${i}/${s1}/${s2}/${s3}/$s4 == *RAW* ]]; then
|
|
||||||
echo "${root_dir}/${i}/${s1}/${s2}/${s3}/$s4 contains JPG/RAW"
|
|
||||||
dedup ${root_dir}/${i}/${s1}/${s2}/${s3}/$s4
|
|
||||||
fi
|
|
||||||
fi
|
|
||||||
done
|
|
||||||
fi
|
|
||||||
fi
|
|
||||||
done
|
|
||||||
fi
|
|
||||||
fi
|
|
||||||
done
|
|
||||||
fi
|
|
||||||
fi
|
|
||||||
done
|
|
||||||
fi
|
|
||||||
fi
|
|
||||||
done
|
|
||||||
|
|
||||||
|
|
||||||
110
import_media.py
110
import_media.py
|
|
@ -1,110 +0,0 @@
|
||||||
#!/usr/bin/env python
|
|
||||||
|
|
||||||
"""
|
|
||||||
Import photos from SD card into folder with today's date + nickname
|
|
||||||
Use: import_media.py (--jpg|--raw|--both) <nickname of folder (optional)>
|
|
||||||
Add script to path
|
|
||||||
|
|
||||||
TODO:
|
|
||||||
8. Optionally allow specification of a backup location on another disk
|
|
||||||
or NAS to ship a 3rd copy to
|
|
||||||
10. Every config option has an arg override
|
|
||||||
11. Optionally rename file if EVENT name was passed in
|
|
||||||
-- STRETCH --
|
|
||||||
12. Make a graphical interface
|
|
||||||
"""
|
|
||||||
|
|
||||||
import argparse
|
|
||||||
import os
|
|
||||||
from tqdm import tqdm
|
|
||||||
|
|
||||||
### Local Imports
|
|
||||||
from configure import CONFIG_FILE, Configure, files
|
|
||||||
from file_stuff import cleanup_sd, validate_config_dir_access, is_file
|
|
||||||
from hashing import gen_xxhashes, validate_xx_checksums
|
|
||||||
from bitmover import copy_files
|
|
||||||
from lumberjack import timber
|
|
||||||
from media import process_file
|
|
||||||
|
|
||||||
c = Configure(CONFIG_FILE)
|
|
||||||
config = c.load_config()
|
|
||||||
log = timber(__name__)
|
|
||||||
log.info("Starting import_media")
|
|
||||||
|
|
||||||
parser = argparse.ArgumentParser()
|
|
||||||
parser.add_argument("-e", "--event", help = "Event Name")
|
|
||||||
parser.add_argument("-s", "--source", help = "Source Directory to search for files")
|
|
||||||
parser.add_argument("-d", "--destination", help = "Destination Directory to put files")
|
|
||||||
parser.add_argument("-o", "--create-originals", help = "For images only, create an originals \
|
|
||||||
folder for safe keeping")
|
|
||||||
parser.add_argument("-b", "--backup-destination", help = "Create a backup of everything at the \
|
|
||||||
specified location")
|
|
||||||
parser.add_argument("-D", "--delete-source-files", help = "Delete files from SD after validating \
|
|
||||||
checksum of copied files")
|
|
||||||
parser.add_argument("-v", "--verify", help = "[True|False] Verify the checksum of \
|
|
||||||
the copied file")
|
|
||||||
parser.add_argument("-c", "--config", help = "Load the specified config file instead \
|
|
||||||
of the default " + CONFIG_FILE)
|
|
||||||
parser.add_argument("-g", "--generate-config", help = "Generate config file based on options \
|
|
||||||
passed from command arguments")
|
|
||||||
|
|
||||||
args = parser.parse_args()
|
|
||||||
|
|
||||||
if args.event:
|
|
||||||
EVENT = args.event
|
|
||||||
else:
|
|
||||||
EVENT = False
|
|
||||||
if args.source:
|
|
||||||
config['folders']['source']['base'] = args.source
|
|
||||||
if args.destination:
|
|
||||||
config['folders']['destination']['base'] = args.source
|
|
||||||
#if args.create-originals:
|
|
||||||
# pass
|
|
||||||
#if args.backup-destination:
|
|
||||||
# pass
|
|
||||||
#if args.delete-source-files:
|
|
||||||
# pass
|
|
||||||
#if args.config:
|
|
||||||
# pass
|
|
||||||
#if args.generate-config:
|
|
||||||
# pass
|
|
||||||
|
|
||||||
|
|
||||||
def find_files(directory):
|
|
||||||
""" find files to build a dictionary out of """
|
|
||||||
log.debug(f'find_files({directory})')
|
|
||||||
os.system('clear')
|
|
||||||
for folder, subfolders, filename in os.walk(directory):
|
|
||||||
log.debug(f'{folder},{filename}')
|
|
||||||
for f_type in config['file_types']:
|
|
||||||
log.debug(f'Type: {f_type}')
|
|
||||||
for ext in config['file_types'][f_type]:
|
|
||||||
log.debug(f'Extension: {ext}')
|
|
||||||
os.system('clear')
|
|
||||||
for file in tqdm(filename,
|
|
||||||
desc = 'Finding ' + ext + ' Files in ' + folder):
|
|
||||||
log.debug(f'File: {file}')
|
|
||||||
if file.lower().endswith(ext):
|
|
||||||
current_file = os.path.join(folder,file)
|
|
||||||
log.debug(f'Current File: {current_file}')
|
|
||||||
if is_file(current_file):
|
|
||||||
log.debug(f'Is File: {current_file}')
|
|
||||||
log.debug(f'Call function: process_file({folder}, {file}, {EVENT}, {config})')
|
|
||||||
#process_file(folder, f_type, file, ext)
|
|
||||||
process_file(folder, file, EVENT, config)
|
|
||||||
else:
|
|
||||||
log.warn(f"Skipping {current_file} as it does not look like a real file.")
|
|
||||||
|
|
||||||
GO = validate_config_dir_access(config)
|
|
||||||
if GO is True:
|
|
||||||
find_files(config['folders']['source']['base'])
|
|
||||||
copy_files(files,config)
|
|
||||||
gen_xxhashes(files)
|
|
||||||
validate_xx_checksums(files)
|
|
||||||
cleanup_sd(files,config)
|
|
||||||
else:
|
|
||||||
log.critical('There was a problem accessing one or more directories defined in the configuration.')
|
|
||||||
|
|
||||||
|
|
||||||
# dump_yaml(files, 'files_dict.yaml')
|
|
||||||
log.info('done.')
|
|
||||||
116
rename.py
116
rename.py
|
|
@ -1,116 +0,0 @@
|
||||||
#!/usr/bin/env python
|
|
||||||
|
|
||||||
import argparse
|
|
||||||
import sys
|
|
||||||
import os
|
|
||||||
|
|
||||||
from bitmover import copy_from_source
|
|
||||||
from file_stuff import path_exists, cmp_files
|
|
||||||
from get_image_tag import get_image_date, get_exif_tag
|
|
||||||
from hashing import xx_hash
|
|
||||||
|
|
||||||
parser = argparse.ArgumentParser()
|
|
||||||
parser.add_argument("-f", "--folder", help = "folder with files to rename")
|
|
||||||
parser.add_argument("-o", "--keeporiginalname", help = "keeps the original name attached to the file.")
|
|
||||||
parser.add_argument("-d", "--dryrun", help = "dry run, no action")
|
|
||||||
|
|
||||||
args = parser.parse_args()
|
|
||||||
|
|
||||||
if args.folder:
|
|
||||||
FOLDER = args.folder
|
|
||||||
else:
|
|
||||||
print("you need to specify a folder. Duh.")
|
|
||||||
sys.exit()
|
|
||||||
if args.dryrun:
|
|
||||||
dry_run = True
|
|
||||||
else:
|
|
||||||
dry_run = False
|
|
||||||
|
|
||||||
if args.keeporiginalname:
|
|
||||||
keep_orig_name = True
|
|
||||||
else:
|
|
||||||
keep_orig_name = False
|
|
||||||
|
|
||||||
def get_file_size(f):
|
|
||||||
return os.path.getsize(f)
|
|
||||||
|
|
||||||
for file in os.listdir(FOLDER):
|
|
||||||
if file.lower().endswith("gif"):
|
|
||||||
copy_from_source(FOLDER,'/Volumes/VIDEO_ARRAY_01/Multimedia/gif',file)
|
|
||||||
file_match = cmp_files(os.path.join(FOLDER,file),
|
|
||||||
os.path.join('/Volumes/VIDEO_ARRAY_01/Multimedia/gif',file))
|
|
||||||
|
|
||||||
if file_match is True:
|
|
||||||
os.remove(os.path.join(FOLDER,file))
|
|
||||||
|
|
||||||
if file.lower().endswith("png"):
|
|
||||||
copy_from_source(FOLDER,'/Volumes/VIDEO_ARRAY_01/Multimedia/png',file)
|
|
||||||
|
|
||||||
file_match = cmp_files(os.path.join(FOLDER, file),
|
|
||||||
os.path.join('/Volumes/VIDEO_ARRAY_01/Multimedia/gif', file))
|
|
||||||
|
|
||||||
if file_match is True:
|
|
||||||
os.remove(os.path.join(FOLDER, file))
|
|
||||||
|
|
||||||
if file.lower().endswith("heic"):
|
|
||||||
copy_from_source(FOLDER,'/Volumes/VIDEO_ARRAY_01/Multimedia/heic',file)
|
|
||||||
|
|
||||||
file_match = cmp_files(os.path.join(FOLDER, file),
|
|
||||||
os.path.join('/Volumes/VIDEO_ARRAY_01/Multimedia/heic', file))
|
|
||||||
|
|
||||||
if file_match is True:
|
|
||||||
os.remove(os.path.join(FOLDER, file))
|
|
||||||
|
|
||||||
if file.lower().endswith("jpg") or \
|
|
||||||
file.lower().endswith("jpeg") or \
|
|
||||||
file.lower().endswith("nef") or \
|
|
||||||
file.lower().endswith("rw2") or \
|
|
||||||
file.lower().endswith("arw") or \
|
|
||||||
file.lower().endswith("dng"):
|
|
||||||
|
|
||||||
old_path = os.path.join(FOLDER,file)
|
|
||||||
|
|
||||||
lowered_name = file.lower()
|
|
||||||
file_size = get_file_size(old_path)
|
|
||||||
file_extension = os.path.splitext(lowered_name)[1]
|
|
||||||
image_date = get_image_date(old_path)
|
|
||||||
image_date_year = image_date.strftime("%Y")
|
|
||||||
image_date_month = image_date.strftime("%m")
|
|
||||||
image_date_day = image_date.strftime("%d")
|
|
||||||
image_date_hour = image_date.strftime("%H")
|
|
||||||
image_date_minute = image_date.strftime("%M")
|
|
||||||
image_date_second = image_date.strftime("%S")
|
|
||||||
image_date_microsecond = image_date.strftime("%f")
|
|
||||||
image_date_subsecond = str(get_exif_tag(old_path,'EXIF SubSec'))
|
|
||||||
image_hash = xx_hash(old_path)
|
|
||||||
|
|
||||||
if image_date_subsecond:
|
|
||||||
subsecond_desired_length = 6
|
|
||||||
|
|
||||||
if image_date_subsecond == 'None':
|
|
||||||
image_date_subsecond = subsecond_desired_length*str('0')
|
|
||||||
|
|
||||||
l_image_date_subsecond = len(image_date_subsecond)
|
|
||||||
|
|
||||||
if subsecond_desired_length > l_image_date_subsecond:
|
|
||||||
pad = subsecond_desired_length - l_image_date_subsecond
|
|
||||||
image_date_subsecond = image_date_subsecond + pad*str("0")
|
|
||||||
|
|
||||||
new_name = f'{image_date_year}-{image_date_month}-{image_date_day}-{image_date_hour}{image_date_minute}{image_date_second}{image_date_subsecond}_{file_size}_{image_hash}'
|
|
||||||
else:
|
|
||||||
new_name = f'{image_date_year}-{image_date_month}-{image_date_day}-{image_date_hour}{image_date_minute}{image_date_second}000_{file_size}_{image_hash}'
|
|
||||||
|
|
||||||
if keep_orig_name is True:
|
|
||||||
new_file_name = f'{new_name}-{lowered_name}'
|
|
||||||
else:
|
|
||||||
new_file_name = f'{new_name}{file_extension}'
|
|
||||||
|
|
||||||
new_path = os.path.join(FOLDER,new_file_name)
|
|
||||||
|
|
||||||
if path_exists(new_path):
|
|
||||||
print(f"{new_path} exists.. skipping.")
|
|
||||||
elif dry_run is True:
|
|
||||||
print(f'Dry run: {old_path} becomes {new_path}')
|
|
||||||
else:
|
|
||||||
print(f'Renaming {old_path} to: {new_path}')
|
|
||||||
os.rename(old_path,new_path)
|
|
||||||
Loading…
Reference in New Issue