Script to delete duplicate files

Code:

#!/bin/bash

HASH=md5sum

INODE=$(stat -c %i ${1:-.})

MDF="/tmp/deldup-$INODE.md5"

DUP="/tmp/deldup-$INODE.list"

if [ $MDF -nt . ]; then

    echo found cached hash list in $MDF

else 

    echo -n "hashing..."

    FC=$(find ${@:-.} -type f | wc -l)

    while true; do

        P=$(cat $MDF | wc -l)

        echo -en "\rhashing files ($P/$FC)... "

        sleep 0.25

    done &

    find -O2 ${1:-.} -type f -print0 | xargs -L 1 -P 8 -0 md5sum > $MDF

    kill $!

    wait $! 2> /dev/null

fi



cut -f1 -d" " < $MDF | sort | uniq -d > $DUP



T=$(cat $DUP | wc -l)

N=1

echo Found $T duplicate files

for dh in $(cat $DUP); do

    echo

    echo \[$N/$T\] Files with hash $dh:

    grep -a $dh $MDF | cut -f2- -d" " | { 

        readarray -t -O 1 FILES



        C=1 # counter

        for line in $(seq ${#FILES[@]}); do

            printf "%4d\t%s\n" $C "${FILES[$C]}"

            let C++

        done



        echo -n 'Select to delete (eg. "1", "2,3,4", "!", "1,3-5"): '

        IFS=',' read -ra RANGE <&1; 

        if [ "${RANGE[1]}" = "!" ]; then # "!" deletes all matches

            for i in "${FILES[@]}"; do

                rm -v "$(echo $i | xargs -0 echo)"

            done

        else

            for i in "${RANGE[@]}"; do

                if [[ $i =~ ^([0-9]+)-([0-9]+)$ ]]; then

                    START=${BASH_REMATCH[1]}

                    END=${BASH_REMATCH[2]}

                    for x in $(seq $START $END); do

                        rm -v "$(echo ${FILES[$x]} | xargs -0 echo)"

                    done

                elif [[ $i =~ ^[0-9]+$ ]]; then

                    rm -v "$(echo ${FILES[$i]} | xargs -0 echo)"

                else

                    echo Invalid range or number \"$i\"

                fi

            done

        fi

    }

    let N++

done



echo

read -p "delete cache [y/N]? " DC

[[ $DC =~ ^[yY] ]] && rm -v $MDF && rm -v $DUP

Code:

#!/bin/sh

# read file list from input, output all duplicates on a single line, separated by tabs

# eg: find duplicates and only keep one:

# find . -type f | ./getdup | cut -f2- | xargs rm



# write hash + filename in a temp file, sorted by hash

TMP=$(mktemp)

xargs -n1 sha1sum | sort > $TMP



# find duplicate hash, and match them in the list

for SHA1 in $(cut -d' ' -f1 < $TMP | uniq -d); do

    # print all files for each hash separated by tabs

    # assume files don't include spaces, of course...

    grep $SHA1 < $TMP | cut -d' ' -f2- | xargs echo | tr ' ' '\t'

done



rm $TMP