function varargout = hash_files(D,varargin)
%HASH_FILES  Hashes all files and folders in a structure returned by dir2
%
%   This function appends the MD5 hashes of to a  directory structure D.
%   This directory structure should be generated by the dir2 function. 
%
%   If the option store_hash is set to true, the hash itself, the filesize and
%   the modification_date are stored in a file that is put next to the file
%   that is hashed with an .md5 extension. If no changes are found in the
%   the file size and the modification date, the hash is not recomputed but
%   extracted from the .md5 file. As this is not a guarantee the file has
%   not changed, the option store_hash false by default
%
%   Syntax:
%   varargout = hash_files(varargin)
%
%   Input:
%   D  = structure returned by dir2
%
%   Output:
%   varargout =
%
%   Example
%   D = hash_files(dir2('no_dirs',1))
%
%   See also:
%   dir2, CalcMD5

%% Copyright notice
%   --------------------------------------------------------------------
%   Copyright (C) 2012 Van Oord
%       Thijs Damsma
%
%       tda@vanoord.com
%
%       Watermanweg 64
%       3067 GG
%       Rotterdam
%       Netherlands
%
%   This library is free software: you can redistribute it and/or modify
%   it under the terms of the GNU General Public License as published by
%   the Free Software Foundation, either version 3 of the License, or
%   (at your option) any later version.
%
%   This library is distributed in the hope that it will be useful,
%   but WITHOUT ANY WARRANTY; without even the implied warranty of
%   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
%   GNU General Public License for more details.
%
%   You should have received a copy of the GNU General Public License
%   along with this library.  If not, see <http://www.gnu.org/licenses/>.
%   --------------------------------------------------------------------

% This tool is part of <a href="http://www.OpenEarth.eu">OpenEarthTools</a>.
% OpenEarthTools is an online collaboration to share and manage data and
% programming tools in an open source, version controlled environment.
% Sign up to recieve regular updates of this function, and to contribute
% your own tools.

%% Version <http://svnbook.red-bean.com/en/1.5/svn.advanced.props.special.keywords.html>
% Created: 08 Mar 2012
% Created with Matlab version: 7.14.0.834 (R2012a)

% $Id$
% $Date$
% $Author$
% $Revision$
% $HeadURL$
% $Keywords: $

%%
OPT.method      = 'MD5';
OPT.quiet       = false;
OPT.store_hash  = false;

OPT = setproperty(OPT,varargin{:});

if nargin==0;
    varargout = OPT;
    return;
end
%% check input

% check if the input has all of the fields that are output by dir2.
requiredfields =  {
    'name'
    'date'
    'bytes'
    'isdir'
    'datenum'
    'pathname'
    };
if ~all(ismember(requiredfields,fieldnames(D)))
    error('HASH_FILES:input','input must be a struct returned by dir2');
end

% check method
switch upper(OPT.method)
    case 'MD5'
        hashfun = @(filename) uint8(CalcMD5(filename,'File','dec'));
    otherwise
        error('HASH_FILES:input','hash mathod %s not supported',OPT.method);
end

% make sure no hashes of hash files are created, as this probably indicates
% a D structure created by a not strict enough query, and would result in
% an ever growing list of files every time hashes for that directory are
% computed.
if OPT.store_hash
    if any(~cellfun(@isempty,regexp({D.name},['\.' lower(OPT.method) '$'],'once')))
        error('HASH_FILES:input','Stored hash files are found in folder structure D. Add the following to the call of dir2:\n''file_excl'',''\\.%s$''',lower(OPT.method));
    end
end

% cthrow a warning if file hashes are stored for small files (<1mb), as
% this degraded performance
if OPT.store_hash
    if sum([D(~[D.isdir]).bytes])/sum(~[D.isdir]) < 2^20
        warning('HASH_FILES:small_files','storing the hashes of files is not recommended when the average filesize is smaller than 1 MB as this degrades performance');
    end
end

%% initialize waitbar
if ~OPT.quiet
    WB.small_file_correction_bytes = 2e6;
    WB.totalbytes   = sum([D(~[D.isdir]).bytes])+sum(~[D.isdir])*WB.small_file_correction_bytes;
    WB.hashedbytes = 0;
    t = tic;
end


%% loop through files
for ii = find(~[D.isdir])
    already_hashed = false;
    
    % compare size and date of previously computed hash
    if OPT.store_hash
        hashdata_filename = [D(ii).pathname D(ii).name '.' lower(OPT.method)];
        if exist(hashdata_filename,'file')
            % read hashdata file with date, time and bytes
            fid             = fopen(hashdata_filename);
            hashdata_infile = fread(fid,'*char')';
            fclose(fid);
            
            % extract filehash from hashdata_infile with a strict regular
            % expression
            filehash_hex = char(regexp(hashdata_infile,['^' OPT.method '_hash = ([0-9a-z]{32}$)'],'once','tokens','lineanchors'));
            
            % prepare a hashdata_string using the file attributes from the
            % D structure, combined with the hash extracted from the hash
            % file.
            hashdata_string = sprintf('name = %s\ndate = %s\nbytes = %d\n%s_hash = %s',D(ii).name, D(ii).date, D(ii).bytes, OPT.method, filehash_hex);
            
            % if the file attributes date and size remain identical, the
            % hashdata (including attributes) read from the file will be
            % identical to the hashdata_string. In that case assume the
            % file has not changed, and thus the hash is not recomputed
            if strcmp(hashdata_infile,hashdata_string)
                already_hashed = true;
            end
        end
    end
    
    if ~already_hashed
        % compute the hash if the previously computed hash is rejected / not available
        D(ii).hash     = hashfun([D(ii).pathname D(ii).name]);
        if OPT.store_hash
            filehash_hex    = reshape(lower(dec2hex(D(ii).hash)'),1,[]);
            % recopmute hash string with new hash value, and write it to a
            % file
            hashdata_string = sprintf('name = %s\ndate = %s\nbytes = %d\n%s_hash = %s',D(ii).name, D(ii).date, D(ii).bytes, OPT.method, filehash_hex);
            fid             = fopen(hashdata_filename,'w');
            fwrite(fid,hashdata_string,'char*1');
            fclose(fid);
        end
    else
        % compute uint8 value of the hexadecimal hash
        D(ii).hash = uint8(hex2dec(reshape(filehash_hex,2,[])'))';
    end
    
    if ~OPT.quiet
        WB.hashedbytes = WB.hashedbytes + D(ii).bytes + WB.small_file_correction_bytes;
        if toc(t)>0.4
            t = tic;
            multiWaitbar('hashing files',WB.hashedbytes/WB.totalbytes)
        end
    end
end

%% finalize
if ~OPT.quiet
    multiWaitbar('hashing files','close')
end


varargout = {D,OPT};
