rawdatapath = fullfile(fileparts(which(mfilename)) ,'..', '..', 'raw', 'grid');
destpath = fullfile(fileparts(which(mfilename)) ,'..', '..', 'raw', 'odd');

file_incl          = '\.asc$';
dir_excl           = '.svn$';
%cleaning of the vaklodingen raw files
D = dir2(rawdatapath,...
    'file_incl', file_incl,...
    'dir_excl', dir_excl,...
    'case_sensitive', 0,...
    'no_dirs', true,...
    'depth', inf);
multiWaitbar('Processing asc files',0)

cellfun(@(x) mkdir(fullfile(destpath, x)), {'commented' 'shifted' 'wrong_shape'})

for ii = 1:length(D);
    fname = [D(ii).pathname D(ii).name];
    fid = fopen(fname);
    s = textscan(fid,'%s %f',6);
    if isempty(s{2})
        txt = fread(fid, '*char')';
        fclose(fid);
        [id, commentline] = regexp(txt, '^.*(?=ncols)', 'end', 'match');
        fid = fopen(fname, 'w');
        fprintf(fid, '%s', txt(id+1:end));
        disp(D(ii).name);
        disp(['removed: ' commentline]);
        s = textscan(txt(id+1:end),'%s %f',6);
        clear txt;
    end
    ncols        = s{2}(strcmpi(s{1},'ncols'        ));
    nrows        = s{2}(strcmpi(s{1},'nrows'        ));
    xllcorner    = s{2}(strcmpi(s{1},'xllcorner'    ));
    yllcorner    = s{2}(strcmpi(s{1},'yllcorner'    ));
    cellsize     = s{2}(strcmpi(s{1},'cellsize'     ));
    nodata_value = s{2}(strcmpi(s{1},'nodata_value'));
    data = textscan(fid,'%f64', 'CollectOutput',true);
    fclose(fid);
    
    if isempty(ncols)||isempty(nrows)||isempty(xllcorner)||isempty(yllcorner)||isempty(cellsize)||isempty(nodata_value)
        disp(D(ii).name);
        disp('error reading file');
        movefile([D(ii).pathname D(ii).name], fullfile(destpath,'commented', D(ii).name),'f');
    elseif mod(xllcorner,cellsize) >0 || mod(yllcorner,cellsize) >0
        disp(D(ii).name);
        fprintf('dx: %g\ndy: %g\n', mod(xllcorner,cellsize), mod(yllcorner,cellsize));
        movefile([D(ii).pathname D(ii).name],fullfile(destpath,'shifted', D(ii).name),'f');
    elseif ~(numel(data{:}) / ncols == nrows)
        disp(D(ii).name);
        disp('wrong shape')
        movefile([D(ii).pathname D(ii).name],fullfile(destpath,'wrong_shape', D(ii).name),'f');
    end
    multiWaitbar('Processing asc files',ii/length(D))
end
