function rws_waterbase_all %RWS_WATERBASE_ALL download waterbase.nl parameters from web, transform to netCDF, make kml, make catalog. % %See also: KNMI_ALL, NC_CF_HARVEST, RWS_WATERBASE_* % TO DO: merge kmls for al substances % TO DO: move only identical DONAR ID to old %% Initialize OPT.download = 0; % get fresh downloads from rws and move exisitng to sub dir old OPT.make_nc = 0; % makes also temporary mat files, moves exisiting nc to old subdir OPT.make_catalog = 0; % otherwise load existing one OPT.make_kml = 1; % processing all kml only takas about 4 hours OPT.baseurl = 'http://live.waterbase.nl'; OPT.institution = 'rijkswaterstaat'; % for construcitng relative path OPT.dataset = 'waterbase'; % for construcitng relative path OPT.directory_xml = 'd:\checkouts\OpenEarthTools\configurations\dtvirt5.deltares.nl\'; % for static THREDDS catalogs in folder tomcat6/content/thredds %rawbase = '/Users/fedorbaart/Downloads/rws/raw/';% @ local, change this for your own computer %ncbase = '/Users/fedorbaart/Downloads/rws/nc/'; % @ local, change this for your own computer %kmlbase = '/Users/fedorbaart/Downloads/rws/kml/';% @ local, change this... no links to other kml or images any more rawbase = 'd:\checkouts\OpenEarthRawData\'; % @ local ncbase = 'D:\opendap.deltares.nl\thredds\dodsC\opendap\'; % @ local urlbase = 'http://opendap.deltares.nl/thredds/dodsC/opendap/'; % @ server kmlbase = 'D:\kml.deltares.nl\'; % @ local %% Parameter choice: select DONAR code or % 0=all or select number from 'donar_wnsnum' column in rws_waterbase_name2standard_name.xls % DO get 1 always after 54 to make sure catalog and kml of 1 contains 54 as well. donar_wnsnum = [ 559 44 282 410 209 ... % sal T Chl SPM pO2 29 54 22 23 24 ... % Q eta Hs dir Tm 332 346 347 360 363 ... % KjN N N O2 PO4 364 380 491 492 493 ... % P P04 NH4 N02 N03 541 560 1083 1 377 ... % DSe Si DOC zwl pH 1238 713 1082 401 ]; % NO3NO2 E POC TOC donar_wnsnum = [282] %1082 Use this if you want only an update of one some specific parameter. mfilename('fullpath') DONAR = xls2struct([fileparts(mfilename('fullpath')) filesep 'rws_waterbase_name2standard_name.xls']); if donar_wnsnum==0 donar_wnsnum = DONAR.donar_wnsnum; end multiWaitbar(mfilename,0,'label','Looping substances.','color',[0.3 0.8 0.3]) %% Parameter loop n = 0; for ivar=[donar_wnsnum] n = n+1; disp(['Processing donar_wnsnum: ',num2str(ivar)]) index = find(DONAR.donar_wnsnum==ivar); OPT.code = DONAR.donar_wnsnum(index); OPT.donar_wnsnum = DONAR.donar_wnsnum(index); % needed for url OPT.donar_wns_oms = DONAR.donar_wns_oms{index}; % needed for url OPT.donar_parcode = DONAR.donar_parcode{index}; % needed for disp OPT.name = DONAR.name{index}; % needed for directory subdir = OPT.name; OPT.directory_nc = fullfile( ncbase,OPT.institution,OPT.dataset,subdir,filesep); OPT.directory_kml = fullfile(kmlbase,OPT.institution,OPT.dataset,filesep); OPT.directory_raw = fullfile(rawbase,OPT.institution,OPT.dataset,'cache',subdir,filesep); multiWaitbar(mfilename,n/length(donar_wnsnum),'label',['Processing substance: ',OPT.donar_parcode,' (',num2str(ivar),')']) %% Download from waterbase.nl if OPT.download rws_waterbase_get_url_loop('donar_wnsnum' ,OPT.code,... 'directory_raw',OPT.directory_raw,... 'directory_raw_old',[OPT.directory_raw filesep 'old'],... 'cleanup',1); % remove date from file name > version control on cached download too ? end %% Read raw, cache as *.mat and make netCDF if OPT.make_nc if any(OPT.code==[1 54 29 22 23 24]) % physical parameters have long time series: waterlevels(1 54), Q(29) or waves(22 23 24) OPT.method='fgetl'; else OPT.method='textread'; end %% TO DO: copy exisitng nc files to \OLD rws_waterbase2nc('donar_wnsnum' ,OPT.code,... 'directory_nc' ,OPT.directory_nc,... 'directory_raw',OPT.directory_raw,... 'method',OPT.method,... % 'fgetl' for water levels or discharges 'att_name',{},... 'att_val',{},... 'load',1,...% 1 = skip cached mat file, always load zipped txt file 'debug',0,...% check unit conversion and more 'mask',['id' num2str(OPT.code) '*.zip']); % as more ids are in same dir end %% Make catalog.nc (and write human readable subset to catalog.xls) % make sure urlPath alreayd links to place where we are going to put them. % so we can copy catalog.nc together with the other nc files. % For making kml below we use local still files ! % Idea: make a special *_local_machine catalog? Be aware this causes % catalog to contain non-expected files. if OPT.make_catalog CATALOG = nc_cf_harvest(OPT.directory_nc,... % dir where to READ netcdf 'featuretype','timeseries',... 'debug',[],... 'catalog.nc',[OPT.directory_nc ,'catalog.nc'],... % dir where to SAVE catalog 'catalog.xml',[OPT.directory_xml,OPT.institution,'_',mkvar(OPT.dataset),'_',subdir,'.xml'],... % dir where to SAVE catalog 'catalog.xls',[OPT.directory_nc ,'catalog.xls'],... % dir where to SAVE catalog 'catalog.mat',[OPT.directory_nc ,'catalog.mat'],... % dir where to SAVE catalog 'urlPathFcn',@(s) path2os(strrep(s,ncbase,urlbase),'h'),... % dir where to LINK to for netCDF 'disp','multiWaitbar',... 'ID',[OPT.institution,'/',mkvar(OPT.dataset),'/',subdir],... 'name',[OPT.institution,'_',mkvar(OPT.dataset),'_',subdir]); end %% Make KML overview with links to netCDFs on http://opendap.deltares.nl THREDDS if OPT.make_kml if (~OPT.make_catalog) CATALOG = nc2struct([OPT.directory_nc,'catalog.nc']); end OPT2.fileName = [OPT.directory_kml,filesep,subdir,'.kml']; OPT2.kmlName = ['Rijkswaterstaat time series ' subdir]; OPT2.text = {['',OPT.name,'']}; %OPT2.iconnormalState = 'http://maps.google.com/mapfiles/kml/shapes/placemark_square.png'; %OPT2.iconhighlightState = 'http://www.rijkswaterstaat.nl/images/favicon.ico'; %OPT2.description = {['data: Rijkswaterstaat (http://www.rws.nl) via (',OPT.baseurl,'), presentation: http://www.OpenEarth.eu']}; OPT2.description = ['
Credits: | |
data source | Rijkswaterstaat |
data source url | http://www.rws.nl |
data provider | ',OPT.baseurl,' | ',... '
data distributor | http://www.OpenEarth.eu | ',... '
$HeadURL$ $Id$
']; OPT2.lon = 1; OPT2.lat = 54; OPT2.z = 100e4; OPT2.logokmlName = {}; %{'Rijkswaterstaat logo','OpenEarth logo'}; OPT2.overlayXY = {[.5 1],[0 0.00]}; OPT2.screenXY = {[.5 1],[0.01 0.03]}; OPT2.imName = {'http://www.rws.nl/en/images/ENRO_VW_RW~LI.png',[fileparts(oetlogo),filesep,'OpenEarth-logo-blurred-white-background4kml.png'];}; OPT2.logoName = {'overheid4GE.png','OpenEarth-logo-blurred-white-background4GE.png'}; OPT2.varPathFcn = @(s) path2os(strrep(s,urlbase,ncbase),filesep); % use local netCDF files for preview/statistics when CATALOG refers already to server OPT2.resolveUrl = cellfun(@(x) ['http://live.waterbase.nl/waterbase_dbh.cfm?loc=',upper(x),'&page=start.locaties.databeschikbaarheid&taal=nl&loc=&wbwns=',num2str(OPT.donar_wnsnum),'|',strtrim(strrep(OPT.donar_wns_oms,' ','+')),'&whichform=2'],CATALOG.platform_id,'un',0); OPT2.resolveName = 'www.rws.nl (waterbase)'; OPT2.iconnormalStateScale = 1; OPT2.iconhighlightStateScale = 1; OPT2.iconnormalState = 'http://www.ndbc.noaa.gov//images/maps/markers/tiny_active_marker.png'; OPT2.iconhighlightState = 'http://www.ndbc.noaa.gov//images/maps/markers/tiny_active_marker.png'; OPT2.credit = ' data: www.rws.nl plot: www.OpenEarth.eu'; OPT2.varname = subdir; OPT2.name = OPT.name; OPT2.preview = 1; OPT2.open = 0; % too slow nc_cf_harvest_matrix2kml(CATALOG,OPT2); % inside urlPath is used to read netCDF data for plotting previews close all end end multiWaitbar(mfilename,1,'label','Looping substances.')