%%NOAA_NDBC  crawl, harvest and kml overview of NDBC buoys from opendap server
%
% http://www.ndbc.noaa.gov/
% http://dods.ndbc.noaa.gov/
% http://dods.ndbc.noaa.gov/thredds/catalog/data/
%
%See aLSO: RWS_WATERBASE_ALL, KNMI_ALL, NC_CF_HARVEST

%% crawl, harvest and cache
% http://www.whoi.edu/virtual/oceansites/data/index.html
% http://www.eurosites.info/data.php
% http://dods.ndbc.noaa.gov/thredds/catalog/data/oceansites/catalog.html
% http://www.jcommops.org/dbcp/data/access.html
clear all

OPT.crawl   = 0;
OPT.harvest = 0;

OPT.data_type = 'dart';
OPT.data_type = 'ocean'; % TO DO: 'oceansites', 'swden

if OPT.crawl
   L = opendap_catalog(['http://dods.ndbc.noaa.gov/thredds/catalog/data/',OPT.data_type,'/'],'maxlevel',2,'debug',0)
   savestr(['ndbc_',OPT.data_type,'_locations.txt'],L)
end

%% harvest and cache

if OPT.harvest
   L = loadstr(['ndbc_',OPT.data_type,'_locations.txt'])
   CATALOG = nc_cf_harvest(L)
   save CATALOG
else
   load CATALOG
end

   CATALOG.platform_id   = cellfun(@(x)last_subdir(x,1),CATALOG.urlPath,'UniformOutput',0);
   CATALOG.platform_name = CATALOG.platform_id;
   nc_cf_harvest_matrix2nc (['ndbc_',OPT.data_type,'_locations.nc' ],CATALOG);
   nc_cf_harvest_matrix2xml(['ndbc_',OPT.data_type,'_locations.xml'],CATALOG);
   nc_cf_harvest_matrix2xls(['ndbc_',OPT.data_type,'_locations.xls'],CATALOG);

%% kml

if strcmpi(OPT.data_type,'dart')
   warning('TO DO: remove buoys without location')
   warning('TO DO: merge nc files from same buoy')
end

   CATALOG = nc2struct(['ndbc_',OPT.data_type,'_locations.nc']);

   OPT2.fileName           = ['ndbc_',OPT.data_type,'_locations.kml'];
   OPT2.kmlName            = ['NOAA time series: ',OPT.data_type];
   OPT2.text               = {['<B>NDBC data</B>']};
   OPT2.description        = ['<hr> This is a proof-of-concept demo of how time series of the DODS data distribution'...
                              'from NOAA could be presented in Google Earth for easy navigation in time-space. '...
                              'The data in this proof-of-concept demo is a cache that is updated a few times per year.'...
                              '<hr><table bgcolor="#333333" cellpadding="3" cellspacing="1"><tbody><tr><td colspan="2" bgcolor="#666666"><div style="color:#FFFFFF;">Credits:</div></td></tr>',...
                              '<tr><td    bgcolor="#FFFFFF">data source     </td><td bgcolor="#FFFFFF">NOAA</td></tr>',...
                              '<tr><td    bgcolor="#FFFFFF">data source url </td><td bgcolor="#FFFFFF">http://www.ndbc.noaa.gov/</td></tr>',...
                              '<tr><td    bgcolor="#FFFFFF">data provider   </td><td bgcolor="#FFFFFF">OpenEarth</td>',...
                              '<tr><td    bgcolor="#FFFFFF">data distributor</td><td bgcolor="#FFFFFF">http://www.OpenEarth.eu</td>',...
                              '</tr></tbody></table><hr>'];      
                          
   OPT2.lon                = -90;
   OPT2.lat                = 25;
   OPT2.z                  = 10e6;
   
   OPT2.logokmlName        = {'NOAA logo','OpenEarth logo'};
   OPT2.overlayXY          = {[0    0   ],[0    0   ]};
   OPT2.screenXY           = {[0.01 0.12],[0.01 0.03]};
   OPT2.imName             = {'https://www.ess.uci.edu/sites/ess.uci.edu/files/images/NOAA-Transparent-Logo.thumbnail.png',[fileparts(oetlogo),filesep,'OpenEarth-logo-blurred-white-background4kml.png'];};
   OPT2.logoName           = {'noaalogo4GE.png','oet4GE.png'}; % http://oceanservice.noaa.gov/facts/noaalogo.jpg
   
   OPT2.resolveUrl         = cellfun(@(x) ['http://www.ndbc.noaa.gov/station_page.php?station=',x],CATALOG.platform_id,'un',0);
   OPT2.resolveName        = 'National Buoy Data Centre';
   
   OPT2.iconnormalStateScale    = 1;
   OPT2.iconhighlightStateScale = 1;  
   OPT2.iconnormalState    = 'http://www.ndbc.noaa.gov//images/maps/markers/tiny_active_marker.png';
   OPT2.iconhighlightState = 'http://www.ndbc.noaa.gov//images/maps/markers/tiny_active_marker.png';

   nc_cf_harvest_matrix2kml(CATALOG,OPT2);
   nc_cf_harvest_matrix2nc ('ndbc_catalog.nc',CATALOG);nc_dump('ndbc_catalog.nc')

%% THREDDS catalog

   OPT.ID                      = ['NOAA/',OPT.data_type];
   OPT.name                    = ['NOAA_',OPT.data_type];
   
   OPT.creator.name            = 'OpenEarth';
   OPT.creator.contact.url     = 'http://www.openearth.eu';
   OPT.creator.contact.email   = 'gerben.deboer@deltares.nl';
   
   OPT.publisher.name          = 'NOAA NDBC';
   OPT.publisher.contact.url   = 'http://www.ndbc.noaa.gov/';
   OPT.publisher.contact.email = '';
   
   OPT.dataType                = ''; % 'GRID'
   
   OPT.documentation.summary   = 'National Data Buoy Center';
   OPT.documentation.title     = 'National Data Buoy Center';
   OPT.documentation.url       = 'http://dods.ndbc.noaa.gov/';
   
   nc_cf_harvest_matrix2xml('ndbc_catalog_static.xml',CATALOG,'ID',OPT.data_type)


