{ "metadata": { "name": "RWS Open Data" }, "nbformat": 3, "nbformat_minor": 0, "worksheets": [ { "cells": [ { "cell_type": "code", "collapsed": false, "input": [ "import urllib2\n", "import pandas\n", "from zipfile import ZipFile\n", "import io\n", "import dateutil\n", "import re\n", "import pyproj\n", "\n", "url = 'http://www.rws.nl/rws/opendata/meetdata/meetdata.zip'" ], "language": "python", "metadata": {}, "outputs": [], "prompt_number": 689 }, { "cell_type": "code", "collapsed": false, "input": [ "f = urllib2.urlopen(url)\n", "bytes = io.BytesIO(f.read())\n", "zipfile = ZipFile(bytes)\n", "f.close()" ], "language": "python", "metadata": {}, "outputs": [], "prompt_number": 690 }, { "cell_type": "code", "collapsed": false, "input": [ "header = zipfile.read('update.adm')\n", "data = zipfile.read('update.dat')" ], "language": "python", "metadata": {}, "outputs": [], "prompt_number": 691 }, { "cell_type": "code", "collapsed": false, "input": [ "names = ('source',\n", " 'location_code',\n", " 'empty',\n", " 'parameter_code',\n", " 'location_name',\n", " 'parameter_name',\n", " 'units',\n", " 'date_start',\n", " 'date_end',\n", " 'format')\n", "\n", "df_header = pandas.read_csv(io.BytesIO(header),names=names)\n", "df_data = pandas.read_csv(io.BytesIO(data),names=range(6),index_col=False)\n", "\n", "for k in df_header.keys():\n", " df_header[k] = df_header[k].apply(str.strip)\n", "\n", "df = pandas.concat((df_data, df_header),axis=1)\n", "melted = pandas.melt(df,id_vars=names, value_vars=range(6), var_name='time_index')\n", "\n", "melted['date_start'] = melted['date_start'].apply(dateutil.parser.parse)\n", "melted['date_end'] = melted['date_end'].apply(dateutil.parser.parse)\n", "melted['time_delta'] = melted['time_index'].apply(lambda x: datetime.timedelta(minutes=int(x)*10))\n", "melted['date'] = melted['date_start'] + melted['time_delta']\n", "melted['value'] = melted['value'].apply(str.strip)\n", "melted['value'] = melted['value'].replace(['f', 'n'],None).astype('float')\n", "melted['time'] = melted['date'].apply(lambda x: time.mktime(x.timetuple()))\n", "\n", "del melted['empty']" ], "language": "python", "metadata": {}, "outputs": [], "prompt_number": 692 }, { "cell_type": "code", "collapsed": false, "input": [ "h = melted[(melted['parameter_code'] == 'H10') & (melted['location_code'] == 'SCHE')]\n", "h" ], "language": "python", "metadata": {}, "outputs": [ { "html": [ "
\n", "\n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", "
sourcelocation_codeparameter_codelocation_nameparameter_nameunitsdate_startdate_endformattime_indexvaluetime_deltadatetime
348 LMW SCHE H10 Scheveningen Waterhoogte cm NAP2013-08-14 17:00:002013-08-14 17:50:00 ASCII 0-6600:00:002013-08-14 17:00:00 1376492400
806 LMW SCHE H10 Scheveningen Waterhoogte cm NAP2013-08-14 17:00:002013-08-14 17:50:00 ASCII 1-6600:10:002013-08-14 17:10:00 1376493000
1264 LMW SCHE H10 Scheveningen Waterhoogte cm NAP2013-08-14 17:00:002013-08-14 17:50:00 ASCII 2-6300:20:002013-08-14 17:20:00 1376493600
1722 LMW SCHE H10 Scheveningen Waterhoogte cm NAP2013-08-14 17:00:002013-08-14 17:50:00 ASCII 3-6300:30:002013-08-14 17:30:00 1376494200
2180 LMW SCHE H10 Scheveningen Waterhoogte cm NAP2013-08-14 17:00:002013-08-14 17:50:00 ASCII 4-5900:40:002013-08-14 17:40:00 1376494800
2638 LMW SCHE H10 Scheveningen Waterhoogte cm NAP2013-08-14 17:00:002013-08-14 17:50:00 ASCII 5-5600:50:002013-08-14 17:50:00 1376495400
\n", "
" ], "output_type": "pyout", "prompt_number": 693, "text": [ " source location_code parameter_code location_name parameter_name units \\\n", "348 LMW SCHE H10 Scheveningen Waterhoogte cm NAP \n", "806 LMW SCHE H10 Scheveningen Waterhoogte cm NAP \n", "1264 LMW SCHE H10 Scheveningen Waterhoogte cm NAP \n", "1722 LMW SCHE H10 Scheveningen Waterhoogte cm NAP \n", "2180 LMW SCHE H10 Scheveningen Waterhoogte cm NAP \n", "2638 LMW SCHE H10 Scheveningen Waterhoogte cm NAP \n", "\n", " date_start date_end format time_index value \\\n", "348 2013-08-14 17:00:00 2013-08-14 17:50:00 ASCII 0 -66 \n", "806 2013-08-14 17:00:00 2013-08-14 17:50:00 ASCII 1 -66 \n", "1264 2013-08-14 17:00:00 2013-08-14 17:50:00 ASCII 2 -63 \n", "1722 2013-08-14 17:00:00 2013-08-14 17:50:00 ASCII 3 -63 \n", "2180 2013-08-14 17:00:00 2013-08-14 17:50:00 ASCII 4 -59 \n", "2638 2013-08-14 17:00:00 2013-08-14 17:50:00 ASCII 5 -56 \n", "\n", " time_delta date time \n", "348 00:00:00 2013-08-14 17:00:00 1376492400 \n", "806 00:10:00 2013-08-14 17:10:00 1376493000 \n", "1264 00:20:00 2013-08-14 17:20:00 1376493600 \n", "1722 00:30:00 2013-08-14 17:30:00 1376494200 \n", "2180 00:40:00 2013-08-14 17:40:00 1376494800 \n", "2638 00:50:00 2013-08-14 17:50:00 1376495400 " ] } ], "prompt_number": 693 }, { "cell_type": "code", "collapsed": false, "input": [ "h.to_json(orient='records')" ], "language": "python", "metadata": {}, "outputs": [ { "output_type": "pyout", "prompt_number": 694, "text": [ "'[{\"source\":\"LMW\",\"location_code\":\"SCHE\",\"parameter_code\":\"H10\",\"location_name\":\"Scheveningen\",\"parameter_name\":\"Waterhoogte\",\"units\":\"cm NAP\",\"date_start\":1376499600000000000,\"date_end\":1376502600000000000,\"format\":\"ASCII\",\"time_index\":0,\"value\":-66.0,\"time_delta\":0,\"date\":1376499600000000000,\"time\":1376492400.0},{\"source\":\"LMW\",\"location_code\":\"SCHE\",\"parameter_code\":\"H10\",\"location_name\":\"Scheveningen\",\"parameter_name\":\"Waterhoogte\",\"units\":\"cm NAP\",\"date_start\":1376499600000000000,\"date_end\":1376502600000000000,\"format\":\"ASCII\",\"time_index\":1,\"value\":-66.0,\"time_delta\":600000000000,\"date\":1376500200000000000,\"time\":1376493000.0},{\"source\":\"LMW\",\"location_code\":\"SCHE\",\"parameter_code\":\"H10\",\"location_name\":\"Scheveningen\",\"parameter_name\":\"Waterhoogte\",\"units\":\"cm NAP\",\"date_start\":1376499600000000000,\"date_end\":1376502600000000000,\"format\":\"ASCII\",\"time_index\":2,\"value\":-63.0,\"time_delta\":1200000000000,\"date\":1376500800000000000,\"time\":1376493600.0},{\"source\":\"LMW\",\"location_code\":\"SCHE\",\"parameter_code\":\"H10\",\"location_name\":\"Scheveningen\",\"parameter_name\":\"Waterhoogte\",\"units\":\"cm NAP\",\"date_start\":1376499600000000000,\"date_end\":1376502600000000000,\"format\":\"ASCII\",\"time_index\":3,\"value\":-63.0,\"time_delta\":1800000000000,\"date\":1376501400000000000,\"time\":1376494200.0},{\"source\":\"LMW\",\"location_code\":\"SCHE\",\"parameter_code\":\"H10\",\"location_name\":\"Scheveningen\",\"parameter_name\":\"Waterhoogte\",\"units\":\"cm NAP\",\"date_start\":1376499600000000000,\"date_end\":1376502600000000000,\"format\":\"ASCII\",\"time_index\":4,\"value\":-59.0,\"time_delta\":2400000000000,\"date\":1376502000000000000,\"time\":1376494800.0},{\"source\":\"LMW\",\"location_code\":\"SCHE\",\"parameter_code\":\"H10\",\"location_name\":\"Scheveningen\",\"parameter_name\":\"Waterhoogte\",\"units\":\"cm NAP\",\"date_start\":1376499600000000000,\"date_end\":1376502600000000000,\"format\":\"ASCII\",\"time_index\":5,\"value\":-56.0,\"time_delta\":3000000000000,\"date\":1376502600000000000,\"time\":1376495400.0}]'" ] } ], "prompt_number": 694 }, { "cell_type": "code", "collapsed": false, "input": [ "import netCDF4\n", "import os\n", "import time\n", "\n", "h = df_merged\n", "\n", "sources = h['source'].drop_duplicates()\n", "locations = h['location_code'].drop_duplicates()\n", "parameters = h['parameter_code'].drop_duplicates()\n", "tstamps = h['time'].drop_duplicates()\n", "\n", "n = 0\n", "for l, location in enumerate(locations):\n", " \n", " fname = '%s.nc' % location\n", " \n", " if not os.path.exists(fname):\n", " nc = netCDF4.Dataset(fname,'w')\n", " \n", " nc.createDimension('source',sources.count())\n", " #nc.createDimension('location',locations.count())\n", " nc.createDimension('location',1)\n", " nc.createDimension('parameter',parameters.count())\n", " nc.createDimension('time',None)\n", " \n", " nc.createVariable('source',str,('source'))\n", " nc.createVariable('location_code',str,('location'))\n", " nc.createVariable('location_name',str,('location'))\n", " nc.createVariable('x',np.int32,('location'))\n", " nc.createVariable('y',np.int32,('location'))\n", " nc.createVariable('lat',np.float32,('location'))\n", " nc.createVariable('lon',np.float32,('location'))\n", " nc.createVariable('parameter_code',str,('parameter'))\n", " nc.createVariable('parameter_name',str,('parameter'))\n", " nc.createVariable('units',str,('parameter'))\n", " nc.createVariable('time',np.int32,('time'))\n", " nc.createVariable('value',np.float32,('time','parameter','location','source'))\n", " \n", " nc.variables['source'][:] = sources.values\n", " nc.variables['location_code'][0] = locations.values[l]\n", " nc.variables['location_name'][0] = h['location_name'][locations.index].values[l]\n", " nc.variables['x'][0] = h['x'][locations.index].values[l]\n", " nc.variables['y'][0] = h['y'][locations.index].values[l]\n", " nc.variables['lat'][0] = h['lat'][locations.index].values[l]\n", " nc.variables['lon'][0] = h['lon'][locations.index].values[l]\n", " nc.variables['parameter_code'][:] = parameters.values\n", " nc.variables['parameter_name'][:] = h['parameter_name'][parameters.index].values\n", " nc.variables['units'][:] = h['units'][parameters.index].values\n", " \n", " nc.variables['x'].setncattr('EPSG',28992)\n", " nc.variables['y'].setncattr('EPSG',28992)\n", " nc.variables['lat'].setncattr('EPSG',4326)\n", " nc.variables['lon'].setncattr('EPSG',4326)\n", " \n", " nc.description = '10 min interval measurements from Rijkswaterstaat Open Data: http://www.rws.nl/rws/opendata/'\n", " nc.inistitution = 'Rijkswaterstaat'\n", " nc.history = 'Created ' + time.ctime(time.time())\n", " nc.source = url\n", " nc.location = h['location_name'][locations.index].values[l]\n", "\n", " else:\n", " nc = netCDF4.Dataset(fname,'a')\n", " \n", " # add time\n", " new_records = [(i,t) for i,t in enumerate(tstamps) if t not in nc.variables['time']]\n", " if len(new_records) > 0:\n", " idx, tstamps = zip(*new_records)\n", " nt = len(nc.dimensions['time'])\n", " nc.variables['time'][nt:nt+len(idx)] = tstamps\n", " \n", " # add data\n", " for i, source in enumerate(nc.variables['source'][:]):\n", " for j, param in enumerate(nc.variables['parameter_code'][:]):\n", " #for k, location in enumerate(nc.variables['location_code'][:]):\n", " k = 0\n", " hh = h[(h['source'] == source) & (h['parameter_code'] == param)] # & (h['location_code'] == location)]\n", " #hp = hh.pivot('time','location_code','value')\n", " if len(hh)>0:\n", " nc.variables['value'][nt:nt+len(idx),j,k,i] = hh['value'].values[list(idx)]\n", " \n", " n = n + len(new_records)\n", " \n", " nc.close()\n", " \n", "if n > 0:\n", " print 'Added %d new records' % n\n", "else:\n", " print 'No new records found'" ], "language": "python", "metadata": {}, "outputs": [ { "output_type": "stream", "stream": "stdout", "text": [ "Added 1434 new records\n" ] } ], "prompt_number": 703 }, { "cell_type": "code", "collapsed": false, "input": [ "from lxml import etree\n", "\n", "xmlurl = 'http://live.waterbase.nl/metis/cgi-bin/mivd.pl?action=value&format=xml&lang=nl&order=code&type=loc'\n", "\n", "f = urllib2.urlopen(xmlurl)\n", "xml = etree.XML(f.read())\n", "f.close()" ], "language": "python", "metadata": {}, "outputs": [], "prompt_number": 315 }, { "cell_type": "code", "collapsed": false, "input": [ "# RDF -> Seq -> li -> Location\n", "# RDF -> Seq -> li -> Location -> code\n", "# RDF -> Seq -> li -> Location -> description\n", "# RDF -> Seq -> li -> Location -> geometry -> _Features -> Featuremember -> Point -> Coordinates" ], "language": "python", "metadata": {}, "outputs": [], "prompt_number": 314 }, { "cell_type": "code", "collapsed": false, "input": [ "locations = {'location_code':[],'location_name':[],'x':[],'y':[],'lat':[],'lon':[]}\n", "\n", "WGS84 = pyproj.Proj('+init=EPSG:4326')\n", "RD = pyproj.Proj('+init=EPSG:28992')\n", " \n", "locs = xml.findall('.//{%s}li' % xml.nsmap['rdf'])\n", "for loc in locs:\n", " geom = loc.find('.//{%s}geometry' % loc.nsmap['wadi'])\n", " coords = [float(x) for x in geom.find('.//{%s}Coordinates' % geom[0].nsmap['gml']).text.split(',')]\n", " epsg = geom.find('.//{%s}Point' % geom[0].nsmap['gml']).attrib['srsName']\n", " \n", " try:\n", " lat, lon = pyproj.transform(pyproj.Proj('+init=%s' % epsg), WGS84, *coords)\n", " x, y = pyproj.transform(pyproj.Proj('+init=%s' % epsg), RD, *coords)\n", " except:\n", " continue\n", " \n", " locations['location_code'].append(loc.find('.//{%s}code' % loc.nsmap['wadi']).text)\n", " locations['location_name'].append(loc.find('.//{%s}description' % loc.nsmap['wadi']).text)\n", " \n", " locations['x'].append(x)\n", " locations['y'].append(y)\n", " \n", " locations['lat'].append(lat)\n", " locations['lon'].append(lon)\n", " " ], "language": "python", "metadata": {}, "outputs": [], "prompt_number": 647 }, { "cell_type": "code", "collapsed": false, "input": [ "df_locations = pandas.DataFrame(locations)\n", "df_locations = df_locations.ix[df_locations['location_name'].drop_duplicates().index]\n", "df_merged = pandas.merge(melted, df_locations, how='left', on='location_name', suffixes=('','_donar'))" ], "language": "python", "metadata": {}, "outputs": [], "prompt_number": 656 }, { "cell_type": "code", "collapsed": false, "input": [ "df_merged[[x is np.nan for x in df_merged['location_code_donar']]]" ], "language": "python", "metadata": {}, "outputs": [ { "html": [ "
\n",
        "<class 'pandas.core.frame.DataFrame'>\n",
        "Int64Index: 1470 entries, 0 to 2693\n",
        "Data columns (total 19 columns):\n",
        "source                 1470  non-null values\n",
        "location_code          1470  non-null values\n",
        "parameter_code         1470  non-null values\n",
        "location_name          1470  non-null values\n",
        "parameter_name         1470  non-null values\n",
        "units                  1470  non-null values\n",
        "date_start             1470  non-null values\n",
        "date_end               1470  non-null values\n",
        "format                 1470  non-null values\n",
        "time_index             1470  non-null values\n",
        "value                  1470  non-null values\n",
        "time_delta             1470  non-null values\n",
        "date                   1470  non-null values\n",
        "time                   1470  non-null values\n",
        "lat                    0  non-null values\n",
        "location_code_donar    0  non-null values\n",
        "lon                    0  non-null values\n",
        "x                      0  non-null values\n",
        "y                      0  non-null values\n",
        "dtypes: datetime64[ns](3), float64(6), int64(1), object(8), timedelta64[ns](1)\n",
        "
" ], "output_type": "pyout", "prompt_number": 672, "text": [ "\n", "Int64Index: 1470 entries, 0 to 2693\n", "Data columns (total 19 columns):\n", "source 1470 non-null values\n", "location_code 1470 non-null values\n", "parameter_code 1470 non-null values\n", "location_name 1470 non-null values\n", "parameter_name 1470 non-null values\n", "units 1470 non-null values\n", "date_start 1470 non-null values\n", "date_end 1470 non-null values\n", "format 1470 non-null values\n", "time_index 1470 non-null values\n", "value 1470 non-null values\n", "time_delta 1470 non-null values\n", "date 1470 non-null values\n", "time 1470 non-null values\n", "lat 0 non-null values\n", "location_code_donar 0 non-null values\n", "lon 0 non-null values\n", "x 0 non-null values\n", "y 0 non-null values\n", "dtypes: datetime64[ns](3), float64(6), int64(1), object(8), timedelta64[ns](1)" ] } ], "prompt_number": 672 }, { "cell_type": "code", "collapsed": false, "input": [ "df_merged[df_merged['location_name']=='Hoek van Holland']" ], "language": "python", "metadata": {}, "outputs": [ { "html": [ "
\n", "\n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", "
sourcelocation_codeparameter_codelocation_nameparameter_nameunitsdate_startdate_endformattime_indexvaluetime_deltadatetimelatlocation_code_donarlonxy
894 LMW HOEK H10 Hoek van Holland Waterhoogte cm NAP2013-08-12 16:20:002013-08-12 17:10:00 ASCII 0 -20.000:00:002013-08-12 16:20:00 1376317200 4.119881 HOEKVHLD 51.977574 67930 444000
895 LMW HOEK TW10 Hoek van Holland Watertemperatuur gr.C2013-08-12 16:20:002013-08-12 17:10:00 ASCII 0 20.100:00:002013-08-12 16:20:00 1376317200 4.119881 HOEKVHLD 51.977574 67930 444000
896 LMW HVH25 CL10 Hoek van Holland chlorositeit mg Cl/l2013-08-12 16:20:002013-08-12 17:10:00 ASCII 0 4317.000:00:002013-08-12 16:20:00 1376317200 4.119881 HOEKVHLD 51.977574 67930 444000
897 LMW HOEK H10 Hoek van Holland Waterhoogte cm NAP2013-08-12 16:20:002013-08-12 17:10:00 ASCII 1 -16.000:10:002013-08-12 16:30:00 1376317800 4.119881 HOEKVHLD 51.977574 67930 444000
898 LMW HOEK TW10 Hoek van Holland Watertemperatuur gr.C2013-08-12 16:20:002013-08-12 17:10:00 ASCII 1 20.300:10:002013-08-12 16:30:00 1376317800 4.119881 HOEKVHLD 51.977574 67930 444000
899 LMW HVH25 CL10 Hoek van Holland chlorositeit mg Cl/l2013-08-12 16:20:002013-08-12 17:10:00 ASCII 1 4452.000:10:002013-08-12 16:30:00 1376317800 4.119881 HOEKVHLD 51.977574 67930 444000
900 LMW HOEK H10 Hoek van Holland Waterhoogte cm NAP2013-08-12 16:20:002013-08-12 17:10:00 ASCII 2 -12.000:20:002013-08-12 16:40:00 1376318400 4.119881 HOEKVHLD 51.977574 67930 444000
901 LMW HOEK TW10 Hoek van Holland Watertemperatuur gr.C2013-08-12 16:20:002013-08-12 17:10:00 ASCII 2 20.300:20:002013-08-12 16:40:00 1376318400 4.119881 HOEKVHLD 51.977574 67930 444000
902 LMW HVH25 CL10 Hoek van Holland chlorositeit mg Cl/l2013-08-12 16:20:002013-08-12 17:10:00 ASCII 2 4344.000:20:002013-08-12 16:40:00 1376318400 4.119881 HOEKVHLD 51.977574 67930 444000
903 LMW HOEK H10 Hoek van Holland Waterhoogte cm NAP2013-08-12 16:20:002013-08-12 17:10:00 ASCII 3 -6.000:30:002013-08-12 16:50:00 1376319000 4.119881 HOEKVHLD 51.977574 67930 444000
904 LMW HOEK TW10 Hoek van Holland Watertemperatuur gr.C2013-08-12 16:20:002013-08-12 17:10:00 ASCII 3 20.300:30:002013-08-12 16:50:00 1376319000 4.119881 HOEKVHLD 51.977574 67930 444000
905 LMW HVH25 CL10 Hoek van Holland chlorositeit mg Cl/l2013-08-12 16:20:002013-08-12 17:10:00 ASCII 3 4264.000:30:002013-08-12 16:50:00 1376319000 4.119881 HOEKVHLD 51.977574 67930 444000
906 LMW HOEK H10 Hoek van Holland Waterhoogte cm NAP2013-08-12 16:20:002013-08-12 17:10:00 ASCII 4 0.000:40:002013-08-12 17:00:00 1376319600 4.119881 HOEKVHLD 51.977574 67930 444000
907 LMW HOEK TW10 Hoek van Holland Watertemperatuur gr.C2013-08-12 16:20:002013-08-12 17:10:00 ASCII 4 20.500:40:002013-08-12 17:00:00 1376319600 4.119881 HOEKVHLD 51.977574 67930 444000
908 LMW HVH25 CL10 Hoek van Holland chlorositeit mg Cl/l2013-08-12 16:20:002013-08-12 17:10:00 ASCII 4 4337.000:40:002013-08-12 17:00:00 1376319600 4.119881 HOEKVHLD 51.977574 67930 444000
909 LMW HOEK H10 Hoek van Holland Waterhoogte cm NAP2013-08-12 16:20:002013-08-12 17:10:00 ASCII 5 7.000:50:002013-08-12 17:10:00 1376320200 4.119881 HOEKVHLD 51.977574 67930 444000
910 LMW HOEK TW10 Hoek van Holland Watertemperatuur gr.C2013-08-12 16:20:002013-08-12 17:10:00 ASCII 5 20.400:50:002013-08-12 17:10:00 1376320200 4.119881 HOEKVHLD 51.977574 67930 444000
911 LMW HVH25 CL10 Hoek van Holland chlorositeit mg Cl/l2013-08-12 16:20:002013-08-12 17:10:00 ASCII 5 3988.000:50:002013-08-12 17:10:00 1376320200 4.119881 HOEKVHLD 51.977574 67930 444000
\n", "
" ], "output_type": "pyout", "prompt_number": 697, "text": [ " source location_code parameter_code location_name parameter_name \\\n", "894 LMW HOEK H10 Hoek van Holland Waterhoogte \n", "895 LMW HOEK TW10 Hoek van Holland Watertemperatuur \n", "896 LMW HVH25 CL10 Hoek van Holland chlorositeit \n", "897 LMW HOEK H10 Hoek van Holland Waterhoogte \n", "898 LMW HOEK TW10 Hoek van Holland Watertemperatuur \n", "899 LMW HVH25 CL10 Hoek van Holland chlorositeit \n", "900 LMW HOEK H10 Hoek van Holland Waterhoogte \n", "901 LMW HOEK TW10 Hoek van Holland Watertemperatuur \n", "902 LMW HVH25 CL10 Hoek van Holland chlorositeit \n", "903 LMW HOEK H10 Hoek van Holland Waterhoogte \n", "904 LMW HOEK TW10 Hoek van Holland Watertemperatuur \n", "905 LMW HVH25 CL10 Hoek van Holland chlorositeit \n", "906 LMW HOEK H10 Hoek van Holland Waterhoogte \n", "907 LMW HOEK TW10 Hoek van Holland Watertemperatuur \n", "908 LMW HVH25 CL10 Hoek van Holland chlorositeit \n", "909 LMW HOEK H10 Hoek van Holland Waterhoogte \n", "910 LMW HOEK TW10 Hoek van Holland Watertemperatuur \n", "911 LMW HVH25 CL10 Hoek van Holland chlorositeit \n", "\n", " units date_start date_end format time_index \\\n", "894 cm NAP 2013-08-12 16:20:00 2013-08-12 17:10:00 ASCII 0 \n", "895 gr.C 2013-08-12 16:20:00 2013-08-12 17:10:00 ASCII 0 \n", "896 mg Cl/l 2013-08-12 16:20:00 2013-08-12 17:10:00 ASCII 0 \n", "897 cm NAP 2013-08-12 16:20:00 2013-08-12 17:10:00 ASCII 1 \n", "898 gr.C 2013-08-12 16:20:00 2013-08-12 17:10:00 ASCII 1 \n", "899 mg Cl/l 2013-08-12 16:20:00 2013-08-12 17:10:00 ASCII 1 \n", "900 cm NAP 2013-08-12 16:20:00 2013-08-12 17:10:00 ASCII 2 \n", "901 gr.C 2013-08-12 16:20:00 2013-08-12 17:10:00 ASCII 2 \n", "902 mg Cl/l 2013-08-12 16:20:00 2013-08-12 17:10:00 ASCII 2 \n", "903 cm NAP 2013-08-12 16:20:00 2013-08-12 17:10:00 ASCII 3 \n", "904 gr.C 2013-08-12 16:20:00 2013-08-12 17:10:00 ASCII 3 \n", "905 mg Cl/l 2013-08-12 16:20:00 2013-08-12 17:10:00 ASCII 3 \n", "906 cm NAP 2013-08-12 16:20:00 2013-08-12 17:10:00 ASCII 4 \n", "907 gr.C 2013-08-12 16:20:00 2013-08-12 17:10:00 ASCII 4 \n", "908 mg Cl/l 2013-08-12 16:20:00 2013-08-12 17:10:00 ASCII 4 \n", "909 cm NAP 2013-08-12 16:20:00 2013-08-12 17:10:00 ASCII 5 \n", "910 gr.C 2013-08-12 16:20:00 2013-08-12 17:10:00 ASCII 5 \n", "911 mg Cl/l 2013-08-12 16:20:00 2013-08-12 17:10:00 ASCII 5 \n", "\n", " value time_delta date time lat \\\n", "894 -20.0 00:00:00 2013-08-12 16:20:00 1376317200 4.119881 \n", "895 20.1 00:00:00 2013-08-12 16:20:00 1376317200 4.119881 \n", "896 4317.0 00:00:00 2013-08-12 16:20:00 1376317200 4.119881 \n", "897 -16.0 00:10:00 2013-08-12 16:30:00 1376317800 4.119881 \n", "898 20.3 00:10:00 2013-08-12 16:30:00 1376317800 4.119881 \n", "899 4452.0 00:10:00 2013-08-12 16:30:00 1376317800 4.119881 \n", "900 -12.0 00:20:00 2013-08-12 16:40:00 1376318400 4.119881 \n", "901 20.3 00:20:00 2013-08-12 16:40:00 1376318400 4.119881 \n", "902 4344.0 00:20:00 2013-08-12 16:40:00 1376318400 4.119881 \n", "903 -6.0 00:30:00 2013-08-12 16:50:00 1376319000 4.119881 \n", "904 20.3 00:30:00 2013-08-12 16:50:00 1376319000 4.119881 \n", "905 4264.0 00:30:00 2013-08-12 16:50:00 1376319000 4.119881 \n", "906 0.0 00:40:00 2013-08-12 17:00:00 1376319600 4.119881 \n", "907 20.5 00:40:00 2013-08-12 17:00:00 1376319600 4.119881 \n", "908 4337.0 00:40:00 2013-08-12 17:00:00 1376319600 4.119881 \n", "909 7.0 00:50:00 2013-08-12 17:10:00 1376320200 4.119881 \n", "910 20.4 00:50:00 2013-08-12 17:10:00 1376320200 4.119881 \n", "911 3988.0 00:50:00 2013-08-12 17:10:00 1376320200 4.119881 \n", "\n", " location_code_donar lon x y \n", "894 HOEKVHLD 51.977574 67930 444000 \n", "895 HOEKVHLD 51.977574 67930 444000 \n", "896 HOEKVHLD 51.977574 67930 444000 \n", "897 HOEKVHLD 51.977574 67930 444000 \n", "898 HOEKVHLD 51.977574 67930 444000 \n", "899 HOEKVHLD 51.977574 67930 444000 \n", "900 HOEKVHLD 51.977574 67930 444000 \n", "901 HOEKVHLD 51.977574 67930 444000 \n", "902 HOEKVHLD 51.977574 67930 444000 \n", "903 HOEKVHLD 51.977574 67930 444000 \n", "904 HOEKVHLD 51.977574 67930 444000 \n", "905 HOEKVHLD 51.977574 67930 444000 \n", "906 HOEKVHLD 51.977574 67930 444000 \n", "907 HOEKVHLD 51.977574 67930 444000 \n", "908 HOEKVHLD 51.977574 67930 444000 \n", "909 HOEKVHLD 51.977574 67930 444000 \n", "910 HOEKVHLD 51.977574 67930 444000 \n", "911 HOEKVHLD 51.977574 67930 444000 " ] } ], "prompt_number": 697 }, { "cell_type": "code", "collapsed": false, "input": [ "import scp\n", "import paramiko\n", "\n", "host = 'opendap.deltares.nl'\n", "user = ''\n", "passwd = ''\n", "remote_dir = '/p/opendap/data/rijkswaterstaat/opendata/'\n", "\n", "ssh = paramiko.SSHClient()\n", "ssh.load_system_host_keys()\n", "ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())\n", "ssh.connect(host, username=user, password=passwd)\n", "\n", "client = scp.SCPClient(ssh.get_transport())\n", "\n", "for fname in os.listdir(os.getcwd()):\n", " if re.search('\\.nc$',fname):\n", " print 'Transferring %s -> %s...' % (fname, remote_dir)\n", " client.put(fname, remote_dir)" ], "language": "python", "metadata": {}, "outputs": [ { "ename": "AuthenticationException", "evalue": "Authentication failed.", "output_type": "pyerr", "traceback": [ "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m\n\u001b[0;31mAuthenticationException\u001b[0m Traceback (most recent call last)", "\u001b[0;32m\u001b[0m in \u001b[0;36m\u001b[0;34m()\u001b[0m\n\u001b[1;32m 10\u001b[0m \u001b[0mssh\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mload_system_host_keys\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 11\u001b[0m \u001b[0mssh\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mset_missing_host_key_policy\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mparamiko\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mAutoAddPolicy\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 12\u001b[0;31m \u001b[0mssh\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mconnect\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mhost\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0musername\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0muser\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mpassword\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mpasswd\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 13\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 14\u001b[0m \u001b[0mclient\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mscp\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mSCPClient\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mssh\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mget_transport\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", "\u001b[0;32m/opt/local/Library/Frameworks/Python.framework/Versions/2.7/lib/python2.7/site-packages/paramiko/client.pyc\u001b[0m in \u001b[0;36mconnect\u001b[0;34m(self, hostname, port, username, password, pkey, key_filename, timeout, allow_agent, look_for_keys, compress, sock)\u001b[0m\n\u001b[1;32m 340\u001b[0m \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 341\u001b[0m \u001b[0mkey_filenames\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mkey_filename\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 342\u001b[0;31m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_auth\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0musername\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mpassword\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mpkey\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mkey_filenames\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mallow_agent\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mlook_for_keys\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 343\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 344\u001b[0m \u001b[0;32mdef\u001b[0m \u001b[0mclose\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", "\u001b[0;32m/opt/local/Library/Frameworks/Python.framework/Versions/2.7/lib/python2.7/site-packages/paramiko/client.pyc\u001b[0m in \u001b[0;36m_auth\u001b[0;34m(self, username, password, pkey, key_filenames, allow_agent, look_for_keys)\u001b[0m\n\u001b[1;32m 531\u001b[0m \u001b[0;31m# if we got an auth-failed exception earlier, re-raise it\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 532\u001b[0m \u001b[0;32mif\u001b[0m \u001b[0msaved_exception\u001b[0m \u001b[0;32mis\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0mNone\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 533\u001b[0;31m \u001b[0;32mraise\u001b[0m \u001b[0msaved_exception\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 534\u001b[0m \u001b[0;32mraise\u001b[0m \u001b[0mSSHException\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m'No authentication methods available'\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 535\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n", "\u001b[0;31mAuthenticationException\u001b[0m: Authentication failed." ] } ], "prompt_number": 705 }, { "cell_type": "code", "collapsed": false, "input": [], "language": "python", "metadata": {}, "outputs": [], "prompt_number": 704 }, { "cell_type": "code", "collapsed": false, "input": [], "language": "python", "metadata": {}, "outputs": [], "prompt_number": 650 }, { "cell_type": "code", "collapsed": false, "input": [], "language": "python", "metadata": {}, "outputs": [], "prompt_number": 651 }, { "cell_type": "code", "collapsed": false, "input": [], "language": "python", "metadata": {}, "outputs": [], "prompt_number": 651 }, { "cell_type": "code", "collapsed": false, "input": [], "language": "python", "metadata": {}, "outputs": [], "prompt_number": 652 }, { "cell_type": "code", "collapsed": false, "input": [], "language": "python", "metadata": {}, "outputs": [], "prompt_number": 652 }, { "cell_type": "code", "collapsed": false, "input": [], "language": "python", "metadata": {}, "outputs": [], "prompt_number": 652 }, { "cell_type": "code", "collapsed": false, "input": [], "language": "python", "metadata": {}, "outputs": [], "prompt_number": 652 }, { "cell_type": "code", "collapsed": false, "input": [], "language": "python", "metadata": {}, "outputs": [] } ], "metadata": {} } ] }