{ "cells": [ { "cell_type": "code", "execution_count": 1, "metadata": { "collapsed": true }, "outputs": [], "source": [ "import io\n", "import re\n", "import itertools\n", "\n", "import netCDF4\n", "import pandas\n", "import numpy as np\n", "\n", "import jinja2" ] }, { "cell_type": "code", "execution_count": 2, "metadata": { "collapsed": true }, "outputs": [], "source": [ "ds = netCDF4.Dataset('../data/transect.nc')" ] }, { "cell_type": "code", "execution_count": 3, "metadata": { "collapsed": true }, "outputs": [], "source": [ "variables = {\n", " 'id': {\"var\": 'id', \"slice\": np.s_[:]},\n", " 'lat_0': {\"var\": 'lat', \"slice\": np.s_[:, 0]},\n", " 'lat_1': {\"var\": 'lat', \"slice\": np.s_[:, -1]},\n", " 'lon_0': {\"var\": 'lon', \"slice\": np.s_[:, 0]},\n", " 'lon_1': {\"var\": 'lon', \"slice\": np.s_[:, -1]},\n", " 'rsp_lon': {\"var\": 'rsp_lon', \"slice\": np.s_[:]},\n", " 'rsp_lat': {\"var\": 'rsp_lat', \"slice\": np.s_[:]}\n", "}\n", "data = {}\n", "for var, props in variables.items():\n", " data[var] = ds.variables[props['var']][props['slice']]\n", "df = pandas.DataFrame(data=data)\n", "df['north'] = df['rsp_lat'] + 0.002\n", "df['south'] = df['rsp_lat'] - 0.002\n", "df['east'] = df['rsp_lon'] + .0025\n", "df['west'] = df['rsp_lon'] - .0025\n" ] }, { "cell_type": "code", "execution_count": 4, "metadata": { "collapsed": true }, "outputs": [], "source": [ "def textcoordinates(x0, y0, z0=None, x1=None, y1=None, z1=None):\n", " \"\"\"\n", " convert the coordinates to a string so they can be used by kml\n", "\n", " # Example usage:\n", " >>> x = np.array([1,1,1])\n", " >>> y = np.array([1,-2e10,3.0000001])\n", " >>> print(textcoordinates(x, y))\n", " 1.0,1.0,0.0\n", " 1.0,-20000000000.0,0.0\n", " 1.0,3.0000001,0.0\n", " \n", " >>> # Allow for coordinate pairs\n", " >>> x0 = x\n", " >>> y0 = y\n", " >>> x1 = x\n", " >>> y1 = y\n", " >>> print(textcoordinates(x0=x0, y0=y0, x1=x1, y1=y1))\n", " 1.0,1.0,0.0 1.0,1.0,0.0\n", " 1.0,-20000000000.0,0.0 1.0,-20000000000.0,0.0\n", " 1.0,3.0000001,0.0 1.0,3.0000001,0.0\n", " \n", " >>> x = np.array([1])\n", " >>> y = np.array([1])\n", " >>> z = np.array([np.nan])\n", " >>> print(textcoordinates(x, y, z))\n", " \n", "\n", " \"\"\"\n", " if z0 is None:\n", " z0 = np.zeros_like(x0)\n", " if x1 is None:\n", " coordinates = np.vstack([x0, y0, z0]).T\n", " else:\n", " if z1 is None:\n", " z1 = np.zeros_like(x1)\n", " coordinates = np.vstack([x0, y0, z0, x1, y1, z1]).T\n", " # only write coordinates where none is nan\n", " filter = np.isnan(coordinates).any(1)\n", " # use cStringIO for performance\n", " output = io.BytesIO()\n", " # save coordinates to string buffer\n", " # I think this is faster than other methods\n", " # Use %s to get reduced string output\n", " np.savetxt(output, coordinates[~filter], delimiter=',', fmt='%s')\n", " coord_bytes = output.getvalue()\n", " coord_str = coord_bytes.decode()\n", " if x1 is not None:\n", " # replace all the 3rd , by a space...\n", " # TODO double check this regex\n", " coord_str = re.sub(r'(.*?,.*?,.*?),(.*)', r'\\1 \\2', coord_str)\n", " return coord_str\n" ] }, { "cell_type": "code", "execution_count": 5, "metadata": { "collapsed": true }, "outputs": [], "source": [ "def line_coords(record):\n", " return textcoordinates(\n", " x0=record.lon_0, \n", " y0=record.lat_0, \n", " x1=record.lon_1, \n", " y1=record.lat_1\n", " )\n", "def point_coords(record):\n", " return textcoordinates(\n", " x0=record.rsp_lat,\n", " y0=record.rsp_lon\n", " )\n", "def bbox(record):\n", " return {\n", " 'north': record.north,\n", " 'south': record.south,\n", " 'east': record.east,\n", " 'west': record.west\n", " }" ] }, { "cell_type": "code", "execution_count": 6, "metadata": { "collapsed": true }, "outputs": [], "source": [ "df['line_coords'] = df.apply(line_coords, axis=1)\n", "df['point_coords'] = df.apply(point_coords, axis=1)\n", "df['bbox'] = df.apply(bbox, axis=1)\n", "n_pixels = itertools.islice(\n", " itertools.cycle([64, 32, 64, 16, 64, 32, 64, 8]),\n", " len(df)\n", ")\n", "df['min_lod_pixels'] = list(n_pixels)" ] }, { "cell_type": "code", "execution_count": 7, "metadata": {}, "outputs": [ { "data": { "text/html": [ "
\n", "\n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", "
idlat_0lat_1lon_0lon_1rsp_latrsp_lonnorthsoutheastwestline_coordspoint_coordsbboxmin_lod_pixels
0200010053.49036253.4040066.1468226.14075253.4714226.14548853.47342253.4694226.1479886.1429886.1468218159,53.4903622618,0.0 6.14075185913,5...53.4714215203,6.14548829642,0.0\\n{'west': 6.142988296424489, 'east': 6.14798829...64
1200010153.49000553.4052766.1517746.12316253.4714226.14548853.47342253.4694226.1479886.1429886.15177363461,53.4900045029,0.0 6.12316167857,...53.4714215203,6.14548829642,0.0\\n{'west': 6.142988296424489, 'east': 6.14798829...32
2200010253.48918953.4081746.1565706.10611953.4714226.14548853.47342253.4694226.1479886.1429886.15657048638,53.489188978,0.0 6.10611872132,5...53.4714215203,6.14548829642,0.0\\n{'west': 6.142988296424489, 'east': 6.14798829...64
3200010353.48793653.4126266.1610946.09004153.4714226.14548853.47342253.4694226.1479886.1429886.16109412018,53.4879357932,0.0 6.09004091911,...53.4714215203,6.14548829642,0.0\\n{'west': 6.142988296424489, 'east': 6.14798829...16
4200010453.48645953.4178736.1648396.07672353.4714226.14548853.47342253.4694226.1479886.1429886.16483945767,53.4864590501,0.0 6.07672286469,...53.4714215203,6.14548829642,0.0\\n{'west': 6.142988296424489, 'east': 6.14798829...64
\n", "
" ], "text/plain": [ " id lat_0 lat_1 lon_0 lon_1 rsp_lat rsp_lon \\\n", "0 2000100 53.490362 53.404006 6.146822 6.140752 53.471422 6.145488 \n", "1 2000101 53.490005 53.405276 6.151774 6.123162 53.471422 6.145488 \n", "2 2000102 53.489189 53.408174 6.156570 6.106119 53.471422 6.145488 \n", "3 2000103 53.487936 53.412626 6.161094 6.090041 53.471422 6.145488 \n", "4 2000104 53.486459 53.417873 6.164839 6.076723 53.471422 6.145488 \n", "\n", " north south east west \\\n", "0 53.473422 53.469422 6.147988 6.142988 \n", "1 53.473422 53.469422 6.147988 6.142988 \n", "2 53.473422 53.469422 6.147988 6.142988 \n", "3 53.473422 53.469422 6.147988 6.142988 \n", "4 53.473422 53.469422 6.147988 6.142988 \n", "\n", " line_coords \\\n", "0 6.1468218159,53.4903622618,0.0 6.14075185913,5... \n", "1 6.15177363461,53.4900045029,0.0 6.12316167857,... \n", "2 6.15657048638,53.489188978,0.0 6.10611872132,5... \n", "3 6.16109412018,53.4879357932,0.0 6.09004091911,... \n", "4 6.16483945767,53.4864590501,0.0 6.07672286469,... \n", "\n", " point_coords \\\n", "0 53.4714215203,6.14548829642,0.0\\n \n", "1 53.4714215203,6.14548829642,0.0\\n \n", "2 53.4714215203,6.14548829642,0.0\\n \n", "3 53.4714215203,6.14548829642,0.0\\n \n", "4 53.4714215203,6.14548829642,0.0\\n \n", "\n", " bbox min_lod_pixels \n", "0 {'west': 6.142988296424489, 'east': 6.14798829... 64 \n", "1 {'west': 6.142988296424489, 'east': 6.14798829... 32 \n", "2 {'west': 6.142988296424489, 'east': 6.14798829... 64 \n", "3 {'west': 6.142988296424489, 'east': 6.14798829... 16 \n", "4 {'west': 6.142988296424489, 'east': 6.14798829... 64 " ] }, "execution_count": 7, "metadata": {}, "output_type": "execute_result" } ], "source": [ "df.head()" ] }, { "cell_type": "code", "execution_count": 8, "metadata": { "collapsed": true }, "outputs": [], "source": [ "template = \"\"\"\n", "\n", "\n", " \n", " Transect Lod\n", " 1\n", " \n", " \n", " \n", " \n", " normal\n", " #thin\n", " \n", " \n", " highlight\n", " #thick\n", " \n", " \n", " \n", " {% for id, line in lines.iterrows() %}\n", " \n", " Transect {{ line.id }}\n", " \n", " \n", " {{ line.bbox.north }}\n", " {{ line.bbox.south }}\n", " {{ line.bbox.east }}\n", " {{ line.bbox.west }}\n", " -20\n", " {# Make 50 meters high, so you get more transects if you fly over them #}\n", " 50\n", " absolute\n", " \n", " \n", " {# Don't show too many lines #}\n", " {# alternate lod, so we get different transects at different levels #}\n", " {{ line.min_lod_pixels }}\n", " \n", " \n", " \n", " {# \n", " Replace these query parameters in the url tag... (or extend url to match these parameters)\n", " Don't forget to escape &.\n", " #}\n", " /transect/{{line.id}}/kml\n", " onRegion\n", " \n", " \n", " \n", " Transect {{ line.id }}\n", " #whiteline\n", " \n", " {{ line.coordinates }}\n", " \n", " \n", " \n", " info\n", " \n", " ]]>\n", " \n", " \n", " {% endfor %}\n", " \n", " {# ensures the style KML is preloaded, so there's no delay when all transects load #}\n", " \n", " Style preloader\n", " /transect/style\n", " \n", " 0,0,0 0.1,0.1,0.1\n", " \n", " \n", " \n", "\n", "\"\"\"" ] }, { "cell_type": "code", "execution_count": 9, "metadata": { "collapsed": true }, "outputs": [], "source": [ "kml = jinja2.Template(template).render(lines=df)" ] }, { "cell_type": "code", "execution_count": 66, "metadata": { "collapsed": true }, "outputs": [], "source": [ "template = \"\"\"\n", "\n", "\n", " \n", " {{ transect.id }}\n", " {% for _, row in transect.years.iterrows() %}\n", " \n", " {{ row.properties.year }}\n", " style\n", " {# first one without extrude #}\n", " \n", " {{ extrude }}\n", " absolute\n", " {{ row.line_coordinates }}\n", " \n", " \n", " {# kml:dateTime #}\n", " {{ row.properties.begin_date }}\n", " {# kml:dateTime #}\n", " {{ row.properties.end_date }}\n", " \n", " \n", " {% endfor %}\n", " \n", "\n", "\"\"\"" ] }, { "cell_type": "code", "execution_count": 75, "metadata": {}, "outputs": [], "source": [ "ids = ds.variables['id'][:]\n", "id_ = 7003900\n", "\n", "def get_transect(id_):\n", " \"\"\"lookup information for transect\"\"\"\n", " transect_idx = np.searchsorted(ids, id_)\n", "\n", " variables = {\n", " 'lat': {\"var\": 'lat', \"slice\": np.s_[transect_idx, :]},\n", " 'lon': {\"var\": 'lon', \"slice\": np.s_[transect_idx, :]},\n", " 'z': {\"var\": 'altitude', \"slice\": np.s_[:, transect_idx, :]},\n", " \"t\": {\"var\": 'time', \"slice\": np.s_[:]}\n", "\n", " }\n", " data = {}\n", " for var, props in variables.items():\n", " data[var] = ds.variables[props['var']][props['slice']]\n", "\n", " # df = pandas.DataFrame(data=data)\n", " years = []\n", " for t, row in zip(data['t'], data['z']):\n", " item = {}\n", " coords = pandas.DataFrame(data=dict(\n", " lon=data['lon'], \n", " lat=data['lat'],\n", " z=row\n", " ))\n", " item['coordinates'] = coords.dropna()\n", " item['line_coordinates'] = textcoordinates(\n", " x0=item['coordinates']['lon'],\n", " y0=item['coordinates']['lat'],\n", " z0=item['coordinates']['z']\n", " )\n", " date = netCDF4.num2date(t, ds.variables['time'].units)\n", " item['properties'] = {\n", " \"t\": date,\n", " }\n", " item['properties']['year'] = date.year\n", " item['properties']['begin_date'] = date\n", " item['properties']['end_date'] = date.replace(year=date.year+1)\n", " years.append(item)\n", " years = pandas.DataFrame.from_records(years)\n", " transect = {\n", " \"years\": years,\n", " \"id\": id_\n", " }\n", " return transect" ] }, { "cell_type": "code", "execution_count": 68, "metadata": {}, "outputs": [ { "data": { "text/plain": [ "1116" ] }, "execution_count": 68, "metadata": {}, "output_type": "execute_result" } ], "source": [ "# ds.variables['altitude']\n", "ds.variables.keys()\n", "np.searchsorted(ids, 7003900)" ] }, { "cell_type": "code", "execution_count": 79, "metadata": {}, "outputs": [], "source": [ "transect = get_transect(7003900)" ] }, { "cell_type": "code", "execution_count": 82, "metadata": {}, "outputs": [], "source": [ "T = jinja2.Template(template)\n", "kml = T.render(transect=transect, extrude=1)" ] }, { "cell_type": "code", "execution_count": 83, "metadata": { "collapsed": true }, "outputs": [], "source": [ "template = \"\"\"\n", "\n", "{% load tools %}\n", "\n", " \n", " Style\n", " {# style that gets applied to our stub preloader linestring #}\n", " \n", " {# this the style for the upper line #}\n", " {% for key, color in colors.items %}\n", " \n", " {% endfor %}\n", " \n", "\n", "\"\"\"" ] }, { "cell_type": "code", "execution_count": 106, "metadata": {}, "outputs": [], "source": [ "import matplotlib.colors\n", "import datetime\n", "\n", "args = {}\n", "template_context = {}\n", "poly_alpha = int(float(args.get('poly_alpha', 1.0)) * 255)\n", "template_context['poly_alpha'] = '{:02X}'.format(poly_alpha)\n", "template_context['outline'] = int(args.get('outline',0))\n", "\n", "# Get a colormap based on the ?colormap parameter\n", "colormap_name = kml_args_dict.get('colormap', 'viridis')\n", "colormap = matplotlib.cm.cmap_d.get(colormap_name, matplotlib.cm.viridis)\n", "colors = {}\n", "# HACK: This can be done a bit nicer and the styles can be references to an external file (through styleurl)\n", "for year in range(1964, datetime.datetime.now().year + 1):\n", " # call with float 0..1 (or int 0 .. 255)\n", " r,g,b, alpha = colormap(float(year-1964)/float(2015-1964))\n", " color = matplotlib.colors.rgb2hex((b, g, r)).replace('#', '') # r and b reversed in the google, don't forget to add alpha\n", " colors['year{0}'.format(year)] = color\n", "template_context['colors'] = colors\n" ] }, { "cell_type": "code", "execution_count": 107, "metadata": {}, "outputs": [ { "data": { "text/plain": [ "{'colors': {'year1964': '540144',\n", " 'year1965': '5c0846',\n", " 'year1966': '631047',\n", " 'year1967': '691748',\n", " 'year1968': '6f1d48',\n", " 'year1969': '752448',\n", " 'year1970': '7a2a47',\n", " 'year1971': '7e3046',\n", " 'year1972': '813745',\n", " 'year1973': '843d43',\n", " 'year1974': '874241',\n", " 'year1975': '89483f',\n", " 'year1976': '8a4e3d',\n", " 'year1977': '8b533a',\n", " 'year1978': '8c5938',\n", " 'year1979': '8d5e35',\n", " 'year1980': '8d6333',\n", " 'year1981': '8e6831',\n", " 'year1982': '8e6d2e',\n", " 'year1983': '8e712c',\n", " 'year1984': '8e762a',\n", " 'year1985': '8e7b29',\n", " 'year1986': '8e8027',\n", " 'year1987': '8e8425',\n", " 'year1988': '8e8923',\n", " 'year1989': '8d8e21',\n", " 'year1990': '8c9220',\n", " 'year1991': '8b971f',\n", " 'year1992': '899c1e',\n", " 'year1993': '88a11f',\n", " 'year1994': '85a521',\n", " 'year1995': '83aa24',\n", " 'year1996': '80ae28',\n", " 'year1997': '7cb32e',\n", " 'year1998': '79b735',\n", " 'year1999': '74bc3d',\n", " 'year2000': '6fc046',\n", " 'year2001': '6ac450',\n", " 'year2002': '64c85a',\n", " 'year2003': '5ecb65',\n", " 'year2004': '57cf70',\n", " 'year2005': '50d27c',\n", " 'year2006': '48d589',\n", " 'year2007': '40d895',\n", " 'year2008': '37daa2',\n", " 'year2009': '2fddb0',\n", " 'year2010': '26dfbd',\n", " 'year2011': '1fe1ca',\n", " 'year2012': '19e2d8',\n", " 'year2013': '19e4e5',\n", " 'year2014': '1de5f1',\n", " 'year2015': '25e7fd',\n", " 'year2016': '25e7fd',\n", " 'year2017': '25e7fd'},\n", " 'outline': 0,\n", " 'poly_alpha': 'FF'}" ] }, "execution_count": 107, "metadata": {}, "output_type": "execute_result" } ], "source": [ "datetime.datetime.now().year\n", "template_context" ] }, { "cell_type": "code", "execution_count": 108, "metadata": { "collapsed": true }, "outputs": [], "source": [ "import requests" ] }, { "cell_type": "code", "execution_count": 109, "metadata": { "collapsed": true }, "outputs": [], "source": [ "resp = requests.get('http://nu.nl')" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "collapsed": true }, "outputs": [], "source": [] } ], "metadata": { "hide_input": false, "kernelspec": { "display_name": "Python 3", "language": "python", "name": "python3" }, "language_info": { "codemirror_mode": { "name": "ipython", "version": 3 }, "file_extension": ".py", "mimetype": "text/x-python", "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", "version": "3.6.5" } }, "nbformat": 4, "nbformat_minor": 2 }