ERA5 - check inputĀ¶

InĀ [1]:
%matplotlib widget

import cartopy
import hvplot.pandas
import hvplot.xarray
import holoviews as hv
import numpy as np
import pandas as pd
import matplotlib.pyplot as plt
import xarray as xr

hv.extension("bokeh")
InĀ [2]:
from dask.distributed import LocalCluster
cluster = LocalCluster()          # Fully-featured local Dask cluster
cluster

LocalCluster

519dca2d

Dashboard: http://127.0.0.1:8787/status Workers: 16
Total threads: 128 Total memory: 503.23 GiB
Status: running Using processes: True

Scheduler Info

Scheduler

Scheduler-c351c464-c09e-4b07-85fc-b46bb5befc69

Comm: tcp://127.0.0.1:37059 Workers: 16
Dashboard: http://127.0.0.1:8787/status Total threads: 128
Started: Just now Total memory: 503.23 GiB

Workers

Worker: 0
Comm: tcp://127.0.0.1:36521 Total threads: 8
Dashboard: http://127.0.0.1:38785/status Memory: 31.45 GiB
Nanny: tcp://127.0.0.1:39839
Local directory: /tmp/dask-scratch-space-101348/worker-z63mdpdf

Worker: 1
Comm: tcp://127.0.0.1:42761 Total threads: 8
Dashboard: http://127.0.0.1:37451/status Memory: 31.45 GiB
Nanny: tcp://127.0.0.1:43043
Local directory: /tmp/dask-scratch-space-101348/worker-d4c2i6dl

Worker: 2
Comm: tcp://127.0.0.1:36953 Total threads: 8
Dashboard: http://127.0.0.1:46213/status Memory: 31.45 GiB
Nanny: tcp://127.0.0.1:37197
Local directory: /tmp/dask-scratch-space-101348/worker-98rgf9a9

Worker: 3
Comm: tcp://127.0.0.1:43711 Total threads: 8
Dashboard: http://127.0.0.1:39249/status Memory: 31.45 GiB
Nanny: tcp://127.0.0.1:38279
Local directory: /tmp/dask-scratch-space-101348/worker-mpy5sdpc

Worker: 4
Comm: tcp://127.0.0.1:34115 Total threads: 8
Dashboard: http://127.0.0.1:33589/status Memory: 31.45 GiB
Nanny: tcp://127.0.0.1:39471
Local directory: /tmp/dask-scratch-space-101348/worker-4i9f1pi8

Worker: 5
Comm: tcp://127.0.0.1:43505 Total threads: 8
Dashboard: http://127.0.0.1:40169/status Memory: 31.45 GiB
Nanny: tcp://127.0.0.1:41291
Local directory: /tmp/dask-scratch-space-101348/worker-l_vibs6v

Worker: 6
Comm: tcp://127.0.0.1:39907 Total threads: 8
Dashboard: http://127.0.0.1:36147/status Memory: 31.45 GiB
Nanny: tcp://127.0.0.1:36495
Local directory: /tmp/dask-scratch-space-101348/worker-ms0c3209

Worker: 7
Comm: tcp://127.0.0.1:43691 Total threads: 8
Dashboard: http://127.0.0.1:36359/status Memory: 31.45 GiB
Nanny: tcp://127.0.0.1:44553
Local directory: /tmp/dask-scratch-space-101348/worker-kiora_y2

Worker: 8
Comm: tcp://127.0.0.1:45865 Total threads: 8
Dashboard: http://127.0.0.1:44993/status Memory: 31.45 GiB
Nanny: tcp://127.0.0.1:42221
Local directory: /tmp/dask-scratch-space-101348/worker-pub4k30e

Worker: 9
Comm: tcp://127.0.0.1:41435 Total threads: 8
Dashboard: http://127.0.0.1:32933/status Memory: 31.45 GiB
Nanny: tcp://127.0.0.1:34277
Local directory: /tmp/dask-scratch-space-101348/worker-bfdvest5

Worker: 10
Comm: tcp://127.0.0.1:35149 Total threads: 8
Dashboard: http://127.0.0.1:40203/status Memory: 31.45 GiB
Nanny: tcp://127.0.0.1:45451
Local directory: /tmp/dask-scratch-space-101348/worker-8398x6xi

Worker: 11
Comm: tcp://127.0.0.1:43611 Total threads: 8
Dashboard: http://127.0.0.1:42795/status Memory: 31.45 GiB
Nanny: tcp://127.0.0.1:45203
Local directory: /tmp/dask-scratch-space-101348/worker-dif3g46o

Worker: 12
Comm: tcp://127.0.0.1:43565 Total threads: 8
Dashboard: http://127.0.0.1:44341/status Memory: 31.45 GiB
Nanny: tcp://127.0.0.1:34125
Local directory: /tmp/dask-scratch-space-101348/worker-jxetj_q2

Worker: 13
Comm: tcp://127.0.0.1:41645 Total threads: 8
Dashboard: http://127.0.0.1:42935/status Memory: 31.45 GiB
Nanny: tcp://127.0.0.1:35747
Local directory: /tmp/dask-scratch-space-101348/worker-ndew7a7i

Worker: 14
Comm: tcp://127.0.0.1:34209 Total threads: 8
Dashboard: http://127.0.0.1:33415/status Memory: 31.45 GiB
Nanny: tcp://127.0.0.1:46773
Local directory: /tmp/dask-scratch-space-101348/worker-qltdbyo1

Worker: 15
Comm: tcp://127.0.0.1:42455 Total threads: 8
Dashboard: http://127.0.0.1:45807/status Memory: 31.45 GiB
Nanny: tcp://127.0.0.1:41149
Local directory: /tmp/dask-scratch-space-101348/worker-fu35373g
InĀ [3]:
client = cluster.get_client()
client
Out[3]:

Client

Client-544adab0-0493-11ef-b4f4-00000f53fe80

Connection method: Cluster object Cluster type: distributed.LocalCluster
Dashboard: http://127.0.0.1:8787/status

Cluster Info

LocalCluster

519dca2d

Dashboard: http://127.0.0.1:8787/status Workers: 16
Total threads: 128 Total memory: 503.23 GiB
Status: running Using processes: True

Scheduler Info

Scheduler

Scheduler-c351c464-c09e-4b07-85fc-b46bb5befc69

Comm: tcp://127.0.0.1:37059 Workers: 16
Dashboard: http://127.0.0.1:8787/status Total threads: 128
Started: Just now Total memory: 503.23 GiB

Workers

Worker: 0
Comm: tcp://127.0.0.1:36521 Total threads: 8
Dashboard: http://127.0.0.1:38785/status Memory: 31.45 GiB
Nanny: tcp://127.0.0.1:39839
Local directory: /tmp/dask-scratch-space-101348/worker-z63mdpdf

Worker: 1
Comm: tcp://127.0.0.1:42761 Total threads: 8
Dashboard: http://127.0.0.1:37451/status Memory: 31.45 GiB
Nanny: tcp://127.0.0.1:43043
Local directory: /tmp/dask-scratch-space-101348/worker-d4c2i6dl

Worker: 2
Comm: tcp://127.0.0.1:36953 Total threads: 8
Dashboard: http://127.0.0.1:46213/status Memory: 31.45 GiB
Nanny: tcp://127.0.0.1:37197
Local directory: /tmp/dask-scratch-space-101348/worker-98rgf9a9

Worker: 3
Comm: tcp://127.0.0.1:43711 Total threads: 8
Dashboard: http://127.0.0.1:39249/status Memory: 31.45 GiB
Nanny: tcp://127.0.0.1:38279
Local directory: /tmp/dask-scratch-space-101348/worker-mpy5sdpc

Worker: 4
Comm: tcp://127.0.0.1:34115 Total threads: 8
Dashboard: http://127.0.0.1:33589/status Memory: 31.45 GiB
Nanny: tcp://127.0.0.1:39471
Local directory: /tmp/dask-scratch-space-101348/worker-4i9f1pi8

Worker: 5
Comm: tcp://127.0.0.1:43505 Total threads: 8
Dashboard: http://127.0.0.1:40169/status Memory: 31.45 GiB
Nanny: tcp://127.0.0.1:41291
Local directory: /tmp/dask-scratch-space-101348/worker-l_vibs6v

Worker: 6
Comm: tcp://127.0.0.1:39907 Total threads: 8
Dashboard: http://127.0.0.1:36147/status Memory: 31.45 GiB
Nanny: tcp://127.0.0.1:36495
Local directory: /tmp/dask-scratch-space-101348/worker-ms0c3209

Worker: 7
Comm: tcp://127.0.0.1:43691 Total threads: 8
Dashboard: http://127.0.0.1:36359/status Memory: 31.45 GiB
Nanny: tcp://127.0.0.1:44553
Local directory: /tmp/dask-scratch-space-101348/worker-kiora_y2

Worker: 8
Comm: tcp://127.0.0.1:45865 Total threads: 8
Dashboard: http://127.0.0.1:44993/status Memory: 31.45 GiB
Nanny: tcp://127.0.0.1:42221
Local directory: /tmp/dask-scratch-space-101348/worker-pub4k30e

Worker: 9
Comm: tcp://127.0.0.1:41435 Total threads: 8
Dashboard: http://127.0.0.1:32933/status Memory: 31.45 GiB
Nanny: tcp://127.0.0.1:34277
Local directory: /tmp/dask-scratch-space-101348/worker-bfdvest5

Worker: 10
Comm: tcp://127.0.0.1:35149 Total threads: 8
Dashboard: http://127.0.0.1:40203/status Memory: 31.45 GiB
Nanny: tcp://127.0.0.1:45451
Local directory: /tmp/dask-scratch-space-101348/worker-8398x6xi

Worker: 11
Comm: tcp://127.0.0.1:43611 Total threads: 8
Dashboard: http://127.0.0.1:42795/status Memory: 31.45 GiB
Nanny: tcp://127.0.0.1:45203
Local directory: /tmp/dask-scratch-space-101348/worker-dif3g46o

Worker: 12
Comm: tcp://127.0.0.1:43565 Total threads: 8
Dashboard: http://127.0.0.1:44341/status Memory: 31.45 GiB
Nanny: tcp://127.0.0.1:34125
Local directory: /tmp/dask-scratch-space-101348/worker-jxetj_q2

Worker: 13
Comm: tcp://127.0.0.1:41645 Total threads: 8
Dashboard: http://127.0.0.1:42935/status Memory: 31.45 GiB
Nanny: tcp://127.0.0.1:35747
Local directory: /tmp/dask-scratch-space-101348/worker-ndew7a7i

Worker: 14
Comm: tcp://127.0.0.1:34209 Total threads: 8
Dashboard: http://127.0.0.1:33415/status Memory: 31.45 GiB
Nanny: tcp://127.0.0.1:46773
Local directory: /tmp/dask-scratch-space-101348/worker-qltdbyo1

Worker: 15
Comm: tcp://127.0.0.1:42455 Total threads: 8
Dashboard: http://127.0.0.1:45807/status Memory: 31.45 GiB
Nanny: tcp://127.0.0.1:41149
Local directory: /tmp/dask-scratch-space-101348/worker-fu35373g
InĀ [4]:
chunks = {}
chunks = dict(
    time=120, 
    longitude=20, 
    latitude=20,
)

era5 = xr.open_mfdataset(
    "/project/home/p200232/02_meteo/era5/lon_lat/netcdf/2023*.nc",
    chunks=chunks,
)
era5
Out[4]:
<xarray.Dataset> Size: 218GB
Dimensions:    (longitude: 1440, latitude: 721, time: 8760)
Coordinates:
  * longitude  (longitude) float32 6kB 0.0 0.25 0.5 0.75 ... 359.2 359.5 359.8
  * latitude   (latitude) float32 3kB 90.0 89.75 89.5 ... -89.5 -89.75 -90.0
  * time       (time) datetime64[ns] 70kB 2023-01-01 ... 2023-12-31T23:00:00
Data variables:
    msl        (time, latitude, longitude) float64 73GB dask.array<chunksize=(120, 20, 20), meta=np.ndarray>
    u10        (time, latitude, longitude) float64 73GB dask.array<chunksize=(120, 20, 20), meta=np.ndarray>
    v10        (time, latitude, longitude) float64 73GB dask.array<chunksize=(120, 20, 20), meta=np.ndarray>
Attributes:
    Conventions:  CF-1.6
    history:      2024-04-04 09:42:07 GMT by grib_to_netcdf-2.25.1: /opt/ecmw...
xarray.Dataset
    • longitude: 1440
    • latitude: 721
    • time: 8760
    • longitude
      (longitude)
      float32
      0.0 0.25 0.5 ... 359.2 359.5 359.8
      units :
      degrees_east
      long_name :
      longitude
      array([0.0000e+00, 2.5000e-01, 5.0000e-01, ..., 3.5925e+02, 3.5950e+02,
       3.5975e+02], dtype=float32)
    • latitude
      (latitude)
      float32
      90.0 89.75 89.5 ... -89.75 -90.0
      units :
      degrees_north
      long_name :
      latitude
      array([ 90.  ,  89.75,  89.5 , ..., -89.5 , -89.75, -90.  ], dtype=float32)
    • time
      (time)
      datetime64[ns]
      2023-01-01 ... 2023-12-31T23:00:00
      long_name :
      time
      array(['2023-01-01T00:00:00.000000000', '2023-01-01T01:00:00.000000000',
       '2023-01-01T02:00:00.000000000', ..., '2023-12-31T21:00:00.000000000',
       '2023-12-31T22:00:00.000000000', '2023-12-31T23:00:00.000000000'],
      dtype='datetime64[ns]')
    • msl
      (time, latitude, longitude)
      float64
      dask.array<chunksize=(120, 20, 20), meta=np.ndarray>
      units :
      Pa
      long_name :
      Mean sea level pressure
      standard_name :
      air_pressure_at_mean_sea_level
      Array Chunk
      Bytes 67.76 GiB 375.00 kiB
      Shape (8760, 721, 1440) (120, 20, 20)
      Dask graph 194472 chunks in 2 graph layers
      Data type float64 numpy.ndarray
      		1440 721 8760
    • u10
      (time, latitude, longitude)
      float64
      dask.array<chunksize=(120, 20, 20), meta=np.ndarray>
      units :
      m s**-1
      long_name :
      10 metre U wind component
      Array Chunk
      Bytes 67.76 GiB 375.00 kiB
      Shape (8760, 721, 1440) (120, 20, 20)
      Dask graph 194472 chunks in 2 graph layers
      Data type float64 numpy.ndarray
      		1440 721 8760
    • v10
      (time, latitude, longitude)
      float64
      dask.array<chunksize=(120, 20, 20), meta=np.ndarray>
      units :
      m s**-1
      long_name :
      10 metre V wind component
      Array Chunk
      Bytes 67.76 GiB 375.00 kiB
      Shape (8760, 721, 1440) (120, 20, 20)
      Dask graph 194472 chunks in 2 graph layers
      Data type float64 numpy.ndarray
      		1440 721 8760
    • longitude
      PandasIndex
      PandasIndex(Index([   0.0,   0.25,    0.5,   0.75,    1.0,   1.25,    1.5,   1.75,    2.0,
         2.25,
       ...
        357.5, 357.75,  358.0, 358.25,  358.5, 358.75,  359.0, 359.25,  359.5,
       359.75],
      dtype='float32', name='longitude', length=1440))
    • latitude
      PandasIndex
      PandasIndex(Index([  90.0,  89.75,   89.5,  89.25,   89.0,  88.75,   88.5,  88.25,   88.0,
        87.75,
       ...
       -87.75,  -88.0, -88.25,  -88.5, -88.75,  -89.0, -89.25,  -89.5, -89.75,
        -90.0],
      dtype='float32', name='latitude', length=721))
    • time
      PandasIndex
      PandasIndex(DatetimeIndex(['2023-01-01 00:00:00', '2023-01-01 01:00:00',
               '2023-01-01 02:00:00', '2023-01-01 03:00:00',
               '2023-01-01 04:00:00', '2023-01-01 05:00:00',
               '2023-01-01 06:00:00', '2023-01-01 07:00:00',
               '2023-01-01 08:00:00', '2023-01-01 09:00:00',
               ...
               '2023-12-31 14:00:00', '2023-12-31 15:00:00',
               '2023-12-31 16:00:00', '2023-12-31 17:00:00',
               '2023-12-31 18:00:00', '2023-12-31 19:00:00',
               '2023-12-31 20:00:00', '2023-12-31 21:00:00',
               '2023-12-31 22:00:00', '2023-12-31 23:00:00'],
              dtype='datetime64[ns]', name='time', length=8760, freq=None))
  • Conventions :
    CF-1.6
    history :
    2024-04-04 09:42:07 GMT by grib_to_netcdf-2.25.1: /opt/ecmwf/mars-client/bin/grib_to_netcdf.bin -S param -o /cache/data5/adaptor.mars.internal-1712223465.2069833-14833-10-170b5d44-2e0e-42df-826b-56c1dd5e4490.nc /cache/tmp/170b5d44-2e0e-42df-826b-56c1dd5e4490-adaptor.mars.internal-1712222898.1675792-14833-10-tmp.grib
InĀ [5]:
window = dict(
    time=slice("2023-07-12", "2023-07-17"),
    latitude=slice(30, 0), 
    longitude=slice(210, 260),
)

era5_window = era5.sel(**window).load()
InĀ [6]:
# Convert MSL to kPa!
if era5_window.msl.max() > 10_000:
    era5_window["msl"] = era5_window.msl / 1000
InĀ [7]:
plot_options = dict(frame_width=500)
InĀ [8]:
msl_min = era5_window.msl.min("time")
InĀ [9]:
msl_min.hvplot(
    x="longitude", 
    y="latitude", 
    title="min MSL (kPa)",
    geo=True,
    coastline=True,
).options(**plot_options)
Out[9]:
InĀ [10]:
# Contours make things a bit clearer
msl_min.hvplot.contourf(
    x="longitude", 
    y="latitude", 
    title="min msl (kPa)",
    levels=10,
    geo=True,
    coastline=True,
).options(**plot_options)
Out[10]:
InĀ [11]:
era5_window.msl.groupby("time", squeeze=False).map(np.min).hvplot(grid=True, title="min MSL over window (kPa)")
Out[11]:
InĀ [12]:
# Animation
era5_window.msl.hvplot.contourf(
    groupby="time",
    x="longitude", 
    y="latitude",
    levels=15,
    geo=True,
    coastline=True,
    widget_type="scrubber",
    widget_location="bottom",
)
Out[12]:
InĀ [13]:
era5_window["mag"] = np.sqrt(era5_window.u10**2 + era5_window.v10**2)
era5_window["angle"] = (np.pi/2.) - np.arctan2(era5_window.v10, era5_window.u10)    # WTF?? I had to switch U and V in order to get the arrows to show correctly...
InĀ [14]:
ww = dict(latitude=slice(20, 10), longitude=slice(230, 240))
era5_window.sel(time="2023-07-14T21:00:00", **ww).hvplot.vectorfield(
    #groupby="time",
    x="longitude", 
    y="latitude",
    angle='angle', 
    mag='mag', 
    hover=False,
    geo=True,
    #widget_type="scrubber",
    #widget_location="bottom",
).opts(magnitude='mag', pivot="tip") + \
era5_window.sel(time="2023-07-14T14:00:00", **ww).hvplot.vectorfield(
    #groupby="time",
    x="longitude", 
    y="latitude",
    angle='angle', 
    mag='mag', 
    hover=False,
    geo=True,
    #widget_type="scrubber",
    #widget_location="bottom",
).opts(magnitude='mag')
Out[14]:
InĀ [15]:
time_slice = slice("2023-07-12", "2023-07-17")
longitude=235
latitude=40
hv.Layout([
    era5.sel(longitude=longitude, latitude=latitude, time=time_slice).msl.hvplot(grid=True),
    era5.sel(longitude=longitude, latitude=0, time=time_slice).msl.hvplot(grid=True),
    era5.sel(longitude=longitude, latitude=-latitude, time=time_slice).msl.hvplot(grid=True),
    
]).cols(1)
Out[15]: