Time lag checks L0 fluxes IRGA75 (IRGA75, 2004-2017, 2019)

Time lag checks L0 fluxes IRGA75 (IRGA75, 2004-2017, 2019)#


Imports#

import warnings
warnings.filterwarnings("ignore", category=UserWarning)
from datetime import datetime
import pandas as pd
from diive.core.plotting.heatmap_datetime import HeatmapDateTime
from diive.core.io.files import load_parquet
from diive.pkgs.analyses.histogram import Histogram
import matplotlib.gridspec as gridspec
import matplotlib.pyplot as plt
import matplotlib.transforms as transforms
from diive.core.plotting.plotfuncs import default_format

Load data#

SOURCEFILE = r"12_OPENLAG_EDDYPRO_FLUXNET_OUTPUT_IRGA75_2004-2017_2019.parquet"
df = load_parquet(filepath=SOURCEFILE)
df

Loaded .parquet file 12_OPENLAG_EDDYPRO_FLUXNET_OUTPUT_IRGA75_2004-2017_2019.parquet (3.891 seconds).
    --> Detected time resolution of <30 * Minutes> / 30min
AIR_CP AIR_DENSITY AIR_MV AIR_RHO_CP AOA_METHOD AXES_ROTATION_METHOD BADM_HEIGHTC BADM_INSTPAIR_EASTWARD_SEP_GA_CH4 BADM_INSTPAIR_EASTWARD_SEP_GA_CO2 BADM_INSTPAIR_EASTWARD_SEP_GA_H2O BADM_INSTPAIR_EASTWARD_SEP_GA_NONE BADM_INSTPAIR_HEIGHT_SEP_GA_CH4 BADM_INSTPAIR_HEIGHT_SEP_GA_CO2 BADM_INSTPAIR_HEIGHT_SEP_GA_H2O BADM_INSTPAIR_HEIGHT_SEP_GA_NONE ... W_T_SONIC_COV_IBROM_N0004 W_T_SONIC_COV_IBROM_N0008 W_T_SONIC_COV_IBROM_N0016 W_T_SONIC_COV_IBROM_N0032 W_T_SONIC_COV_IBROM_N0065 W_T_SONIC_COV_IBROM_N0133 W_T_SONIC_COV_IBROM_N0277 W_T_SONIC_COV_IBROM_N0614 W_T_SONIC_COV_IBROM_N1626 W_UNROT W_U_COV W_VM97_TEST W_ZCD ZL ZL_UNCORR
TIMESTAMP_MIDDLE
2004-03-30 14:15:00 NaN NaN NaN NaN NaN NaN NaN NaN NaN NaN NaN NaN NaN NaN NaN ... NaN NaN NaN NaN NaN NaN NaN NaN NaN NaN NaN NaN NaN NaN NaN
2004-03-30 14:45:00 1005.42 1.12844 0.025668 1134.55 0.0 1.0 37.0 NaN NaN NaN NaN NaN NaN NaN NaN ... NaN NaN NaN NaN NaN NaN NaN NaN NaN -0.227060 -0.209191 800000001.0 14.0 -1.260300 -1.311810
2004-03-30 15:15:00 NaN NaN NaN NaN NaN NaN NaN NaN NaN NaN NaN NaN NaN NaN NaN ... NaN NaN NaN NaN NaN NaN NaN NaN NaN NaN NaN NaN NaN NaN NaN
2004-03-30 15:45:00 1005.43 1.12663 0.025709 1132.74 0.0 1.0 37.0 NaN NaN NaN NaN NaN NaN NaN NaN ... NaN NaN NaN NaN NaN NaN NaN NaN NaN -0.063026 0.002830 800000001.0 39.0 -0.350350 -0.361224
2004-03-30 16:15:00 1005.43 1.12587 0.025726 1131.98 0.0 1.0 37.0 NaN NaN NaN NaN NaN NaN NaN NaN ... NaN NaN NaN NaN NaN NaN NaN NaN NaN -0.028456 -0.034904 800000000.0 76.0 -0.295733 -0.304238
... ... ... ... ... ... ... ... ... ... ... ... ... ... ... ... ... ... ... ... ... ... ... ... ... ... ... ... ... ... ... ...
2019-03-20 16:45:00 1005.38 1.13522 0.025515 1141.33 0.0 1.0 37.0 NaN NaN NaN NaN NaN NaN NaN NaN ... NaN NaN NaN NaN NaN NaN NaN NaN NaN 0.383872 -0.350438 800000000.0 20.0 -0.143543 -0.147249
2019-03-20 17:15:00 1005.38 1.13469 0.025527 1140.80 0.0 1.0 37.0 NaN NaN NaN NaN NaN NaN NaN NaN ... NaN NaN NaN NaN NaN NaN NaN NaN NaN 0.431124 -0.339284 800000000.0 13.0 -0.120186 -0.122775
2019-03-20 17:45:00 1005.39 1.13422 0.025537 1140.33 0.0 1.0 37.0 NaN NaN NaN NaN NaN NaN NaN NaN ... NaN NaN NaN NaN NaN NaN NaN NaN NaN 0.197926 -0.251053 800010000.0 17.0 -0.029753 -0.030451
2019-03-20 18:15:00 1005.39 1.13262 0.025573 1138.73 0.0 1.0 37.0 NaN NaN NaN NaN NaN NaN NaN NaN ... NaN NaN NaN NaN NaN NaN NaN NaN NaN -0.061639 -0.101834 800000000.0 29.0 0.088462 0.089222
2019-03-20 18:45:00 1005.40 1.13146 0.025599 1137.57 0.0 1.0 37.0 NaN NaN NaN NaN NaN NaN NaN NaN ... NaN NaN NaN NaN NaN NaN NaN NaN NaN -0.079800 -0.104907 800000000.0 39.0 0.074985 0.075608

262474 rows × 494 columns

tlag_actual_cols = [c for c in df.columns if c.endswith("_TLAG_ACTUAL")]
tlag_actual_cols

['CH4_TLAG_ACTUAL', 'CO2_TLAG_ACTUAL', 'H2O_TLAG_ACTUAL', 'NONE_TLAG_ACTUAL']

# # Time range
# # locs = (df.index.year == 2019) & (df.index.month >= 5)
# locs = df.index.year == 2021
# # locs = (df.index.year == 2017) | (df.index.year == 2018)
# # locs = (df.index.year == 2017) & (df.index < "2017-03-15 23:59:00")
# # locs = (df.index.year == 2019) & ((df.index > "2019-02-17 23:59:00") & (df.index <= "2019-04-30 23:59:00"))
# # locs = (
# #         ((df.index > "2019-01-01 23:59:00") & (df.index < "2019-02-18 07:00:00")) |
# #         ((df.index > "2019-05-01 07:00:00") & (df.index < "2019-05-22 07:00:00"))
# # )
# # locs = (df.index > "2020-02-28 23:59:00") & (df.index < "2020-05-13 07:00:00")
# # locs = (df.index > "2021-07-23 23:59:00")

# tlag_actual = df[tlag_actual_cols][locs].copy()
# first_date = tlag_actual.index[0].date()
# last_date = tlag_actual.index[-1].date()

Function for plotting time lags and ranges#

def timelags(gascol: str, series: pd.Series, vline1: float, vline2: float, startbin: float, endbin: float):
    # Calculate histogram
    hist = Histogram(
        s=series,
        method='uniques',
        # n_bins=10,
        # ignore_fringe_bins=None
        ignore_fringe_bins=[5, 10]
    )
    results = hist.results
    peakbins = hist.peakbins
    peak = peakbins[0]

    # Restrict shown results
    locs = (results['BIN_START_INCL'] >= startbin) & (results['BIN_START_INCL'] <= endbin)
    results = results[locs].copy()
    hist_bins = results['BIN_START_INCL'].copy()
    hist_counts = results['COUNTS'].copy()

    # Plot
    fig = plt.figure(layout="constrained", facecolor='white', figsize=(21, 8), dpi=72)
    gs = gridspec.GridSpec(1, 1)  # rows, cols
    gs.update(wspace=0.3, hspace=0.3, left=0.03, right=0.97, top=0.97, bottom=0.03)    
    gs = gridspec.GridSpec(2, 1, figure=fig)  # rows, cols
    ax = fig.add_subplot(gs[0, :])
    ax2 = fig.add_subplot(gs[1, :])
    bar_width = .05
    # bar_width = (hist_bins[1] - hist_bins[0]) * 1  # Calculate bar width
    args = dict(width=bar_width, align='edge')
    
    # Axis 1: histogram plot
    ax.bar(x=hist_bins, height=hist_counts, label='counts', zorder=90, color='#78909c', **args)
    title = f"{gascol} (between {first_date} and {last_date})"
    ax.set_title(title, fontsize=24, weight='bold') 
    default_format(ax=ax, ax_xlabel_txt="lag (seconds)", ax_ylabel_txt="counts")
    ax.axvline(peak, color="black")
    ax.axvline(vline1, color="blue")
    ax.axvline(vline2, color="red")
    ax.locator_params(axis='both', nbins=20)
    trans = transforms.blended_transform_factory(ax.transData, ax.transAxes)
    ax.text(peak, 0.98, f"PEAK {peak}s",
            size=16, color='black', backgroundcolor='none', transform=trans,
            alpha=1, horizontalalignment='center', verticalalignment='top', zorder=999)
    trans = transforms.blended_transform_factory(ax.transData, ax.transAxes)
    ax.text(vline1, 0.70, f"start {vline1}s",
            size=16, color='blue', backgroundcolor='none', transform=trans,
            alpha=1, horizontalalignment='right', verticalalignment='top', zorder=999)
    ax.text(vline2, 0.70, f"end {vline2}s",
            size=16, color='red', backgroundcolor='none', transform=trans,
            alpha=1, horizontalalignment='left', verticalalignment='top', zorder=999)

    # Axis 2: time series
    ax2.plot(series.index, series, alpha=0.5, c='#5f87ae', marker='.', ms=5, ls='solid', lw=1)    
    ax2.axhline(peak, color="black")
    ax2.axhline(vline1, color="blue")
    ax2.axhline(vline2, color="red")
    ax2.locator_params(axis='both', nbins=20)
    default_format(ax=ax2, ax_xlabel_txt="date", ax_ylabel_txt="lag (s)")      
    ax2.set_ylim([hist_bins.iloc[0], hist_bins.iloc[-1]])  # Use same scaling as for ax
    
    fig.show()


Time lags from OPENLAG runs#

2004#

locs = df.index.year == 2004
tlag_actual = df[tlag_actual_cols][locs].copy()
first_date = tlag_actual.index[0].date()
last_date = tlag_actual.index[-1].date()

timelags(gascol="CO2", series=tlag_actual["CO2_TLAG_ACTUAL"], vline1=0.05, vline2=0.75, startbin=0, endbin=4)
../../_images/a2595f7bd49f1eeec18356f871209fff170eb28f167607a846e6afb4a4d6007e.png 
timelags(gascol="H2O", series=tlag_actual["H2O_TLAG_ACTUAL"], vline1=0.05, vline2=0.75, startbin=0, endbin=4)
../../_images/8e512af8564a3dd31cf1b6db5775a2476b28b61b5207d1b73b47edaac1bbe420.png

2005_2+3#

locs = (df.index > "2005-09-30 11:30:00") & (df.index <= "2005-11-12 10:48:00")
tlag_actual = df[tlag_actual_cols][locs].copy()
first_date = tlag_actual.index[0].date()
last_date = tlag_actual.index[-1].date()

timelags(gascol="CO2", series=tlag_actual["CO2_TLAG_ACTUAL"], vline1=0.05, vline2=0.75, startbin=0, endbin=4)
../../_images/1bd6da98803a3de73127334d5be34a2f985a8686c6d43b6c95a1c3065ba4473b.png 
timelags(gascol="H2O", series=tlag_actual["H2O_TLAG_ACTUAL"], vline1=0.05, vline2=0.75, startbin=0, endbin=4)
../../_images/b684a88c1d0b3fe6e325b49b930932944f5f8d0c6aa47a5495f7e285fe54fd54.png

2005_1+4#

locs = ((df.index > "2004-12-31 23:00:00") & (df.index <= "2005-09-30 10:48:00")) | ((df.index > "2005-11-12 15:55:00") & (df.index <= "2005-12-30 23:00:00"))
tlag_actual = df[tlag_actual_cols][locs].copy()
first_date = tlag_actual.index[0].date()
last_date = tlag_actual.index[-1].date()

timelags(gascol="CO2", series=tlag_actual["CO2_TLAG_ACTUAL"], vline1=0.05, vline2=0.75, startbin=0, endbin=4)
../../_images/95bf4bd594d5fa6c14fa24c0ab758bbe7fa752d6cadf63a8bf3183ddac2e087e.png 
timelags(gascol="H2O", series=tlag_actual["H2O_TLAG_ACTUAL"], vline1=0.05, vline2=0.75, startbin=0, endbin=4)
../../_images/6b49966eaa416fa2abdaaa2525313323f0da3efecc84a4c24f413846e6161146.png

2008_1+4#

locs = ((df.index > "2007-12-31 23:00:00") & (df.index <= "2008-05-16 11:00:00")) | ((df.index > "2008-06-30 12:58:00") & (df.index <= "2008-12-31 23:00:00"))
tlag_actual = df[tlag_actual_cols][locs].copy()
first_date = tlag_actual.index[0].date()
last_date = tlag_actual.index[-1].date()

timelags(gascol="CO2", series=tlag_actual["CO2_TLAG_ACTUAL"], vline1=0.05, vline2=0.75, startbin=0, endbin=4)
../../_images/feed3fdd981713a17d6147cdcac1f62a71248c022526dc72a0a62042413dd09d.png 
timelags(gascol="H2O", series=tlag_actual["H2O_TLAG_ACTUAL"], vline1=0.05, vline2=0.75, startbin=0, endbin=4)
../../_images/628313028b4847c7559557b7b25a10ec29ccbc0668a4aa0fc4c3dafc09ba3879.png

2008_2+3#

locs = (df.index > "2008-05-16 17:52:00") & (df.index <= "2008-06-30 11:00:00")
tlag_actual = df[tlag_actual_cols][locs].copy()
first_date = tlag_actual.index[0].date()
last_date = tlag_actual.index[-1].date()

timelags(gascol="CO2", series=tlag_actual["CO2_TLAG_ACTUAL"], vline1=0.05, vline2=0.75, startbin=0, endbin=4)
../../_images/bf9b33464906afd7dc03b7542b854761c37d7b8fa9b8845bba6233cbe6162fec.png 
timelags(gascol="H2O", series=tlag_actual["H2O_TLAG_ACTUAL"], vline1=0.05, vline2=0.75, startbin=0, endbin=4)
../../_images/b598beb34026ad41d768e46f20e586e6c8eabf311b907e86fbab78a4fbe5e174.png

2006#

locs = df.index.year == 2006
tlag_actual = df[tlag_actual_cols][locs].copy()
first_date = tlag_actual.index[0].date()
last_date = tlag_actual.index[-1].date()

timelags(gascol="CO2", series=tlag_actual["CO2_TLAG_ACTUAL"], vline1=0.05, vline2=0.75, startbin=0, endbin=4)
../../_images/f4c38ff7fc4307e3d8c2b281b89ce899ce858ad0bd19cc24cb79c3f8fd77de6f.png 
timelags(gascol="H2O", series=tlag_actual["H2O_TLAG_ACTUAL"], vline1=0.05, vline2=0.75, startbin=0, endbin=4)
../../_images/3657582476e9eade5cb904a6ae376087d066aba854f974dd287c7c9566dddd5a.png

2007#

locs = df.index.year == 2007
tlag_actual = df[tlag_actual_cols][locs].copy()
first_date = tlag_actual.index[0].date()
last_date = tlag_actual.index[-1].date()

timelags(gascol="CO2", series=tlag_actual["CO2_TLAG_ACTUAL"], vline1=0.05, vline2=0.75, startbin=0, endbin=4)
../../_images/8f38a4ab4981a837da8837d62ab5116afc65961ffea672bc9f43ad489cce14de.png 
timelags(gascol="H2O", series=tlag_actual["H2O_TLAG_ACTUAL"], vline1=0.05, vline2=0.75, startbin=0, endbin=4)
../../_images/282add5c01b9120f3fb33f95203d3a7f4badd1d6663f0d625606bf74720fdb0d.png

2008_1+4#

locs = df.index.year == 2007
tlag_actual = df[tlag_actual_cols][locs].copy()
first_date = tlag_actual.index[0].date()
last_date = tlag_actual.index[-1].date()

timelags(gascol="CO2", series=tlag_actual["CO2_TLAG_ACTUAL"], vline1=0.05, vline2=0.75, startbin=0, endbin=4)
../../_images/8f38a4ab4981a837da8837d62ab5116afc65961ffea672bc9f43ad489cce14de.png 
timelags(gascol="H2O", series=tlag_actual["H2O_TLAG_ACTUAL"], vline1=0.05, vline2=0.75, startbin=0, endbin=4)
../../_images/282add5c01b9120f3fb33f95203d3a7f4badd1d6663f0d625606bf74720fdb0d.png

2009#

locs = df.index.year == 2009
tlag_actual = df[tlag_actual_cols][locs].copy()
first_date = tlag_actual.index[0].date()
last_date = tlag_actual.index[-1].date()

timelags(gascol="CO2", series=tlag_actual["CO2_TLAG_ACTUAL"], vline1=0.05, vline2=0.75, startbin=0, endbin=4)
../../_images/11ec0781c11c89040dab123c2e5b7a41a5188c0c40f2394ca3b306acc813c090.png 
timelags(gascol="H2O", series=tlag_actual["H2O_TLAG_ACTUAL"], vline1=0.05, vline2=0.75, startbin=0, endbin=4)
../../_images/2b82be3bd5161bae893e4de1ab4a18332167698349afce146d77ebd38c45038a.png

2010#

locs = df.index.year == 2010
tlag_actual = df[tlag_actual_cols][locs].copy()
first_date = tlag_actual.index[0].date()
last_date = tlag_actual.index[-1].date()

timelags(gascol="CO2", series=tlag_actual["CO2_TLAG_ACTUAL"], vline1=0.05, vline2=0.75, startbin=0, endbin=4)
../../_images/30628f33f5bfcd77462705a38e653b0fb7e89d5d37353ed66f21ffaeef29873b.png 
timelags(gascol="H2O", series=tlag_actual["H2O_TLAG_ACTUAL"], vline1=0.05, vline2=0.75, startbin=0, endbin=4)
../../_images/6904c5c3e53e98943c76757e74179f62db6836f6f8fdf2e3fa66ec7610922829.png

2011#

locs = df.index.year == 2011
tlag_actual = df[tlag_actual_cols][locs].copy()
first_date = tlag_actual.index[0].date()
last_date = tlag_actual.index[-1].date()

timelags(gascol="CO2", series=tlag_actual["CO2_TLAG_ACTUAL"], vline1=0.05, vline2=0.75, startbin=0, endbin=4)
../../_images/2e6499ba1a66ed18bdca99bf058f844cf1973069c98957d20cf6330538f65d5b.png 
timelags(gascol="H2O", series=tlag_actual["H2O_TLAG_ACTUAL"], vline1=0.05, vline2=0.75, startbin=0, endbin=4)
../../_images/a8cc75d558b0d087a437b0b663264744bfce875381876d5dcea674336ec6db61.png

2012#

locs = df.index.year == 2012
tlag_actual = df[tlag_actual_cols][locs].copy()
first_date = tlag_actual.index[0].date()
last_date = tlag_actual.index[-1].date()

timelags(gascol="CO2", series=tlag_actual["CO2_TLAG_ACTUAL"], vline1=0.05, vline2=0.75, startbin=0, endbin=4)
../../_images/044a9d70c8e6c5422ab409a3ba38f5e1dbd06b84fe4b44251a226726fc4ea583.png 
timelags(gascol="H2O", series=tlag_actual["H2O_TLAG_ACTUAL"], vline1=0.05, vline2=0.75, startbin=0, endbin=4)
../../_images/6b7797a5d8051bdaea0e3791068701e27e3674756090de049ea793649c95a853.png

2013#

locs = df.index.year == 2013
tlag_actual = df[tlag_actual_cols][locs].copy()
first_date = tlag_actual.index[0].date()
last_date = tlag_actual.index[-1].date()

timelags(gascol="CO2", series=tlag_actual["CO2_TLAG_ACTUAL"], vline1=0.05, vline2=0.75, startbin=0, endbin=4)
../../_images/5e8e4835619536e92794715529165428027e25355db59aa291c9cd5c14e2e667.png 
timelags(gascol="H2O", series=tlag_actual["H2O_TLAG_ACTUAL"], vline1=0.05, vline2=0.75, startbin=0, endbin=4)
../../_images/9f5532fc03808d11bf8f506cd364376eb3602edee33fc11312cf1f4c717a76e3.png

2014#

locs = df.index.year == 2014
tlag_actual = df[tlag_actual_cols][locs].copy()
first_date = tlag_actual.index[0].date()
last_date = tlag_actual.index[-1].date()

timelags(gascol="CO2", series=tlag_actual["CO2_TLAG_ACTUAL"], vline1=0.05, vline2=0.75, startbin=0, endbin=4)
../../_images/e7e94d022469742b51645cb2a3dfe1a46d5798c16b261f427794341af965a7c4.png 
timelags(gascol="H2O", series=tlag_actual["H2O_TLAG_ACTUAL"], vline1=0.05, vline2=0.75, startbin=0, endbin=4)
../../_images/7dc25b75e96835b6d3c8ba76f899b1b56ed0c1a41b46cf50b8028f56bb266b0d.png

2015#

locs = df.index.year == 2015
tlag_actual = df[tlag_actual_cols][locs].copy()
first_date = tlag_actual.index[0].date()
last_date = tlag_actual.index[-1].date()

timelags(gascol="CO2", series=tlag_actual["CO2_TLAG_ACTUAL"], vline1=0.05, vline2=0.75, startbin=0, endbin=4)
../../_images/777060c6fff6427312d0096b5d95fe072370827f7b2c4c40504892b77f451f22.png 
timelags(gascol="H2O", series=tlag_actual["H2O_TLAG_ACTUAL"], vline1=0.05, vline2=0.75, startbin=0, endbin=4)
../../_images/5c3dcb9a7fda614d998019fa7a44ab4cdf3d84734359e3a210f90a691ff7c5c1.png

2016_2+3#

locs = (df.index.year == 2016) & (df.index > "2016-01-11 15:05:00")
tlag_actual = df[tlag_actual_cols][locs].copy()
first_date = tlag_actual.index[0].date()
last_date = tlag_actual.index[-1].date()

timelags(gascol="CO2", series=tlag_actual["CO2_TLAG_ACTUAL"], vline1=0.05, vline2=0.75, startbin=0, endbin=2)
../../_images/eb47975a6860850b939b52972b93ee09876eda072b17880202e28a265d6dbd07.png 
timelags(gascol="H2O", series=tlag_actual["H2O_TLAG_ACTUAL"], vline1=0.05, vline2=0.75, startbin=0, endbin=7)
../../_images/0f690d025e953fb7e4fd7d1d2de0c31f959e64973df8251b106b1035d6fcc721.png

2017_1+2#

locs = df.index.year == 2017
tlag_actual = df[tlag_actual_cols][locs].copy()
first_date = tlag_actual.index[0].date()
last_date = tlag_actual.index[-1].date()

timelags(gascol="CO2", series=tlag_actual["CO2_TLAG_ACTUAL"], vline1=0.05, vline2=0.75, startbin=0, endbin=2)
../../_images/0ba9e1c1c79767ae9ac14ddf1903b307a677f516ae481b5d511ae9715092abb0.png 
timelags(gascol="H2O", series=tlag_actual["H2O_TLAG_ACTUAL"], vline1=0.05, vline2=0.75, startbin=0, endbin=7)
../../_images/653c86d2d59d21eeba88fd6c0024ee0bf7397ad7db233834982cb1358aaf554b.png

2019_3#

locs = df.index.year == 2019
tlag_actual = df[tlag_actual_cols][locs].copy()
first_date = tlag_actual.index[0].date()
last_date = tlag_actual.index[-1].date()

timelags(gascol="CO2", series=tlag_actual["CO2_TLAG_ACTUAL"], vline1=11.9, vline2=12.7, startbin=11, endbin=14)
../../_images/40084b768f787e2e9a874174dbaf3e2be3db8d8ce6efc208565a8463c74913e1.png 
timelags(gascol="H2O", series=tlag_actual["H2O_TLAG_ACTUAL"], vline1=11.9, vline2=12.7, startbin=11, endbin=14)
../../_images/b615e7125605be69152c0bbbd5ae91655633c87b35597bcd2717102d521d3c26.png

End of notebook.#

dt_string = datetime.now().strftime("%Y-%m-%d %H:%M:%S")
print(f"Finished. {dt_string}")
Contents