notebook version: 3 (4 Jul 2024)
new in this version: added check if downloaded data is indeed in 30MIN time resolution
This notebook can be used to download data from the database
InfluxDBData are stored to a
.csvfile in this folder
Data used in the manuscript https://doi.org/10.1016/j.agrformet.2023.109613.
Auto-settings#
Data settings#
DIRCONF = r'F:\Sync\luhk_work\20 - CODING\22 - POET\configs'
# DIRCONF = r'P:\Flux\RDS_calculations\_scripts\_configs\configs' # Folder with configuration files: needed e.g. for connection to database
TIMEZONE_OFFSET_TO_UTC_HOURS = 1 # Timezone, e.g. "1" is translated to timezone "UTC+01:00" (CET, winter time)
REQUIRED_TIME_RESOLUTION = '30min' # 30MIN time resolution
Imports#
import importlib.metadata
from datetime import datetime
%matplotlib inline
import seaborn as sns
from pathlib import Path
from diive.core.io.files import save_parquet
sns.set_theme('notebook')
from dbc_influxdb import dbcInflux
from diive.core.plotting.heatmap_datetime import HeatmapDateTime
import warnings
warnings.filterwarnings(action='ignore', category=FutureWarning)
warnings.filterwarnings(action='ignore', category=UserWarning)
dt_string = datetime.now().strftime("%Y-%m-%d %H:%M:%S")
version_diive = importlib.metadata.version("diive")
print(f"diive version: v{version_diive}")
version_dbc = importlib.metadata.version("dbc_influxdb")
print(f"dbc-influxdb version: v{version_dbc}")
dbc = dbcInflux(dirconf=DIRCONF) # Connect to database
diive version: v0.80.0
dbc-influxdb version: v0.12.0
Reading configuration files was successful.
Connection to database works.
DOWNLOAD Data from Feigenwinter et al., 2023 (2005-2020)#
Explore data online here
Manuscript: https://doi.org/10.1016/j.agrformet.2023.109613
Download data from ETH Research Collection: https://doi.org/10.3929/ethz-b-000524199
Assembling meteo data from Feigenwinter et al. (2023) for years between 2005 and 2020
Using newly screened data from the database for the years between 2021 and 2023
%%time
data_simple_fw23, data_detailed_fw23, assigned_measurements_fw23 = dbc.download(
bucket=f'ch-cha_processed',
measurements=['G', 'TA', 'RH', 'LW', 'PPFD', 'SW', 'PA', 'PREC', 'SWC', 'TS'],
fields=['G_0.03', 'LW_IN', 'LW_OUT', 'PA', 'PA_SOURCE', 'PPFD_IN', 'PREC_RAIN', 'PREC_RAIN_SOURCE', 'RH', 'RH_SOURCE', 'SW_IN', 'SW_IN_SOURCE', 'SW_OUT',
'SWC_0.05', 'SWC_0.15', 'SWC_0.75', 'TA', 'TA_SOURCE', 'TS_0.04', 'TS_0.15', 'TS_0.4'],
start='2005-01-01 00:00:01', # Download data starting with this date (the start date itself IS included),
stop='2021-01-01 00:00:01', # Download data before this date (the stop date itself IS NOT included),
timezone_offset_to_utc_hours=TIMEZONE_OFFSET_TO_UTC_HOURS,
data_version='ms_feigenwinter2023a'
)
DOWNLOADING
from bucket ch-cha_processed
variables ['G_0.03', 'LW_IN', 'LW_OUT', 'PA', 'PA_SOURCE', 'PPFD_IN', 'PREC_RAIN', 'PREC_RAIN_SOURCE', 'RH', 'RH_SOURCE', 'SW_IN', 'SW_IN_SOURCE', 'SW_OUT', 'SWC_0.05', 'SWC_0.15', 'SWC_0.75', 'TA', 'TA_SOURCE', 'TS_0.04', 'TS_0.15', 'TS_0.4']
from measurements ['G', 'TA', 'RH', 'LW', 'PPFD', 'SW', 'PA', 'PREC', 'SWC', 'TS']
from data version ms_feigenwinter2023a
between 2005-01-01 00:00:01 and 2021-01-01 00:00:01
with timezone offset to UTC of 1
Used querystring: from(bucket: "ch-cha_processed") |> range(start: 2005-01-01T00:00:01+01:00, stop: 2021-01-01T00:00:01+01:00) |> filter(fn: (r) => r["_measurement"] == "G" or r["_measurement"] == "TA" or r["_measurement"] == "RH" or r["_measurement"] == "LW" or r["_measurement"] == "PPFD" or r["_measurement"] == "SW" or r["_measurement"] == "PA" or r["_measurement"] == "PREC" or r["_measurement"] == "SWC" or r["_measurement"] == "TS") |> filter(fn: (r) => r["data_version"] == "ms_feigenwinter2023a") |> filter(fn: (r) => r["_field"] == "G_0.03" or r["_field"] == "LW_IN" or r["_field"] == "LW_OUT" or r["_field"] == "PA" or r["_field"] == "PA_SOURCE" or r["_field"] == "PPFD_IN" or r["_field"] == "PREC_RAIN" or r["_field"] == "PREC_RAIN_SOURCE" or r["_field"] == "RH" or r["_field"] == "RH_SOURCE" or r["_field"] == "SW_IN" or r["_field"] == "SW_IN_SOURCE" or r["_field"] == "SW_OUT" or r["_field"] == "SWC_0.05" or r["_field"] == "SWC_0.15" or r["_field"] == "SWC_0.75" or r["_field"] == "TA" or r["_field"] == "TA_SOURCE" or r["_field"] == "TS_0.04" or r["_field"] == "TS_0.15" or r["_field"] == "TS_0.4") |> pivot(rowKey:["_time"], columnKey: ["_field"], valueColumn: "_value")
querystring was constructed from:
bucketstring: from(bucket: "ch-cha_processed")
rangestring: |> range(start: 2005-01-01T00:00:01+01:00, stop: 2021-01-01T00:00:01+01:00)
measurementstring: |> filter(fn: (r) => r["_measurement"] == "G" or r["_measurement"] == "TA" or r["_measurement"] == "RH" or r["_measurement"] == "LW" or r["_measurement"] == "PPFD" or r["_measurement"] == "SW" or r["_measurement"] == "PA" or r["_measurement"] == "PREC" or r["_measurement"] == "SWC" or r["_measurement"] == "TS")
dataversionstring: |> filter(fn: (r) => r["data_version"] == "ms_feigenwinter2023a")
fieldstring: |> filter(fn: (r) => r["_field"] == "G_0.03" or r["_field"] == "LW_IN" or r["_field"] == "LW_OUT" or r["_field"] == "PA" or r["_field"] == "PA_SOURCE" or r["_field"] == "PPFD_IN" or r["_field"] == "PREC_RAIN" or r["_field"] == "PREC_RAIN_SOURCE" or r["_field"] == "RH" or r["_field"] == "RH_SOURCE" or r["_field"] == "SW_IN" or r["_field"] == "SW_IN_SOURCE" or r["_field"] == "SW_OUT" or r["_field"] == "SWC_0.05" or r["_field"] == "SWC_0.15" or r["_field"] == "SWC_0.75" or r["_field"] == "TA" or r["_field"] == "TA_SOURCE" or r["_field"] == "TS_0.04" or r["_field"] == "TS_0.15" or r["_field"] == "TS_0.4")
pivotstring: |> pivot(rowKey:["_time"], columnKey: ["_field"], valueColumn: "_value")
Download finished.
Downloaded data for 21 variables:
<-- G_0.03 (257233 records) first date: 2005-09-09 10:00:00 last date: 2021-01-01 00:00:00
<-- LW_IN (264553 records) first date: 2005-09-09 10:00:00 last date: 2021-01-01 00:00:00
<-- LW_OUT (264726 records) first date: 2005-09-09 10:00:00 last date: 2021-01-01 00:00:00
<-- PA (275885 records) first date: 2005-01-01 00:30:00 last date: 2021-01-01 00:00:00
<-- PA_SOURCE (280512 records) first date: 2005-01-01 00:30:00 last date: 2021-01-01 00:00:00
<-- PPFD_IN (265182 records) first date: 2005-09-09 10:00:00 last date: 2021-01-01 00:00:00
<-- PREC_RAIN (280512 records) first date: 2005-01-01 00:30:00 last date: 2021-01-01 00:00:00
<-- PREC_RAIN_SOURCE (280512 records) first date: 2005-01-01 00:30:00 last date: 2021-01-01 00:00:00
<-- RH (280512 records) first date: 2005-01-01 00:30:00 last date: 2021-01-01 00:00:00
<-- RH_SOURCE (280439 records) first date: 2005-01-01 00:30:00 last date: 2021-01-01 00:00:00
<-- SW_IN (280512 records) first date: 2005-01-01 00:30:00 last date: 2021-01-01 00:00:00
<-- SW_IN_SOURCE (280512 records) first date: 2005-01-01 00:30:00 last date: 2021-01-01 00:00:00
<-- SW_OUT (265645 records) first date: 2005-09-09 10:00:00 last date: 2021-01-01 00:00:00
<-- SWC_0.05 (255743 records) first date: 2005-09-09 10:00:00 last date: 2021-01-01 00:00:00
<-- SWC_0.15 (261403 records) first date: 2005-09-09 10:00:00 last date: 2021-01-01 00:00:00
<-- SWC_0.75 (261394 records) first date: 2005-09-09 10:00:00 last date: 2021-01-01 00:00:00
<-- TA (280512 records) first date: 2005-01-01 00:30:00 last date: 2021-01-01 00:00:00
<-- TA_SOURCE (280512 records) first date: 2005-01-01 00:30:00 last date: 2021-01-01 00:00:00
<-- TS_0.04 (261097 records) first date: 2005-09-09 10:00:00 last date: 2021-01-01 00:00:00
<-- TS_0.15 (261812 records) first date: 2005-09-09 10:00:00 last date: 2021-01-01 00:00:00
<-- TS_0.4 (261914 records) first date: 2005-09-09 10:00:00 last date: 2021-01-01 00:00:00
========================================
Fields in measurement G of bucket ch-cha_processed:
#1 ch-cha_processed G G_0.03
#2 ch-cha_processed G G_F_MDS
#3 ch-cha_processed G G_F_MDS_QC
#4 ch-cha_processed G G_GF1_0.03_1
#5 ch-cha_processed G G_GF1_0.03_2
#6 ch-cha_processed G G_GF1_0.05_1
#7 ch-cha_processed G G_GF1_0.05_2
Found 7 fields in measurement G of bucket ch-cha_processed.
========================================
========================================
Fields in measurement TA of bucket ch-cha_processed:
#1 ch-cha_processed TA TA
#2 ch-cha_processed TA TA_1_1_1
#3 ch-cha_processed TA TA_EP
#4 ch-cha_processed TA TA_ERA
#5 ch-cha_processed TA TA_F
#6 ch-cha_processed TA TA_F_MDS
#7 ch-cha_processed TA TA_F_MDS_QC
#8 ch-cha_processed TA TA_F_QC
#9 ch-cha_processed TA TA_SOURCE
#10 ch-cha_processed TA TA_T1_2_1
#11 ch-cha_processed TA T_SONIC
Found 11 fields in measurement TA of bucket ch-cha_processed.
========================================
========================================
Fields in measurement RH of bucket ch-cha_processed:
#1 ch-cha_processed RH RH
#2 ch-cha_processed RH RH_1_1_1
#3 ch-cha_processed RH RH_EP
#4 ch-cha_processed RH RH_SCALED_T1_2_1
#5 ch-cha_processed RH RH_SOURCE
#6 ch-cha_processed RH RH_T1_2_1
Found 6 fields in measurement RH of bucket ch-cha_processed.
========================================
========================================
Fields in measurement LW of bucket ch-cha_processed:
#1 ch-cha_processed LW LW_IN
#2 ch-cha_processed LW LW_IN_1_1_1
#3 ch-cha_processed LW LW_IN_ERA
#4 ch-cha_processed LW LW_IN_F
#5 ch-cha_processed LW LW_IN_F_MDS
#6 ch-cha_processed LW LW_IN_F_MDS_QC
#7 ch-cha_processed LW LW_IN_F_QC
#8 ch-cha_processed LW LW_IN_JSB
#9 ch-cha_processed LW LW_IN_JSB_ERA
#10 ch-cha_processed LW LW_IN_JSB_F
#11 ch-cha_processed LW LW_IN_JSB_F_QC
#12 ch-cha_processed LW LW_IN_JSB_QC
#13 ch-cha_processed LW LW_IN_T1_2_1
#14 ch-cha_processed LW LW_OUT
#15 ch-cha_processed LW LW_OUT_T1_2_1
Found 15 fields in measurement LW of bucket ch-cha_processed.
========================================
========================================
Fields in measurement PPFD of bucket ch-cha_processed:
#1 ch-cha_processed PPFD PPFD_IN
#2 ch-cha_processed PPFD PPFD_IN_1_1_1
#3 ch-cha_processed PPFD PPFD_IN_CORRECTED_SETTO_0_T1_2_1
#4 ch-cha_processed PPFD PPFD_IN_CORRECTED_SETTO_0_T1_2_2
#5 ch-cha_processed PPFD PPFD_IN_CORRECTED_T1_2_1
#6 ch-cha_processed PPFD PPFD_IN_CORRECTED_T1_2_2
#7 ch-cha_processed PPFD PPFD_IN_T1_2_1
#8 ch-cha_processed PPFD PPFD_IN_T1_2_2
#9 ch-cha_processed PPFD PPFD_OUT_CORRECTED_SETTO_0_T1_2_2
#10 ch-cha_processed PPFD PPFD_OUT_CORRECTED_T1_2_2
#11 ch-cha_processed PPFD PPFD_OUT_T1_2_2
Found 11 fields in measurement PPFD of bucket ch-cha_processed.
========================================
========================================
Fields in measurement SW of bucket ch-cha_processed:
#1 ch-cha_processed SW NIGHT
#2 ch-cha_processed SW SW_IN
#3 ch-cha_processed SW SW_IN_1_1_1
#4 ch-cha_processed SW SW_IN_AZI_4
#5 ch-cha_processed SW SW_IN_CORRECTED_SETTO_0_T1_2_1
#6 ch-cha_processed SW SW_IN_CORRECTED_T1_2_1
#7 ch-cha_processed SW SW_IN_ELE_4
#8 ch-cha_processed SW SW_IN_ERA
#9 ch-cha_processed SW SW_IN_F
#10 ch-cha_processed SW SW_IN_F_MDS
#11 ch-cha_processed SW SW_IN_F_MDS_QC
#12 ch-cha_processed SW SW_IN_F_QC
#13 ch-cha_processed SW SW_IN_POT
#14 ch-cha_processed SW SW_IN_SOURCE
#15 ch-cha_processed SW SW_IN_T1_2_1
#16 ch-cha_processed SW SW_OUT
#17 ch-cha_processed SW SW_OUT_CORRECTED_SETTO_0_T1_2_1
#18 ch-cha_processed SW SW_OUT_CORRECTED_T1_2_1
#19 ch-cha_processed SW SW_OUT_T1_2_1
Found 19 fields in measurement SW of bucket ch-cha_processed.
========================================
========================================
Fields in measurement PA of bucket ch-cha_processed:
#1 ch-cha_processed PA CUSTOM_AIR_P_MEAN
#2 ch-cha_processed PA PA
#3 ch-cha_processed PA PA_1_1_1
#4 ch-cha_processed PA PA_EP
#5 ch-cha_processed PA PA_ERA
#6 ch-cha_processed PA PA_F
#7 ch-cha_processed PA PA_F_QC
#8 ch-cha_processed PA PA_GF1_0.9_1
#9 ch-cha_processed PA PA_SOURCE
Found 9 fields in measurement PA of bucket ch-cha_processed.
========================================
========================================
Fields in measurement PREC of bucket ch-cha_processed:
#1 ch-cha_processed PREC PREC
#2 ch-cha_processed PREC PREC_ERA
#3 ch-cha_processed PREC PREC_F
#4 ch-cha_processed PREC PREC_F_QC
#5 ch-cha_processed PREC PREC_RAIN
#6 ch-cha_processed PREC PREC_RAIN_SOURCE
#7 ch-cha_processed PREC PREC_TOT_M1_1_1
#8 ch-cha_processed PREC P_RAIN_TOT_GF1_0.5_1
Found 8 fields in measurement PREC of bucket ch-cha_processed.
========================================
========================================
Fields in measurement SWC of bucket ch-cha_processed:
#1 ch-cha_processed SWC SWC_0.05
#2 ch-cha_processed SWC SWC_0.15
#3 ch-cha_processed SWC SWC_0.75
#4 ch-cha_processed SWC SWC_F_MDS_1
#5 ch-cha_processed SWC SWC_F_MDS_1_QC
#6 ch-cha_processed SWC SWC_F_MDS_2
#7 ch-cha_processed SWC SWC_F_MDS_2_QC
#8 ch-cha_processed SWC SWC_F_MDS_3
#9 ch-cha_processed SWC SWC_F_MDS_3_QC
#10 ch-cha_processed SWC SWC_F_MDS_4
#11 ch-cha_processed SWC SWC_F_MDS_4_QC
#12 ch-cha_processed SWC SWC_F_MDS_5
#13 ch-cha_processed SWC SWC_F_MDS_5_QC
#14 ch-cha_processed SWC SWC_F_MDS_6
#15 ch-cha_processed SWC SWC_F_MDS_6_QC
#16 ch-cha_processed SWC SWC_F_MDS_7
#17 ch-cha_processed SWC SWC_F_MDS_7_QC
#18 ch-cha_processed SWC SWC_F_MDS_8
#19 ch-cha_processed SWC SWC_F_MDS_8_QC
#20 ch-cha_processed SWC SWC_F_MDS_9
#21 ch-cha_processed SWC SWC_F_MDS_9_QC
#22 ch-cha_processed SWC SWC_GF1_0.05_1
#23 ch-cha_processed SWC SWC_GF1_0.05_2
#24 ch-cha_processed SWC SWC_GF1_0.05_3
#25 ch-cha_processed SWC SWC_GF1_0.15_1
#26 ch-cha_processed SWC SWC_GF1_0.1_1
#27 ch-cha_processed SWC SWC_GF1_0.1_2
#28 ch-cha_processed SWC SWC_GF1_0.1_3
#29 ch-cha_processed SWC SWC_GF1_0.25_1
#30 ch-cha_processed SWC SWC_GF1_0.2_1
#31 ch-cha_processed SWC SWC_GF1_0.2_2
#32 ch-cha_processed SWC SWC_GF1_0.2_3
#33 ch-cha_processed SWC SWC_GF1_0.3_1
#34 ch-cha_processed SWC SWC_GF1_0.3_2
#35 ch-cha_processed SWC SWC_GF1_0.3_3
#36 ch-cha_processed SWC SWC_GF1_0.4_1
#37 ch-cha_processed SWC SWC_GF1_0.4_3
#38 ch-cha_processed SWC SWC_GF1_0.5_1
#39 ch-cha_processed SWC SWC_GF1_0.5_3
#40 ch-cha_processed SWC SWC_GF1_0.6_3
#41 ch-cha_processed SWC SWC_GF1_0.75_1
#42 ch-cha_processed SWC SWC_GF1_0.75_3
#43 ch-cha_processed SWC SWC_GF1_1_3
#44 ch-cha_processed SWC SWC_GF4_0.05_1
#45 ch-cha_processed SWC SWC_GF4_0.05_2
#46 ch-cha_processed SWC SWC_GF4_0.05_3
#47 ch-cha_processed SWC SWC_GF4_0.05_4
#48 ch-cha_processed SWC SWC_GF4_0.05_5
#49 ch-cha_processed SWC SWC_GF4_0.1_1
#50 ch-cha_processed SWC SWC_GF4_0.1_2
#51 ch-cha_processed SWC SWC_GF4_0.1_3
#52 ch-cha_processed SWC SWC_GF4_0.1_4
#53 ch-cha_processed SWC SWC_GF4_0.1_5
#54 ch-cha_processed SWC SWC_GF4_0.3_4
#55 ch-cha_processed SWC SWC_GF4_0.3_5
#56 ch-cha_processed SWC SWC_GF4_0.5_4
#57 ch-cha_processed SWC SWC_GF4_0.5_5
#58 ch-cha_processed SWC SWC_GF5_0.05_1
#59 ch-cha_processed SWC SWC_GF5_0.05_2
#60 ch-cha_processed SWC SWC_GF5_0.05_3
#61 ch-cha_processed SWC SWC_GF5_0.05_4
#62 ch-cha_processed SWC SWC_GF5_0.1_1
#63 ch-cha_processed SWC SWC_GF5_0.1_2
#64 ch-cha_processed SWC SWC_GF5_0.1_3
#65 ch-cha_processed SWC SWC_GF5_0.1_4
Found 65 fields in measurement SWC of bucket ch-cha_processed.
========================================
========================================
Fields in measurement TS of bucket ch-cha_processed:
#1 ch-cha_processed TS TS_0.04
#2 ch-cha_processed TS TS_0.15
#3 ch-cha_processed TS TS_0.4
#4 ch-cha_processed TS TS_AVG_GF1_0.025_2
#5 ch-cha_processed TS TS_AVG_GF1_1_3
#6 ch-cha_processed TS TS_F_MDS_1
#7 ch-cha_processed TS TS_F_MDS_10
#8 ch-cha_processed TS TS_F_MDS_10_QC
#9 ch-cha_processed TS TS_F_MDS_11
#10 ch-cha_processed TS TS_F_MDS_11_QC
#11 ch-cha_processed TS TS_F_MDS_12
#12 ch-cha_processed TS TS_F_MDS_12_QC
#13 ch-cha_processed TS TS_F_MDS_13
#14 ch-cha_processed TS TS_F_MDS_13_QC
#15 ch-cha_processed TS TS_F_MDS_14
#16 ch-cha_processed TS TS_F_MDS_14_QC
#17 ch-cha_processed TS TS_F_MDS_1_QC
#18 ch-cha_processed TS TS_F_MDS_2
#19 ch-cha_processed TS TS_F_MDS_2_QC
#20 ch-cha_processed TS TS_F_MDS_3
#21 ch-cha_processed TS TS_F_MDS_3_QC
#22 ch-cha_processed TS TS_F_MDS_4
#23 ch-cha_processed TS TS_F_MDS_4_QC
#24 ch-cha_processed TS TS_F_MDS_5
#25 ch-cha_processed TS TS_F_MDS_5_QC
#26 ch-cha_processed TS TS_F_MDS_6
#27 ch-cha_processed TS TS_F_MDS_6_QC
#28 ch-cha_processed TS TS_F_MDS_7
#29 ch-cha_processed TS TS_F_MDS_7_QC
#30 ch-cha_processed TS TS_F_MDS_8
#31 ch-cha_processed TS TS_F_MDS_8_QC
#32 ch-cha_processed TS TS_F_MDS_9
#33 ch-cha_processed TS TS_F_MDS_9_QC
#34 ch-cha_processed TS TS_GF1_0.01_1
#35 ch-cha_processed TS TS_GF1_0.01_2
#36 ch-cha_processed TS TS_GF1_0.025_2
#37 ch-cha_processed TS TS_GF1_0.02_1
#38 ch-cha_processed TS TS_GF1_0.04_1
#39 ch-cha_processed TS TS_GF1_0.05_1
#40 ch-cha_processed TS TS_GF1_0.05_2
#41 ch-cha_processed TS TS_GF1_0.05_3
#42 ch-cha_processed TS TS_GF1_0.07_1
#43 ch-cha_processed TS TS_GF1_0.15_1
#44 ch-cha_processed TS TS_GF1_0.1_1
#45 ch-cha_processed TS TS_GF1_0.1_2
#46 ch-cha_processed TS TS_GF1_0.1_3
#47 ch-cha_processed TS TS_GF1_0.25_1
#48 ch-cha_processed TS TS_GF1_0.2_1
#49 ch-cha_processed TS TS_GF1_0.2_2
#50 ch-cha_processed TS TS_GF1_0.2_3
#51 ch-cha_processed TS TS_GF1_0.3_1
#52 ch-cha_processed TS TS_GF1_0.3_2
#53 ch-cha_processed TS TS_GF1_0.3_3
#54 ch-cha_processed TS TS_GF1_0.4_1
#55 ch-cha_processed TS TS_GF1_0.4_3
#56 ch-cha_processed TS TS_GF1_0.5_1
#57 ch-cha_processed TS TS_GF1_0.5_3
#58 ch-cha_processed TS TS_GF1_0.6_3
#59 ch-cha_processed TS TS_GF1_0.75_3
#60 ch-cha_processed TS TS_GF1_0.95_1
#61 ch-cha_processed TS TS_GF1_1_3
#62 ch-cha_processed TS TS_GF4_0.05_1
#63 ch-cha_processed TS TS_GF4_0.05_2
#64 ch-cha_processed TS TS_GF4_0.05_3
#65 ch-cha_processed TS TS_GF4_0.05_4
#66 ch-cha_processed TS TS_GF4_0.05_5
#67 ch-cha_processed TS TS_GF4_0.1_1
#68 ch-cha_processed TS TS_GF4_0.1_2
#69 ch-cha_processed TS TS_GF4_0.1_3
#70 ch-cha_processed TS TS_GF4_0.1_4
#71 ch-cha_processed TS TS_GF4_0.1_5
#72 ch-cha_processed TS TS_GF4_0.3_4
#73 ch-cha_processed TS TS_GF4_0.3_5
#74 ch-cha_processed TS TS_GF4_0.5_4
#75 ch-cha_processed TS TS_GF4_0.5_5
#76 ch-cha_processed TS TS_GF5_0.05_1
#77 ch-cha_processed TS TS_GF5_0.05_2
#78 ch-cha_processed TS TS_GF5_0.05_3
#79 ch-cha_processed TS TS_GF5_0.05_4
#80 ch-cha_processed TS TS_GF5_0.1_1
#81 ch-cha_processed TS TS_GF5_0.1_2
#82 ch-cha_processed TS TS_GF5_0.1_3
#83 ch-cha_processed TS TS_GF5_0.1_4
Found 83 fields in measurement TS of bucket ch-cha_processed.
========================================
CPU times: total: 18min 20s
Wall time: 19min 13s
data_simple_fw23
| G_0.03 | LW_IN | LW_OUT | PA | PA_SOURCE | PPFD_IN | PREC_RAIN | PREC_RAIN_SOURCE | RH | RH_SOURCE | SWC_0.05 | SWC_0.15 | SWC_0.75 | SW_IN | SW_IN_SOURCE | SW_OUT | TA | TA_SOURCE | TS_0.04 | TS_0.15 | TS_0.4 | |
|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|
| TIMESTAMP_END | |||||||||||||||||||||
| 2005-01-01 00:30:00 | NaN | NaN | NaN | 978.100000 | 1.0 | NaN | 0.0 | 1.0 | 85.400000 | 1.0 | NaN | NaN | NaN | 3.0 | 3.0 | NaN | 1.566667 | 3.0 | NaN | NaN | NaN |
| 2005-01-01 01:00:00 | NaN | NaN | NaN | 977.933333 | 1.0 | NaN | 0.0 | 1.0 | 85.700000 | 1.0 | NaN | NaN | NaN | 3.0 | 3.0 | NaN | 1.533333 | 3.0 | NaN | NaN | NaN |
| 2005-01-01 01:30:00 | NaN | NaN | NaN | 977.900000 | 1.0 | NaN | 0.1 | 1.0 | 86.600000 | 1.0 | NaN | NaN | NaN | 3.0 | 3.0 | NaN | 1.566667 | 3.0 | NaN | NaN | NaN |
| 2005-01-01 02:00:00 | NaN | NaN | NaN | 977.833333 | 1.0 | NaN | 0.0 | 1.0 | 89.600000 | 1.0 | NaN | NaN | NaN | 3.0 | 3.0 | NaN | 1.566667 | 3.0 | NaN | NaN | NaN |
| 2005-01-01 02:30:00 | NaN | NaN | NaN | 977.833333 | 1.0 | NaN | 0.1 | 1.0 | 91.433333 | 1.0 | NaN | NaN | NaN | 0.0 | 3.0 | NaN | 1.500000 | 3.0 | NaN | NaN | NaN |
| ... | ... | ... | ... | ... | ... | ... | ... | ... | ... | ... | ... | ... | ... | ... | ... | ... | ... | ... | ... | ... | ... |
| 2020-12-31 22:00:00 | -5.246918 | 327.0590 | 305.8658 | 958.613800 | 0.0 | 0.012177 | 0.0 | 0.0 | 100.000000 | 0.0 | 43.15512 | 37.83649 | 42.34799 | 0.0 | 0.0 | -13.17378 | -0.059646 | 0.0 | 7.059905 | 4.751553 | 5.665280 |
| 2020-12-31 22:30:00 | -5.688780 | 325.6822 | 299.9425 | 958.361800 | 0.0 | 0.015688 | 0.0 | 0.0 | 100.000000 | 0.0 | 43.14826 | 37.83116 | 42.34475 | 0.0 | 0.0 | -14.05419 | -0.747990 | 0.0 | 7.037867 | 4.753286 | 5.666638 |
| 2020-12-31 23:00:00 | -6.256253 | 325.8016 | 303.5924 | 958.257100 | 0.0 | 0.015338 | 0.0 | 0.0 | 100.000000 | 0.0 | 43.14190 | 37.82694 | 42.34330 | 0.0 | 0.0 | -13.52161 | -0.606677 | 0.0 | 7.021161 | 4.754451 | 5.668180 |
| 2020-12-31 23:30:00 | -6.492525 | 325.8260 | 307.0624 | 958.275800 | 0.0 | 0.010785 | 0.0 | 0.0 | 100.000000 | 0.0 | 43.13696 | 37.82402 | 42.34280 | 0.0 | 0.0 | -12.03449 | -0.063648 | 0.0 | 6.998506 | 4.753085 | 5.669790 |
| 2021-01-01 00:00:00 | -6.446632 | 328.6548 | 309.0171 | 958.237500 | 0.0 | 0.007877 | 0.0 | 0.0 | 100.000000 | 0.0 | 43.13130 | 37.82018 | 42.30388 | 0.0 | 0.0 | -12.31394 | 0.177185 | 0.0 | 6.990282 | 4.749334 | 5.671485 |
280512 rows × 21 columns
Plot downloaded data#
data_simple_fw23.plot(subplots=True, x_compat=True, title="Data from Feigenwinter et al. (2023)", figsize=(20, 12));
RENAME VARIABLES to group convention#
renaming_dict = {
'G_0.03': 'G_GF1_0.03_1',
'LW_IN': 'LW_IN_T1_2_1',
'LW_OUT': 'LW_OUT_T1_2_1',
'PA': 'PA_GF1_0.9_1',
'PA_SOURCE': 'FLAG_PA_GF1_0.9_1_ISFILLED', # units: "0=measured; 1=MeteoSwiss CHZ"
'PPFD_IN': 'PPFD_IN_T1_2_2',
'PREC_RAIN': 'PREC_RAIN_TOT_GF1_0.5_1',
'PREC_RAIN_SOURCE': 'FLAG_PREC_RAIN_TOT_GF1_0.5_1_ISFILLED', # units: "0=measured; 1=MeteoSwiss CHZ; 2=set missing value to zero"
'RH': 'RH_T1_2_1',
'RH_SOURCE': 'FLAG_RH_T1_2_1_ISFILLED', # units: "0=measured; 1=MeteoSwiss CHZ; 2=running median"
'SW_IN': 'SW_IN_T1_2_1',
'SW_IN_SOURCE': 'FLAG_SW_IN_T1_2_1_ISFILLED', # units: "0=measured; 1=sFillMDC; 2=MeteoSwiss CHZ; 3=MeteoSwiss WAE"
'SW_OUT': 'SW_OUT_T1_2_1',
'SWC_0.05': 'SWC_GF1_0.05_1',
'SWC_0.15': 'SWC_GF1_0.15_1',
'SWC_0.75': 'SWC_GF1_0.75_1',
'TA': 'TA_T1_2_1',
'TA_SOURCE': 'FLAG_TA_T1_2_1_ISFILLED', # units: "0=measured; 1=sFillLUT_Rg; 2=sFillMDC; 3=MeteoSwiss CHZ"
'TS_0.04': 'TS_GF1_0.04_1',
'TS_0.15': 'TS_GF1_0.15_1',
'TS_0.4': 'TS_GF1_0.4_1'
}
data_simple_renamed_fw23 = data_simple_fw23.copy()
data_simple_renamed_fw23 = data_simple_renamed_fw23.rename(columns=renaming_dict)
data_simple_renamed_fw23
| G_GF1_0.03_1 | LW_IN_T1_2_1 | LW_OUT_T1_2_1 | PA_GF1_0.9_1 | FLAG_PA_GF1_0.9_1_ISFILLED | PPFD_IN_T1_2_2 | PREC_RAIN_TOT_GF1_0.5_1 | FLAG_PREC_RAIN_TOT_GF1_0.5_1_ISFILLED | RH_T1_2_1 | FLAG_RH_T1_2_1_ISFILLED | SWC_GF1_0.05_1 | SWC_GF1_0.15_1 | SWC_GF1_0.75_1 | SW_IN_T1_2_1 | FLAG_SW_IN_T1_2_1_ISFILLED | SW_OUT_T1_2_1 | TA_T1_2_1 | FLAG_TA_T1_2_1_ISFILLED | TS_GF1_0.04_1 | TS_GF1_0.15_1 | TS_GF1_0.4_1 | |
|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|
| TIMESTAMP_END | |||||||||||||||||||||
| 2005-01-01 00:30:00 | NaN | NaN | NaN | 978.100000 | 1.0 | NaN | 0.0 | 1.0 | 85.400000 | 1.0 | NaN | NaN | NaN | 3.0 | 3.0 | NaN | 1.566667 | 3.0 | NaN | NaN | NaN |
| 2005-01-01 01:00:00 | NaN | NaN | NaN | 977.933333 | 1.0 | NaN | 0.0 | 1.0 | 85.700000 | 1.0 | NaN | NaN | NaN | 3.0 | 3.0 | NaN | 1.533333 | 3.0 | NaN | NaN | NaN |
| 2005-01-01 01:30:00 | NaN | NaN | NaN | 977.900000 | 1.0 | NaN | 0.1 | 1.0 | 86.600000 | 1.0 | NaN | NaN | NaN | 3.0 | 3.0 | NaN | 1.566667 | 3.0 | NaN | NaN | NaN |
| 2005-01-01 02:00:00 | NaN | NaN | NaN | 977.833333 | 1.0 | NaN | 0.0 | 1.0 | 89.600000 | 1.0 | NaN | NaN | NaN | 3.0 | 3.0 | NaN | 1.566667 | 3.0 | NaN | NaN | NaN |
| 2005-01-01 02:30:00 | NaN | NaN | NaN | 977.833333 | 1.0 | NaN | 0.1 | 1.0 | 91.433333 | 1.0 | NaN | NaN | NaN | 0.0 | 3.0 | NaN | 1.500000 | 3.0 | NaN | NaN | NaN |
| ... | ... | ... | ... | ... | ... | ... | ... | ... | ... | ... | ... | ... | ... | ... | ... | ... | ... | ... | ... | ... | ... |
| 2020-12-31 22:00:00 | -5.246918 | 327.0590 | 305.8658 | 958.613800 | 0.0 | 0.012177 | 0.0 | 0.0 | 100.000000 | 0.0 | 43.15512 | 37.83649 | 42.34799 | 0.0 | 0.0 | -13.17378 | -0.059646 | 0.0 | 7.059905 | 4.751553 | 5.665280 |
| 2020-12-31 22:30:00 | -5.688780 | 325.6822 | 299.9425 | 958.361800 | 0.0 | 0.015688 | 0.0 | 0.0 | 100.000000 | 0.0 | 43.14826 | 37.83116 | 42.34475 | 0.0 | 0.0 | -14.05419 | -0.747990 | 0.0 | 7.037867 | 4.753286 | 5.666638 |
| 2020-12-31 23:00:00 | -6.256253 | 325.8016 | 303.5924 | 958.257100 | 0.0 | 0.015338 | 0.0 | 0.0 | 100.000000 | 0.0 | 43.14190 | 37.82694 | 42.34330 | 0.0 | 0.0 | -13.52161 | -0.606677 | 0.0 | 7.021161 | 4.754451 | 5.668180 |
| 2020-12-31 23:30:00 | -6.492525 | 325.8260 | 307.0624 | 958.275800 | 0.0 | 0.010785 | 0.0 | 0.0 | 100.000000 | 0.0 | 43.13696 | 37.82402 | 42.34280 | 0.0 | 0.0 | -12.03449 | -0.063648 | 0.0 | 6.998506 | 4.753085 | 5.669790 |
| 2021-01-01 00:00:00 | -6.446632 | 328.6548 | 309.0171 | 958.237500 | 0.0 | 0.007877 | 0.0 | 0.0 | 100.000000 | 0.0 | 43.13130 | 37.82018 | 42.30388 | 0.0 | 0.0 | -12.31394 | 0.177185 | 0.0 | 6.990282 | 4.749334 | 5.671485 |
280512 rows × 21 columns
Plot renamed data#
data_simple_renamed_fw23.plot(subplots=True, x_compat=True, title="Data from Feigenwinter et al. (2023)", figsize=(20, 12));
SAVE TO FILE#
OUTNAME = "13.1_CH-CHA_FEIGENW_meteo_2005-2020"
OUTPATH = r"F:\Sync\luhk_work\20 - CODING\29 - WORKBENCH\dataset_cha_fp2024_2005-2023\10_METEO\13_merge_meteo_for_analyses_2005-2023"
filepath = save_parquet(filename=OUTNAME, data=data_simple_renamed_fw23, outpath=OUTPATH)
data_simple_renamed_fw23.to_csv(Path(OUTPATH) / f"{OUTNAME}.csv")
Saved file F:\Sync\luhk_work\20 - CODING\29 - WORKBENCH\dataset_cha_fp2024_2005-2023\10_METEO\13_merge_meteo_for_analyses_2005-2023\13.1_CH-CHA_FEIGENW_meteo_2005-2020.parquet (0.330 seconds).
PLOT HEATMAPS#
for col in data_simple_renamed_fw23.columns:
series = data_simple_renamed_fw23[col]
series.name = col
HeatmapDateTime(series, figsize=(6, 9)).show()
F:\Sync\luhk_work\20 - CODING\21 - DIIVE\diive\diive\core\plotting\heatmap_base.py:190: RuntimeWarning: More than 20 figures have been opened. Figures created through the pyplot interface (`matplotlib.pyplot.figure`) are retained until explicitly closed and may consume too much memory. (To control this warning, see the rcParam `figure.max_open_warning`). Consider using `matplotlib.pyplot.close()`.
fig = plt.figure(facecolor='white', figsize=self.figsize)
Stats#
data_simple_renamed_fw23.describe()
| G_GF1_0.03_1 | LW_IN_T1_2_1 | LW_OUT_T1_2_1 | PA_GF1_0.9_1 | FLAG_PA_GF1_0.9_1_ISFILLED | PPFD_IN_T1_2_2 | PREC_RAIN_TOT_GF1_0.5_1 | FLAG_PREC_RAIN_TOT_GF1_0.5_1_ISFILLED | RH_T1_2_1 | FLAG_RH_T1_2_1_ISFILLED | SWC_GF1_0.05_1 | SWC_GF1_0.15_1 | SWC_GF1_0.75_1 | SW_IN_T1_2_1 | FLAG_SW_IN_T1_2_1_ISFILLED | SW_OUT_T1_2_1 | TA_T1_2_1 | FLAG_TA_T1_2_1_ISFILLED | TS_GF1_0.04_1 | TS_GF1_0.15_1 | TS_GF1_0.4_1 | |
|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|
| count | 257233.000000 | 264553.000000 | 264726.000000 | 275885.000000 | 280512.000000 | 265182.000000 | 280512.000000 | 280512.000000 | 280512.000000 | 280439.000000 | 255743.000000 | 261403.000000 | 261394.000000 | 280512.000000 | 280512.000000 | 265645.000000 | 280512.000000 | 280512.000000 | 261097.000000 | 261812.000000 | 261914.000000 |
| mean | -1.294757 | 322.764134 | 363.862656 | 967.455135 | 0.697350 | 280.551417 | 0.064690 | 0.063266 | 83.921786 | 0.242998 | 50.799695 | 42.715561 | 43.008481 | 141.457347 | 0.188224 | 26.672217 | 9.865343 | 0.237284 | 11.635724 | 11.692454 | 11.546517 |
| std | 13.078049 | 43.931499 | 49.620005 | 8.000886 | 0.459406 | 467.401993 | 0.408662 | 0.243457 | 18.188887 | 0.428894 | 7.900669 | 4.602187 | 1.596751 | 235.038633 | 0.684141 | 49.826913 | 8.464063 | 0.809652 | 6.528476 | 6.248735 | 5.482197 |
| min | -47.055000 | 162.974472 | 219.459778 | 926.800000 | 0.000000 | -42.389820 | 0.000000 | 0.000000 | 9.561100 | 0.000000 | 24.511940 | 24.468729 | 37.764030 | -0.158850 | 0.000000 | -29.716660 | -18.246000 | 0.000000 | -2.576900 | 0.045730 | 1.378100 |
| 25% | -8.839500 | 295.478600 | 325.215393 | 962.900000 | 0.000000 | -0.104510 | 0.000000 | 0.000000 | 72.523962 | 0.000000 | 47.020676 | 40.653645 | 41.844761 | 0.000000 | 0.000000 | -3.722600 | 3.156075 | 0.000000 | 5.872834 | 5.830100 | 6.261362 |
| 50% | -4.027900 | 325.410523 | 357.583692 | 967.800000 | 1.000000 | 9.042426 | 0.000000 | 0.000000 | 91.100000 | 0.000000 | 53.084404 | 44.082874 | 42.524414 | 3.000000 | 0.000000 | 0.698370 | 9.589400 | 0.000000 | 11.853000 | 11.827865 | 11.589000 |
| 75% | 3.322900 | 354.076446 | 393.994850 | 972.533333 | 1.000000 | 365.505924 | 0.000000 | 0.000000 | 99.659190 | 0.000000 | 56.925919 | 45.981319 | 43.705841 | 185.192501 | 0.000000 | 39.070100 | 15.896067 | 0.000000 | 17.216000 | 17.303924 | 16.733999 |
| max | 144.000000 | 454.941700 | 588.454834 | 1004.485000 | 1.000000 | 2306.300049 | 36.480904 | 2.000000 | 100.000000 | 1.000000 | 62.706470 | 50.708080 | 48.056732 | 1140.900024 | 3.000000 | 396.149994 | 36.348891 | 3.000000 | 29.000000 | 25.957001 | 21.813999 |
List of variables (without flags)#
[print(ix, c) for ix, c in enumerate(data_simple_renamed_fw23.columns) if not str(c).startswith("FLAG_")];
0 G_GF1_0.03_1
1 LW_IN_T1_2_1
2 LW_OUT_T1_2_1
3 PA_GF1_0.9_1
5 PPFD_IN_T1_2_2
6 PREC_RAIN_TOT_GF1_0.5_1
8 RH_T1_2_1
10 SWC_GF1_0.05_1
11 SWC_GF1_0.15_1
12 SWC_GF1_0.75_1
13 SW_IN_T1_2_1
15 SW_OUT_T1_2_1
16 TA_T1_2_1
18 TS_GF1_0.04_1
19 TS_GF1_0.15_1
20 TS_GF1_0.4_1
End of notebook.#
dt_string = datetime.now().strftime("%Y-%m-%d %H:%M:%S")
print(f"Finished. {dt_string}")
Finished. 2024-09-02 12:09:20