Sub workflow for biometeorological data to process raw data and send it to ICOS Carbon Portal

subWorkflowBM_ICOS(
  confVariablePath,
  confDataCorrection,
  fileToProcess,
  skipRow,
  timestampColumn,
  timestampFormat,
  timestampTZ,
  extensionRawFile,
  stationLoggerName,
  code_logger,
  nom_frequence,
  code_table,
  ICOSPath,
  out_rootPath,
  sendDataICOS,
  userName = "FR-Lqu",
  password = ""
)

Arguments

confVariablePath

path and names of mappingFiles

confDataCorrection

path and names of confDataCorrection file

fileToProcess

path and name of data to process.

skipRow

Number of row to skip (e.g header and unit c(1,2))

timestampColumn

Name of timestamp column

timestampFormat

Timestamp format (eg. "%d/%m/%Y %H:%M")

timestampTZ

Timestamp timezone (eg. "Africa/Algiers")

extensionRawFile

extension file to process

stationLoggerName

station logger name folder (bm-cent_L03-atm)

code_logger

Code logger ICOS (e.g L03)

nom_frequence

Name of frequence of table without space (e.g 30s)

code_table

Code table of ICOS (e.g F01)

ICOSPath

station logger name folder (bm-cent_L03-atm)

out_rootPath

root folder

sendDataICOS

TRUE OR FALSE to send data via curl command (configuration in yml configuration file)

userName

String, user name used for carbon portal connection

password

String, password used for carbon portal connection

Value

Write ICOS file and send it to Carbon Portal with curl command (http put)

Examples

library(aigoual) skipRow <- c(1,2) timestampColumn <- "TIMESTAMP" timestampFormat <- "%Y-%m-%d %H:%M:%S" timestampTZ <- "Africa/Algiers" extensionRawFile <- ".dat" code_table <- "F01" nom_frequence <- "30s" code_logger <- "L03" out_rootPath <- tempdir() ICOSPath <- "" stationLoggerName <- "lqu_bm-cent_L03" confVariablePath <- system.file("extdata/_config/bmVariablesMapping.csv", package = "aigoual") confDataCorrection <- system.file("extdata/_config/bmDataCorrection.csv", package = "aigoual") fileToProcess <- system.file("extdata/bm-cent_L03-atm/0_rawdata/F01-30s/lqu_bm-cent_L03-atm_F01-30s_2021_02_22_0000.dat", package = "aigoual") subWorkflowBM_ICOS(confVariablePath=confVariablePath,confDataCorrection=confDataCorrection,fileToProcess=fileToProcess, skipRow=skipRow,timestampColumn=timestampColumn,timestampFormat=timestampFormat, timestampTZ=timestampTZ,extensionRawFile=extensionRawFile, stationLoggerName=stationLoggerName,code_logger=code_logger,nom_frequence=nom_frequence, code_table=code_table,ICOSPath=ICOSPath,out_rootPath=out_rootPath,sendDataICOS=FALSE)
#> Warning: Coercing 'character' RHS to 'double' to match the type of the target column (column 2 named 'WD_1_1_1').
#> Warning: Coercing 'character' RHS to 'double' to match the type of the target column (column 3 named 'WS_1_1_1').
#> Warning: Coercing 'character' RHS to 'logical' to match the type of the target column (column 4 named 'WD_2_1_1').
#> Warning: Coercing 'character' RHS to 'logical' to match the type of the target column (column 5 named 'WS_2_1_1').
#> Warning: Coercing 'character' RHS to 'double' to match the type of the target column (column 6 named 'WD_2_2_1').
#> Warning: Coercing 'character' RHS to 'double' to match the type of the target column (column 7 named 'WS_2_2_1').
#> Warning: Coercing 'character' RHS to 'double' to match the type of the target column (column 8 named 'PA_1_1_1').
#> Warning: Coercing 'character' RHS to 'double' to match the type of the target column (column 9 named 'TA_1_1_1').
#> Warning: Coercing 'character' RHS to 'double' to match the type of the target column (column 10 named 'TA_1_2_1').
#> Warning: Coercing 'character' RHS to 'double' to match the type of the target column (column 11 named 'RH_1_1_1').
#> Warning: Coercing 'character' RHS to 'double' to match the type of the target column (column 12 named 'RH_1_2_1').
#> Warning: Coercing 'character' RHS to 'logical' to match the type of the target column (column 13 named 'D_SNOW').
#> Warning: Coercing 'character' RHS to 'logical' to match the type of the target column (column 14 named 'P_1_1_1').
#> [1] "INFO [2021-09-27 13:19:49] --Loading : /tmp/RtmpCt1rpZ/temp_libpathebafc9f60/aigoual/extdata/bm-cent_L03-atm/0_rawdata/F01-30s/lqu_bm-cent_L03-atm_F01-30s_2021_02_22_0000.dat--" #> [2] "INFO [2021-09-27 13:19:49] --Split data by day--" #> [3] "INFO [2021-09-27 13:19:49] File contain : 2021-02-22" #> [4] "INFO [2021-09-27 13:19:49] File contain : 2021-02-23" #> [5] "INFO [2021-09-27 13:19:49] --Mapping ICOS variable--" #> [6] "INFO [2021-09-27 13:19:49] ICOS variables : TIMESTAMP" #> [7] "INFO [2021-09-27 13:19:49] ICOS variables : WD_1_1_1" #> [8] "INFO [2021-09-27 13:19:49] ICOS variables : WS_1_1_1" #> [9] "INFO [2021-09-27 13:19:49] ICOS variables : WD_2_1_1" #> [10] "INFO [2021-09-27 13:19:49] ICOS variables : WS_2_1_1" #> [11] "INFO [2021-09-27 13:19:49] ICOS variables : WD_2_2_1" #> [12] "INFO [2021-09-27 13:19:49] ICOS variables : WS_2_2_1" #> [13] "INFO [2021-09-27 13:19:49] ICOS variables : PA_1_1_1" #> [14] "INFO [2021-09-27 13:19:49] ICOS variables : TA_1_1_1" #> [15] "INFO [2021-09-27 13:19:49] ICOS variables : TA_1_2_1" #> [16] "INFO [2021-09-27 13:19:49] ICOS variables : RH_1_1_1" #> [17] "INFO [2021-09-27 13:19:49] ICOS variables : RH_1_2_1" #> [18] "INFO [2021-09-27 13:19:49] ICOS variables : D_SNOW" #> [19] "INFO [2021-09-27 13:19:49] ICOS variables : P_1_1_1" #> [20] "INFO [2021-09-27 13:19:49] --Make a complete timeseries--" #> [21] "INFO [2021-09-27 13:19:49] nom_frequence defined in file name : 30s" #> [22] "INFO [2021-09-27 13:19:49] nom_frequence detected : 30s" #> [23] "INFO [2021-09-27 13:19:49] Begin day:2021-02-22 00:00:30" #> [24] "INFO [2021-09-27 13:19:49] End day:2021-02-23" #> [25] "INFO [2021-09-27 13:19:49] --Data correction--" #> [26] "INFO [2021-09-27 13:19:49] --Quality control--" #> [27] "INFO [2021-09-27 13:19:49] --ICOS Format for TIMESTAMP AND NaN" #> [28] "INFO [2021-09-27 13:19:49] First date line : 20210222000030" #> [29] "INFO [2021-09-27 13:19:49] --Write ICOS File--" #> [30] "INFO [2021-09-27 13:19:49] ICOSFileNamePath : /tmp/RtmpssvyzX//FR-Lqu_BM_20210222_L03_F01.csv" #> [31] "SUCCESS [2021-09-27 13:19:49] End of processing for /tmp/RtmpCt1rpZ/temp_libpathebafc9f60/aigoual/extdata/bm-cent_L03-atm/0_rawdata/F01-30s/lqu_bm-cent_L03-atm_F01-30s_2021_02_22_0000.dat"
library(aigoual) skipRow <- c(1,2) timestampColumn <- "TIMESTAMP" timestampFormat <- "%Y-%m-%d %H:%M:%S" timestampTZ <- "Africa/Algiers" extensionRawFile <- ".dat" code_table <- "F01" nom_frequence <- "60s" code_logger <- "L11" out_rootPath <- tempdir() ICOSPath <- "" stationLoggerName <- "lqu_bm-cent_L11" confVariablePath <- system.file("extdata/_config/bmVariablesMapping.csv", package = "aigoual") confDataCorrection <- system.file("extdata/_config/bmDataCorrection.csv", package = "aigoual") fileToProcess <- system.file("extdata/ec-int_L11-sol/0_rawdata/F01-60s/lqu_ec-int_L11-sol_F01-60s_2021_05_30_0000.dat", package = "aigoual") subWorkflowBM_ICOS(confVariablePath=confVariablePath,confDataCorrection=confDataCorrection,fileToProcess=fileToProcess, skipRow=skipRow,timestampColumn=timestampColumn,timestampFormat=timestampFormat, timestampTZ=timestampTZ,extensionRawFile=extensionRawFile, stationLoggerName=stationLoggerName,code_logger=code_logger,nom_frequence=nom_frequence, code_table=code_table,ICOSPath=ICOSPath,out_rootPath=out_rootPath,sendDataICOS=FALSE)
#> Warning: Coercing 'character' RHS to 'double' to match the type of the target column (column 2 named 'G_3_1_1').
#> Warning: Coercing 'character' RHS to 'double' to match the type of the target column (column 3 named 'G_SF_3_1_1').
#> Warning: Coercing 'character' RHS to 'double' to match the type of the target column (column 4 named 'G_ISCAL_3_1_1').
#> Warning: Coercing 'character' RHS to 'double' to match the type of the target column (column 5 named 'G_4_1_1').
#> Warning: Coercing 'character' RHS to 'double' to match the type of the target column (column 6 named 'G_SF_4_1_1').
#> Warning: Coercing 'character' RHS to 'double' to match the type of the target column (column 7 named 'G_ISCAL_4_1_1').
#> Warning: Coercing 'character' RHS to 'double' to match the type of the target column (column 8 named 'SWC_3_1_1').
#> Warning: Coercing 'character' RHS to 'double' to match the type of the target column (column 9 named 'TS_3_1_1').
#> Warning: Coercing 'character' RHS to 'double' to match the type of the target column (column 10 named 'SWC_3_2_1').
#> Warning: Coercing 'character' RHS to 'double' to match the type of the target column (column 11 named 'TS_3_2_1').
#> Warning: Coercing 'character' RHS to 'double' to match the type of the target column (column 12 named 'SWC_3_3_1').
#> Warning: Coercing 'character' RHS to 'double' to match the type of the target column (column 13 named 'TS_3_3_1').
#> Warning: Coercing 'character' RHS to 'double' to match the type of the target column (column 14 named 'SWC_3_4_1').
#> Warning: Coercing 'character' RHS to 'double' to match the type of the target column (column 15 named 'TS_3_4_1').
#> Warning: Coercing 'character' RHS to 'double' to match the type of the target column (column 16 named 'SWC_3_5_1').
#> Warning: Coercing 'character' RHS to 'double' to match the type of the target column (column 17 named 'TS_3_5_1').
#> Warning: Coercing 'character' RHS to 'double' to match the type of the target column (column 18 named 'SWC_3_6_1').
#> Warning: Coercing 'character' RHS to 'double' to match the type of the target column (column 19 named 'TS_3_6_1').
#> Warning: Coercing 'character' RHS to 'double' to match the type of the target column (column 20 named 'SWC_4_1_1').
#> Warning: Coercing 'character' RHS to 'double' to match the type of the target column (column 21 named 'TS_4_1_1').
#> Warning: Coercing 'character' RHS to 'double' to match the type of the target column (column 22 named 'SWC_4_2_1').
#> Warning: Coercing 'character' RHS to 'double' to match the type of the target column (column 23 named 'TS_4_2_1').
#> Warning: Coercing 'character' RHS to 'double' to match the type of the target column (column 24 named 'SWC_4_3_1').
#> Warning: Coercing 'character' RHS to 'double' to match the type of the target column (column 25 named 'TS_4_3_1').
#> Warning: Coercing 'character' RHS to 'double' to match the type of the target column (column 26 named 'SWC_4_4_1').
#> Warning: Coercing 'character' RHS to 'double' to match the type of the target column (column 27 named 'TS_4_4_1').
#> Warning: Coercing 'character' RHS to 'double' to match the type of the target column (column 28 named 'SWC_4_5_1').
#> Warning: Coercing 'character' RHS to 'double' to match the type of the target column (column 29 named 'TS_4_5_1').
#> Warning: Coercing 'character' RHS to 'double' to match the type of the target column (column 30 named 'SWC_4_6_2').
#> Warning: Coercing 'character' RHS to 'double' to match the type of the target column (column 31 named 'TS_4_6_2').
#> [1] "INFO [2021-09-27 13:19:49] --Loading : /tmp/RtmpCt1rpZ/temp_libpathebafc9f60/aigoual/extdata/ec-int_L11-sol/0_rawdata/F01-60s/lqu_ec-int_L11-sol_F01-60s_2021_05_30_0000.dat--" #> [2] "INFO [2021-09-27 13:19:49] --Split data by day--" #> [3] "INFO [2021-09-27 13:19:49] File contain : 2021-05-30" #> [4] "INFO [2021-09-27 13:19:49] --Mapping ICOS variable--" #> [5] "INFO [2021-09-27 13:19:49] ICOS variables : TIMESTAMP" #> [6] "INFO [2021-09-27 13:19:49] ICOS variables : G_3_1_1" #> [7] "INFO [2021-09-27 13:19:49] ICOS variables : G_SF_3_1_1" #> [8] "INFO [2021-09-27 13:19:49] ICOS variables : G_ISCAL_3_1_1" #> [9] "INFO [2021-09-27 13:19:49] ICOS variables : G_4_1_1" #> [10] "INFO [2021-09-27 13:19:49] ICOS variables : G_SF_4_1_1" #> [11] "INFO [2021-09-27 13:19:49] ICOS variables : G_ISCAL_4_1_1" #> [12] "INFO [2021-09-27 13:19:49] ICOS variables : SWC_3_1_1" #> [13] "INFO [2021-09-27 13:19:49] ICOS variables : TS_3_1_1" #> [14] "INFO [2021-09-27 13:19:49] ICOS variables : SWC_3_2_1" #> [15] "INFO [2021-09-27 13:19:49] ICOS variables : TS_3_2_1" #> [16] "INFO [2021-09-27 13:19:49] ICOS variables : SWC_3_3_1" #> [17] "INFO [2021-09-27 13:19:49] ICOS variables : TS_3_3_1" #> [18] "INFO [2021-09-27 13:19:49] ICOS variables : SWC_3_4_1" #> [19] "INFO [2021-09-27 13:19:49] ICOS variables : TS_3_4_1" #> [20] "INFO [2021-09-27 13:19:49] ICOS variables : SWC_3_5_1" #> [21] "INFO [2021-09-27 13:19:49] ICOS variables : TS_3_5_1" #> [22] "INFO [2021-09-27 13:19:49] ICOS variables : SWC_3_6_1" #> [23] "INFO [2021-09-27 13:19:49] ICOS variables : TS_3_6_1" #> [24] "INFO [2021-09-27 13:19:49] ICOS variables : SWC_4_1_1" #> [25] "INFO [2021-09-27 13:19:49] ICOS variables : TS_4_1_1" #> [26] "INFO [2021-09-27 13:19:49] ICOS variables : SWC_4_2_1" #> [27] "INFO [2021-09-27 13:19:49] ICOS variables : TS_4_2_1" #> [28] "INFO [2021-09-27 13:19:49] ICOS variables : SWC_4_3_1" #> [29] "INFO [2021-09-27 13:19:49] ICOS variables : TS_4_3_1" #> [30] "INFO [2021-09-27 13:19:49] ICOS variables : SWC_4_4_1" #> [31] "INFO [2021-09-27 13:19:49] ICOS variables : TS_4_4_1" #> [32] "INFO [2021-09-27 13:19:49] ICOS variables : SWC_4_5_1" #> [33] "INFO [2021-09-27 13:19:49] ICOS variables : TS_4_5_1" #> [34] "INFO [2021-09-27 13:19:49] ICOS variables : SWC_4_6_2" #> [35] "INFO [2021-09-27 13:19:49] ICOS variables : TS_4_6_2" #> [36] "INFO [2021-09-27 13:19:49] --Make a complete timeseries--" #> [37] "INFO [2021-09-27 13:19:49] nom_frequence defined in file name : 60s" #> [38] "INFO [2021-09-27 13:19:49] nom_frequence detected : 1min" #> [39] "INFO [2021-09-27 13:19:49] Begin day:2021-05-30 00:01:00" #> [40] "INFO [2021-09-27 13:19:49] End day:2021-05-31" #> [41] "INFO [2021-09-27 13:19:49] --Data correction--" #> [42] "INFO [2021-09-27 13:19:49] --Quality control--" #> [43] "INFO [2021-09-27 13:19:49] --ICOS Format for TIMESTAMP AND NaN" #> [44] "INFO [2021-09-27 13:19:49] First date line : 20210530000100" #> [45] "INFO [2021-09-27 13:19:49] --Write ICOS File--" #> [46] "INFO [2021-09-27 13:19:49] ICOSFileNamePath : /tmp/RtmpssvyzX//FR-Lqu_BM_20210530_L11_F01.csv" #> [47] "SUCCESS [2021-09-27 13:19:50] End of processing for /tmp/RtmpCt1rpZ/temp_libpathebafc9f60/aigoual/extdata/ec-int_L11-sol/0_rawdata/F01-60s/lqu_ec-int_L11-sol_F01-60s_2021_05_30_0000.dat"