@ -11,7 +11,17 @@ from datetime import datetime
def _read_globals_attrs ( variable_attrs , context = None ) :
""" Combine attributes from different variables according to combine_attrs """
""" Find global parameters of shots, including scan axes.
: param variable_attrs : The attrs of current shot .
: type variable_attrs : dict
: param context : _description_ , defaults to None
: type context : _type_ , optional
: return : The globals attrs of the whole shot .
: rtype : dict
"""
# Combine attributes from different variables according to combine_attrs
if not variable_attrs :
# no attributes to merge
return None
@ -86,12 +96,31 @@ def _read_globals_attrs(variable_attrs, context=None):
def _read_shot_number_from_hdf5 ( x ) :
""" Add the current shot number to the data read from HDF5 file.
: param x : The data of current shot
: type x : xarray DataArray
: return : The data with current shot number
: rtype : xarray DataArray
"""
filePath = x . encoding [ " source " ]
shotNum = filePath . split ( " _ " ) [ - 1 ] . split ( " _ " ) [ - 1 ] . split ( " . " ) [ 0 ]
return x . assign ( shotNum = shotNum )
def _assign_scan_axis_partial ( x , datesetOfGlobal , fullFilePath ) :
""" Find and add the scan axes to the data read from HDF5 file.
: param x : The data of current shot
: type x : xarray DataArray
: param datesetOfGlobal : The xarray DataSet stored the information of global parameters
: type datesetOfGlobal : xarray DataSet
: param fullFilePath : The full and absolute file path of current shot
: type fullFilePath : str
: return : The data of current shot with scan axes
: rtype : xarray DataArray
"""
scanAxis = datesetOfGlobal . scanAxis
filePath = x . encoding [ " source " ] . replace ( " \\ " , " / " )
shotNum = np . where ( fullFilePath == filePath )
@ -109,14 +138,41 @@ def _assign_scan_axis_partial(x, datesetOfGlobal, fullFilePath):
def _update_globals_attrs ( variable_attrs , context = None ) :
# for live plot panel
pass
def update_hdf5_file ( ) :
# for live plot panel
pass
def read_hdf5_file ( filePath , group = None , datesetOfGlobal = None , preprocess = None , join = " outer " , parallel = True , engine = " h5netcdf " , phony_dims = " access " , excludeAxis = [ ] , maxFileNum = None , * * kwargs ) :
""" Read the data from HDF5 files in given path.
: param filePath : The path of HDF5 files , which python glob . glob ( ) can read . It has to end with ' .h5 ' .
: type filePath : str
: param group : The path of the group in HDF5 file where data is , defaults to None . Please use ' / ' , instead of ' \\ '
: type group : str , optional
: param datesetOfGlobal : A xarry dataSet stored the global parameters of the data , defaults to None
: type datesetOfGlobal : xarry DataSet , optional
: param preprocess : The function you want to run for each file after read before combination , defaults to None
: type preprocess : a handle to function , optional
: param join : over write of the same argument in xarray . open_mfdataset , defaults to " outer "
: type join : str , optional
: param parallel : over write of the same argument in xarray . open_mfdataset , defaults to True
: type parallel : bool , optional
: param engine : The engine to read HDF5 file , defaults to " h5netcdf "
: type engine : str , optional
: param phony_dims : Please read the introduction of h5netcdf package , defaults to " access "
: type phony_dims : str , optional
: param excludeAxis : The name of axes , whose value changes together with scan axes , defaults to [ ]
: type excludeAxis : list , optional
: param maxFileNum : The maximal number of files to read , defaults to None
: type maxFileNum : int , optional
: return : A xarray dataSet contain the data read from specified HDF5 file , including scan axes and shot number .
: rtype : xarray DataSet
"""
filePath = np . sort ( np . atleast_1d ( filePath ) )
@ -192,6 +248,18 @@ def read_hdf5_file(filePath, group=None, datesetOfGlobal=None, preprocess=None,
def _assign_scan_axis_partial_and_remove_everything ( x , datesetOfGlobal , fullFilePath ) :
""" Find ONLY and add ONLY the scan axes to the data read from HDF5 file.
: param x : The data of current shot
: type x : xarray DataArray
: param datesetOfGlobal : The xarray DataSet stored the information of global parameters
: type datesetOfGlobal : xarray DataSet
: param fullFilePath : The full and absolute file path of current shot
: type fullFilePath : str
: return : The data of current shot with scan axes
: rtype : xarray DataArray
"""
scanAxis = datesetOfGlobal . scanAxis
filePath = x . encoding [ " source " ] . replace ( " \\ " , " / " )
shotNum = np . where ( fullFilePath == filePath )
@ -209,11 +277,43 @@ def _assign_scan_axis_partial_and_remove_everything(x, datesetOfGlobal, fullFile
def _read_run_time_from_hdf5 ( x ) :
""" Find the run time of give data read from HDF5 file.
: param x : The data of current shot
: type x : xarray DataArray
: return : The data of current shot with last modification time
: rtype : xarray DataArray
"""
runTime = datetime . strptime ( x . attrs [ ' run time ' ] , ' % Y % m %d T % H % M % S ' )
return runTime
def read_hdf5_run_time ( filePath , group = None , datesetOfGlobal = None , preprocess = None , join = " outer " , parallel = True , engine = " h5netcdf " , phony_dims = " access " , excludeAxis = [ ] , maxFileNum = None , * * kwargs ) :
""" Read the run time from HDF5 files in given path.
: param filePath : The path of HDF5 files , which python glob . glob ( ) can read . It has to end with ' .h5 ' .
: type filePath : str
: param group : The path of the group in HDF5 file where run time is , defaults to None . Please use ' / ' , instead of ' \\ '
: type group : str , optional
: param datesetOfGlobal : A xarry dataSet stored the global parameters of the data , defaults to None
: type datesetOfGlobal : xarry DataSet , optional
: param preprocess : The function you want to run for each file after read before combination , defaults to None
: type preprocess : a handle to function , optional
: param join : over write of the same argument in xarray . open_mfdataset , defaults to " outer "
: type join : str , optional
: param parallel : over write of the same argument in xarray . open_mfdataset , defaults to True
: type parallel : bool , optional
: param engine : The engine to read HDF5 file , defaults to " h5netcdf "
: type engine : str , optional
: param phony_dims : Please read the introduction of h5netcdf package , defaults to " access "
: type phony_dims : str , optional
: param excludeAxis : The name of axes , whose value changes together with scan axes , defaults to [ ]
: type excludeAxis : list , optional
: param maxFileNum : The maximal number of files to read , defaults to None
: type maxFileNum : int , optional
: return : A xarray dataSet contain the data read from specified HDF5 file.
: rtype : xarray DataSet
"""
filePath = np . sort ( np . atleast_1d ( filePath ) )
@ -289,6 +389,29 @@ def read_hdf5_run_time(filePath, group=None, datesetOfGlobal=None, preprocess=No
def read_hdf5_global ( filePath , preprocess = None , join = " outer " , combine = " nested " , parallel = True , engine = " h5netcdf " , phony_dims = " access " , excludeAxis = [ ] , maxFileNum = None , * * kwargs ) :
""" Read the global parameters and find scan axes, from HDF5 files in given path.
: param filePath : The path of HDF5 files , which python glob . glob ( ) can read . It has to end with ' .h5 ' .
: type filePath : str
: param preprocess : The function you want to run for each file after read before combination , defaults to None
: type preprocess : a handle to function , optional
: param join : over write of the same argument in xarray . open_mfdataset , defaults to " outer "
: type join : str , optional
: param combine : over write of the same argument in xarray . open_mfdataset , defaults to " nested "
: type combine : str , optional
: param parallel : over write of the same argument in xarray . open_mfdataset , defaults to True
: type parallel : bool , optional
: param engine : The engine to read HDF5 file , defaults to " h5netcdf "
: type engine : str , optional
: param phony_dims : Please read the introduction of h5netcdf package , defaults to " access "
: type phony_dims : str , optional
: param excludeAxis : The name of axes , whose value changes together with scan axes , defaults to [ ]
: type excludeAxis : list , optional
: param maxFileNum : The maximal number of files to read , defaults to None
: type maxFileNum : int , optional
: return : A xarray dataSet contain the data read from specified HDF5 file.
: rtype : xarray DataSet
"""
filePath = np . sort ( np . atleast_1d ( filePath ) )
@ -330,6 +453,13 @@ def read_hdf5_global(filePath, preprocess=None, join="outer", combine="nested",
def _read_csv_file_pandas ( filePath , * * kwargs ) :
""" Read csv file using pandas package function read_csv()
: param filePath : The path of csv files .
: type filePath : str
: return : A xarray DataSet stored the data
: rtype : xarray DataSet
"""
res = pd . read_csv ( filePath , * * kwargs )
@ -338,6 +468,13 @@ def _read_csv_file_pandas(filePath, **kwargs):
def _read_csv_file_dask ( filePath , * * kwargs ) :
""" Read csv file using dask package function read_csv()
: param filePath : The path of csv files .
: type filePath : str
: return : A xarray DataSet stored the data
: rtype : xarray DataSet
"""
res = df . read_csv ( filePath , * * kwargs )
@ -346,6 +483,26 @@ def _read_csv_file_dask(filePath, **kwargs):
def read_csv_file ( filePath , maxFileNum = None , dask = ' parallelized ' , vectorize = True , csvEngine = ' pandas ' , daskKwargs = { } , csvKwargs = { } , * * kwargs ) :
""" Read the data from csv files in given path.
: param filePath : The path of csv files , which python glob . glob ( ) can read . It has to end with ' .csv ' .
: type filePath : str
: param maxFileNum : The maximal number of files to read , defaults to None
: type maxFileNum : int , optional
: param dask : over write of the same argument in xarray . apply_ufunc , defaults to ' parallelized '
: type dask : str , optional
: param vectorize : over write of the same argument in xarray . apply_ufunc , defaults to True
: type vectorize : bool , optional
: param csvEngine : The engine to read csv file , defaults to ' pandas '
: type csvEngine : str , optional
: param daskKwargs : over write of the same argument in xarray . apply_ufunc , defaults to { }
: type daskKwargs : dict , optional
: param csvKwargs : The kwargs send to csvEngine , defaults to { }
: type csvKwargs : dict , optional
: return : A xarray DataSet stored the data
: rtype : xarray DataSet
"""
filePath = np . sort ( np . atleast_1d ( filePath ) )
filePathAbs = [ ]