gao/analyseScript
1
0
Fork 0
Code Issues Pull Requests repo.project_board Releases Wiki Activity

regular backup

Browse Source
This commit is contained in:
Jianshun Gao 2023-06-13 18:16:24 +02:00
parent 7351593cbb
commit 09fdb09fd3
3 changed files with 5676 additions and 92114 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

131
DataContainer/MongoDB.py
View File

@ -37,6 +37,17 @@ class MongoDB:
pass
return data
def _break_dataSet(self, dataSet, scanAxis=None):
if scanAxis is None:
scanAxis = dataSet.attrs['scanAxis']
dataArray = dataSet.shotNum
stackedDataArray = dataArray.stack(_scanAxis=tuple(scanAxis))
return stackedDataArray
def set_date(self, date):
date = date.split("/")
self.year = int(date[0])
@ -52,6 +63,7 @@ class MongoDB:
'month': self.month,
'day': self.day,
'shotNum': shotNum,
'runNum': 0,
'global_parameters' : {},
}
@ -62,19 +74,72 @@ class MongoDB:
data = self._convert_numpy_type(data)
if 'scanAxis' in dataSet.attrs:
del data['global_parameters']['scanAxis']
del data['global_parameters']['scanAxisLength']
scanAxis = dataSet.attrs['scanAxis']
data['global_parameters'].update(
{
key:0
for key in scanAxis
}
)
stackedDataArray = self._break_dataSet(dataSet)
stackedDataArray = stackedDataArray.groupby('_scanAxis')
for i in stackedDataArray:
stackedDataArray_single = i[1]
data.update(
{
'runNum': int(stackedDataArray_single.item())
}
)
data['global_parameters'].update(
{
key: stackedDataArray_single[key].item()
for key in scanAxis
}
)
if '_id' in data:
del data['_id']
self.mongoDB['global'].insert_one(data)
def _add_data_normal(self, shotNum, data):
else:
self.mongoDB['global'].insert_one(data)
def _add_data_normal(self, shotNum, runNum, data):
if runNum is None:
runNum = 0
filter = {
'year': self.year,
'month': self.month,
'day': self.day,
'shotNum': shotNum,
'runNum': runNum,
}
self.mongoDB['global'].update_one(filter, {"$set": data}, upsert=False)
def _add_data_xarray_dataArray(self, shotNum, dataArray):
def _add_data_xarray_dataArray(self, shotNum, dataArray, scanAxis=None):
if scanAxis is None:
scanAxis = list(dataArray.coords)
dataArray.attrs = self._convert_numpy_type(dataArray.attrs)
stackedDataArray = dataArray.stack(_scanAxis=tuple(scanAxis))
stackedDataArray = stackedDataArray.groupby('_scanAxis')
filter = {
'year': self.year,
'month': self.month,
@ -82,9 +147,18 @@ class MongoDB:
'shotNum': shotNum,
}
dataArray.attrs = self._convert_numpy_type(dataArray.attrs)
for i in stackedDataArray:
mongoID, _ = self.xdb.put(dataArray)
stackedDataArray_single = i[1].drop('_scanAxis')
global_parameters = {
'global_parameters.' + key: stackedDataArray_single[key].item()
for key in scanAxis
}
filter.update(global_parameters)
mongoID, _ = self.xdb.put(stackedDataArray_single)
data_label = {
dataArray.name:
@ -98,7 +172,19 @@ class MongoDB:
self.mongoDB['global'].update_one(filter, {"$set": data_label}, upsert=False)
def _add_data_xarray_dataSet(self, shotNum, dataSet, name):
def _add_data_xarray_dataSet(self, shotNum, dataSet, name, scanAxis=None):
if scanAxis is None:
scanAxis = list(dataSet.coords)
dataSet.attrs = self._convert_numpy_type(dataSet.attrs)
for key in list(dataSet.data_vars):
dataSet[key].attrs = self._convert_numpy_type(dataSet[key].attrs)
stackedDataSet = dataSet.stack(_scanAxis=tuple(scanAxis))
stackedDataSet = stackedDataSet.groupby('_scanAxis')
filter = {
'year': self.year,
'month': self.month,
@ -106,11 +192,16 @@ class MongoDB:
'shotNum': shotNum,
}
dataSet.attrs = self._convert_numpy_type(dataSet.attrs)
for i in stackedDataSet:
for key in list(dataSet.data_vars):
dataSet[key].attrs = self._convert_numpy_type(dataSet[key].attrs)
print(key)
stackedDataSet_single = i[1].drop('_scanAxis')
global_parameters = {
'global_parameters.' + key: stackedDataSet_single[key].item()
for key in scanAxis
}
filter.update(global_parameters)
mongoID, _ = self.xdb.put(dataSet)
@ -126,12 +217,17 @@ class MongoDB:
self.mongoDB['global'].update_one(filter, {"$set": data_label}, upsert=False)
def _add_data_additional(self, shotNum, data, name):
def _add_data_additional(self, shotNum, runNum, data, name):
if runNum is None:
runNum = 0
filter = {
'year': self.year,
'month': self.month,
'day': self.day,
'shotNum': shotNum,
'runNum': runNum,
}
mongoID = self.mongoDB.additional.insert_one(data).inserted_id
@ -148,21 +244,21 @@ class MongoDB:
self.mongoDB['global'].update_one(filter, {"$set": data_label}, upsert=False)
def add_data(self, shotNum, data, date=None, name=None, engine='normal'):
def add_data(self, shotNum, data, runNum=None, date=None, name=None, engine='normal'):
if not date is None:
self.set_date(date)
if engine == 'normal':
self._add_data_normal(shotNum=shotNum, data=data)
self._add_data_normal(shotNum=shotNum, runNum=runNum, data=data)
elif engine == 'xarray':
if isinstance(data, type(xr.Dataset())):
self._add_data_xarray_dataSet(shotNum=shotNum, data=data, name=name)
else:
self._add_data_xarray_dataArray(shotNum=shotNum, data=data, name=name)
elif engine == 'additional':
self._add_data_additional(shotNum=shotNum, data=data, name=name)
self._add_data_additional(shotNum=shotNum, runNum=runNum, data=data, name=name)
def read_global(self, shotNum, date=None):
def read_global_single(self, shotNum, runNum, date=None):
if not date is None:
self.set_date(date)
@ -172,6 +268,7 @@ class MongoDB:
'month': self.month,
'day': self.day,
'shotNum': shotNum,
'runNum': runNum,
}
return self.mongoDB['global'].find_one(filter)
@ -182,18 +279,18 @@ class MongoDB:
if engine == 'additional':
return self.mongoDB.additional.find_one({'_id': mongoID})
def load_data_single(self, shotNum=None, globalDict=None, date=None, field=None):
def load_data_single(self, shotNum=None, runNum=None, globalDict=None, date=None, field=None):
if not date is None:
self.set_date(date)
if globalDict is None:
globalDict = self.read_global(shotNum)
globalDict = self.read_global_single(shotNum=shotNum, runNum=runNum)
if field is None:
field = globalDict
res = {}
res = field
for key in field:
if isinstance(globalDict[key], dict) and ('mongoID' in globalDict[key]):

2
DataContainer/ReadData.py
View File

@ -326,8 +326,6 @@ def read_hdf5_global(filePath, preprocess=None, join="outer", combine="nested",
datesetOfGlobal.attrs['scanAxis'] = np.setdiff1d(datesetOfGlobal.attrs['scanAxis'], excludeAxis)
datesetOfGlobal.
return datesetOfGlobal

96511
testMongoDB.ipynb
View File

File diff suppressed because one or more lines are too long