diff --git a/Analyser/ImagingAnalyser.py b/Analyser/ImagingAnalyser.py index 7e3e815..fb09570 100644 --- a/Analyser/ImagingAnalyser.py +++ b/Analyser/ImagingAnalyser.py @@ -72,7 +72,7 @@ class ImageAnalyser(): def substract_offset(self, dataArray, **kwargs): return dataArray - self.get_offset_from_corner(dataArray, **kwargs) - def crop_image(self, dataset, center=None, span=None): + def crop_image(self, dataSet, center=None, span=None): if center is None: center = self._center @@ -84,7 +84,7 @@ class ImageAnalyser(): y_end = int(center[1] + span[1] / 2) y_start = int(center[1] - span[1] / 2) - return dataset.isel(x=slice(x_start, x_end), y=slice(y_start, y_end)) + return dataSet.isel(x=slice(x_start, x_end), y=slice(y_start, y_end)) def get_OD(self, imageAtom, imageBackground, imageDrak): @@ -104,7 +104,7 @@ class ImageAnalyser(): def get_Ncount(self, imageOD): return np.sum(imageOD) - def get_absorption_images(self, dataset, dask='allowed', keep_attrs=True, **kwargs): + def get_absorption_images(self, dataSet, dask='allowed', keep_attrs=True, **kwargs): kwargs.update( { @@ -113,15 +113,15 @@ class ImageAnalyser(): } ) - dataset = dataset.assign( + dataSet = dataSet.assign( { - self._image_name['OD']: xr.apply_ufunc(self.get_OD, dataset[self._image_name['atoms']], dataset[self._image_name['background']], dataset[self._image_name['dark']], **kwargs) + self._image_name['OD']: xr.apply_ufunc(self.get_OD, dataSet[self._image_name['atoms']], dataSet[self._image_name['background']], dataSet[self._image_name['dark']], **kwargs) } ) - return dataset + return dataSet - def remove_background(self, dataset, dask='allowed', keep_attrs=True, **kwargs): + def remove_background(self, dataSet, dask='allowed', keep_attrs=True, **kwargs): kwargs.update( { @@ -130,7 +130,7 @@ class ImageAnalyser(): } ) - xr.apply_ufunc(self.get_OD, dataset[self._image_name['atoms']], dataset[self._image_name['background']], dataset[self._image_name['dark']], **kwargs) + xr.apply_ufunc(self.get_OD, dataSet[self._image_name['atoms']], dataSet[self._image_name['background']], dataSet[self._image_name['dark']], **kwargs) \ No newline at end of file diff --git a/ToolFunction/ToolFunction.py b/ToolFunction/ToolFunction.py index 64c279a..f4ef047 100644 --- a/ToolFunction/ToolFunction.py +++ b/ToolFunction/ToolFunction.py @@ -1,18 +1,32 @@ -def get_mask(): - pass +import numpy as np +import glob +from datetime import date -def remove_bad_shots(): - pass +def get_mask(dataArray): + return np.ones(dataArray.shape, dtype=bool) -def auto_rechunk(): - pass +def remove_bad_shots(dataArray, **kwargs): + dataArray.loc[dict(kwargs)] = np.nan -def get_h5_file_path(folderpath, maxFileNum): - pass +def auto_rechunk(dataSet): + kwargs = { + key: "auto" + for key in dataSet.dims.keys() + } + return dataSet.chunk(**kwargs) -def get_folder_path(): - pass \ No newline at end of file +def get_h5_file_path(folderpath, maxFileNum=None, filename='*.h5',): + filepath = np.sort(glob.glob(folderpath + filename)) + if maxFileNum is None: + return filepath + else: + return filepath[:maxFileNum] + + +def get_date(): + today = date.today() + return today.strftime("%y/%m/%d") \ No newline at end of file diff --git a/test.ipynb b/test.ipynb index 432ed41..ca14bdd 100644 --- a/test.ipynb +++ b/test.ipynb @@ -10,7 +10,7 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 1, "metadata": {}, "outputs": [], "source": [ @@ -36,9 +36,412 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 2, "metadata": {}, - "outputs": [], + "outputs": [ + { + "data": { + "text/html": [ + "
\n", + "
\n", + "
\n", + "

Client

\n", + "

Client-1a44c769-ea9b-11ed-88cc-80e82ce2fa8e

\n", + " \n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "\n", + "
Connection method: Cluster objectCluster type: distributed.LocalCluster
\n", + " Dashboard: http://127.0.0.1:8787/status\n", + "
\n", + "\n", + " \n", + "\n", + " \n", + "
\n", + "

Cluster Info

\n", + "
\n", + "
\n", + "
\n", + "
\n", + "

LocalCluster

\n", + "

b16f0063

\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "\n", + "\n", + " \n", + "
\n", + " Dashboard: http://127.0.0.1:8787/status\n", + " \n", + " Workers: 6\n", + "
\n", + " Total threads: 60\n", + " \n", + " Total memory: 55.88 GiB\n", + "
Status: runningUsing processes: True
\n", + "\n", + "
\n", + " \n", + "

Scheduler Info

\n", + " \n", + "\n", + "
\n", + "
\n", + "
\n", + "
\n", + "

Scheduler

\n", + "

Scheduler-8571d7e2-e6e5-4263-a9ed-50018e90379b

\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
\n", + " Comm: tcp://127.0.0.1:55356\n", + " \n", + " Workers: 6\n", + "
\n", + " Dashboard: http://127.0.0.1:8787/status\n", + " \n", + " Total threads: 60\n", + "
\n", + " Started: Just now\n", + " \n", + " Total memory: 55.88 GiB\n", + "
\n", + "
\n", + "
\n", + "\n", + "
\n", + " \n", + "

Workers

\n", + " \n", + "\n", + " \n", + "
\n", + "
\n", + "
\n", + "
\n", + " \n", + "

Worker: 0

\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "\n", + " \n", + "\n", + " \n", + "\n", + "
\n", + " Comm: tcp://127.0.0.1:55387\n", + " \n", + " Total threads: 10\n", + "
\n", + " Dashboard: http://127.0.0.1:55388/status\n", + " \n", + " Memory: 9.31 GiB\n", + "
\n", + " Nanny: tcp://127.0.0.1:55359\n", + "
\n", + " Local directory: C:\\Users\\data\\AppData\\Local\\Temp\\dask-worker-space\\worker-646bafev\n", + "
\n", + "
\n", + "
\n", + "
\n", + " \n", + "
\n", + "
\n", + "
\n", + "
\n", + " \n", + "

Worker: 1

\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "\n", + " \n", + "\n", + " \n", + "\n", + "
\n", + " Comm: tcp://127.0.0.1:55374\n", + " \n", + " Total threads: 10\n", + "
\n", + " Dashboard: http://127.0.0.1:55385/status\n", + " \n", + " Memory: 9.31 GiB\n", + "
\n", + " Nanny: tcp://127.0.0.1:55360\n", + "
\n", + " Local directory: C:\\Users\\data\\AppData\\Local\\Temp\\dask-worker-space\\worker-i7cnpql6\n", + "
\n", + "
\n", + "
\n", + "
\n", + " \n", + "
\n", + "
\n", + "
\n", + "
\n", + " \n", + "

Worker: 2

\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "\n", + " \n", + "\n", + " \n", + "\n", + "
\n", + " Comm: tcp://127.0.0.1:55390\n", + " \n", + " Total threads: 10\n", + "
\n", + " Dashboard: http://127.0.0.1:55392/status\n", + " \n", + " Memory: 9.31 GiB\n", + "
\n", + " Nanny: tcp://127.0.0.1:55361\n", + "
\n", + " Local directory: C:\\Users\\data\\AppData\\Local\\Temp\\dask-worker-space\\worker-cm12p6g3\n", + "
\n", + "
\n", + "
\n", + "
\n", + " \n", + "
\n", + "
\n", + "
\n", + "
\n", + " \n", + "

Worker: 3

\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "\n", + " \n", + "\n", + " \n", + "\n", + "
\n", + " Comm: tcp://127.0.0.1:55391\n", + " \n", + " Total threads: 10\n", + "
\n", + " Dashboard: http://127.0.0.1:55393/status\n", + " \n", + " Memory: 9.31 GiB\n", + "
\n", + " Nanny: tcp://127.0.0.1:55362\n", + "
\n", + " Local directory: C:\\Users\\data\\AppData\\Local\\Temp\\dask-worker-space\\worker-ift1r1_n\n", + "
\n", + "
\n", + "
\n", + "
\n", + " \n", + "
\n", + "
\n", + "
\n", + "
\n", + " \n", + "

Worker: 4

\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "\n", + " \n", + "\n", + " \n", + "\n", + "
\n", + " Comm: tcp://127.0.0.1:55396\n", + " \n", + " Total threads: 10\n", + "
\n", + " Dashboard: http://127.0.0.1:55397/status\n", + " \n", + " Memory: 9.31 GiB\n", + "
\n", + " Nanny: tcp://127.0.0.1:55363\n", + "
\n", + " Local directory: C:\\Users\\data\\AppData\\Local\\Temp\\dask-worker-space\\worker-n0ircagt\n", + "
\n", + "
\n", + "
\n", + "
\n", + " \n", + "
\n", + "
\n", + "
\n", + "
\n", + " \n", + "

Worker: 5

\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "\n", + " \n", + "\n", + " \n", + "\n", + "
\n", + " Comm: tcp://127.0.0.1:55399\n", + " \n", + " Total threads: 10\n", + "
\n", + " Dashboard: http://127.0.0.1:55400/status\n", + " \n", + " Memory: 9.31 GiB\n", + "
\n", + " Nanny: tcp://127.0.0.1:55364\n", + "
\n", + " Local directory: C:\\Users\\data\\AppData\\Local\\Temp\\dask-worker-space\\worker-tym7j5xi\n", + "
\n", + "
\n", + "
\n", + "
\n", + " \n", + "\n", + "
\n", + "
\n", + "\n", + "
\n", + "
\n", + "
\n", + "
\n", + " \n", + "\n", + "
\n", + "
" + ], + "text/plain": [ + "" + ] + }, + "execution_count": 2, + "metadata": {}, + "output_type": "execute_result" + } + ], "source": [ "from dask.distributed import Client\n", "client = Client(n_workers=6, threads_per_worker=10, processes=True, memory_limit='10GB')\n", @@ -63,7 +466,7 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 3, "metadata": {}, "outputs": [], "source": [ @@ -79,7 +482,7 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 4, "metadata": {}, "outputs": [], "source": [ @@ -93,12 +496,38 @@ " # \"images/ODT_1_Axis_Camera/in_situ_absorption\": \"camera_2\",\n", "}\n", "\n", - "datasetDict = {\n", + "dataSetDict = {\n", " dskey[groupList[i]]: read_hdf5_file(filepath, groupList[i])\n", " for i in range(len(groupList))\n", "}" ] }, + { + "cell_type": "code", + "execution_count": 37, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Today's date: 2023-05-04\n", + "d3 = 23/05/04\n", + "True\n" + ] + } + ], + "source": [ + "from datetime import date\n", + "\n", + "today = date.today()\n", + "print(\"Today's date:\", today)\n", + "\n", + "d3 = today.strftime(\"%y/%m/%d\")\n", + "print(\"d3 =\", d3)\n", + "print(isinstance(d3, str))" + ] + }, { "attachments": {}, "cell_type": "markdown", @@ -109,15 +538,15 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 5, "metadata": {}, "outputs": [], "source": [ - "dataset = datasetDict[\"camera_1\"]\n", + "dataSet = dataSetDict[\"camera_1\"]\n", "\n", - "scanAxis = dataset.scanAxis\n", + "scanAxis = dataSet.scanAxis\n", "\n", - "# dataset = dataset.chunk(\n", + "# dataSet = dataSet.chunk(\n", "# {\n", "# # \"compZ_current_sg\": \"auto\",\n", "# \"sin_mod_freq\": \"auto\",\n", @@ -128,6 +557,27 @@ "# )" ] }, + { + "cell_type": "code", + "execution_count": 27, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "final_amp\n", + "runs\n", + "x\n", + "y\n" + ] + } + ], + "source": [ + "for key in dataset.dims.keys():\n", + " print(key)" + ] + }, { "attachments": {}, "cell_type": "markdown", @@ -146,13 +596,1506 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 13, "metadata": {}, - "outputs": [], + "outputs": [ + { + "data": { + "text/html": [ + "
\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "
<xarray.Dataset>\n",
+       "Dimensions:     (final_amp: 11, runs: 3, x: 1200, y: 1920)\n",
+       "Coordinates:\n",
+       "  * final_amp   (final_amp) float64 3e-05 5.5e-05 8e-05 ... 0.000255 0.00028\n",
+       "  * runs        (runs) float64 0.0 1.0 2.0\n",
+       "Dimensions without coordinates: x, y\n",
+       "Data variables:\n",
+       "    atoms       (final_amp, runs, x, y) uint16 dask.array<chunksize=(1, 1, 1200, 1920), meta=np.ndarray>\n",
+       "    background  (final_amp, runs, x, y) uint16 dask.array<chunksize=(1, 1, 1200, 1920), meta=np.ndarray>\n",
+       "    dark        (final_amp, runs, x, y) uint16 dask.array<chunksize=(1, 1, 1200, 1920), meta=np.ndarray>\n",
+       "    shotNum     (final_amp, runs) <U2 '00' '11' '22' '01' ... '10' '21' '32'\n",
+       "    OD          (final_amp, runs, x, y) float64 dask.array<chunksize=(1, 1, 1200, 1920), meta=np.ndarray>\n",
+       "Attributes: (12/96)\n",
+       "    TOF_free:                          0.01\n",
+       "    abs_img_freq:                      110.858\n",
+       "    absorption_imaging_flag:           True\n",
+       "    backup_data:                       True\n",
+       "    blink_off_time:                    nan\n",
+       "    blink_on_time:                     nan\n",
+       "    ...                                ...\n",
+       "    z_offset:                          0.189\n",
+       "    z_offset_img:                      0.189\n",
+       "    final_amp:                         [3.00e-05 5.50e-05 8.00e-05 1.05e-04 1...\n",
+       "    runs:                              [0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 1. 1...\n",
+       "    scanAxis:                          ['final_amp', 'runs']\n",
+       "    scanAxisLength:                    [33. 33.]
" + ], + "text/plain": [ + "\n", + "Dimensions: (final_amp: 11, runs: 3, x: 1200, y: 1920)\n", + "Coordinates:\n", + " * final_amp (final_amp) float64 3e-05 5.5e-05 8e-05 ... 0.000255 0.00028\n", + " * runs (runs) float64 0.0 1.0 2.0\n", + "Dimensions without coordinates: x, y\n", + "Data variables:\n", + " atoms (final_amp, runs, x, y) uint16 dask.array\n", + " background (final_amp, runs, x, y) uint16 dask.array\n", + " dark (final_amp, runs, x, y) uint16 dask.array\n", + " shotNum (final_amp, runs) \n", + "Attributes: (12/96)\n", + " TOF_free: 0.01\n", + " abs_img_freq: 110.858\n", + " absorption_imaging_flag: True\n", + " backup_data: True\n", + " blink_off_time: nan\n", + " blink_on_time: nan\n", + " ... ...\n", + " z_offset: 0.189\n", + " z_offset_img: 0.189\n", + " final_amp: [3.00e-05 5.50e-05 8.00e-05 1.05e-04 1...\n", + " runs: [0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 1. 1...\n", + " scanAxis: ['final_amp', 'runs']\n", + " scanAxisLength: [33. 33.]" + ] + }, + "execution_count": 13, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "dataSet = imageAnalyser.get_absorption_images(dataSet)\n", + "\n", + "dataSet" + ] + }, + { + "cell_type": "code", + "execution_count": 18, + "metadata": {}, + "outputs": [ + { + "data": { + "text/html": [ + "
\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "
<xarray.Dataset>\n",
+       "Dimensions:     (final_amp: 11, runs: 3, x: 1200, y: 1920)\n",
+       "Coordinates:\n",
+       "  * final_amp   (final_amp) float64 3e-05 5.5e-05 8e-05 ... 0.000255 0.00028\n",
+       "  * runs        (runs) float64 0.0 1.0 2.0\n",
+       "Dimensions without coordinates: x, y\n",
+       "Data variables:\n",
+       "    atoms       (final_amp, runs, x, y) uint16 98 101 109 106 ... 146 140 140\n",
+       "    background  (final_amp, runs, x, y) uint16 99 101 113 103 ... 147 137 137\n",
+       "    dark        (final_amp, runs, x, y) uint16 50 51 51 50 50 ... 50 51 50 49 49\n",
+       "    shotNum     (final_amp, runs) <U2 '00' '11' '22' '01' ... '10' '21' '32'\n",
+       "    OD          (final_amp, runs, x, y) float64 nan nan ... -0.03352 -0.03352\n",
+       "Attributes: (12/96)\n",
+       "    TOF_free:                          0.01\n",
+       "    abs_img_freq:                      110.858\n",
+       "    absorption_imaging_flag:           True\n",
+       "    backup_data:                       True\n",
+       "    blink_off_time:                    nan\n",
+       "    blink_on_time:                     nan\n",
+       "    ...                                ...\n",
+       "    z_offset:                          0.189\n",
+       "    z_offset_img:                      0.189\n",
+       "    final_amp:                         [3.00e-05 5.50e-05 8.00e-05 1.05e-04 1...\n",
+       "    runs:                              [0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 1. 1...\n",
+       "    scanAxis:                          ['final_amp', 'runs']\n",
+       "    scanAxisLength:                    [33. 33.]
" + ], + "text/plain": [ + "\n", + "Dimensions: (final_amp: 11, runs: 3, x: 1200, y: 1920)\n", + "Coordinates:\n", + " * final_amp (final_amp) float64 3e-05 5.5e-05 8e-05 ... 0.000255 0.00028\n", + " * runs (runs) float64 0.0 1.0 2.0\n", + "Dimensions without coordinates: x, y\n", + "Data variables:\n", + " atoms (final_amp, runs, x, y) uint16 98 101 109 106 ... 146 140 140\n", + " background (final_amp, runs, x, y) uint16 99 101 113 103 ... 147 137 137\n", + " dark (final_amp, runs, x, y) uint16 50 51 51 50 50 ... 50 51 50 49 49\n", + " shotNum (final_amp, runs)