70 lines
2.6 KiB
Matlab
70 lines
2.6 KiB
Matlab
function [SAVE_TO_WORKSPACE, runMemoryGB] = estimateDatasetMemory(dataSources, options)
|
|
% Estimate per-run memory and decide whether to save dataset to workspace
|
|
|
|
% --- Measured memory per image (bytes) ---
|
|
bytesPerFullImage = 37.75 * 1e6; % full OD image
|
|
bytesPerCroppedImage = 0.16 * 1e6; % cropped OD image
|
|
|
|
% --- Check available RAM on Windows ---
|
|
if ispc
|
|
[~, sys] = memory;
|
|
availableRAM = sys.PhysicalMemory.Available;
|
|
else
|
|
availableRAM = 16e9; % fallback: 16 GB if not Windows
|
|
end
|
|
|
|
SAVE_TO_WORKSPACE = true; % default, may change per run
|
|
runMemoryGB = []; % store per-run memory
|
|
|
|
% --- Loop over all data sources and runs ---
|
|
for i_ds = 1:numel(dataSources)
|
|
ds = dataSources{i_ds};
|
|
|
|
if isfield(ds, 'baseFolder') && ~isempty(ds.baseFolder)
|
|
baseFolder = fullfile(ds.baseFolder, ds.sequence, ds.date);
|
|
else
|
|
baseFolder = fullfile(options.baseDataFolder, ds.sequence, ds.date);
|
|
end
|
|
|
|
for j_run = 1:numel(ds.runs)
|
|
runItem = ds.runs(j_run);
|
|
|
|
% Convert runItem to string runID
|
|
if isnumeric(runItem)
|
|
runID = sprintf('%04d', runItem);
|
|
elseif isstring(runItem)
|
|
runID = runItem;
|
|
elseif ischar(runItem)
|
|
runID = string(runItem);
|
|
elseif iscell(runItem)
|
|
runID = string(runItem{1});
|
|
else
|
|
error('Unsupported run type');
|
|
end
|
|
|
|
runFolder = fullfile(baseFolder, runID);
|
|
|
|
if isfolder(runFolder)
|
|
files = dir(fullfile(runFolder, '*.h5'));
|
|
nFiles = numel(files);
|
|
if nFiles == 0
|
|
continue;
|
|
end
|
|
|
|
% Memory estimate for this run (full + cropped)
|
|
runBytes = nFiles * (bytesPerFullImage + bytesPerCroppedImage);
|
|
runMemoryGB(end+1,1) = runBytes/1e9;
|
|
|
|
% Decide workspace flag per run by comparing with 50% of available RAM
|
|
if runBytes > 0.75 * availableRAM
|
|
SAVE_TO_WORKSPACE = false;
|
|
fprintf('[INFO] Estimated size on memory of Run %s/%s too large (%.2f GB). Not saving to workspace.\n', ...
|
|
ds.sequence, runID, runBytes/1e9);
|
|
else
|
|
fprintf('[INFO] Estimated size on memory of Run %s/%s = %.2f GB. Will save to workspace.\n', ...
|
|
ds.sequence, runID, runBytes/1e9);
|
|
end
|
|
end
|
|
end
|
|
end
|
|
end |