ensemble_jobman

 Top-level job manager for the Ensemble Matlab analysis toolkit.

 analysis_list = ensemble_jobman(analysis_list,params)

 analysis_list is a cell array of analysis structures that
 conform to the ensemble analysis structure spec (see wiki for details).
 possible params are:

 params.ensemble.conn_id: integer that specifies a mysql connection to use
                          the jobman will use this id to open a connection
 params.run_analyses:     a vector of indices that specify which analysis
                          jobs to run. Analysis jobs whose indices are not
                          listed will not run, but their results (from a
                          previous run) will be available.

 params.resultNameIsAnalysisName: if True, results.name will be set to the
                          name of the analysis



 This function serves as a controller for the Ensemble Matlab
 analysis toolkit. It processes an analysis_list and calls the
 appropriate analysis functions as specified by the function
 handles, and with the appropriate parameters specified in the
 params struct.Results from the analysis are stored in the
 'results' field of the analysis list structure.

0001 function analysis_list = ensemble_jobman(analysis_list,params)
0002 % Top-level job manager for the Ensemble Matlab analysis toolkit.
0003 %
0004 % analysis_list = ensemble_jobman(analysis_list,params)
0005 %
0006 % analysis_list is a cell array of analysis structures that
0007 % conform to the ensemble analysis structure spec (see wiki for details).
0008 % possible params are:
0009 %
0010 % params.ensemble.conn_id: integer that specifies a mysql connection to use
0011 %                          the jobman will use this id to open a connection
0012 % params.run_analyses:     a vector of indices that specify which analysis
0013 %                          jobs to run. Analysis jobs whose indices are not
0014 %                          listed will not run, but their results (from a
0015 %                          previous run) will be available.
0016 %
0017 % params.resultNameIsAnalysisName: if True, results.name will be set to the
0018 %                          name of the analysis
0019 %
0020 %
0021 %
0022 % This function serves as a controller for the Ensemble Matlab
0023 % analysis toolkit. It processes an analysis_list and calls the
0024 % appropriate analysis functions as specified by the function
0025 % handles, and with the appropriate parameters specified in the
0026 % params struct.Results from the analysis are stored in the
0027 % 'results' field of the analysis list structure.
0028 %
0029 
0030 
0031 % 16 Feb 2007 - First version, built on code by PJ, Stefan Tomic
0032 % 01 May 2007 - PJ Added better host, database, conn_id support
0033 % 11/27/07 - improved conn_id support
0034 % 8/25/08  - ST, if conn_id is empty, conn_local is now set to
0035 %            false. Reverted last version, since 'dbstop if error' didn't take
0036 %            you to the desired breakpoint
0037 % 06/15/10 - PJ cleaning mysql_make_conn support
0038 % 04/11/11 - PJ added alternative loading of results struct from file
0039 % 10Nov2012 - PJ added and fixed insertion of analysis name into the
0040 %             results structure
0041 % 02Sep2014 - Fixed handling of run_analyses vector passed in as column
0042 %             rather than row vector
0043 % 01Sep2016 - Added option to force the name of the result structure to
0044 %             take on the analysis name
0045 
0046 
0047 %if conn_id is set in params, then open a mysql connection
0048 %this mysql connection may be used by analysis functions that
0049 %specify the same conn_id in their params
0050 conn_local = false;
0051 
0052 try
0053   conn_id = params.ensemble.conn_id;
0054 catch
0055   conn_id = [];
0056   conn_local = false;
0057 end
0058 
0059 try ignoreEmptyConnID = params.ensemble.ignoreEmptyConnID;
0060 catch ignoreEmptyConnID = 0;
0061 end
0062 
0063 % If connection ID is not empty, check to see if it is open
0064 if ~isempty(conn_id) && ~ignoreEmptyConnID
0065   if mysql(conn_id,'status')
0066     if conn_id == 0
0067       conn_local = true;
0068     end
0069     mysql_make_conn(params.ensemble);
0070   end
0071 end
0072 
0073 if(isfield(params,'run_analyses'))
0074   idxs = params.run_analyses;
0075 else
0076   idxs = 1:length(analysis_list);
0077 end
0078 
0079 for aidx = 1:length(idxs)
0080   ia = idxs(aidx);
0081   analysis_name = analysis_list{ia}.name;
0082   
0083   fprintf('\nPerforming analysis %d/%d: %s\n', ia, length(analysis_list), analysis_name);
0084   indata = {};
0085   
0086   %assign the input data structure for the function based on the
0087   %required fieldname
0088   if isfield(analysis_list{ia},'requires') && ~isempty(analysis_list{ia}.requires)
0089     
0090     for requiredIdx = 1:length(analysis_list{ia}.requires)
0091       requiredData = analysis_list{ia}.requires{requiredIdx};
0092       anSearchCrit.name = requiredData.name;
0093       anListIdx = ensemble_find_analysis_struct(analysis_list,anSearchCrit);
0094       if(isempty(anListIdx))
0095         error(sprintf('Analysis %s specifies a non-existent required analysis %s',...
0096           analysis_list{ia}.name,anSearchCrit.name));
0097       end
0098       
0099       numIdx = length(anListIdx);
0100       if numIdx > 1
0101         error('Found %d analysis structs (%s) that matched the name of the required structure: %s', ...
0102           numIdx, sprintf('%d ', anListIdx), anSearchCrit.name)
0103       end
0104       
0105       if(isfield(analysis_list{anListIdx},'results') )
0106         %search for the required data type
0107         
0108         dataList = analysis_list{anListIdx}.results;
0109         
0110         % Handle the possibility that the results is simply a flag and that the
0111         % results are stored to disk, e.g. if the analysis was run using
0112         % ensemble_jobman_parallel.
0113         if isnumeric(dataList) && dataList == 1 && ...
0114             isfield(analysis_list{anListIdx},'path') && ~isempty(analysis_list{anListIdx}.path)
0115           tmp = load(analysis_list{anListIdx}.path);
0116           dataList = tmp.job_data.results;
0117         end
0118         
0119         %see if 'vars' was specified, in which case we need to
0120         %search for the variable to return
0121         if(isfield(requiredData,'vars') && ~isempty(requiredData.vars))
0122           
0123           
0124           %find the index of the result we need
0125           dataListIdx = strmatch(requiredData.vars,dataList.vars);
0126           
0127           %if result data struct was found then assign to indata
0128           %otherwise, throw an error indicating the data is not there.
0129           if(~isempty(dataListIdx))
0130             indata{requiredIdx} = dataList.data{dataListIdx};
0131             
0132           else
0133             error([sprintf('Analysis %s specifies a non-existent required',analysis_name)...
0134               sprintf(' datatype %s for analysis %s',requiredData.vars,analysis_list{anListIdx}.name)]);
0135           end
0136           
0137         else
0138           %if the 'vars' field of the required data list was not
0139           %specified, then return the whole results data structure
0140           indata{requiredIdx} = analysis_list{anListIdx}.results;
0141         end
0142         
0143         
0144       else
0145         
0146         %results field wasn't populated so required data wasn't available
0147         error(sprintf('Required data for analysis ''%s'' was not available.',analysis_name));
0148         
0149       end
0150       
0151     end % for requiredIdx
0152     
0153   end %if isfield(analysis_list{ia},'requires')
0154   
0155   %if there was only one required data struct, then don't pass this
0156   %as a cell
0157   if(length(indata) == 1)
0158     indata = indata{1};
0159   end
0160   
0161   % Call the function that is registered with this analysis
0162   fh = analysis_list{ia}.fun;
0163   
0164   if(isfield(analysis_list{ia},'params'))
0165     analysisParams = analysis_list{ia}.params;
0166   else
0167     analysisParams = [];
0168   end
0169   
0170   result = fh(indata,analysisParams);
0171   
0172   % Set the name to the analysis name if isfield(analysisParams,'resultNameIsAnalysisName')
0173   if isfield(analysisParams,'resultNameIsAnalysisName') && analysisParams.resultNameIsAnalysisName
0174     result.name = analysis_name;
0175   end
0176   
0177   analysis_list{ia}.results = result;
0178   
0179   % Check to see if the name of the result is empty, and if so, populate it
0180   % with the name of this analysis
0181   numResults = length(result);
0182   for ires = 1:numResults
0183     if isstruct(result(ires))
0184       if ~isfield(result(ires),'name') || isempty(result(ires).name) || isfield(params,'resultNameIsAnalysisName') && params.resultNameIsAnalysisName
0185         analysis_list{ia}.results(ires).name = analysis_name;
0186       else
0187         continue
0188       end
0189     elseif ~isfield(analysis_list{ia}.results{ires},'name') || isempty(analysis_list{ia}.results{ires}.name) || isfield('params','resultNameIsAnalysisName') && params.resultNameIsAnalysisName
0190       analysis_list{ia}.results{ires}.name = analysis_name;
0191     end
0192   end
0193   %    check_conn_exit(params)
0194   %    err = lasterror;
0195   %    fprintf('Failed on analysis %d ...\n%s\nReturning completed analyses ...\n', ia, err.message);
0196   %    return
0197   
0198 end % for ia - analysis list
0199 
0200 if conn_local
0201   mysql(conn_id,'close');
0202 end
0203 end