ensemble_load_expinfo

 returns the response table and metadata associated with an experiment

   result = ensemble_load_expinfo(indata,params)
 
 REQUIRES
   params.ensemble.conn_id
   params.ensemble.experiment_title
   params.ensemble.remove_incomplete_sessions
   params.ensemble.terminal_form
   params.extract_vars
   params.filt
 
 the 'indata' variable is not used yet, but is simply there as a
 placeholder. It may be used in future versions of this function

0001 function result = ensemble_load_expinfo(indata,params) 
0002 % returns the response table and metadata associated with an experiment
0003 %
0004 %   result = ensemble_load_expinfo(indata,params)
0005 %
0006 % REQUIRES
0007 %   params.ensemble.conn_id
0008 %   params.ensemble.experiment_title
0009 %   params.ensemble.remove_incomplete_sessions
0010 %   params.ensemble.terminal_form
0011 %   params.extract_vars
0012 %   params.filt
0013 %
0014 % the 'indata' variable is not used yet, but is simply there as a
0015 % placeholder. It may be used in future versions of this function
0016 %
0017 
0018 % Original version by Petr Janata
0019 % 16 Feb 2007 - reorganized code by PJ into a function, Stefan Tomic
0020 %
0021 % 15 March 2007 - edited function to conform to ensemble data structures
0022 % 04 April 2007 - PJ implemented incremental buildup of result structure.
0023 %                 Added check to make sure there are stimuli to load
0024 %                 information about.
0025 % 03 May 2007 - Added option to pass in a list of variables to extract from the
0026 %               response table.
0027 % 05/18/07 PJ - Added handling of terminal_form which is used in conjunction
0028 %               with remove_incomplete_sessions to specify experiment
0029 %               completion relative to a particular form ID rather than an
0030 %               end_datetime entry in the session table.
0031 %
0032 % 08/06/08 JG - Added getDefaultParams feature
0033 %
0034 % 2009.05.18 FB - added some additional header documentation
0035 % 2009.10.18 PJ - added check to make sure response table was found
0036 % 2010.01.20 FB - now also returns 'misc_info' in the 'response_data' struct
0037 % 2010.05.08 PJ - isolated subject_summary_stats in try/catch
0038 % 2010.06.14 PJ - call to mysql_get_subinfo now requirese that encryption
0039 %                 key information be passed along
0040 % 2010.06.18 FB - now accepts inData={'task','return_outdir'}, for
0041 %                   ensemble_jobman_parallel
0042 % 2011.11.23 PJ - added filtering of subject info
0043 % 2014.08.31 PJ - added experiment_id to list of variables extracted from
0044 %                 response table to accommodate settings with multiple
0045 %                 experiments per response table
0046 
0047 if( ischar( indata ) && strcmp( indata, 'getDefaultParams' ) )
0048     result.ensemble.experiment_title = 'use local settings';
0049     result.mysql.host = 'use local settings';
0050     result.mysql.database = 'use local settings';
0051     result.mysql.conn_id = -1;
0052     return;
0053 end
0054 
0055 if (iscell(indata) && ~isempty(indata) && isfield(indata{1},'task') && ...
0056         strcmp('return_outdir',indata{1}.task)) || ...
0057         (isstruct(indata) && isfield(indata,'task') && ...
0058         strcmp('return_outdir',indata.task))
0059     result = '';
0060     return
0061 end
0062 
0063 % Initialize the output data struct
0064 result = ensemble_init_data_struct;
0065 result.type = 'experiment_info';
0066 
0067 % Check to see if we have a valid connection
0068 if ~isfield(params,'mysql')
0069     params.mysql = params.ensemble;
0070 end
0071 conn_id = mysql_make_conn(params.mysql);
0072 
0073 expname = params.ensemble.experiment_title;
0074 
0075 %% Get the experiment metadata
0076 fprintf('Getting experiment information for: %s\n', expname);
0077 exp_meta = mysql_extract_metadata('table','experiment', ...
0078     'experiment_title',expname, ...
0079     'conn_id', conn_id);
0080 
0081 if isempty(exp_meta.response_table)
0082     warning(['Failed to locate response_table for experiment (%s)\n' ...
0083         'Check to make sure that you are connecting to the database ' ...
0084         'you think you are connecting to!\n'], expname);
0085     return
0086 end
0087 
0088 % Pull desired parts of the response table into a response structure
0089 fprintf('Extracting the response table: %s\n', exp_meta.response_table);
0090 if ~isfield(params.ensemble, 'extract_vars') || isempty(params.ensemble.extract_vars)
0091     extract_vars = {'experiment_id','session_id','subject_id', ...
0092         'response_order','response_id', 'date_time', ...
0093         'form_id','form_order','question_id','subquestion', ...
0094         'stimulus_id','trial_id','response_enum','response_text','misc_info','decline'};
0095 else
0096     extract_vars = params.ensemble.extract_vars;
0097 end
0098 
0099 exp_meta_dataStruct = ensemble_tree2datastruct(exp_meta);
0100 exp_meta_dataStruct.name = 'experiment_metadata';
0101 exp_meta_dataStruct.type = 'experiment_metadata';
0102 
0103 % update the output structure
0104 result.vars{end+1} = 'experiment_metadata';
0105 result.data{end+1} = exp_meta_dataStruct;
0106 
0107 %get the experiment ID
0108 exp_meta_cols = set_var_col_const(exp_meta_dataStruct.vars);
0109 experiment_id = exp_meta_dataStruct.data{exp_meta_cols.experiment_id};
0110 
0111 %% Load session table info
0112 sessInfo = ensemble_init_data_struct;
0113 sessInfo.name = 'session_info';
0114 sessInfo.type = 'session_info';
0115 [sessInfo.data,sessInfo.vars] = mysql_extract_data('table','session', ...
0116     'experiment_id',experiment_id, ...
0117     'conn_id',conn_id);
0118 
0119 %apply any filters to the sessions. This could trap subjects to exclude
0120 if(isfield(params,'filt'))
0121     sessFilt = params.filt;
0122     sessInfo = ensemble_filter(sessInfo,sessFilt);
0123 end
0124 
0125 %filter out any session that have NaN as the end_datetime (these
0126 %sessions were not completed)
0127 remove_incomplete = 0;
0128 if(isfield(params.ensemble,'remove_incomplete_sessions') && ...
0129         params.ensemble.remove_incomplete_sessions == 1)
0130     remove_incomplete = 1;
0131     if ~isfield(params.ensemble,'terminal_form') || isempty(params.ensemble.terminal_form)
0132         sessFilt.exclude.any.end_datetime = NaN;
0133         sessInfo = ensemble_filter(sessInfo,sessFilt);
0134     end
0135 end
0136 
0137 %get the filtered list of session IDs
0138 sess_colNames = set_var_col_const(sessInfo.vars);
0139 sessIDList = sessInfo.data{sess_colNames.session_id};
0140 
0141 %% Load response table
0142 fprintf('Loading response table information ...\n');
0143 respinfo = ensemble_init_data_struct;
0144 
0145 % Check to see if we are dealing with encrypted fields
0146 if isfield(params.ensemble, 'encrypted_fields')
0147   encrypted_flds = params.ensemble.encrypted_fields;
0148   enc_key = params.mysql.enc_key;
0149 else
0150   encrypted_flds = {};
0151   enc_key = '';
0152 end
0153 
0154 [respinfo.data,respinfo.vars] = mysql_extract_data('table', exp_meta.response_table, ...
0155     'extract_flds', extract_vars, 'order_by','response_id', ...
0156     'session_id', sessIDList, ...
0157   'encrypted_flds', encrypted_flds, ...
0158   'enc_key', enc_key, ...
0159     'conn_id', conn_id);
0160 respinfo.name = 'response_data';
0161 respinfo.type = 'response_data';
0162 respcols = set_var_col_const(respinfo.vars);
0163 
0164 % Check to make sure that subquestion information isn't NaN
0165 if any(isnan(respinfo.data{respcols.subquestion}))
0166     fprintf('Replacing NaNs in subquestion field with 1\n');
0167     respinfo.data{respcols.subquestion}(isnan(respinfo.data{respcols.subquestion})) = 1;
0168 end
0169 
0170 % Check to see if we are filtering using 'terminal_form'
0171 respFilt = [];
0172 if isfield(params.ensemble,'terminal_form')
0173   term_form = params.ensemble.terminal_form;
0174 else
0175   term_form = [];
0176 end
0177 
0178 if ~isempty(term_form) && remove_incomplete
0179     form_mask = respinfo.data{respcols.form_id} == term_form;
0180     finished_sessids = unique(respinfo.data{respcols.session_id}(form_mask));
0181     respFilt.include.any.session_id = finished_sessids;
0182 end
0183 
0184 if isfield(params,'filt')
0185     respFilt = add2filt(respFilt,params.filt);
0186 end
0187 
0188 if(~isempty(respFilt))
0189     fprintf('Filtering response table ...\n');
0190     respinfo = ensemble_filter(respinfo,respFilt);
0191 end
0192 
0193 % update the output structure
0194 result.vars{end+1} = 'response_data';
0195 result.data{end+1} = respinfo;
0196 
0197 % Refilter session information if necessary
0198 if remove_incomplete && ~isempty(term_form)
0199     fprintf('Removing incomplete sessions from session_info ...\n');
0200     clear tmpFilt
0201     tmpFilt.include.any.session_id = finished_sessids;
0202     sessInfo = ensemble_filter(sessInfo, tmpFilt);
0203 end
0204 
0205 % update the output structure
0206 result.vars{end+1} = 'session_info';
0207 result.data{end+1} = sessInfo;
0208 outCols = set_var_col_const(result.vars);
0209 
0210 %% Load subject table info
0211 % Pull the subject information for all of the subjects
0212 if isfield(params, 'ignoreSubjectTable') && params.ignoreSubjectTable
0213     fprintf('%s: ignoring subject table information\n', mfilename);
0214 else
0215     fprintf('Loading subject information ...\n');
0216     if ~isfield(params.ensemble,'enc_key')
0217         if isfield(params, 'mysql') && isfield(params.mysql, 'enc_key')
0218             params.ensemble.enc_key = params.mysql.enc_key;
0219         else
0220             params.ensemble.enc_key = '';
0221         end
0222     end
0223     
0224     if ~isfield(params.ensemble, 'conn_id') || isempty(params.ensemble.conn_id)
0225         try
0226             params.ensemble.conn_id = params.mysql.conn_id;
0227         catch
0228             fprintf('Failed to locate connection ID');
0229         end
0230     end
0231     
0232     if isfield(params,'mysql')
0233         login_params = params.mysql;
0234     elseif isfield(params,'ensemble')
0235         login_params = params.ensemble;
0236     else
0237         login_params = struct();
0238     end
0239     
0240     subInfo = mysql_get_subinfo('subject_id', sessInfo.data{sess_colNames.subject_id}, ...
0241         'mysql', login_params);
0242     subInfo.name = 'subject_info';
0243     subInfo.type = 'subject_info';
0244     
0245     % Perform filtering on subject info
0246     if isfield(params,'filt')
0247         subInfo = ensemble_filter(subInfo, params.filt);
0248     end
0249     scols = set_var_col_const(subInfo.vars);
0250     
0251     % Get list of retained subject IDs
0252     goodSubIDs = unique(subInfo.data{scols.subject_id});
0253     
0254     % Refilter session and response table structures
0255     if isfield(params,'ensemble') && isfield(params.ensemble,'refilterBySubjectTableIDs') && ~params.ensemble.refilterBySubjectTableIDs
0256         fprintf('Not refiltering session and response data using subject table information ...\n');
0257     else
0258         fprintf('Refiltering session and response data using subject table information ...\n');
0259         tmpfilt = struct();
0260         tmpfilt.include.all.subject_id = goodSubIDs;
0261         
0262         result.data{outCols.session_info} = ...
0263             ensemble_filter(result.data{outCols.session_info}, tmpfilt);
0264         sessInfo = result.data{outCols.session_info};
0265         
0266         result.data{outCols.response_data} = ...
0267             ensemble_filter(result.data{outCols.response_data}, tmpfilt);
0268         respinfo = result.data{outCols.response_data};
0269     end
0270     
0271     % update the output structure
0272     result.vars{end+1} = 'subject_info';
0273     result.data{end+1} = subInfo;
0274 end % if ignoreSubjectTable
0275 
0276 %% Get stimulus info
0277 % Get stimulus information if there are any stimuli associated with the
0278 % response table
0279 if ~all(isnan(respinfo.data{respcols.stimulus_id}))
0280     fprintf('Loading stimulus and attribute information...\n');
0281     stimulusDataStruct = ensemble_get_stiminfo(respinfo, params);
0282     result.vars = [result.vars stimulusDataStruct.vars];
0283     result.data = [result.data stimulusDataStruct.data];
0284 end 
0285 
0286 %get subject summary stats
0287 try
0288   subSummaryStats = ensemble_summary_subject_stats({subInfo sessInfo});
0289   result.vars{end+1} = 'subject_summary_stats';
0290   result.data{end+1} = subSummaryStats;
0291 catch
0292   warning('Unable to run ensemble_summary_subject_stats');
0293 end
0294 
0295