%% This documentation needs to be updated. %% %% results = general(trials,classes,parameterSets,validateFraction,validateReps,... %% makeFeaturizeF,featurizeF,makeClassifierF,classifyF) %% trials: ntasks x ntrials cell array of EEG data matrices, each nchannels x nsamples %% classes: ntasks x ntrials matrix of class labels %% parameterSets: nParameterSets vector of structs containing named ... %% parameters, like parameterSets(1).nlags %% validateFraction: fraction of training data samples to randomly hold ... %% out for validation samples %% validateReps: repeat partition of training data into tuning and ... %% validating subsets this many times and average the results %% %% featurizeF: function to create parameters for making features and for %% transforming samples into features %% %% To make feature parameters, must have form like %% CV = featurizeF(trials,CV,'make') %% where %% trials: ntasks x ntrials cell array of data %% CV.parms contains named fields like CV.parms.nlags %% returns %% CV: with fields to be used by next call to featurizeF %% %% To transform samples to features, must have form like %% features = featurizeF(trials,CV) %% where %% trials: ntasks x ntrials cell array of data %% CV.parms contains named fields like CV.parms.nlags %% CV... other fields made by featurizeF(..,..,'make') %% returns %% features: cell array of features %% %% classifyF: function to classify features %% Must have form like %% [fractionCorrect,classifier] = classifyF(traindata,CV,selectInOneTrialF,'make') %% where %% traindata: ntasks x ntrials cell array of featurized data %% CV contains fields for parameters like CV.parms.firstMode %% selectInOneTrialF: See below %% returns %% fractionCorrect: fraction validate samples correctly classified %% classifier: classifier parameters %% %% Must have form like %% [fractionCorrect,classes] = classifyF(testdata,CV,selectInOneTrialF) %% where %% testdata: ntasks x ntrials cell array of featurized data %% CV contains fields for parameters like CV.parms.firstMode %% and CV.classifier %% selectInOneTrialF: See below. %% returns %% fractionCorrect: fraction validate samples correctly classified %% classes: classes for test data %% %% selectInOneTrialF: Must have form like %% [X,Y] = oneTrialF(oneTrial,indices,oneTrialClass,CV); %% where oneTrial: is matrix of one trial of featurized data. %% indices: is vector of sample (column) indices %% oneTrialClass: is class for this trial %% CV.parms: fields for use here, such as firstMode, nModes %% returns X: nSamples x nFeatures of samples %% Y: nSamples x 1 class %% %% Returns %% results.
0001 function CV = runCV(trials,classes,parameterSets,validateFraction,validateReps,... 0002 featurizeF,classifyF,selectInOneTrialF) 0003 %%% This documentation needs to be updated. 0004 %%% 0005 %%% results = general(trials,classes,parameterSets,validateFraction,validateReps,... 0006 %%% makeFeaturizeF,featurizeF,makeClassifierF,classifyF) 0007 %%% trials: ntasks x ntrials cell array of EEG data matrices, each nchannels x nsamples 0008 %%% classes: ntasks x ntrials matrix of class labels 0009 %%% parameterSets: nParameterSets vector of structs containing named ... 0010 %%% parameters, like parameterSets(1).nlags 0011 %%% validateFraction: fraction of training data samples to randomly hold ... 0012 %%% out for validation samples 0013 %%% validateReps: repeat partition of training data into tuning and ... 0014 %%% validating subsets this many times and average the results 0015 %%% 0016 %%% featurizeF: function to create parameters for making features and for 0017 %%% transforming samples into features 0018 %%% 0019 %%% To make feature parameters, must have form like 0020 %%% CV = featurizeF(trials,CV,'make') 0021 %%% where 0022 %%% trials: ntasks x ntrials cell array of data 0023 %%% CV.parms contains named fields like CV.parms.nlags 0024 %%% returns 0025 %%% CV: with fields to be used by next call to featurizeF 0026 %%% 0027 %%% To transform samples to features, must have form like 0028 %%% features = featurizeF(trials,CV) 0029 %%% where 0030 %%% trials: ntasks x ntrials cell array of data 0031 %%% CV.parms contains named fields like CV.parms.nlags 0032 %%% CV... other fields made by featurizeF(..,..,'make') 0033 %%% returns 0034 %%% features: cell array of features 0035 %%% 0036 %%% classifyF: function to classify features 0037 %%% Must have form like 0038 %%% [fractionCorrect,classifier] = classifyF(traindata,CV,selectInOneTrialF,'make') 0039 %%% where 0040 %%% traindata: ntasks x ntrials cell array of featurized data 0041 %%% CV contains fields for parameters like CV.parms.firstMode 0042 %%% selectInOneTrialF: See below 0043 %%% returns 0044 %%% fractionCorrect: fraction validate samples correctly classified 0045 %%% classifier: classifier parameters 0046 %%% 0047 %%% Must have form like 0048 %%% [fractionCorrect,classes] = classifyF(testdata,CV,selectInOneTrialF) 0049 %%% where 0050 %%% testdata: ntasks x ntrials cell array of featurized data 0051 %%% CV contains fields for parameters like CV.parms.firstMode 0052 %%% and CV.classifier 0053 %%% selectInOneTrialF: See below. 0054 %%% returns 0055 %%% fractionCorrect: fraction validate samples correctly classified 0056 %%% classes: classes for test data 0057 %%% 0058 %%% selectInOneTrialF: Must have form like 0059 %%% [X,Y] = oneTrialF(oneTrial,indices,oneTrialClass,CV); 0060 %%% where oneTrial: is matrix of one trial of featurized data. 0061 %%% indices: is vector of sample (column) indices 0062 %%% oneTrialClass: is class for this trial 0063 %%% CV.parms: fields for use here, such as firstMode, nModes 0064 %%% returns X: nSamples x nFeatures of samples 0065 %%% Y: nSamples x 1 class 0066 %%% 0067 %%% Returns 0068 %%% results. 0069 0070 fprintf(1,'Given data from %d classes, each with %d trials.\n',... 0071 size(trials,1),size(trials,2)); 0072 fprintf(1,' Using %s to define and create features\n',... 0073 char(featurizeF)); 0074 fprintf(1,' and %s to make and use classifier.\n',... 0075 char(classifyF)); 0076 fprintf(1,' and %s to collect train, validate, test data in classifier.\n',... 0077 char(selectInOneTrialF)); 0078 fprintf(1,' Finding best of %d parameter sets using %.2f fraction\n',... 0079 length(parameterSets),validateFraction); 0080 fprintf(1,' of samples from each trial as validation set by averaging over\n'); 0081 fprintf(1,' %d random partitionings.\n',validateReps); 0082 0083 CV.trials = trials; 0084 CV.classes = classes; 0085 CV.validate.fraction = validateFraction; 0086 CV.validate.repetitions = validateReps; 0087 0088 CV.cvresults = []; 0089 [v,names] = getfields(parameterSets(1)); 0090 CV.cvresultsNames = {'TestTrial' names{:}}; 0091 0092 CV.testResults = {}; 0093 0094 %%% testi is test trial index 0095 ntrials = size(CV.trials,2); 0096 possibleTestTrials = 1:ntrials; 0097 0098 %% testTrials set globally to control which trials to use as test 0099 %% added for running Jose Millan's data from competition 2005 0100 global testTrials 0101 0102 if ~isempty(testTrials) 0103 possibleTestTrials = testTrials(:) 0104 end 0105 testTrials 0106 possibleTestTrials 0107 for testi = possibleTestTrials 0108 0109 testi = testi' 0110 0111 resultsOneTestTrial = []; 0112 0113 fprintf(1,'Test trial %d\n',testi); 0114 0115 %% Partition trials in cell array into train and test structures. 0116 0117 train.trials = CV.trials(:,setdiff(1:ntrials,testi)); 0118 train.classes = CV.classes(:,setdiff(1:ntrials,testi)); 0119 test.trials = CV.trials(:,testi); 0120 test.classes = CV.classes(:,testi); 0121 size(test.trials) 0122 0123 %% Find best values for feature and classifier parameters. 0124 %% Here we loop over possible values of nlags and mode. For each 0125 %% set of values, average validation classification results over 0126 %% validateReps random partitions of train data into train and 0127 %% validate sets. 0128 0129 bestValCorrect = 0; 0130 0131 for parmsi = 1:length(parameterSets) 0132 CV.parms = parameterSets(parmsi); 0133 0134 % Print out parameter values 0135 [values,names] = getfields(CV.parms); 0136 for i = 1:length(names) 0137 if ~(isnan(values(i)) || prod(size(values(i))) > 1) 0138 fprintf(1,'%s= %g ',names{i},values(i)); 0139 end 0140 end 0141 0142 resultsOneParmSet = [testi values]; 0143 0144 CV = featurizeF(train.trials,CV,'make'); 0145 train.features = featurizeF(train.trials,CV); 0146 0147 for valReps = 1:CV.validate.repetitions 0148 CV.validateIndices = partition(train.features,CV.validate.fraction); 0149 [validateCorrect] = classifyF(train,CV,selectInOneTrialF,'make'); 0150 fprintf(1,' %.2f',validateCorrect); 0151 resultsOneParmSet = [resultsOneParmSet validateCorrect]; 0152 end 0153 avgcorr = mean(resultsOneParmSet(end-valReps+1:end)); 0154 fprintf(1,' av %.2f\n',avgcorr); 0155 if avgcorr > bestValCorrect 0156 bestparms = CV.parms; 0157 bestValCorrect = avgcorr; 0158 end 0159 resultsOneTestTrial = [resultsOneTestTrial; resultsOneParmSet]; 0160 end 0161 0162 %% Set best parameters for validation sets. 0163 0164 CV.parms = bestparms; 0165 0166 %% Use these best parameters to build classifer and apply to the 0167 %% test trial. 0168 0169 CV = featurizeF(train.trials,CV,'make'); 0170 train.features = featurizeF(train.trials,CV); 0171 test.features = featurizeF(test.trials,CV); 0172 0173 CV.validateIndices = []; 0174 [vc,classifier] = classifyF(train,CV,selectInOneTrialF,'make'); 0175 %imagesc(X);drawnow; 0176 CV.classifier = classifier; 0177 [testerr,classes] = classifyF(test,CV,selectInOneTrialF); 0178 CV.testResults = {CV.testResults{:}, {CV.parms CV.classifier ... 0179 train.features test.features classes testerr}}; 0180 0181 CV.cvresults = [CV.cvresults; resultsOneTestTrial]; 0182 0183 end 0184 0185 0186 %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% 0187 function validateIndices = partition(trials,validateFraction) 0188 0189 [r,c] = size(trials); 0190 validateIndices = cell(r,c); 0191 for ri = 1:r 0192 for ci = 1:c 0193 num = size(trials{ri,ci},2); 0194 randorder = randperm(num); 0195 nv = round(num * validateFraction); 0196 validateIndices{ri,ci} = randorder(1:nv); 0197 end 0198 end 0199 0200 0201