function CV = runCV(trials,classes,parameterSets,validateFraction,validateReps,...
    featurizeF,classifyF,selectInOneTrialF)
%%%   This documentation needs to be updated.
%%%
%%% results = general(trials,classes,parameterSets,validateFraction,validateReps,...
%%%    makeFeaturizeF,featurizeF,makeClassifierF,classifyF)
%%%  trials: ntasks x ntrials cell array of EEG data matrices, each nchannels x nsamples
%%%  classes: ntasks x ntrials matrix of class labels
%%%  parameterSets: nParameterSets vector of structs containing named ...
%%%    parameters, like  parameterSets(1).nlags
%%%  validateFraction: fraction of training data samples to randomly hold ...
%%%    out for validation samples
%%%  validateReps: repeat partition of training data into tuning and ...
%%%    validating subsets this many times and average the results
%%%
%%%  featurizeF: function to create parameters for making features and for
%%%              transforming samples into features
%%%
%%%     To make feature parameters, must have form like
%%%        CV = featurizeF(trials,CV,'make')
%%%     where
%%%          trials: ntasks x ntrials cell array of data
%%%          CV.parms contains named fields like CV.parms.nlags
%%%     returns
%%%          CV: with fields to be used by next call to featurizeF
%%%
%%%     To transform samples to features, must have form like
%%%        features = featurizeF(trials,CV)
%%%     where
%%%          trials: ntasks x ntrials cell array of data
%%%          CV.parms contains named fields like CV.parms.nlags
%%%          CV... other fields made by featurizeF(..,..,'make')
%%%     returns
%%%          features: cell array of features
%%%
%%%  classifyF: function to classify features
%%%     Must have form like
%%%          [fractionCorrect,classifier] = classifyF(traindata,CV,selectInOneTrialF,'make')
%%%     where
%%%          traindata: ntasks x ntrials cell array of featurized data
%%%          CV contains fields for parameters like CV.parms.firstMode
%%%          selectInOneTrialF: See below
%%%     returns
%%%          fractionCorrect: fraction validate samples correctly classified
%%%          classifier: classifier parameters
%%%
%%%     Must have form like
%%%          [fractionCorrect,classes] = classifyF(testdata,CV,selectInOneTrialF)
%%%     where
%%%          testdata: ntasks x ntrials cell array of featurized data
%%%          CV contains fields for parameters like CV.parms.firstMode
%%%             and CV.classifier
%%%          selectInOneTrialF: See below.
%%%     returns
%%%          fractionCorrect: fraction validate samples correctly classified
%%%          classes: classes for test data
%%%
%%%     selectInOneTrialF: Must have form like
%%%         [X,Y] = oneTrialF(oneTrial,indices,oneTrialClass,CV);
%%%         where oneTrial: is matrix of one trial of featurized data.
%%%               indices: is vector of sample (column) indices
%%%               oneTrialClass: is class for this trial
%%%               CV.parms:  fields for use here, such as firstMode, nModes
%%%         returns X: nSamples x nFeatures of samples
%%%                 Y: nSamples x 1 class 
%%%
%%%   Returns 
%%%     results.

fprintf(1,'Given data from %d classes, each with %d trials.\n',...
    size(trials,1),size(trials,2));
fprintf(1,' Using %s to define and create features\n',...
    char(featurizeF));
fprintf(1,' and %s to make and use classifier.\n',...
    char(classifyF));
fprintf(1,' and %s to collect train, validate, test data in classifier.\n',...
    char(selectInOneTrialF));
fprintf(1,' Finding best of %d parameter sets using %.2f fraction\n',...
    length(parameterSets),validateFraction);
fprintf(1,' of samples from each trial as validation set by averaging over\n');
fprintf(1,' %d random partitionings.\n',validateReps);

CV.trials = trials;
CV.classes = classes;
CV.validate.fraction = validateFraction;
CV.validate.repetitions = validateReps;

CV.cvresults = [];
[v,names] = getfields(parameterSets(1));
CV.cvresultsNames = {'TestTrial' names{:}};

CV.testResults = {};

%%% testi is test trial index
ntrials = size(CV.trials,2);
possibleTestTrials = 1:ntrials;

%% testTrials set globally to control which trials to use as test
%% added for running Jose Millan's data from competition 2005
global testTrials

if ~isempty(testTrials)
  possibleTestTrials = testTrials(:)
end
testTrials
possibleTestTrials
for testi = possibleTestTrials

  testi = testi'
  
  resultsOneTestTrial = [];

  fprintf(1,'Test trial %d\n',testi);
  
  %% Partition trials in cell array into train and test structures.

  train.trials = CV.trials(:,setdiff(1:ntrials,testi));
  train.classes = CV.classes(:,setdiff(1:ntrials,testi));
  test.trials = CV.trials(:,testi);
  test.classes = CV.classes(:,testi);
size(test.trials)

  %% Find best values for feature and classifier parameters.
  %% Here we loop over possible values of nlags and mode. For each
  %% set of values, average validation classification results over
  %% validateReps random partitions of train data into train and
  %% validate sets.
  
  bestValCorrect = 0;
  
  for parmsi = 1:length(parameterSets)
    CV.parms = parameterSets(parmsi);

    % Print out parameter values
    [values,names] = getfields(CV.parms);
    for i = 1:length(names)
      if ~(isnan(values(i)) || prod(size(values(i))) > 1)
	fprintf(1,'%s= %g ',names{i},values(i));
      end
    end    

    resultsOneParmSet = [testi values];
    
    CV = featurizeF(train.trials,CV,'make'); 
    train.features = featurizeF(train.trials,CV); 
    
    for valReps = 1:CV.validate.repetitions
      CV.validateIndices = partition(train.features,CV.validate.fraction);
      [validateCorrect] = classifyF(train,CV,selectInOneTrialF,'make');
      fprintf(1,' %.2f',validateCorrect);
      resultsOneParmSet = [resultsOneParmSet validateCorrect];
    end
      avgcorr = mean(resultsOneParmSet(end-valReps+1:end));
    fprintf(1,' av %.2f\n',avgcorr);
    if avgcorr > bestValCorrect
      bestparms = CV.parms;
      bestValCorrect = avgcorr;
    end
    resultsOneTestTrial = [resultsOneTestTrial; resultsOneParmSet];
  end

  %% Set best  parameters for validation sets.
  
  CV.parms = bestparms;

  %% Use these best parameters to build classifer and apply to the
  %% test trial.
  
  CV = featurizeF(train.trials,CV,'make'); 
  train.features = featurizeF(train.trials,CV); 
  test.features = featurizeF(test.trials,CV);

  CV.validateIndices = [];
  [vc,classifier] = classifyF(train,CV,selectInOneTrialF,'make');
  %imagesc(X);drawnow;
  CV.classifier = classifier;
  [testerr,classes] = classifyF(test,CV,selectInOneTrialF);
  CV.testResults = {CV.testResults{:}, {CV.parms CV.classifier ...
      train.features test.features classes testerr}};

  CV.cvresults = [CV.cvresults; resultsOneTestTrial];

end


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
function validateIndices = partition(trials,validateFraction)

[r,c] = size(trials);
validateIndices = cell(r,c);
for ri = 1:r
  for ci = 1:c
    num = size(trials{ri,ci},2);
    randorder = randperm(num);
    nv = round(num * validateFraction);
    validateIndices{ri,ci} = randorder(1:nv);
  end
end



