Daniel@0: function cwr  = cwr_em(X, Y, nc, varargin)
Daniel@0: % CWR_LEARN Fit the parameters of a cluster weighted regression model using EM
Daniel@0: % function cwr  = cwr_learn(X, Y, ...)
Daniel@0: %
Daniel@0: % X(:, t) is the t'th input example
Daniel@0: % Y(:, t) is the t'th output example
Daniel@0: % nc is the number of clusters
Daniel@0: %
Daniel@0: % Kevin Murphy, May 2003
Daniel@0: 
Daniel@0: [max_iter, thresh, cov_typeX, cov_typeY, clamp_weights, ...
Daniel@0:  muX, muY, SigmaX, SigmaY, weightsY, priorC, create_init_params, ...
Daniel@0: cov_priorX, cov_priorY, verbose, regress, clamp_covX, clamp_covY] = process_options(...
Daniel@0:     varargin, 'max_iter', 10, 'thresh', 1e-2, 'cov_typeX', 'full', ...
Daniel@0:      'cov_typeY', 'full', 'clamp_weights', 0, ...
Daniel@0:      'muX', [], 'muY', [], 'SigmaX', [], 'SigmaY', [], 'weightsY', [], 'priorC', [], ...
Daniel@0:      'create_init_params', 1, 'cov_priorX', [], 'cov_priorY', [], 'verbose', 0, ...
Daniel@0:     'regress', 1, 'clamp_covX', 0, 'clamp_covY', 0);
Daniel@0:      
Daniel@0: [nx N] = size(X);
Daniel@0: [ny N2] = size(Y);
Daniel@0: if N ~= N2
Daniel@0:   error(sprintf('nsamples X (%d) ~= nsamples Y (%d)', N, N2));
Daniel@0: end
Daniel@0: %if N < nx 
Daniel@0: %  fprintf('cwr_em warning: dim X (%d) > nsamples X (%d)\n', nx, N);
Daniel@0: %end
Daniel@0: if (N < nx) & regress
Daniel@0:   fprintf('cwr_em warning: dim X = %d, nsamples X = %d\n', nx, N);
Daniel@0: end
Daniel@0: if (N < ny) 
Daniel@0:   fprintf('cwr_em warning: dim Y = %d, nsamples Y = %d\n', ny, N);
Daniel@0: end
Daniel@0: if (nc > N) 
Daniel@0:   error(sprintf('cwr_em: more centers (%d) than data', nc))
Daniel@0: end
Daniel@0: 
Daniel@0: if nc==1
Daniel@0:   % No latent variable, so there is a closed-form solution
Daniel@0:   w = 1/N;
Daniel@0:   WYbig = Y*w;
Daniel@0:   WYY = WYbig * Y'; 
Daniel@0:   WY = sum(WYbig, 2);
Daniel@0:   WYTY = sum(diag(WYbig' * Y));
Daniel@0:   cwr.priorC = 1;
Daniel@0:   cwr.SigmaX = [];
Daniel@0:   if ~regress
Daniel@0:     % This is just fitting an unconditional Gaussian
Daniel@0:     cwr.weightsY = [];
Daniel@0:     [cwr.muY, cwr.SigmaY] = ...
Daniel@0: 	mixgauss_Mstep(1, WY, WYY, WYTY, ...
Daniel@0: 		       'cov_type', cov_typeY, 'cov_prior', cov_priorY);
Daniel@0:     % There is a much easier way...
Daniel@0:     assert(approxeq(cwr.muY, mean(Y')))
Daniel@0:     assert(approxeq(cwr.SigmaY, cov(Y') + 0.01*eye(ny)))
Daniel@0:   else
Daniel@0:     % This is just linear regression
Daniel@0:     WXbig = X*w;
Daniel@0:     WXX = WXbig * X';
Daniel@0:    WX = sum(WXbig, 2);
Daniel@0:     WXTX = sum(diag(WXbig' * X));
Daniel@0:     WXY = WXbig * Y';
Daniel@0:     [cwr.muY, cwr.SigmaY, cwr.weightsY] = ...
Daniel@0: 	clg_Mstep(1, WY, WYY, WYTY, WX, WXX, WXY, ...
Daniel@0: 		  'cov_type', cov_typeY, 'cov_prior', cov_priorY);
Daniel@0:   end
Daniel@0:   if clamp_covY, cwr.SigmaY = SigmaY; end
Daniel@0:   if clamp_weights,  cwr.weightsY = weightsY; end
Daniel@0:   return;
Daniel@0: end
Daniel@0: 
Daniel@0: 
Daniel@0: if create_init_params
Daniel@0:   [cwr.muX, cwr.SigmaX] = mixgauss_init(nc, X, cov_typeX);
Daniel@0:   [cwr.muY, cwr.SigmaY] = mixgauss_init(nc, Y, cov_typeY);
Daniel@0:   cwr.weightsY = zeros(ny, nx, nc);
Daniel@0:   cwr.priorC = normalize(ones(nc,1));
Daniel@0: else
Daniel@0:   cwr.muX = muX;  cwr.muY = muY; cwr.SigmaX = SigmaX; cwr.SigmaY = SigmaY;
Daniel@0:   cwr.weightsY = weightsY; cwr.priorC = priorC;
Daniel@0: end
Daniel@0: 
Daniel@0: 
Daniel@0: if clamp_covY, cwr.SigmaY = SigmaY; end
Daniel@0: if clamp_covX,  cwr.SigmaX = SigmaX; end
Daniel@0: if clamp_weights,  cwr.weightsY = weightsY; end
Daniel@0: 
Daniel@0: previous_loglik = -inf;
Daniel@0: num_iter = 1;
Daniel@0: converged = 0;
Daniel@0: 
Daniel@0: while (num_iter <= max_iter) & ~converged
Daniel@0: 
Daniel@0:   % E step
Daniel@0:   
Daniel@0:   [likXandY, likYgivenX, post] = cwr_prob(cwr, X, Y);
Daniel@0:   loglik = sum(log(likXandY));
Daniel@0:   % extract expected sufficient statistics
Daniel@0:   w = sum(post,2);  % post(c,t)
Daniel@0:   WYY = zeros(ny, ny, nc);
Daniel@0:   WY = zeros(ny, nc);
Daniel@0:   WYTY = zeros(nc,1);
Daniel@0:   
Daniel@0:   WXX = zeros(nx, nx, nc);
Daniel@0:   WX = zeros(nx, nc);
Daniel@0:   WXTX = zeros(nc, 1);
Daniel@0:   WXY = zeros(nx,ny,nc);
Daniel@0:   %WYY = repmat(reshape(w, [1 1 nc]), [ny ny 1]) .*  repmat(Y*Y', [1 1 nc]);
Daniel@0:   for c=1:nc
Daniel@0:     weights = repmat(post(c,:), ny, 1);
Daniel@0:     WYbig = Y .* weights;
Daniel@0:     WYY(:,:,c) = WYbig * Y'; 
Daniel@0:     WY(:,c) = sum(WYbig, 2);
Daniel@0:     WYTY(c) = sum(diag(WYbig' * Y));
Daniel@0: 
Daniel@0:     weights = repmat(post(c,:), nx, 1); % weights(nx, nsamples)
Daniel@0:     WXbig = X .* weights;
Daniel@0:     WXX(:,:,c) = WXbig * X';
Daniel@0:     WX(:,c) = sum(WXbig, 2);
Daniel@0:     WXTX(c) = sum(diag(WXbig' * X));
Daniel@0:     WXY(:,:,c) = WXbig * Y';
Daniel@0:   end
Daniel@0: 
Daniel@0:   % M step
Daniel@0:   % Q -> X is called Q->Y in Mstep_clg
Daniel@0:   [cwr.muX, cwr.SigmaX] = mixgauss_Mstep(w, WX, WXX, WXTX, ...
Daniel@0: 			    'cov_type', cov_typeX, 'cov_prior', cov_priorX);
Daniel@0:   for c=1:nc
Daniel@0:     assert(is_psd(cwr.SigmaX(:,:,c)))
Daniel@0:   end
Daniel@0:   
Daniel@0:   if clamp_weights % affects estimate of mu and Sigma
Daniel@0:     W = cwr.weightsY;
Daniel@0:   else
Daniel@0:     W = [];
Daniel@0:   end
Daniel@0:   [cwr.muY, cwr.SigmaY, cwr.weightsY] = ...
Daniel@0:       clg_Mstep(w, WY, WYY, WYTY, WX, WXX, WXY, ...
Daniel@0: 		'cov_type', cov_typeY, 'clamped_weights', W, ...
Daniel@0: 		'cov_prior', cov_priorY);
Daniel@0:   %'xs', X, 'ys', Y, 'post', post); % debug
Daniel@0:   %a = linspace(min(Y(2,:)), max(Y(2,:)), nc+2);
Daniel@0:   %cwr.muY(2,:) = a(2:end-1);
Daniel@0: 
Daniel@0:   cwr.priorC = normalize(w);
Daniel@0: 
Daniel@0:   for c=1:nc
Daniel@0:     assert(is_psd(cwr.SigmaY(:,:,c)))
Daniel@0:   end
Daniel@0: 
Daniel@0:   if clamp_covY, cwr.SigmaY = SigmaY; end
Daniel@0:   if clamp_covX,  cwr.SigmaX = SigmaX; end
Daniel@0:   if clamp_weights,  cwr.weightsY = weightsY; end
Daniel@0: 
Daniel@0:   if verbose, fprintf(1, 'iteration %d, loglik = %f\n', num_iter, loglik); end
Daniel@0:   num_iter =  num_iter + 1;
Daniel@0:   converged = em_converged(loglik, previous_loglik, thresh);
Daniel@0:   previous_loglik = loglik;
Daniel@0:   
Daniel@0: end
Daniel@0: