view nonExposed/generateScene.m @ 14:b1901e8d8f5f

initial commit

function [w,sceneSchedule] = generateScene (sceneSchedule,sceneObjects,score,inputPath,outputPath,outputFileName,displayFigures,timeMode,endCut)
% function [w,sceneSchedule] = generateScene (sceneSchedule,sceneObjects,score,inputPath,outputPath,outputFileName,displayFigures,timeMode,endCut)
% This function does the actual job of loading, scaling, positioning and
% mixing the various samples to generate a scene, based on the
% specifications given by its parameters. 

% This program was written by Mathias Rossignol & Grégoire Lafay
% is Copyright (C) 2015 IRCAM <http://www.ircam.fr>
%
% This program is free software: you can redistribute it and/or modify it
% under the terms of the GNU General Public License as published by the Free
% Software Foundation, either version 3 of the License, or (at your option)
% any later version.
%
% This program is distributed in the hope that it will be useful, but
% WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY
% or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
% for more details.
%
% You should have received a copy of the GNU General Public License along
% with this program.  If not, see <http://www.gnu.org/licenses/>.

sr = 44100;
trackLength = sr*score.sceneDuration;
nbTracks = length(sceneObjects);

% All instances of one sample will appear on a given track, before
% everything is mixed together, making it possible to apply track-specific
% effects or generate nice colorful representations
tracks = zeros(length(sceneObjects), trackLength);

% We also generate a "bg" track with all background samples, against which
% the snr of fg sounds will be evaluated
bg = zeros(trackLength, 1);

%% Preloading all wav samples into memory
waves = {};
for i=1:nbTracks
    if(sceneObjects(i).isBackground)
        path = [inputPath,'background/'];
    else
        path = [inputPath,'event/'];
    end
    sample = dir([path '*wav']);
    sampleId = find(cellfun('isempty',regexpi({sample.name},sceneObjects(i).query,'match'))==0);
    for j=1:length(sampleId)
        % this test is necessary because mp3 conversion does not
        % always maintain exact length, and wavread will crash if asked to
        % read too many samples.
        waves{i}{j}= wavread([path sample(sampleId(j)).name]);
        if(size(waves{i}{j},2)==2)
            waves{i}{j}= mean(waves{i}{j},2);
        end
        waves{i}{j} = waves{i}{j}(:);
    end
end

dominantTrack = zeros(1,trackLength) + 1;
dominantObject = zeros(1,trackLength) + 1;
dominantEnergy = zeros(1,trackLength);
simulatedEBR = zeros(length(sceneSchedule),2);
objNum = 1;

%% Create sceneSchedule
id2remove=[];
for i=1:length(sceneSchedule)
    id = sceneSchedule(i).classId;
    if (sceneSchedule(i).isBackground)
        scale = 1;
        if (i>1)
            scale = adjustScaleForEBR(waves{id}{1}, bg, 1, sceneSchedule(i).ebr, 1, 0.01);
        end
        if (length(waves{id}{1}) < trackLength)
            % need to loop the sample to fill the bg. Linear crossfade
            loop = waves{id}{1};
            fadeLen = 20000;
            fadeIn = transpose(0:1/fadeLen:1);
            fadeOut = transpose(1:-1/fadeLen:0);
            loop(1:fadeLen) = loop(1:fadeLen) .* fadeIn(1:fadeLen);
            loop(length(loop)-fadeLen+1:length(loop)) = loop(length(loop)+1-fadeLen:length(loop)) .* fadeOut(1:fadeLen);
            loop = scale * loop.';
            for l=0:floor(trackLength/(length(loop)-fadeLen))
                t1 = 1+l*(length(loop)-fadeLen);
                t2 = min(length(bg), t1+length(loop)-1);
                tracks(id,t1:t2) = tracks(id,t1:t2) + loop(1:t2-t1+1);
            end
        else
            dim = min(trackLength, length(waves{id}{1}));
            tracks(id,1:dim) = tracks(id,1:dim) + scale*(waves{id}{1}(1:dim).');
        end
        if (i==1)
            bg = tracks(id,:);
        else
            bg = bg+tracks(id,:);
        end
    else
        objNum = objNum+1;
        inst = sceneSchedule(i).instance;
        pos = max(1,floor(sr*sceneSchedule(i).position));
        
        switch timeMode
            case {'replicate'}
                if(length(waves{id}{inst})/sr - sceneSchedule(i).duration > 0.5)
                    endTime = round(sceneSchedule(i).duration*sr);
                else
                    endTime = length(waves{id}{inst});
                end
            case {'abstract'}
                endTime = round(sceneSchedule(i).duration*sr);
            otherwise
                endTime = length(waves{id}{inst});
        end
  
        if endCut
                pos2Test=pos;
        else
            
                pos2Test=pos+endTime-1;
        end
        
        if(pos2Test<round(sceneObjects(sceneSchedule(i).classId).trackLength*sr))
            t2 = min(pos+endTime-1, round(sceneObjects(sceneSchedule(i).classId).trackLength*sr));
            wav2use=waves{id}{inst}(1:t2-pos+1);
            sceneSchedule(i).duration=length(wav2use)/sr;
            scale = adjustScaleForEBR(wav2use, bg, pos, sceneSchedule(i).ebr, 1, 0.01);
            
            [~,bgEnergy] = powspec(bg(pos:t2));
            [~,evEnergy] = powspec(scale*wav2use.');
            labelStart = min(find(bgEnergy<evEnergy));
            labelEnd = max(find(bgEnergy<evEnergy));
            for t=labelStart:labelEnd
                if evEnergy(t)>dominantEnergy(pos+t)
                    dominantObject(pos+round(t*(t2-pos)/length(bgEnergy))) = objNum;
                    dominantEnergy(pos+round(t*(t2-pos)/length(bgEnergy))) = evEnergy(t);
                end
            end
            dominantObject(min(find(dominantObject==objNum)):max(find(dominantObject==objNum))) = objNum;
            dominantTrack(dominantObject==objNum) = id;
            
            tracks(id, pos:t2) = tracks(id, pos:t2) + scale*wav2use.';
            %% Store EBR and event locations
            simulatedEBR(i,1) = ebr(tracks(id, pos:t2),tracks(1, pos:t2));
            simulatedEBR(i,2)=pos;
        else
            id2remove=[id2remove i];
        end
    end
end

sceneSchedule(id2remove)=[];
% checkClassPresence(sceneSchedule,sceneObjects);

save ([outputPath 'annotation/' outputFileName '.mat'],'score', 'sceneObjects', 'sceneSchedule','dominantTrack', 'dominantObject','simulatedEBR');
saveAnnotationTxt(sceneSchedule,outputPath,outputFileName);

w = sum(tracks);
w = w/(max(abs(w))+0.001); %Normalize to [-1,1]

if displayFigures
    cmap = generateColormap(sceneObjects);
    
    timeDomainVisualization(tracks, cmap, 1, 10000, 1, [outputPath 'annotation/' outputFileName,'-timeDomain.png']);
    
    % Producing a colored spectrum visualization
    for i=1:nbTracks
        spec = log(1+abs(spectrogram(tracks(i,:), hanning(2048), 1024, 2048)));
        spec = min(1, spec ./ max(spec(:)));
        spec = spec(1:400,:);
        spec = flipud(spec);
        for colorComp=1:3
            if (i==1)
                img(:,:,colorComp) = cmap(i,colorComp)*spec;
            else
                img(:,:,colorComp) = img(:,:,colorComp)+cmap(i,colorComp)*spec;
            end
        end
    end
    img = img/max(img(:));
    f = figure(2);
    clf;
    set(gca,'YTick', []);
    set(f, 'Position', [0, 0, 2*size(img,1), 600]);
    imagesc(img);
    imwrite(img, [outputPath 'annotation/' outputFileName,'-spectrum.png'], 'png');
    
    
    %% Producing a "piano roll" visualization
    f = figure(3);
    clf;
    set(gca,'YTick', []);
    grid on;
    set(f, 'Position', [0, 0, trackLength/512, 300]);
    for i=1:length(sceneObjects)
        text(0, i+.4, [num2str(sceneObjects(i).classLabel), '  '], 'HorizontalAlignment', 'right');
    end
    
    for i=1:length(sceneSchedule)
        id = sceneSchedule(i).classId;
        if (sceneSchedule(i).isBackground)
            rectangle('Position', [0, id+.2, score.sceneDuration, .6], 'FaceColor', cmap(id,:));
        else
            t1 = sceneSchedule(i).position;
            rectangle('Position', [t1, id+.1, sceneSchedule(i).duration, .8], 'FaceColor', cmap(id,:));
        end
    end
    set(f, 'PaperPositionMode', 'auto');
    print ('-dpng', '-r150', [outputPath 'annotation/' outputFileName,'-piano_roll.png']);
end
end