view notebooks/sensitivity_experiment.ipynb @ 21:206fdeff4641 branch-tests

rerunning notebook

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/homes/mp305/anaconda/lib/python2.7/site-packages/librosa/core/audio.py:33: UserWarning: Could not import scikits.samplerate. Falling back to scipy.signal\n",
      "  warnings.warn('Could not import scikits.samplerate. '\n"
     ]
    }
   ],
   "source": [
    "import numpy as np\n",
    "\n",
    "%matplotlib inline\n",
    "import matplotlib.pyplot as plt\n",
    "\n",
    "%load_ext autoreload\n",
    "%autoreload 2\n",
    "\n",
    "import sys\n",
    "sys.path.append('../')\n",
    "import scripts.load_dataset as load_dataset\n",
    "import scripts.map_and_average as mapper\n",
    "import scripts.classification\n",
    "import scripts.outliers as outliers"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/homes/mp305/anaconda/lib/python2.7/site-packages/IPython/core/interactiveshell.py:2822: DtypeWarning: Columns (0,1,2,4,5,6,7,8,10,11,12,13,14,15,16,17,19,21,22,23,24,25,26,27,29,31,35,38,39,40,41,44,45,48,55,56,58,59,60,61,62,63,64,65,66,67,68,69,70,71,72,73,74,75,76,77,78,79,80,81,82,83,84,85,86,93,95,96) have mixed types. Specify dtype option on import or set low_memory=False.\n",
      "  if self.run_code(code, result):\n"
     ]
    }
   ],
   "source": [
    "OUTPUT_FILES = load_dataset.OUTPUT_FILES\n",
    "n_iters = 10\n",
    "df = load_dataset.sample_dataset(csv_file=load_dataset.METADATA_FILE)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "iteration 0\n",
      "file 0 of 5037\n",
      "loading speech/music segments...\n",
      "extracting onset patterns and mfccs...\n",
      "extracting chroma...\n",
      "load precomputed pitch bihist /import/c4dm-05/mariap/Melodia-melody-8sec/\n",
      "file 1 of 5037\n",
      "loading speech/music segments...\n",
      "extracting onset patterns and mfccs...\n",
      "extracting chroma...\n",
      "load precomputed pitch bihist /import/c4dm-05/mariap/Melodia-melody-8sec/\n",
      "file 2 of 5037\n",
      "loading speech/music segments...\n",
      "extracting onset patterns and mfccs...\n",
      "extracting chroma...\n",
      "load precomputed pitch bihist /import/c4dm-05/mariap/Melodia-melody-8sec/\n",
      "file 3 of 5037\n",
      "loading speech/music segments...\n",
      "extracting onset patterns and mfccs...\n",
      "extracting chroma...\n",
      "load precomputed pitch bihist /import/c4dm-05/mariap/Melodia-melody-8sec/\n",
      "file 4 of 5037\n",
      "loading speech/music segments...\n",
      "extracting onset patterns and mfccs...\n",
      "extracting chroma...\n",
      "load precomputed pitch bihist /import/c4dm-05/mariap/Melodia-melody-8sec/\n",
      "file 5 of 5037\n",
      "loading speech/music segments...\n",
      "extracting onset patterns and mfccs...\n",
      "extracting chroma...\n",
      "load precomputed pitch bihist /import/c4dm-05/mariap/Melodia-melody-8sec/\n",
      "file 6 of 5037\n",
      "loading speech/music segments...\n",
      "extracting onset patterns and mfccs...\n",
      "extracting chroma...\n",
      "load precomputed pitch bihist /import/c4dm-05/mariap/Melodia-melody-8sec/\n",
      "file 7 of 5037\n",
      "loading speech/music segments...\n",
      "extracting onset patterns and mfccs...\n",
      "extracting chroma...\n",
      "load precomputed pitch bihist /import/c4dm-05/mariap/Melodia-melody-8sec/\n",
      "file 8 of 5037\n",
      "loading speech/music segments...\n",
      "extracting onset patterns and mfccs...\n",
      "extracting chroma...\n",
      "load precomputed pitch bihist /import/c4dm-05/mariap/Melodia-melody-8sec/\n",
      "file 9 of 5037\n",
      "loading speech/music segments...\n",
      "extracting onset patterns and mfccs...\n",
      "extracting chroma...\n",
      "load precomputed pitch bihist /import/c4dm-05/mariap/Melodia-melody-8sec/\n",
      "file 10 of 5037\n",
      "loading speech/music segments...\n",
      "extracting onset patterns and mfccs...\n",
      "extracting chroma...\n",
      "load precomputed pitch bihist /import/c4dm-05/mariap/Melodia-melody-8sec/\n",
      "file 11 of 5037\n",
      "loading speech/music segments...\n",
      "extracting onset patterns and mfccs...\n",
      "extracting chroma...\n",
      "load precomputed pitch bihist /import/c4dm-05/mariap/Melodia-melody-8sec/\n",
      "file 12 of 5037\n",
      "loading speech/music segments...\n",
      "extracting onset patterns and mfccs...\n",
      "extracting chroma...\n",
      "load precomputed pitch bihist /import/c4dm-05/mariap/Melodia-melody-8sec/\n",
      "file 13 of 5037\n",
      "loading speech/music segments...\n",
      "extracting onset patterns and mfccs...\n",
      "extracting chroma...\n",
      "load precomputed pitch bihist /import/c4dm-05/mariap/Melodia-melody-8sec/\n",
      "file 14 of 5037\n",
      "loading speech/music segments...\n",
      "extracting onset patterns and mfccs...\n",
      "extracting chroma...\n",
      "load precomputed pitch bihist /import/c4dm-05/mariap/Melodia-melody-8sec/\n",
      "file 15 of 5037\n",
      "loading speech/music segments...\n",
      "extracting onset patterns and mfccs...\n",
      "extracting chroma...\n",
      "load precomputed pitch bihist /import/c4dm-05/mariap/Melodia-melody-8sec/\n",
      "file 16 of 5037\n",
      "loading speech/music segments...\n",
      "extracting onset patterns and mfccs...\n",
      "file 17 of 5037\n",
      "loading speech/music segments...\n",
      "extracting onset patterns and mfccs...\n",
      "extracting chroma...\n",
      "load precomputed pitch bihist /import/c4dm-05/mariap/Melodia-melody-8sec/\n",
      "file 18 of 5037\n",
      "loading speech/music segments...\n",
      "extracting onset patterns and mfccs...\n",
      "extracting chroma...\n",
      "load precomputed pitch bihist /import/c4dm-05/mariap/Melodia-melody-8sec/\n",
      "file 19 of 5037\n",
      "loading speech/music segments...\n",
      "extracting onset patterns and mfccs...\n",
      "extracting chroma...\n",
      "load precomputed pitch bihist /import/c4dm-05/mariap/Melodia-melody-8sec/\n",
      "file 20 of 5037\n",
      "loading speech/music segments...\n",
      "extracting onset patterns and mfccs...\n",
      "extracting chroma...\n",
      "load precomputed pitch bihist /import/c4dm-05/mariap/Melodia-melody-8sec/\n",
      "file 21 of 5037\n",
      "loading speech/music segments...\n",
      "extracting onset patterns and mfccs...\n",
      "extracting chroma...\n",
      "load precomputed pitch bihist /import/c4dm-05/mariap/Melodia-melody-8sec/\n",
      "file 22 of 5037\n",
      "loading speech/music segments...\n",
      "file 23 of 5037\n",
      "loading speech/music segments...\n",
      "extracting onset patterns and mfccs...\n",
      "extracting chroma...\n",
      "load precomputed pitch bihist /import/c4dm-05/mariap/Melodia-melody-8sec/\n",
      "file 24 of 5037\n",
      "loading speech/music segments...\n",
      "extracting onset patterns and mfccs...\n",
      "extracting chroma...\n",
      "load precomputed pitch bihist /import/c4dm-05/mariap/Melodia-melody-8sec/\n",
      "file 25 of 5037\n",
      "loading speech/music segments...\n",
      "extracting onset patterns and mfccs...\n",
      "extracting chroma...\n",
      "load precomputed pitch bihist /import/c4dm-05/mariap/Melodia-melody-8sec/\n",
      "file 26 of 5037\n",
      "loading speech/music segments...\n",
      "extracting onset patterns and mfccs...\n",
      "extracting chroma...\n",
      "load precomputed pitch bihist /import/c4dm-05/mariap/Melodia-melody-8sec/\n",
      "file 27 of 5037\n",
      "loading speech/music segments...\n",
      "extracting onset patterns and mfccs...\n",
      "extracting chroma...\n",
      "load precomputed pitch bihist /import/c4dm-05/mariap/Melodia-melody-8sec/\n",
      "file 28 of 5037\n",
      "loading speech/music segments...\n",
      "extracting onset patterns and mfccs...\n",
      "extracting chroma...\n",
      "load precomputed pitch bihist /import/c4dm-05/mariap/Melodia-melody-8sec/\n",
      "file 29 of 5037\n",
      "loading speech/music segments...\n",
      "extracting onset patterns and mfccs...\n",
      "extracting chroma...\n",
      "load precomputed pitch bihist /import/c4dm-05/mariap/Melodia-melody-8sec/\n",
      "file 30 of 5037\n",
      "loading speech/music segments...\n",
      "extracting onset patterns and mfccs...\n",
      "extracting chroma...\n",
      "load precomputed pitch bihist /import/c4dm-05/mariap/Melodia-melody-8sec/\n",
      "file 31 of 5037\n",
      "loading speech/music segments...\n",
      "extracting onset patterns and mfccs...\n",
      "extracting chroma...\n",
      "load precomputed pitch bihist /import/c4dm-05/mariap/Melodia-melody-8sec/\n",
      "file 32 of 5037\n",
      "loading speech/music segments...\n",
      "extracting onset patterns and mfccs...\n",
      "extracting chroma...\n",
      "load precomputed pitch bihist /import/c4dm-05/mariap/Melodia-melody-8sec/\n",
      "file 33 of 5037\n",
      "loading speech/music segments...\n",
      "extracting onset patterns and mfccs...\n",
      "extracting chroma...\n",
      "load precomputed pitch bihist /import/c4dm-05/mariap/Melodia-melody-8sec/\n",
      "file 34 of 5037\n",
      "loading speech/music segments...\n",
      "extracting onset patterns and mfccs...\n",
      "extracting chroma...\n",
      "load precomputed pitch bihist /import/c4dm-05/mariap/Melodia-melody-8sec/\n",
      "file 35 of 5037\n",
      "loading speech/music segments...\n",
      "extracting onset patterns and mfccs...\n",
      "extracting chroma...\n",
      "load precomputed pitch bihist /import/c4dm-05/mariap/Melodia-melody-8sec/\n",
      "file 36 of 5037\n",
      "loading speech/music segments...\n",
      "extracting onset patterns and mfccs...\n",
      "extracting chroma...\n",
      "load precomputed pitch bihist /import/c4dm-05/mariap/Melodia-melody-8sec/\n",
      "file 37 of 5037\n",
      "loading speech/music segments...\n",
      "extracting onset patterns and mfccs...\n",
      "extracting chroma...\n",
      "load precomputed pitch bihist /import/c4dm-05/mariap/Melodia-melody-8sec/\n",
      "file 38 of 5037\n",
      "loading speech/music segments...\n",
      "extracting onset patterns and mfccs...\n",
      "extracting chroma...\n",
      "load precomputed pitch bihist /import/c4dm-05/mariap/Melodia-melody-8sec/\n",
      "file 39 of 5037\n",
      "loading speech/music segments...\n",
      "extracting onset patterns and mfccs...\n",
      "extracting chroma...\n",
      "load precomputed pitch bihist /import/c4dm-05/mariap/Melodia-melody-8sec/\n",
      "file 40 of 5037\n",
      "loading speech/music segments...\n",
      "extracting onset patterns and mfccs...\n",
      "extracting chroma...\n",
      "load precomputed pitch bihist /import/c4dm-05/mariap/Melodia-melody-8sec/\n",
      "file 41 of 5037\n",
      "loading speech/music segments...\n",
      "extracting onset patterns and mfccs...\n",
      "extracting chroma...\n",
      "load precomputed pitch bihist /import/c4dm-05/mariap/Melodia-melody-8sec/\n",
      "file 42 of 5037\n",
      "loading speech/music segments...\n",
      "extracting onset patterns and mfccs...\n",
      "extracting chroma...\n",
      "load precomputed pitch bihist /import/c4dm-05/mariap/Melodia-melody-8sec/\n",
      "file 43 of 5037\n",
      "loading speech/music segments...\n",
      "extracting onset patterns and mfccs...\n",
      "extracting chroma...\n",
      "load precomputed pitch bihist /import/c4dm-05/mariap/Melodia-melody-8sec/\n",
      "file 44 of 5037\n",
      "loading speech/music segments...\n",
      "extracting onset patterns and mfccs...\n",
      "extracting chroma...\n",
      "load precomputed pitch bihist /import/c4dm-05/mariap/Melodia-melody-8sec/\n",
      "file 45 of 5037\n",
      "loading speech/music segments...\n",
      "extracting onset patterns and mfccs...\n",
      "extracting chroma...\n",
      "load precomputed pitch bihist /import/c4dm-05/mariap/Melodia-melody-8sec/\n",
      "file 46 of 5037\n",
      "loading speech/music segments...\n",
      "extracting onset patterns and mfccs...\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "file 47 of 5037\n",
      "loading speech/music segments...\n",
      "extracting onset patterns and mfccs...\n",
      "file 48 of 5037\n",
      "loading speech/music segments...\n",
      "extracting onset patterns and mfccs...\n",
      "file 49 of 5037\n",
      "loading speech/music segments...\n",
      "extracting onset patterns and mfccs...\n",
      "file 50 of 5037\n",
      "loading speech/music segments...\n",
      "extracting onset patterns and mfccs...\n",
      "file 51 of 5037\n",
      "loading speech/music segments...\n",
      "extracting onset patterns and mfccs...\n",
      "file 52 of 5037\n",
      "loading speech/music segments...\n",
      "extracting onset patterns and mfccs...\n",
      "file 53 of 5037\n",
      "loading speech/music segments...\n",
      "extracting onset patterns and mfccs...\n",
      "file 54 of 5037\n",
      "loading speech/music segments...\n",
      "extracting onset patterns and mfccs...\n",
      "file 55 of 5037\n",
      "loading speech/music segments...\n",
      "extracting onset patterns and mfccs...\n",
      "extracting chroma...\n",
      "load precomputed pitch bihist /import/c4dm-05/mariap/Melodia-melody-8sec/\n",
      "file 56 of 5037\n",
      "loading speech/music segments...\n",
      "extracting onset patterns and mfccs...\n",
      "extracting chroma...\n",
      "load precomputed pitch bihist /import/c4dm-05/mariap/Melodia-melody-8sec/\n",
      "file 57 of 5037\n",
      "loading speech/music segments...\n",
      "extracting onset patterns and mfccs...\n",
      "extracting chroma...\n",
      "load precomputed pitch bihist /import/c4dm-05/mariap/Melodia-melody-8sec/\n",
      "file 58 of 5037\n",
      "loading speech/music segments...\n",
      "extracting onset patterns and mfccs...\n",
      "extracting chroma...\n",
      "load precomputed pitch bihist /import/c4dm-05/mariap/Melodia-melody-8sec/\n",
      "file 59 of 5037\n",
      "loading speech/music segments...\n",
      "extracting onset patterns and mfccs...\n",
      "extracting chroma...\n",
      "load precomputed pitch bihist /import/c4dm-05/mariap/Melodia-melody-8sec/\n",
      "file 60 of 5037\n",
      "loading speech/music segments...\n",
      "extracting onset patterns and mfccs...\n",
      "extracting chroma...\n",
      "load precomputed pitch bihist /import/c4dm-05/mariap/Melodia-melody-8sec/\n",
      "file 61 of 5037\n",
      "loading speech/music segments...\n",
      "extracting onset patterns and mfccs...\n",
      "extracting chroma...\n",
      "load precomputed pitch bihist /import/c4dm-05/mariap/Melodia-melody-8sec/\n",
      "file 62 of 5037\n",
      "loading speech/music segments...\n",
      "extracting onset patterns and mfccs...\n",
      "extracting chroma...\n",
      "load precomputed pitch bihist /import/c4dm-05/mariap/Melodia-melody-8sec/\n",
      "file 63 of 5037\n",
      "loading speech/music segments...\n",
      "extracting onset patterns and mfccs...\n",
      "extracting chroma...\n",
      "load precomputed pitch bihist /import/c4dm-05/mariap/Melodia-melody-8sec/\n",
      "file 64 of 5037\n",
      "loading speech/music segments...\n",
      "extracting onset patterns and mfccs...\n",
      "extracting chroma...\n",
      "load precomputed pitch bihist /import/c4dm-05/mariap/Melodia-melody-8sec/\n",
      "file 65 of 5037\n",
      "loading speech/music segments...\n",
      "extracting onset patterns and mfccs...\n",
      "extracting chroma...\n",
      "load precomputed pitch bihist /import/c4dm-05/mariap/Melodia-melody-8sec/\n",
      "file 66 of 5037\n",
      "loading speech/music segments...\n",
      "extracting onset patterns and mfccs...\n",
      "extracting chroma...\n",
      "load precomputed pitch bihist /import/c4dm-05/mariap/Melodia-melody-8sec/\n",
      "file 67 of 5037\n",
      "loading speech/music segments...\n",
      "extracting onset patterns and mfccs...\n",
      "extracting chroma...\n",
      "load precomputed pitch bihist /import/c4dm-05/mariap/Melodia-melody-8sec/\n",
      "file 68 of 5037\n",
      "loading speech/music segments...\n",
      "extracting onset patterns and mfccs...\n",
      "extracting chroma...\n",
      "load precomputed pitch bihist /import/c4dm-05/mariap/Melodia-melody-8sec/\n",
      "file 69 of 5037\n",
      "loading speech/music segments...\n",
      "extracting onset patterns and mfccs...\n",
      "extracting chroma...\n",
      "load precomputed pitch bihist /import/c4dm-05/mariap/Melodia-melody-8sec/\n",
      "file 70 of 5037\n",
      "loading speech/music segments...\n",
      "extracting onset patterns and mfccs...\n",
      "extracting chroma...\n",
      "load precomputed pitch bihist /import/c4dm-05/mariap/Melodia-melody-8sec/\n",
      "file 71 of 5037\n",
      "loading speech/music segments...\n",
      "extracting onset patterns and mfccs...\n",
      "extracting chroma...\n",
      "load precomputed pitch bihist /import/c4dm-05/mariap/Melodia-melody-8sec/\n",
      "file 72 of 5037\n",
      "loading speech/music segments...\n",
      "extracting onset patterns and mfccs...\n",
      "extracting chroma...\n",
      "load precomputed pitch bihist /import/c4dm-05/mariap/Melodia-melody-8sec/\n",
      "file 73 of 5037\n",
      "loading speech/music segments...\n",
      "extracting onset patterns and mfccs...\n",
      "extracting chroma...\n",
      "load precomputed pitch bihist /import/c4dm-05/mariap/Melodia-melody-8sec/\n",
      "file 74 of 5037\n",
      "loading speech/music segments...\n",
      "extracting onset patterns and mfccs...\n",
      "extracting chroma...\n",
      "load precomputed pitch bihist /import/c4dm-05/mariap/Melodia-melody-8sec/\n",
      "file 75 of 5037\n",
      "loading speech/music segments...\n",
      "extracting onset patterns and mfccs...\n",
      "extracting chroma...\n",
      "load precomputed pitch bihist /import/c4dm-05/mariap/Melodia-melody-8sec/\n",
      "file 76 of 5037\n",
      "loading speech/music segments...\n",
      "extracting onset patterns and mfccs...\n",
      "extracting chroma...\n",
      "load precomputed pitch bihist /import/c4dm-05/mariap/Melodia-melody-8sec/\n",
      "file 77 of 5037\n",
      "loading speech/music segments...\n",
      "extracting onset patterns and mfccs...\n",
      "extracting chroma...\n",
      "load precomputed pitch bihist /import/c4dm-05/mariap/Melodia-melody-8sec/\n",
      "file 78 of 5037\n",
      "loading speech/music segments...\n",
      "extracting onset patterns and mfccs...\n",
      "extracting chroma...\n",
      "load precomputed pitch bihist /import/c4dm-05/mariap/Melodia-melody-8sec/\n",
      "file 79 of 5037\n",
      "loading speech/music segments...\n",
      "extracting onset patterns and mfccs...\n",
      "extracting chroma...\n",
      "load precomputed pitch bihist /import/c4dm-05/mariap/Melodia-melody-8sec/\n",
      "file 80 of 5037\n",
      "loading speech/music segments...\n",
      "extracting onset patterns and mfccs...\n",
      "extracting chroma...\n",
      "load precomputed pitch bihist /import/c4dm-05/mariap/Melodia-melody-8sec/\n",
      "file 81 of 5037\n",
      "loading speech/music segments...\n",
      "extracting onset patterns and mfccs...\n",
      "extracting chroma...\n",
      "load precomputed pitch bihist /import/c4dm-05/mariap/Melodia-melody-8sec/\n",
      "file 82 of 5037\n",
      "loading speech/music segments...\n",
      "extracting onset patterns and mfccs...\n",
      "extracting chroma...\n",
      "load precomputed pitch bihist /import/c4dm-05/mariap/Melodia-melody-8sec/\n",
      "file 83 of 5037\n",
      "loading speech/music segments...\n",
      "extracting onset patterns and mfccs...\n",
      "extracting chroma...\n",
      "load precomputed pitch bihist /import/c4dm-05/mariap/Melodia-melody-8sec/\n",
      "file 84 of 5037\n",
      "loading speech/music segments...\n",
      "extracting onset patterns and mfccs...\n",
      "extracting chroma...\n",
      "load precomputed pitch bihist /import/c4dm-05/mariap/Melodia-melody-8sec/\n",
      "file 85 of 5037\n",
      "loading speech/music segments...\n",
      "extracting onset patterns and mfccs...\n",
      "extracting chroma...\n",
      "load precomputed pitch bihist /import/c4dm-05/mariap/Melodia-melody-8sec/\n",
      "file 86 of 5037\n",
      "loading speech/music segments...\n",
      "extracting onset patterns and mfccs...\n",
      "extracting chroma...\n",
      "load precomputed pitch bihist /import/c4dm-05/mariap/Melodia-melody-8sec/\n",
      "file 87 of 5037\n",
      "loading speech/music segments...\n",
      "extracting onset patterns and mfccs...\n",
      "extracting chroma...\n",
      "load precomputed pitch bihist /import/c4dm-05/mariap/Melodia-melody-8sec/\n",
      "file 88 of 5037\n",
      "loading speech/music segments...\n",
      "extracting onset patterns and mfccs...\n",
      "extracting chroma...\n",
      "load precomputed pitch bihist /import/c4dm-05/mariap/Melodia-melody-8sec/\n",
      "file 89 of 5037\n",
      "loading speech/music segments...\n",
      "extracting onset patterns and mfccs...\n",
      "extracting chroma...\n",
      "load precomputed pitch bihist /import/c4dm-05/mariap/Melodia-melody-8sec/\n",
      "file 90 of 5037\n",
      "loading speech/music segments...\n",
      "extracting onset patterns and mfccs...\n",
      "extracting chroma...\n",
      "load precomputed pitch bihist /import/c4dm-05/mariap/Melodia-melody-8sec/\n",
      "file 91 of 5037\n",
      "loading speech/music segments...\n",
      "extracting onset patterns and mfccs...\n",
      "extracting chroma...\n",
      "load precomputed pitch bihist /import/c4dm-05/mariap/Melodia-melody-8sec/\n",
      "file 92 of 5037\n",
      "loading speech/music segments...\n",
      "extracting onset patterns and mfccs...\n",
      "extracting chroma...\n",
      "load precomputed pitch bihist /import/c4dm-05/mariap/Melodia-melody-8sec/\n",
      "file 93 of 5037\n",
      "loading speech/music segments...\n",
      "extracting onset patterns and mfccs...\n",
      "extracting chroma...\n",
      "load precomputed pitch bihist /import/c4dm-05/mariap/Melodia-melody-8sec/\n",
      "file 94 of 5037\n",
      "loading speech/music segments...\n",
      "extracting onset patterns and mfccs...\n",
      "extracting chroma...\n",
      "load precomputed pitch bihist /import/c4dm-05/mariap/Melodia-melody-8sec/\n",
      "file 95 of 5037\n",
      "loading speech/music segments...\n",
      "extracting onset patterns and mfccs...\n",
      "extracting chroma...\n",
      "load precomputed pitch bihist /import/c4dm-05/mariap/Melodia-melody-8sec/\n",
      "file 96 of 5037\n",
      "loading speech/music segments...\n",
      "extracting onset patterns and mfccs...\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "extracting chroma...\n",
      "load precomputed pitch bihist /import/c4dm-05/mariap/Melodia-melody-8sec/\n",
      "file 97 of 5037\n",
      "loading speech/music segments...\n",
      "extracting onset patterns and mfccs...\n",
      "extracting chroma...\n",
      "load precomputed pitch bihist /import/c4dm-05/mariap/Melodia-melody-8sec/\n",
      "file 98 of 5037\n",
      "loading speech/music segments...\n",
      "extracting onset patterns and mfccs...\n",
      "extracting chroma...\n",
      "load precomputed pitch bihist /import/c4dm-05/mariap/Melodia-melody-8sec/\n",
      "file 99 of 5037\n",
      "loading speech/music segments...\n",
      "extracting onset patterns and mfccs...\n",
      "extracting chroma...\n",
      "load precomputed pitch bihist /import/c4dm-05/mariap/Melodia-melody-8sec/\n",
      "file 100 of 5037\n",
      "loading speech/music segments...\n",
      "extracting onset patterns and mfccs...\n",
      "extracting chroma...\n",
      "load precomputed pitch bihist /import/c4dm-05/mariap/Melodia-melody-8sec/\n",
      "file 101 of 5037\n",
      "loading speech/music segments...\n",
      "extracting onset patterns and mfccs...\n",
      "extracting chroma...\n",
      "load precomputed pitch bihist /import/c4dm-05/mariap/Melodia-melody-8sec/\n",
      "file 102 of 5037\n",
      "loading speech/music segments...\n",
      "extracting onset patterns and mfccs...\n",
      "extracting chroma...\n",
      "load precomputed pitch bihist /import/c4dm-05/mariap/Melodia-melody-8sec/\n",
      "file 103 of 5037\n",
      "loading speech/music segments...\n",
      "extracting onset patterns and mfccs...\n",
      "extracting chroma...\n",
      "load precomputed pitch bihist /import/c4dm-05/mariap/Melodia-melody-8sec/\n",
      "file 104 of 5037\n",
      "loading speech/music segments...\n",
      "extracting onset patterns and mfccs...\n",
      "extracting chroma...\n",
      "load precomputed pitch bihist /import/c4dm-05/mariap/Melodia-melody-8sec/\n",
      "file 105 of 5037\n",
      "loading speech/music segments...\n",
      "extracting onset patterns and mfccs...\n",
      "extracting chroma...\n",
      "load precomputed pitch bihist /import/c4dm-05/mariap/Melodia-melody-8sec/\n",
      "file 106 of 5037\n",
      "loading speech/music segments...\n",
      "extracting onset patterns and mfccs...\n",
      "extracting chroma...\n",
      "load precomputed pitch bihist /import/c4dm-05/mariap/Melodia-melody-8sec/\n",
      "file 107 of 5037\n",
      "loading speech/music segments...\n",
      "extracting onset patterns and mfccs...\n",
      "extracting chroma...\n",
      "load precomputed pitch bihist /import/c4dm-05/mariap/Melodia-melody-8sec/\n",
      "file 108 of 5037\n",
      "loading speech/music segments...\n",
      "extracting onset patterns and mfccs...\n",
      "extracting chroma...\n",
      "load precomputed pitch bihist /import/c4dm-05/mariap/Melodia-melody-8sec/\n",
      "file 109 of 5037\n",
      "loading speech/music segments...\n",
      "extracting onset patterns and mfccs...\n",
      "extracting chroma...\n",
      "load precomputed pitch bihist /import/c4dm-05/mariap/Melodia-melody-8sec/\n",
      "file 110 of 5037\n",
      "loading speech/music segments...\n",
      "extracting onset patterns and mfccs...\n",
      "extracting chroma...\n",
      "load precomputed pitch bihist /import/c4dm-05/mariap/Melodia-melody-8sec/\n",
      "file 111 of 5037\n",
      "loading speech/music segments...\n",
      "extracting onset patterns and mfccs...\n",
      "extracting chroma...\n",
      "load precomputed pitch bihist /import/c4dm-05/mariap/Melodia-melody-8sec/\n",
      "file 112 of 5037\n",
      "loading speech/music segments...\n",
      "extracting onset patterns and mfccs...\n",
      "extracting chroma...\n",
      "load precomputed pitch bihist /import/c4dm-05/mariap/Melodia-melody-8sec/\n",
      "file 113 of 5037\n",
      "loading speech/music segments...\n",
      "extracting onset patterns and mfccs...\n",
      "extracting chroma...\n",
      "load precomputed pitch bihist /import/c4dm-05/mariap/Melodia-melody-8sec/\n",
      "file 114 of 5037\n",
      "loading speech/music segments...\n",
      "extracting onset patterns and mfccs...\n",
      "extracting chroma...\n",
      "load precomputed pitch bihist /import/c4dm-05/mariap/Melodia-melody-8sec/\n",
      "file 115 of 5037\n",
      "loading speech/music segments...\n",
      "extracting onset patterns and mfccs...\n",
      "extracting chroma...\n",
      "load precomputed pitch bihist /import/c4dm-05/mariap/Melodia-melody-8sec/\n",
      "file 116 of 5037\n",
      "loading speech/music segments...\n",
      "extracting onset patterns and mfccs...\n",
      "extracting chroma...\n",
      "load precomputed pitch bihist /import/c4dm-05/mariap/Melodia-melody-8sec/\n",
      "file 117 of 5037\n",
      "loading speech/music segments...\n",
      "extracting onset patterns and mfccs...\n",
      "extracting chroma...\n",
      "load precomputed pitch bihist /import/c4dm-05/mariap/Melodia-melody-8sec/\n",
      "file 118 of 5037\n",
      "loading speech/music segments...\n",
      "extracting onset patterns and mfccs...\n",
      "extracting chroma...\n",
      "load precomputed pitch bihist /import/c4dm-05/mariap/Melodia-melody-8sec/\n",
      "file 119 of 5037\n",
      "loading speech/music segments...\n",
      "extracting onset patterns and mfccs...\n",
      "extracting chroma...\n",
      "load precomputed pitch bihist /import/c4dm-05/mariap/Melodia-melody-8sec/\n",
      "file 120 of 5037\n",
      "loading speech/music segments...\n",
      "extracting onset patterns and mfccs...\n",
      "extracting chroma...\n",
      "load precomputed pitch bihist /import/c4dm-05/mariap/Melodia-melody-8sec/\n",
      "file 121 of 5037\n",
      "loading speech/music segments...\n",
      "extracting onset patterns and mfccs...\n",
      "extracting chroma...\n",
      "load precomputed pitch bihist /import/c4dm-05/mariap/Melodia-melody-8sec/\n",
      "file 122 of 5037\n",
      "loading speech/music segments...\n",
      "extracting onset patterns and mfccs...\n",
      "extracting chroma...\n",
      "load precomputed pitch bihist /import/c4dm-05/mariap/Melodia-melody-8sec/\n",
      "file 123 of 5037\n",
      "loading speech/music segments...\n",
      "extracting onset patterns and mfccs...\n",
      "extracting chroma...\n",
      "load precomputed pitch bihist /import/c4dm-05/mariap/Melodia-melody-8sec/\n",
      "file 124 of 5037\n",
      "loading speech/music segments...\n",
      "extracting onset patterns and mfccs...\n",
      "extracting chroma...\n",
      "load precomputed pitch bihist /import/c4dm-05/mariap/Melodia-melody-8sec/\n",
      "file 125 of 5037\n",
      "loading speech/music segments...\n",
      "extracting onset patterns and mfccs...\n"
     ]
    }
   ],
   "source": [
    "for n in range(n_iters):\n",
    "    print \"iteration %d\" % n\n",
    "    load_dataset.OUTPUT_FILES = [output_file.split('.pickle')[0]+'_'+str(n)+'.pickle' for \n",
    "                                 output_file in OUTPUT_FILES]\n",
    "    load_dataset.features_for_train_test_sets(df, write_output=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "for n in range(n_iters):\n",
    "    print \"iteration %d\" % n\n",
    "    \n",
    "    print \"mapping...\"\n",
    "    mapper.INPUT_FILES = [output_file.split('.pickle')[0]+'_'+str(n)+'.pickle' for \n",
    "                                 output_file in OUTPUT_FILES]\n",
    "    _, _, ldadata_list, _, _, Y, Yaudio = mapper.lda_map_and_average_frames(min_variance=0.99)\n",
    "    X = np.concatenate(ldadata_list)\n",
    "    \n",
    "    # classification and confusion\n",
    "    print \"classifying...\"\n",
    "    traininds, testinds = classification.get_train_test_indices()\n",
    "    X_train, Y_train, X_test, Y_test = classification.get_train_test_sets(X, Y, traininds, testinds)\n",
    "    accuracy, _ = classification.confusion_matrix(X_train, Y_train, X_test, Y_test, saveCF=False, plots=False)\n",
    "    print accuracy\n",
    "    \n",
    "    # outliers\n",
    "    print \"detecting outliers...\"\n",
    "    ddf = outliers.load_metadata(Yaudio, metadata_file=load_dataset.METADATA_FILE)\n",
    "    df_global, threshold, MD = get_outliers_df(X, Y, chi2thr=0.999)\n",
    "    print_most_least_outliers_topN(df_global, N=10)\n",
    "    \n",
    "    # write output\n",
    "    print \"writing file\"\n",
    "    df_global.to_csv('../data/outliers_'+str(n)+'.csv', index=False)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 2",
   "language": "python",
   "name": "python2"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.12"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}