filtering.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "%load_ext autoreload\n",
    "%autoreload 2\n",
    "%matplotlib inline"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "The autoreload extension is already loaded. To reload it, use:\n",
      "  %reload_ext autoreload\n"
     ]
    }
   ],
   "source": [
    "import pandas\n",
    "import json\n",
    "from os import path, listdir\n",
    "import itertools\n",
    "from collections import Counter\n",
    "import matplotlib.pyplot as plt\n",
    "from nltk.corpus import stopwords\n",
    "from unidiff import PatchSet, PatchedFile\n",
    "import sys\n",
    "import numpy as np\n",
    "import pickle"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "from commitgen.code import tokenize_code\n",
    "from commitgen.nlp import tokenize_nlp\n",
    "from commitgen.settings import data_path"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def get_added_lines(parsed_diff):\n",
    "    added_lines = []\n",
    "    for modfile in parsed_diff.modified_files:\n",
    "        for hunk in modfile:\n",
    "            added_lines += [line.value for line in hunk if line.is_added]\n",
    "    for addfile in parsed_diff.added_files:\n",
    "        for hunk in addfile:\n",
    "            added_lines += [line.value for line in hunk if line.is_added]\n",
    "    return added_lines\n",
    "\n",
    "def get_removed_lines(parsed_diff):\n",
    "    removed_lines = []\n",
    "    for modfile in parsed_diff.modified_files:\n",
    "        for hunk in modfile:\n",
    "            removed_lines += [line.value for line in hunk if line.is_removed]\n",
    "    for remfile in parsed_diff.removed_files:\n",
    "        for hunk in remfile:\n",
    "            removed_lines += [line.value for line in hunk if line.is_removed]\n",
    "    return removed_lines"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "json_path = path.join(data_path, \"json\")\n",
    "diffs_path = path.join(data_path, \"diffs\")\n",
    "\n",
    "diff_files = listdir(diffs_path)\n",
    "json_files = listdir(json_path)\n",
    "\n",
    "data_dict = {}\n",
    "for filename in json_files:\n",
    "    sha = filename.replace('.json','')\n",
    "    filepath = path.join(json_path, filename)\n",
    "    with open(filepath, 'r') as json_file:\n",
    "        json_data = json.load(json_file)\n",
    "    data_dict[sha] = json_data\n",
    "\n",
    "diff_dict = {}\n",
    "for filename in diff_files:\n",
    "    sha = filename.replace('.diff','')\n",
    "    filepath = path.join(diffs_path, filename)\n",
    "    with open(filepath, 'r') as diff_file:\n",
    "        diff = diff_file.read().decode('utf-8')\n",
    "    parsed_diff = PatchSet(diff.splitlines())\n",
    "    diff_dict[sha] = parsed_diff"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "22426\n"
     ]
    }
   ],
   "source": [
    "# initial filter \n",
    "tokenized_comments = {}\n",
    "for sha, commit in data_dict.items():\n",
    "    t = tokenize_nlp(commit['commit']['message'])\n",
    "    if len(t) > 1 and len(t) <= 30 :\n",
    "        tokenized_comments[sha] = tokenize_nlp(commit['commit']['message'])\n",
    "\n",
    "print len(tokenized_comments)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "filtered = [] \n",
    "for sha, comment in tokenized_comments.items():\n",
    "    parsed_diff = diff_dict[sha] \n",
    "    added_lines   = get_added_lines(parsed_diff)\n",
    "    removed_lines = get_removed_lines(parsed_diff)\n",
    "    code = (added_lines , removed_lines)\n",
    "    filtered.append((sha,comment, code))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "pickle.dump( filtered, open( \"filtered_original.pickle\", \"wb\" ) )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 54,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "# ahora , descartemos los cambios que tienen mas de un archivo (generamos un dataset atomico, atoumico) \n",
    "\n",
    "a = diff_dict.items()\n",
    "\n",
    "atomic_diff = {}\n",
    "for aa in a:\n",
    "    if len(aa[1]) > 1:\n",
    "        atomic_diff[aa[0]] = aa[1]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 55,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "8290"
      ]
     },
     "execution_count": 55,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(atomic_diff)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 56,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "atomic_diff_keys = atomic_diff.keys()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 60,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "# checking *\n",
    "def intersect(a, b):\n",
    "    return list(set(a) & set(b))\n",
    "\n",
    "valid_keys = intersect(atomic_diff_keys, tokenized_comments.keys() )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 61,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "7458"
      ]
     },
     "execution_count": 61,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(valid_keys)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 62,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "filtered_atomic = [] \n",
    "for k in valid_keys:\n",
    "    parsed_diff = atomic_diff[k] \n",
    "    added_lines   = get_added_lines(parsed_diff)\n",
    "    removed_lines = get_removed_lines(parsed_diff)\n",
    "    code = (added_lines , removed_lines)\n",
    "    \n",
    "    comment = tokenized_comments[k]\n",
    "    filtered_atomic.append((k,comment, code))\n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 64,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "pickle.dump( filtered_atomic, open( \"filtered_atomic.pickle\", \"wb\" ) )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 2",
   "language": "python",
   "name": "python2"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 0
}