20_utils.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# default_exp utils"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Utils\n",
    "\n",
    "> some utils :) "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/dperezrada/anaconda3/envs/keywords2vec/lib/python3.7/site-packages/fastprogress/fastprogress.py:102: UserWarning: Couldn't import ipywidgets properly, progress bar will use console behavior\n",
      "  warn(\"Couldn't import ipywidgets properly, progress bar will use console behavior\")\n"
     ]
    }
   ],
   "source": [
    "#export\n",
    "from keywords2vec.imports import *\n",
    "\n",
    "from fastprogress.fastprogress import progress_bar\n",
    "from concurrent.futures import ProcessPoolExecutor, as_completed\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#export\n",
    "\n",
    "# BEGIN From fastai\n",
    "def parallel(func, arr, max_workers=-1):\n",
    "    if max_workers == -1:\n",
    "        max_workers = num_cpus(2)\n",
    "    with ProcessPoolExecutor(max_workers=max_workers) as ex:\n",
    "        futures = [ex.submit(func, arr_el) for arr_el in arr]\n",
    "        results = []\n",
    "        for f in progress_bar(as_completed(futures), total=len(arr)):\n",
    "            results.append(f.result())\n",
    "        return results\n",
    "\n",
    "def num_cpus(n_cpus):\n",
    "    try:\n",
    "        return len(os.sched_getaffinity(0))\n",
    "    except AttributeError:\n",
    "        return os.cpu_count()\n",
    "    if n_cpus > 0:\n",
    "        return n_cpus\n",
    "    \"\"\"Get number of cpus.\"\"\"\n",
    "# END From fastai\n",
    "\n",
    "\n",
    "def open_file(filepath, options):\n",
    "    if filepath[-3:] == \".gz\":\n",
    "        return gzip.open(filepath, options)\n",
    "    return open(filepath, options)\n",
    "\n",
    "\n",
    "def chunk_of_text(_file, chunk_size=-1):\n",
    "    index = 0\n",
    "    if chunk_size == -1:\n",
    "        chunk_size = 200\n",
    "    while True:\n",
    "        line = _file.readline()\n",
    "        if not line:\n",
    "            break\n",
    "        for sentence in line.split(\".\"):\n",
    "            if sentence.strip():\n",
    "                yield sentence.strip()\n",
    "        if index >= chunk_size:\n",
    "            break\n",
    "        index += 1\n",
    "\n",
    "\n",
    "def get_file_chunks(start_index, filepath, lines_chunk, sample_size):\n",
    "    _file = open_file(filepath, 'rt')\n",
    "    texts = []\n",
    "    break_by_sample = False\n",
    "    while True:\n",
    "        next_n_lines = list(chunk_of_text(_file, lines_chunk))\n",
    "        texts.append(\"\\n\".join(next_n_lines) + \"\\n\")\n",
    "        if not next_n_lines:\n",
    "            break\n",
    "        start_index += lines_chunk\n",
    "        if sample_size > 0 and start_index >= sample_size:\n",
    "            break_by_sample = True\n",
    "            break\n",
    "    _file.close()\n",
    "    return (start_index, texts, break_by_sample)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}