notebooks/api/0.8/09-blob-storage.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "SYFT_VERSION = \">=0.8.2.b0,<0.9\"\n",
    "package_string = f'\"syft{SYFT_VERSION}\"'\n",
    "# %pip install {package_string} -q"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# stdlib\n",
    "import json\n",
    "\n",
    "# syft absolute\n",
    "import syft as sy\n",
    "from syft import autocache\n",
    "\n",
    "sy.requires(SYFT_VERSION)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "node = sy.orchestra.launch(\n",
    "    name=\"test-domain-1\",\n",
    "    dev_mode=True,\n",
    "    reset=True,\n",
    "    create_producer=True,\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "domain_client = node.login(email=\"info@openmined.org\", password=\"changethis\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Uploading a Blobfile from multiple local files"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "a_dataset_url = (\n",
    "    \"https://raw.githubusercontent.com/OpenMined/datasets/main/testdatasyft/a.json\"\n",
    ")\n",
    "a_file = autocache(a_dataset_url, \"json\")\n",
    "a_file"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "b_dataset_url = (\n",
    "    \"https://raw.githubusercontent.com/OpenMined/datasets/main/testdatasyft/b.json\"\n",
    ")\n",
    "b_file = autocache(b_dataset_url, \"json\")\n",
    "b_file"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "data_ptr = domain_client.upload_files([a_file, b_file])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "data_ptr[0].read()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Iterating over lines of a file"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "x_dataset_url = (\n",
    "    \"https://raw.githubusercontent.com/OpenMined/datasets/main/testdatasyft/lines.jsonl\"\n",
    ")\n",
    "x_file = autocache(x_dataset_url, \"jsonl\")\n",
    "x_file"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "lines_file_pptr = domain_client.upload_files(x_file)[0].syft_action_data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "res = []\n",
    "for y in lines_file_pptr.iter_lines():\n",
    "    d = json.loads(y)\n",
    "    print(d)\n",
    "    res.append(d[\"x\"])\n",
    "assert sum(res) == 3"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Creating a dataset with a list of files as Asset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "ds = sy.Dataset(\n",
    "    name=\"my dataset\",\n",
    "    asset_list=[sy.Asset(name=\"data\", data=data_ptr, mock=sy.ActionObject.empty())],\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "domain_client.upload_dataset(ds)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Running a syft_function on the files"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "@sy.syft_function_single_use(files=data_ptr)\n",
    "def sum_trade_value_mil(files):\n",
    "    # stdlib\n",
    "    import json\n",
    "\n",
    "    a_file, b_file = files\n",
    "    data_1 = json.loads(a_file.read())\n",
    "    data_2 = json.loads(b_file.read())\n",
    "    res = data_1[\"x\"] + data_2[\"x\"]\n",
    "    return res"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "assert sum_trade_value_mil(files=data_ptr).syft_action_data == 3"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# From a single file"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "single_data_ptr = domain_client.upload_files(a_file)\n",
    "single_data_ptr"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# From a folder"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "if False:\n",
    "    domain_client.upload_files(\"./path/to/folder\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Cleanup local domain server"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "if node.node_type.value == \"python\":\n",
    "    node.land()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.16"
  },
  "toc": {
   "base_numbering": 1,
   "nav_menu": {},
   "number_sections": true,
   "sideBar": true,
   "skip_h1_title": false,
   "title_cell": "Table of Contents",
   "title_sidebar": "Contents",
   "toc_cell": false,
   "toc_position": {},
   "toc_section_display": true,
   "toc_window_display": false
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}