scrapy the subscript.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "f6b0f4e4-38ba-440e-a7a2-0ba161631a36",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2024-03-16T07:41:41.674914300Z",
     "start_time": "2024-03-16T07:41:41.644836800Z"
    }
   },
   "outputs": [],
   "source": [
    "import urllib3\n",
    "from bs4 import BeautifulSoup\n",
    "import re\n",
    "import json"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "2c90bdfc2f79bac5",
   "metadata": {
    "collapsed": false,
    "ExecuteTime": {
     "end_time": "2024-03-16T07:42:07.571408400Z",
     "start_time": "2024-03-16T07:42:06.863388400Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "获取TFTS主页状态： 200\n"
     ]
    }
   ],
   "source": [
    "url = \"https://www.gsb.stanford.edu/business-podcasts/think-fast-talk-smart-podcast\"\n",
    "# 获取TFTS主页\n",
    "\n",
    "http = urllib3.PoolManager()\n",
    "r = http.request('GET', url)\n",
    "print(\"获取TFTS主页状态：\", r.status)\n",
    "\n",
    "# 解析主页\n",
    "soup = BeautifulSoup(r.data, 'html.parser')\n",
    "soup.title.string\n",
    "\n",
    "# 定位大体位置\n",
    "\n",
    "# target_pos = soup.find('h3', class_='views-field views-field-title')\n",
    "target_pos = soup.find('div', class_='idea-stories-list-view__content-wrapper').h3\n",
    "\n",
    "# 解析标题和网址\n",
    "the_episode_title = str(target_pos.a.string)\n",
    "the_episode_url = \"https://www.gsb.stanford.edu\" + str(target_pos.a[\"href\"])\n",
    "print(\"文章标题：\", the_episode_title)\n",
    "print(\"文章网址：\", the_episode_url)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "id": "86380740-2bb5-4510-98eb-406c7b013360",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2024-03-16T08:06:19.974100100Z",
     "start_time": "2024-03-16T08:05:19.278575400Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "获取文章页面状态： 200\n"
     ]
    }
   ],
   "source": [
    "# 获取文章页面\n",
    "page = http.request(\"GET\", the_episode_url)\n",
    "print(\"获取文章页面状态：\", page.status)\n",
    "\n",
    "# 解析文章页面\n",
    "page_soup = BeautifulSoup(page.data)\n",
    "\n",
    "# 定位总体位置\n",
    "pos = page_soup.find(\"div\", class_='announcement-stories__wrapper-information')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "d92ac544-11d0-4f8c-b113-ec0f6bc4043f",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "获取标题、总结、时间、作者、介绍以及正文完成！等待写入tex文件\n"
     ]
    }
   ],
   "source": [
    "# 获取标题\n",
    "title_pos = pos.find('h1', class_='heading has-icon icon-audio-before has-key-taxonomy-above')\n",
    "title = str(title_pos.string.strip())\n",
    "\n",
    "# 获取总结\n",
    "summary_pos = pos.find('p', class_='intro')\n",
    "summary = str( summary_pos.string.strip() )\n",
    "\n",
    "# 获取时间和作者列表\n",
    "infor_pos = pos.find('div', class_='author-info-wrapper')\n",
    "infor_time = str(infor_pos.p.string) \n",
    "infor_author = [str(x.string) for x in infor_pos.find_all('span', class_='author')]\n",
    "\n",
    "# 获取介绍和正文\n",
    "content_pos = pos.find('div', class_='announcement-stories__idea-story-description as-description')\n",
    "intro_pos, conver_pos = content_pos.find_all('div', class_='field__item field--item-text_block')\n",
    "\n",
    "intro = ''.join( map(str, intro_pos.find_all('p')) )\n",
    "intro = re.sub('<p.*?>', '', str(intro))\n",
    "intro = re.sub('</p>', '\\n\\n', intro)\n",
    "\n",
    "conver = ''.join( map(str, conver_pos.find_all(\"p\")) )\n",
    "conver = re.sub('<p.*?>', '', str(conver))\n",
    "conver = re.sub('</p>', '\\n\\n', conver)\n",
    "\n",
    "print(\"获取标题、总结、时间、作者、介绍以及正文完成！等待写入tex文件\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "3fff5076-9197-4ba2-b57d-218aeee30314",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "将内容写入tex文件成功！等待编译pdf\n"
     ]
    }
   ],
   "source": [
    "# tex文件模板\n",
    "temple = \"\"\"\\\\documentclass{article}\n",
    "\\\\usepackage[UTF8]{ctex}\n",
    "\\\\usepackage{hyperref}\n",
    "\n",
    "\\\\linespread{1.3}\n",
    "\\\\setlength{\\parskip}{0.5em}\n",
    "\\\\setlength{\\parindent}{0em}\n",
    "\n",
    "\\\\begin{document}\n",
    "\n",
    "\\\\section*{%s}\n",
    "\n",
    "%s\n",
    "\n",
    "\\\\begin{flushright}\n",
    "    \\\\textit{%s}\n",
    "\\\\end{flushright}\n",
    "\n",
    "%s\n",
    "\n",
    "\\\\end{document}\n",
    "\"\"\"\n",
    "\n",
    "# 格式化内容并写入subscript.tex\n",
    "infor = infor_time + ' | by ' + ' '.join(infor_author)\n",
    "\n",
    "content = intro +'''\\\\vspace{1em}\n",
    "\\\\large{\\\\textbf{TRANSCRIPT}}\n",
    "\\\\vspace{1em}\n",
    "\\n\\n\n",
    "''' +conver\n",
    "content = re.sub(\"<em>(.*?)</em>\", \"\\\\\\\\textsl{\\\\1}\", content)\n",
    "content = re.sub(\"<strong>(.*?)</strong>\", \"\\\\\\\\textbf{\\\\1}\", content)\n",
    "content = re.sub('<a href=\\\"(.*?)\\\">(.*?)</a>', \"\\\\\\\\href{\\\\1}{\\\\2}\", content)\n",
    "content = re.sub('<br/>', \"\\n\", content)\n",
    "\n",
    "tex = temple % (title, summary, infor, content)\n",
    "\n",
    "with open(\"subscript.tex\", \"w+\", encoding=\"utf-8\") as f:\n",
    "    f.write(tex)\n",
    "    \n",
    "print(\"将内容写入tex文件成功！等待编译pdf\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "db7772be-d17c-4e79-99b3-dcd62d526e0c",
   "metadata": {
    "scrolled": true,
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "This is XeTeX, Version 3.14159265-2.6-0.99998 (TeX Live 2017/W32TeX) (preloaded format=xelatex)\n",
      " restricted \\write18 enabled.\n",
      "entering extended mode\n",
      "(./subscript.tex\n",
      "LaTeX2e <2017-04-15>\n",
      "Babel <3.10> and hyphenation patterns for 6 language(s) loaded.\n",
      "(d:/App/install/texlive2017/2017/texmf-dist/tex/latex/base/article.cls\n",
      "Document Class: article 2014/09/29 v1.4h Standard LaTeX document class\n",
      "(d:/App/install/texlive2017/2017/texmf-dist/tex/latex/base/size10.clo))\n",
      "(d:/App/install/texlive2017/2017/texmf-dist/tex/latex/ctex/ctex.sty\n",
      "(d:/App/install/texlive2017/2017/texmf-dist/tex/latex/l3kernel/expl3.sty\n",
      "(d:/App/install/texlive2017/2017/texmf-dist/tex/latex/l3kernel/expl3-code.tex)\n",
      "(d:/App/install/texlive2017/2017/texmf-dist/tex/latex/l3kernel/l3xdvipdfmx.def)\n",
      ")\n",
      "(d:/App/install/texlive2017/2017/texmf-dist/tex/latex/l3packages/xparse/xparse.\n",
      "sty)\n",
      "(d:/App/install/texlive2017/2017/texmf-dist/tex/latex/l3packages/l3keys2e/l3key\n",
      "s2e.sty)\n",
      "(d:/App/install/texlive2017/2017/texmf-dist/tex/latex/ctex/ctexhook.sty)\n",
      "(d:/App/install/texlive2017/2017/texmf-dist/tex/latex/ctex/ctexpatch.sty)\n",
      "(d:/App/install/texlive2017/2017/texmf-dist/tex/latex/base/fix-cm.sty\n",
      "(d:/App/install/texlive2017/2017/texmf-dist/tex/latex/base/ts1enc.def))\n",
      "(d:/App/install/texlive2017/2017/texmf-dist/tex/latex/ms/everysel.sty)\n",
      "(d:/App/install/texlive2017/2017/texmf-dist/tex/latex/ctex/config/ctexopts.cfg)\n",
      "\n",
      "(d:/App/install/texlive2017/2017/texmf-dist/tex/latex/ctex/engine/ctex-engine-x\n",
      "etex.def\n",
      "(d:/App/install/texlive2017/2017/texmf-dist/tex/xelatex/xecjk/xeCJK.sty\n",
      "(d:/App/install/texlive2017/2017/texmf-dist/tex/latex/l3packages/xtemplate/xtem\n",
      "plate.sty)\n",
      "(d:/App/install/texlive2017/2017/texmf-dist/tex/latex/fontspec/fontspec.sty\n",
      "(d:/App/install/texlive2017/2017/texmf-dist/tex/latex/fontspec/fontspec-xetex.s\n",
      "ty (d:/App/install/texlive2017/2017/texmf-dist/tex/latex/base/fontenc.sty\n",
      "(d:/App/install/texlive2017/2017/texmf-dist/tex/latex/base/tuenc.def))\n",
      "(d:/App/install/texlive2017/2017/texmf-dist/tex/latex/fontspec/fontspec.cfg)))\n",
      "(d:/App/install/texlive2017/2017/texmf-dist/tex/xelatex/xecjk/xunicode-addon.st\n",
      "y (d:/App/install/texlive2017/2017/texmf-dist/tex/xelatex/xunicode/xunicode.sty\n",
      "(d:/App/install/texlive2017/2017/texmf-dist/tex/latex/tipa/t3enc.def)\n",
      "(d:/App/install/texlive2017/2017/texmf-dist/tex/latex/graphics/graphicx.sty\n",
      "(d:/App/install/texlive2017/2017/texmf-dist/tex/latex/graphics/keyval.sty)\n",
      "(d:/App/install/texlive2017/2017/texmf-dist/tex/latex/graphics/graphics.sty\n",
      "(d:/App/install/texlive2017/2017/texmf-dist/tex/latex/graphics/trig.sty)\n",
      "(d:/App/install/texlive2017/2017/texmf-dist/tex/latex/graphics-cfg/graphics.cfg\n",
      ") (d:/App/install/texlive2017/2017/texmf-dist/tex/latex/graphics-def/xetex.def)\n",
      "))))\n",
      "(d:/App/install/texlive2017/2017/texmf-dist/tex/xelatex/xunicode/xunicode.sty\n",
      "*** Reloading Xunicode for encoding 'TU' ***\n",
      ")\n",
      "(d:/App/install/texlive2017/2017/texmf-dist/tex/xelatex/xecjk/xunicode-extra.de\n",
      "f) (d:/App/install/texlive2017/2017/texmf-dist/tex/xelatex/xecjk/xeCJK.cfg))\n",
      "(d:/App/install/texlive2017/2017/texmf-dist/tex/xelatex/xecjk/xeCJKfntef.sty\n",
      "(d:/App/install/texlive2017/2017/texmf-dist/tex/generic/ulem/ulem.sty)\n",
      "(d:/App/install/texlive2017/2017/texmf-dist/tex/latex/environ/environ.sty\n",
      "(d:/App/install/texlive2017/2017/texmf-dist/tex/latex/trimspaces/trimspaces.sty\n",
      "))\n",
      "(d:/App/install/texlive2017/2017/texmf-dist/tex/latex/cjk/texinput/CJKfntef.sty\n",
      "))) (d:/App/install/texlive2017/2017/texmf-dist/tex/latex/zhnumber/zhnumber.sty\n",
      "\n",
      "(d:/App/install/texlive2017/2017/texmf-dist/tex/latex/zhnumber/zhnumber-utf8.cf\n",
      "g))\n",
      "(d:/App/install/texlive2017/2017/texmf-dist/tex/latex/ctex/scheme/ctex-scheme-c\n",
      "hinese.def\n",
      "(d:/App/install/texlive2017/2017/texmf-dist/tex/latex/ctex/config/ctex-name-utf\n",
      "8.cfg))\n",
      "(d:/App/install/texlive2017/2017/texmf-dist/tex/latex/tools/indentfirst.sty)\n",
      "(d:/App/install/texlive2017/2017/texmf-dist/tex/latex/ctex/ctex-c5size.clo)\n",
      "(d:/App/install/texlive2017/2017/texmf-dist/tex/latex/ctex/fontset/ctex-fontset\n",
      "-windows.def\n",
      "(d:/App/install/texlive2017/2017/texmf-dist/tex/latex/ctex/fontset/ctex-fontset\n",
      "-windowsnew.def)))\n",
      "(d:/App/install/texlive2017/2017/texmf-dist/tex/latex/ctex/config/ctex.cfg)\n",
      "(d:/App/install/texlive2017/2017/texmf-dist/tex/latex/hyperref/hyperref.sty\n",
      "(d:/App/install/texlive2017/2017/texmf-dist/tex/generic/oberdiek/hobsub-hyperre\n",
      "f.sty\n",
      "(d:/App/install/texlive2017/2017/texmf-dist/tex/generic/oberdiek/hobsub-generic\n",
      ".sty))\n",
      "(d:/App/install/texlive2017/2017/texmf-dist/tex/generic/ifxetex/ifxetex.sty)\n",
      "(d:/App/install/texlive2017/2017/texmf-dist/tex/latex/oberdiek/auxhook.sty)\n",
      "(d:/App/install/texlive2017/2017/texmf-dist/tex/latex/oberdiek/kvoptions.sty)\n",
      "(d:/App/install/texlive2017/2017/texmf-dist/tex/latex/hyperref/pd1enc.def)\n",
      "(d:/App/install/texlive2017/2017/texmf-dist/tex/latex/latexconfig/hyperref.cfg)\n",
      "(d:/App/install/texlive2017/2017/texmf-dist/tex/latex/hyperref/puenc.def)\n",
      "(d:/App/install/texlive2017/2017/texmf-dist/tex/latex/url/url.sty))\n",
      "\n",
      "Package hyperref Message: Driver (autodetected): hxetex.\n",
      "\n",
      "(d:/App/install/texlive2017/2017/texmf-dist/tex/latex/hyperref/hxetex.def\n",
      "(d:/App/install/texlive2017/2017/texmf-dist/tex/generic/oberdiek/stringenc.sty)\n",
      "\n",
      "(d:/App/install/texlive2017/2017/texmf-dist/tex/latex/oberdiek/rerunfilecheck.s\n",
      "ty))\n",
      "No file subscript.aux.\n",
      "(d:/App/install/texlive2017/2017/texmf-dist/tex/latex/tipa/t3cmr.fd)\n",
      "ABD: EverySelectfont initializing macros\n",
      "(d:/App/install/texlive2017/2017/texmf-dist/tex/latex/hyperref/nameref.sty\n",
      "(d:/App/install/texlive2017/2017/texmf-dist/tex/generic/oberdiek/gettitlestring\n",
      ".sty))\n",
      "\n",
      "Package hyperref Warning: Rerun to get /PageLabels entry.\n",
      "\n",
      "\n",
      "LaTeX Font Warning: Font shape `TU/SimSun(0)/bx/sl' undefined\n",
      "(Font)              using `TU/SimSun(0)/bx/n' instead on input line 21.\n",
      "\n",
      "[1] [2] [3] [4] [5] [6] (./subscript.aux)\n",
      "\n",
      "LaTeX Font Warning: Some font shapes were not available, defaults substituted.\n",
      "\n",
      " )\n",
      "Output written on subscript.pdf (6 pages).\n",
      "Transcript written on subscript.log.\n",
      "编译subscript.pdf成功！等待发送邮箱\n"
     ]
    }
   ],
   "source": [
    "!xelatex subscript.tex\n",
    "!del *.aux *.log *.out\n",
    "print(\"编译subscript.pdf成功！等待发送邮箱\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "534e211f-9c40-427c-8b87-c60a2e656c61",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[NbConvertApp] Converting notebook scrapy the subscript.ipynb to script\n",
      "[NbConvertApp] Writing 3089 bytes to scrapy the subscript.py\n"
     ]
    }
   ],
   "source": [
    "# # 转化为.py脚本\n",
    "# !jupyter nbconvert --to script \"scrapy the subscript.ipynb\"\n",
    "# 将必要变量存入json文件\n",
    "with open(\"./info.json\", \"a\", encoding='utf-8') as f:\n",
    "    json.dump({\"title\":title, \"summary\":summary}, f)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.9"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}