Hugging Face's logo

Spaces:
lizhen30
/
LangChainGo
Runtime error

App Files Community
远兮 commited on
Commit
dd85bee
·
1 Parent(s): c7e0b67

add indexes

Browse files
Files changed (3) hide show
  1. index_url_loader.ipynb +76 -0
  2. index_web_base.ipynb +73 -0
  3. index_youtube.ipynb +82 -0
index_url_loader.ipynb ADDED
@@ -0,0 +1,76 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "cells": [
3
+ {
4
+ "cell_type": "code",
5
+ "execution_count": 21,
6
+ "metadata": {},
7
+ "outputs": [],
8
+ "source": [
9
+ "from langchain.document_loaders import SeleniumURLLoader"
10
+ ]
11
+ },
12
+ {
13
+ "cell_type": "code",
14
+ "execution_count": 22,
15
+ "metadata": {},
16
+ "outputs": [],
17
+ "source": [
18
+ "urls = [\n",
19
+ " \"https://www.youtube.com/shorts/XB4I8wy5C3Y\"\n",
20
+ "]"
21
+ ]
22
+ },
23
+ {
24
+ "cell_type": "code",
25
+ "execution_count": 23,
26
+ "metadata": {},
27
+ "outputs": [],
28
+ "source": [
29
+ "loader = SeleniumURLLoader(urls=urls)"
30
+ ]
31
+ },
32
+ {
33
+ "cell_type": "code",
34
+ "execution_count": 24,
35
+ "metadata": {},
36
+ "outputs": [
37
+ {
38
+ "data": {
39
+ "text/plain": [
40
+ "[Document(page_content=\"NaN / NaN\\n\\nTW\\n\\nSkip navigation\\n\\nSearch\\n\\nSearch with your voice\\n\\nSign in\\n\\nTW\\n\\nHome\\n \\n \\n Home\\n\\nShorts\\n \\n \\n Shorts\\n\\nSubscriptions\\n \\n \\n Subscriptions\\n\\nLibrary\\n \\n \\n Library\\n\\nHistory\\n \\n \\n History\\n\\nComments\\n 414\\n\\nTop comments\\n\\nNewest first\\n\\nDescription\\n\\n妮妮吃不饱\\n\\n我說怎麼突然變現這麼好\\n\\nInfo\\n\\nShopping\\n\\nTap to unmute\\n\\nIf playback doesn't begin shortly, try restarting your device.\\n\\nShare\\n\\nAn error occurred while retrieving sharing information. Please try again later.\\n\\nWatch later\\n\\nShare\\n\\nCopy link\\n\\n我說怎麼突然變現這麼好\\n\\n@niniLL1314\\n\\n@niniLL1314\\n\\n50K\\n\\nI like this\\n\\nDislike\\n\\nI dislike this\\n\\n414\\n\\nComments\\n\\nShare\\n\\nShare\\n\\nLike\\n\\nDislike\\n\\nComment\\n\\nShare\\n\\nLike\\n\\nDislike\\n\\nComment\\n\\nShare\\n\\nLike\\n\\nDislike\\n\\nComment\\n\\nShare\\n\\nLike\\n\\nDislike\\n\\nComment\\n\\nShare\\n\\nLike\\n\\nDislike\\n\\nComment\\n\\nShare\\n\\nLike\\n\\nDislike\\n\\nComment\\n\\nShare\\n\\nLike\\n\\nDislike\\n\\nComment\\n\\nShare\\n\\nLike\\n\\nDislike\\n\\nComment\\n\\nShare\\n\\nLike\\n\\nDislike\\n\\nComment\\n\\nShare\\n\\nLike\\n\\nDislike\\n\\nComment\\n\\nShare\\n\\nLike\\n\\nDislike\\n\\nComment\\n\\nShare\\n\\n•\", metadata={'source': 'https://www.youtube.com/shorts/XB4I8wy5C3Y'})]"
41
+ ]
42
+ },
43
+ "execution_count": 24,
44
+ "metadata": {},
45
+ "output_type": "execute_result"
46
+ }
47
+ ],
48
+ "source": [
49
+ "data = loader.load()\n",
50
+ "data"
51
+ ]
52
+ }
53
+ ],
54
+ "metadata": {
55
+ "kernelspec": {
56
+ "display_name": "base",
57
+ "language": "python",
58
+ "name": "python3"
59
+ },
60
+ "language_info": {
61
+ "codemirror_mode": {
62
+ "name": "ipython",
63
+ "version": 3
64
+ },
65
+ "file_extension": ".py",
66
+ "mimetype": "text/x-python",
67
+ "name": "python",
68
+ "nbconvert_exporter": "python",
69
+ "pygments_lexer": "ipython3",
70
+ "version": "3.10.10"
71
+ },
72
+ "orig_nbformat": 4
73
+ },
74
+ "nbformat": 4,
75
+ "nbformat_minor": 2
76
+ }
index_web_base.ipynb ADDED
@@ -0,0 +1,73 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "cells": [
3
+ {
4
+ "cell_type": "code",
5
+ "execution_count": 4,
6
+ "metadata": {},
7
+ "outputs": [],
8
+ "source": [
9
+ "from langchain.document_loaders import WebBaseLoader"
10
+ ]
11
+ },
12
+ {
13
+ "cell_type": "code",
14
+ "execution_count": 5,
15
+ "metadata": {},
16
+ "outputs": [],
17
+ "source": [
18
+ "loader = WebBaseLoader(\"https://www.baidu.com/\")"
19
+ ]
20
+ },
21
+ {
22
+ "cell_type": "code",
23
+ "execution_count": 6,
24
+ "metadata": {},
25
+ "outputs": [],
26
+ "source": [
27
+ "data = loader.load()"
28
+ ]
29
+ },
30
+ {
31
+ "cell_type": "code",
32
+ "execution_count": 7,
33
+ "metadata": {},
34
+ "outputs": [
35
+ {
36
+ "data": {
37
+ "text/plain": [
38
+ "[Document(page_content='\\n ç\\x99¾åº¦ä¸\\x80ä¸\\x8bï¼\\x8cä½\\xa0å°±ç\\x9f¥é\\x81\\x93 æ\\x96°é\\x97» hao123 å\\x9c°å\\x9b¾ è§\\x86é¢\\x91 è´´å\\x90§ ç\\x99»å½\\x95 æ\\x9b´å¤\\x9a产å\\x93\\x81 å\\x85³äº\\x8eç\\x99¾åº¦ About Baidu ©2017\\xa0Baidu\\xa0使ç\\x94¨ç\\x99¾åº¦å\\x89\\x8då¿\\x85读\\xa0 æ\\x84\\x8fè§\\x81å\\x8f\\x8dé¦\\x88\\xa0京ICPè¯\\x81030173å\\x8f·\\xa0 \\n', metadata={'source': 'https://www.baidu.com/', 'title': 'ç\\x99¾åº¦ä¸\\x80ä¸\\x8bï¼\\x8cä½\\xa0å°±ç\\x9f¥é\\x81\\x93', 'language': None})]"
39
+ ]
40
+ },
41
+ "execution_count": 7,
42
+ "metadata": {},
43
+ "output_type": "execute_result"
44
+ }
45
+ ],
46
+ "source": [
47
+ "data"
48
+ ]
49
+ }
50
+ ],
51
+ "metadata": {
52
+ "kernelspec": {
53
+ "display_name": "base",
54
+ "language": "python",
55
+ "name": "python3"
56
+ },
57
+ "language_info": {
58
+ "codemirror_mode": {
59
+ "name": "ipython",
60
+ "version": 3
61
+ },
62
+ "file_extension": ".py",
63
+ "mimetype": "text/x-python",
64
+ "name": "python",
65
+ "nbconvert_exporter": "python",
66
+ "pygments_lexer": "ipython3",
67
+ "version": "3.10.10"
68
+ },
69
+ "orig_nbformat": 4
70
+ },
71
+ "nbformat": 4,
72
+ "nbformat_minor": 2
73
+ }
index_youtube.ipynb ADDED
@@ -0,0 +1,82 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "cells": [
3
+ {
4
+ "cell_type": "code",
5
+ "execution_count": 1,
6
+ "metadata": {},
7
+ "outputs": [],
8
+ "source": [
9
+ "# pip install youtube-transcript-api\n",
10
+ "# pip install pytube"
11
+ ]
12
+ },
13
+ {
14
+ "cell_type": "code",
15
+ "execution_count": 2,
16
+ "metadata": {},
17
+ "outputs": [],
18
+ "source": [
19
+ "from langchain.document_loaders import YoutubeLoader"
20
+ ]
21
+ },
22
+ {
23
+ "cell_type": "code",
24
+ "execution_count": 3,
25
+ "metadata": {},
26
+ "outputs": [],
27
+ "source": [
28
+ "loader = YoutubeLoader.from_youtube_url(\"https://www.youtube.com/shorts/6Z-JeQ7ILhA\", add_video_info=True)"
29
+ ]
30
+ },
31
+ {
32
+ "cell_type": "code",
33
+ "execution_count": 4,
34
+ "metadata": {},
35
+ "outputs": [
36
+ {
37
+ "ename": "PytubeError",
38
+ "evalue": "Exception while accessing title of https://youtube.com/watch?v=6Z-JeQ7ILhA. Please file a bug report at https://github.com/pytube/pytube",
39
+ "output_type": "error",
40
+ "traceback": [
41
+ "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
42
+ "\u001b[0;31mKeyError\u001b[0m Traceback (most recent call last)",
43
+ "File \u001b[0;32m~/anaconda3/lib/python3.10/site-packages/pytube/__main__.py:341\u001b[0m, in \u001b[0;36mYouTube.title\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m 340\u001b[0m \u001b[39mtry\u001b[39;00m:\n\u001b[0;32m--> 341\u001b[0m \u001b[39mself\u001b[39m\u001b[39m.\u001b[39m_title \u001b[39m=\u001b[39m \u001b[39mself\u001b[39;49m\u001b[39m.\u001b[39;49mvid_info[\u001b[39m'\u001b[39;49m\u001b[39mvideoDetails\u001b[39;49m\u001b[39m'\u001b[39;49m][\u001b[39m'\u001b[39m\u001b[39mtitle\u001b[39m\u001b[39m'\u001b[39m]\n\u001b[1;32m 342\u001b[0m \u001b[39mexcept\u001b[39;00m \u001b[39mKeyError\u001b[39;00m:\n\u001b[1;32m 343\u001b[0m \u001b[39m# Check_availability will raise the correct exception in most cases\u001b[39;00m\n\u001b[1;32m 344\u001b[0m \u001b[39m# if it doesn't, ask for a report.\u001b[39;00m\n",
44
+ "\u001b[0;31mKeyError\u001b[0m: 'videoDetails'",
45
+ "\nDuring handling of the above exception, another exception occurred:\n",
46
+ "\u001b[0;31mPytubeError\u001b[0m Traceback (most recent call last)",
47
+ "Cell \u001b[0;32mIn[4], line 1\u001b[0m\n\u001b[0;32m----> 1\u001b[0m loader\u001b[39m.\u001b[39;49mload()\n",
48
+ "File \u001b[0;32m~/anaconda3/lib/python3.10/site-packages/langchain/document_loaders/youtube.py:141\u001b[0m, in \u001b[0;36mYoutubeLoader.load\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m 136\u001b[0m metadata \u001b[39m=\u001b[39m {\u001b[39m\"\u001b[39m\u001b[39msource\u001b[39m\u001b[39m\"\u001b[39m: \u001b[39mself\u001b[39m\u001b[39m.\u001b[39mvideo_id}\n\u001b[1;32m 138\u001b[0m \u001b[39mif\u001b[39;00m \u001b[39mself\u001b[39m\u001b[39m.\u001b[39madd_video_info:\n\u001b[1;32m 139\u001b[0m \u001b[39m# Get more video meta info\u001b[39;00m\n\u001b[1;32m 140\u001b[0m \u001b[39m# Such as title, description, thumbnail url, publish_date\u001b[39;00m\n\u001b[0;32m--> 141\u001b[0m video_info \u001b[39m=\u001b[39m \u001b[39mself\u001b[39;49m\u001b[39m.\u001b[39;49m_get_video_info()\n\u001b[1;32m 142\u001b[0m metadata\u001b[39m.\u001b[39mupdate(video_info)\n\u001b[1;32m 144\u001b[0m \u001b[39mtry\u001b[39;00m:\n",
49
+ "File \u001b[0;32m~/anaconda3/lib/python3.10/site-packages/langchain/document_loaders/youtube.py:182\u001b[0m, in \u001b[0;36mYoutubeLoader._get_video_info\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m 176\u001b[0m \u001b[39mraise\u001b[39;00m \u001b[39mImportError\u001b[39;00m(\n\u001b[1;32m 177\u001b[0m \u001b[39m\"\u001b[39m\u001b[39mCould not import pytube python package. \u001b[39m\u001b[39m\"\u001b[39m\n\u001b[1;32m 178\u001b[0m \u001b[39m\"\u001b[39m\u001b[39mPlease install it with `pip install pytube`.\u001b[39m\u001b[39m\"\u001b[39m\n\u001b[1;32m 179\u001b[0m )\n\u001b[1;32m 180\u001b[0m yt \u001b[39m=\u001b[39m YouTube(\u001b[39mf\u001b[39m\u001b[39m\"\u001b[39m\u001b[39mhttps://www.youtube.com/watch?v=\u001b[39m\u001b[39m{\u001b[39;00m\u001b[39mself\u001b[39m\u001b[39m.\u001b[39mvideo_id\u001b[39m}\u001b[39;00m\u001b[39m\"\u001b[39m)\n\u001b[1;32m 181\u001b[0m video_info \u001b[39m=\u001b[39m {\n\u001b[0;32m--> 182\u001b[0m \u001b[39m\"\u001b[39m\u001b[39mtitle\u001b[39m\u001b[39m\"\u001b[39m: yt\u001b[39m.\u001b[39;49mtitle,\n\u001b[1;32m 183\u001b[0m \u001b[39m\"\u001b[39m\u001b[39mdescription\u001b[39m\u001b[39m\"\u001b[39m: yt\u001b[39m.\u001b[39mdescription,\n\u001b[1;32m 184\u001b[0m \u001b[39m\"\u001b[39m\u001b[39mview_count\u001b[39m\u001b[39m\"\u001b[39m: yt\u001b[39m.\u001b[39mviews,\n\u001b[1;32m 185\u001b[0m \u001b[39m\"\u001b[39m\u001b[39mthumbnail_url\u001b[39m\u001b[39m\"\u001b[39m: yt\u001b[39m.\u001b[39mthumbnail_url,\n\u001b[1;32m 186\u001b[0m \u001b[39m\"\u001b[39m\u001b[39mpublish_date\u001b[39m\u001b[39m\"\u001b[39m: yt\u001b[39m.\u001b[39mpublish_date,\n\u001b[1;32m 187\u001b[0m \u001b[39m\"\u001b[39m\u001b[39mlength\u001b[39m\u001b[39m\"\u001b[39m: yt\u001b[39m.\u001b[39mlength,\n\u001b[1;32m 188\u001b[0m \u001b[39m\"\u001b[39m\u001b[39mauthor\u001b[39m\u001b[39m\"\u001b[39m: yt\u001b[39m.\u001b[39mauthor,\n\u001b[1;32m 189\u001b[0m }\n\u001b[1;32m 190\u001b[0m \u001b[39mreturn\u001b[39;00m video_info\n",
50
+ "File \u001b[0;32m~/anaconda3/lib/python3.10/site-packages/pytube/__main__.py:346\u001b[0m, in \u001b[0;36mYouTube.title\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m 342\u001b[0m \u001b[39mexcept\u001b[39;00m \u001b[39mKeyError\u001b[39;00m:\n\u001b[1;32m 343\u001b[0m \u001b[39m# Check_availability will raise the correct exception in most cases\u001b[39;00m\n\u001b[1;32m 344\u001b[0m \u001b[39m# if it doesn't, ask for a report.\u001b[39;00m\n\u001b[1;32m 345\u001b[0m \u001b[39mself\u001b[39m\u001b[39m.\u001b[39mcheck_availability()\n\u001b[0;32m--> 346\u001b[0m \u001b[39mraise\u001b[39;00m exceptions\u001b[39m.\u001b[39mPytubeError(\n\u001b[1;32m 347\u001b[0m (\n\u001b[1;32m 348\u001b[0m \u001b[39mf\u001b[39m\u001b[39m'\u001b[39m\u001b[39mException while accessing title of \u001b[39m\u001b[39m{\u001b[39;00m\u001b[39mself\u001b[39m\u001b[39m.\u001b[39mwatch_url\u001b[39m}\u001b[39;00m\u001b[39m. \u001b[39m\u001b[39m'\u001b[39m\n\u001b[1;32m 349\u001b[0m \u001b[39m'\u001b[39m\u001b[39mPlease file a bug report at https://github.com/pytube/pytube\u001b[39m\u001b[39m'\u001b[39m\n\u001b[1;32m 350\u001b[0m )\n\u001b[1;32m 351\u001b[0m )\n\u001b[1;32m 353\u001b[0m \u001b[39mreturn\u001b[39;00m \u001b[39mself\u001b[39m\u001b[39m.\u001b[39m_title\n",
51
+ "\u001b[0;31mPytubeError\u001b[0m: Exception while accessing title of https://youtube.com/watch?v=6Z-JeQ7ILhA. Please file a bug report at https://github.com/pytube/pytube"
52
+ ]
53
+ }
54
+ ],
55
+ "source": [
56
+ "loader.load()"
57
+ ]
58
+ }
59
+ ],
60
+ "metadata": {
61
+ "kernelspec": {
62
+ "display_name": "base",
63
+ "language": "python",
64
+ "name": "python3"
65
+ },
66
+ "language_info": {
67
+ "codemirror_mode": {
68
+ "name": "ipython",
69
+ "version": 3
70
+ },
71
+ "file_extension": ".py",
72
+ "mimetype": "text/x-python",
73
+ "name": "python",
74
+ "nbconvert_exporter": "python",
75
+ "pygments_lexer": "ipython3",
76
+ "version": "3.10.10"
77
+ },
78
+ "orig_nbformat": 4
79
+ },
80
+ "nbformat": 4,
81
+ "nbformat_minor": 2
82
+ }