Python\u306e\u30e9\u30a4\u30d6\u30e9\u30ea(Beautiful Soup)\u3092\u5229\u7528\u3057\u3066\u30b9\u30af\u30ec\u30a4\u30d4\u30f3\u30b0\u3057\u3066\u307f\u305f<\/p>\n
\r\nroot@hostname:\/home\/shimizu\/python# aptitude install python-bs4\r\n\u4ee5\u4e0b\u306e\u65b0\u898f\u30d1\u30c3\u30b1\u30fc\u30b8\u304c\u30a4\u30f3\u30b9\u30c8\u30fc\u30eb\u3055\u308c\u307e\u3059:\r\n python-bs4 python-chardet{a} python-lxml{a}\r\n...\r\n\r\nroot@hostname:\/home\/shimizu\/python# cat scraping-bs4.py\r\n# coding: UTF-8\r\nimport urllib2\r\nfrom bs4 import BeautifulSoup\r\nres = urllib2.urlopen("http:\/\/ll.jus.or.jp\/2014\/program.html")\r\n# \u30aa\u30d6\u30b8\u30a7\u30af\u30c8<class 'bs4.element.ResultSet'>\u3092\u53d6\u5f97\u3057\u3001list\u306e\u3088\u3046\u306b\u6271\u3048\u308b\r\nsoup = BeautifulSoup(res.read())\r\n\r\n\r\n# \u5148\u982d\u306e\u30bf\u30b0\u3092\u8868\u793a\u3059\u308b\r\nprint soup.find('a')\r\n# \u30bf\u30b0\u5185\u306e\u5c5e\u6027\u3092\u8868\u793a\u3059\u308b\r\nprint soup.a.get('href')\r\n# \u30bf\u30b0\u306e\u4e2d\u306e\u6587\u5b57\u3092\u53d6\u5f97\u3059\u308b\r\nprint soup.a.string\r\n\r\nroot@hostname:\/home\/shimizu\/python# python scraping-bs4.py\r\n<a href="index.html" rel="home">LL Diver<\/a>\r\nindex.html\r\nLL Diver\r\n<\/pre>\n\u3059\u3079\u3066\u306ea\u30bf\u30b0\u306e\u8981\u7d20\u3092\u8868\u793a\u3059\u308b<\/h2>\n\r\nroot@hostname:\/home\/shimizu\/python# cat scraping-bs4-2.py\r\n# coding: UTF-8\r\nimport urllib2,sys\r\nreload(sys)\r\nsys.setdefaultencoding('utf-8')\r\n\r\nfrom bs4 import BeautifulSoup\r\nres = urllib2.urlopen("http:\/\/ll.jus.or.jp\/2014\/program.html")\r\n# \u30aa\u30d6\u30b8\u30a7\u30af\u30c8<class 'bs4.element.ResultSet'>\u3092\u53d6\u5f97\u3057\u3001list\u306e\u3088\u3046\u306b\u6271\u3048\u308b\r\nsoup = BeautifulSoup(res.read())\r\n\r\n# \u30ea\u30f3\u30af\u62bd\u51fa\r\nfor link in soup.find_all('a'):\r\n if link.string is not None:\r\n print link.string + ": " + link.get('href')\r\n\r\nroot@hostname:\/home\/shimizu\/python# python scraping-bs4-2.py\r\nLL Diver: index.html\r\n\u691c\u7d22: index.html%3Fp=49.html#search-container\r\n\u30b3\u30f3\u30c6\u30f3\u30c4\u3078\u79fb\u52d5: index.html%3Fp=49.html#content\r\nLL Diver: index.html\r\nLL Diver\u958b\u50ac\u6848\u5185: index.html%3Fp=9.html\r\n\u30d7\u30ed\u30b0\u30e9\u30e0: index.html%3Fp=49.html\r\n\u30bf\u30a4\u30e0\u30c6\u30fc\u30d6\u30eb: index.html%3Fp=198.html\r\n\u30a2\u30f3\u30b1\u30fc\u30c8: index.html%3Fp=260.html\r\n\u30a2\u30fc\u30ab\u30a4\u30d6: index.html%3Fp=77.html\r\n\u663c\u306e\u90e8: index.html%3Fp=49.html#day\r\n...\r\n<\/pre>\n\u305d\u306e\u4ed6\u64cd\u4f5c<\/h2>\n\r\nroot@hostname:\/home\/shimizu\/python# python\r\nPython 2.7.9 (default, Mar 1 2015, 12:57:24)\r\n[GCC 4.9.2] on linux2\r\nType "help", "copyright", "credits" or "license" for more information.\r\n>>> import urllib2,sys\r\n>>> reload(sys)\r\n<module 'sys' (built-in)>\r\n>>> sys.setdefaultencoding('utf-8')\r\n>>> from bs4 import BeautifulSoup\r\n>>> res = urllib2.urlopen("http:\/\/ll.jus.or.jp\/2014\/program.html")\r\n>>> soup = BeautifulSoup(res.read())\r\n>>> print soup.title\r\n<title>\u30d7\u30ed\u30b0\u30e9\u30e0 | LL Diver<\/title>\r\n>>> print soup.title.name\r\ntitle\r\n>>> print soup.title.string\r\n\u30d7\u30ed\u30b0\u30e9\u30e0 | LL Diver\r\n>>> soup.p\r\n<p>LL Diver\u306e\u30d7\u30ed\u30b0\u30e9\u30e0\u306f\u3001<a href="index.html%3Fp=49.html#day">\u663c\u306e\u90e8<\/a>\u30fb<a href="index.html%3Fp=49.html#night">\u591c\u306e\u90e8<\/a>\u306b\u5206\u3051\u3066\u884c\u3044\u307e\u3059\u3002<br\/>\r\n<a href="index.html%3Fp=198.html" target="_blank" title="\u30bf\u30a4\u30e0\u30c6\u30fc\u30d6\u30eb">\u30bf\u30a4\u30e0\u30c6\u30fc\u30d6\u30eb<\/a>\u3082\u3042\u308f\u305b\u3066\u3054\u89a7\u304f\u3060\u3055\u3044\u3002<\/p>\r\n>>> print soup.find(id="genericons-css")\r\n<link href="wp-content\/themes\/twentyfourteen\/genericons\/genericons.css%3Fver=3.0.2.css" id="genericons-css" media="all" rel="stylesheet" type="text\/css"\/>\r\n### \u30bf\u30b0\u3092\u9664\u53bb\u3057\u3066\u5168\u30c6\u30ad\u30b9\u30c8\u3092\u62bd\u51fa ###\r\n>>> print(soup.get_text())\r\n...\r\n### soup\u306b\u3064\u3044\u3066 ###\r\n>>> print type(soup)\r\n<class 'bs4.BeautifulSoup'>\r\n>>> print soup.prettify\r\n...\r\n\r\n### Tag obj ###\r\n>>> type(soup)\r\n<class 'bs4.BeautifulSoup'>\r\n>>> type(tag)\r\n<class 'bs4.element.Tag'>\r\n>>> tag.name\r\n'a'\r\n### \u8f9e\u66f8\u306b\u30a2\u30af\u30bb\u30b9\u3059\u308b ###\r\n>>> tag.attrs\r\n{'href': 'index.html', 'rel': ['home']}\r\n\r\n### class\u306f\u4e88\u7d04\u5f8c\u306e\u305f\u3081 class_ \u3068\u3044\u3046\u30ad\u30fc\u30ef\u30fc\u30c9\u5f15\u6570\u3067CSS\u306e\u30af\u30e9\u30b9\u3092\u691c\u7d22 ###\r\n>>> soup.find_all(class_="page_item page-item-9")\r\n[<li class="page_item page-item-9"><a href="index.html%3Fp=9.html">LL Diver\u958b\u50ac\u6848\u5185<\/a><\/li>]\r\n<\/pre>\n\u5b9f\u969b\u306b\u30c7\u30fc\u30bf\u3092\u53d6\u5f97\u3059\u308b-\u5929\u6c17\u4e88\u5831\u30c7\u30fc\u30bf-<\/h1>\n
Python \u3067\u7c21\u5358\u306a\u30c6\u30ad\u30b9\u30c8\u51e6\u7406 (3) – Beautiful Soup \u3092\u4f7f\u3063\u3066\u30b9\u30af\u30ec\u30a4\u30d4\u30f3\u30b0
\nhttp:\/\/jutememo.blogspot.jp\/2008\/06\/python-3-beautiful-soup.html
\n\u3088\u308a\u62dd\u501f<\/p>\n
\r\nroot@hostname:\/home\/shimizu\/python# cat scraping-bs4-3.py\r\n# coding: UTF-8\r\nimport urllib2,sys\r\nreload(sys)\r\nsys.setdefaultencoding('utf-8')\r\n\r\nfrom bs4 import BeautifulSoup\r\nURL = "http:\/\/www.data.jma.go.jp\/obd\/stats\/etrn\/view\/daily_s1.php?prec_no=44&prec_ch=%93%8C%8B%9E%93s&block_no=47662&block_ch=%93%8C%8B%9E&year=2008&month=01&day=12&view=p1"\r\n\r\n# \u51fa\u529b\u3059\u308b\u65e5\u4ed8\r\nDATE_FROM = 1\r\nDATE_TO = 5\r\n\r\n# \u30c7\u30fc\u30bf\u53d6\u5f97\r\nres = urllib2.urlopen(URL)\r\nsoup = BeautifulSoup(res.read())\r\n\r\n#print soup.prettify\r\n#print type(soup)\r\n\r\nrecords = []\r\n# class \u5c5e\u6027\u304c mtx \u3067\u3042\u308b tr \u30bf\u30b0\u3092\u5bfe\u8c61\u306b\r\nfor tr in soup('tr',{'class':'mtx'}):\r\n rec = []\r\n # id \u5c5e\u6027\u304cp_print \u306ediv\u30bf\u30b0\u3092\u5bfe\u8c61\u306b\r\n for div in tr('div',{'class':'a_print'}):\r\n # a \u30bf\u30b0\u3092\u5bfe\u8c61\u306b\r\n for a in div('a'):\r\n # \u65e5\u4ed8\u3092\u53d6\u5f97\r\n rec.append(a.renderContents())\r\n # class \u5c5e\u6027\u304c data_0_0 \u3067\u3042\u308b td \u30bf\u30b0\u3092\u5bfe\u8c61\u306b\r\n for td in tr('td',{'class':'data_0_0'}):\r\n # \u5404\u30c7\u30fc\u30bf\u3092\u53d6\u5f97\r\n data = td.renderContents().strip()\r\n rec.append(data)\r\n if rec != []: records.append(rec)\r\n\r\n# \u53d6\u5f97\u3057\u305f\u30c7\u30fc\u30bf\u3092\u51fa\u529b\r\nfor rec in records:\r\n # \u6307\u5b9a\u3055\u308c\u305f\u65e5\u4ed8\u306e\u671f\u9593\u4ee5\u5916\u306f\u51fa\u529b\u3057\u306a\u3044\r\n if int(rec[0]) not in range(DATE_FROM, DATE_TO + 1): continue\r\n for i,data in enumerate(rec):\r\n if i in [1,6]: print unicode(data, 'utf-8'), "\\t",\r\n print\r\n\r\nroot@hostname:\/home\/shimizu\/python# python scraping-bs4-3.py\r\n998.7 6.0\r\n1007.2 6.2\r\n1011.6 5.9\r\n1014.6 7.0\r\n1014.0 6.0\r\nroot@hostname:\/home\r\n<\/pre>\n\u5b9f\u969b\u306b\u30c7\u30fc\u30bf\u3092\u53d6\u5f97\u3059\u308b-\u5b85\u914d\u4fbf\u306e\u914d\u9001\u72b6\u6cc1-<\/h1>\n
Python\u3067\u30b9\u30af\u30ec\u30a4\u30d4\u30f3\u30b0
\nhttp:\/\/i2bskn.hateblo.jp\/entry\/20120820\/1345478918
\n\u3088\u308a\u62dd\u501f<\/p>\n
\r\nroot@hostname:\/home\/shimizu\/python# cat scraping-bs4-4.py\r\n# coding: UTF-8\r\nimport urllib,urllib2,sys,re\r\nreload(sys)\r\nsys.setdefaultencoding('utf-8')\r\n\r\nfrom bs4 import BeautifulSoup\r\n\r\ndef kuroneko_check(n):\r\n r = re.compile(r'>([^<]+)<br')\r\n url = "http:\/\/toi.kuronekoyamato.co.jp\/cgi-bin\/tneko"\r\n data = urllib.urlencode({"number00": "1", "number01": n})\r\n #html = urllib2.urlopen(url,data).read().decode('shift_jis').encode('utf-8')\r\n html = urllib2.urlopen(url,data).read()\r\n soup = BeautifulSoup(html)\r\n #print soup.prettify\r\n s = str(soup.find_all("td",class_="ct")[0].font)\r\n return r.findall(s)[0]\r\n\r\nprint(kuroneko_check('4320-xxxx-xxxx'))\r\n\r\nroot@hostname:\/home\/shimizu\/python# python scraping-bs4-4.py\r\n\u914d\u9054\u5b8c\u4e86\r\n<\/pre>\n\u5b9f\u969b\u306b\u30c7\u30fc\u30bf\u3092\u53d6\u5f97\u3059\u308b-\u6587\u7ae0\u3092\u62bd\u51fa\u3059\u308b-<\/h1>\n
\u7b2c3\u56de \u30b9\u30af\u30ec\u30a4\u30d4\u30f3\u30b0\u306b\u30c1\u30e3\u30ec\u30f3\u30b8\uff01 \uff082\/3\uff09
\nhttp:\/\/itpro.nikkeibp.co.jp\/article\/COLUMN\/20080407\/298191\/?ST=develop&P=2
\n\u3088\u308a\u62dd\u501f<\/p>\n
\r\nroot@hostname:\/home\/shimizu\/python# cat scraping-bs4-5.py\r\n# coding: UTF-8\r\nimport urllib2,sys\r\nreload(sys)\r\nsys.setdefaultencoding('utf-8')\r\n\r\nfrom bs4 import BeautifulSoup,NavigableString\r\n\r\nurl = 'http:\/\/www.linux.or.jp\/'\r\n\r\nhtml = urllib2.urlopen(url).read()\r\n\r\nsoup = BeautifulSoup(html)\r\n\r\ndef printText(tags):\r\n for tag in tags:\r\n if tag.__class__ == NavigableString:\r\n print tag,\r\n else:\r\n printText(tag)\r\n print ""\r\n\r\nprintText(soup.findAll("p"))\r\n\r\nroot@hostname:\/home\/shimizu\/python# python scraping-bs4-5.py\r\n\u3059\u3050\u306b\u4f7f\u3048\u308b\u30ea\u30ca\u30c3\u30af\u30b9\u30c1\u30e5\u30fc\u30c8\u30ea\u30a2\u30eb\u3092\u3054\u7d39\u4ecb\r\n\u6700\u65b0\u306elinux\u30cb\u30e5\u30fc\u30b9\u3084\u30a4\u30f3\u30bf\u30d3\u30e5\u30fc\u8a18\u4e8b\u3092\u304a\u5c4a\u3051\u3057\u307e\u3059\u3002\r\n\r\n30\u9031\u9023\u7d9a\u4f01\u753b\uff1a\u4eca\u9031\u306fShuah Khan\u6c0f\u306e\u4ed5\u4e8b\u5834\u306b\u6f5c\u5165\uff01\r\n\u3000\u3000\u7d9a\u304d\u3092\u8aad\u3080...\r\n...\r\n<\/pre>\n\u5b9f\u969b\u306b\u30c7\u30fc\u30bf\u3092\u53d6\u5f97\u3059\u308b-\u30cb\u30e5\u30fc\u30b9\u30b5\u30a4\u30c8-<\/h1>\n
BeautifulSoup\u5b9f\u8df5 -python\u3067\u8d85\u30ab\u30f3\u30bf\u30f3\u306b\u30b9\u30af\u30ec\u30a4\u30d4\u30f3\u30b0
\nhttp:\/\/d.hatena.ne.jp\/lolloo-htn\/20090128\/1233155734
\n\u3088\u308a\u62dd\u501f<\/p>\n
\r\nroot@hostname:\/home\/shimizu\/python# cat scraping-bs4-6.py\r\n# coding: UTF-8\r\nimport urllib2,sys\r\nreload(sys)\r\nsys.setdefaultencoding('utf-8')\r\n\r\nfrom bs4 import BeautifulSoup,NavigableString\r\nurl = "http:\/\/www.yomiuri.co.jp\/sports\/"\r\n\r\n# \u30c7\u30fc\u30bf\u53d6\u5f97\r\nres = urllib2.urlopen(url)\r\nsoup = BeautifulSoup(res.read())\r\n\r\ndef printText(tags):\r\n for tag in tags:\r\n if tag.__class__ == NavigableString:\r\n print tag\r\n else:\r\n printText(tag)\r\n\r\nfor data in soup('span',{'class':'headline'}):\r\n printText(data)\r\n\r\nroot@hostname:\/home\/shimizu\/python# python scraping-bs4-6.py\r\n\u56fd\u969b\u9678\u9023\u304c\u4e0d\u6b63\u306e\u8a3c\u62e0\u96a0\u853d\u304b\u2026\u30c9\u30fc\u30d4\u30f3\u30b0\u7591\u60d1\r\n\uff082015\u5e7408\u670816\u65e5 21\u664252\u5206)\r\n\u8208\u5357\u3001\uff12\u70b9\u5dee\u304b\u3089\u7c98\u308a\u2026\u9ce5\u7fbd\u306b\u9006\u8ee2\u52dd\u3061\u3067\uff18\u5f37\r\n\uff082015\u5e7408\u670816\u65e5 18\u664234\u5206)\r\n\uff12\u304b\u6708\u3076\u308a\u5148\u767a\u306e\uff27\u30fb\u5927\u7af9\u3001\uff11\u5931\u70b9\u3082\u767d\u661f\u306a\u3089\u305a\r\n\uff082015\u5e7408\u670816\u65e5 18\u664227\u5206)\r\n...\r\n<\/pre>\n\u53c2\u8003URL<\/h1>\n
Python\u3068Beautiful Soup\u3067\u30b9\u30af\u30ec\u30a4\u30d4\u30f3\u30b0
\nhttp:\/\/qiita.com\/itkr\/items\/513318a9b5b92bd56185
\nBeautiful Soup
\nhttp:\/\/kondou.com\/BS4\/
\nPython \u3067\u7c21\u5358\u306a\u30c6\u30ad\u30b9\u30c8\u51e6\u7406 (3) – Beautiful Soup \u3092\u4f7f\u3063\u3066\u30b9\u30af\u30ec\u30a4\u30d4\u30f3\u30b0
\nhttp:\/\/jutememo.blogspot.jp\/2008\/06\/python-3-beautiful-soup.html
\nPython\u3067\u30b9\u30af\u30ec\u30a4\u30d4\u30f3\u30b0
\nhttp:\/\/i2bskn.hateblo.jp\/entry\/20120820\/1345478918
\n\u30b9\u30af\u30ec\u30a4\u30d4\u30f3\u30b0\u307e\u3068\u3081
\nhttp:\/\/matome.naver.jp\/odai\/2136441394137974201<\/p>\n","protected":false},"excerpt":{"rendered":"
Python\u306e\u30e9\u30a4\u30d6\u30e9\u30ea(Beautiful Soup)\u3092\u5229\u7528\u3057\u3066\u30b9\u30af\u30ec\u30a4\u30d4\u30f3\u30b0\u3057\u3066\u307f\u305f \u30a4\u30f3\u30b9\u30c8\u30fc\u30eb\u3068\u30bf\u30b0\u64cd\u4f5c root@hostname:\/home\/shimizu\/python# aptitude install […]<\/p>\n","protected":false},"author":1,"featured_media":0,"comment_status":"open","ping_status":"closed","sticky":false,"template":"","format":"standard","meta":{"_monsterinsights_skip_tracking":false,"_monsterinsights_sitenote_active":false,"_monsterinsights_sitenote_note":"","_monsterinsights_sitenote_category":0,"footnotes":""},"categories":[77,76],"tags":[],"_links":{"self":[{"href":"https:\/\/tech.akat.info\/index.php?rest_route=\/wp\/v2\/posts\/2047"}],"collection":[{"href":"https:\/\/tech.akat.info\/index.php?rest_route=\/wp\/v2\/posts"}],"about":[{"href":"https:\/\/tech.akat.info\/index.php?rest_route=\/wp\/v2\/types\/post"}],"author":[{"embeddable":true,"href":"https:\/\/tech.akat.info\/index.php?rest_route=\/wp\/v2\/users\/1"}],"replies":[{"embeddable":true,"href":"https:\/\/tech.akat.info\/index.php?rest_route=%2Fwp%2Fv2%2Fcomments&post=2047"}],"version-history":[{"count":3,"href":"https:\/\/tech.akat.info\/index.php?rest_route=\/wp\/v2\/posts\/2047\/revisions"}],"predecessor-version":[{"id":2050,"href":"https:\/\/tech.akat.info\/index.php?rest_route=\/wp\/v2\/posts\/2047\/revisions\/2050"}],"wp:attachment":[{"href":"https:\/\/tech.akat.info\/index.php?rest_route=%2Fwp%2Fv2%2Fmedia&parent=2047"}],"wp:term":[{"taxonomy":"category","embeddable":true,"href":"https:\/\/tech.akat.info\/index.php?rest_route=%2Fwp%2Fv2%2Fcategories&post=2047"},{"taxonomy":"post_tag","embeddable":true,"href":"https:\/\/tech.akat.info\/index.php?rest_route=%2Fwp%2Fv2%2Ftags&post=2047"}],"curies":[{"name":"wp","href":"https:\/\/api.w.org\/{rel}","templated":true}]}}