import json
import pickle
import requests
from bs4 import BeautifulSoup
import sys


'''携程旅游景点爬虫'''
class ctripSpider():
	def __init__(self):
		self.proxy = {'http': ''}
		self.count = 1
		self.pageCount = 1
	'''运行爬虫'''
	def start(self):
		headers = {
					'User-Agent': 'user-agent: Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/76.0.3809.100 Safari/537.36'
					}
		url = 'https://you.ctrip.com/sight/xiamen21/s0-p%s.html#sightname'
		max_pages = 102
		page_count = 1
		results = {}
		while True:
			try:
				self.__updateProxy()
				res = requests.get(url % page_count, headers=headers, proxies=self.proxy)
				# res = requests.get(url % page_count, headers=headers)			
				print("Try....Proxy::"+self.proxy['http']+"\n")
				# break
				soup = BeautifulSoup(res.text, features='lxml')
				list_wide_mod2 = soup.find_all('div', class_='list_wide_mod2')[0]
				for each1, each2 in zip(list_wide_mod2.find_all('dl'), list_wide_mod2.find_all('ul', class_='r_comment')):
					name = each1.dt.a.text
					addr = each1.find_all('dd')[0].text.strip()
					level = each1.find_all('dd')[1].text.strip().split('|')[0].strip()
					if '携程' in level:
						level = 'unknow'
					try:
						price = each1.find_all('span', class_='price')[0].text.strip().replace('¥', '')
					except:
						price = 'unknow'
					score = each2.find_all('a', class_='score')[0].text.strip().replace('\xa0分', '')
					num_comments = each2.find_all('a', class_='recomment')[0].text.strip()[1: -3]
					results[name] = [addr, level, price, score, num_comments]
				page_count += 1
				print('[INFO]:爬取进度: %s/%s...' % (page_count-1, max_pages))
			except:		
				print("expected error:", sys.exc_info())						
				# sys.exit()
				# print("\n....except....\n")				
				self.__updateProxy()				

			if page_count == max_pages:
				break
		print('[INFO]:数据爬取完毕, 将保存在data.pkl中...')
		with open('data.pkl', 'wb') as f:
			pickle.dump(results, f)
	'''更新代理'''
	def __updateProxy(self):
		# print("===========__updateProxy OR EXception===============\n")				
		url = "https://proxyapi.horocn.com/api/v2/proxies?order_id=CPMO1661411152778373&num=10&format=json&line_separator=win&can_repeat=yes&user_token=ed61bf900b76274d26bb240bd279e9d6"
		headers = {
					'User-Agent': 'user-agent: Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/76.0.3809.100 Safari/537.36'
					}

		self.pageCount += 1
		changeProxyCount = 5 #每5次更换一次代理

		if self.count > changeProxyCount:
			print("Proxy IP GET.........\n")
			# res = requests.get(url, headers=headers)
			# print(res.text)
			# res_json = json.loads(res.text)
			# self.proxy['http'] = 'http://%s:%s' % (res_json['data'][0]['host'], res_json['data'][0]['port'])

			with open("proxy.json",'r',encoding='utf-8') as pf:
				res_json = json.load(pf)
			proxyIndex = (self.pageCount // changeProxyCount) - 1
			self.proxy['http'] = 'http://%s:%s' % (res_json['data'][proxyIndex]['host'], res_json['data'][proxyIndex]['port'])			
			self.count = 1
		else:
			self.count += 1


if __name__ == '__main__':
	spider = ctripSpider()
	spider.start()

'''
携程景点数据可视化
jm
'''
import os
import json
import jieba
import pickle
from pyecharts.globals import ThemeType
from pyecharts import options as opts
from pyecharts.charts import Bar
from pyecharts.charts import Pie
from pyecharts.charts import Funnel
from wordcloud import WordCloud



'''柱状图(2维)'''
def drawBar(title, data, savepath='.\\results'):
	if not os.path.exists(savepath):
		print("dir is not exists")		
		os.mkdir(savepath)
	else:
		print("dir is exists")

	bar = Bar()

	attrs = [i for i, j in data.items()]
	values = [j for i, j in data.items()]

	try:		
		print(os.path.join(savepath,'%s.html' % title))
		bar.add_xaxis(attrs).add_yaxis(title,values).set_global_opts(title_opts=opts.TitleOpts(title=title)).render(os.path.join(savepath,'%s.html' % title))
	except Exception as e:
		print(e)
		exit(0)
	

'''饼图'''
def drawPie(title, data, savepath='.\\results'):
	
	if not os.path.exists(savepath):
		os.mkdir(savepath)

	pie = Pie(init_opts=opts.InitOpts(theme=ThemeType.WESTEROS))
	# pie = Pie()	
	itemList = sorted(data.items(),key=lambda x: x[1],reverse=True)
	# attrs = [i for i, j in data.items()]
	# values = [j for i, j in data.items()]
	pie.add(title,itemList,radius=["50%","70%"],label_opts=opts.LabelOpts(is_show=True))\
	.set_global_opts(title_opts=opts.TitleOpts(title=title),legend_opts=opts.LegendOpts(pos_left="right",orient="vertical"))\
	.render(os.path.join(savepath,'%s.html' % title))


'''漏斗图'''
def drawFunnel(title, data, savepath='.\\results'):
	if not os.path.exists(savepath):
		os.mkdir(savepath)

	funnel = Funnel()

	items = sorted(data.items(),key= lambda x: x[1],reverse=True)
	funnel.add(title,items,label_opts=opts.LabelOpts(is_show=True))\
	.set_global_opts(title_opts=opts.TitleOpts(title=title),legend_opts=opts.LegendOpts(pos_left="right",orient="vertical"))\
	.render(os.path.join(savepath, '%s.html' % title))	


'''词云'''
def drawWordCloud(words, title, savepath='.\\results'):
	if not os.path.exists(savepath):
		os.mkdir(savepath)
	wc = WordCloud(font_path='.\\simkai.ttf', background_color='white', max_words=2000, width=1920, height=1080, margin=5)
	wc.generate_from_frequencies(words)
	wc.to_file(os.path.join(savepath, title+'.png'))


'''统计词频'''
def statistics(texts, stopwords):
	words_dict = {}
	for text in texts:
		temp = jieba.cut(text)
		for t in temp:
			if t in stopwords or t == 'unknow':
				continue
			if t in words_dict.keys():
				words_dict[t] += 1
			else:
				words_dict[t] = 1
	return words_dict


'''run'''
if __name__ == '__main__':
	with open('data.pkl', 'rb') as f:
		all_data = pickle.load(f)
	'''词云'''
	# stopwords = open('stopwords.txt', 'r', encoding='utf-8').read().split('\n')[:-1]
	# texts = [each[1][0] for each in all_data]
	# words_dict = statistics(texts, stopwords)
	# drawWordCloud(words_dict, '景点位置词云', savepath='.\\results')
	'''评分分布'''
	scores = {}
	for key, value in all_data.items():
		if value[3] in scores:
			scores[value[3]] += 1
		else:
			scores[value[3]] = 1
	# print(scores)
	# exit(0)
	drawPie('景区评分分布', scores)
	'''评级分布'''
	levels = {}
	print('AAAAA级景区有: ')
	for key, value in all_data.items():
		if not value[1] or value[1] == 'unknow':
			continue
		if 'AAAAA' in value[1]:
			print(key)
		if value[1] in levels:
			levels[value[1]] += 1
		else:
			levels[value[1]] = 1	
	drawBar('景区评级分布', levels)
	'''价格分布'''
	prices = {'50元以下': 0, '50-100元': 0, '100-200元': 0, '200元以上': 0}
	for key, value in all_data.items():
		if value[2] == 'unknow' or not value[2]:
			continue
		price = float(value[2])
		if price < 50:
			prices['50元以下'] += 1
		elif price >= 50 and price < 100:
			prices['50-100元'] += 1
		elif price >= 100 and price < 200:
			prices['100-200元'] += 1
		elif price >= 200:
			prices['200元以上'] += 1
	
	drawFunnel('景区价格分布', prices)
	'''评论量最多的10个景区'''
	comments = {}
	for key, value in all_data.items():
		if value[-1] == '暂无' or not value[-1]:
			continue
		value[-1] = value[-1].replace('条', '')
		if len(comments.keys()) < 10:
			comments[key] = int(value[-1])
			continue
		if int(value[-1]) > min(list(comments.values())):
			comments[key] = int(value[-1])
			abandoned_key = list(comments.keys())[list(comments.values()).index(min(list(comments.values())))]
			del comments[abandoned_key]
	comments = sorted(comments.items(),key=lambda x: x[1],reverse=True)
	comments = dict(comments)
	
	drawBar('评论人数最多的10个景区', comments)