#读取TXT中的单词
def txt_file():
    with open(filename)  as file:
        raw_words = file.read().lower()
        words = re.findall('[a-z]+',raw_words) #得到所有的英文单词，排除了汉语单词，各种符号
    return words 

#统计单词数目，按照词频逆排序，然后从出现个数最多的单词开始记忆，迅速提高学习效率
def count(words):
    words_set = set(words)
    number_words = len(words) #六级题总共出现的单词数目，包括重复的单词数目
    number_words_set = len(words_set) #需要记忆的单词个数
    print('需要记忆的单词个数',number_words_set)
    print('总共出现的单词数目',number_words)

    '''
    #利用函数Counter统计次数
    count_words = Counter(words)
    print(count_words)
    number_per_words = count_words.items()
    #print(number_per_words)
    '''

    #用hash计算每个单词出现次数
    hash = {}
    for word in words:
        if word not in hash:
            hash[word] = 1
        else:
            hash[word] += 1
    sorted_hash = sorted(hash,  reverse = False) #按照ABC顺序排列,没有重复单词,只有英文单词
    #sorted_hash = sorted(hash.values(), reverse = True)
    #print(sorted_hash)
    data = sorted(hash.items(), key=lambda x: x[1], reverse=True) #按照出现次数排列，没有重复单词
    return data #返回列表，每个元素是一个元组，左边是单词，右边是单词出现次数

#爬取有道翻译网页进行翻译
import urllib.request
import urllib.parse
import json
import time
#进行单词翻译
#https://blog.csdn.net/qaz3171210/article/details/51247230
def translate(word):
    content = word 
    url = 'http://fanyi.youdao.com/translate?smartresult=dict&smartresult=rule'
    data = {}
    data['i'] = content
    data['from']= 'AUTO'
    data['to'] = 'AUTO'
    data['smartresult'] = 'dict'
    data['client'] = 'fanyideskweb'
    data['salt'] = '1527480457870'
    data['sign'] = 'abf2d2301303ac6d8ae48c532c631060'
    data['doctype'] = 'json'
    data['version'] = '2.1'
    data['keyfrom'] = 'fanyi.web'
    data['action'] = 'FY_BY_REALTIME'
    data['typoResult'] = 'false'
    data = urllib.parse.urlencode(data).encode('utf-8')
    time.sleep( 6 )  #间隔几秒调取网页翻译功能，防止被屏蔽
    response = urllib.request.urlopen(url,data)
    html = response.read().decode('utf-8')
    target = json.loads(html)
    #print(target['translateResult'][0][0]['tgt'])
    return target['translateResult'][0][0]['tgt']
    #print(word,end = '              ')
    #print("翻译结果：%s" % (target['translateResult'][0][0]['tgt']))

#主函数开始
import os
file_output = '/Users/apple/Documents/ST/python/英语项目/雅思红宝书.csv'
filenames = os.listdir(os.getcwd()) #all the filenames in the current folder
all_words = []
#words = []
for filename in filenames:

    #排除多余的文件，防止重复计数
    if '.csv' in filename:
        continue
    if '.py' in filename:
        continue
    print(filename)

    #统计每个pdf文件里面的单词，并且放到一个列表words里面
    if '.pdf' in filename: 
        words = pdf_file()
        print('len_words is :', len(words))

    #统计每个Word文件里面的单词，并且放到一个列表words里面
    if '.docx' in filename:
        words = docx_file()
        print('len_words is :', len(words))

    #统计每个TXT文件里面的单词，并且放到一个列表words里面
    if '.txt' in filename:
        words = txt_file()
        print('len_words is :', len(words))

    #统计所有文件中的英文单词，并且放进列表里
    #all_words = all_words.extend(words)
    all_words = all_words + words 
    print('len_all_words is :', len(all_words))

data = count(all_words) ##得到列表，每个元素是一个元组，左边是单词，右边是单词出现次数
word_text = ''
for w in data:
    word = w[0] #获取每个元组里的单词
    #print(chinese)
    word_text = word_text + word + '\n'#所有单词按照词频降序排列写入一个字符串里

    这里需要爬取网页翻译，耗费大量时间，可以自己选择运行
    chinese = translate(word)
    word_text = word_text + chinese + '\n'#所有单词写入一个

open(file_output,'a').write(word_text)