{"id":550,"date":"2019-03-27T06:03:14","date_gmt":"2019-03-26T22:03:14","guid":{"rendered":"https:\/\/luke6887.me\/?p=550"},"modified":"2019-03-28T19:59:06","modified_gmt":"2019-03-28T11:59:06","slug":"sit742-%e5%85%b3%e4%ba%8e%e4%bd%9c%e4%b8%9aa1","status":"publish","type":"post","link":"https:\/\/blog.luke6887.me\/?p=550","title":{"rendered":"SIT742 \u5173\u4e8e\u4f5c\u4e1aA1"},"content":{"rendered":"

\u554a\u3002\u3002\u6211\u4e00\u4e2a\u5b8c\u5168\u6ca1\u57fa\u7840\u7684\u4eba\uff0c\u600e\u4e48\u5c31\u9009\u4e86\u4e2aPython\u7684\u8bfe\u5462\u3002\u3002\u3002<\/p>\n

SIT742\u7684A1\uff0c\u5927\u81f4\u5c31\u662f\u6570\u636e\u7684\u9884\u5904\u7406\uff0c\u5305\u62ec\u6570\u5b57\u8ddf\u6587\u672c\u7684\u9884\u5904\u7406\uff0c\u4f46\u56e0\u4e3a\u7ed9\u7684\u6570\u636e\u662fkaggle\u7684\u7ea2\u9152\u6d4b\u8bc4\uff08\u94fe\u63a5\uff1ahttps:\/\/www.kaggle.com\/zynicide\/wine-review<\/a>s\uff09\uff0c\u611f\u89c9\u5c31\u662f\u3002\u3002\u6709\u70b9\u96be\u641e\u3002\u3002<\/p>\n

\u538b\u529b\u9a6c\u65af\u5185_(:\u0437\u300d\u2220)_<\/p>\n

<\/p>\n

\u9996\u5148\u662f\u6570\u636e\u7684\u83b7\u53d6\u3002\u3002\u56e0\u4e3a\u8001\u5e08\u5728GitHub\u653e\u4e86\u6240\u4ee5\u5c31\u5f88\u7b80\u5355\u3002\u3002<\/p>\n

!pip install wget\r\nimport wget\r\nlink_to_data = 'https:\/\/github.com\/tulip-lab\/sit742\/raw\/master\/Assessment\/2019\/data\/wine.json'\r\nDataSet = wget.download(link_to_data)\r\nlink_to_data = 'https:\/\/github.com\/tulip-lab\/sit742\/raw\/master\/Assessment\/2019\/data\/stopwords.txt'\r\nDataSet = wget.download(link_to_data)<\/pre>\n
\u7136\u540e\u5c31\u662f\u65e5\u5e38\u7684\u6a21\u5757\u5bfc\u5165<\/p>\n
import json as js\r\nimport numpy as np\r\nimport pandas as pd\r\nimport matplotlib.pyplot as plt\r\nfile = 'wine.json'<\/pre>\n
\u7136\u540e\u3002\u3002\u3002\u56e0\u4e3a\u539f\u6765\u6570\u636e\u662fjson\u683c\u5f0f\u7684\u3002\u3002\u3002\u7a0d\u5fae\u7528jq\u770b\u4e00\u4e0b\u7ed3\u6784\uff0c\u5206\u8fa8\u51facolumn name\u5c31\u7ee7\u7eed\u64cd\u4f5c\u4e86\u3002\u3002<\/p>\n
import json\r\nfrom pandas import DataFrame\r\n\r\nwith open(\"wine.json\") as f:\r\n    wine=json.load(f)\r\ntype(wine)\r\n#\u5b9e\u9645\u4e0a\u4e0a\u9762\u4e09\u884c\u90fd\u4e0d\u77e5\u9053\u6709\u6ca1\u6709\u7528\u3002\u3002\u3002\r\n\r\ndf = DataFrame(wine, columns = ['points','title','description','taster_name','taster_twitter_handle','price','designation','variety','region_1','region_2','province','country','winery'])\r\n#\u5148\u6784\u9020dataframe\r\ndf1 = df.dropna(subset=['points', 'price', 'variety', 'country'])\r\n#\u7136\u540e\u628a\u67d0\u4e9b\u5217\u7684\u7f3a\u5931\u503c\u53bb\u6389\u3002\u3002\u6bd5\u7adf\u4ea7\u5730\u554a\u4ef7\u683c\u4ec0\u4e48\u7684\u4fe1\u606f\u7f3a\u4e86\u5c31\u6ca1\u4e86\r\ndf1.shape[0] - df1.count()\r\ndf1['points'] = df1['points'].astype('float')\r\n#\u56e0\u4e3apoints\u5f88\u5947\u602a\u3002\u3002\u660e\u660e\u662ffloat\u5374\u8ba4\u6210object\u3002\u3002\u5f97\u8f6c\u8fc7\u6765\r\ndf1.dtypes<\/pre>\n
\u524d\u7f6e\u5de5\u4f5c\u5c31\u5b8c\u6210\u4e86\u3002\u63a5\u4e0b\u6765\u5c31\u662f\u7b2c\u4e00\u90e8\u5206\u7684Numerical analysis\u4e86\uff08\u5176\u5b9e\u5c31\u662f\u9884\u5904\u7406\u3002\u3002\u3002\uff09<\/p>\n
1.1 Explore the data distribution for each column.<\/p>\n
# write your code here\r\n# you may use functions such as describe() on each attribute.\r\ndf1.describe()\r\n#\u7ed3\u679c\u53ea\u6709points\u8ddfprice\u7684\u7edf\u8ba1\u3002\u3002\u8fd9\u4e2a\u6162\u6162\u6539<\/pre>\n
1.2 Find the 10 varieties of wine which receives the highest number of reviews<\/p>\n
# write your code here\r\n# you may use functions such as value_counts()  \r\ndf1['variety'].value_counts().head(10)<\/pre>\n
1.3 Find varieties of wine having the average price less than 20, with the average pointsat least 90<\/p>\n
# write your code here\r\n# you may use functions such as groupby() \r\n##groupby variety -> calculate poings\/price -> select by 20\/90\r\ndf1.dtypes\r\ndf2 = df1.groupby('variety').agg({'points':'mean', 'price':'mean'})\r\n#\u5148\u7528groupby\u6309variety\u7ed9\u91522\u5206\u7c7b\uff0c\u7136\u540e\u540e\u9762\u63a5agg\u6765\u505a\u8fd0\u7b97\u3002\u3002\u3002\r\ndf2[(df2[\"points\"]>90) & (df2[\"price\"]<20)]\r\n#\u7136\u540e\u518d\u6309\u6761\u4ef6\u7b5b\u9009\u3002\u3002\u3002<\/pre>\n
\n
\n
1.4 Build statistic table<\/p>\n
#\u5176\u5b9e\u8981\u6c42\u5c31\u662f4\u5217\uff1a\u56fd\u5bb6\uff0c\u8be5\u56fd\u5bb6\u6700\u6d41\u884c\u7684\u9152\u7684\u54c1\u79cd\uff0c\u8fd9\u54c1\u79cd\u7684\u5e73\u5747\u8bc4\u5206\uff0c\u8fd8\u6709\u5e73\u5747\u4ef7\u683c\r\n# write your code here\r\n# you may use functions such as groupby() and round(decimals=2)\r\ndef top_value_count(x, n=1):\r\n    return x.value_counts().head(n)\r\n#\u8fd9\u4e2a\u6211\u6709\u7528\u5417\u3002\u3002\u3002_(:\u0437\u300d\u2220)_\r\n\r\ndf3 = df1.groupby('country').agg({'points':'mean', 'price':'mean'}).round(decimals=2)\r\n#\u5bf9\u4e8e\u5e73\u5747\u8bc4\u5206\u8ddf\u4ef7\u683c\uff0c\u6545\u6280\u91cd\u65bd\u7528groupby\u5728\u8ba1\u7b97\u5c31\u597d\u3002\u3002\u3002\u5e76\u4e14\u7528decimals=2\u9650\u5b9a\u4e24\u4f4d\u5c0f\u6570\r\n\r\nse1 = df1.groupby('country')['variety'].value_counts()\r\nse2 = se1.groupby('country').head(1)\r\n#\u8fd9\u91cc\u5c31\u5f88\u90a3\u5565\u4e86\u3002\u3002\u6309\u56fd\u5bb6\u6765\u6392\u9152\u7684\u54c1\u79cd\u6392\u540d\uff0c\u7136\u540e\u7528head(1)\u6765\u53ea\u4fdd\u7559\u6bcf\u4e2a\u56fd\u5bb6\u7684\u7b2c\u4e00\u884c\u3002\u3002\r\ndf4 = se2.to_frame()\r\ndf5 = df4.drop(['variety'], axis=1)\r\n#\u7ecf\u8fc7\u8fd9\u79cd\u64cd\u4f5c\u4ee5\u540e\u6570\u636e\u662f\u53d8\u6210series\u4e86\uff0c\u6362\u56dedataframe\u7136\u540e\u518d\u628a\u90a3\u4e2a\u8ba1\u6570\u5220\u6389\u3002\u3002\u3002\r\n\r\ndf6 = df5.join(df3.reindex(df5.index, level=0))\r\ndf6.rename(columns={'country':'Country', 'variety':'Variety', 'points':'AvgPoints', 'price':'AvgPrice'}, inplace = True)\r\ndf6\r\n#\u6700\u540e\u628a\u56fd\u5bb6+\u54c1\u79cd(df5)\u8ddf\u5e73\u5747\u8bc4\u5206\u4ef7\u683c(df3)\u62fc\u5728\u4ee5\u524d\uff0c\u6539\u4e0bcolumn name,\u5c31\u7b97\u662f\u6210\u4e86\u3002\u3002\u3002_(:\u0437\u300d\u2220)_<\/pre>\n
# save your table to 'statisticByState.csv'\r\ndf6.to_csv('statisticByState.csv', \r\n            encoding='utf-8', \r\n            index=True, \r\n            header=True)\r\n!dir\r\n#\u6700\u540e\u5199\u5165\u6587\u4ef6\u3002\u3002\u3002<\/pre>\n
\u8fd9\u6837\u5c31\u7b97\u641e\u5b8c\u7b2c\u4e00\u90e8\u5206\u7684\u6570\u636e\u9884\u5904\u7406\/\u5206\u6790\u4e86\uff0c\u63a5\u4e0b\u6765\u5c31\u662f\u7b2c\u4e8c\u90e8\u5206\u7684\u6587\u672c\u5904\u7406\u4e86\u3002\u3002<\/p>\n
2.1 extract high requency words in description<\/p>\n
import re\r\nimport nltk\r\nfrom nltk.tokenize import RegexpTokenizer\r\nfrom nltk.probability import *\r\nfrom itertools import chain\r\n#from tqdm import tqdm\r\nimport codecs\r\nfrom nltk.tokenize import word_tokenize as WordTokenizer\r\nnltk.download('wordnet')\r\n#\u5176\u5b9e\u4e5f\u641e\u4e0d\u6e05\u695a\u8981\u54ea\u51e0\u4e2a\u3002\u3002\u3002_(:\u0437\u300d\u2220)_<\/pre>\n
with open('stopwords.txt') as f:\r\n    stop_words = f.read().splitlines()\r\nstop_words = set(stop_words)<\/pre>\n
#pre-process-description-only\r\ndf10 = df1.drop(['points','title','taster_name','taster_twitter_handle','price','designation','variety','region_1','region_2','province','country','winery'], axis=1)<\/pre>\n
\u63a5\u4e0b\u6765\u624d\u662f\u771f\u6b63\u7684\u6587\u672c\u9884\u5904\u7406\u3002\u3002\u3002<\/p>\n
# write your code here\r\n# define your tokenize\r\n# tokenization->lower case->stop words->stemming\/lemma->Sentence Segmentation\r\n# transformm case->tokenize->stem->stop_word->filter length\/sentence...\r\ntokenizer = RegexpTokenizer(r\"\\w+(?:[-']\\w+)?\") \r\ntokens = df10['description'].apply(tokenizer.tokenize)\r\n#\u7b2c\u4e00\u6b65\uff0ctokenization\uff0c\u5176\u5b9e\u5c31\u662f\u628a\u8bcd\u5f00\uff0c\u628a\u53e5\u5b50\u62c6\u6210\u5355\u8bcd\uff0c\u7528\u7684\u901a\u7528\u8868\u8fbe\u5f0f\u662f\u8001\u5e08\u7ed9\u7684\u3002\u3002_(:\u0437\u300d\u2220)_\r\n\r\ntokens = [k.lower() for l in tokens for k in l]\r\n#\u7b2c\u4e8c\u6b65\uff0clowercase\uff0c\u5927\u5199\u53d8\u5c0f\u5199\r\n\r\ntokens = [word for word in tokens if word not in stop_words]\r\n#\u7b2c\u4e09\u6b65\uff0c\u8fc7\u6ee4stopwords\uff0c\u5c31\u662f\u628aa\uff0cthe\u4e4b\u7c7b\u7684\u73a9\u610f\u8fc7\u6ee4\u6389\u3002\u3002\u3002\r\n\r\nfrom nltk.stem import WordNetLemmatizer\r\nlemmatizer = WordNetLemmatizer()\r\n['{0} -> {1}'.format(w, lemmatizer.lemmatize(w)) for w in tokens]\r\n#\u7b2c\u56db\u6b65\uff0c\u8f6c\u8bcd\u6839\uff0c\u5176\u5b9e\u4e5f\u6709stemming\u7684\uff0c\u4e0d\u8fc7\u542c\u8bf4\u8fd9\u4e2a\u597d\u4e00\u70b9\u3002\u3002\u3002\r\n\r\ntokens = [word for word in tokens if word.isalpha()]\r\n#\u6700\u540e\u4e00\u6b65\uff0c\u628atokenization\u6ca1\u8fc7\u6ee4\u5e72\u51c0\u7684\u6570\u5b57\u4ec0\u4e48\u7684\u8fc7\u6ee4\u6389\u3002\u3002\r\ntokens<\/pre>\n
\u7136\u540e\u5c31\u662f\u770b\u54ea\u4e9b\u8bcd\u662f\u9ad8\u9891\u8bcd\uff08\u51fa\u73b0\u8d85\u8fc75000\u6b21\uff09\uff0c\u7136\u540e\u5bfc\u51fa\u6765txt\u4e86\u3002\u3002\u3002<\/p>\n
# find top common words with document frequencies > 5000\r\n# you may use function FreqDist() and sort()\r\nfdist = FreqDist(tokens)\r\nlist1 = list(filter(lambda x: x[1]>=5000,fdist.items()))\r\n#\u5148\u7528FreqDist\u7edf\u8ba1\uff0c\u7136\u540e\u7528filter\u8fc7\u6ee4\u3002\u3002\u3002lambda\u5c31\u662f\u5f53\u573a\u505a\u4e9b\u5c0f\u51fd\u6570\uff0c\u50cf\u8fd9\u6837\u8bbe\u5b9a\u6761\u4ef6\u4ec0\u4e48\u7684\u3002\u3002\u3002\r\n\r\n# save your table to 'top_common_words.txt'  \r\nimport codecs\r\nf = codecs.open('top_common_words.txt','w')\r\nf.writelines(str(list1)+'\\r\\n')\r\n#\u7528\u7684codecs\u5199text\u3002\u3002\u611f\u89c9\u8fd8\u662f\u6709\u70b9\u4e11_(:\u0437\u300d\u2220)_<\/pre>\n
2.2 Find key words for describing Shiraz using TF-IDF<\/p>\n
\u4f20\u8bf4\u4e2d\u7684\u7279\u5f81\u751f\u6210\uff0c\u8fd8\u662f\u6700\u7ecf\u5178\u7684tf-idf<\/p>\n
\uff08\u5f53\u7136\u56e0\u4e3a\u8fd9\u4e2a\u662f\u5e26\u4e86\u6761\u4ef6\u7684\uff0c\u53ea\u770b\u770b\u8bcd\u8ddfShiraz\u8fd9\u4e2a\u54c1\u79cd\u6709\u4ec0\u4e48\u5173\u7cfb\u3002\u3002\u8fd8\u662f\u8981\u9884\u5904\u7406\u4e00\u4e0b\u7684\u3002\u3002\uff09<\/p>\n
# select 'description' from 'variety' eqaul to  'Shiraz' \r\n\r\ndf20 = df1.drop(['points','title','taster_name','taster_twitter_handle','price','designation','region_1','region_2','province','country','winery'], axis=1)\r\ndf21 = df20[df20['variety'].str.match('Shiraz')]\r\ndf22 = df21.drop(['variety'],axis=1)\r\n\r\ntokenizer = RegexpTokenizer(r\"\\w+(?:[-']\\w+)?\") \r\ntokens20 = df22['description'].apply(tokenizer.tokenize)\r\n\r\ntokens20 = [k.lower() for l in tokens20 for k in l]\r\n\r\ntokens20 = [word for word in tokens20 if word not in stop_words]\r\n\r\nfrom nltk.stem import WordNetLemmatizer\r\nlemmatizer = WordNetLemmatizer()\r\n['{0} -> {1}'.format(w, lemmatizer.lemmatize(w)) for w in tokens20]\r\n\r\ntokens20 = [word for word in tokens20 if word.isalpha()]\r\ntokens20<\/pre>\n
\u7136\u540e\u5c31\u662f\u641etf-idf\u4e86\u3002\u3002\u3002<\/p>\n
# use TfidfVectorizer to calculate TF-IDF score\r\nstr20 = \" \".join(tokens20)\r\nstr20 = [str20]\r\n\r\nfrom sklearn.feature_extraction.text import TfidfVectorizer\r\ntfidf = TfidfVectorizer(analyzer = \"word\")\r\ntfs = tfidf.fit_transform(str20)\r\n\r\nvocab = tfidf.get_feature_names()\r\n\r\nimport pandas as pd\r\ndf30 = pd.DataFrame({'word': [], 'weight': []})\r\nfor word, weight in zip(vocab, tfs.toarray()[0]):\r\n    df30 = df30.append({'word': word, 'weight': weight}, ignore_index=True)<\/pre>\n
\u7136\u540e\u5c31\u662f\u6309\u9898\u76ee\u7b5b\u9009\uff0c\u9ad8vector\u7684\u3002\u3002<\/p>\n
# find words with TF-IDF score >0.4 and sort them\r\ndf31 = df30[(df30[\"weight\"]>0.1)]\r\ndf31.sort_values([\"weight\"], axis=0, ascending=False, inplace=True) \r\ndf31\r\n#\u5176\u5b9e\u672c\u6765\u5e94\u8be5\u7b5b0,4\u4ee5\u4e0a\u7684\uff0c\u7ed3\u679c\u53d1\u73b0\u6211\u505a\u51fa\u6765\u6700\u9ad8\u7684\u4e5f\u5c310.3.\u3002\u3002\u611f\u89c9\u662f\u54ea\u91cc\u6709\u95ee\u9898\u3002\u3002\u3002\r\n#\u5148\u8fd9\u6837\u5427_(:\u0437\u300d\u2220)_\r\n\r\n# save your table to 'key_Shiraz.txt'   \r\ndf31.to_csv('key_Shiraz.txt', sep=' ', index=False)\r\n#\u5c45\u7136\u7528\u5199csv\u7684\u6a21\u5757\u5199\u4e86txt\uff0c\u8fd8\u80fd\u770b\uff0c\u8212\u670d_(:\u0437\u300d\u2220)_<\/pre>\n
\u603b\u4f53\u5c31\u5148\u8fd9\u6837\u3002\u3002\u80af\u5b9a\u6709\u4e1c\u897f\u80fd\u6539\u7684\u3002\u3002\u3002<\/p>\n
\u8fd9\u6837\u4ea4\u4e0a\u53bb\u6ca1\u95ee\u9898\u5427\u3002\u3002\u3002_(:\u0437\u300d\u2220)_<\/p>\n<\/div>\n<\/div>\n","protected":false},"excerpt":{"rendered":"
\u554a\u3002\u3002\u6211\u4e00\u4e2a\u5b8c\u5168\u6ca1\u57fa\u7840\u7684\u4eba\uff0c\u600e\u4e48\u5c31\u9009\u4e86\u4e2aPython\u7684\u8bfe\u5462\u3002\u3002\u3002 SIT742\u7684A1\uff0c\u5927\u81f4\u5c31\u662f\u6570\u636e\u7684\u9884\u5904\u7406\uff0c\u5305 … <\/p>\n
\u7ee7\u7eed\u9605\u8bfb\u201cSIT742 \u5173\u4e8e\u4f5c\u4e1aA1\u201d<\/span><\/a><\/p>\n","protected":false},"author":1,"featured_media":0,"comment_status":"open","ping_status":"open","sticky":false,"template":"","format":"standard","meta":{"footnotes":""},"categories":[2],"tags":[],"class_list":["post-550","post","type-post","status-publish","format-standard","hentry","category-school-things"],"_links":{"self":[{"href":"https:\/\/blog.luke6887.me\/index.php?rest_route=\/wp\/v2\/posts\/550","targetHints":{"allow":["GET"]}}],"collection":[{"href":"https:\/\/blog.luke6887.me\/index.php?rest_route=\/wp\/v2\/posts"}],"about":[{"href":"https:\/\/blog.luke6887.me\/index.php?rest_route=\/wp\/v2\/types\/post"}],"author":[{"embeddable":true,"href":"https:\/\/blog.luke6887.me\/index.php?rest_route=\/wp\/v2\/users\/1"}],"replies":[{"embeddable":true,"href":"https:\/\/blog.luke6887.me\/index.php?rest_route=%2Fwp%2Fv2%2Fcomments&post=550"}],"version-history":[{"count":23,"href":"https:\/\/blog.luke6887.me\/index.php?rest_route=\/wp\/v2\/posts\/550\/revisions"}],"predecessor-version":[{"id":580,"href":"https:\/\/blog.luke6887.me\/index.php?rest_route=\/wp\/v2\/posts\/550\/revisions\/580"}],"wp:attachment":[{"href":"https:\/\/blog.luke6887.me\/index.php?rest_route=%2Fwp%2Fv2%2Fmedia&parent=550"}],"wp:term":[{"taxonomy":"category","embeddable":true,"href":"https:\/\/blog.luke6887.me\/index.php?rest_route=%2Fwp%2Fv2%2Fcategories&post=550"},{"taxonomy":"post_tag","embeddable":true,"href":"https:\/\/blog.luke6887.me\/index.php?rest_route=%2Fwp%2Fv2%2Ftags&post=550"}],"curies":[{"name":"wp","href":"https:\/\/api.w.org\/{rel}","templated":true}]}}