\u55e8\u5440A1\u597d\u6c14\u554a.jpg<\/p>\n
\u767d\u7ed9\u767d\u7ed9\uff0c\u8fd9\u6b21A2\u968f\u4fbf\u5199\u7b97\u4e86\uff0c\u53cd\u6b63\u600e\u4e48\u641e\u90fd\u662f70\u591a\u3002\u3002\u3002\u5c31\u5f88\u96be\u53d7_(:\u0437\u300d\u2220)_<\/p>\n
<\/p>\n
\u53ef\u80fd\u4ee5\u4e3a\u5206\u6570\u95ee\u9898\uff0c\u6240\u4ee5A2\u52a0\u4e86\u5f88\u591a\u6279\u6ce8\uff0c\u5c31\u4e0d\u5f80\u6587\u7ae0\u91cc\u585e\u522b\u7684\u6ce8\u91ca\u4e86\u3002\u3002\u55ef<\/p>\n
!pip install wget\r\n!apt-get install openjdk-8-jdk-headless -qq > \/dev\/null\r\n!wget -q https:\/\/archive.apache.org\/dist\/spark\/spark-2.4.0\/spark-2.4.0-bin-hadoop2.7.tgz\r\n!tar xf spark-2.4.0-bin-hadoop2.7.tgz\r\n!pip install -q findspark\r\nimport os\r\nos.environ[\"JAVA_HOME\"] = \"\/usr\/lib\/jvm\/java-8-openjdk-amd64\"\r\nos.environ[\"SPARK_HOME\"] = \"\/content\/spark-2.4.0-bin-hadoop2.7\"\r\n\r\nimport findspark\r\nfindspark.init()\r\nfrom pyspark.sql import SparkSession \r\nfrom pyspark import SparkContext\r\nfrom pyspark.sql import SQLContext\r\nsc = SparkContext.getOrCreate()<\/pre>\nimport wget\r\nlink_to_data = 'https:\/\/github.com\/tulip-lab\/sit742\/raw\/master\/Assessment\/2019\/data\/bank.csv'\r\nDataSet = wget.download(link_to_data)\r\n\r\n!ls\r\n\r\n# Import the 'bank.csv' as a Spark dataframe and name it as df\r\nspark = SparkSession.builder.appName('ml-bank').getOrCreate()\r\ndf = spark.read.csv('bank.csv', header = True, inferSchema = True) \r\n\r\n# Check data distribution\r\n# You may use printSchema()\r\n\r\n#Use printSchema() to show the content of dataset\r\ndf.printSchema()\r\n\r\n#Showing first 5 rows of data\r\ndf.show(5)\r\n\r\n#Using describe() to show distribution of numerical data.\r\ndf.describe(['age', 'balance', 'day', 'duration', 'campaign', 'pdays', 'previous']).show()<\/pre>\n#Select features ('age', 'job', 'marital', 'education', 'default', 'balance', 'housing', 'loan', 'campaign', 'pdays', 'previous', 'poutcome', 'deposit') as df2\r\n#Selecting required attributes as df2 \r\ndf2=df.select('age', 'job', 'marital', 'education', 'default', 'balance', 'housing', 'loan', 'campaign', 'pdays', 'previous', 'poutcome', 'deposit')\r\n\r\n#Showing contents of df2\r\ndf2.printSchema()\r\n\r\n#Showing first 5 rows of df2\r\ndf2.show(5)\r\n\r\n\r\n#Remove invalid rows\/records using spark.sql \r\n#Register df2 as table \"bank\"\r\ndf2.registerTempTable(\"bank\")\r\n\r\n#Using spark.sql filter out rows with attributes with \"unknown\"\r\n#Not all attribute are having \"unknown\" value, so not all attributes put into this section.\r\n#Hence, poutcome can only be \"success\" or \"failure\", so row with \"other\" in this attribute also will be filter out. \r\ndf3=spark.sql(\"SELECT * FROM bank WHERE job <> 'unknown' AND education <> 'unknown' AND poutcome <> 'unknown' AND poutcome <> 'other'\")\r\n\r\n#Showing contents of filtered dataframe\r\ndf3.printSchema() \r\n\r\n#..and first 5 rows\r\ndf3.show(5)\r\n\r\n\r\n#Covert categorical features to metric features using One hot encoding\r\nfrom pyspark.ml import Pipeline\r\nfrom pyspark.ml.feature import StringIndexer, OneHotEncoderEstimator\r\nfrom pyspark.ml.feature import VectorAssembler\r\n\r\n#Transforme categorical data into indexes for further transformation by StringIndexer.\r\n#This step is to selecting categorical attributes for transforme into indexes\r\nindexers=[StringIndexer(inputCol=column, outputCol=column+\"_index\").fit(df3) for column in list(set(df3.columns)-set(['age', 'balance', 'campaign', 'pdays', 'previous','deposit']))]\r\n\r\n#Transforme multiple categorical attributres by with the help of Pipeline.\r\npipeline=Pipeline(stages=indexers)\r\ndf4=pipeline.fit(df3).transform(df3)\r\n\r\n#Defining what columns should use One-Hot Encoding, and also the output columns.\r\nencoder=OneHotEncoderEstimator(inputCols=[\"housing_index\", \"marital_index\", \"default_index\", \"loan_index\", \"job_index\", \"education_index\", \"poutcome_index\"],\r\n outputCols=[\"housing_vec\", \"marital_vec\", \"default_vec\", \"loan_vec\", \"job_vec\", \"education_vec\", \"poutcome_vec\"])\r\n\r\n#Step of One-Hot Encoding transformation.\r\nmodel_ohe=encoder.fit(df4)\r\ndf5=model_ohe.transform(df4)\r\n\r\n#Delect the original and index data for categorical attribues, only left the transformed result and numerical data.\r\ndf6=df5.select([column for column in df5.columns if column not in [\"housing\", \"marital\", \"default\", \"loan\", \"job\", \"education\", \"poutcome\", \r\n \"housing_index\", \"marital_index\", \"default_index\", \"loan_index\", \"job_index\", \"education_index\", \"poutcome_index\"]])\r\n\r\n#Assemble all features into one attribute, for prepare of normalization.\r\nassembler=VectorAssembler(inputCols=[\"age\", \"balance\", \"campaign\", \"pdays\", \"previous\", \"job_vec\", \"education_vec\", \"marital_vec\", \"poutcome_vec\", \"housing_vec\", \"loan_vec\", \"default_vec\"], outputCol=\"features\")\r\ndf7=assembler.transform(df6)\r\n\r\n#Only left the feature attribute.\r\ndf8=df7.select(\"features\")\r\n\r\n#...and also showing the content\r\ndf8.printSchema()\r\n\r\n#Bibliography\r\n#https:\/\/stackoverflow.com\/questions\/32277576\/how-to-handle-categorical-features-with-spark-ml\r\n#https:\/\/stackoverflow.com\/questions\/36942233\/apply-stringindexer-to-several-columns-in-a-pyspark-dataframe\r\n#https:\/\/spark.apache.org\/docs\/latest\/ml-features.html#onehotencoderestimator\r\n\r\n\r\nfrom pyspark.ml.feature import StringIndexer\r\n#Because the attribute \"deposit\" is also the label, so we select \"deposit\" to a single dataframe as y\r\ny_df1=df3.select(\"deposit\")\r\n\r\n#...also transforme label into index type by using StringIndexer\r\nindexer=StringIndexer(inputCol=\"deposit\", outputCol=\"deposit_index\")\r\ny_df2=indexer.fit(y_df1).transform(y_df1)\r\n\r\n#...and store as y\r\ny=y_df2.select('deposit_index')\r\n\r\ny.printSchema()<\/pre>\n#Apply Min-Max normalisation on each attribute using MinMaxScaler \r\n\r\nfrom pyspark.ml.feature import MinMaxScaler\r\n\r\n#Using MinMaxScaler for column \"features\" normalnization\r\nscaler=MinMaxScaler(inputCol=\"features\", outputCol=\"scaledFeatures\")\r\nscalerModel=scaler.fit(df8)\r\nscaled=scalerModel.transform(df8)\r\n\r\n#Print the scaled range\r\nprint(\"Features scaled to range: [%f, %f]\" % (scaler.getMin(), scaler.getMax()))\r\n\r\n#...and also store as dataframe X\r\nX=scaled.select(\"scaledFeatures\")\r\nX.show(truncate=False)\r\n\r\n#Bibliography\r\n#https:\/\/spark.apache.org\/docs\/2.2.0\/ml-features.html#minmaxscaler\r\n\r\n\r\nfrom pyspark.sql.functions import monotonically_increasing_id \r\n\r\n#Generating \"id\" for both X and y dataframe for merging\r\nX_index = X.select(\"*\").withColumn(\"id\", monotonically_increasing_id())\r\ny_index = y.select(\"*\").withColumn(\"id\", monotonically_increasing_id())\r\n\r\n#Merging X and y into one dataframe\r\ndata_model = y_index.join(X_index, \"id\", \"outer\").drop(\"id\") \r\n\r\n#Transform into dataframe, store as df100\r\ndf100=data_model.toDF('label', 'features')\r\ndf100.show()<\/pre>\n# Perform unsupervised learning on df2 with k-means \r\n# You can use whole df2 as both training and testing data, \r\n# Evaluate the clustering result using Accuracy. \r\n\r\nfrom pyspark.ml.clustering import KMeans\r\nfrom pyspark.ml.evaluation import ClusteringEvaluator\r\nfrom pyspark.ml.evaluation import MulticlassClassificationEvaluator\r\nfrom pyspark.ml.evaluation import BinaryClassificationEvaluator \r\nfrom pyspark.ml.feature import StringIndexer\r\nfrom pyspark.sql.types import DoubleType\r\n\r\n#Setting up k-means cluster parameter\r\n#Because we need to \"estimate\" the label of data, so we only performe 2 cluster for getting close to the binomial label...\r\nk_means=KMeans().setK(2).setSeed(1)\r\n\r\n#Applying k-means cluster\r\ncluster=k_means.fit(df100)\r\ncluster_predictions=cluster.transform(df100)\r\n\r\n#First evalustor for cluster: ClusteringEvaluator, by using silhouette criteria\r\nevaluator=ClusteringEvaluator()\r\nsilhouette = evaluator.evaluate(cluster_predictions)\r\nprint(\"Silhouette with squared distnce is :\" + str(silhouette))\r\n\r\n#The second evalustor: MulticlassClassificationEvaluator, by using accuracy criteria\r\nkmeans_acc_evaluator = MulticlassClassificationEvaluator()\r\n\r\n#But first we need to transfrome datatype for k-means model's prediction: integer to binomial\r\ncluster_predictions.printSchema()\r\ncluster_predictions_1 = cluster_predictions.withColumn(\"prediction\", cluster_predictions[\"prediction\"].cast(\"double\"))\r\n\r\n#Then apply accuracy criteria for cluster model\r\nprint(\"Test Accuracy: \" + str(kmeans_acc_evaluator.evaluate(cluster_predictions_1, {kmeans_acc_evaluator.metricName: \"accuracy\"})))\r\n\r\n#Showing the centroids of cluster.\r\ncenters = cluster.clusterCenters()\r\nprint(\"Cluster Centers: \")\r\nfor center in centers:\r\n print(center)\r\n\r\n#...and showing some prediction result for k-means cluster model\r\ncluster_predictions_1.show()\r\n\r\n#Bibliography\r\n#https:\/\/stackoverflow.com\/questions\/32284620\/how-to-change-a-dataframe-column-from-string-type-to-double-type-in-pyspark\r\n#https:\/\/spark.apache.org\/docs\/2.4.2\/ml-clustering.html\r\n#https:\/\/jakevdp.github.io\/PythonDataScienceHandbook\/05.11-k-means.html<\/pre>\n#Generate a scatter plot using the first two PCA components to investigate the data distribution.\r\nfrom pyspark.ml.feature import PCA, VectorAssembler\r\nfrom pyspark.ml.linalg import Vectors\r\nimport matplotlib.pyplot as plt\r\nimport numpy as np\r\n\r\n#Setting up PCA parameters, only 2 compinents for plotting\r\npca = PCA(k=2, inputCol=\"scaledFeatures\", outputCol=\"pcaFeatures\")\r\n\r\n#Applying PCA for dataset (df100)\r\nmodel_pca = pca.fit(X)\r\nresult_pca = model_pca.transform(X)\r\n\r\n#Showing some result for PCA\r\nresult_pca.show(truncate=False)\r\n\r\n#We need to transforme PCA components into numpy array format for plotting...\r\n#Transforme to numpy array also benefits for splitting two PCA components.\r\nnp_pca_1 = np.array(result_pca.select('pcaFeatures').collect())\r\n\r\n#Some format setting for numpy array for plotting...removing 1 level for array\r\nnp_pca = np_pca_1[ :, 0, :]\r\n\r\n#Scatter plot settings, including size of graph, title, x\/y labels, and dot size\r\nplt.figure(dpi=120)\r\nplt.suptitle('PCA features Scatter plot', fontsize=15)\r\nplt.xlabel('components0', fontsize=10)\r\nplt.ylabel('components1', fontsize=10)\r\nplt.scatter(np_pca[:,0], np_pca[:,1], s=5, alpha=1)\r\n\r\n#Bibliography\r\n#https:\/\/stackoverflow.com\/questions\/12444716\/how-do-i-set-the-figure-title-and-axes-labels-font-size-in-matplotlib\r\n#https:\/\/stackoverflow.com\/questions\/34007632\/how-to-remove-a-column-in-a-numpy-array\/34008274\r\n#https:\/\/stackoverflow.com\/questions\/42116143\/extracting-numpy-array-from-pyspark-dataframe<\/pre>\n#Splitting dataframe into traning set and test set\r\ntrain, test = df100.randomSplit([0.7, 0.3], seed = 742)\r\n\r\n#...and print how many rows in both traning and test set. \r\nprint(\"Training Dataset Count: \" + str(train.count()))\r\nprint(\"Test Dataset Count: \" + str(test.count()))\r\n\r\n#Bibliography\r\n#https:\/\/towardsdatascience.com\/machine-learning-with-pyspark-and-mllib-solving-a-binary-classification-problem-96396065d2aa\r\n##This Bibliography can be use for almost of the Supervised learning part.\r\n\r\n\r\n# Logistic Regression\r\nfrom pyspark.ml.classification import LogisticRegression\r\nfrom pyspark.ml.evaluation import MulticlassClassificationEvaluator\r\nfrom pyspark.ml.evaluation import BinaryClassificationEvaluator\r\n\r\n#Setting up LogisticRegression parametrs\r\nlr = LogisticRegression(featuresCol = 'features', labelCol = 'label', maxIter=10)\r\n\r\n#Training LogisticRegression model\r\nlrModel = lr.fit(train)\r\n\r\n#...and apply to test set\r\nlr_predictions = lrModel.transform(test)\r\n\r\n#Showing some prediction result for LogisticRegression\r\nlr_predictions.select('label', 'features', 'rawPrediction', 'prediction', 'probability').show(10)\r\n\r\n#Selecitng BinaryClassificationEvaluator and MulticlassClassificationEvaluator as the ROC\/accuracy evaluator\r\nlr_evaluator = BinaryClassificationEvaluator()\r\nlr_acc_evaluator = MulticlassClassificationEvaluator()\r\n\r\n#Generating traning summary for LogisticRegression\r\nlr_trainingSummary = lrModel.summary\r\n\r\n#Transforming summary's ROC part to dataframe\r\nlr_roc = lr_trainingSummary.roc.toPandas()\r\n\r\n#...and start plotting\r\nplt.figure(dpi=120)\r\nplt.title('ROC Curve', fontsize=15)\r\nplt.xlabel('True Positive Rate', fontsize=10)\r\nplt.ylabel('False Positive Rate', fontsize=10)\r\nplt.plot(lr_roc['FPR'],lr_roc['TPR'])\r\nplt.show()\r\n\r\n#Showing AUC for training set\r\nprint('Training set areaUnderROC: ' + str(lr_trainingSummary.areaUnderROC))\r\n\r\n#Showing AUC for test set\r\nprint(\"Test Area Under ROC: \" + str(lr_evaluator.evaluate(lr_predictions, {lr_evaluator.metricName: \"areaUnderROC\"})))\r\n\r\n#Showing accuracy for Logistic Regression predicting test set.\r\nprint(\"Test Accuracy: \" + str(lr_acc_evaluator.evaluate(lr_predictions, {lr_acc_evaluator.metricName: \"accuracy\"})))\r\n\r\n#Exam the coefficients\r\nimport matplotlib.pyplot as plt\r\nimport numpy as np\r\n\r\n#Storing coefficients into numpy array\r\nbeta = np.sort(lrModel.coefficients)\r\n\r\n#...and plotting\r\nplt.plot(beta)\r\nplt.ylabel('Beta Coefficients')\r\nplt.show()<\/pre>\n#Decision tree\r\nfrom pyspark.ml.classification import DecisionTreeClassifier \r\n\r\n#Setting up Decision Tree Classifier\r\ndt = DecisionTreeClassifier(featuresCol = 'features', labelCol = 'label', maxDepth = 5)\r\n\r\n#Training Decision Tree model...\r\ndtModel = dt.fit(train)\r\n\r\n#...and apply to test set\r\ndt_predictions = dtModel.transform(test)\r\n\r\n#Showing some predictions for Decision Tree model on test set\r\ndt_predictions.select('label', 'features', 'rawPrediction', 'prediction', 'probability').show(10)\r\n\r\n#Importing two evaluator as same as LogisticRegression\r\ndt_evaluator = BinaryClassificationEvaluator()\r\ndt_acc_evaluator = MulticlassClassificationEvaluator()\r\n\r\n#Showing AUC for test set\r\nprint(\"Test Area Under ROC: \" + str(dt_evaluator.evaluate(dt_predictions, {dt_evaluator.metricName: \"areaUnderROC\"})))\r\n\r\n#Showing accuracy for test set\r\nprint(\"Test Accuracy: \" + str(dt_acc_evaluator.evaluate(dt_predictions, {dt_acc_evaluator.metricName: \"accuracy\"})))<\/pre>\n#NaiveBayes\r\nfrom pyspark.ml.classification import NaiveBayes\r\nfrom pyspark.ml.evaluation import MulticlassClassificationEvaluator\r\n\r\n#Setting up NaiveBayes model\r\nby = NaiveBayes(featuresCol = 'features', labelCol = 'label')\r\n\r\n#Training NaiveBayes model...\r\nbyModel = by.fit(train)\r\n\r\n#...and apply to test set\r\nby_predictions = byModel.transform(test)\r\n\r\n#Showing some predictions for aiveBayes model on test set\r\nby_predictions.select('label', 'features', 'rawPrediction', 'prediction', 'probability').show(10)\r\n\r\n#Importing two evaluator as same as LogisticRegression\r\nby_evaluator = BinaryClassificationEvaluator()\r\nby_acc_evaluator = MulticlassClassificationEvaluator()\r\n\r\n#Showing AUC for test set\r\nprint(\"Test Area Under ROC: \" + str(by_evaluator.evaluate(by_predictions, {by_evaluator.metricName: \"areaUnderROC\"})))\r\n\r\n#Showing accuracy for test set\r\nprint(\"Test Accuracy: \" + str(by_acc_evaluator.evaluate(by_predictions, {by_acc_evaluator.metricName: \"accuracy\"})))<\/pre>\n\u5c31\u8fd9\u6837\u5427\u3002\u3002\u3002\u53cd\u6b63\u6709\u6279\u6ce8\u3002\u3002<\/p>\n
\u6700\u8fd1\u592a\u61d2\u4e86\u3002\u3002\u5509_(:\u0437\u300d\u2220)_<\/p>\n","protected":false},"excerpt":{"rendered":"
\u55e8\u5440A1\u597d\u6c14\u554a.jpg \u767d\u7ed9\u767d\u7ed9\uff0c\u8fd9\u6b21A2\u968f\u4fbf\u5199\u7b97\u4e86\uff0c\u53cd\u6b63\u600e\u4e48\u641e\u90fd\u662f70\u591a\u3002\u3002\u3002\u5c31\u5f88\u96be\u53d7_(:\u0437\u300d\u2220)_<\/p>\n","protected":false},"author":1,"featured_media":0,"comment_status":"open","ping_status":"open","sticky":false,"template":"","format":"standard","meta":{"footnotes":""},"categories":[2,1],"tags":[],"class_list":["post-592","post","type-post","status-publish","format-standard","hentry","category-school-things","category-uncategorized"],"_links":{"self":[{"href":"https:\/\/blog.luke6887.me\/index.php?rest_route=\/wp\/v2\/posts\/592","targetHints":{"allow":["GET"]}}],"collection":[{"href":"https:\/\/blog.luke6887.me\/index.php?rest_route=\/wp\/v2\/posts"}],"about":[{"href":"https:\/\/blog.luke6887.me\/index.php?rest_route=\/wp\/v2\/types\/post"}],"author":[{"embeddable":true,"href":"https:\/\/blog.luke6887.me\/index.php?rest_route=\/wp\/v2\/users\/1"}],"replies":[{"embeddable":true,"href":"https:\/\/blog.luke6887.me\/index.php?rest_route=%2Fwp%2Fv2%2Fcomments&post=592"}],"version-history":[{"count":8,"href":"https:\/\/blog.luke6887.me\/index.php?rest_route=\/wp\/v2\/posts\/592\/revisions"}],"predecessor-version":[{"id":600,"href":"https:\/\/blog.luke6887.me\/index.php?rest_route=\/wp\/v2\/posts\/592\/revisions\/600"}],"wp:attachment":[{"href":"https:\/\/blog.luke6887.me\/index.php?rest_route=%2Fwp%2Fv2%2Fmedia&parent=592"}],"wp:term":[{"taxonomy":"category","embeddable":true,"href":"https:\/\/blog.luke6887.me\/index.php?rest_route=%2Fwp%2Fv2%2Fcategories&post=592"},{"taxonomy":"post_tag","embeddable":true,"href":"https:\/\/blog.luke6887.me\/index.php?rest_route=%2Fwp%2Fv2%2Ftags&post=592"}],"curies":[{"name":"wp","href":"https:\/\/api.w.org\/{rel}","templated":true}]}}