print(datas)

#去掉全为 NAN 的行
datas = datas.dropna(axis=0, how = "all")
#去掉全为 NAN 的列
datas = datas.dropna(axis = 1, how = "all")

    for index,row in datas.iterrows():
        #print(row["hobbies"])
        row.iloc[2] = row["hobbies"].replace(",", "，")
        row.iloc[2] = row["hobbies"].replace(" ", "，")
        row.iloc[2] = row["hobbies"].replace("、", "，")

row.iloc[2] = row["hobbies"].split("，")

['程序设计', '美食品鉴', '影视欣赏']
['时事政治', '佳肴美酒', '美与生活']
['听音乐', '看电影', '打篮球', 'rushB']
['看电影', '阅读', '听歌']
['唱歌', '篮球', '阅读', '演讲']
['美食', '电影', '乒乓球']

arr = []
for index,row in datas.iterrows():
	row.iloc[2] = row["hobbies"].replace(",", "，")
    row.iloc[2] = row["hobbies"].replace(" ", "，")
    row.iloc[2] = row["hobbies"].replace("、", "，")

    row.iloc[2] = row["hobbies"].split("，")
   	arr += [x for x in row.iloc[2] if x != ""]
    #if 判断条件用于删除空白值

['高雅艺术', '欧洲文学', '人生哲学', '程序设计', '美食品鉴', '影视欣赏', '时事政治', '佳肴美酒', '美与生活', '听音乐'......]

my_dict = {"Name": arr}
df = pandas.DataFrame(my_dict)
df.to_csv("test.csv",encoding='utf_8_sig')

#!/usr/bin/python
# -*- coding: utf-8 -*-


import os
import csv
import re
import logging
import optparse

import dedupe
from unidecode import unidecode


def preProcess(column):
    """
    Do a little bit of data cleaning with the help of Unidecode and Regex.
    Things like casing, extra spaces, quotes and new lines can be ignored.
    """
    column = unidecode(column)
    column = re.sub('  +', ' ', column)
    column = re.sub('\n', ' ', column)
    column = column.strip().strip('"').strip("'").lower().strip()
    # If data is missing, indicate that by setting the value to `None`
    if not column:
        column = None
    return column


def readData(filename):
    """
    Read in our data from a CSV file and create a dictionary of records,
    where the key is a unique record ID and each value is dict
    """

    data_d = {}
    with open(filename, encoding="utf-8") as f:
        reader = csv.DictReader(f)
        for row in reader:
            clean_row = [(k, preProcess(v)) for (k, v) in row.items()]
            print(clean_row)
            print("row:", row)
            row_id = int(row['\ufeffId'])
            data_d[row_id] = dict(clean_row)

    return data_d


if __name__ == '__main__':

    # ## Logging

    # Dedupe uses Python logging to show or suppress verbose output. This
    # code block lets you change the level of loggin on the command
    # line. You don't need it if you don't want that. To enable verbose
    # logging, run `python examples/csv_example/csv_example.py -v`
    optp = optparse.OptionParser()
    optp.add_option('-v', '--verbose', dest='verbose', action='count',
                    help='Increase verbosity (specify multiple times for more)'
                    )
    (opts, args) = optp.parse_args()
    log_level = logging.WARNING
    if opts.verbose:
        if opts.verbose == 1:
            log_level = logging.INFO
        elif opts.verbose >= 2:
            log_level = logging.DEBUG
    logging.getLogger().setLevel(log_level)

    # ## Setup

    #input_file = 'csv_example_messy_input.csv'
    #output_file = 'csv_example_output.csv'
    input_file = "test.csv"
    output_file = "testout.csv"
    settings_file = 'csv_example_learned_settings'
    training_file = 'csv_example_training.json'

    print('importing data ...')
    data_d = readData(input_file)

    # If a settings file already exists, we'll just load that and skip training
    if os.path.exists(settings_file):
        print('reading from', settings_file)
        with open(settings_file, 'rb') as f:
            deduper = dedupe.StaticDedupe(f)
    else:
        # ## Training

        # Define the fields dedupe will pay attention to
        fields = [
            {'field': '\ufeffId', 'type':"Text"},
            {'field': "Name", 'type': 'String'}
            #{'field': 'Site name', 'type': 'String'},
            #{'field': 'Address', 'type': 'String'},
            #{'field': 'Zip', 'type': 'Exact', 'has missing': True},
            #{'field': 'Phone', 'type': 'String', 'has missing': True},
            ]

        # Create a new deduper object and pass our data model to it.
        deduper = dedupe.Dedupe(fields)

        # If we have training data saved from a previous run of dedupe,
        # look for it and load it in.
        # __Note:__ if you want to train from scratch, delete the training_file
        if os.path.exists(training_file):
            print('reading labeled examples from ', training_file)
            with open(training_file, 'rb') as f:
                deduper.prepare_training(data_d, f)
        else:
            deduper.prepare_training(data_d)

        # ## Active learning
        # Dedupe will find the next pair of records
        # it is least certain about and ask you to label them as duplicates
        # or not.
        # use 'y', 'n' and 'u' keys to flag duplicates
        # press 'f' when you are finished
        print('starting active labeling...')

        dedupe.console_label(deduper)

        # Using the examples we just labeled, train the deduper and learn
        # blocking predicates
        deduper.train()

        # When finished, save our training to disk
        with open(training_file, 'w') as tf:
            deduper.write_training(tf)

        # Save our weights and predicates to disk.  If the settings file
        # exists, we will skip all the training and learning next time we run
        # this file.
        with open(settings_file, 'wb') as sf:
            deduper.write_settings(sf)

    # ## Clustering

    # `partition` will return sets of records that dedupe
    # believes are all referring to the same entity.

    print('clustering...')
    clustered_dupes = deduper.partition(data_d, 0.5)

    print('# duplicate sets', len(clustered_dupes))
    print(clustered_dupes)
    # ## Writing Results

    # Write our original data back out to a CSV with a new column called
    # 'Cluster ID' which indicates which records refer to each other.

    cluster_membership = {}
    for cluster_id, (records, scores) in enumerate(clustered_dupes):
        for record_id, score in zip(records, scores):
            cluster_membership[record_id] = {
                "Cluster ID": cluster_id,
                "confidence_score": score
            }
    print()
    print(cluster_membership)

    with open(output_file,  "w",encoding="utf_8_sig") as f_output, open(input_file, encoding="utf-8") as f_input:

        reader = csv.DictReader(f_input)
        fieldnames = ['Cluster ID', 'confidence_score'] + reader.fieldnames

        writer = csv.DictWriter(f_output, fieldnames=fieldnames)
        writer.writeheader()

        for row in reader:
            row_id = int(row['\ufeffId'])
            row.update(cluster_membership[row_id])
            writer.writerow(row)

result_df = pandas.read_csv("testout.csv", encoding="utf-8")
    cluster_id_dict = {}
    #创建字典，方便查找
    for index, row in result_df.iterrows():
        key = row["Cluster ID"]
        value = row["Name"]

        if key in cluster_id_dict.keys():
            cluster_id_dict[key].add(value)
        else:
            cluster_id_dict[key] = set()
            cluster_id_dict[key].add(value)

g = Graph('http://localhost:7474', user='neo4j', password='*********')

g = Graph('http://localhost:7474', user='neo4j', password='*********')
used_id = []
    for index, row in datas.iterrows():
        name = row["name"]
        id = row["id"]
        name_node = Node("Person", name=name)
        g.create(name_node)
        #暂时不存储 id_node（学号）
        # id_node = Node("A", name=id)
        # g.create(id_node)
        # re = Relationship(name_node, "学号", id_node)
        # g.create(re)
        for hobby in row["hobbies"]:
            temp_cluster_id = -1
            for key, value in cluster_id_dict.items():
                if hobby in value:
                    temp_cluster_id = key
                    if temp_cluster_id in used_id:    #之前已经用过
                        hobby = list(value)[0]                                        #统一使用第一个元素
                    else:                                                       #没用过
                       used_id.append(key)
                    break                             #找到即只有一个，跳出

            hobby_node = Node("Hobby", name=hobby)
            g.create(hobby_node)
            re = Relationship(name_node, "LIKES", hobby_node)
            g.create(re)