164 lines
6.7 KiB
Python
164 lines
6.7 KiB
Python
import pandas as pd
|
||
import os
|
||
import datetime
|
||
import time
|
||
import pymysql
|
||
|
||
"""
|
||
1.读取excel/csv文件，需判定导入数据格式为excel或csv或txt
|
||
2.
|
||
"""
|
||
|
||
def get_dates(files):
|
||
star_time = time.time()
|
||
file_count = range(len(files))
|
||
file_zip = zip(file_count, files)
|
||
names = locals()
|
||
file_list = []
|
||
print('---{}开始读取文件夹中数据---'.format(datetime.datetime.now()))
|
||
for i, j in file_zip:
|
||
if os.path.splitext(j)[-1][1:] == 'xlsx':
|
||
names['df' + str(i)] = pd.read_excel(file_path + '/' + j)
|
||
names['df' + str(i)].drop_duplicates(inplace=True)
|
||
print('---{} {}共计{}条数据---'.format(datetime.datetime.now(), j, len(names['df' + str(i)])))
|
||
elif os.path.splitext(j)[-1][1:] == 'csv':
|
||
names['df' + str(i)] = pd.read_csv(file_path + '/' + j)
|
||
names['df' + str(i)].drop_duplicates(inplace=True)
|
||
print('---{} {}共计{}条数据---'.format(datetime.datetime.now(), j, len(names['df' + str(i)])))
|
||
file_list.append(names['df' + str(i)])
|
||
return file_list
|
||
|
||
def get_drop_dates(): # 这里可以直接调用链接数据库的类
|
||
conn = pymysql.connect(
|
||
host = 'cori0108.top',
|
||
user = 'cori', # 数据库帐号
|
||
password = 'corition0108', # 数据库访问密码
|
||
db = 'ln_database', # 数据库名
|
||
port = 3306
|
||
)
|
||
df_drop = pd.read_sql('select OPENID from `官微推文-基准去除数据`', conn)
|
||
return df_drop
|
||
|
||
# def get_drop_dates(): # 这里可以直接调用链接数据库的类
|
||
# conn = pymysql.connect(
|
||
# host = 'nasofcori.fun',
|
||
# user = 'root', # 数据库帐号
|
||
# password = '123456', # 数据库访问密码
|
||
# db = 'cori_database', # 数据库名
|
||
# port = 3306
|
||
# )
|
||
# df_drop = pd.read_sql('select OPENID from `官微推文-基准去除数据`', conn)
|
||
# return df_drop
|
||
|
||
# def get_drop_dates(): # 这里可以直接调用链接数据库的类
|
||
# conn = pymysql.connect(
|
||
# host = 'localhost',
|
||
# user = 'root', # 数据库帐号
|
||
# password = 'corition0108', # 数据库访问密码
|
||
# db = 'database', # 数据库名
|
||
# port = 3306
|
||
# )
|
||
# df_drop = pd.read_sql('select OPENID from `常规推文数据`', conn)
|
||
# return df_drop
|
||
|
||
# 数据去重
|
||
def duplicates_dates(file_list, df_drop):
|
||
print('---{} 开始合并{}组推文---'.format(datetime.datetime.now(), len(files)))
|
||
df_all = pd.concat(file_list).drop_duplicates() #合并去重
|
||
print('---{} 合并完成！{}组推文合并去重后合计{}条数据---'.format(datetime.datetime.now(), len(files), len(df_all)))
|
||
df_all_filter = df_all[~ df_all['OPENID'].isin(df_drop['OPENID'])] #获取去除基准数据后的df
|
||
print('---{} 去除12月关注名单后后合计剩余{}条数据---'.format(datetime.datetime.now(), len(df_all_filter)))
|
||
return df_all_filter
|
||
|
||
# 合并受限名单
|
||
def duplicates_drop_dates(file_list, df_drop, limit):
|
||
file_list.append(df_drop)
|
||
print('---{} 开始合并常规刨除组推文---'.format(datetime.datetime.now()))
|
||
df_all = pd.concat(file_list).drop_duplicates() # 合并去重
|
||
print('---{} 合并完成！常规刨除推文合并去重后合计{}条数据---'.format(datetime.datetime.now(), len(df_all)))
|
||
|
||
df_all.columns = ['用户识别码']
|
||
if len(df_all) > limit:
|
||
n = len(df_all) // limit + 1
|
||
print(n)
|
||
for i in range(0, n):
|
||
df_save = df_all.iloc[i * limit + 1:(i + 1) * limit + 1, :]
|
||
new_file_name = save_path + '/常规刨除' + str(i) + '_new.xlsx'
|
||
print('---{} 开始导出文件，去重后合计{}条数据'.format(datetime.datetime.now(), len(df_save)))
|
||
df_save.to_excel(new_file_name, index=False)
|
||
print('---{} 已成功导出{}文件--'.format(datetime.datetime.now(), new_file_name))
|
||
else:
|
||
new_file_name = save_path + '/' + '常规刨除_new.xlsx'
|
||
print('---{} 开始导出文件，去重后合计{}条数据'.format(datetime.datetime.now(), len(df_all)))
|
||
df_all.to_excel(new_file_name, index=False)
|
||
print('---{} 已成功导出{}文件--'.format(datetime.datetime.now(), new_file_name))
|
||
|
||
|
||
|
||
def eachgroup_dates(file_list, df_all_filter, save_path):
|
||
names = locals()
|
||
|
||
j = 0
|
||
for dfs in file_list:
|
||
names['df' + str(j)] = dfs
|
||
j += 1
|
||
|
||
new_file_list =[]
|
||
i = 0
|
||
for file in files:
|
||
print('---{} 开始进行分组去重---'.format(datetime.datetime.now()))
|
||
if i == 0: # 剥离第一组数据
|
||
df_new = pd.merge(names['df' + str(i)], df_all_filter, on='OPENID')
|
||
|
||
elif i > 0 & i < len(files) - 1:
|
||
df_all_filter = df_all_filter[
|
||
~ df_all_filter['OPENID'].isin(names['df' + str(i - 1)]['OPENID'])]
|
||
df_new = pd.merge(names['df' + str(i)], df_all_filter, on='OPENID')
|
||
|
||
else:
|
||
df_new = df_all_filter[~ df_all_filter['OPENID'].isin(names['df' + str(i - 1)]['OPENID'])]
|
||
|
||
print(df_new)
|
||
df_new.rename(columns={'OPENID': '用户识别码'}, inplace=True)
|
||
# df_new.columns = ['用户识别码']
|
||
df_new['备注（选填）'] = datetime.date.today()
|
||
new_file_list.append(df_new)
|
||
i += 1
|
||
|
||
return new_file_list
|
||
|
||
def export_dates(new_file_list, save_path, files, limit):
|
||
j = 0
|
||
for file in new_file_list:
|
||
filename = files[j].split('.')
|
||
if len(file) > limit:
|
||
n = len(file) // limit + 1
|
||
print(n)
|
||
for i in range(0, n + 1):
|
||
df_save = file.iloc[i * limit + 1:(i + 1) * limit + 1, :]
|
||
new_file_name = save_path + '/' + filename[0] + str(i) + '_new.xlsx'
|
||
print('---{} 开始导出文件，去重后合计{}条数据'.format(datetime.datetime.now(), len(df_save)))
|
||
df_save.to_excel(new_file_name, index=False)
|
||
print('---{} 已成功导出{}文件--'.format(datetime.datetime.now(), new_file_name))
|
||
else:
|
||
new_file_name = save_path + '/' + filename[0] + '_new.xlsx'
|
||
print('---{} 开始导出文件，去重后合计{}条数据'.format(datetime.datetime.now(), len(file)))
|
||
file.to_excel(new_file_name, index=False)
|
||
print('---{} 已成功导出{}文件--'.format(datetime.datetime.now(), new_file_name))
|
||
|
||
j += 1
|
||
|
||
file_path = r'D:\常用文件\1-数据分组待处理'
|
||
save_path = r'D:\常用文件\2-数据分组已处理'
|
||
files = os.listdir(file_path)
|
||
limit = 300000
|
||
|
||
file_list = get_dates(files)
|
||
df_drop = get_drop_dates()
|
||
df_all_filter = duplicates_dates(file_list, df_drop)
|
||
new_file_list = eachgroup_dates(file_list, df_all_filter, save_path)
|
||
export_dates(new_file_list, save_path, files, limit)
|
||
duplicates_drop_dates(file_list, df_drop, limit)
|
||
|
||
|