开源部分的baseline，不提供任何技术支持，拒绝伸手党

Discussion

Home Discussion Board KDD CUP of Fresh Air 开源部分的baseline，不提供任何技术支持，拒绝伸手党

智浪淘沙

开源部分的baseline，不提供任何技术支持，拒绝伸手党

posted in KDD CUP of Fresh Air

2018-04-27 02:02

```python
# -*- coding: utf-8 -*-
"""
Created on Thu Apr  5 19:37:01 2018

@author: ZHILANGTAOSHA
"""

import pandas as pd

from lightgbm.sklearn import LGBMRegressor
import numpy as np

import os
from sklearn.preprocessing import LabelEncoder
import requests

headers = {
    'User-Agent':'Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/57.0.2987.133 Safari/537.36'
}

cw = {'wanshouxig_aq':'wanshouxigong_aq', 'aotizhongx_aq':'aotizhongxin_aq', 'nongzhangu_aq':'nongzhanguan_aq', 'fengtaihua_aq':'fengtaihuayuan_aq',
       'miyunshuik_aq':'miyunshuiku_aq', 'yongdingme_aq':'yongdingmennei_aq', 'xizhimenbe_aq':'xizhimenbei_aq'}
gz = {value:key for key,value in cw.items()}
ycsj = (pd.datetime.now() + pd.DateOffset(days=1)).strftime('%Y-%m-%d 00:00:00')
stat_latlon = pd.read_csv('../stat.csv',header = None)
stat_latlon.columns = ['dq','stationId','lat','lon']

def colchuli(x):
    if x == 'time':return 'utc_time'
    if x == 'station_id':return 'stationId'
    x = str(x).replace('_Concentration','').replace('.','').upper()
    if x == 'PM25':
        return 'PM2.5'
    else:
        return x
    
    
def colchulil(x):
    if x == 'time':return 'utc_time'
    if x == 'station_id':return 'stationName'
    x = str(x).replace('_Concentration','').replace('.','').lower()
    if x == 'PM25':
        return 'PM2.5'
    else:
        return x
def chulifloat(x):
    try:
        return float(x)
    except:
        return np.NaN

def smape(y_true, y_pred):
    denominator = (np.abs(y_true) + np.abs(y_pred)) / 2.0
    diff = np.abs(y_true - y_pred) / denominator
    diff[denominator == 0] = 0.0
    return np.mean(diff)

def get_bjaq_shuju():
    if os.path.exists('beijingaq.csv') & 0:
        beijingaq = pd.read_csv('beijingaq.csv',parse_dates = ['utc_time'])
        beijingaq.drop(['lat', 'lon'],axis = 1,inplace = True)
    else:
        beijing_aq_ls = pd.read_csv('../beijing_17_18_aq.csv',parse_dates = ['utc_time'])
        beijing_aq = pd.read_csv('../beijing_201802_201803_aq.csv',parse_dates = ['utc_time'])
        beijingaq = pd.concat([beijing_aq_ls,beijing_aq])
        beijingaq = beijingaq.drop_duplicates()
    time_start = beijingaq.utc_time.max().strftime('%Y-%m-%d-%H')
    time_end = pd.datetime.now().strftime('%Y-%m-%d-%H')
    url = 'https://biendata.com/competition/airquality/bj/%s/%s/2k0d1d8'%(time_start,time_end)
    respones= requests.get(url)
    df = pd.DataFrame([i.decode('utf8').split(',') for i in respones.content.splitlines()])
    df.columns = df.loc[0]
    df = df.shift(-1,axis = 0).dropna()
    df.columns = df.columns.map(colchuli)
    df = df[beijingaq.columns]
    df.utc_time = pd.to_datetime(df.utc_time)
    floatcl = [i for i in df.columns if i not in ['stationId', 'utc_time']]
    df.loc[:,floatcl] = df.loc[:,floatcl].applymap(chulifloat)
    beijingaq = pd.concat([beijingaq,df])
    stat_latlon = pd.read_csv('../stat.csv',header = None)
    stat_latlon.columns = ['dq','stationId','lat','lon']
    beijingaq = pd.merge(beijingaq,stat_latlon[['stationId','lat','lon']],on = 'stationId',how = 'left')
    beijingaq = beijingaq.sort_values(['utc_time','lat','lon','stationId'])
    for i in beijingaq.columns:
        beijingaq[i] = beijingaq[i].map(lambda x:x if x > 0 else np.NaN)
        beijingaq[i] = beijingaq[i].interpolate()
    beijingaq.to_csv('beijingaq.csv',index = False)
    return beijingaq

def get_ldaq_shuju():
    if os.path.exists('londonaq.csv') & 0:
        londonaq = pd.read_csv('londonaq.csv',parse_dates = ['utc_time'])
        londonaq.drop(['lat', 'lon'],axis = 1,inplace = True)
    else:
        londonaq = pd.read_csv('../London_historical_aqi_forecast_stations_20180331.csv',parse_dates = ['MeasurementDateGMT'])
        del londonaq['Unnamed: 0']
        londonaq.columns = ['utc_time','stationId','PM2.5','PM10','NO2']
        londonaq = londonaq.drop_duplicates()
    time_start = londonaq.utc_time.max().strftime('%Y-%m-%d-%H')
    time_end = pd.datetime.now().strftime('%Y-%m-%d-%H')
    url = 'https://biendata.com/competition/airquality/ld/%s/%s/2k0d1d8'%(time_start,time_end)
    respones= requests.get(url)
    df = pd.DataFrame([i.decode('utf8').split(',') for i in respones.content.splitlines()])
    df.columns = df.loc[0]
    df = df.shift(-1,axis = 0).dropna()
    df.columns = df.columns.map(colchuli)
    df = df[londonaq.columns]
    df.utc_time = pd.to_datetime(df.utc_time)
    floatcl = [i for i in df.columns if i not in ['stationId', 'utc_time']]
    df.loc[:,floatcl] = df.loc[:,floatcl].applymap(chulifloat)
    londonaq = pd.concat([londonaq,df])  
    stat_latlon = pd.read_csv('../stat.csv',header = None)
    stat_latlon.columns = ['dq','stationId','lat','lon']
    londonaq = pd.merge(londonaq,stat_latlon[['stationId','lat','lon']],on = 'stationId',how = 'left')
    londonaq = londonaq.sort_values(['utc_time','lat','lon','stationId'])
    for i in londonaq.columns:
        if i in ['utc_time','lat','lon','stationId'] : continue
        londonaq[i] = londonaq[i].map(lambda x:x if x > 0 else np.NaN)
        londonaq[i] = londonaq[i].interpolate()
    londonaq.to_csv('londonaq.csv',index = False)
    return londonaq

def get_time_feature(df,time = 'utc_time'):
    df['day'] = df[time].dt.day
    df['dayofweek'] = df[time].dt.dayofweek
    df['month'] = df[time].dt.month
    df['hour'] = df[time].dt.hour
    df['year'] = df[time].dt.year
    df['weekofyear'] = df[time].dt.weekofyear
    return df

def trainmodellv1(xlj,label,city,time):
    alldata  = xlj[['stationId', 'utc_time', label]]
    feature2 = alldata.copy()
    feature2.utc_time = feature2.utc_time + pd.DateOffset(days=2)
    feature2.columns = ['stationId', 'utc_time', 'qlt']
    alldata = get_time_feature(alldata)
    alldata = pd.merge(alldata,stat_latlon[['stationId','lat','lon']],on = ['stationId'],how = 'left')
    test = stat_latlon[stat_latlon.dq == city]
    if city == 'bj':
        ycj = pd.DataFrame(np.zeros([35,48]))
    else:
        ycj = pd.DataFrame(np.zeros([13,48]))
    ycj.index =test.stationId
    ycj.columns = pd.date_range(start = ycsj,periods = 48,freq = 'H')
    ycj = ycj.stack().reset_index()
    ycj.columns = ['stationId', 'utc_time', label]
    ycj = get_time_feature(ycj)
    ycj = pd.merge(ycj,stat_latlon[['stationId','lat','lon']],on = ['stationId'],how = 'left')
    ycj = pd.merge(ycj,feature2,on = ['stationId', 'utc_time'],how = 'left')
    ycj = ycj.sort_values(['utc_time','lat','lon','stationId'])
    for i in ycj.columns:
        ycj[i] = ycj[i].interpolate()
    alldata = pd.merge(alldata,feature2,on = ['stationId', 'utc_time'],how = 'inner')    
    features = [i for i in ycj.columns if i not in ['stationId', 'utc_time', label]]
    rf = LGBMRegressor()
    rf.fit(alldata[features],alldata[label])
    rf.predict(ycj[features])
    ycj[label] = rf.predict(ycj[features])
    time = pd.DataFrame(pd.date_range(start = ycsj,periods = 48,freq = 'H'))
    ycj['bh'] = ycj.utc_time.map(lambda x:list(time[time[0] == x].index)[0])    
    ycj['test_id'] = ycj.stationId.map(lambda x:gz[x] if x in gz.keys() else x ) + '#' + ycj.bh.astype(str)
    return ycj

```

23 comments