Crawl weather data

ACM SIGKDD • $36,500 • 6072 Team • 6662 participants

KDD CUP of Fresh Air

2018-03-15 - Launch

2018-05-07 - Team Merger Deadline

2018-06-01 - Close

Home Competitions

周杰

Crawl weather data

posted in KDD CUP of Fresh Air

2018-04-17 11:43

```python
#!/usr/bin/python
# -*- coding: UTF-8 -*-
import bs4
import re
import requests
import io
import sys
import unicodedata
import time
from dateutil import parser
import json

reload(sys)
sys.setdefaultencoding('utf-8')

# 抓取城市的数据，每小时的数据，以及一天的数据
# https://www.wunderground.com/history/airport/EGMC/2018/4/4/DailyHistory.html?req_city=London&req_statename=England
'''
ans{
	"hours_data": [[time,temp,windchill,dew_point,humidity,pressure,visiability,wind_dir,wind_speed,gust_speed,precip,events,conditions], [], ..., []]
	"day_data":[temp_avg,temp_max,temp_min,degree_day,dew_point,humidity_avg,humidity_max,humidity_min,rain_value,pressure,speed_avg,speed_max,visiability,events]
}
hours_data: 时间 (BST)	气温	风冷温	露点	湿度	气压	    能见度	    Wind Dir	风速	                    瞬间风速	Precip	活动	状况
样例：      12:20 AM	5.0 °C	2.5 °C	3.0 °C	87%	    1010 百帕	7.0 千米	东南偏南	11.1 公里/小时 / 3.1 m/s	-	        N/A	    中雨	小雨
day_data:   平均温度	最高温度	最低温度	采温度日数	露点	平均湿度	最高湿度	最小湿度	降水量	气压	风速	最快风速	能见度	活动
'''

def city_weather(year=2018, month=4, day=7, city="beijing"):
    ans = {}
    if city == "beijing":
        link = 'https://www.wunderground.com/history/airport/ZBAA/%d/%02d/%02d/DailyHistory.html?req_city=Beijing&req_statename=China' % (
        year, month, day)
    else:
        link = 'https://www.wunderground.com/history/airport/EGMC/%d/%02d/%02d/DailyHistory.html?req_city=London&req_statename=England' % (
        year, month, day)
    headers = {'User-Agent': 'Mozilla/5.0 (Windows; U; Windows NT 6.1; en-US; rv:1.9.1.6) Gecko/20091201 Firefox/3.5.6',
               'referer': 'link'}
    beautiful = requests.get(link, headers=headers).text.decode('utf-8', 'ignore')
    soup = bs4.BeautifulSoup(unicodedata.normalize("NFKD", beautiful), 'html.parser')
    table = soup.find(id="obsTable")
    # print table
    tr_list = table.find("tbody").find_all("tr")
    # print tr_list[0]
    tmp = []
    for tr in tr_list:
        # print tr
        td_list = tr.find_all("td")
        # print td_list
        time_ = td_list[0].get_text(strip=True).strip()  # 时间
        temp_ = td_list[1].get_text(strip=True).strip("\n").strip("\r").strip().replace("\t", "")  # 气温
        windchill_ = td_list[2].get_text(strip=True).replace("\n", "").replace("\r", "").replace("\t",
                                                                                                 "").strip()  # 风冷温
        dew_point_ = td_list[3].get_text(strip=True).replace("\n", "").replace("\r", "").replace("\t", "").strip()  # 露点
        humidity_ = td_list[4].get_text(strip=True).replace("\n", "").replace("\r", "").replace("\t", "").strip()  # 湿度
        pressure_ = td_list[5].get_text(strip=True).replace("\n", "").replace("\r", "").replace("\t", "").strip()  # 气压
        visiability_ = td_list[6].get_text(strip=True).replace("\n", "").replace("\r", "").replace("\t",
                                                                                                   "").strip()  # 能见度
        wind_dir_ = td_list[7].get_text(strip=True).replace("\n", "").replace("\r", "").replace("\t", "").strip()  # 风向
        wind_speed_ = td_list[8].get_text(strip=True).replace("\n", "").replace("\r", "").replace("\t",
                                                                                                  "").strip()  # 风速度
        gust_speed_ = td_list[9].get_text(strip=True).replace("\n", "").replace("\r", "").replace("\t",
                                                                                                  "").strip()  # 瞬间风速
        precip_ = td_list[10].get_text(strip=True).replace("\n", "").replace("\r", "").replace("\t",
                                                                                               "").strip()  # precip
        events_ = td_list[11].get_text(strip=True).replace("\n", "").replace("\r", "").replace("\t", "").strip()  # 活动
        conditions_ = td_list[12].get_text(strip=True).replace("\n", "").replace("\r", "").replace("\t",
                                                                                                   "").strip()  # 状况
        tmp_1 = time_ + ',' + temp_ + ',' + windchill_ + ',' + dew_point_ + ',' + humidity_ + ',' + pressure_ + ',' + visiability_ + ',' + wind_dir_ + ',' + wind_speed_ + ',' + gust_speed_ + ',' + precip_ + ',' + events_ + ',' + conditions_
        tmp_1 = tmp_1.encode('unicode-escape').replace(r'\xb0C', r'')
        tmp_1 = tmp_1.replace("m/s", "").replace("%", "").replace("hPa", "").replace("km/h()", " ").replace(
            "kilometers", "").replace("km/h", "").replace("mm", "")
        tmp_1 = tmp_1.strip()
        tmp.append(tmp_1.split(","))
    ans["hours_data"] = tmp

tmp = ""
    table = soup.find(id="historyTable")
    tr_list = table.find("tbody").find_all("tr")
    temp_avg_ = tr_list[1].find_all("td")[1].get_text(strip=True).replace("\n", "").replace("\r", "").replace("\t",
                                                                                                              "").strip()
    temp_max_ = tr_list[2].find_all("td")[1].get_text(strip=True).replace("\n", "").replace("\r", "").replace("\t",
                                                                                                              "").strip()
    temp_min_ = tr_list[3].find_all("td")[1].get_text(strip=True).replace("\n", "").replace("\r", "").replace("\t",
                                                                                                              "").strip()
    degree_day_ = tr_list[5].find_all("td")[1].get_text(strip=True).replace("\n", "").replace("\r", "").replace("\t",
                                                                                                                "").strip()
    dew_point_ = tr_list[7].find_all("td")[1].get_text(strip=True).replace("\n", "").replace("\r", "").replace("\t",
                                                                                                               "").strip()
    humidity_avg_ = tr_list[8].find_all("td")[1].get_text(strip=True).replace("\n", "").replace("\r", "").replace("\t",
                                                                                                                  "").strip()
    humidity_max_ = tr_list[9].find_all("td")[1].get_text(strip=True).replace("\n", "").replace("\r", "").replace("\t",
                                                                                                                  "").strip()
    humidity_min_ = tr_list[10].find_all("td")[1].get_text(strip=True).replace("\n", "").replace("\r", "").replace("\t",
                                                                                                                   "").strip()
    rain_value_ = tr_list[12].find_all("td")[1].get_text(strip=True).replace("\n", "").replace("\r", "").replace("\t",
                                                                                                                 "").strip()
    pressure_ = tr_list[14].find_all("td")[1].get_text(strip=True).replace("\n", "").replace("\r", "").replace("\t",
                                                                                                               "").strip()
    speed_avg_ = tr_list[16].find_all("td")[1].get_text(strip=True).replace("\n", "").replace("\r", "").replace("\t",
                                                                                                                "").strip()
    speed_max_ = tr_list[17].find_all("td")[1].get_text(strip=True).replace("\n", "").replace("\r", "").replace("\t",
                                                                                                                "").strip()
    visiability_ = tr_list[19].find_all("td")[1].get_text(strip=True).replace("\n", "").replace("\r", "").replace("\t",
                                                                                                                  "").strip()
    events_ = tr_list[20].find_all("td")[1].get_text(strip=True).replace("\n", "").replace("\r", "").replace("\t",
                                                                                                             "").strip()
    tmp_1 = temp_avg_ + "," + temp_max_ + "," + temp_min_ + "," + degree_day_ + "," + dew_point_ + "," + humidity_avg_ + "," + humidity_max_ + "," + humidity_min_ + "," + rain_value_ + "," + pressure_ + "," + speed_avg_ + "," + speed_max_ + "," + visiability_ + "," + events_
    tmp_1 = tmp_1.encode('unicode-escape').replace(r'\xb0C', r'')
    tmp_1 = tmp_1.replace("m/s", "").replace("%", "").replace("hPa", "").replace("km/h()", " ").replace("kilometers",
                                                                                                        "").replace(
        "km/h", "").replace("mm", "")
    tmp_1 = tmp_1.strip()
    ans["day_data"] = tmp_1.split(",")
    return ans

# https://www.wunderground.com/personal-weather-station/dashboard?ID=ILONDON636#history/s20180405/e20180405/mdaily
# https://www.wunderground.com/hourly/cn/baizhifang/date/2018-04-08/I11BAIZH2?cm_ven=localwx_hour
'''
# 根据站点的id获取天气,包含历史,当天和预测

'''

data = {
    "history": {
        "days": {
            [{
                "observations": [{},{}], # 一天每小时的天气情况
                "summary": {} #一天整体的天气
            }]
        },
        "end_date": {},
        "start_date": {},
    },
    "response": {
        "date":{} #查询当天的天气情况
    }
}

def weather_station(year=2018, month=4, day=6, station_id="I11BAIZH2"):
    time_str = "%d%02d%02d" % (year, month, day)
    time_find = parser.parse(time_str)
    time_find = time_find.strftime('%Y-%m-%d')

time_now = time.time()
    time_now = time.localtime(time_now)
    time_now = time.strftime('%Y-%m-%d', time_now)
    if time_now > time_find:
        link = 'https://www.wunderground.com/personal-weather-station/dashboard?ID=' + station_id + '#history/s' + time_str + '/e' + time_str + '/mdaily'
    else:
        link = 'https://www.wunderground.com/hourly/date/' + str(time_find) + '/' + station_id + '?cm_ven=localwx_hour'
    headers = {'User-Agent': 'Mozilla/5.0 (Windows; U; Windows NT 6.1; en-US; rv:1.9.1.6) Gecko/20091201 Firefox/3.5.6',
               'referer': 'link'}
    beautiful = requests.get(link, headers=headers).text.decode('utf-8', 'ignore')
    soup = bs4.BeautifulSoup(unicodedata.normalize("NFKD", beautiful), 'html.parser')
    ans = {}
    print time_now, time_find
    if time_now > time_find:
        link1 = 'https://api-ak.wunderground.com/api/606f3f6977348613/history_' + time_str + 'null/units:metric/v:2.0/q/pws:' + station_id + '.json?'
        print link1
        respones1 = requests.get(link1)
        text =  respones1.text
        # print text
        data = json.loads(text)
        # print data
        ans["history"] = data["history"]["days"][0]
        ans["today"] = data["response"]["date"]
    else:
        lat = float(soup.find("span", {"class": "subheading"}).find_all("strong")[1].get_text(strip=True).strip())
        lng = float(soup.find("span", {"class": "subheading"}).find_all("strong")[2].get_text(strip=True).strip())
        ans = weather_lat_lng(lat, lng)
        # table = soup.find(id = "hourly-forecast-table")
        # tr_list = table.find("tbody").find_all("tr")
        # # print tr_list
        # tmp = []
        # for tr in tr_list:
        # 	td_list = tr.find_all('td')
        # 	time_ = td_list[0].get_text(strip=True).replace("\n","").replace("\r","").replace("\t","").strip()
        # 	conditions_ = td_list[1].find("span",{"class":"show-for-medium conditions"}).get_text(strip=True).replace("\n","").replace("\r","").replace("\t","").strip()
        # 	temp_ = td_list[2].get_text(strip=True).replace("\n","").replace("\r","").replace("\t","").strip()
        # 	feels_like_ = td_list[3].get_text(strip=True).replace("\n","").replace("\r","").replace("\t","").strip()
        # 	precip_ = td_list[4].get_text(strip=True).replace("\n","").replace("\r","").replace("\t","").strip()
        # 	amount_ = td_list[5].get_text(strip=True).replace(" in","").replace("\n","").replace("\r","").replace("\t","").strip()
        # 	cloud_cover_ = td_list[6].get_text(strip=True).replace("\n","").replace("\r","").replace("\t","").strip()
        # 	dew_point_ = td_list[7].get_text(strip=True).replace("\n","").replace("\r","").replace("\t","").strip()
        # 	humidity_ = td_list[8].get_text(strip=True).replace("\n","").replace("\r","").replace("\t","").strip()
        # 	wind_ = td_list[9].get_text(strip=True).replace("\n","").replace("\r","").replace("\t","").strip()
        # 	wind_speed_ = wind_.split(" ")[0]
        # 	wind_dir_ = wind_.split(" ")[2]
        # 	pressure_ = td_list[10].find("span",{"class":"wu-value wu-value-to"}).get_text(strip=True).replace("\n","").replace("\r","").replace("\t","").strip()
        # 	tmp_1 = time_+","+conditions_+","+temp_+","+feels_like_+","+precip_+","+amount_+","+cloud_cover_+","+dew_point_+","+humidity_+","+wind_speed_+","+wind_dir_+","+pressure_
        # 	tmp_1 = tmp_1.encode('unicode-escape').replace(r'\xb0F',r'')
        # 	tmp_1 = tmp_1.replace("m/s","").replace("%","").replace("hPa","").replace("km/h()"," ").replace("kilometers","").replace("km/h","").replace("mm","")
        # 	tmp_1 = tmp_1.strip()
        # 	tmp.append(tmp_1.split(","))
        # ans["hours_data"] = tmp
        pass
    return ans

# https://stationdata.wunderground.com/cgi-bin/stationdata?v=2.0&type=ICAO%2CPWS&units=english&format=json&maxage=1800&maxstations=35&minstations=10&centerLat=39.873821&centerLon=116.339943&height=400&width=400&iconsize=2
# https://api.weather.com/v1/geocode/40.039/116.395/forecast/hourly/240hour.json?apiKey=6532d6454b8aa370768e63d6ba5a832e&units=e
'''
根据经纬度获取附近站点的天气情况
根据经纬度获取近10天24小时的天气数据
'''

def weather_lat_lng(lat, lng):
    link1 = 'https://stationdata.wunderground.com/cgi-bin/stationdata?v=2.0&type=ICAO%2CPWS&units=english&format=json&maxage=1800&maxstations=35&minstations=10&centerLat=' + str(
        lat) + '&centerLon=' + str(lng) + '&height=400&width=400&iconsize=2'
    link2 = 'https://api.weather.com/v1/geocode/' + str(lat) + '/' + str(
        lng) + '/forecast/hourly/240hour.json?apiKey=6532d6454b8aa370768e63d6ba5a832e&units=e'
    link3 = 'https://api.weather.com/v1/geocode/' + str(lat) + '/' + str(
        lng) + '/forecast/daily/10day.json?apiKey=6532d6454b8aa370768e63d6ba5a832e&units=e'
    respones1 = requests.get(link1)
    stations = json.loads(respones1.text)["stations"]
    respones2 = requests.get(link2)
    forecasts_hours = json.loads(respones2.text)["forecasts"]
    respones3 = requests.get(link3)
    forecasts_daily = json.loads(respones3.text)["forecasts"]
    ans = {}
    ans["stations"] = stations
    ans["forecasts_hours"] = forecasts_hours
    ans['forecasts_daily'] = forecasts_daily
    return ans

# https://api-ak.wunderground.com/api/606f3f6977348613/history_20180404null/units:metric/v:2.0/q/pws:I11DONGX2.json?

def find_station_information():
    # fr_to = open("stataion_beijing.txt","w")
    fr_to = open("stataion_london.txt", "w")
    seed_lat = 51.51
    seed_lng = 0.13
    station_informain = {}
    import Queue
    q = Queue.Queue()
    q.put([seed_lat, seed_lng])
    while (True):
        tmp = q.get()
        ans = weather_lat_lng(tmp[0], tmp[1])
        stations = ans["stations"]
        for i in range(len(stations)):
            id = stations[i]['id']
            latitude = stations[i]['latitude']
            longitude = stations[i]['longitude']
            adm1 = stations[i]['adm1']
            if station_informain.has_key(id) == False:
                station_informain[id] = {}
                q.put([latitude, longitude])
            station_informain[id]["latitude"] = latitude
            station_informain[id]["longitude"] = longitude
            station_informain[id]["adm1"] = adm1
        if q.empty():
            break
        if len(station_informain.keys()) > 100:
            break
    print station_informain
    print len(station_informain.keys())
    for key in station_informain.keys():
        fr_to.write(str(key) + "," + str(station_informain[key]['latitude']) + "," + str(
            station_informain[key]['longitude']) + "," + station_informain[key]['adm1'] + "\n")
    fr_to.close()

# https://api.waqi.info/mapq/bounds/?bounds=39.39799959542146,115.04608154296876,40.073868105094846,117.59765625000001&inc=placeholders&k=_2Y2EzUR9IAR8fDStDSElWXmldWkc+EzMZFngzZA==&_=1523093789592
# http://aqicn.org/city/beijing
# https://api.waqi.info/api/feed/@1451/obs.en.json
# http://aqicn.org/city/london/
# https://api.waqi.info/api/feed/@5724/obs.en.json

def aqi_idx(idx=5724):
    link = 'https://api.waqi.info/api/feed/@' + str(idx) + '/obs.en.json'
    respones = requests.get(link)
    msg = json.loads(respones.text)["rxs"]["obs"][0]["msg"]
    print msg

def main():
    year = 2018
    month = 4
    day = 8
    city = "London"
    # ans = city_weather(year=year, month=month, day=day, city=city)
    # print ans
    weather_station(year=year, month= month, day = day)
    lat = 40.039
    lng = 116.395
    # weather_lat_lng(lat = lat, lng = lng)
    # find_station_information()
    # aqi_idx()

if __name__ == '__main__':
    main()

```

station in beijing
I11HAIDI3,39.97919083,116.29973602,Haidian
IHAIDIAN9,39.9978447,116.36952209,Haidian Qu
ICHAOYAN11,39.96221161,116.44536591,Chaoyang Qu
IBEIJING355,39.89273453,116.33809662,Beijing
I11BAIZH2,39.87382126,116.33994293,Baizhifang
IBEIJING184,40.03221893,116.50910187,Beijing
IXICHENG8,39.89440155,116.34385681,Xicheng Qu
I11HAIDI2,39.97648621,116.30276489,Haidian
IHAIDIAN6,39.95755386,116.34825134,Haidian Qu
IHAIDIAN7,39.94497299,116.30096436,Haidian Qu
ISHUNYIQ5,40.07844925,116.51275635,Shunyi Qu
ICHANGPI3,40.09079742,116.33363342,Changping Qu
ITONGZHO3,39.91837311,116.6907959,Tongzhou Qu
ISHUNYIQ3,40.08337784,116.5056839,Shunyi Qu
ICHAOYAN28,40.0286293,116.51416016,Chaoyang Qu
I11DONGX2,40.03866959,116.39524841,Dongxiaokou
I11HOUSH2,40.09382629,116.54441071,Houshayu
ISHUNYIQ8,40.06755447,116.53660583,Shunyi Qu
IBEIJING17,39.98901367,116.48651123,Beijing
IBEIJING250,39.91112518,116.25349426,Beijing

station in london
ILONDON1124,51.47687912,0.068984,London
ILONDON1169,51.47953796,0.1002982,London
ILONDONM4,51.4369812,0.0511718,Mottingham
IDARTFOR12,51.42866898,0.17570126,Dartford
ICHISLEH2,51.42477798,0.065824,Chislehurst
IGLAUPMI3,51.54998016,0.25253499,Upminster
IKENTERI2,51.47380066,0.1617,Erith
IILFORD2,51.55250549,0.083651,Ilford
IBEXLEYH3,51.46260452,0.117337,Bexleyheath
ILONDON234,51.46895218,0.046274,London
IWELLING234,51.46497726,0.11156,Welling
IBEXLEYH6,51.46427536,0.146369,Bexleyheath
IGLABECO2,51.53381348,0.117337,Becontree
ILONDON726,51.45943832,0.19951232,London
IBEXLEY13,51.44404221,0.14159916,Bexley
IBEXLEY16,51.4752388,0.17316701,Bexley
IBEXLEY14,51.4461937,0.13829599,Bexley
IGLAWELL2,51.45271301,0.088085,Welling
IENGLAND346,51.47603989,0.14876094,Erith
IHAVERIN3,51.55194473,0.19453192,Havering
ILONDON522,51.56309891,0.17084134,London
IRAINHAM5,51.53635025,0.200808,Rainham
IRAINHAM7,51.53620529,0.20069073,Rainham
IRAINHAM8,51.5499649,0.20133001,Rainham
IBEXLEYH10,51.46513748,0.147806,Bexleyheath
IGLACHAR5,51.47551727,0.034443,Charlton
IROMFORD7,51.57570267,0.126587,Romford
IKENTBEX3,51.45648193,0.15228479,Bexleyheath
IKENTBEX2,51.43861008,0.16249999,Bexley
IKENTBEX4,51.46320724,0.15664101,Bexleyheath
IHORNCHU10,51.56059647,0.19448701,Hornchurch
IROMFORD17,51.5711441,0.17890701,Romford
ILONDON172,51.47330093,0.04749,London
IWELLING170,51.46265411,0.11737342,Welling
IGLAUPMI2,51.55229568,0.25212601,Upminster
IROMFORD13,51.58579254,0.198581,Romford
IBEXLEY18,51.43543625,0.130968,Bexley
IROMFORD10,51.59189987,0.190137,Romford
IGLAYIEW96,51.48517609,0.030198,Yiewsley
IGLACRAY4,51.44298172,0.179445,Crayford
ILONDON610,51.58948135,0.17981441,London
IDAGENHA3,51.53533936,0.129619,Dagenham
IGLACRAY3,51.44725418,0.19536699,Crayford
ILONDON265,51.45254898,0.058577,London
ILONDON633,51.47533035,0.03219949,London
IENGLAND869,51.43728638,0.21009997,Dartford
IGLARAIN2,51.53604126,0.19437499,Rainham
IHORNCHU5,51.58985901,0.232631,Hornchurch
ILONDON557,51.46236801,0.09156118,London
ISIDCUP4,51.42630005,0.1024,Sidcup
IGLABEXL3,51.46281815,0.132551,Bexleyheath
ILONDON552,51.55987167,0.25750893,London
IBARKING2,51.52777863,0.119927,Barking
ILONDON1023,51.47974777,0.030706,London
ILONDON559,51.46854782,0.10738425,London
IHORNCHU7,51.54951859,0.21128133,Hornchurch
ISIDCUP2,51.4402771,0.098433,Sidcup

23 comments