5.4 Working with web requests
Sometimes it’s necessary to work with web requests on a lower level, to do fancier things like setting cookies, building more complex web requests or using authentication. The httr packages provides relatively easy functionality for making more complex web requests.
A full explanation of httr is out of our scope (see the httr quickstart guide for a more thorough introduction), but a simple example below shows how we could access the BoM JSON data using our own web request.
bom_url <- "http://www.bom.gov.au/fwo/IDV60901/IDV60901.94866.json"
response <- httr::GET(bom_url)
response
#> Response [http://www.bom.gov.au/fwo/IDV60901/IDV60901.94866.json]
#> Date: 2020-12-01 03:15
#> Status: 200
#> Content-Type: application/json
#> Size: 142 kB
#> {
#> "observations": {
#> "notice": [
#> {
#> "copyright": "Copyright Commonwealth of Australia 2020, Bureau of Meteoro...
#> "copyright_url": "http://www.bom.gov.au/other/copyright.shtml",
#> "disclaimer_url": "http://www.bom.gov.au/other/disclaimer.shtml",
#> "feedback_url": "http://www.bom.gov.au/other/feedback"
#> }
#> ],
#> ...
bom_json <- httr::content(response, as = "text")
#> No encoding supplied: defaulting to UTF-8.
substr(bom_json, 1, 1000)
#> [1] "{\n\t\"observations\": {\n\t\t\"notice\": [\n\t\t\t{\n\t\t\t\t\"copyright\": \"Copyright Commonwealth of Australia 2020, Bureau of Meteorology. For more information see: http://www.bom.gov.au/other/copyright.shtml http://www.bom.gov.au/other/disclaimer.shtml\",\n\t\t\t\t\"copyright_url\": \"http://www.bom.gov.au/other/copyright.shtml\",\n\t\t\t\t\"disclaimer_url\": \"http://www.bom.gov.au/other/disclaimer.shtml\",\n\t\t\t\t\"feedback_url\": \"http://www.bom.gov.au/other/feedback\"\n\t\t\t}\n\t\t],\n\t\t\"header\": [\n\t\t\t{\n\t\t\t\t\"refresh_message\": \"Issued at 2:11 pm EDT Tuesday 1 December 2020\",\n\t\t\t\t\"ID\": \"IDV60901\",\n\t\t\t\t\"main_ID\": \"IDV60900\",\n\t\t\t\t\"name\": \"Melbourne Airport\",\n\t\t\t\t\"state_time_zone\": \"VIC\",\n\t\t\t\t\"time_zone\": \"EDT\",\n\t\t\t\t\"product_name\": \"Capital City Observations\",\n\t\t\t\t\"state\": \"Victoria\"\n\t\t\t}\n\t\t],\n\t\t\"data\": [\n\t\t\t{\n\t\t\t\t\"sort_order\": 0,\n\t\t\t\t\"wmo\": 94866,\n\t\t\t\t\"name\": \"Melbourne Airport\",\n\t\t\t\t\"history_product\": \"IDV60901\",\n\t\t\t\t\"local_date_time\": \"01/02:00pm\",\n\t\t\t\t\"local_date_time_full\": \"20201201140000\",\n\t\t\t\t\"aifstime_utc\": \"2020120103000"
cat(substr(bom_json, 1, 1000))
#> {
#> "observations": {
#> "notice": [
#> {
#> "copyright": "Copyright Commonwealth of Australia 2020, Bureau of Meteorology. For more information see: http://www.bom.gov.au/other/copyright.shtml http://www.bom.gov.au/other/disclaimer.shtml",
#> "copyright_url": "http://www.bom.gov.au/other/copyright.shtml",
#> "disclaimer_url": "http://www.bom.gov.au/other/disclaimer.shtml",
#> "feedback_url": "http://www.bom.gov.au/other/feedback"
#> }
#> ],
#> "header": [
#> {
#> "refresh_message": "Issued at 2:11 pm EDT Tuesday 1 December 2020",
#> "ID": "IDV60901",
#> "main_ID": "IDV60900",
#> "name": "Melbourne Airport",
#> "state_time_zone": "VIC",
#> "time_zone": "EDT",
#> "product_name": "Capital City Observations",
#> "state": "Victoria"
#> }
#> ],
#> "data": [
#> {
#> "sort_order": 0,
#> "wmo": 94866,
#> "name": "Melbourne Airport",
#> "history_product": "IDV60901",
#> "local_date_time": "01/02:00pm",
#> "local_date_time_full": "20201201140000",
#> "aifstime_utc": "2020120103000
bom_data <- jsonlite::fromJSON(bom_json)
str(bom_data)
#> List of 1
#> $ observations:List of 3
#> ..$ notice:'data.frame': 1 obs. of 4 variables:
#> .. ..$ copyright : chr "Copyright Commonwealth of Australia 2020, Bureau of Meteorology. For more information see: http://www.bom.gov.a"| __truncated__
#> .. ..$ copyright_url : chr "http://www.bom.gov.au/other/copyright.shtml"
#> .. ..$ disclaimer_url: chr "http://www.bom.gov.au/other/disclaimer.shtml"
#> .. ..$ feedback_url : chr "http://www.bom.gov.au/other/feedback"
#> ..$ header:'data.frame': 1 obs. of 8 variables:
#> .. ..$ refresh_message: chr "Issued at 2:11 pm EDT Tuesday 1 December 2020"
#> .. ..$ ID : chr "IDV60901"
#> .. ..$ main_ID : chr "IDV60900"
#> .. ..$ name : chr "Melbourne Airport"
#> .. ..$ state_time_zone: chr "VIC"
#> .. ..$ time_zone : chr "EDT"
#> .. ..$ product_name : chr "Capital City Observations"
#> .. ..$ state : chr "Victoria"
#> ..$ data :'data.frame': 161 obs. of 35 variables:
#> .. ..$ sort_order : int [1:161] 0 1 2 3 4 5 6 7 8 9 ...
#> .. ..$ wmo : int [1:161] 94866 94866 94866 94866 94866 94866 94866 94866 94866 94866 ...
#> .. ..$ name : chr [1:161] "Melbourne Airport" "Melbourne Airport" "Melbourne Airport" "Melbourne Airport" ...
#> .. ..$ history_product : chr [1:161] "IDV60901" "IDV60901" "IDV60901" "IDV60901" ...
#> .. ..$ local_date_time : chr [1:161] "01/02:00pm" "01/01:30pm" "01/01:00pm" "01/12:30pm" ...
#> .. ..$ local_date_time_full: chr [1:161] "20201201140000" "20201201133000" "20201201130000" "20201201123000" ...
#> .. ..$ aifstime_utc : chr [1:161] "20201201030000" "20201201023000" "20201201020000" "20201201013000" ...
#> .. ..$ lat : num [1:161] -37.7 -37.7 -37.7 -37.7 -37.7 -37.7 -37.7 -37.7 -37.7 -37.7 ...
#> .. ..$ lon : num [1:161] 145 145 145 145 145 ...
#> .. ..$ apparent_t : num [1:161] 18.8 17.3 15.3 14 14.1 14.2 13.9 15.6 17.5 19.4 ...
#> .. ..$ cloud : chr [1:161] "Mostly clear" "Mostly clear" "Mostly clear" "Cloudy" ...
#> .. ..$ cloud_base_m : int [1:161] 600 600 600 3300 3300 2610 2500 1110 1110 2340 ...
#> .. ..$ cloud_oktas : int [1:161] 1 1 1 8 8 3 8 1 1 3 ...
#> .. ..$ cloud_type_id : int [1:161] 6 6 6 NA NA NA 35 6 6 NA ...
#> .. ..$ cloud_type : chr [1:161] "Stratocumulus" "Stratocumulus" "Stratocumulus" "-" ...
#> .. ..$ delta_t : num [1:161] 4.5 4 3.2 2.7 2.8 3.3 3.6 7.5 7.4 7.7 ...
#> .. ..$ gust_kmh : int [1:161] 33 33 39 39 37 52 52 59 50 59 ...
#> .. ..$ gust_kt : int [1:161] 18 18 21 21 20 28 28 32 27 32 ...
#> .. ..$ air_temp : num [1:161] 21.7 20.4 19.4 18.5 18.6 19.3 19.5 23.1 24.2 25 ...
#> .. ..$ dewpt : num [1:161] 14 13.5 13.9 13.8 13.7 13.6 13.2 9.3 11.1 11.5 ...
#> .. ..$ press : num [1:161] 1005 1006 1005 1004 1004 ...
#> .. ..$ press_qnh : num [1:161] 1006 1006 1005 1005 1005 ...
#> .. ..$ press_msl : num [1:161] 1005 1006 1005 1004 1004 ...
#> .. ..$ press_tend : chr [1:161] "-" "-" "-" "-" ...
#> .. ..$ rain_trace : chr [1:161] "0.0" "0.0" "0.0" "0.0" ...
#> .. ..$ rel_hum : int [1:161] 61 64 70 74 73 69 67 41 43 42 ...
#> .. ..$ sea_state : chr [1:161] "-" "-" "-" "-" ...
#> .. ..$ swell_dir_worded : chr [1:161] "-" "-" "-" "-" ...
#> .. ..$ swell_height : logi [1:161] NA NA NA NA NA NA ...
#> .. ..$ swell_period : logi [1:161] NA NA NA NA NA NA ...
#> .. ..$ vis_km : chr [1:161] "10" "10" "10" "10" ...
#> .. ..$ weather : chr [1:161] "-" "-" "-" "Showers" ...
#> .. ..$ wind_dir : chr [1:161] "N" "NNW" "N" "NNE" ...
#> .. ..$ wind_spd_kmh : int [1:161] 22 22 28 30 30 33 35 39 37 32 ...
#> .. ..$ wind_spd_kt : int [1:161] 12 12 15 16 16 18 19 21 20 17 ...