5.4 Working with web requests

Sometimes it’s necessary to work with web requests on a lower level, to do fancier things like setting cookies, building more complex web requests or using authentication. The httr packages provides relatively easy functionality for making more complex web requests.

A full explanation of httr is out of our scope (see the httr quickstart guide for a more thorough introduction), but a simple example below shows how we could access the BoM JSON data using our own web request.

bom_url <- "http://www.bom.gov.au/fwo/IDV60901/IDV60901.94866.json"

response <- httr::GET(bom_url)
response
#> Response [http://www.bom.gov.au/fwo/IDV60901/IDV60901.94866.json]
#>   Date: 2020-12-01 03:15
#>   Status: 200
#>   Content-Type: application/json
#>   Size: 142 kB
#> {
#>  "observations": {
#>      "notice": [
#>          {
#>              "copyright": "Copyright Commonwealth of Australia 2020, Bureau of Meteoro...
#>              "copyright_url": "http://www.bom.gov.au/other/copyright.shtml",
#>              "disclaimer_url": "http://www.bom.gov.au/other/disclaimer.shtml",
#>              "feedback_url": "http://www.bom.gov.au/other/feedback"
#>          }
#>      ],
#> ...

bom_json <- httr::content(response, as = "text")
#> No encoding supplied: defaulting to UTF-8.
substr(bom_json, 1, 1000)
#> [1] "{\n\t\"observations\": {\n\t\t\"notice\": [\n\t\t\t{\n\t\t\t\t\"copyright\": \"Copyright Commonwealth of Australia 2020, Bureau of Meteorology. For more information see: http://www.bom.gov.au/other/copyright.shtml http://www.bom.gov.au/other/disclaimer.shtml\",\n\t\t\t\t\"copyright_url\": \"http://www.bom.gov.au/other/copyright.shtml\",\n\t\t\t\t\"disclaimer_url\": \"http://www.bom.gov.au/other/disclaimer.shtml\",\n\t\t\t\t\"feedback_url\": \"http://www.bom.gov.au/other/feedback\"\n\t\t\t}\n\t\t],\n\t\t\"header\": [\n\t\t\t{\n\t\t\t\t\"refresh_message\": \"Issued at  2:11 pm EDT Tuesday  1 December 2020\",\n\t\t\t\t\"ID\": \"IDV60901\",\n\t\t\t\t\"main_ID\": \"IDV60900\",\n\t\t\t\t\"name\": \"Melbourne Airport\",\n\t\t\t\t\"state_time_zone\": \"VIC\",\n\t\t\t\t\"time_zone\": \"EDT\",\n\t\t\t\t\"product_name\": \"Capital City Observations\",\n\t\t\t\t\"state\": \"Victoria\"\n\t\t\t}\n\t\t],\n\t\t\"data\": [\n\t\t\t{\n\t\t\t\t\"sort_order\": 0,\n\t\t\t\t\"wmo\": 94866,\n\t\t\t\t\"name\": \"Melbourne Airport\",\n\t\t\t\t\"history_product\": \"IDV60901\",\n\t\t\t\t\"local_date_time\": \"01/02:00pm\",\n\t\t\t\t\"local_date_time_full\": \"20201201140000\",\n\t\t\t\t\"aifstime_utc\": \"2020120103000"
cat(substr(bom_json, 1, 1000))
#> {
#>  "observations": {
#>      "notice": [
#>          {
#>              "copyright": "Copyright Commonwealth of Australia 2020, Bureau of Meteorology. For more information see: http://www.bom.gov.au/other/copyright.shtml http://www.bom.gov.au/other/disclaimer.shtml",
#>              "copyright_url": "http://www.bom.gov.au/other/copyright.shtml",
#>              "disclaimer_url": "http://www.bom.gov.au/other/disclaimer.shtml",
#>              "feedback_url": "http://www.bom.gov.au/other/feedback"
#>          }
#>      ],
#>      "header": [
#>          {
#>              "refresh_message": "Issued at  2:11 pm EDT Tuesday  1 December 2020",
#>              "ID": "IDV60901",
#>              "main_ID": "IDV60900",
#>              "name": "Melbourne Airport",
#>              "state_time_zone": "VIC",
#>              "time_zone": "EDT",
#>              "product_name": "Capital City Observations",
#>              "state": "Victoria"
#>          }
#>      ],
#>      "data": [
#>          {
#>              "sort_order": 0,
#>              "wmo": 94866,
#>              "name": "Melbourne Airport",
#>              "history_product": "IDV60901",
#>              "local_date_time": "01/02:00pm",
#>              "local_date_time_full": "20201201140000",
#>              "aifstime_utc": "2020120103000

bom_data <- jsonlite::fromJSON(bom_json)
str(bom_data)
#> List of 1
#>  $ observations:List of 3
#>   ..$ notice:'data.frame':   1 obs. of  4 variables:
#>   .. ..$ copyright     : chr "Copyright Commonwealth of Australia 2020, Bureau of Meteorology. For more information see: http://www.bom.gov.a"| __truncated__
#>   .. ..$ copyright_url : chr "http://www.bom.gov.au/other/copyright.shtml"
#>   .. ..$ disclaimer_url: chr "http://www.bom.gov.au/other/disclaimer.shtml"
#>   .. ..$ feedback_url  : chr "http://www.bom.gov.au/other/feedback"
#>   ..$ header:'data.frame':   1 obs. of  8 variables:
#>   .. ..$ refresh_message: chr "Issued at  2:11 pm EDT Tuesday  1 December 2020"
#>   .. ..$ ID             : chr "IDV60901"
#>   .. ..$ main_ID        : chr "IDV60900"
#>   .. ..$ name           : chr "Melbourne Airport"
#>   .. ..$ state_time_zone: chr "VIC"
#>   .. ..$ time_zone      : chr "EDT"
#>   .. ..$ product_name   : chr "Capital City Observations"
#>   .. ..$ state          : chr "Victoria"
#>   ..$ data  :'data.frame':   161 obs. of  35 variables:
#>   .. ..$ sort_order          : int [1:161] 0 1 2 3 4 5 6 7 8 9 ...
#>   .. ..$ wmo                 : int [1:161] 94866 94866 94866 94866 94866 94866 94866 94866 94866 94866 ...
#>   .. ..$ name                : chr [1:161] "Melbourne Airport" "Melbourne Airport" "Melbourne Airport" "Melbourne Airport" ...
#>   .. ..$ history_product     : chr [1:161] "IDV60901" "IDV60901" "IDV60901" "IDV60901" ...
#>   .. ..$ local_date_time     : chr [1:161] "01/02:00pm" "01/01:30pm" "01/01:00pm" "01/12:30pm" ...
#>   .. ..$ local_date_time_full: chr [1:161] "20201201140000" "20201201133000" "20201201130000" "20201201123000" ...
#>   .. ..$ aifstime_utc        : chr [1:161] "20201201030000" "20201201023000" "20201201020000" "20201201013000" ...
#>   .. ..$ lat                 : num [1:161] -37.7 -37.7 -37.7 -37.7 -37.7 -37.7 -37.7 -37.7 -37.7 -37.7 ...
#>   .. ..$ lon                 : num [1:161] 145 145 145 145 145 ...
#>   .. ..$ apparent_t          : num [1:161] 18.8 17.3 15.3 14 14.1 14.2 13.9 15.6 17.5 19.4 ...
#>   .. ..$ cloud               : chr [1:161] "Mostly clear" "Mostly clear" "Mostly clear" "Cloudy" ...
#>   .. ..$ cloud_base_m        : int [1:161] 600 600 600 3300 3300 2610 2500 1110 1110 2340 ...
#>   .. ..$ cloud_oktas         : int [1:161] 1 1 1 8 8 3 8 1 1 3 ...
#>   .. ..$ cloud_type_id       : int [1:161] 6 6 6 NA NA NA 35 6 6 NA ...
#>   .. ..$ cloud_type          : chr [1:161] "Stratocumulus" "Stratocumulus" "Stratocumulus" "-" ...
#>   .. ..$ delta_t             : num [1:161] 4.5 4 3.2 2.7 2.8 3.3 3.6 7.5 7.4 7.7 ...
#>   .. ..$ gust_kmh            : int [1:161] 33 33 39 39 37 52 52 59 50 59 ...
#>   .. ..$ gust_kt             : int [1:161] 18 18 21 21 20 28 28 32 27 32 ...
#>   .. ..$ air_temp            : num [1:161] 21.7 20.4 19.4 18.5 18.6 19.3 19.5 23.1 24.2 25 ...
#>   .. ..$ dewpt               : num [1:161] 14 13.5 13.9 13.8 13.7 13.6 13.2 9.3 11.1 11.5 ...
#>   .. ..$ press               : num [1:161] 1005 1006 1005 1004 1004 ...
#>   .. ..$ press_qnh           : num [1:161] 1006 1006 1005 1005 1005 ...
#>   .. ..$ press_msl           : num [1:161] 1005 1006 1005 1004 1004 ...
#>   .. ..$ press_tend          : chr [1:161] "-" "-" "-" "-" ...
#>   .. ..$ rain_trace          : chr [1:161] "0.0" "0.0" "0.0" "0.0" ...
#>   .. ..$ rel_hum             : int [1:161] 61 64 70 74 73 69 67 41 43 42 ...
#>   .. ..$ sea_state           : chr [1:161] "-" "-" "-" "-" ...
#>   .. ..$ swell_dir_worded    : chr [1:161] "-" "-" "-" "-" ...
#>   .. ..$ swell_height        : logi [1:161] NA NA NA NA NA NA ...
#>   .. ..$ swell_period        : logi [1:161] NA NA NA NA NA NA ...
#>   .. ..$ vis_km              : chr [1:161] "10" "10" "10" "10" ...
#>   .. ..$ weather             : chr [1:161] "-" "-" "-" "Showers" ...
#>   .. ..$ wind_dir            : chr [1:161] "N" "NNW" "N" "NNE" ...
#>   .. ..$ wind_spd_kmh        : int [1:161] 22 22 28 30 30 33 35 39 37 32 ...
#>   .. ..$ wind_spd_kt         : int [1:161] 12 12 15 16 16 18 19 21 20 17 ...