Merge branch 'master' of https://github.com/ledgetech/lua-resty-http into ledgetech-master

Author: spacewander

.github/workflows/test.yml

@@ -1,11 +1,6 @@
 name: Test
 
-on:
-  push:
-    paths-ignore: # Skip if only docs are updated
-        - '*.md'
-  pull_request:
-    branches: [master]
+on: [push, pull_request]
 
 jobs:
   luacheck:

.gitignore

@@ -1,4 +1,6 @@
 t/servroot/
 t/error.log
-lua-resty-http*
 luacov*
+lua-resty-http-*/
+lua-resty-http-*.src.rock
+lua-resty-http-*.tar.gz 

README.md

@@ -93,7 +93,7 @@ local httpc = require("resty.http").new()
 -- First establish a connection
 local ok, err = httpc:connect({
     scheme = "https",
-    host = "127.0.0.1"
+    host = "127.0.0.1",
     port = 8080,
 })
 if not ok then
@@ -241,7 +241,7 @@ The `params` table expects the following fields:
 * `path`: The path string. Defaults to `/`.
 * `query`: The query string, presented as either a literal string or Lua table..
 * `headers`: A table of request headers.
-* `body`: The request body as a string, or an iterator function (see [get\_client\_body\_reader](#get_client_body_reader)).
+* `body`: The request body as a string, a table of strings, or an iterator function yielding strings until nil when exhausted. Note that you must specify a `Content-Length` for the request body, or specify `Transfer-Encoding: chunked` and have your function implement the encoding. See also: [get\_client\_body\_reader](#get_client_body_reader)).
 
 When the request is successful, `res` will contain the following fields:
 
@@ -257,7 +257,7 @@ When the request is successful, `res` will contain the following fields:
 
 `syntax: res, err = httpc:request_uri(uri, params)`
 
-The single-shot interface (see [usage](#Usage)). Since this method performs an entire end-to-end request, options specified in the `params` can include anything found in both [connect](#connect) and [request](#request) documented above. Note also that fields in `params` will override relevant components of the `uri` if specified.
+The single-shot interface (see [usage](#Usage)). Since this method performs an entire end-to-end request, options specified in the `params` can include anything found in both [connect](#connect) and [request](#request) documented above. Note also that fields `path`, and `query`, in `params` will override relevant components of the `uri` if specified (`scheme`, `host`, and `port` will always be taken from the `uri`).
 
 There are 3 additional parameters for controlling keepalives:
 

lib/resty/http.lua

@@ -106,7 +106,7 @@ end
 
 
 local _M = {
-    _VERSION = '0.14',
+    _VERSION = '0.16.1',
 }
 _M._USER_AGENT = "lua-resty-http/" .. _M._VERSION .. " (Lua) ngx_lua/" .. ngx.config.ngx_lua_version
 
@@ -362,7 +362,21 @@ local function _receive_status(sock)
         return nil, nil, nil, err
     end
 
-    return tonumber(str_sub(line, 10, 12)), tonumber(str_sub(line, 6, 8)), str_sub(line, 14)
+    local version = tonumber(str_sub(line, 6, 8))
+    if not version then
+        return nil, nil, nil,
+               "couldn't parse HTTP version from response status line: " .. line
+    end
+
+    local status = tonumber(str_sub(line, 10, 12))
+    if not status then
+        return nil, nil, nil,
+               "couldn't parse status code from response status line: " .. line
+    end
+
+    local reason = str_sub(line, 14)
+
+    return status, version, reason
 end
 
 
@@ -398,6 +412,23 @@ local function _receive_headers(sock)
 end
 
 
+local function transfer_encoding_is_chunked(headers)
+    local te = headers["Transfer-Encoding"]
+    if not te then
+        return false
+    end
+
+    -- Handle duplicate headers
+    -- This shouldn't happen but can in the real world
+    if type(te) ~= "string" then
+        te = tbl_concat(te, ",")
+    end
+
+    return str_find(str_lower(te), "chunked", 1, true) ~= nil
+end
+_M.transfer_encoding_is_chunked = transfer_encoding_is_chunked
+
+
 local function _chunked_body_reader(sock, default_chunk_size)
     return co_wrap(function(max_chunk_size)
         local remaining = 0
@@ -575,7 +606,7 @@ end
 
 
 local function _send_body(sock, body)
-    if type(body) == 'function' then
+    if type(body) == "function" then
         repeat
             local chunk, err, partial = body()
 
@@ -627,12 +658,13 @@ function _M.send_request(self, params)
     local body = params.body
     local headers = http_headers.new()
 
-    local params_headers = params.headers or {}
-    -- We assign one by one so that the metatable can handle case insensitivity
+    -- We assign one-by-one so that the metatable can handle case insensitivity
     -- for us. You can blame the spec for this inefficiency.
+    local params_headers = params.headers or {}
     for k, v in pairs(params_headers) do
         headers[k] = v
     end
+
     if not headers["Proxy-Authorization"] then
         -- TODO: next major, change this to always override the provided
         -- header. Can't do that yet because it would be breaking.
@@ -641,15 +673,40 @@ function _M.send_request(self, params)
         headers["Proxy-Authorization"] = self.http_proxy_auth
     end
 
-    -- Ensure minimal headers are set
+    -- Ensure we have appropriate message length or encoding.
+    do
+        local is_chunked = transfer_encoding_is_chunked(headers)
+
+        if is_chunked then
+            -- If we have both Transfer-Encoding and Content-Length we MUST
+            -- drop the Content-Length, to help prevent request smuggling.
+            -- https://tools.ietf.org/html/rfc7230#section-3.3.3
+            headers["Content-Length"] = nil
 
-    if not headers["Content-Length"] then
-        if type(body) == 'string' then
-            headers["Content-Length"] = #body
-        elseif body == nil and EXPECTING_BODY[str_upper(params.method)] then
-            headers["Content-Length"] = 0
+        elseif not headers["Content-Length"] then
+            -- A length was not given, try to calculate one.
+
+            local body_type = type(body)
+
+            if body_type == "function" then
+                return nil, "Request body is a function but a length or chunked encoding is not specified"
+
+            elseif body_type == "table" then
+                local length = 0
+                for _, v in ipairs(body) do
+                    length = length + #tostring(v)
+                end
+                headers["Content-Length"] = length
+
+            elseif body == nil and EXPECTING_BODY[str_upper(params.method)] then
+                headers["Content-Length"] = 0
+
+            elseif body ~= nil then
+                headers["Content-Length"] = #tostring(body)
+            end
         end
     end
+
     if not headers["Host"] then
         if (str_sub(self.host, 1, 5) == "unix:") then
             return nil, "Unable to generate a useful Host header for a unix domain socket. Please provide one."
@@ -751,22 +808,8 @@ function _M.read_response(self, params)
     if _should_receive_body(params.method, status) then
         has_body = true
 
-        local te = res_headers["Transfer-Encoding"]
-
-        -- Handle duplicate headers
-        -- This shouldn't happen but can in the real world
-        if type(te) == "table" then
-            te = tbl_concat(te, "")
-        end
-
-        local ok, encoding = pcall(str_lower, te)
-        if not ok then
-            encoding = ""
-        end
-
-        if version == 1.1 and str_find(encoding, "chunked", 1, true) ~= nil then
+        if version == 1.1 and transfer_encoding_is_chunked(res_headers) then
             body_reader, err = _chunked_body_reader(sock)
-
         else
             local ok, length = pcall(tonumber, res_headers["Content-Length"])
             if not ok then
@@ -775,9 +818,7 @@ function _M.read_response(self, params)
             end
 
             body_reader, err = _body_reader(sock, length)
-
         end
-
     end
 
     if res_headers["Trailer"] then
@@ -872,6 +913,7 @@ function _M.request_uri(self, uri, params)
         params.scheme, params.host, params.port, path, query = unpack(parsed_uri)
         params.path = params.path or path
         params.query = params.query or query
+        params.ssl_server_name = params.ssl_server_name or params.host
     end
 
     do
@@ -941,10 +983,9 @@ function _M.get_client_body_reader(_, chunksize, sock)
 
     local headers = ngx_req_get_headers()
     local length = headers.content_length
-    local encoding = headers.transfer_encoding
     if length then
         return _body_reader(sock, tonumber(length), chunksize)
-    elseif encoding and str_lower(encoding) == 'chunked' then
+    elseif transfer_encoding_is_chunked(headers) then
         -- Not yet supported by ngx_lua but should just work...
         return _chunked_body_reader(sock, chunksize)
     else

lib/resty/http_headers.lua

@@ -4,7 +4,7 @@ local rawget, rawset, setmetatable =
 local str_lower = string.lower
 
 local _M = {
-    _VERSION = '0.14',
+    _VERSION = '0.16.1',
 }
 
 

lua-resty-http-0.15-0.rockspec renamed to lua-resty-http-0.16.1-0.rockspec

@@ -1,8 +1,8 @@
 package = "lua-resty-http"
-version = "0.15-0"
+version = "0.16.1-0"
 source = {
     url = "git://github.com/ledgetech/lua-resty-http",
-    tag = "v0.15"
+    tag = "v0.16.1"
 }
 description = {
     summary = "Lua HTTP client cosocket driver for OpenResty / ngx_lua.",

t/01-basic.t

@@ -341,3 +341,51 @@ OK
 --- no_error_log
 [error]
 [warn]
+
+=== TEST 13: Should return error on invalid HTTP version in response status line
+--- http_config eval: $::HttpConfig
+--- config
+    location = /a {
+        content_by_lua_block {
+            local http = require "resty.http"
+            local httpc = http.new()
+            local res, err = httpc:request_uri("http://127.0.0.1:12345")
+
+            assert(err == "couldn't parse HTTP version from response status line: TEAPOT/1.1 OMG")
+        }
+    }
+--- tcp_listen: 12345
+--- tcp_reply
+TEAPOT/1.1 OMG
+Server: Teapot
+
+OK
+--- request
+GET /a
+--- no_error_log
+[error]
+[warn]
+
+=== TEST 14: Should return error on invalid status code in response status line
+--- http_config eval: $::HttpConfig
+--- config
+    location = /a {
+        content_by_lua_block {
+            local http = require "resty.http"
+            local httpc = http.new()
+            local res, err = httpc:request_uri("http://127.0.0.1:12345")
+
+            assert(err == "couldn't parse status code from response status line: HTTP/1.1 OMG")
+        }
+    }
+--- tcp_listen: 12345
+--- tcp_reply
+HTTP/1.1 OMG
+Server: Teapot
+
+OK
+--- request
+GET /a
+--- no_error_log
+[error]
+[warn]

t/02-chunked.t

@@ -238,3 +238,87 @@ table
 --- no_error_log
 [error]
 [warn]
+
+
+=== TEST 5: transfer_encoding_is_chunked utility.
+--- http_config eval: $::HttpConfig
+--- config
+    location = /a {
+        content_by_lua_block {
+            local http_headers = require("resty.http_headers")
+            local http = require("resty.http")
+
+            local headers = http_headers:new()
+            assert(http.transfer_encoding_is_chunked(headers) == false,
+                "empty headers should return false")
+
+            headers["Transfer-Encoding"] = "chunked"
+            assert(http.transfer_encoding_is_chunked(headers) == true,
+                "te set to `chunked` should return true`")
+
+            headers["Transfer-Encoding"] = " ChuNkEd "
+            assert(http.transfer_encoding_is_chunked(headers) == true,
+                "te set to ` ChuNkEd ` should return true`")
+
+            headers["Transfer-Encoding"] = { "chunked", " ChuNkEd " }
+            assert(http.transfer_encoding_is_chunked(headers) == true,
+                "te set to table values containing `chunked` should return true`")
+
+            headers["Transfer-Encoding"] = "chunked"
+            headers["Content-Length"] = 10
+            assert(http.transfer_encoding_is_chunked(headers) == true,
+                "transfer encoding should override content-length`")
+        }
+    }
+--- request
+GET /a
+--- no_error_log
+[error]
+[warn]
+
+
+=== TEST 6: Don't send Content-Length if Transfer-Encoding is specified
+--- http_config eval: $::HttpConfig
+--- config
+    location = /a {
+        content_by_lua_block {
+            local httpc = require("resty.http").new()
+            local yield = coroutine.yield
+
+            local uri = "http://127.0.0.1:" .. ngx.var.server_port .. "/b"
+
+            local res, err = assert(httpc:request_uri(uri, {
+                body = coroutine.wrap(function()
+                    yield("3\r\n")
+                    yield("foo\r\n")
+
+                    yield("3\r\n")
+                    yield("bar\r\n")
+
+                    yield("0\r\n")
+                    yield("\r\n")
+                end),
+                headers = {
+                    ["Transfer-Encoding"] = "chunked",
+                    ["Content-Length"] = 42,
+                },
+            }))
+
+            ngx.say(res.body)
+        }
+    }
+    location = /b {
+        content_by_lua_block {
+            ngx.req.read_body()
+            ngx.say(ngx.req.get_headers()["Content-Length"])
+            ngx.print(ngx.req.get_body_data())
+        }
+    }
+--- request
+GET /a
+--- response_body
+nil
+foobar
+--- no_error_log
+[error]
+[warn]