libphobos/src/std/net/curl.d

// Written in the D programming language.

/**
Networking client functionality as provided by $(HTTP _curl.haxx.se/libcurl,
libcurl). The libcurl library must be installed on the system in order to use
this module.

$(SCRIPT inhibitQuickIndex = 1;)

$(DIVC quickindex,
$(BOOKTABLE ,
$(TR $(TH Category) $(TH Functions)
)
$(TR $(TDNW High level) $(TD $(MYREF download) $(MYREF upload) $(MYREF get)
$(MYREF post) $(MYREF put) $(MYREF del) $(MYREF options) $(MYREF trace)
$(MYREF connect) $(MYREF byLine) $(MYREF byChunk)
$(MYREF byLineAsync) $(MYREF byChunkAsync) )
)
$(TR $(TDNW Low level) $(TD $(MYREF HTTP) $(MYREF FTP) $(MYREF
SMTP) )
)
)
)

Note:
You may need to link to the $(B curl) library, e.g. by adding $(D "libs": ["curl"])
to your $(B dub.json) file if you are using $(LINK2 http://code.dlang.org, DUB).

Windows x86 note:
A DMD compatible libcurl static library can be downloaded from the dlang.org
$(LINK2 http://dlang.org/download.html, download page).

Compared to using libcurl directly this module allows simpler client code for
common uses, requires no unsafe operations, and integrates better with the rest
of the language. Futhermore it provides <a href="std_range.html">$(D range)</a>
access to protocols supported by libcurl both synchronously and asynchronously.

A high level and a low level API are available. The high level API is built
entirely on top of the low level one.

The high level API is for commonly used functionality such as HTTP/FTP get. The
$(LREF byLineAsync) and $(LREF byChunkAsync) provides asynchronous <a
href="std_range.html">$(D ranges)</a> that performs the request in another
thread while handling a line/chunk in the current thread.

The low level API allows for streaming and other advanced features.

$(BOOKTABLE Cheat Sheet,
$(TR $(TH Function Name) $(TH Description)
)
$(LEADINGROW High level)
$(TR $(TDNW $(LREF download)) $(TD $(D
download("ftp.digitalmars.com/sieve.ds", "/tmp/downloaded-ftp-file"))
downloads file from URL to file system.)
)
$(TR $(TDNW $(LREF upload)) $(TD $(D
upload("/tmp/downloaded-ftp-file", "ftp.digitalmars.com/sieve.ds");)
uploads file from file system to URL.)
)
$(TR $(TDNW $(LREF get)) $(TD $(D
get("dlang.org")) returns a char[] containing the dlang.org web page.)
)
$(TR $(TDNW $(LREF put)) $(TD $(D
put("dlang.org", "Hi")) returns a char[] containing
the dlang.org web page. after a HTTP PUT of "hi")
)
$(TR $(TDNW $(LREF post)) $(TD $(D
post("dlang.org", "Hi")) returns a char[] containing
the dlang.org web page. after a HTTP POST of "hi")
)
$(TR $(TDNW $(LREF byLine)) $(TD $(D
byLine("dlang.org")) returns a range of char[] containing the
dlang.org web page.)
)
$(TR $(TDNW $(LREF byChunk)) $(TD $(D
byChunk("dlang.org", 10)) returns a range of ubyte[10] containing the
dlang.org web page.)
)
$(TR $(TDNW $(LREF byLineAsync)) $(TD $(D
byLineAsync("dlang.org")) returns a range of char[] containing the dlang.org web
 page asynchronously.)
)
$(TR $(TDNW $(LREF byChunkAsync)) $(TD $(D
byChunkAsync("dlang.org", 10)) returns a range of ubyte[10] containing the
dlang.org web page asynchronously.)
)
$(LEADINGROW Low level
)
$(TR $(TDNW $(LREF HTTP)) $(TD $(D HTTP) struct for advanced usage))
$(TR $(TDNW $(LREF FTP)) $(TD $(D FTP) struct for advanced usage))
$(TR $(TDNW $(LREF SMTP)) $(TD $(D SMTP) struct for advanced usage))
)


Example:
---
import std.net.curl, std.stdio;

// Return a char[] containing the content specified by a URL
auto content = get("dlang.org");

// Post data and return a char[] containing the content specified by a URL
auto content = post("mydomain.com/here.cgi", ["name1" : "value1", "name2" : "value2"]);

// Get content of file from ftp server
auto content = get("ftp.digitalmars.com/sieve.ds");

// Post and print out content line by line. The request is done in another thread.
foreach (line; byLineAsync("dlang.org", "Post data"))
    writeln(line);

// Get using a line range and proxy settings
auto client = HTTP();
client.proxy = "1.2.3.4";
foreach (line; byLine("dlang.org", client))
    writeln(line);
---

For more control than the high level functions provide, use the low level API:

Example:
---
import std.net.curl, std.stdio;

// GET with custom data receivers
auto http = HTTP("dlang.org");
http.onReceiveHeader =
    (in char[] key, in char[] value) { writeln(key, ": ", value); };
http.onReceive = (ubyte[] data) { /+ drop +/ return data.length; };
http.perform();
---

First, an instance of the reference-counted HTTP struct is created. Then the
custom delegates are set. These will be called whenever the HTTP instance
receives a header and a data buffer, respectively. In this simple example, the
headers are written to stdout and the data is ignored. If the request should be
stopped before it has finished then return something less than data.length from
the onReceive callback. See $(LREF onReceiveHeader)/$(LREF onReceive) for more
information. Finally the HTTP request is effected by calling perform(), which is
synchronous.

Source: $(PHOBOSSRC std/net/_curl.d)

Copyright: Copyright Jonas Drewsen 2011-2012
License: $(HTTP www.boost.org/LICENSE_1_0.txt, Boost License 1.0).
Authors: Jonas Drewsen. Some of the SMTP code contributed by Jimmy Cao.

Credits: The functionally is based on $(HTTP _curl.haxx.se/libcurl, libcurl).
         LibCurl is licensed under an MIT/X derivative license.
*/
/*
         Copyright Jonas Drewsen 2011 - 2012.
Distributed under the Boost Software License, Version 1.0.
   (See accompanying file LICENSE_1_0.txt or copy at
         http://www.boost.org/LICENSE_1_0.txt)
*/
module std.net.curl;

import core.thread;
import etc.c.curl;
import std.concurrency;
import std.encoding;
import std.exception;
import std.meta;
import std.range.primitives;
import std.socket : InternetAddress;
import std.traits;
import std.typecons;

import std.internal.cstring;

public import etc.c.curl : CurlOption;

version (unittest)
{
    // Run unit test with the PHOBOS_TEST_ALLOW_NET=1 set in order to
    // allow net traffic
    import std.range;
    import std.stdio;

    import std.socket : Address, INADDR_LOOPBACK, Socket, SocketShutdown, TcpSocket;

    private struct TestServer
    {
        string addr() { return _addr; }

        void handle(void function(Socket s) dg)
        {
            tid.send(dg);
        }

    private:
        string _addr;
        Tid tid;
        TcpSocket sock;

        static void loop(shared TcpSocket listener)
        {
            try while (true)
            {
                void function(Socket) handler = void;
                try
                    handler = receiveOnly!(typeof(handler));
                catch (OwnerTerminated)
                    return;
                handler((cast() listener).accept);
            }
            catch (Throwable e)
            {
                stderr.writeln(e);  // Bugzilla 7018
            }
        }
    }

    private TestServer startServer()
    {
        tlsInit = true;
        auto sock = new TcpSocket;
        sock.bind(new InternetAddress(INADDR_LOOPBACK, InternetAddress.PORT_ANY));
        sock.listen(1);
        auto addr = sock.localAddress.toString();
        auto tid = spawn(&TestServer.loop, cast(shared) sock);
        return TestServer(addr, tid, sock);
    }

    __gshared TestServer server;
    bool tlsInit;

    private ref TestServer testServer()
    {
        return initOnce!server(startServer());
    }

    static ~this()
    {
        // terminate server from a thread local dtor of the thread that started it,
        //  because thread_joinall is called before shared module dtors
        if (tlsInit && server.sock)
        {
            server.sock.shutdown(SocketShutdown.RECEIVE);
            server.sock.close();
        }
    }

    private struct Request(T)
    {
        string hdrs;
        immutable(T)[] bdy;
    }

    private Request!T recvReq(T=char)(Socket s)
    {
        import std.algorithm.comparison : min;
        import std.algorithm.searching : find, canFind;
        import std.conv : to;
        import std.regex : ctRegex, matchFirst;

        ubyte[1024] tmp=void;
        ubyte[] buf;

        while (true)
        {
            auto nbytes = s.receive(tmp[]);
            assert(nbytes >= 0);

            immutable beg = buf.length > 3 ? buf.length - 3 : 0;
            buf ~= tmp[0 .. nbytes];
            auto bdy = buf[beg .. $].find(cast(ubyte[])"\r\n\r\n");
            if (bdy.empty)
                continue;

            auto hdrs = cast(string) buf[0 .. $ - bdy.length];
            bdy.popFrontN(4);
            // no support for chunked transfer-encoding
            if (auto m = hdrs.matchFirst(ctRegex!(`Content-Length: ([0-9]+)`, "i")))
            {
                import std.uni : asUpperCase;
                if (hdrs.asUpperCase.canFind("EXPECT: 100-CONTINUE"))
                    s.send(httpContinue);

                size_t remain = m.captures[1].to!size_t - bdy.length;
                while (remain)
                {
                    nbytes = s.receive(tmp[0 .. min(remain, $)]);
                    assert(nbytes >= 0);
                    buf ~= tmp[0 .. nbytes];
                    remain -= nbytes;
                }
            }
            else
            {
                assert(bdy.empty);
            }
            bdy = buf[hdrs.length + 4 .. $];
            return typeof(return)(hdrs, cast(immutable(T)[])bdy);
        }
    }

    private string httpOK(string msg)
    {
        import std.conv : to;

        return "HTTP/1.1 200 OK\r\n"~
            "Content-Type: text/plain\r\n"~
            "Content-Length: "~msg.length.to!string~"\r\n"~
            "\r\n"~
            msg;
    }

    private string httpOK()
    {
        return "HTTP/1.1 200 OK\r\n"~
            "Content-Length: 0\r\n"~
            "\r\n";
    }

    private string httpNotFound()
    {
        return "HTTP/1.1 404 Not Found\r\n"~
            "Content-Length: 0\r\n"~
            "\r\n";
    }

    private enum httpContinue = "HTTP/1.1 100 Continue\r\n\r\n";
}
version (StdDdoc) import std.stdio;

// Default data timeout for Protocols
private enum _defaultDataTimeout = dur!"minutes"(2);

/**
Macros:

CALLBACK_PARAMS = $(TABLE ,
    $(DDOC_PARAM_ROW
        $(DDOC_PARAM_ID $(DDOC_PARAM dlTotal))
        $(DDOC_PARAM_DESC total bytes to download)
        )
    $(DDOC_PARAM_ROW
        $(DDOC_PARAM_ID $(DDOC_PARAM dlNow))
        $(DDOC_PARAM_DESC currently downloaded bytes)
        )
    $(DDOC_PARAM_ROW
        $(DDOC_PARAM_ID $(DDOC_PARAM ulTotal))
        $(DDOC_PARAM_DESC total bytes to upload)
        )
    $(DDOC_PARAM_ROW
        $(DDOC_PARAM_ID $(DDOC_PARAM ulNow))
        $(DDOC_PARAM_DESC currently uploaded bytes)
        )
)
*/

/** Connection type used when the URL should be used to auto detect the protocol.
  *
  * This struct is used as placeholder for the connection parameter when calling
  * the high level API and the connection type (HTTP/FTP) should be guessed by
  * inspecting the URL parameter.
  *
  * The rules for guessing the protocol are:
  * 1, if URL starts with ftp://, ftps:// or ftp. then FTP connection is assumed.
  * 2, HTTP connection otherwise.
  *
  * Example:
  * ---
  * import std.net.curl;
  * // Two requests below will do the same.
  * string content;
  *
  * // Explicit connection provided
  * content = get!HTTP("dlang.org");
  *
  * // Guess connection type by looking at the URL
  * content = get!AutoProtocol("ftp://foo.com/file");
  * // and since AutoProtocol is default this is the same as
  * content = get("ftp://foo.com/file");
  * // and will end up detecting FTP from the url and be the same as
  * content = get!FTP("ftp://foo.com/file");
  * ---
  */
struct AutoProtocol { }

// Returns true if the url points to an FTP resource
private bool isFTPUrl(const(char)[] url)
{
    import std.algorithm.searching : startsWith;
    import std.uni : toLower;

    return startsWith(url.toLower(), "ftp://", "ftps://", "ftp.") != 0;
}

// Is true if the Conn type is a valid Curl Connection type.
private template isCurlConn(Conn)
{
    enum auto isCurlConn = is(Conn : HTTP) ||
        is(Conn : FTP) || is(Conn : AutoProtocol);
}

/** HTTP/FTP download to local file system.
 *
 * Params:
 * url = resource to download
 * saveToPath = path to store the downloaded content on local disk
 * conn = connection to use e.g. FTP or HTTP. The default AutoProtocol will
 *        guess connection type and create a new instance for this call only.
 *
 * Example:
 * ----
 * import std.net.curl;
 * download("d-lang.appspot.com/testUrl2", "/tmp/downloaded-http-file");
 * ----
 */
void download(Conn = AutoProtocol)(const(char)[] url, string saveToPath, Conn conn = Conn())
if (isCurlConn!Conn)
{
    static if (is(Conn : HTTP) || is(Conn : FTP))
    {
        import std.stdio : File;
        conn.url = url;
        auto f = File(saveToPath, "wb");
        conn.onReceive = (ubyte[] data) { f.rawWrite(data); return data.length; };
        conn.perform();
    }
    else
    {
        if (isFTPUrl(url))
            return download!FTP(url, saveToPath, FTP());
        else
            return download!HTTP(url, saveToPath, HTTP());
    }
}

@system unittest
{
    import std.algorithm.searching : canFind;
    static import std.file;

    foreach (host; [testServer.addr, "http://"~testServer.addr])
    {
        testServer.handle((s) {
            assert(s.recvReq.hdrs.canFind("GET /"));
            s.send(httpOK("Hello world"));
        });
        auto fn = std.file.deleteme;
        scope (exit)
        {
            if (std.file.exists(fn))
                std.file.remove(fn);
        }
        download(host, fn);
        assert(std.file.readText(fn) == "Hello world");
    }
}

/** Upload file from local files system using the HTTP or FTP protocol.
 *
 * Params:
 * loadFromPath = path load data from local disk.
 * url = resource to upload to
 * conn = connection to use e.g. FTP or HTTP. The default AutoProtocol will
 *        guess connection type and create a new instance for this call only.
 *
 * Example:
 * ----
 * import std.net.curl;
 * upload("/tmp/downloaded-ftp-file", "ftp.digitalmars.com/sieve.ds");
 * upload("/tmp/downloaded-http-file", "d-lang.appspot.com/testUrl2");
 * ----
 */
void upload(Conn = AutoProtocol)(string loadFromPath, const(char)[] url, Conn conn = Conn())
if (isCurlConn!Conn)
{
    static if (is(Conn : HTTP))
    {
        conn.url = url;
        conn.method = HTTP.Method.put;
    }
    else static if (is(Conn : FTP))
    {
        conn.url = url;
        conn.handle.set(CurlOption.upload, 1L);
    }
    else
    {
        if (isFTPUrl(url))
            return upload!FTP(loadFromPath, url, FTP());
        else
            return upload!HTTP(loadFromPath, url, HTTP());
    }

    static if (is(Conn : HTTP) || is(Conn : FTP))
    {
        import std.stdio : File;
        auto f = File(loadFromPath, "rb");
        conn.onSend = buf => f.rawRead(buf).length;
        immutable sz = f.size;
        if (sz != ulong.max)
            conn.contentLength = sz;
        conn.perform();
    }
}

@system unittest
{
    import std.algorithm.searching : canFind;
    static import std.file;

    foreach (host; [testServer.addr, "http://"~testServer.addr])
    {
        auto fn = std.file.deleteme;
        scope (exit)
        {
            if (std.file.exists(fn))
                std.file.remove(fn);
        }
        std.file.write(fn, "upload data\n");
        testServer.handle((s) {
            auto req = s.recvReq;
            assert(req.hdrs.canFind("PUT /path"));
            assert(req.bdy.canFind("upload data"));
            s.send(httpOK());
        });
        upload(fn, host ~ "/path");
    }
}

/** HTTP/FTP get content.
 *
 * Params:
 * url = resource to get
 * conn = connection to use e.g. FTP or HTTP. The default AutoProtocol will
 *        guess connection type and create a new instance for this call only.
 *
 * The template parameter $(D T) specifies the type to return. Possible values
 * are $(D char) and $(D ubyte) to return $(D char[]) or $(D ubyte[]). If asking
 * for $(D char), content will be converted from the connection character set
 * (specified in HTTP response headers or FTP connection properties, both ISO-8859-1
 * by default) to UTF-8.
 *
 * Example:
 * ----
 * import std.net.curl;
 * auto content = get("d-lang.appspot.com/testUrl2");
 * ----
 *
 * Returns:
 * A T[] range containing the content of the resource pointed to by the URL.
 *
 * Throws:
 *
 * $(D CurlException) on error.
 *
 * See_Also: $(LREF HTTP.Method)
 */
T[] get(Conn = AutoProtocol, T = char)(const(char)[] url, Conn conn = Conn())
if ( isCurlConn!Conn && (is(T == char) || is(T == ubyte)) )
{
    static if (is(Conn : HTTP))
    {
        conn.method = HTTP.Method.get;
        return _basicHTTP!(T)(url, "", conn);

    }
    else static if (is(Conn : FTP))
    {
        return _basicFTP!(T)(url, "", conn);
    }
    else
    {
        if (isFTPUrl(url))
            return get!(FTP,T)(url, FTP());
        else
            return get!(HTTP,T)(url, HTTP());
    }
}

@system unittest
{
    import std.algorithm.searching : canFind;

    foreach (host; [testServer.addr, "http://"~testServer.addr])
    {
        testServer.handle((s) {
            assert(s.recvReq.hdrs.canFind("GET /path"));
            s.send(httpOK("GETRESPONSE"));
        });
        auto res = get(host ~ "/path");
        assert(res == "GETRESPONSE");
    }
}


/** HTTP post content.
 *
 * Params:
 *     url = resource to post to
 *     postDict = data to send as the body of the request. An associative array
 *                of $(D string) is accepted and will be encoded using
 *                www-form-urlencoding
 *     postData = data to send as the body of the request. An array
 *                of an arbitrary type is accepted and will be cast to ubyte[]
 *                before sending it.
 *     conn = HTTP connection to use
 *     T    = The template parameter $(D T) specifies the type to return. Possible values
 *            are $(D char) and $(D ubyte) to return $(D char[]) or $(D ubyte[]). If asking
 *            for $(D char), content will be converted from the connection character set
 *            (specified in HTTP response headers or FTP connection properties, both ISO-8859-1
 *            by default) to UTF-8.
 *
 * Examples:
 * ----
 * import std.net.curl;
 *
 * auto content1 = post("d-lang.appspot.com/testUrl2", ["name1" : "value1", "name2" : "value2"]);
 * auto content2 = post("d-lang.appspot.com/testUrl2", [1,2,3,4]);
 * ----
 *
 * Returns:
 * A T[] range containing the content of the resource pointed to by the URL.
 *
 * See_Also: $(LREF HTTP.Method)
 */
T[] post(T = char, PostUnit)(const(char)[] url, const(PostUnit)[] postData, HTTP conn = HTTP())
if (is(T == char) || is(T == ubyte))
{
    conn.method = HTTP.Method.post;
    return _basicHTTP!(T)(url, postData, conn);
}

@system unittest
{
    import std.algorithm.searching : canFind;

    foreach (host; [testServer.addr, "http://"~testServer.addr])
    {
        testServer.handle((s) {
            auto req = s.recvReq;
            assert(req.hdrs.canFind("POST /path"));
            assert(req.bdy.canFind("POSTBODY"));
            s.send(httpOK("POSTRESPONSE"));
        });
        auto res = post(host ~ "/path", "POSTBODY");
        assert(res == "POSTRESPONSE");
    }
}

@system unittest
{
    import std.algorithm.searching : canFind;

    auto data = new ubyte[](256);
    foreach (i, ref ub; data)
        ub = cast(ubyte) i;

    testServer.handle((s) {
        auto req = s.recvReq!ubyte;
        assert(req.bdy.canFind(cast(ubyte[])[0, 1, 2, 3, 4]));
        assert(req.bdy.canFind(cast(ubyte[])[253, 254, 255]));
        s.send(httpOK(cast(ubyte[])[17, 27, 35, 41]));
    });
    auto res = post!ubyte(testServer.addr, data);
    assert(res == cast(ubyte[])[17, 27, 35, 41]);
}

/// ditto
T[] post(T = char)(const(char)[] url, string[string] postDict, HTTP conn = HTTP())
if (is(T == char) || is(T == ubyte))
{
    import std.uri : urlEncode;

    return post(url, urlEncode(postDict), conn);
}

@system unittest
{
    foreach (host; [testServer.addr, "http://" ~ testServer.addr])
    {
        testServer.handle((s) {
            auto req = s.recvReq!char;
            s.send(httpOK(req.bdy));
        });
        auto res = post(host ~ "/path", ["name1" : "value1", "name2" : "value2"]);
        assert(res == "name1=value1&name2=value2" || res == "name2=value2&name1=value1");
    }
}

/** HTTP/FTP put content.
 *
 * Params:
 * url = resource to put
 * putData = data to send as the body of the request. An array
 *           of an arbitrary type is accepted and will be cast to ubyte[]
 *           before sending it.
 * conn = connection to use e.g. FTP or HTTP. The default AutoProtocol will
 *        guess connection type and create a new instance for this call only.
 *
 * The template parameter $(D T) specifies the type to return. Possible values
 * are $(D char) and $(D ubyte) to return $(D char[]) or $(D ubyte[]). If asking
 * for $(D char), content will be converted from the connection character set
 * (specified in HTTP response headers or FTP connection properties, both ISO-8859-1
 * by default) to UTF-8.
 *
 * Example:
 * ----
 * import std.net.curl;
 * auto content = put("d-lang.appspot.com/testUrl2",
 *                      "Putting this data");
 * ----
 *
 * Returns:
 * A T[] range containing the content of the resource pointed to by the URL.
 *
 * See_Also: $(LREF HTTP.Method)
 */
T[] put(Conn = AutoProtocol, T = char, PutUnit)(const(char)[] url, const(PutUnit)[] putData,
                                                  Conn conn = Conn())
if ( isCurlConn!Conn && (is(T == char) || is(T == ubyte)) )
{
    static if (is(Conn : HTTP))
    {
        conn.method = HTTP.Method.put;
        return _basicHTTP!(T)(url, putData, conn);
    }
    else static if (is(Conn : FTP))
    {
        return _basicFTP!(T)(url, putData, conn);
    }
    else
    {
        if (isFTPUrl(url))
            return put!(FTP,T)(url, putData, FTP());
        else
            return put!(HTTP,T)(url, putData, HTTP());
    }
}

@system unittest
{
    import std.algorithm.searching : canFind;

    foreach (host; [testServer.addr, "http://"~testServer.addr])
    {
        testServer.handle((s) {
            auto req = s.recvReq;
            assert(req.hdrs.canFind("PUT /path"));
            assert(req.bdy.canFind("PUTBODY"));
            s.send(httpOK("PUTRESPONSE"));
        });
        auto res = put(host ~ "/path", "PUTBODY");
        assert(res == "PUTRESPONSE");
    }
}


/** HTTP/FTP delete content.
 *
 * Params:
 * url = resource to delete
 * conn = connection to use e.g. FTP or HTTP. The default AutoProtocol will
 *        guess connection type and create a new instance for this call only.
 *
 * Example:
 * ----
 * import std.net.curl;
 * del("d-lang.appspot.com/testUrl2");
 * ----
 *
 * See_Also: $(LREF HTTP.Method)
 */
void del(Conn = AutoProtocol)(const(char)[] url, Conn conn = Conn())
if (isCurlConn!Conn)
{
    static if (is(Conn : HTTP))
    {
        conn.method = HTTP.Method.del;
        _basicHTTP!char(url, cast(void[]) null, conn);
    }
    else static if (is(Conn : FTP))
    {
        import std.algorithm.searching : findSplitAfter;
        import std.conv : text;

        auto trimmed = url.findSplitAfter("ftp://")[1];
        auto t = trimmed.findSplitAfter("/");
        enum minDomainNameLength = 3;
        enforce!CurlException(t[0].length > minDomainNameLength,
                                text("Invalid FTP URL for delete ", url));
        conn.url = t[0];

        enforce!CurlException(!t[1].empty,
                                text("No filename specified to delete for URL ", url));
        conn.addCommand("DELE " ~ t[1]);
        conn.perform();
    }
    else
    {
        if (isFTPUrl(url))
            return del!FTP(url, FTP());
        else
            return del!HTTP(url, HTTP());
    }
}

@system unittest
{
    import std.algorithm.searching : canFind;

    foreach (host; [testServer.addr, "http://"~testServer.addr])
    {
        testServer.handle((s) {
            auto req = s.recvReq;
            assert(req.hdrs.canFind("DELETE /path"));
            s.send(httpOK());
        });
        del(host ~ "/path");
    }
}


/** HTTP options request.
 *
 * Params:
 * url = resource make a option call to
 * conn = connection to use e.g. FTP or HTTP. The default AutoProtocol will
 *        guess connection type and create a new instance for this call only.
 *
 * The template parameter $(D T) specifies the type to return. Possible values
 * are $(D char) and $(D ubyte) to return $(D char[]) or $(D ubyte[]).
 *
 * Example:
 * ----
 * import std.net.curl;
 * auto http = HTTP();
 * options("d-lang.appspot.com/testUrl2", http);
 * writeln("Allow set to " ~ http.responseHeaders["Allow"]);
 * ----
 *
 * Returns:
 * A T[] range containing the options of the resource pointed to by the URL.
 *
 * See_Also: $(LREF HTTP.Method)
 */
T[] options(T = char)(const(char)[] url, HTTP conn = HTTP())
if (is(T == char) || is(T == ubyte))
{
    conn.method = HTTP.Method.options;
    return _basicHTTP!(T)(url, null, conn);
}

@system unittest
{
    import std.algorithm.searching : canFind;

    testServer.handle((s) {
        auto req = s.recvReq;
        assert(req.hdrs.canFind("OPTIONS /path"));
        s.send(httpOK("OPTIONSRESPONSE"));
    });
    auto res = options(testServer.addr ~ "/path");
    assert(res == "OPTIONSRESPONSE");
}


/** HTTP trace request.
 *
 * Params:
 * url = resource make a trace call to
 * conn = connection to use e.g. FTP or HTTP. The default AutoProtocol will
 *        guess connection type and create a new instance for this call only.
 *
 * The template parameter $(D T) specifies the type to return. Possible values
 * are $(D char) and $(D ubyte) to return $(D char[]) or $(D ubyte[]).
 *
 * Example:
 * ----
 * import std.net.curl;
 * trace("d-lang.appspot.com/testUrl1");
 * ----
 *
 * Returns:
 * A T[] range containing the trace info of the resource pointed to by the URL.
 *
 * See_Also: $(LREF HTTP.Method)
 */
T[] trace(T = char)(const(char)[] url, HTTP conn = HTTP())
if (is(T == char) || is(T == ubyte))
{
    conn.method = HTTP.Method.trace;
    return _basicHTTP!(T)(url, cast(void[]) null, conn);
}

@system unittest
{
    import std.algorithm.searching : canFind;

    testServer.handle((s) {
        auto req = s.recvReq;
        assert(req.hdrs.canFind("TRACE /path"));
        s.send(httpOK("TRACERESPONSE"));
    });
    auto res = trace(testServer.addr ~ "/path");
    assert(res == "TRACERESPONSE");
}


/** HTTP connect request.
 *
 * Params:
 * url = resource make a connect to
 * conn = HTTP connection to use
 *
 * The template parameter $(D T) specifies the type to return. Possible values
 * are $(D char) and $(D ubyte) to return $(D char[]) or $(D ubyte[]).
 *
 * Example:
 * ----
 * import std.net.curl;
 * connect("d-lang.appspot.com/testUrl1");
 * ----
 *
 * Returns:
 * A T[] range containing the connect info of the resource pointed to by the URL.
 *
 * See_Also: $(LREF HTTP.Method)
 */
T[] connect(T = char)(const(char)[] url, HTTP conn = HTTP())
if (is(T == char) || is(T == ubyte))
{
    conn.method = HTTP.Method.connect;
    return _basicHTTP!(T)(url, cast(void[]) null, conn);
}

@system unittest
{
    import std.algorithm.searching : canFind;

    testServer.handle((s) {
        auto req = s.recvReq;
        assert(req.hdrs.canFind("CONNECT /path"));
        s.send(httpOK("CONNECTRESPONSE"));
    });
    auto res = connect(testServer.addr ~ "/path");
    assert(res == "CONNECTRESPONSE");
}


/** HTTP patch content.
 *
 * Params:
 * url = resource to patch
 * patchData = data to send as the body of the request. An array
 *           of an arbitrary type is accepted and will be cast to ubyte[]
 *           before sending it.
 * conn = HTTP connection to use
 *
 * The template parameter $(D T) specifies the type to return. Possible values
 * are $(D char) and $(D ubyte) to return $(D char[]) or $(D ubyte[]).
 *
 * Example:
 * ----
 * auto http = HTTP();
 * http.addRequestHeader("Content-Type", "application/json");
 * auto content = patch("d-lang.appspot.com/testUrl2", `{"title": "Patched Title"}`, http);
 * ----
 *
 * Returns:
 * A T[] range containing the content of the resource pointed to by the URL.
 *
 * See_Also: $(LREF HTTP.Method)
 */
T[] patch(T = char, PatchUnit)(const(char)[] url, const(PatchUnit)[] patchData,
                               HTTP conn = HTTP())
if (is(T == char) || is(T == ubyte))
{
    conn.method = HTTP.Method.patch;
    return _basicHTTP!(T)(url, patchData, conn);
}

@system unittest
{
    import std.algorithm.searching : canFind;

    testServer.handle((s) {
        auto req = s.recvReq;
        assert(req.hdrs.canFind("PATCH /path"));
        assert(req.bdy.canFind("PATCHBODY"));
        s.send(httpOK("PATCHRESPONSE"));
    });
    auto res = patch(testServer.addr ~ "/path", "PATCHBODY");
    assert(res == "PATCHRESPONSE");
}


/*
 * Helper function for the high level interface.
 *
 * It performs an HTTP request using the client which must have
 * been setup correctly before calling this function.
 */
private auto _basicHTTP(T)(const(char)[] url, const(void)[] sendData, HTTP client)
{
    import std.algorithm.comparison : min;
    import std.format : format;

    immutable doSend = sendData !is null &&
        (client.method == HTTP.Method.post ||
         client.method == HTTP.Method.put ||
         client.method == HTTP.Method.patch);

    scope (exit)
    {
        client.onReceiveHeader = null;
        client.onReceiveStatusLine = null;
        client.onReceive = null;

        if (doSend)
        {
            client.onSend = null;
            client.handle.onSeek = null;
            client.contentLength = 0;
        }
    }
    client.url = url;
    HTTP.StatusLine statusLine;
    import std.array : appender;
    auto content = appender!(ubyte[])();
    client.onReceive = (ubyte[] data)
    {
        content ~= data;
        return data.length;
    };

    if (doSend)
    {
        client.contentLength = sendData.length;
        auto remainingData = sendData;
        client.onSend = delegate size_t(void[] buf)
        {
            size_t minLen = min(buf.length, remainingData.length);
            if (minLen == 0) return 0;
            buf[0 .. minLen] = remainingData[0 .. minLen];
            remainingData = remainingData[minLen..$];
            return minLen;
        };
        client.handle.onSeek = delegate(long offset, CurlSeekPos mode)
        {
            switch (mode)
            {
                case CurlSeekPos.set:
                    remainingData = sendData[cast(size_t) offset..$];
                    return CurlSeek.ok;
                default:
                    // As of curl 7.18.0, libcurl will not pass
                    // anything other than CurlSeekPos.set.
                    return CurlSeek.cantseek;
            }
        };
    }

    client.onReceiveHeader = (in char[] key,
                              in char[] value)
    {
        if (key == "content-length")
        {
            import std.conv : to;
            content.reserve(value.to!size_t);
        }
    };
    client.onReceiveStatusLine = (HTTP.StatusLine l) { statusLine = l; };
    client.perform();
    enforce(statusLine.code / 100 == 2, new HTTPStatusException(statusLine.code,
            format("HTTP request returned status code %d (%s)", statusLine.code, statusLine.reason)));

    return _decodeContent!T(content.data, client.p.charset);
}

@system unittest
{
    import std.algorithm.searching : canFind;

    testServer.handle((s) {
        auto req = s.recvReq;
        assert(req.hdrs.canFind("GET /path"));
        s.send(httpNotFound());
    });
    auto e = collectException!HTTPStatusException(get(testServer.addr ~ "/path"));
    assert(e.msg == "HTTP request returned status code 404 (Not Found)");
    assert(e.status == 404);
}

// Bugzilla 14760 - content length must be reset after post
@system unittest
{
    import std.algorithm.searching : canFind;

    testServer.handle((s) {
        auto req = s.recvReq;
        assert(req.hdrs.canFind("POST /"));
        assert(req.bdy.canFind("POSTBODY"));
        s.send(httpOK("POSTRESPONSE"));

        req = s.recvReq;
        assert(req.hdrs.canFind("TRACE /"));
        assert(req.bdy.empty);
        s.blocking = false;
        ubyte[6] buf = void;
        assert(s.receive(buf[]) < 0);
        s.send(httpOK("TRACERESPONSE"));
    });
    auto http = HTTP();
    auto res = post(testServer.addr, "POSTBODY", http);
    assert(res == "POSTRESPONSE");
    res = trace(testServer.addr, http);
    assert(res == "TRACERESPONSE");
}

@system unittest // charset detection and transcoding to T
{
    testServer.handle((s) {
        s.send("HTTP/1.1 200 OK\r\n"~
        "Content-Length: 4\r\n"~
        "Content-Type: text/plain; charset=utf-8\r\n" ~
        "\r\n" ~
        "äbc");
    });
    auto client = HTTP();
    auto result = _basicHTTP!char(testServer.addr, "", client);
    assert(result == "äbc");

    testServer.handle((s) {
        s.send("HTTP/1.1 200 OK\r\n"~
        "Content-Length: 3\r\n"~
        "Content-Type: text/plain; charset=iso-8859-1\r\n" ~
        "\r\n" ~
        0xE4 ~ "bc");
    });
    client = HTTP();
    result = _basicHTTP!char(testServer.addr, "", client);
    assert(result == "äbc");
}

/*
 * Helper function for the high level interface.
 *
 * It performs an FTP request using the client which must have
 * been setup correctly before calling this function.
 */
private auto _basicFTP(T)(const(char)[] url, const(void)[] sendData, FTP client)
{
    import std.algorithm.comparison : min;

    scope (exit)
    {
        client.onReceive = null;
        if (!sendData.empty)
            client.onSend = null;
    }

    ubyte[] content;

    if (client.encoding.empty)
        client.encoding = "ISO-8859-1";

    client.url = url;
    client.onReceive = (ubyte[] data)
    {
        content ~= data;
        return data.length;
    };

    if (!sendData.empty)
    {
        client.handle.set(CurlOption.upload, 1L);
        client.onSend = delegate size_t(void[] buf)
        {
            size_t minLen = min(buf.length, sendData.length);
            if (minLen == 0) return 0;
            buf[0 .. minLen] = sendData[0 .. minLen];
            sendData = sendData[minLen..$];
            return minLen;
        };
    }

    client.perform();

    return _decodeContent!T(content, client.encoding);
}

/* Used by _basicHTTP() and _basicFTP() to decode ubyte[] to
 * correct string format
 */
private auto _decodeContent(T)(ubyte[] content, string encoding)
{
    static if (is(T == ubyte))
    {
        return content;
    }
    else
    {
        import std.format : format;

        // Optimally just return the utf8 encoded content
        if (encoding == "UTF-8")
            return cast(char[])(content);

        // The content has to be re-encoded to utf8
        auto scheme = EncodingScheme.create(encoding);
        enforce!CurlException(scheme !is null,
                                format("Unknown encoding '%s'", encoding));

        auto strInfo = decodeString(content, scheme);
        enforce!CurlException(strInfo[0] != size_t.max,
                                format("Invalid encoding sequence for encoding '%s'",
                                       encoding));

        return strInfo[1];
    }
}

alias KeepTerminator = Flag!"keepTerminator";
/+
struct ByLineBuffer(Char)
{
    bool linePresent;
    bool EOF;
    Char[] buffer;
    ubyte[] decodeRemainder;

    bool append(const(ubyte)[] data)
    {
        byLineBuffer ~= data;
    }

    @property bool linePresent()
    {
        return byLinePresent;
    }

    Char[] get()
    {
        if (!linePresent)
        {
            // Decode ubyte[] into Char[] until a Terminator is found.
            // If not Terminator is found and EOF is false then raise an
            // exception.
        }
        return byLineBuffer;
    }

}
++/
/** HTTP/FTP fetch content as a range of lines.
 *
 * A range of lines is returned when the request is complete. If the method or
 * other request properties is to be customized then set the $(D conn) parameter
 * with a HTTP/FTP instance that has these properties set.
 *
 * Example:
 * ----
 * import std.net.curl, std.stdio;
 * foreach (line; byLine("dlang.org"))
 *     writeln(line);
 * ----
 *
 * Params:
 * url = The url to receive content from
 * keepTerminator = $(D Yes.keepTerminator) signals that the line terminator should be
 *                  returned as part of the lines in the range.
 * terminator = The character that terminates a line
 * conn = The connection to use e.g. HTTP or FTP.
 *
 * Returns:
 * A range of Char[] with the content of the resource pointer to by the URL
 */
auto byLine(Conn = AutoProtocol, Terminator = char, Char = char)
           (const(char)[] url, KeepTerminator keepTerminator = No.keepTerminator,
            Terminator terminator = '\n', Conn conn = Conn())
if (isCurlConn!Conn && isSomeChar!Char && isSomeChar!Terminator)
{
    static struct SyncLineInputRange
    {

        private Char[] lines;
        private Char[] current;
        private bool currentValid;
        private bool keepTerminator;
        private Terminator terminator;

        this(Char[] lines, bool kt, Terminator terminator)
        {
            this.lines = lines;
            this.keepTerminator = kt;
            this.terminator = terminator;
            currentValid = true;
            popFront();
        }

        @property @safe bool empty()
        {
            return !currentValid;
        }

        @property @safe Char[] front()
        {
            enforce!CurlException(currentValid, "Cannot call front() on empty range");
            return current;
        }

        void popFront()
        {
            import std.algorithm.searching : findSplitAfter, findSplit;

            enforce!CurlException(currentValid, "Cannot call popFront() on empty range");
            if (lines.empty)
            {
                currentValid = false;
                return;
            }

            if (keepTerminator)
            {
                auto r = findSplitAfter(lines, [ terminator ]);
                if (r[0].empty)
                {
                    current = r[1];
                    lines = r[0];
                }
                else
                {
                    current = r[0];
                    lines = r[1];
                }
            }
            else
            {
                auto r = findSplit(lines, [ terminator ]);
                current = r[0];
                lines = r[2];
            }
        }
    }

    auto result = _getForRange!Char(url, conn);
    return SyncLineInputRange(result, keepTerminator == Yes.keepTerminator, terminator);
}

@system unittest
{
    import std.algorithm.comparison : equal;

    foreach (host; [testServer.addr, "http://"~testServer.addr])
    {
        testServer.handle((s) {
            auto req = s.recvReq;
            s.send(httpOK("Line1\nLine2\nLine3"));
        });
        assert(byLine(host).equal(["Line1", "Line2", "Line3"]));
    }
}

/** HTTP/FTP fetch content as a range of chunks.
 *
 * A range of chunks is returned when the request is complete. If the method or
 * other request properties is to be customized then set the $(D conn) parameter
 * with a HTTP/FTP instance that has these properties set.
 *
 * Example:
 * ----
 * import std.net.curl, std.stdio;
 * foreach (chunk; byChunk("dlang.org", 100))
 *     writeln(chunk); // chunk is ubyte[100]
 * ----
 *
 * Params:
 * url = The url to receive content from
 * chunkSize = The size of each chunk
 * conn = The connection to use e.g. HTTP or FTP.
 *
 * Returns:
 * A range of ubyte[chunkSize] with the content of the resource pointer to by the URL
 */
auto byChunk(Conn = AutoProtocol)
            (const(char)[] url, size_t chunkSize = 1024, Conn conn = Conn())
if (isCurlConn!(Conn))
{
    static struct SyncChunkInputRange
    {
        private size_t chunkSize;
        private ubyte[] _bytes;
        private size_t offset;

        this(ubyte[] bytes, size_t chunkSize)
        {
            this._bytes = bytes;
            this.chunkSize = chunkSize;
        }

        @property @safe auto empty()
        {
            return offset == _bytes.length;
        }

        @property ubyte[] front()
        {
            size_t nextOffset = offset + chunkSize;
            if (nextOffset > _bytes.length) nextOffset = _bytes.length;
            return _bytes[offset .. nextOffset];
        }

        @safe void popFront()
        {
            offset += chunkSize;
            if (offset > _bytes.length) offset = _bytes.length;
        }
    }

    auto result = _getForRange!ubyte(url, conn);
    return SyncChunkInputRange(result, chunkSize);
}

@system unittest
{
    import std.algorithm.comparison : equal;

    foreach (host; [testServer.addr, "http://"~testServer.addr])
    {
        testServer.handle((s) {
            auto req = s.recvReq;
            s.send(httpOK(cast(ubyte[])[0, 1, 2, 3, 4, 5]));
        });
        assert(byChunk(host, 2).equal([[0, 1], [2, 3], [4, 5]]));
    }
}

private T[] _getForRange(T,Conn)(const(char)[] url, Conn conn)
{
    static if (is(Conn : HTTP))
    {
        conn.method = conn.method == HTTP.Method.undefined ? HTTP.Method.get : conn.method;
        return _basicHTTP!(T)(url, null, conn);
    }
    else static if (is(Conn : FTP))
    {
        return _basicFTP!(T)(url, null, conn);
    }
    else
    {
        if (isFTPUrl(url))
            return get!(FTP,T)(url, FTP());
        else
            return get!(HTTP,T)(url, HTTP());
    }
}

/*
  Main thread part of the message passing protocol used for all async
  curl protocols.
 */
private mixin template WorkerThreadProtocol(Unit, alias units)
{
    @property bool empty()
    {
        tryEnsureUnits();
        return state == State.done;
    }

    @property Unit[] front()
    {
        import std.format : format;
        tryEnsureUnits();
        assert(state == State.gotUnits,
               format("Expected %s but got $s",
                      State.gotUnits, state));
        return units;
    }

    void popFront()
    {
        import std.format : format;
        tryEnsureUnits();
        assert(state == State.gotUnits,
               format("Expected %s but got $s",
                      State.gotUnits, state));
        state = State.needUnits;
        // Send to worker thread for buffer reuse
        workerTid.send(cast(immutable(Unit)[]) units);
        units = null;
    }

    /** Wait for duration or until data is available and return true if data is
         available
    */
    bool wait(Duration d)
    {
        import std.datetime.stopwatch : StopWatch;

        if (state == State.gotUnits)
            return true;

        enum noDur = dur!"hnsecs"(0);
        StopWatch sw;
        sw.start();
        while (state != State.gotUnits && d > noDur)
        {
            final switch (state)
            {
            case State.needUnits:
                receiveTimeout(d,
                        (Tid origin, CurlMessage!(immutable(Unit)[]) _data)
                        {
                            if (origin != workerTid)
                                return false;
                            units = cast(Unit[]) _data.data;
                            state = State.gotUnits;
                            return true;
                        },
                        (Tid origin, CurlMessage!bool f)
                        {
                            if (origin != workerTid)
                                return false;
                            state = state.done;
                            return true;
                        }
                        );
                break;
            case State.gotUnits: return true;
            case State.done:
                return false;
            }
            d -= sw.peek();
            sw.reset();
        }
        return state == State.gotUnits;
    }

    enum State
    {
        needUnits,
        gotUnits,
        done
    }
    State state;

    void tryEnsureUnits()
    {
        while (true)
        {
            final switch (state)
            {
            case State.needUnits:
                receive(
                        (Tid origin, CurlMessage!(immutable(Unit)[]) _data)
                        {
                            if (origin != workerTid)
                                return false;
                            units = cast(Unit[]) _data.data;
                            state = State.gotUnits;
                            return true;
                        },
                        (Tid origin, CurlMessage!bool f)
                        {
                            if (origin != workerTid)
                                return false;
                            state = state.done;
                            return true;
                        }
                        );
                break;
            case State.gotUnits: return;
            case State.done:
                return;
            }
        }
    }
}

// @@@@BUG 15831@@@@
// this should be inside byLineAsync
// Range that reads one line at a time asynchronously.
private static struct AsyncLineInputRange(Char)
{
    private Char[] line;
    mixin WorkerThreadProtocol!(Char, line);

    private Tid workerTid;
    private State running;

    private this(Tid tid, size_t transmitBuffers, size_t bufferSize)
    {
        workerTid = tid;
        state = State.needUnits;

        // Send buffers to other thread for it to use.  Since no mechanism is in
        // place for moving ownership a cast to shared is done here and casted
        // back to non-shared in the receiving end.
        foreach (i ; 0 .. transmitBuffers)
        {
            auto arr = new Char[](bufferSize);
            workerTid.send(cast(immutable(Char[]))arr);
        }
    }
}

/** HTTP/FTP fetch content as a range of lines asynchronously.
 *
 * A range of lines is returned immediately and the request that fetches the
 * lines is performed in another thread. If the method or other request
 * properties is to be customized then set the $(D conn) parameter with a
 * HTTP/FTP instance that has these properties set.
 *
 * If $(D postData) is non-_null the method will be set to $(D post) for HTTP
 * requests.
 *
 * The background thread will buffer up to transmitBuffers number of lines
 * before it stops receiving data from network. When the main thread reads the
 * lines from the range it frees up buffers and allows for the background thread
 * to receive more data from the network.
 *
 * If no data is available and the main thread accesses the range it will block
 * until data becomes available. An exception to this is the $(D wait(Duration)) method on
 * the $(LREF AsyncLineInputRange). This method will wait at maximum for the
 * specified duration and return true if data is available.
 *
 * Example:
 * ----
 * import std.net.curl, std.stdio;
 * // Get some pages in the background
 * auto range1 = byLineAsync("www.google.com");
 * auto range2 = byLineAsync("www.wikipedia.org");
 * foreach (line; byLineAsync("dlang.org"))
 *     writeln(line);
 *
 * // Lines already fetched in the background and ready
 * foreach (line; range1) writeln(line);
 * foreach (line; range2) writeln(line);
 * ----
 *
 * ----
 * import std.net.curl, std.stdio;
 * // Get a line in a background thread and wait in
 * // main thread for 2 seconds for it to arrive.
 * auto range3 = byLineAsync("dlang.com");
 * if (range3.wait(dur!"seconds"(2)))
 *     writeln(range3.front);
 * else
 *     writeln("No line received after 2 seconds!");
 * ----
 *
 * Params:
 * url = The url to receive content from
 * postData = Data to HTTP Post
 * keepTerminator = $(D Yes.keepTerminator) signals that the line terminator should be
 *                  returned as part of the lines in the range.
 * terminator = The character that terminates a line
 * transmitBuffers = The number of lines buffered asynchronously
 * conn = The connection to use e.g. HTTP or FTP.
 *
 * Returns:
 * A range of Char[] with the content of the resource pointer to by the
 * URL.
 */
auto byLineAsync(Conn = AutoProtocol, Terminator = char, Char = char, PostUnit)
            (const(char)[] url, const(PostUnit)[] postData,
             KeepTerminator keepTerminator = No.keepTerminator,
             Terminator terminator = '\n',
             size_t transmitBuffers = 10, Conn conn = Conn())
if (isCurlConn!Conn && isSomeChar!Char && isSomeChar!Terminator)
{
    static if (is(Conn : AutoProtocol))
    {
        if (isFTPUrl(url))
            return byLineAsync(url, postData, keepTerminator,
                               terminator, transmitBuffers, FTP());
        else
            return byLineAsync(url, postData, keepTerminator,
                               terminator, transmitBuffers, HTTP());
    }
    else
    {
        // 50 is just an arbitrary number for now
        setMaxMailboxSize(thisTid, 50, OnCrowding.block);
        auto tid = spawn(&_spawnAsync!(Conn, Char, Terminator));
        tid.send(thisTid);
        tid.send(terminator);
        tid.send(keepTerminator == Yes.keepTerminator);

        _asyncDuplicateConnection(url, conn, postData, tid);

        return AsyncLineInputRange!Char(tid, transmitBuffers,
                                        Conn.defaultAsyncStringBufferSize);
    }
}

/// ditto
auto byLineAsync(Conn = AutoProtocol, Terminator = char, Char = char)
            (const(char)[] url, KeepTerminator keepTerminator = No.keepTerminator,
             Terminator terminator = '\n',
             size_t transmitBuffers = 10, Conn conn = Conn())
{
    static if (is(Conn : AutoProtocol))
    {
        if (isFTPUrl(url))
            return byLineAsync(url, cast(void[]) null, keepTerminator,
                               terminator, transmitBuffers, FTP());
        else
            return byLineAsync(url, cast(void[]) null, keepTerminator,
                               terminator, transmitBuffers, HTTP());
    }
    else
    {
        return byLineAsync(url, cast(void[]) null, keepTerminator,
                           terminator, transmitBuffers, conn);
    }
}

@system unittest
{
    import std.algorithm.comparison : equal;

    foreach (host; [testServer.addr, "http://"~testServer.addr])
    {
        testServer.handle((s) {
            auto req = s.recvReq;
            s.send(httpOK("Line1\nLine2\nLine3"));
        });
        assert(byLineAsync(host).equal(["Line1", "Line2", "Line3"]));
    }
}

// @@@@BUG 15831@@@@
// this should be inside byLineAsync
// Range that reads one chunk at a time asynchronously.
private static struct AsyncChunkInputRange
{
    private ubyte[] chunk;
    mixin WorkerThreadProtocol!(ubyte, chunk);

    private Tid workerTid;
    private State running;

    private this(Tid tid, size_t transmitBuffers, size_t chunkSize)
    {
        workerTid = tid;
        state = State.needUnits;

        // Send buffers to other thread for it to use.  Since no mechanism is in
        // place for moving ownership a cast to shared is done here and a cast
        // back to non-shared in the receiving end.
        foreach (i ; 0 .. transmitBuffers)
        {
            ubyte[] arr = new ubyte[](chunkSize);
            workerTid.send(cast(immutable(ubyte[]))arr);
        }
    }
}

/** HTTP/FTP fetch content as a range of chunks asynchronously.
 *
 * A range of chunks is returned immediately and the request that fetches the
 * chunks is performed in another thread. If the method or other request
 * properties is to be customized then set the $(D conn) parameter with a
 * HTTP/FTP instance that has these properties set.
 *
 * If $(D postData) is non-_null the method will be set to $(D post) for HTTP
 * requests.
 *
 * The background thread will buffer up to transmitBuffers number of chunks
 * before is stops receiving data from network. When the main thread reads the
 * chunks from the range it frees up buffers and allows for the background
 * thread to receive more data from the network.
 *
 * If no data is available and the main thread access the range it will block
 * until data becomes available. An exception to this is the $(D wait(Duration))
 * method on the $(LREF AsyncChunkInputRange). This method will wait at maximum for the specified
 * duration and return true if data is available.
 *
 * Example:
 * ----
 * import std.net.curl, std.stdio;
 * // Get some pages in the background
 * auto range1 = byChunkAsync("www.google.com", 100);
 * auto range2 = byChunkAsync("www.wikipedia.org");
 * foreach (chunk; byChunkAsync("dlang.org"))
 *     writeln(chunk); // chunk is ubyte[100]
 *
 * // Chunks already fetched in the background and ready
 * foreach (chunk; range1) writeln(chunk);
 * foreach (chunk; range2) writeln(chunk);
 * ----
 *
 * ----
 * import std.net.curl, std.stdio;
 * // Get a line in a background thread and wait in
 * // main thread for 2 seconds for it to arrive.
 * auto range3 = byChunkAsync("dlang.com", 10);
 * if (range3.wait(dur!"seconds"(2)))
 *     writeln(range3.front);
 * else
 *     writeln("No chunk received after 2 seconds!");
 * ----
 *
 * Params:
 * url = The url to receive content from
 * postData = Data to HTTP Post
 * chunkSize = The size of the chunks
 * transmitBuffers = The number of chunks buffered asynchronously
 * conn = The connection to use e.g. HTTP or FTP.
 *
 * Returns:
 * A range of ubyte[chunkSize] with the content of the resource pointer to by
 * the URL.
 */
auto byChunkAsync(Conn = AutoProtocol, PostUnit)
           (const(char)[] url, const(PostUnit)[] postData,
            size_t chunkSize = 1024, size_t transmitBuffers = 10,
            Conn conn = Conn())
if (isCurlConn!(Conn))
{
    static if (is(Conn : AutoProtocol))
    {
        if (isFTPUrl(url))
            return byChunkAsync(url, postData, chunkSize,
                                transmitBuffers, FTP());
        else
            return byChunkAsync(url, postData, chunkSize,
                                transmitBuffers, HTTP());
    }
    else
    {
        // 50 is just an arbitrary number for now
        setMaxMailboxSize(thisTid, 50, OnCrowding.block);
        auto tid = spawn(&_spawnAsync!(Conn, ubyte));
        tid.send(thisTid);

        _asyncDuplicateConnection(url, conn, postData, tid);

        return AsyncChunkInputRange(tid, transmitBuffers, chunkSize);
    }
}

/// ditto
auto byChunkAsync(Conn = AutoProtocol)
           (const(char)[] url,
            size_t chunkSize = 1024, size_t transmitBuffers = 10,
            Conn conn = Conn())
if (isCurlConn!(Conn))
{
    static if (is(Conn : AutoProtocol))
    {
        if (isFTPUrl(url))
            return byChunkAsync(url, cast(void[]) null, chunkSize,
                                transmitBuffers, FTP());
        else
            return byChunkAsync(url, cast(void[]) null, chunkSize,
                                transmitBuffers, HTTP());
    }
    else
    {
        return byChunkAsync(url, cast(void[]) null, chunkSize,
                            transmitBuffers, conn);
    }
}

@system unittest
{
    import std.algorithm.comparison : equal;

    foreach (host; [testServer.addr, "http://"~testServer.addr])
    {
        testServer.handle((s) {
            auto req = s.recvReq;
            s.send(httpOK(cast(ubyte[])[0, 1, 2, 3, 4, 5]));
        });
        assert(byChunkAsync(host, 2).equal([[0, 1], [2, 3], [4, 5]]));
    }
}


/* Used by byLineAsync/byChunkAsync to duplicate an existing connection
 * that can be used exclusively in a spawned thread.
 */
private void _asyncDuplicateConnection(Conn, PostData)
    (const(char)[] url, Conn conn, PostData postData, Tid tid)
{
    // no move semantic available in std.concurrency ie. must use casting.
    auto connDup = conn.dup();
    connDup.url = url;

    static if ( is(Conn : HTTP) )
    {
        connDup.p.headersOut = null;
        connDup.method = conn.method == HTTP.Method.undefined ?
            HTTP.Method.get : conn.method;
        if (postData !is null)
        {
            if (connDup.method == HTTP.Method.put)
            {
                connDup.handle.set(CurlOption.infilesize_large,
                                   postData.length);
            }
            else
            {
                // post
                connDup.method = HTTP.Method.post;
                connDup.handle.set(CurlOption.postfieldsize_large,
                                   postData.length);
            }
            connDup.handle.set(CurlOption.copypostfields,
                               cast(void*) postData.ptr);
        }
        tid.send(cast(ulong) connDup.handle.handle);
        tid.send(connDup.method);
    }
    else
    {
        enforce!CurlException(postData is null,
                                "Cannot put ftp data using byLineAsync()");
        tid.send(cast(ulong) connDup.handle.handle);
        tid.send(HTTP.Method.undefined);
    }
    connDup.p.curl.handle = null; // make sure handle is not freed
}

/*
  Mixin template for all supported curl protocols. This is the commom
  functionallity such as timeouts and network interface settings. This should
  really be in the HTTP/FTP/SMTP structs but the documentation tool does not
  support a mixin to put its doc strings where a mixin is done. Therefore docs
  in this template is copied into each of HTTP/FTP/SMTP below.
*/
private mixin template Protocol()
{

    /// Value to return from $(D onSend)/$(D onReceive) delegates in order to
    /// pause a request
    alias requestPause = CurlReadFunc.pause;

    /// Value to return from onSend delegate in order to abort a request
    alias requestAbort = CurlReadFunc.abort;

    static uint defaultAsyncStringBufferSize = 100;

    /**
       The curl handle used by this connection.
    */
    @property ref Curl handle() return
    {
        return p.curl;
    }

    /**
       True if the instance is stopped. A stopped instance is not usable.
    */
    @property bool isStopped()
    {
        return p.curl.stopped;
    }

    /// Stop and invalidate this instance.
    void shutdown()
    {
        p.curl.shutdown();
    }

    /** Set verbose.
        This will print request information to stderr.
     */
    @property void verbose(bool on)
    {
        p.curl.set(CurlOption.verbose, on ? 1L : 0L);
    }

    // Connection settings

    /// Set timeout for activity on connection.
    @property void dataTimeout(Duration d)
    {
        p.curl.set(CurlOption.low_speed_limit, 1);
        p.curl.set(CurlOption.low_speed_time, d.total!"seconds");
    }

    /** Set maximum time an operation is allowed to take.
        This includes dns resolution, connecting, data transfer, etc.
     */
    @property void operationTimeout(Duration d)
    {
        p.curl.set(CurlOption.timeout_ms, d.total!"msecs");
    }

    /// Set timeout for connecting.
    @property void connectTimeout(Duration d)
    {
        p.curl.set(CurlOption.connecttimeout_ms, d.total!"msecs");
    }

    // Network settings

    /** Proxy
     *  See: $(HTTP curl.haxx.se/libcurl/c/curl_easy_setopt.html#CURLOPTPROXY, _proxy)
     */
    @property void proxy(const(char)[] host)
    {
        p.curl.set(CurlOption.proxy, host);
    }

    /** Proxy port
     *  See: $(HTTP curl.haxx.se/libcurl/c/curl_easy_setopt.html#CURLOPTPROXYPORT, _proxy_port)
     */
    @property void proxyPort(ushort port)
    {
        p.curl.set(CurlOption.proxyport, cast(long) port);
    }

    /// Type of proxy
    alias CurlProxy = etc.c.curl.CurlProxy;

    /** Proxy type
     *  See: $(HTTP curl.haxx.se/libcurl/c/curl_easy_setopt.html#CURLOPTPROXY, _proxy_type)
     */
    @property void proxyType(CurlProxy type)
    {
        p.curl.set(CurlOption.proxytype, cast(long) type);
    }

    /// DNS lookup timeout.
    @property void dnsTimeout(Duration d)
    {
        p.curl.set(CurlOption.dns_cache_timeout, d.total!"msecs");
    }

    /**
     * The network interface to use in form of the the IP of the interface.
     *
     * Example:
     * ----
     * theprotocol.netInterface = "192.168.1.32";
     * theprotocol.netInterface = [ 192, 168, 1, 32 ];
     * ----
     *
     * See: $(REF InternetAddress, std,socket)
     */
    @property void netInterface(const(char)[] i)
    {
        p.curl.set(CurlOption.intrface, i);
    }

    /// ditto
    @property void netInterface(const(ubyte)[4] i)
    {
        import std.format : format;
        const str = format("%d.%d.%d.%d", i[0], i[1], i[2], i[3]);
        netInterface = str;
    }

    /// ditto
    @property void netInterface(InternetAddress i)
    {
        netInterface = i.toAddrString();
    }

    /**
       Set the local outgoing port to use.
       Params:
       port = the first outgoing port number to try and use
    */
    @property void localPort(ushort port)
    {
        p.curl.set(CurlOption.localport, cast(long) port);
    }

    /**
       Set the no proxy flag for the specified host names.
       Params:
       test = a list of comma host names that do not require
              proxy to get reached
    */
    void setNoProxy(string hosts)
    {
        p.curl.set(CurlOption.noproxy, hosts);
    }

    /**
       Set the local outgoing port range to use.
       This can be used together with the localPort property.
       Params:
       range = if the first port is occupied then try this many
               port number forwards
    */
    @property void localPortRange(ushort range)
    {
        p.curl.set(CurlOption.localportrange, cast(long) range);
    }

    /** Set the tcp no-delay socket option on or off.
        See: $(HTTP curl.haxx.se/libcurl/c/curl_easy_setopt.html#CURLOPTTCPNODELAY, nodelay)
    */
    @property void tcpNoDelay(bool on)
    {
        p.curl.set(CurlOption.tcp_nodelay, cast(long) (on ? 1 : 0) );
    }

    /** Sets whether SSL peer certificates should be verified.
        See: $(HTTP curl.haxx.se/libcurl/c/curl_easy_setopt.html#CURLOPTSSLVERIFYPEER, verifypeer)
    */
    @property void verifyPeer(bool on)
    {
      p.curl.set(CurlOption.ssl_verifypeer, on ? 1 : 0);
    }

    /** Sets whether the host within an SSL certificate should be verified.
        See: $(HTTP curl.haxx.se/libcurl/c/curl_easy_setopt.html#CURLOPTSSLVERIFYHOST, verifypeer)
    */
    @property void verifyHost(bool on)
    {
      p.curl.set(CurlOption.ssl_verifyhost, on ? 2 : 0);
    }

    // Authentication settings

    /**
       Set the user name, password and optionally domain for authentication
       purposes.

       Some protocols may need authentication in some cases. Use this
       function to provide credentials.

       Params:
       username = the username
       password = the password
       domain = used for NTLM authentication only and is set to the NTLM domain
                name
    */
    void setAuthentication(const(char)[] username, const(char)[] password,
                           const(char)[] domain = "")
    {
        import std.format : format;
        if (!domain.empty)
            username = format("%s/%s", domain, username);
        p.curl.set(CurlOption.userpwd, format("%s:%s", username, password));
    }

    @system unittest
    {
        import std.algorithm.searching : canFind;

        testServer.handle((s) {
            auto req = s.recvReq;
            assert(req.hdrs.canFind("GET /"));
            assert(req.hdrs.canFind("Basic dXNlcjpwYXNz"));
            s.send(httpOK());
        });

        auto http = HTTP(testServer.addr);
        http.onReceive = (ubyte[] data) { return data.length; };
        http.setAuthentication("user", "pass");
        http.perform();

        // Bugzilla 17540
        http.setNoProxy("www.example.com");
    }

    /**
       Set the user name and password for proxy authentication.

       Params:
       username = the username
       password = the password
    */
    void setProxyAuthentication(const(char)[] username, const(char)[] password)
    {
        import std.array : replace;
        import std.format : format;

        p.curl.set(CurlOption.proxyuserpwd,
            format("%s:%s",
                username.replace(":", "%3A"),
                password.replace(":", "%3A"))
        );
    }

    /**
     * The event handler that gets called when data is needed for sending. The
     * length of the $(D void[]) specifies the maximum number of bytes that can
     * be sent.
     *
     * Returns:
     * The callback returns the number of elements in the buffer that have been
     * filled and are ready to send.
     * The special value $(D .abortRequest) can be returned in order to abort the
     * current request.
     * The special value $(D .pauseRequest) can be returned in order to pause the
     * current request.
     *
     * Example:
     * ----
     * import std.net.curl;
     * string msg = "Hello world";
     * auto client = HTTP("dlang.org");
     * client.onSend = delegate size_t(void[] data)
     * {
     *     auto m = cast(void[]) msg;
     *     size_t length = m.length > data.length ? data.length : m.length;
     *     if (length == 0) return 0;
     *     data[0 .. length] = m[0 .. length];
     *     msg = msg[length..$];
     *     return length;
     * };
     * client.perform();
     * ----
     */
    @property void onSend(size_t delegate(void[]) callback)
    {
        p.curl.clear(CurlOption.postfields); // cannot specify data when using callback
        p.curl.onSend = callback;
    }

    /**
      * The event handler that receives incoming data. Be sure to copy the
      * incoming ubyte[] since it is not guaranteed to be valid after the
      * callback returns.
      *
      * Returns:
      * The callback returns the number of incoming bytes read. If the entire array is
      * not read the request will abort.
      * The special value .pauseRequest can be returned in order to pause the
      * current request.
      *
      * Example:
      * ----
      * import std.net.curl, std.stdio;
      * auto client = HTTP("dlang.org");
      * client.onReceive = (ubyte[] data)
      * {
      *     writeln("Got data", to!(const(char)[])(data));
      *     return data.length;
      * };
      * client.perform();
      * ----
      */
    @property void onReceive(size_t delegate(ubyte[]) callback)
    {
        p.curl.onReceive = callback;
    }

    /**
      * The event handler that gets called to inform of upload/download progress.
      *
      * Params:
      * dlTotal = total bytes to download
      * dlNow = currently downloaded bytes
      * ulTotal = total bytes to upload
      * ulNow = currently uploaded bytes
      *
      * Returns:
      * Return 0 from the callback to signal success, return non-zero to abort
      *          transfer
      *
      * Example:
      * ----
      * import std.net.curl, std.stdio;
      * auto client = HTTP("dlang.org");
      * client.onProgress = delegate int(size_t dl, size_t dln, size_t ul, size_t ult)
      * {
      *     writeln("Progress: downloaded ", dln, " of ", dl);
      *     writeln("Progress: uploaded ", uln, " of ", ul);
      * };
      * client.perform();
      * ----
      */
    @property void onProgress(int delegate(size_t dlTotal, size_t dlNow,
                                           size_t ulTotal, size_t ulNow) callback)
    {
        p.curl.onProgress = callback;
    }
}

/*
  Decode $(D ubyte[]) array using the provided EncodingScheme up to maxChars
  Returns: Tuple of ubytes read and the $(D Char[]) characters decoded.
           Not all ubytes are guaranteed to be read in case of decoding error.
*/
private Tuple!(size_t,Char[])
decodeString(Char = char)(const(ubyte)[] data,
                          EncodingScheme scheme,
                          size_t maxChars = size_t.max)
{
    Char[] res;
    immutable startLen = data.length;
    size_t charsDecoded = 0;
    while (data.length && charsDecoded < maxChars)
    {
        immutable dchar dc = scheme.safeDecode(data);
        if (dc == INVALID_SEQUENCE)
        {
            return typeof(return)(size_t.max, cast(Char[]) null);
        }
        charsDecoded++;
        res ~= dc;
    }
    return typeof(return)(startLen-data.length, res);
}

/*
  Decode $(D ubyte[]) array using the provided $(D EncodingScheme) until a the
  line terminator specified is found. The basesrc parameter is effectively
  prepended to src as the first thing.

  This function is used for decoding as much of the src buffer as
  possible until either the terminator is found or decoding fails. If
  it fails as the last data in the src it may mean that the src buffer
  were missing some bytes in order to represent a correct code
  point. Upon the next call to this function more bytes have been
  received from net and the failing bytes should be given as the
  basesrc parameter. It is done this way to minimize data copying.

  Returns: true if a terminator was found
           Not all ubytes are guaranteed to be read in case of decoding error.
           any decoded chars will be inserted into dst.
*/
private bool decodeLineInto(Terminator, Char = char)(ref const(ubyte)[] basesrc,
                                                     ref const(ubyte)[] src,
                                                     ref Char[] dst,
                                                     EncodingScheme scheme,
                                                     Terminator terminator)
{
    import std.algorithm.searching : endsWith;

    // if there is anything in the basesrc then try to decode that
    // first.
    if (basesrc.length != 0)
    {
        // Try to ensure 4 entries in the basesrc by copying from src.
        immutable blen = basesrc.length;
        immutable len = (basesrc.length + src.length) >= 4 ?
                     4 : basesrc.length + src.length;
        basesrc.length = len;

        immutable dchar dc = scheme.safeDecode(basesrc);
        if (dc == INVALID_SEQUENCE)
        {
            enforce!CurlException(len != 4, "Invalid code sequence");
            return false;
        }
        dst ~= dc;
        src = src[len-basesrc.length-blen .. $]; // remove used ubytes from src
        basesrc.length = 0;
    }

    while (src.length)
    {
        const lsrc = src;
        dchar dc = scheme.safeDecode(src);
        if (dc == INVALID_SEQUENCE)
        {
            if (src.empty)
            {
                // The invalid sequence was in the end of the src.  Maybe there
                // just need to be more bytes available so these last bytes are
                // put back to src for later use.
                src = lsrc;
                return false;
            }
            dc = '?';
        }
        dst ~= dc;

        if (dst.endsWith(terminator))
            return true;
    }
    return false; // no terminator found
}

/**
  * HTTP client functionality.
  *
  * Example:
  * ---
  * import std.net.curl, std.stdio;
  *
  * // Get with custom data receivers
  * auto http = HTTP("dlang.org");
  * http.onReceiveHeader =
  *     (in char[] key, in char[] value) { writeln(key ~ ": " ~ value); };
  * http.onReceive = (ubyte[] data) { /+ drop +/ return data.length; };
  * http.perform();
  *
  * // Put with data senders
  * auto msg = "Hello world";
  * http.contentLength = msg.length;
  * http.onSend = (void[] data)
  * {
  *     auto m = cast(void[]) msg;
  *     size_t len = m.length > data.length ? data.length : m.length;
  *     if (len == 0) return len;
  *     data[0 .. len] = m[0 .. len];
  *     msg = msg[len..$];
  *     return len;
  * };
  * http.perform();
  *
  * // Track progress
  * http.method = HTTP.Method.get;
  * http.url = "http://upload.wikimedia.org/wikipedia/commons/"
  *            "5/53/Wikipedia-logo-en-big.png";
  * http.onReceive = (ubyte[] data) { return data.length; };
  * http.onProgress = (size_t dltotal, size_t dlnow,
  *                    size_t ultotal, size_t ulnow)
  * {
  *     writeln("Progress ", dltotal, ", ", dlnow, ", ", ultotal, ", ", ulnow);
  *     return 0;
  * };
  * http.perform();
  * ---
  *
  * See_Also: $(_HTTP www.ietf.org/rfc/rfc2616.txt, RFC2616)
  *
  */
struct HTTP
{
    mixin Protocol;

    import std.datetime.systime : SysTime;

    /// Authentication method equal to $(REF CurlAuth, etc,c,curl)
    alias AuthMethod = CurlAuth;

    static private uint defaultMaxRedirects = 10;

    private struct Impl
    {
        ~this()
        {
            if (headersOut !is null)
                Curl.curl.slist_free_all(headersOut);
            if (curl.handle !is null) // work around RefCounted/emplace bug
                curl.shutdown();
        }
        Curl curl;
        curl_slist* headersOut;
        string[string] headersIn;
        string charset;

        /// The status line of the final sub-request in a request.
        StatusLine status;
        private void delegate(StatusLine) onReceiveStatusLine;

        /// The HTTP method to use.
        Method method = Method.undefined;

        @system @property void onReceiveHeader(void delegate(in char[] key,
                                                     in char[] value) callback)
        {
            import std.algorithm.searching : startsWith;
            import std.regex : regex, match;
            import std.uni : toLower;

            // Wrap incoming callback in order to separate http status line from
            // http headers.  On redirected requests there may be several such
            // status lines. The last one is the one recorded.
            auto dg = (in char[] header)
            {
                import std.utf : UTFException;
                try
                {
                    if (header.empty)
                    {
                        // header delimiter
                        return;
                    }
                    if (header.startsWith("HTTP/"))
                    {
                        headersIn.clear();
                        if (parseStatusLine(header, status))
                        {
                            if (onReceiveStatusLine != null)
                                onReceiveStatusLine(status);
                        }
                        return;
                    }

                    // Normal http header
                    auto m = match(cast(char[]) header, regex("(.*?): (.*)$"));

                    auto fieldName = m.captures[1].toLower().idup;
                    if (fieldName == "content-type")
                    {
                        auto mct = match(cast(char[]) m.captures[2],
                                         regex("charset=([^;]*)", "i"));
                        if (!mct.empty && mct.captures.length > 1)
                            charset = mct.captures[1].idup;
                    }

                    if (!m.empty && callback !is null)
                        callback(fieldName, m.captures[2]);
                    headersIn[fieldName] = m.captures[2].idup;
                }
                catch (UTFException e)
                {
                    //munch it - a header should be all ASCII, any "wrong UTF" is broken header
                }
            };

            curl.onReceiveHeader = dg;
        }
    }

    private RefCounted!Impl p;

    /// Parse status line, as received from / generated by cURL.
    private static bool parseStatusLine(in char[] header, out StatusLine status) @safe
    {
        import std.conv : to;
        import std.regex : regex, match;

        const m = match(header, regex(r"^HTTP/(\d+)(?:\.(\d+))? (\d+)(?: (.*))?$"));
        if (m.empty)
            return false; // Invalid status line
        else
        {
            status.majorVersion = to!ushort(m.captures[1]);
            status.minorVersion = m.captures[2].length ? to!ushort(m.captures[2]) : 0;
            status.code = to!ushort(m.captures[3]);
            status.reason = m.captures[4].idup;
            return true;
        }
    }

    @safe unittest
    {
        StatusLine status;
        assert(parseStatusLine("HTTP/1.1 200 OK", status)
            && status == StatusLine(1, 1, 200, "OK"));
        assert(parseStatusLine("HTTP/1.0 304 Not Modified", status)
            && status == StatusLine(1, 0, 304, "Not Modified"));
        // The HTTP2 protocol is binary; cURL generates this fake text header.
        assert(parseStatusLine("HTTP/2 200", status)
            && status == StatusLine(2, 0, 200, null));
    }

    /** Time condition enumeration as an alias of $(REF CurlTimeCond, etc,c,curl)

        $(HTTP www.w3.org/Protocols/rfc2616/rfc2616-sec14.html#sec14.25, _RFC2616 Section 14.25)
    */
    alias TimeCond = CurlTimeCond;

    /**
       Constructor taking the url as parameter.
    */
    static HTTP opCall(const(char)[] url)
    {
        HTTP http;
        http.initialize();
        http.url = url;
        return http;
    }

    ///
    static HTTP opCall()
    {
        HTTP http;
        http.initialize();
        return http;
    }

    ///
    HTTP dup()
    {
        HTTP copy;
        copy.initialize();
        copy.p.method = p.method;
        curl_slist* cur = p.headersOut;
        curl_slist* newlist = null;
        while (cur)
        {
            newlist = Curl.curl.slist_append(newlist, cur.data);
            cur = cur.next;
        }
        copy.p.headersOut = newlist;
        copy.p.curl.set(CurlOption.httpheader, copy.p.headersOut);
        copy.p.curl = p.curl.dup();
        copy.dataTimeout = _defaultDataTimeout;
        copy.onReceiveHeader = null;
        return copy;
    }

    private void initialize()
    {
        p.curl.initialize();
        maxRedirects = HTTP.defaultMaxRedirects;
        p.charset = "ISO-8859-1"; // Default charset defined in HTTP RFC
        p.method = Method.undefined;
        setUserAgent(HTTP.defaultUserAgent);
        dataTimeout = _defaultDataTimeout;
        onReceiveHeader = null;
        verifyPeer = true;
        verifyHost = true;
    }

    /**
       Perform a http request.

       After the HTTP client has been setup and possibly assigned callbacks the
       $(D perform()) method will start performing the request towards the
       specified server.

       Params:
       throwOnError = whether to throw an exception or return a CurlCode on error
    */
    CurlCode perform(ThrowOnError throwOnError = Yes.throwOnError)
    {
        p.status.reset();

        CurlOption opt;
        final switch (p.method)
        {
        case Method.head:
            p.curl.set(CurlOption.nobody, 1L);
            opt = CurlOption.nobody;
            break;
        case Method.undefined:
        case Method.get:
            p.curl.set(CurlOption.httpget, 1L);
            opt = CurlOption.httpget;
            break;
        case Method.post:
            p.curl.set(CurlOption.post, 1L);
            opt = CurlOption.post;
            break;
        case Method.put:
            p.curl.set(CurlOption.upload, 1L);
            opt = CurlOption.upload;
            break;
        case Method.del:
            p.curl.set(CurlOption.customrequest, "DELETE");
            opt = CurlOption.customrequest;
            break;
        case Method.options:
            p.curl.set(CurlOption.customrequest, "OPTIONS");
            opt = CurlOption.customrequest;
            break;
        case Method.trace:
            p.curl.set(CurlOption.customrequest, "TRACE");
            opt = CurlOption.customrequest;
            break;
        case Method.connect:
            p.curl.set(CurlOption.customrequest, "CONNECT");
            opt = CurlOption.customrequest;
            break;
        case Method.patch:
            p.curl.set(CurlOption.customrequest, "PATCH");
            opt = CurlOption.customrequest;
            break;
        }

        scope (exit) p.curl.clear(opt);
        return p.curl.perform(throwOnError);
    }

    /// The URL to specify the location of the resource.
    @property void url(const(char)[] url)
    {
        import std.algorithm.searching : startsWith;
        import std.uni : toLower;
        if (!startsWith(url.toLower(), "http://", "https://"))
            url = "http://" ~ url;
        p.curl.set(CurlOption.url, url);
    }

    /// Set the CA certificate bundle file to use for SSL peer verification
    @property void caInfo(const(char)[] caFile)
    {
        p.curl.set(CurlOption.cainfo, caFile);
    }

    // This is a workaround for mixed in content not having its
    // docs mixed in.
    version (StdDdoc)
    {
        /// Value to return from $(D onSend)/$(D onReceive) delegates in order to
        /// pause a request
        alias requestPause = CurlReadFunc.pause;

        /// Value to return from onSend delegate in order to abort a request
        alias requestAbort = CurlReadFunc.abort;

        /**
           True if the instance is stopped. A stopped instance is not usable.
        */
        @property bool isStopped();

        /// Stop and invalidate this instance.
        void shutdown();

        /** Set verbose.
            This will print request information to stderr.
        */
        @property void verbose(bool on);

        // Connection settings

        /// Set timeout for activity on connection.
        @property void dataTimeout(Duration d);

        /** Set maximum time an operation is allowed to take.
            This includes dns resolution, connecting, data transfer, etc.
          */
        @property void operationTimeout(Duration d);

        /// Set timeout for connecting.
        @property void connectTimeout(Duration d);

        // Network settings

        /** Proxy
         *  See: $(HTTP curl.haxx.se/libcurl/c/curl_easy_setopt.html#CURLOPTPROXY, _proxy)
         */
        @property void proxy(const(char)[] host);

        /** Proxy port
         *  See: $(HTTP curl.haxx.se/libcurl/c/curl_easy_setopt.html#CURLOPTPROXYPORT, _proxy_port)
         */
        @property void proxyPort(ushort port);

        /// Type of proxy
        alias CurlProxy = etc.c.curl.CurlProxy;

        /** Proxy type
         *  See: $(HTTP curl.haxx.se/libcurl/c/curl_easy_setopt.html#CURLOPTPROXY, _proxy_type)
         */
        @property void proxyType(CurlProxy type);

        /// DNS lookup timeout.
        @property void dnsTimeout(Duration d);

        /**
         * The network interface to use in form of the the IP of the interface.
         *
         * Example:
         * ----
         * theprotocol.netInterface = "192.168.1.32";
         * theprotocol.netInterface = [ 192, 168, 1, 32 ];
         * ----
         *
         * See: $(REF InternetAddress, std,socket)
         */
        @property void netInterface(const(char)[] i);

        /// ditto
        @property void netInterface(const(ubyte)[4] i);

        /// ditto
        @property void netInterface(InternetAddress i);

        /**
           Set the local outgoing port to use.
           Params:
           port = the first outgoing port number to try and use
        */
        @property void localPort(ushort port);

        /**
           Set the local outgoing port range to use.
           This can be used together with the localPort property.
           Params:
           range = if the first port is occupied then try this many
           port number forwards
        */
        @property void localPortRange(ushort range);

        /** Set the tcp no-delay socket option on or off.
            See: $(HTTP curl.haxx.se/libcurl/c/curl_easy_setopt.html#CURLOPTTCPNODELAY, nodelay)
        */
        @property void tcpNoDelay(bool on);

        // Authentication settings

        /**
           Set the user name, password and optionally domain for authentication
           purposes.

           Some protocols may need authentication in some cases. Use this
           function to provide credentials.

           Params:
           username = the username
           password = the password
           domain = used for NTLM authentication only and is set to the NTLM domain
           name
        */
        void setAuthentication(const(char)[] username, const(char)[] password,
                               const(char)[] domain = "");

        /**
           Set the user name and password for proxy authentication.

           Params:
           username = the username
           password = the password
        */
        void setProxyAuthentication(const(char)[] username, const(char)[] password);

        /**
         * The event handler that gets called when data is needed for sending. The
         * length of the $(D void[]) specifies the maximum number of bytes that can
         * be sent.
         *
         * Returns:
         * The callback returns the number of elements in the buffer that have been
         * filled and are ready to send.
         * The special value $(D .abortRequest) can be returned in order to abort the
         * current request.
         * The special value $(D .pauseRequest) can be returned in order to pause the
         * current request.
         *
         * Example:
         * ----
         * import std.net.curl;
         * string msg = "Hello world";
         * auto client = HTTP("dlang.org");
         * client.onSend = delegate size_t(void[] data)
         * {
         *     auto m = cast(void[]) msg;
         *     size_t length = m.length > data.length ? data.length : m.length;
         *     if (length == 0) return 0;
         *     data[0 .. length] = m[0 .. length];
         *     msg = msg[length..$];
         *     return length;
         * };
         * client.perform();
         * ----
         */
        @property void onSend(size_t delegate(void[]) callback);

        /**
         * The event handler that receives incoming data. Be sure to copy the
         * incoming ubyte[] since it is not guaranteed to be valid after the
         * callback returns.
         *
         * Returns:
         * The callback returns the incoming bytes read. If not the entire array is
         * the request will abort.
         * The special value .pauseRequest can be returned in order to pause the
         * current request.
         *
         * Example:
         * ----
         * import std.net.curl, std.stdio;
         * auto client = HTTP("dlang.org");
         * client.onReceive = (ubyte[] data)
         * {
         *     writeln("Got data", to!(const(char)[])(data));
         *     return data.length;
         * };
         * client.perform();
         * ----
         */
        @property void onReceive(size_t delegate(ubyte[]) callback);

        /**
         * Register an event handler that gets called to inform of
         * upload/download progress.
         *
         * Callback_parameters:
         * $(CALLBACK_PARAMS)
         *
         * Callback_returns: Return 0 to signal success, return non-zero to
         * abort transfer.
         *
         * Example:
         * ----
         * import std.net.curl, std.stdio;
         * auto client = HTTP("dlang.org");
         * client.onProgress = delegate int(size_t dl, size_t dln, size_t ul, size_t ult)
         * {
         *     writeln("Progress: downloaded ", dln, " of ", dl);
         *     writeln("Progress: uploaded ", uln, " of ", ul);
         * };
         * client.perform();
         * ----
         */
        @property void onProgress(int delegate(size_t dlTotal, size_t dlNow,
                                               size_t ulTotal, size_t ulNow) callback);
    }

    /** Clear all outgoing headers.
    */
    void clearRequestHeaders()
    {
        if (p.headersOut !is null)
            Curl.curl.slist_free_all(p.headersOut);
        p.headersOut = null;
        p.curl.clear(CurlOption.httpheader);
    }

    /** Add a header e.g. "X-CustomField: Something is fishy".
     *
     * There is no remove header functionality. Do a $(LREF clearRequestHeaders)
     * and set the needed headers instead.
     *
     * Example:
     * ---
     * import std.net.curl;
     * auto client = HTTP();
     * client.addRequestHeader("X-Custom-ABC", "This is the custom value");
     * auto content = get("dlang.org", client);
     * ---
     */
    void addRequestHeader(const(char)[] name, const(char)[] value)
    {
        import std.format : format;
        import std.uni : icmp;

        if (icmp(name, "User-Agent") == 0)
            return setUserAgent(value);
        string nv = format("%s: %s", name, value);
        p.headersOut = Curl.curl.slist_append(p.headersOut,
                                              nv.tempCString().buffPtr);
        p.curl.set(CurlOption.httpheader, p.headersOut);
    }

    /**
     * The default "User-Agent" value send with a request.
     * It has the form "Phobos-std.net.curl/$(I PHOBOS_VERSION) (libcurl/$(I CURL_VERSION))"
     */
    static string defaultUserAgent() @property
    {
        import std.compiler : version_major, version_minor;
        import std.format : format, sformat;

        // http://curl.haxx.se/docs/versions.html
        enum fmt = "Phobos-std.net.curl/%d.%03d (libcurl/%d.%d.%d)";
        enum maxLen = fmt.length - "%d%03d%d%d%d".length + 10 + 10 + 3 + 3 + 3;

        static char[maxLen] buf = void;
        static string userAgent;

        if (!userAgent.length)
        {
            auto curlVer = Curl.curl.version_info(CURLVERSION_NOW).version_num;
            userAgent = cast(immutable) sformat(
                buf, fmt, version_major, version_minor,
                curlVer >> 16 & 0xFF, curlVer >> 8 & 0xFF, curlVer & 0xFF);
        }
        return userAgent;
    }

    /** Set the value of the user agent request header field.
     *
     * By default a request has it's "User-Agent" field set to $(LREF
     * defaultUserAgent) even if $(D setUserAgent) was never called.  Pass
     * an empty string to suppress the "User-Agent" field altogether.
     */
    void setUserAgent(const(char)[] userAgent)
    {
        p.curl.set(CurlOption.useragent, userAgent);
    }

    /**
     * Get various timings defined in $(REF CurlInfo, etc, c, curl).
     * The value is usable only if the return value is equal to $(D etc.c.curl.CurlError.ok).
     *
     * Params:
     *      timing = one of the timings defined in $(REF CurlInfo, etc, c, curl).
     *               The values are:
     *               $(D etc.c.curl.CurlInfo.namelookup_time),
     *               $(D etc.c.curl.CurlInfo.connect_time),
     *               $(D etc.c.curl.CurlInfo.pretransfer_time),
     *               $(D etc.c.curl.CurlInfo.starttransfer_time),
     *               $(D etc.c.curl.CurlInfo.redirect_time),
     *               $(D etc.c.curl.CurlInfo.appconnect_time),
     *               $(D etc.c.curl.CurlInfo.total_time).
     *      val    = the actual value of the inquired timing.
     *
     * Returns:
     *      The return code of the operation. The value stored in val
     *      should be used only if the return value is $(D etc.c.curl.CurlInfo.ok).
     *
     * Example:
     * ---
     * import std.net.curl;
     * import etc.c.curl : CurlError, CurlInfo;
     *
     * auto client = HTTP("dlang.org");
     * client.perform();
     *
     * double val;
     * CurlCode code;
     *
     * code = http.getTiming(CurlInfo.namelookup_time, val);
     * assert(code == CurlError.ok);
     * ---
     */
    CurlCode getTiming(CurlInfo timing, ref double val)
    {
        return p.curl.getTiming(timing, val);
    }

    /** The headers read from a successful response.
     *
     */
    @property string[string] responseHeaders()
    {
        return p.headersIn;
    }

    /// HTTP method used.
    @property void method(Method m)
    {
        p.method = m;
    }

    /// ditto
    @property Method method()
    {
        return p.method;
    }

    /**
       HTTP status line of last response. One call to perform may
       result in several requests because of redirection.
    */
    @property StatusLine statusLine()
    {
        return p.status;
    }

    /// Set the active cookie string e.g. "name1=value1;name2=value2"
    void setCookie(const(char)[] cookie)
    {
        p.curl.set(CurlOption.cookie, cookie);
    }

    /// Set a file path to where a cookie jar should be read/stored.
    void setCookieJar(const(char)[] path)
    {
        p.curl.set(CurlOption.cookiefile, path);
        if (path.length)
            p.curl.set(CurlOption.cookiejar, path);
    }

    /// Flush cookie jar to disk.
    void flushCookieJar()
    {
        p.curl.set(CurlOption.cookielist, "FLUSH");
    }

    /// Clear session cookies.
    void clearSessionCookies()
    {
        p.curl.set(CurlOption.cookielist, "SESS");
    }

    /// Clear all cookies.
    void clearAllCookies()
    {
        p.curl.set(CurlOption.cookielist, "ALL");
    }

    /**
       Set time condition on the request.

       Params:
       cond =  $(D CurlTimeCond.{none,ifmodsince,ifunmodsince,lastmod})
       timestamp = Timestamp for the condition

       $(HTTP www.w3.org/Protocols/rfc2616/rfc2616-sec14.html#sec14.25, _RFC2616 Section 14.25)
    */
    void setTimeCondition(HTTP.TimeCond cond, SysTime timestamp)
    {
        p.curl.set(CurlOption.timecondition, cond);
        p.curl.set(CurlOption.timevalue, timestamp.toUnixTime());
    }

    /** Specifying data to post when not using the onSend callback.
      *
      * The data is NOT copied by the library.  Content-Type will default to
      * application/octet-stream.  Data is not converted or encoded by this
      * method.
      *
      * Example:
      * ----
      * import std.net.curl, std.stdio;
      * auto http = HTTP("http://www.mydomain.com");
      * http.onReceive = (ubyte[] data) { writeln(to!(const(char)[])(data)); return data.length; };
      * http.postData = [1,2,3,4,5];
      * http.perform();
      * ----
      */
    @property void postData(const(void)[] data)
    {
        setPostData(data, "application/octet-stream");
    }

    /** Specifying data to post when not using the onSend callback.
      *
      * The data is NOT copied by the library.  Content-Type will default to
      * text/plain.  Data is not converted or encoded by this method.
      *
      * Example:
      * ----
      * import std.net.curl, std.stdio;
      * auto http = HTTP("http://www.mydomain.com");
      * http.onReceive = (ubyte[] data) { writeln(to!(const(char)[])(data)); return data.length; };
      * http.postData = "The quick....";
      * http.perform();
      * ----
      */
    @property void postData(const(char)[] data)
    {
        setPostData(data, "text/plain");
    }

    /**
     * Specify data to post when not using the onSend callback, with
     * user-specified Content-Type.
     * Params:
     *  data = Data to post.
     *  contentType = MIME type of the data, for example, "text/plain" or
     *      "application/octet-stream". See also:
     *      $(LINK2 http://en.wikipedia.org/wiki/Internet_media_type,
     *      Internet media type) on Wikipedia.
     * -----
     * import std.net.curl;
     * auto http = HTTP("http://onlineform.example.com");
     * auto data = "app=login&username=bob&password=s00perS3kret";
     * http.setPostData(data, "application/x-www-form-urlencoded");
     * http.onReceive = (ubyte[] data) { return data.length; };
     * http.perform();
     * -----
     */
    void setPostData(const(void)[] data, string contentType)
    {
        // cannot use callback when specifying data directly so it is disabled here.
        p.curl.clear(CurlOption.readfunction);
        addRequestHeader("Content-Type", contentType);
        p.curl.set(CurlOption.postfields, cast(void*) data.ptr);
        p.curl.set(CurlOption.postfieldsize, data.length);
        if (method == Method.undefined)
            method = Method.post;
    }

    @system unittest
    {
        import std.algorithm.searching : canFind;

        testServer.handle((s) {
            auto req = s.recvReq!ubyte;
            assert(req.hdrs.canFind("POST /path"));
            assert(req.bdy.canFind(cast(ubyte[])[0, 1, 2, 3, 4]));
            assert(req.bdy.canFind(cast(ubyte[])[253, 254, 255]));
            s.send(httpOK(cast(ubyte[])[17, 27, 35, 41]));
        });
        auto data = new ubyte[](256);
        foreach (i, ref ub; data)
            ub = cast(ubyte) i;

        auto http = HTTP(testServer.addr~"/path");
        http.postData = data;
        ubyte[] res;
        http.onReceive = (data) { res ~= data; return data.length; };
        http.perform();
        assert(res == cast(ubyte[])[17, 27, 35, 41]);
    }

    /**
      * Set the event handler that receives incoming headers.
      *
      * The callback will receive a header field key, value as parameter. The
      * $(D const(char)[]) arrays are not valid after the delegate has returned.
      *
      * Example:
      * ----
      * import std.net.curl, std.stdio;
      * auto http = HTTP("dlang.org");
      * http.onReceive = (ubyte[] data) { writeln(to!(const(char)[])(data)); return data.length; };
      * http.onReceiveHeader = (in char[] key, in char[] value) { writeln(key, " = ", value); };
      * http.perform();
      * ----
      */
    @property void onReceiveHeader(void delegate(in char[] key,
                                                 in char[] value) callback)
    {
        p.onReceiveHeader = callback;
    }

    /**
       Callback for each received StatusLine.

       Notice that several callbacks can be done for each call to
       $(D perform()) due to redirections.

       See_Also: $(LREF StatusLine)
     */
    @property void onReceiveStatusLine(void delegate(StatusLine) callback)
    {
        p.onReceiveStatusLine = callback;
    }

    /**
       The content length in bytes when using request that has content
       e.g. POST/PUT and not using chunked transfer. Is set as the
       "Content-Length" header.  Set to ulong.max to reset to chunked transfer.
    */
    @property void contentLength(ulong len)
    {
        import std.conv : to;

        CurlOption lenOpt;

        // Force post if necessary
        if (p.method != Method.put && p.method != Method.post &&
            p.method != Method.patch)
            p.method = Method.post;

        if (p.method == Method.post || p.method == Method.patch)
            lenOpt = CurlOption.postfieldsize_large;
        else
            lenOpt = CurlOption.infilesize_large;

        if (size_t.max != ulong.max && len == size_t.max)
            len = ulong.max; // check size_t.max for backwards compat, turn into error

        if (len == ulong.max)
        {
            // HTTP 1.1 supports requests with no length header set.
            addRequestHeader("Transfer-Encoding", "chunked");
            addRequestHeader("Expect", "100-continue");
        }
        else
        {
            p.curl.set(lenOpt, to!curl_off_t(len));
        }
    }

    /**
       Authentication method as specified in $(LREF AuthMethod).
    */
    @property void authenticationMethod(AuthMethod authMethod)
    {
        p.curl.set(CurlOption.httpauth, cast(long) authMethod);
    }

    /**
       Set