dlibs/dlib/http_client/http_client.cpp

#include "../sockets.h"
#include "../string.h"
#include "../logger.h"
#include "../sockstreambuf.h"
#include "../timeout.h"
#include "http_client.h"
#include <time.h>
#include <stdio.h>
#include <fstream>
#include <sstream>
#include <iostream>

namespace dlib
{

    typedef std::shared_ptr<dlib::timeout> timeout_ptr;


#ifdef _MSC_VER
#define BR_CASECMP strnicmp
#else
#define BR_CASECMP strncasecmp
#endif
// Default timeout after 60 seconds
#define DEFAULT_TIMEOUT 60000

// ----------------------------------------------------------------------------------------

    inline bool isXdigit( char c )
    {
        return  (c >= '0' && c <= '9') ||
                (c >= 'A' && c <= 'Z') ||
                (c >= 'a' && c <= 'z');
    }

// ----------------------------------------------------------------------------------------

    std::string http_client::urldecode( const std::string& s )
    {
        std::stringstream ss;

        for ( char const * p_read = s.c_str(), * p_end = (s.c_str() + s.size()); p_read < p_end; p_read++ )
        {
            if ( p_read[0] == '%' && p_read+1 != p_end && p_read+2 != p_end && isXdigit(p_read[1]) && isXdigit(p_read[2]) )
            {
                ss << static_cast<char>((( (p_read[1] & 0xf) + ((p_read[1] >= 'A') ? 9 : 0) ) << 4 ) | ( (p_read[2] & 0xf) + ((p_read[2] >= 'A') ? 9 : 0) ));
                p_read += 2;
            }
            else if ( p_read[0] == '+' )
            {
                // Undo the encoding that replaces spaces with plus signs.
                ss << ' ';
            }
            else
            {
                ss << p_read[0];
            }
        }

        return ss.str();
    }

// ----------------------------------------------------------------------------------------

//! \return modified string ``s'' with spaces trimmed from left
    inline std::string& triml(std::string& s)
    {
        std::string::size_type pos(0);
        for ( ; s[pos] == ' ' || s[pos] == '\t' || s[pos] == '\r' || s[pos] == '\n' ; ++pos );
        s.erase(0, pos);
        return s;
    }

// ----------------------------------------------------------------------------------------

//! \return modified string ``s'' with spaces trimmed from right
    inline std::string& trimr(std::string& s)
    {
        std::string::size_type pos(s.size());
        for ( ; pos && (s[pos-1] == ' ' || s[pos-1] == '\t' || s[pos-1] == '\r' || s[pos-1] == '\n') ; --pos );
        s.erase(pos, s.size()-pos);
        return s;
    }

// ----------------------------------------------------------------------------------------

//! \return modified string ``s'' with spaces trimmed from edges
    inline std::string& trim(std::string& s)
    {
        return triml(trimr(s));
    }

// ----------------------------------------------------------------------------------------

    http_client::
    http_client(
    ) : 
        http_return(0),
        timeout(DEFAULT_TIMEOUT),
        OnDownload(0)
    {
    }

// ----------------------------------------------------------------------------------------

    std::string http_client::get_header(const std::string& header_name) const
    {
        stringmap::const_iterator ci = headers.find(header_name);
        return ci != headers.end() ? ci->second : std::string();
    }

// ----------------------------------------------------------------------------------------

    void http_client::set_header(const std::string& header_name, long header_value)
    {
        char buf[21] = { 0 };
#ifdef __WXMSW__
        ::ltoa(header_value, buf, 10);
#else
        sprintf(buf, "%ld", header_value);
#endif
        set_header(header_name, buf);
    }

// ----------------------------------------------------------------------------------------

    void http_client::set_header(const std::string& header_name, const std::string& header_value)
    {
        headers[header_name] = header_value;
    }

// ----------------------------------------------------------------------------------------

    bool http_client::is_header_set(const std::string& header_name) const
    {
        stringmap::const_iterator ci = headers.find(header_name);
        return ci != headers.end() && !ci->second.empty();
    }

// ----------------------------------------------------------------------------------------

    void http_client::remove_header(const std::string& header_name)
    {
        headers.erase(header_name);
    }

// ----------------------------------------------------------------------------------------

    void http_client::set_cookie(const std::string& cookie_name, long cookie_value)
    {
        char buf[21] = { 0 };
#ifdef __WXMSW__
        ::ltoa(cookie_value, buf, 10);
#else
        sprintf(buf, "%ld", cookie_value);
#endif
        set_cookie(cookie_name, buf);
    }

// ----------------------------------------------------------------------------------------

    void http_client::set_cookie(const std::string& cookie_name, const std::string& cookie_value)
    {
        cookies[cookie_name] = cookie_value;
    }

// ----------------------------------------------------------------------------------------

    void http_client::remove_cookie(const std::string& cookie_name)
    {
        cookies.erase(cookie_name);
    }

// ----------------------------------------------------------------------------------------

// POST
    const std::string& http_client::post_url (const std::string& url, const string_to_stringmap& postvars, const string_to_stringmap& filenames)
    {
        std::string CT;
        std::string postBody = build_post(CT, postvars, filenames);
        set_header("Content-Type", CT);
        set_header("Content-Length", static_cast<long>(postBody.size()));

        grab_url(url, "POST", postBody);

        return returned_body;
    }

// ----------------------------------------------------------------------------------------

    const std::string& http_client::post_url (const std::string& url, const std::string& postbuffer)
    {
        if ( !is_header_set("Content-Type") ) // Maybe they just forgot it?
            set_header("Content-Type", "application/x-www-form-urlencoded");

        set_header("Content-Length", static_cast<long>(postbuffer.size()));

        grab_url(url, "POST", postbuffer);

        return returned_body;
    }

// ----------------------------------------------------------------------------------------

    std::string http_client::get_random_string( size_t length ) const
    {
        static bool has_seeded(false);
        static std::string allowed_chars("abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789");

        if ( !has_seeded )
        {
            has_seeded = true;
            ::srand( static_cast<unsigned int>(::time(NULL)) );
        }

        std::string retVal; retVal.reserve(length);
        while ( retVal.size() < length )
        {
            retVal += allowed_chars[(rand() % allowed_chars.size())];
        }

        return retVal;
    }

// ----------------------------------------------------------------------------------------

// static
    std::string http_client::urlencode(const std::string& in, bool post_encode)
    {
        static std::string allowed_chars("abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789_");

        std::stringstream ss;
        ss << std::hex;
        for (std::string::const_iterator ci = in.begin(); ci != in.end(); ++ci)
        {
            if ( allowed_chars.find(*ci) != std::string::npos )
            {
                ss << *ci;
            }
            else if ( post_encode && *ci == ' ' )
            {
                ss << '+';
            }
            else
            {
                ss << '%' << std::setfill('0') << std::setw(2) << std::right << static_cast<int>(*ci);
            }
        }

        return ss.str();
    }

// ----------------------------------------------------------------------------------------

    std::string http_client::get_basename( const std::string& filename ) const
    {
        std::string::size_type pos = filename.find_last_of("\\/");
        if ( pos == std::string::npos )
            return filename;
        else
            return filename.substr(pos+1);
    }

// ----------------------------------------------------------------------------------------

    bool http_client::parse_url(
        const std::string& url,
        std::string& scheme,
        std::string& user,
        std::string& pass,
        std::string& host,
        short& port,
        std::string& path
    ) const
    {
        scheme.clear();
        user.clear();
        pass.clear();
        host.clear();
        path.clear();
        port = 0;

        // Find scheme
        std::string::size_type pos_scheme = url.find("://");
        if ( pos_scheme == std::string::npos )
        {
            pos_scheme = 0;
        }
        else
        {
            scheme = strtolower(url.substr(0, pos_scheme));
            pos_scheme += 3;
        }

        std::string::size_type pos_path = url.find('/', pos_scheme);
        if ( pos_path == std::string::npos )
        {
            host = url.substr(pos_scheme);
        }
        else
        {
            host = url.substr(pos_scheme, pos_path - pos_scheme);
            path = url.substr(pos_path);
        }

        std::string::size_type pos_at = host.find('@');
        if ( pos_at != std::string::npos )
        {
            std::string::size_type pos_dp = host.find(':');
            if ( pos_dp != std::string::npos && pos_dp < pos_at )
            {
                user = host.substr(0, pos_dp);
                pass = host.substr(pos_dp+1, pos_at-pos_dp-1);
            }
            else
            {
                user = host.substr(0, pos_at);
            }
            host = host.substr(pos_at+1);
        }

        std::string::size_type pos_dp = host.find(':');
        if ( pos_dp != std::string::npos )
        {
            port = dlib::string_cast<short>(host.substr(pos_dp+1));
            host = host.substr(0, pos_dp);
        }

        host = strtolower(host);

        if ( port == 0 )
        {
            if ( scheme == "http" )
                port = 80;
            else if ( scheme == "ftp" )
                port = 21;
            else if ( scheme == "https" )
                port = 443;
        }

        return !host.empty();
    }

// ----------------------------------------------------------------------------------------

    std::string http_client::strtolower(const std::string& in) const
    {
        std::string retVal = in;

        for (std::string::iterator ii = retVal.begin(); ii != retVal.end(); ++ii)
        {
            *ii = ::tolower(*ii);
        }

        return retVal;
    }

// ----------------------------------------------------------------------------------------

    std::string http_client::strtoupper(const std::string& in) const
    {
        std::string retVal = in;

        for (std::string::iterator ii = retVal.begin(); ii != retVal.end(); ++ii)
        {
            *ii = ::toupper(*ii);
        }

        return retVal;
    }

// ----------------------------------------------------------------------------------------

// GET
    const std::string& http_client::get_url(const std::string& url)
    {
        std::string CT = get_header("Content-Type");

        // You do a GET with a POST header??
        if ( CT == "application/x-www-form-urlencoded" || CT == "multipart/form-data" )
            remove_header("Content-Type");

        grab_url(url);

        return returned_body;
    }

// ----------------------------------------------------------------------------------------

    std::string http_client::build_post(std::string& content_type, const string_to_stringmap& postvars, const string_to_stringmap& filenames_in) const
    {
        if ( postvars.empty() && filenames_in.empty() )
            return std::string();

        string_to_stringmap filenames = filenames_in;

        // sanitize the files
        if ( !filenames.empty() )
        {
            string_to_stringmap::iterator var_names = filenames.begin();
            while (var_names != filenames.end())
            {
                stringmap::iterator fnames = var_names->second.begin();

                while( fnames != var_names->second.end() )
                {
                    FILE *fp = ::fopen(fnames->second.c_str(), "rb");
                    if ( fp == NULL )
                    {
                        stringmap::iterator old_one = fnames++;
                        var_names->second.erase(old_one);
                    }
                    else
                    {
                        fclose(fp);
                        ++fnames;
                    }
                }

                if ( fnames->second.empty() )
                {
                    string_to_stringmap::iterator old_one = var_names++;
                    filenames.erase(old_one);
                }
                else
                {
                    ++var_names;
                }
            }
        }

        content_type = !filenames.empty() ? "multipart/form-data" : "application/x-www-form-urlencoded";
        std::stringstream postBody;
        if ( !filenames.empty() )
        {
            std::string mime_boundary = get_random_string(32);

            // First add the form vars
            for (string_to_stringmap::const_iterator ci = postvars.begin(); ci != postvars.end(); ++ci)
            {
                for (stringmap::const_iterator si = ci->second.begin(); si != ci->second.end(); ++si)
                {
                    postBody << "--" << mime_boundary << "\r\n"
                        "Content-Disposition: form-data; name=\"" << ci->first << "\"\r\n\r\n"
                        << si->second << "\r\n";
                }
            }

            // Then add the files
            for (string_to_stringmap::const_iterator ci = filenames.begin(); ci != filenames.end(); ++ci)
            {
                for (stringmap::const_iterator si = ci->second.begin(); si != ci->second.end(); ++si)
                {
                    std::ifstream in(si->second.c_str());
                    postBody << "--" << mime_boundary << "\r\n"
                        "Content-Disposition: form-data; name=\"" << ci->first << "\"; filename=\"" << get_basename(si->second) << "\"\r\n\r\n"
                        << in.rdbuf() << "\r\n";
                }
            }

            postBody << "--" << mime_boundary << "--\r\n";
        }
        else
        {
            // No files...
            for (string_to_stringmap::const_iterator ci = postvars.begin(); ci != postvars.end(); ++ci)
            {
                for (stringmap::const_iterator si = ci->second.begin(); si != ci->second.end(); ++si)
                {
                    postBody << urlencode(ci->first) << '=' << urlencode(si->second) << '&';
                }
            }

            // read the last '&'
            char c;
            postBody.read(&c, 1);
        }

        return postBody.str();
    }

// ----------------------------------------------------------------------------------------

    bool http_client::grab_url(const std::string& url, const std::string& method, const std::string& post_body)
    {
        error_field.clear();
        returned_headers.clear();
        http_return = 0;
        returned_body.clear();

        std::string to_use_method = strtoupper(method);

        std::string scheme, user, pass, host, path;
        short port;
        if ( !parse_url(url, scheme, user, pass, host, port, path) )
        {
            error_field = "Couldn't parse the URL!";
            return false;
        }

        // Build request
        std::stringstream ret;
        ret << to_use_method << ' ' << path << " HTTP/1.0\r\n"
            << "Host: " << host;
        if (port != 80 && port != 443) ret << ':' << port;
        ret << "\r\n";

        bool content_length_said = false;

        set_header("Connection", "Close");
        for (stringmap::iterator ci = headers.begin(); ci != headers.end(); ++ci)
        {
            std::string head = strtolower(ci->first);

            if ( head == "content-length" )
            {
                content_length_said = true;
            }

            ret << ci->first << ':' << ' ' << ci->second << "\r\n";
        }

        if ( !content_length_said && to_use_method != "GET" )
            ret << "Content-Length: " << static_cast<unsigned int>(post_body.size()) << "\r\n";

        std::stringstream cookie_ss;
        for (stringmap::iterator ci = cookies.begin(); ci != cookies.end(); ++ci)
        {
            std::string var = ci->first ; trim(var);
            std::string val = ci->second; trim(val);

            if ( val.empty() || var.empty() )
                continue;

            if ( !cookie_ss.str().empty() )
                cookie_ss << ';' << ' ';

            cookie_ss << urlencode(var) << '=' << urlencode(val);
        }

        if ( !cookie_ss.str().empty() )
            ret << "Cookie: " << cookie_ss.str() << "\r\n";

        ret << "\r\n";
        ret << post_body;

        std::string request_build = ret.str();

        std::stringstream ss;
        {
            dlib::connection * conn(0);
            try
            {
                if (timeout > 0)
                    conn = dlib::connect(host, port, timeout);
                else
                    conn = dlib::connect(host, port);
            }
            catch (const dlib::socket_error& e)
            {
                error_field = e.what();
                return false;
            }

            // Implement a timeout
            timeout_ptr t;
            if ( timeout > 0 )
                t.reset( new dlib::timeout(*conn, &dlib::connection::shutdown, timeout) );

            // Write our request
            conn->write(request_build.c_str(), static_cast<long>(request_build.size()));

            t.reset();

            // And read the response
            char buf[512];
            long bytes_read(0), bytes_total(0);
            bool read_headers(true);

            if ( timeout > 0 )
                t.reset( new dlib::timeout(*conn, &dlib::connection::shutdown, timeout) );

            while ( (bytes_read = conn->read(buf, 512)) > 0 )
            {
                ss.write(buf, bytes_read);

                // Incremental read headers
                if ( read_headers )
                {
                    std::string body_with_headers = ss.str();
                    std::string::size_type ctr(0);

                    while ( true )
                    {
                        std::string::size_type pos = body_with_headers.find("\r\n", ctr);
                        if ( pos == std::string::npos )
                        {
                            // This is our last position of "\r\n"
                            ss.str("");
                            ss.write( body_with_headers.substr(ctr).c_str(), body_with_headers.size() - ctr );
                            break;
                        }

                        std::string header = body_with_headers.substr(ctr, pos-ctr);
                        if ( header.empty() )
                        {
                            // Ok, we're done reading the headers
                            read_headers = false;
                            // What follows now is the body
                            ss.str("");
                            ss.write( body_with_headers.substr(pos + 2).c_str(), body_with_headers.size() - pos - 2 );
                            break;
                        }
                        ctr = pos + 2;

                        if ( returned_headers.empty() )
                        {
                            if (
                                header[0] == 'H' &&
                                header[1] == 'T' &&
                                header[2] == 'T' &&
                                header[3] == 'P' &&
                                header[4] == '/' &&
                                (header[5] >= '0' && header[5] <= '9') &&
                                header[6] == '.' &&
                                (header[7] >= '0' && header[7] <= '9') &&
                                header[8] == ' '
                            )
                            {
                                http_return = (header[9 ] - '0') * 100 +
                                    (header[10] - '0') * 10 +
                                    (header[11] - '0');
                                continue;
                            }
                        }

                        std::string::size_type pos_dp = header.find_first_of(':');
                        std::string header_name, header_value;
                        if ( pos_dp == std::string::npos )
                        {
                            // **TODO** what should I do here??
                            header_name = header;
                        }
                        else
                        {
                            header_name  = trim(header.substr(0, pos_dp));
                            header_value = trim(header.substr(pos_dp+1));
                        }

                        returned_headers[ header_name ].push_back(header_value);

                        if ( BR_CASECMP(header_name.c_str(), "Content-Length", 14) == 0 )
                        {
                            bytes_total = atol( header_value.c_str() );
                        }
                        else if ( BR_CASECMP(header_name.c_str(), "Set-Cookie", 10) == 0 )
                        {
                            std::string::size_type cur_pos(0), pos_pk, pos_is;
                            std::string work, var, val;
                            for ( cur_pos = 0; cur_pos < header_value.size(); cur_pos++ )
                            {
                                pos_pk = header_value.find(';', cur_pos);
                                work   = trim( header_value.substr(cur_pos, pos_pk - cur_pos) );

                                pos_is = work.find('=');
                                if ( pos_is != std::string::npos )
                                { // Hmmm? what in the else case?
                                    var = trim( http_client::urldecode( work.substr(0, pos_is) ) );
                                    val = trim( http_client::urldecode( work.substr(pos_is + 1) ) );

                                    if ( var != "expires" && var != "domain" && var != "path" )
                                        set_cookie( var, val );
                                }
                                cur_pos = pos_pk == std::string::npos ? pos_pk - 1 : pos_pk;
                            }
                        } // Set-Cookie?

                    } // while (true)
                } // read_headers?

                // Call the OnDownload function if it's set
                if ( OnDownload && !read_headers )
                {
                    if ( (*OnDownload)(static_cast<long>(ss.tellp()), bytes_total, user_info) == false )
                    {
                        t.reset();
                        break;
                    }
                }

                if ( bytes_total != 0 && static_cast<long>(ss.tellp()) == bytes_total )
                {
                    t.reset();
                    break;
                }

                if ( timeout > 0 )
                    t.reset( new dlib::timeout(*conn, &dlib::connection::shutdown, timeout) );
            } // while still data to read

            t.reset();

            delete conn;


            switch ( bytes_read )
            {
                case dlib::TIMEOUT:      error_field = "Timeout";     return false; break;
                case dlib::WOULDBLOCK:   error_field = "Would block"; return false; break;
                case dlib::OTHER_ERROR:  error_field = "Other error"; return false; break;
                case dlib::SHUTDOWN:     error_field = "Timeout";     return false; break;
                case dlib::PORTINUSE:    error_field = "Port in use"; return false; break;
            }
        }

        returned_body = ss.str();

        return true;
    }

// ----------------------------------------------------------------------------------------

    void http_client::clear()
    {
        headers.clear();
        cookies.clear();
    }

// ----------------------------------------------------------------------------------------

    void http_client::prepare_for_next_url( )
    {
        remove_header("Content-Type");
        remove_header("Content-Length");
    }

// ----------------------------------------------------------------------------------------

}