mdl_session.c

Go to the documentation of this file.

#include "mdl_session.h"
 
#include <stdio.h>
#include <string.h>
 
#include "mdl_stream_internal.h"
#include "mdl_url_guard.h"
#include "ra8_attributes.h"

typedef enum : uint32_t {
  k_robots_timeout_ms = 10000U, 
} mdl_session_net_t;

typedef enum : uint16_t {
  k_http_too_many       = 429, 
  k_http_client_err_min = 400, 
  k_http_client_err_max = 499, 
  k_http_server_err_min = 500, 
  k_http_server_err_max = 599, 
} mdl_session_status_t;

static const char* const s_ua_token = "mdl";
static const char* const s_ua_version = "0.1.0";
static const char* const s_ua_url = "https://github.com/bsikar/ra8-firmware";
 
const char* mdl_session_ua_token(void)
{
  return s_ua_token;
}
 
bool mdl_session_build_ua(const char* contact, char* out, size_t cap)
{
  if ((out == nullptr) || (cap == 0U)) {
    return false;
  }
  const bool have = (contact != nullptr) && (contact[0] != '\0');
  if (have) {
    (void)snprintf(out, cap, "%s/%s (+%s; %s)", s_ua_token, s_ua_version, s_ua_url, contact);
  } else {
    (void)snprintf(out, cap, "%s/%s (+%s)", s_ua_token, s_ua_version, s_ua_url);
  }
  return have;
}
 
void mdl_session_init(mdl_session_t*   session,
                      mdl_net_iface_t* net,
                      const char*      user_agent,
                      ra8_io_stream_t* diagnostic,
                      bool             honor_robots)
{
  if (session == nullptr) {
    return;
  }
  session->net          = net;
  session->user_agent   = user_agent;
  session->diagnostic   = diagnostic;
  session->honor_robots = honor_robots;
  session->cache        = (mdl_robots_cache_t){};
}

RA8_INTERNAL static bool internal_url_prefix(const char* url, const char* prefix)
{
  size_t i = 0U;
  while (prefix[i] != '\0') {
    char c = url[i];
    if ((c >= 'A') && (c <= 'Z')) {
      c = (char)(c + ('a' - 'A'));
    }
    if (c != prefix[i]) {
      return false;
    }
    ++i;
  }
  return true;
}

RA8_INTERNAL static const char* internal_url_scheme(const char* url)
{
  if (internal_url_prefix(url, "https://")) {
    return "https";
  }
  if (internal_url_prefix(url, "http://")) {
    return "http";
  }
  return nullptr;
}

RA8_INTERNAL static bool internal_robots_target(const char* url, char* out, size_t cap)
{
  if ((url == nullptr) || (out == nullptr) || (cap < 2U)) {
    return false;
  }
  const char* sep = strstr(url, "://");
  if (sep == nullptr) {
    return false;
  }
  const char* authority = sep + 3U;
  const char* start     = strpbrk(authority, "/?#");
  size_t      n         = 0U;
  if ((start == nullptr) || (*start == '#')) {
    out[0] = '/';
    out[1] = '\0';
    return true;
  }
  if (*start == '?') {
    out[n] = '/';
    ++n;
  }
  const char* p = start;
  while ((*p != '\0') && (*p != '#')) {
    if ((n + 1U) >= cap) {
      out[0] = '\0';
      return false;
    }
    out[n] = *p;
    ++n;
    ++p;
  }
  out[n] = '\0';
  return true;
}

RA8_INTERNAL static mdl_robots_fetch_result_t
internal_session_fetch(void* ctx, const char* robots_url, char* buf, size_t cap, size_t* out_len)
{
  mdl_session_t*      s    = (mdl_session_t*)ctx;
  const mdl_net_req_t req  = {.user_agent = s->user_agent,
                              .referer    = nullptr,
                              .timeout_ms = k_robots_timeout_ms};
  mdl_net_resp_t      resp = {};
  if (mdl_net_get_buf(s->net, robots_url, &req, buf, cap, out_len, &resp) == k_ra8_ok) {
    return k_mdl_robots_fetch_ok;
  }
  if ((resp.status >= (long)k_http_client_err_min) &&
      (resp.status <= (long)k_http_client_err_max) && (resp.status != (long)k_http_too_many)) {
    return k_mdl_robots_fetch_absent;
  }
  if ((resp.status >= (long)k_http_server_err_min) &&
      (resp.status <= (long)k_http_server_err_max)) {
    return k_mdl_robots_fetch_denied;
  }
  return k_mdl_robots_fetch_denied;
}

RA8_INTERNAL static void
internal_report_robots_unavailable(mdl_session_t* session, const char* host, const char* path)
{
  ra8_err_t error = priv_mdl_stream_text(k_ra8_ok,
                                         session->diagnostic,
                                         "mdl: robots.txt unavailable, rate-limited, or "
                                         "oversized for ");
  error           = priv_mdl_stream_text(error, session->diagnostic, host);
  error           = priv_mdl_stream_text(error, session->diagnostic, "; refusing ");
  error           = priv_mdl_stream_text(error, session->diagnostic, path);
  (void)priv_mdl_stream_text(error, session->diagnostic, "\n");
}

RA8_INTERNAL static void internal_report_robots_disallow(mdl_session_t* session,
                                                         const char*    path,
                                                         const char*    host,
                                                         const char*    reason)
{
  ra8_err_t error =
    priv_mdl_stream_text(k_ra8_ok, session->diagnostic, "mdl: robots.txt disallows ");
  error = priv_mdl_stream_text(error, session->diagnostic, path);
  error = priv_mdl_stream_text(error, session->diagnostic, " on ");
  error = priv_mdl_stream_text(error, session->diagnostic, host);
  error = priv_mdl_stream_text(error, session->diagnostic, " (rule \"");
  error = priv_mdl_stream_text(error, session->diagnostic, reason);
  (void)priv_mdl_stream_text(error, session->diagnostic, "\"); skipping\n");
}
 
bool mdl_session_url_allowed(mdl_session_t* session, const char* url, uint32_t* crawl_delay_ms)
{
  if (crawl_delay_ms != nullptr) {
    *crawl_delay_ms = 0U;
  }
  if ((session == nullptr) || (url == nullptr)) {
    return false;
  }
  if (!session->honor_robots) {
    return true;
  }
  const char* scheme = internal_url_scheme(url);
  if (scheme == nullptr) {
    (void)priv_mdl_stream_text(k_ra8_ok,
                               session->diagnostic,
                               "mdl: cannot apply robots.txt to malformed/non-HTTP URL; "
                               "refusing\n");
    return false;
  }
  char host[k_mdl_robots_host_max];
  if (!mdl_url_host(url, host, sizeof(host))) {
    (void)priv_mdl_stream_text(k_ra8_ok,
                               session->diagnostic,
                               "mdl: cannot identify URL origin for robots.txt; refusing\n");
    return false;
  }
  char path[k_mdl_robots_path_max];
  if (!internal_robots_target(url, path, sizeof(path))) {
    (void)priv_mdl_stream_text(k_ra8_ok,
                               session->diagnostic,
                               "mdl: URL path/query exceeds robots.txt bound; refusing\n");
    return false;
  }
  const mdl_robots_t* rules = mdl_robots_cache_consult(&session->cache,
                                                       scheme,
                                                       host,
                                                       s_ua_token,
                                                       internal_session_fetch,
                                                       session,
                                                       session->scratch,
                                                       sizeof(session->scratch));
  if (rules == nullptr) {
    internal_report_robots_unavailable(session, host, path);
    return false;
  }
  if ((crawl_delay_ms != nullptr) && rules->have_crawl_delay) {
    *crawl_delay_ms = rules->crawl_delay_ms;
  }
  const char* reason = mdl_robots_disallow_reason(rules, path);
  if (reason != nullptr) {
    internal_report_robots_disallow(session, path, host, reason);
    return false;
  }
  return true;
}